From 042e36da65e353df2bd65e9aa7a74cef2d9699ea Mon Sep 17 00:00:00 2001 From: Mike Malone Date: Thu, 17 Jan 2019 16:07:27 -0800 Subject: [PATCH] autocert --- autocert/INSTALL.md | 132 ++++++ autocert/README.md | 247 +++++++++++ autocert/bootstrapper/Dockerfile | 10 + autocert/bootstrapper/bootstrapper.sh | 7 + autocert/controller/Dockerfile | 19 + autocert/controller/client.go | 124 ++++++ autocert/controller/main.go | 601 ++++++++++++++++++++++++++ autocert/controller/provisioner.go | 143 ++++++ autocert/install/01-step-ca.yaml | 87 ++++ autocert/install/02-autocert.yaml | 106 +++++ autocert/install/03-rbac.yaml | 36 ++ autocert/renewer/Dockerfile | 8 + 12 files changed, 1520 insertions(+) create mode 100644 autocert/INSTALL.md create mode 100644 autocert/README.md create mode 100644 autocert/bootstrapper/Dockerfile create mode 100644 autocert/bootstrapper/bootstrapper.sh create mode 100644 autocert/controller/Dockerfile create mode 100644 autocert/controller/client.go create mode 100644 autocert/controller/main.go create mode 100644 autocert/controller/provisioner.go create mode 100644 autocert/install/01-step-ca.yaml create mode 100644 autocert/install/02-autocert.yaml create mode 100644 autocert/install/03-rbac.yaml create mode 100644 autocert/renewer/Dockerfile diff --git a/autocert/INSTALL.md b/autocert/INSTALL.md new file mode 100644 index 00000000..93066a18 --- /dev/null +++ b/autocert/INSTALL.md @@ -0,0 +1,132 @@ +### Create a CA + +Set your `STEPPATH` to a working directory where we can stage our CA artifacts before we push them to kubernetes. You can delete this directory once installation is complete. + +``` +$ export STEPPATH=$(mktemp -d /tmp/step.XXX) +$ step path +/tmp/step.0kE +``` + +Run `step ca init` to generate a root certificate and CA configuration for your cluster. You'll be prompted for a password that will be used to encrypt key material. + +``` +$ step ca init \ + --name Autocert \ + --dns "ca.step.svc.cluster.local,127.0.0.1" \ + --address ":4443" \ + --provisioner admin \ + --with-ca-url "ca.step.svc.cluster.local" +``` + +For older versions of `step` run this command without the flags. + +Add provisioning credentials for use by `autocert`. You'll be prompted for a password for `autocert`. + +``` +$ step ca provisioner add autocert --create +``` + +For older versions of `step`: + +* Run `step ca init` and follow prompts +* Edit `$(step path)/config/ca.json` and change base paths to `/home/step` +* Edit `$(step path)/config/defaults.json` to change base paths to `/home/step` and remove port from CA URL + +``` +$ sed -i "" "s|$(step path)|/home/step/.step|g" $(step path)/config/ca.json +$ sed -i "" "s|$(step path)|/home/step/.step|g" $(step path)/config/defaults.json +$ sed -i "" "s|ca.step.svc.cluster.local:4443|ca.step.svc.cluster.local|" $(step path)/config/defaults.json +``` + +### Install the CA in Kubernetes + +We'll install our CA and the `autocert` controller in the `step` namespace. + +``` +$ kubectl create namespace step +``` + +To install the CA we need to configmap the CA certificates, signing keys, and configuration artifacts. Note that key material is encrypted so we don't need to use secrets. + +``` +$ kubectl -n step create configmap config --from-file $(step path)/config +$ kubectl -n step create configmap certs --from-file $(step path)/certs +$ kubectl -n step create configmap secrets --from-file $(step path)/secrets +``` + +But we will need to create secrets for the CA and autocert to decrypt their keys: + +``` +$ kubectl -n step create secret generic ca-password --from-literal password= +$ kubectl -n step create secret generic autocert-password --from-literal password= +``` + +Where `` is the password you entered during `step ca init` and `` is the password you entered during `step ca provisioner add`. + +Next, we'll install the CA. + +``` +$ kubectl apply -f https://raw.githubusercontent.com/smallstep/certificates/master/autocert/install/01-step-ca.yaml +``` + +Once you've done this you can delete the temporary `$STEPPATH` directory and `unset STEPPATH` (though you may want to retain it as a backup). + +### Install `autocert` in Kubernetes + +Install the `autocert` controller. + +``` +$ kubectl apply -f https://raw.githubusercontent.com/smallstep/certificates/master/autocert/install/02-autocert.yaml +``` + +Autocert creates secrets containing single-use bootstrap tokens for pods to authenticate with the CA and obtain a certificate. The tokens are automatically cleaned up after they expire. To do this, `autocert` needs permission to create and delete secrets in your cluster. + +If you have RBAC enabled in your cluster, apply `rbac.yaml` to give `autocert` these permissions. + +``` +$ kubectl apply -f https://raw.githubusercontent.com/smallstep/certificates/master/autocert/install/03-rbac.yaml +``` + +Finally, register the `autocert` mutation webhook with kubernetes. + +``` +$ cat < + +Annotations: autocert.step.sm/name: sleep.default.svc.cluster.local + autocert.step.sm/status: injected +Status: Running + +<... snip ...> + +Init Containers: + autocert-bootstrapper: + Image: step-k8s/bootstrapper + +<... snip ...> + +Containers: + sleep: + Image: alpine + +<... snip ...> + + Mounts: + /var/run/autocert.step.sm from certs (ro) + /var/run/secrets/kubernetes.io/serviceaccount from default-token-jn988 (ro) + autocert-renewer: + Image: step-k8s/renewer + +<... snip ...> + +Volumes: + certs: + Type: EmptyDir (a temporary directory that shares a pod's lifetime) + +<... snip ...> + +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 4m2s default-scheduler Successfully assigned sleep-f996bd578-nch7c to docker-for-desktop + Normal SuccessfulMountVolume 4m2s kubelet, docker-for-desktop MountVolume.SetUp succeeded for volume "certs" + Normal SuccessfulMountVolume 4m2s kubelet, docker-for-desktop MountVolume.SetUp succeeded for volume "default-token-jn988" + Normal Pulled 4m1s kubelet, docker-for-desktop Container image "step-k8s/bootstrapper" already present on machine + Normal Created 4m1s kubelet, docker-for-desktop Created container + Normal Started 4m kubelet, docker-for-desktop Started container + Normal Pulled 4m kubelet, docker-for-desktop Container image "alpine" already present on machine + Normal Created 4m kubelet, docker-for-desktop Created container + Normal Started 3m59s kubelet, docker-for-desktop Started container + Normal Pulled 3m59s kubelet, docker-for-desktop Container image "step-k8s/renewer" already present on machine + Normal Created 3m59s kubelet, docker-for-desktop Created container + Normal Started 3m59s kubelet, docker-for-desktop Started container +``` + +Certificates are mounted to `/var/run/autocert.step.sm`. We can inspect this directory to make sure everything worked correctly: + +``` +$ kubectl exec -it sleep-f996bd578-nch7c -c sleep -- ls -lias /var/run/autocert.step.sm +total 20 +1593393 4 drwxrwxrwx 2 root root 4096 Jan 17 21:27 . +1339651 4 drwxr-xr-x 1 root root 4096 Jan 17 21:27 .. +1593451 4 -rw------- 1 root root 574 Jan 17 21:27 root.crt +1593442 4 -rw-r--r-- 1 root root 1352 Jan 17 21:41 site.crt +1593443 4 -rw-r--r-- 1 root root 227 Jan 17 21:27 site.key +``` + +The `autocert-renewer` sidecare installs the `step` CLI tool, which we can use to inspect the issued certificate: + +``` +$ kubectl exec -it sleep-f996bd578-nch7c -c autocert-renewer -- step certificate inspect /var/run/autocert.step.sm/site.crt +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 46935033335539540860078000614852612373 (0x234f5bce23705f015a8377ab1cfd5115) + Signature Algorithm: ECDSA-SHA256 + Issuer: CN=Autocert Intermediate CA + Validity + Not Before: Jan 17 21:41:04 2019 UTC + Not After : Jan 17 21:46:14 2019 UTC + Subject: CN=sleep.default.svc.cluster.local + Subject Public Key Info: + Public Key Algorithm: ECDSA + Public-Key: (256 bit) + X: + 31:aa:a1:7f:c8:b4:c6:da:90:fc:b8:3a:e9:cc:48: + f9:89:b9:5d:d7:a4:63:80:76:9f:21:6d:e5:88:4c: + a8:e4 + Y: + ed:21:38:57:cd:3f:32:71:6f:ca:81:34:b0:4a:bd: + a3:c4:8d:d1:87:bc:2c:4c:42:79:e5:35:49:38:3f: + b7:c8 + Curve: P-256 + X509v3 extensions: + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Subject Key Identifier: + 43:0E:0A:50:30:A5:5B:AF:22:AC:28:49:26:53:2A:B4:D4:20:E0:E0 + X509v3 Authority Key Identifier: + keyid:61:45:1E:E4:95:4C:0A:6B:37:4C:43:41:FD:54:2E:8E:5E:A2:24:EF + X509v3 Subject Alternative Name: + DNS:sleep.default.svc.cluster.local + + Signature Algorithm: ECDSA-SHA256 + 30:44:02:20:0c:c5:ab:0d:22:17:a2:04:9f:ff:5f:b1:c0:a5: + 8b:94:88:e0:40:66:e1:19:e9:34:2f:67:74:12:4f:bb:51:8b: + 02:20:01:7e:0d:44:ce:b2:92:41:d5:78:0d:02:5a:68:05:7c: + c2:a9:81:28:71:5c:95:6d:56:51:49:e0:37:b7:09:87 +``` + +### Test your installation + +To test your installation you can install the `hello-mtls` demo app. + +* Install app, which uses mTLS and responds "hello, `identity`" +* Do a `kubectl run` of `step-cli` then get a certificate using `step` and `curl hello-mtls` from within the cluster +* Port forward from localhost to get a certificate then `curl` with `--resolve` + +### Further reading + +* Link to ExternalDNS example +* Link to multiple cluster with Service type ExternalDNS so they can communicate + +### Uninstall + +* Delete the `sleep` deployment (if you created it) +* Remove labels (show how to find labelled namespaces) +* Remove annotations (show how to find any annotated pods) +* Remove secrets (show how to find labelled secrets) +* Delete `step` namespace diff --git a/autocert/bootstrapper/Dockerfile b/autocert/bootstrapper/Dockerfile new file mode 100644 index 00000000..baca6386 --- /dev/null +++ b/autocert/bootstrapper/Dockerfile @@ -0,0 +1,10 @@ +FROM smallstep/step-cli:0.8.3 + +USER root +ENV CRT="/var/run/autocert.step.sm/site.crt" +ENV KEY="/var/run/autocert.step.sm/site.key" +ENV STEP_ROOT="/var/run/autocert.step.sm/root.crt" + +COPY bootstrapper.sh /home/step/ +RUN chmod +x /home/step/bootstrapper.sh +CMD ["/home/step/bootstrapper.sh"] diff --git a/autocert/bootstrapper/bootstrapper.sh b/autocert/bootstrapper/bootstrapper.sh new file mode 100644 index 00000000..e90d26c8 --- /dev/null +++ b/autocert/bootstrapper/bootstrapper.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Download the root certificate and set permissions +step ca certificate $COMMON_NAME $CRT $KEY +chmod 644 $CRT $KEY + +step ca root $STEP_ROOT diff --git a/autocert/controller/Dockerfile b/autocert/controller/Dockerfile new file mode 100644 index 00000000..f51f820a --- /dev/null +++ b/autocert/controller/Dockerfile @@ -0,0 +1,19 @@ +# build stage +FROM golang:alpine AS build-env +RUN apk update && apk upgrade && \ + apk add --no-cache git +RUN go get -u github.com/golang/dep/cmd/dep +WORKDIR $GOPATH/src/github.com/step-certificates-k8s/controller +# copy dep files and run dep separately from code for better caching +COPY Gopkg.toml Gopkg.lock ./ +RUN dep ensure --vendor-only +COPY . ./ +RUN go build -o /server . + +# final stage +FROM smallstep/step-cli:0.8.3 +ENV STEPPATH="/home/step/.step" +ENV PWDPATH="/home/step/password/password" +ENV CONFIGPATH="/home/step/autocert/config.yaml" +COPY --from=build-env /server . +ENTRYPOINT ./server $CONFIGPATH diff --git a/autocert/controller/client.go b/autocert/controller/client.go new file mode 100644 index 00000000..6a443168 --- /dev/null +++ b/autocert/controller/client.go @@ -0,0 +1,124 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "strings" + "time" +) + +const ( + serviceAccountToken = "/var/run/secrets/kubernetes.io/serviceaccount/token" + serviceAccountCACert = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +) + +// Client is minimal kubernetes client interface +type Client interface { + Do(req *http.Request) (*http.Response, error) + GetRequest(url string) (*http.Request, error) + PostRequest(url, body, contentType string) (*http.Request, error) + DeleteRequest(url string) (*http.Request, error) + Host() string +} + +type k8sClient struct { + host string + token string + httpClient *http.Client +} + +func (kc *k8sClient) GetRequest(url string) (*http.Request, error) { + if !strings.HasPrefix(url, kc.host) { + url = fmt.Sprintf("%s/%s", kc.host, url) + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + if len(kc.token) > 0 { + req.Header.Set("Authorization", "Bearer "+kc.token) + } + return req, nil +} + +func (kc *k8sClient) PostRequest(url string, body string, contentType string) (*http.Request, error) { + if !strings.HasPrefix(url, kc.host) { + url = fmt.Sprintf("%s/%s", kc.host, url) + } + req, err := http.NewRequest("POST", url, strings.NewReader(body)) + if err != nil { + return nil, err + } + if len(kc.token) > 0 { + req.Header.Set("Authorization", "Bearer "+kc.token) + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + return req, nil +} + +func (kc *k8sClient) DeleteRequest(url string) (*http.Request, error) { + if !strings.HasPrefix(url, kc.host) { + url = fmt.Sprintf("%s/%s", kc.host, url) + } + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return nil, err + } + if len(kc.token) > 0 { + req.Header.Set("Authorization", "Bearer "+kc.token) + } + return req, nil +} + +func (kc *k8sClient) Do(req *http.Request) (*http.Response, error) { + return kc.httpClient.Do(req) +} + +func (kc *k8sClient) Host() string { + return kc.host +} + +// NewInClusterK8sClient creates K8sClient if it is inside Kubernetes +func NewInClusterK8sClient() (Client, error) { + host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT") + if len(host) == 0 || len(port) == 0 { + return nil, fmt.Errorf("unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined") + } + token, err := ioutil.ReadFile(serviceAccountToken) + if err != nil { + return nil, err + } + ca, err := ioutil.ReadFile(serviceAccountCACert) + if err != nil { + return nil, err + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(ca) + transport := &http.Transport{TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS10, + RootCAs: certPool, + }} + httpClient := &http.Client{Transport: transport, Timeout: time.Nanosecond * 0} + + return &k8sClient{ + host: "https://" + net.JoinHostPort(host, port), + token: string(token), + httpClient: httpClient, + }, nil +} + +// NewInsecureK8sClient creates an insecure k8s client which is suitable +// to connect kubernetes api behind proxy +func NewInsecureK8sClient(apiURL string) Client { + return &k8sClient{ + host: apiURL, + httpClient: http.DefaultClient, + } +} \ No newline at end of file diff --git a/autocert/controller/main.go b/autocert/controller/main.go new file mode 100644 index 00000000..cb849aa1 --- /dev/null +++ b/autocert/controller/main.go @@ -0,0 +1,601 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "encoding/hex" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "time" + + "github.com/ghodss/yaml" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/smallstep/certificates/ca" + "github.com/smallstep/cli/crypto/pemutil" + "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" +) + +var ( + runtimeScheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(runtimeScheme) + deserializer = codecs.UniversalDeserializer() + // GetRootCAPath() is broken; points to secrets not certs. So + // we'll hard code instead for now. + //rootCAPath = pki.GetRootCAPath() + rootCAPath = "/home/step/.step/certs/root_ca.crt" +) + +const ( + admissionWebhookAnnotationKey = "autocert.step.sm/name" + admissionWebhookStatusKey = "autocert.step.sm/status" + provisionerPasswordFile = "/home/step/password/password" + volumeMountPath = "/var/run/autocert.step.sm" + tokenSecretKey = "token" + tokenSecretLabel = "autocert.step.sm/token" +) + +// Config options for the autocert admission controller. +type Config struct { + LogFormat string `yaml:"logFormat"` + CaUrl string `yaml:"caUrl"` + CertLifetime string `yaml:"certLifetime"` + Bootstrapper corev1.Container `yaml:"bootstrapper"` + Renewer corev1.Container `yaml:"renewer"` + CertsVolume corev1.Volume `yaml:"certsVolume"` +} + +// RFC6902 JSONPatch Operation +type PatchOperation struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +// RFC6901 JSONPath Escaping -- https://tools.ietf.org/html/rfc6901 +func escapeJsonPath(path string) string { + // Replace`~` with `~0` then `/` with `~1`. Note that the order + // matters otherwise we'll turn a `/` into a `~/`. + path = strings.Replace(path, "~", "~0", -1) + path = strings.Replace(path, "/", "~1", -1) + return path +} + +func loadConfig(file string) (*Config, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + + var cfg Config + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, err + } + + return &cfg, nil +} + +// createTokenSecret generates a kubernetes Secret object containing a bootstrap token +// in the specified namespce. The secret name is randomly generated with a given prefix. +// A goroutine is scheduled to cleanup the secret after the token expires. The secret +// is also labelled for easy identification and manual cleanup. +func createTokenSecret(prefix, namespace, token string) (string, error) { + secret := corev1.Secret { + TypeMeta: metav1.TypeMeta { + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta { + GenerateName: prefix, + Namespace: namespace, + Labels: map[string]string { + tokenSecretLabel: "true", + }, + }, + StringData: map[string]string { + tokenSecretKey: token, + }, + Type: corev1.SecretTypeOpaque, + } + + client, err := NewInClusterK8sClient() + if err != nil { + return "", err + } + + body, err := json.Marshal(secret) + if err != nil { + return "", err + } + log.WithField("secret", string(body)).Debug("Creating secret") + + req, err := client.PostRequest(fmt.Sprintf("api/v1/namespaces/%s/secrets", namespace), string(body), "application/json") + if err != nil { + return "", err + } + + resp, err := client.Do(req) + if err != nil { + log.Errorf("Secret creation error. Response: %v", resp) + return "", errors.Wrap(err, "secret creation") + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Errorf("Secret creation error (!2XX). Response: %v", resp) + var rbody []byte + if resp.Body != nil { + if data, err := ioutil.ReadAll(resp.Body); err == nil { + rbody = data + } + } + log.Error("Error body: ", string(rbody)) + return "", errors.New("Not 200") + } + + var rbody []byte + if resp.Body != nil { + if data, err := ioutil.ReadAll(resp.Body); err == nil { + rbody = data + } + } + if len(rbody) == 0 { + return "", errors.New("Empty response body") + } + + var created *corev1.Secret + if err := json.Unmarshal(rbody, &created); err != nil { + return "", errors.Wrap(err, "Error unmarshalling secret response") + } + + // Clean up after ourselves by deleting the Secret after the bootstrap + // token expires. This is best effort -- obviously we'll miss some stuff + // if this process goes away -- but the secrets are also labelled so + // it's also easy to clean them up in bulk using kubectl if we miss any. + go func() { + time.Sleep(tokenLifetime) + req, err := client.DeleteRequest(fmt.Sprintf("api/v1/namespaces/%s/secrets/%s", namespace, created.Name)) + ctxLog := log.WithFields(log.Fields{ + "name": created.Name, + "namespace": namespace, + }) + if err != nil { + ctxLog.WithField("error", err).Error("Error deleting expired boostrap token secret") + return + } + resp, err := client.Do(req) + if err != nil { + ctxLog.WithField("error", err).Error("Error deleting expired boostrap token secret") + return + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + ctxLog.WithFields(log.Fields{ + "status": resp.Status, + "statusCode": resp.StatusCode, + }).Error("Error deleting expired boostrap token secret") + return + } + ctxLog.Info("Deleted expired bootstrap token secret") + }() + + return created.Name, err +} + +// mkBootstrapper generates a bootstrap container based on the template defined in Config. It +// generates a new bootstrap token and mounts it, along with other required coniguration, as +// environment variables in the returned bootstrap container. +func mkBootstrapper(config *Config, commonName string, namespace string, provisioner Provisioner) (corev1.Container, error) { + b := config.Bootstrapper + + token, err := provisioner.Token(commonName) + if err != nil { + return b, errors.Wrap(err, "token generation") + } + + // Generate CA fingerprint + crt, err := pemutil.ReadCertificate(rootCAPath) + if err != nil { + return b, errors.Wrap(err, "CA fingerprint") + } + sum := sha256.Sum256(crt.Raw) + fingerprint := strings.ToLower(hex.EncodeToString(sum[:])) + + secretName, err := createTokenSecret(commonName + "-", namespace, token) + if err != nil { + return b, errors.Wrap(err, "create token secret") + } + log.Infof("Secret name is: %s", secretName) + + b.Env = append(b.Env, corev1.EnvVar { + Name: "COMMON_NAME", + Value: commonName, + }) + b.Env = append(b.Env, corev1.EnvVar { + Name: "STEP_TOKEN", + ValueFrom: &corev1.EnvVarSource { + SecretKeyRef: &corev1.SecretKeySelector { + LocalObjectReference: corev1.LocalObjectReference { + Name: secretName, + }, + Key: tokenSecretKey, + }, + }, + }) + b.Env = append(b.Env, corev1.EnvVar { + Name: "STEP_CA_URL", + Value: config.CaUrl, + }) + b.Env = append(b.Env, corev1.EnvVar { + Name: "STEP_FINGERPRINT", + Value: fingerprint, + }) + b.Env = append(b.Env, corev1.EnvVar { + Name: "STEP_NOT_AFTER", + Value: config.CertLifetime, + }) + return b, nil +} + +// mkRenewer generates a new renewer based on the template provided in Config. +func mkRenewer(config *Config) (corev1.Container) { + r := config.Renewer + r.Env = append(r.Env, corev1.EnvVar { + Name: "STEP_CA_URL", + Value: config.CaUrl, + }) + return r +} + +func addContainers(existing, new []corev1.Container, path string) (ops []PatchOperation) { + if len(existing) == 0 { + return []PatchOperation { + PatchOperation { + Op: "add", + Path: path, + Value: new, + }, + } + } else { + for _, add := range new { + ops = append(ops, PatchOperation { + Op: "add", + Path: path + "/-", + Value: add, + }) + } + return ops + } +} + +func addVolumes(existing, new []corev1.Volume, path string) (ops []PatchOperation) { + if len(existing) == 0 { + return []PatchOperation{ + PatchOperation { + Op: "add", + Path: path, + Value: new, + }, + } + } else { + for _, add := range new { + ops = append(ops, PatchOperation { + Op: "add", + Path: path + "/-", + Value: add, + }) + } + return ops + } +} + +func addCertsVolumeMount(volumeName string, containers []corev1.Container) (ops []PatchOperation) { + volumeMount := corev1.VolumeMount { + Name: volumeName, + MountPath: volumeMountPath, + ReadOnly: true, + } + for i, container := range containers { + if len(container.VolumeMounts) == 0 { + ops = append(ops, PatchOperation { + Op: "add", + Path: fmt.Sprintf("/spec/containers/%v/volumeMounts", i), + Value: []corev1.VolumeMount{volumeMount}, + }) + } else { + ops = append(ops, PatchOperation { + Op: "add", + Path: fmt.Sprintf("/spec/containers/%v/volumeMounts/-", i), + Value: volumeMount, + }) + } + } + return ops +} + +func addAnnotations(existing, new map[string]string) (ops []PatchOperation) { + if len(existing) == 0 { + return []PatchOperation{ + PatchOperation { + Op: "add", + Path: "/metadata/annotations", + Value: new, + }, + } + } + for k, v := range new { + if existing[k] == "" { + ops = append(ops, PatchOperation { + Op: "add", + Path: "/metadata/annotations/" + escapeJsonPath(k), + Value: v, + }) + } else { + ops = append(ops, PatchOperation { + Op: "replace", + Path: "/metadata/annotations/" + escapeJsonPath(k), + Value: v, + }) + } + } + return ops +} + +// patch produces a list of patches to apply to a pod to inject a certificate. In particular, +// we patch the pod in order to: +// - Mount the `certs` volume in existing containers defined in the pod +// - Add the autocert-renewer as a container (a sidecar) +// - Add the autocert-bootstrapper as an initContainer +// - Add the `certs` volume definition +// - Annotate the pod to indicate that it's been processed by this controller +// The result is a list of serialized JSONPatch objects (or an error). +func patch(pod *corev1.Pod, namespace string, config *Config, provisioner Provisioner) ([]byte, error) { + var ops[] PatchOperation + + commonName := pod.ObjectMeta.GetAnnotations()[admissionWebhookAnnotationKey] + renewer := mkRenewer(config) + bootstrapper, err := mkBootstrapper(config, commonName, namespace, provisioner) + if err != nil { + return nil, err + } + + ops = append(ops, addCertsVolumeMount(config.CertsVolume.Name, pod.Spec.Containers)...) + ops = append(ops, addContainers(pod.Spec.Containers, []corev1.Container{renewer}, "/spec/containers")...) + ops = append(ops, addContainers(pod.Spec.InitContainers, []corev1.Container{bootstrapper}, "/spec/initContainers")...) + ops = append(ops, addVolumes(pod.Spec.Volumes, []corev1.Volume{config.CertsVolume}, "/spec/volumes")...) + ops = append(ops, addAnnotations(pod.Annotations, map[string]string{admissionWebhookStatusKey: "injected"})...) + + return json.Marshal(ops) +} + +// shouldMutate checks whether a pod is subject to mutation by this admission controller. A pod +// is subject to mutation if it's annotated with the `admissionWebhookAnnotationKey` and if it +// has not already been processed (indicated by `admissionWebhookStatusKey` set to `injected`). +func shouldMutate(metadata *metav1.ObjectMeta) bool { + annotations := metadata.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + + // Only mutate if the object is annotated appropriately (annotation key set) and we haven't + // mutated already (status key isn't set). + if annotations[admissionWebhookAnnotationKey] == "" || annotations[admissionWebhookStatusKey] == "injected" { + return false + } else { + return true + } +} + +// mutate takes an `AdmissionReview`, determines whether it is subject to mutation, and returns +// an appropriate `AdmissionResponse` including patches or any errors that occurred. +func mutate(review *v1beta1.AdmissionReview, config *Config, provisioner Provisioner) *v1beta1.AdmissionResponse { + ctxLog := log.WithField("uid", review.Request.UID) + + request := review.Request + var pod corev1.Pod + if err := json.Unmarshal(request.Object.Raw, &pod); err != nil { + ctxLog.WithField("error", err).Error("Error unmarshalling pod") + return &v1beta1.AdmissionResponse { + Allowed: false, + UID: request.UID, + Result: &metav1.Status { + Message: err.Error(), + }, + } + } + + ctxLog = ctxLog.WithFields(log.Fields{ + "kind": request.Kind, + "operation": request.Operation, + "name": pod.Name, + "generateName": pod.GenerateName, + "namespace": request.Namespace, + "user": request.UserInfo, + }) + + if !shouldMutate(&pod.ObjectMeta) { + ctxLog.WithField("annotations", pod.Annotations).Info("Skipping mutation") + return &v1beta1.AdmissionResponse { + Allowed: true, + UID: request.UID, + } + } + + patchBytes, err := patch(&pod, request.Namespace, config, provisioner) + if err != nil { + ctxLog.WithField("error", err).Error("Error generating patch") + return &v1beta1.AdmissionResponse { + Allowed: false, + UID: request.UID, + Result: &metav1.Status { + Message: err.Error(), + }, + } + } + + ctxLog.WithField("patch", string(patchBytes)).Info("Generated patch") + return &v1beta1.AdmissionResponse { + Allowed: true, + Patch: patchBytes, + UID: request.UID, + PatchType: func() *v1beta1.PatchType { + pt := v1beta1.PatchTypeJSONPatch + return &pt + }(), + } +} + +func main() { + if len(os.Args) != 2 { + log.Errorf("Usage: %s \n", os.Args[0]) + os.Exit(1) + } + + config, err := loadConfig(os.Args[1]) + if err != nil { + panic(err) + } + + log.SetOutput(os.Stdout) + if config.LogFormat == "json" { + log.SetFormatter(&log.JSONFormatter{}) + } + if config.LogFormat == "text" { + log.SetFormatter(&log.TextFormatter{}) + } + + log.WithFields(log.Fields{ + "config": config, + }).Info("Loaded config") + + provisionerName := os.Getenv("PROVISIONER_NAME") + provisionerKid := os.Getenv("PROVISIONER_KID") + log.WithFields(log.Fields{ + "provisionerName": provisionerName, + "provisionerKid": provisionerKid, + }).Info("Loaded provisioner configuration") + + provisioner, err := NewProvisioner(provisionerName, provisionerKid, config.CaUrl, rootCAPath, provisionerPasswordFile) + if err != nil { + log.Errorf("Error loading provisioner: %v", err) + os.Exit(1) + } + log.WithFields(log.Fields{ + "name": provisioner.Name(), + "kid": provisioner.Kid(), + }).Info("Loaded provisioner") + + namespace := os.Getenv("NAMESPACE") + if namespace == "" { + log.Errorf("$NAMESPACE not set") + os.Exit(1) + } + + name := fmt.Sprintf("autocert.%s.svc", namespace) + token, err := provisioner.Token(name) + if err != nil { + log.WithField("error", err).Errorf("Error generating bootstrap token during controller startup") + os.Exit(1) + } + log.WithField("name", name).Infof("Generated bootstrap token for controller") + + // make sure to cancel the renew goroutine + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + srv, err := ca.BootstrapServer(ctx, token, &http.Server{ + Addr: ":4443", + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" { + log.Info("/healthz") + fmt.Fprintf(w, "ok") + w.WriteHeader(http.StatusOK) + return + } + + /* + var name string + if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { + name = r.TLS.PeerCertificates[0].Subject.CommonName + } + */ + + if r.URL.Path != "/mutate" { + log.WithField("path", r.URL.Path).Error("Bad Request: 404 Not Found") + http.NotFound(w, r) + return + } + + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + if len(body) == 0 { + log.Error("Bad Request: 400 (Empty Body)") + http.Error(w, "Bad Request (Empty Body)", http.StatusBadRequest) + return + } + + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + log.WithField("Content-Type", contentType).Error("Bad Request: 415 (Unsupported Media Type)") + http.Error(w, fmt.Sprintf("Bad Request: 415 Unsupported Media Type (Expected Content-Type 'application/json' but got '%s')", contentType), http.StatusUnsupportedMediaType) + return + } + + var response *v1beta1.AdmissionResponse + review := v1beta1.AdmissionReview{} + if _, _, err := deserializer.Decode(body, nil, &review); err != nil { + log.WithFields(log.Fields{ + "body": body, + "error": err, + }).Error("Can't decode body") + response = &v1beta1.AdmissionResponse { + Allowed: false, + Result: &metav1.Status { + Message: err.Error(), + }, + } + } else { + response = mutate(&review, config, provisioner) + } + + resp, err := json.Marshal(v1beta1.AdmissionReview { + Response: response, + }) + if err != nil { + log.WithFields(log.Fields{ + "uid": review.Request.UID, + "error": err, + }).Info("Marshal error") + http.Error(w, fmt.Sprintf("Marshal Error: %v", err), http.StatusInternalServerError) + } else { + log.WithFields(log.Fields{ + "uid": review.Request.UID, + "response": string(resp), + }).Info("Returning review") + if _, err := w.Write(resp); err != nil { + log.WithFields(log.Fields{ + "uid": review.Request.UID, + "error": err, + }).Info("Write error") + } + } + }), + }, ca.VerifyClientCertIfGiven()) + if err != nil { + panic(err) + } + + log.Info("Listening on :4443 ...") + if err := srv.ListenAndServeTLS("", ""); err != nil { + panic(err) + } +} diff --git a/autocert/controller/provisioner.go b/autocert/controller/provisioner.go new file mode 100644 index 00000000..eb739fe1 --- /dev/null +++ b/autocert/controller/provisioner.go @@ -0,0 +1,143 @@ +package main + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/smallstep/cli/crypto/pki" + "github.com/smallstep/cli/crypto/randutil" + "github.com/smallstep/cli/jose" + "github.com/smallstep/cli/token" + "github.com/smallstep/cli/token/provision" +) + +const ( + tokenLifetime = 5 * time.Minute +) + +type Provisioner interface { + Name() string + Kid() string + Token(subject string) (string, error) +} + +type provisioner struct { + name string + kid string + caUrl string + caRoot string + jwk *jose.JSONWebKey + tokenLifetime time.Duration +} + +// Name returns the provisioner's name. +func (p *provisioner) Name() string { + return p.name +} + +// Kid returns the provisioners key ID. +func (p *provisioner) Kid() string { + return p.kid +} + +// Token generates a bootstrap token for a subject. +func (p *provisioner) Token(subject string) (string, error) { + // A random jwt id will be used to identify duplicated tokens + jwtID, err := randutil.Hex(64) // 256 bits + if err != nil { + return "", err + } + + notBefore := time.Now() + notAfter := notBefore.Add(tokenLifetime) + signUrl := fmt.Sprintf("%v/1.0/sign", p.caUrl) + + tokOptions := []token.Options{ + token.WithJWTID(jwtID), + token.WithKid(p.kid), + token.WithIssuer(p.name), + token.WithAudience(signUrl), + token.WithValidity(notBefore, notAfter), + token.WithRootCA(p.caRoot), + } + + tok, err := provision.New(subject, tokOptions...) + if err != nil { + return "", err + } + + return tok.SignedString(p.jwk.Algorithm, p.jwk.Key) +} + +func decryptProvisionerJWK(encryptedKey, passFile string) (*jose.JSONWebKey, error) { + decrypted, err := jose.Decrypt("", []byte(encryptedKey), jose.WithPasswordFile(passFile)) + if err != nil { + return nil, err + } + + jwk := new(jose.JSONWebKey) + if err := json.Unmarshal(decrypted, jwk); err != nil { + return nil, errors.Wrap(err, "error unmarshalling provisioning key") + } + return jwk, nil +} + +// loadProvisionerJWKByKid retrieves a provisioner key from the CA by key ID and +// decrypts it using the specified password file. +func loadProvisionerJWKByKid(kid, caUrl, caRoot, passFile string) (*jose.JSONWebKey, error) { + encrypted, err := pki.GetProvisionerKey(caUrl, caRoot, kid) + if err != nil { + return nil, err + } + + return decryptProvisionerJWK(encrypted, passFile) +} + +// loadProvisionerJWKByName retrieves the list of provisioners and encrypted key then +// returns the key of the first provisioner with a matching name that can be successfully +// decrypted with the specified password file. +func loadProvisionerJWKByName(name, caUrl, caRoot, passFile string) (key *jose.JSONWebKey, err error) { + provisioners, err := pki.GetProvisioners(caUrl, caRoot) + if err != nil { + err = errors.Wrap(err, "error getting the provisioners") + return + } + + for _, provisioner := range provisioners { + if provisioner.Name == name { + key, err = decryptProvisionerJWK(provisioner.EncryptedKey, passFile) + if err == nil { + return + } + } + } + return nil, errors.New(fmt.Sprintf("provisioner '%s' not found (or your password is wrong)", name)) +} + +// NewProvisioner loads and decrypts key material from the CA for the named +// provisioner. The key identified by `kid` will be used if specified. If `kid` +// is the empty string we'll use the first key for the named provisioner that +// decrypts using `passFile`. +func NewProvisioner(name, kid, caUrl, caRoot, passFile string) (Provisioner, error) { + var jwk *jose.JSONWebKey + var err error + if kid != "" { + jwk, err = loadProvisionerJWKByKid(kid, caUrl, caRoot, passFile) + } else { + jwk, err = loadProvisionerJWKByName(name, caUrl, caRoot, passFile) + } + if err != nil { + return nil, err + } + + return &provisioner{ + name: name, + kid: jwk.KeyID, + caUrl: caUrl, + caRoot: caRoot, + jwk: jwk, + tokenLifetime: tokenLifetime, + }, nil +} \ No newline at end of file diff --git a/autocert/install/01-step-ca.yaml b/autocert/install/01-step-ca.yaml new file mode 100644 index 00000000..09772adb --- /dev/null +++ b/autocert/install/01-step-ca.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: ca + name: ca + namespace: step +spec: + type: ClusterIP + ports: + - port: 443 + targetPort: 4443 + selector: + app: ca + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ca + namespace: step + labels: + app: ca +spec: + replicas: 1 + selector: + matchLabels: + app: ca + template: + metadata: + labels: + app: ca + spec: + containers: + - name: ca + image: smallstep/step-ca:0.8.3 + env: + - name: PWDPATH + value: /home/step/password/password + resources: + requests: + cpu: 100m + memory: 20Mi + readinessProbe: + httpGet: + path: /health + port: 4443 + scheme: HTTPS + initialDelaySeconds: 3 + periodSeconds: 3 + livenessProbe: + httpGet: + path: /health + port: 4443 + scheme: HTTPS + initialDelaySeconds: 3 + periodSeconds: 3 + volumeMounts: + - name: config + mountPath: /home/step/.step/config + readOnly: true + - name: certs + mountPath: /home/step/.step/certs + readOnly: true + - name: secrets + mountPath: /home/step/.step/secrets + readOnly: true + - name: ca-password + mountPath: /home/step/password + readOnly: true + securityContext: + runAsUser: 1000 + allowPrivilegeEscalation: false + volumes: + - name: certs + configMap: + name: certs + - name: config + configMap: + name: config + - name: secrets + configMap: + name: secrets + - name: ca-password + secret: + secretName: ca-password diff --git a/autocert/install/02-autocert.yaml b/autocert/install/02-autocert.yaml new file mode 100644 index 00000000..f6453ca2 --- /dev/null +++ b/autocert/install/02-autocert.yaml @@ -0,0 +1,106 @@ +apiVersion: v1 +kind: Service +metadata: + labels: {app: autocert} + name: autocert + namespace: step +spec: + type: ClusterIP + ports: + - port: 443 + targetPort: 4443 + selector: {app: autocert} + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: autocert-config + namespace: step +data: + config.yaml: | + logFormat: json # or text + caUrl: https://ca.step.svc.cluster.local + certLifetime: 24h + renewer: + name: autocert-renewer + image: smallstep/autocert-renewer:0.8.3 + resources: {requests: {cpu: 10m, memory: 20Mi}} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: certs + mountPath: /var/run/autocert.step.sm + bootstrapper: + name: autocert-bootstrapper + image: smallstep/autocert-bootstrapper:0.8.3 + resources: {requests: {cpu: 10m, memory: 20Mi}} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: certs + mountPath: /var/run/autocert.step.sm + certsVolume: + name: certs + emptyDir: {} + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: autocert + namespace: step + labels: {app: autocert} +spec: + replicas: 1 + selector: {matchLabels: {app: autocert}} + template: + metadata: {labels: {app: autocert}} + spec: + containers: + - name: autocert + image: smallstep/autocert-controller:0.8.3 + resources: {requests: {cpu: 100m, memory: 20Mi}} + env: + - name: PROVISIONER_NAME + value: autocert + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config + mountPath: /home/step/.step/config + readOnly: true + - name: certs + mountPath: /home/step/.step/certs + readOnly: true + - name: autocert-password + mountPath: /home/step/password + readOnly: true + - name: autocert-config + mountPath: /home/step/autocert + readOnly: true + securityContext: + runAsUser: 1000 + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 4443 + scheme: HTTPS + readinessProbe: + httpGet: + path: /healthz + port: 4443 + scheme: HTTPS + volumes: + - name: config + configMap: {name: config} + - name: certs + configMap: {name: certs} + - name: autocert-password + secret: {secretName: autocert-password} + - name: autocert-config + configMap: {name: autocert-config} + diff --git a/autocert/install/03-rbac.yaml b/autocert/install/03-rbac.yaml new file mode 100644 index 00000000..8d108e1c --- /dev/null +++ b/autocert/install/03-rbac.yaml @@ -0,0 +1,36 @@ +# Create a ClusterRole for managing autocert secrets, which should +# only exist in namespaces with autocert enabled and should always +# be labeled `autocert.step.sm/token: true`. +# +# To create this ClusterRole you need cluster-admin privileges. On +# GKE you can give yourself cluster-admin privileges using the +# following command: +# +# kubectl create clusterrolebinding cluster-admin-binding \ +# --clusterrole cluster-admin \ +# --user $(gcloud config get-value account) + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: autocert-secret-management +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: autocert-secret-management +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: autocert-secret-management +subjects: +- kind: ServiceAccount + name: default + namespace: step + diff --git a/autocert/renewer/Dockerfile b/autocert/renewer/Dockerfile new file mode 100644 index 00000000..900b2f60 --- /dev/null +++ b/autocert/renewer/Dockerfile @@ -0,0 +1,8 @@ +FROM smallstep/step-cli:0.8.3 + +USER root +ENV CRT="/var/run/autocert.step.sm/site.crt" +ENV KEY="/var/run/autocert.step.sm/site.key" +ENV STEP_ROOT="/var/run/autocert.step.sm/root.crt" + +ENTRYPOINT ["/bin/bash", "-c", "step ca renew --daemon $CRT $KEY"]