certificates/autocert/controller/main.go

package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/ghodss/yaml"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	"github.com/smallstep/certificates/ca"
	"github.com/smallstep/cli/crypto/pemutil"
	"k8s.io/api/admission/v1beta1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
)

var (
	runtimeScheme = runtime.NewScheme()
	codecs        = serializer.NewCodecFactory(runtimeScheme)
	deserializer  = codecs.UniversalDeserializer()
	// GetRootCAPath() is broken; points to secrets not certs. So
	// we'll hard code instead for now.
	//rootCAPath  = pki.GetRootCAPath()
	rootCAPath = "/home/step/.step/certs/root_ca.crt"
)

const (
	admissionWebhookAnnotationKey = "autocert.step.sm/name"
	admissionWebhookStatusKey     = "autocert.step.sm/status"
	provisionerPasswordFile       = "/home/step/password/password"
	volumeMountPath               = "/var/run/autocert.step.sm"
	tokenSecretKey                = "token"
	tokenSecretLabel              = "autocert.step.sm/token"
)

// Config options for the autocert admission controller.
type Config struct {
	LogFormat                       string           `yaml:"logFormat"`
	CaURL                           string           `yaml:"caUrl"`
	CertLifetime                    string           `yaml:"certLifetime"`
	Bootstrapper                    corev1.Container `yaml:"bootstrapper"`
	Renewer                         corev1.Container `yaml:"renewer"`
	CertsVolume                     corev1.Volume    `yaml:"certsVolume"`
	RestrictCertificatesToNamespace bool             `yaml:"restrictCertificatesToNamespace"`
	ClusterDomain                   string           `yaml:"clusterDomain"`
}

// GetClusterDomain returns the Kubernetes cluster domain, defaults to
// "cluster.local" if not specified in the configuration.
func (c Config) GetClusterDomain() string {
	if c.ClusterDomain != "" {
		return c.ClusterDomain
	}

	return "cluster.local"
}

// PatchOperation represents a RFC6902 JSONPatch Operation
type PatchOperation struct {
	Op    string      `json:"op"`
	Path  string      `json:"path"`
	Value interface{} `json:"value,omitempty"`
}

// RFC6901 JSONPath Escaping -- https://tools.ietf.org/html/rfc6901
func escapeJSONPath(path string) string {
	// Replace`~` with `~0` then `/` with `~1`. Note that the order
	// matters otherwise we'll turn a `/` into a `~/`.
	path = strings.Replace(path, "~", "~0", -1)
	path = strings.Replace(path, "/", "~1", -1)
	return path
}

func loadConfig(file string) (*Config, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}

	var cfg Config
	if err := yaml.Unmarshal(data, &cfg); err != nil {
		return nil, err
	}

	return &cfg, nil
}

// createTokenSecret generates a kubernetes Secret object containing a bootstrap token
// in the specified namespce. The secret name is randomly generated with a given prefix.
// A goroutine is scheduled to cleanup the secret after the token expires. The secret
// is also labelled for easy identification and manual cleanup.
func createTokenSecret(prefix, namespace, token string) (string, error) {
	secret := corev1.Secret{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Secret",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: prefix,
			Namespace:    namespace,
			Labels: map[string]string{
				tokenSecretLabel: "true",
			},
		},
		StringData: map[string]string{
			tokenSecretKey: token,
		},
		Type: corev1.SecretTypeOpaque,
	}

	client, err := NewInClusterK8sClient()
	if err != nil {
		return "", err
	}

	body, err := json.Marshal(secret)
	if err != nil {
		return "", err
	}
	log.WithField("secret", string(body)).Debug("Creating secret")

	req, err := client.PostRequest(fmt.Sprintf("api/v1/namespaces/%s/secrets", namespace), string(body), "application/json")
	if err != nil {
		return "", err
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Errorf("Secret creation error. Response: %v", resp)
		return "", errors.Wrap(err, "secret creation")
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		log.Errorf("Secret creation error (!2XX). Response: %v", resp)
		var rbody []byte
		if resp.Body != nil {
			if data, err := ioutil.ReadAll(resp.Body); err == nil {
				rbody = data
			}
		}
		log.Error("Error body: ", string(rbody))
		return "", errors.New("Not 200")
	}

	var rbody []byte
	if resp.Body != nil {
		if data, err := ioutil.ReadAll(resp.Body); err == nil {
			rbody = data
		}
	}
	if len(rbody) == 0 {
		return "", errors.New("Empty response body")
	}

	var created *corev1.Secret
	if err := json.Unmarshal(rbody, &created); err != nil {
		return "", errors.Wrap(err, "Error unmarshalling secret response")
	}

	// Clean up after ourselves by deleting the Secret after the bootstrap
	// token expires. This is best effort -- obviously we'll miss some stuff
	// if this process goes away -- but the secrets are also labelled so
	// it's also easy to clean them up in bulk using kubectl if we miss any.
	go func() {
		time.Sleep(tokenLifetime)
		req, err := client.DeleteRequest(fmt.Sprintf("api/v1/namespaces/%s/secrets/%s", namespace, created.Name))
		ctxLog := log.WithFields(log.Fields{
			"name":      created.Name,
			"namespace": namespace,
		})
		if err != nil {
			ctxLog.WithField("error", err).Error("Error deleting expired boostrap token secret")
			return
		}
		resp, err := client.Do(req)
		if err != nil {
			ctxLog.WithField("error", err).Error("Error deleting expired boostrap token secret")
			return
		}
		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			ctxLog.WithFields(log.Fields{
				"status":     resp.Status,
				"statusCode": resp.StatusCode,
			}).Error("Error deleting expired boostrap token secret")
			return
		}
		ctxLog.Info("Deleted expired bootstrap token secret")
	}()

	return created.Name, err
}

// mkBootstrapper generates a bootstrap container based on the template defined in Config. It
// generates a new bootstrap token and mounts it, along with other required coniguration, as
// environment variables in the returned bootstrap container.
func mkBootstrapper(config *Config, commonName string, namespace string, provisioner Provisioner) (corev1.Container, error) {
	b := config.Bootstrapper

	token, err := provisioner.Token(commonName)
	if err != nil {
		return b, errors.Wrap(err, "token generation")
	}

	// Generate CA fingerprint
	crt, err := pemutil.ReadCertificate(rootCAPath)
	if err != nil {
		return b, errors.Wrap(err, "CA fingerprint")
	}
	sum := sha256.Sum256(crt.Raw)
	fingerprint := strings.ToLower(hex.EncodeToString(sum[:]))

	secretName, err := createTokenSecret(commonName+"-", namespace, token)
	if err != nil {
		return b, errors.Wrap(err, "create token secret")
	}
	log.Infof("Secret name is: %s", secretName)

	b.Env = append(b.Env, corev1.EnvVar{
		Name:  "COMMON_NAME",
		Value: commonName,
	})

	b.Env = append(b.Env, corev1.EnvVar{
		Name: "STEP_TOKEN",
		ValueFrom: &corev1.EnvVarSource{
			SecretKeyRef: &corev1.SecretKeySelector{
				LocalObjectReference: corev1.LocalObjectReference{
					Name: secretName,
				},
				Key: tokenSecretKey,
			},
		},
	})
	b.Env = append(b.Env, corev1.EnvVar{
		Name:  "STEP_CA_URL",
		Value: config.CaURL,
	})
	b.Env = append(b.Env, corev1.EnvVar{
		Name:  "STEP_FINGERPRINT",
		Value: fingerprint,
	})
	b.Env = append(b.Env, corev1.EnvVar{
		Name:  "STEP_NOT_AFTER",
		Value: config.CertLifetime,
	})
	return b, nil
}

// mkRenewer generates a new renewer based on the template provided in Config.
func mkRenewer(config *Config) corev1.Container {
	r := config.Renewer
	r.Env = append(r.Env, corev1.EnvVar{
		Name:  "STEP_CA_URL",
		Value: config.CaURL,
	})
	return r
}

func addContainers(existing, new []corev1.Container, path string) (ops []PatchOperation) {
	if len(existing) == 0 {
		return []PatchOperation{
			{
				Op:    "add",
				Path:  path,
				Value: new,
			},
		}
	}

	for _, add := range new {
		ops = append(ops, PatchOperation{
			Op:    "add",
			Path:  path + "/-",
			Value: add,
		})
	}
	return ops
}

func addVolumes(existing, new []corev1.Volume, path string) (ops []PatchOperation) {
	if len(existing) == 0 {
		return []PatchOperation{
			{
				Op:    "add",
				Path:  path,
				Value: new,
			},
		}
	}

	for _, add := range new {
		ops = append(ops, PatchOperation{
			Op:    "add",
			Path:  path + "/-",
			Value: add,
		})
	}
	return ops
}

func addCertsVolumeMount(volumeName string, containers []corev1.Container) (ops []PatchOperation) {
	volumeMount := corev1.VolumeMount{
		Name:      volumeName,
		MountPath: volumeMountPath,
		ReadOnly:  true,
	}
	for i, container := range containers {
		if len(container.VolumeMounts) == 0 {
			ops = append(ops, PatchOperation{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%v/volumeMounts", i),
				Value: []corev1.VolumeMount{volumeMount},
			})
		} else {
			ops = append(ops, PatchOperation{
				Op:    "add",
				Path:  fmt.Sprintf("/spec/containers/%v/volumeMounts/-", i),
				Value: volumeMount,
			})
		}
	}
	return ops
}

func addAnnotations(existing, new map[string]string) (ops []PatchOperation) {
	if len(existing) == 0 {
		return []PatchOperation{
			{
				Op:    "add",
				Path:  "/metadata/annotations",
				Value: new,
			},
		}
	}
	for k, v := range new {
		if existing[k] == "" {
			ops = append(ops, PatchOperation{
				Op:    "add",
				Path:  "/metadata/annotations/" + escapeJSONPath(k),
				Value: v,
			})
		} else {
			ops = append(ops, PatchOperation{
				Op:    "replace",
				Path:  "/metadata/annotations/" + escapeJSONPath(k),
				Value: v,
			})
		}
	}
	return ops
}

// patch produces a list of patches to apply to a pod to inject a certificate. In particular,
// we patch the pod in order to:
//  - Mount the `certs` volume in existing containers defined in the pod
//  - Add the autocert-renewer as a container (a sidecar)
//  - Add the autocert-bootstrapper as an initContainer
//  - Add the `certs` volume definition
//  - Annotate the pod to indicate that it's been processed by this controller
// The result is a list of serialized JSONPatch objects (or an error).
func patch(pod *corev1.Pod, namespace string, config *Config, provisioner Provisioner) ([]byte, error) {
	var ops []PatchOperation

	annotations := pod.ObjectMeta.GetAnnotations()
	commonName := annotations[admissionWebhookAnnotationKey]
	renewer := mkRenewer(config)
	bootstrapper, err := mkBootstrapper(config, commonName, namespace, provisioner)
	if err != nil {
		return nil, err
	}

	ops = append(ops, addCertsVolumeMount(config.CertsVolume.Name, pod.Spec.Containers)...)
	ops = append(ops, addContainers(pod.Spec.Containers, []corev1.Container{renewer}, "/spec/containers")...)
	ops = append(ops, addContainers(pod.Spec.InitContainers, []corev1.Container{bootstrapper}, "/spec/initContainers")...)
	ops = append(ops, addVolumes(pod.Spec.Volumes, []corev1.Volume{config.CertsVolume}, "/spec/volumes")...)
	ops = append(ops, addAnnotations(pod.Annotations, map[string]string{admissionWebhookStatusKey: "injected"})...)

	return json.Marshal(ops)
}

// shouldMutate checks whether a pod is subject to mutation by this admission controller. A pod
// is subject to mutation if it's annotated with the `admissionWebhookAnnotationKey` and if it
// has not already been processed (indicated by `admissionWebhookStatusKey` set to `injected`).
// If the pod requests a certificate with a subject matching a namespace other than its own
// and restrictToNamespace is true, then shouldMutate will return a validation error
// that should be returned to the client.
func shouldMutate(metadata *metav1.ObjectMeta, namespace string, clusterDomain string, restrictToNamespace bool) (bool, error) {
	annotations := metadata.GetAnnotations()
	if annotations == nil {
		annotations = map[string]string{}
	}

	// Only mutate if the object is annotated appropriately (annotation key set) and we haven't
	// mutated already (status key isn't set).
	if annotations[admissionWebhookAnnotationKey] == "" || annotations[admissionWebhookStatusKey] == "injected" {
		return false, nil
	}

	if !restrictToNamespace {
		return true, nil
	}

	subject := strings.Trim(annotations[admissionWebhookAnnotationKey], ".")

	err := fmt.Errorf("subject \"%s\" matches a namespace other than \"%s\" and is not permitted. This check can be disabled by setting restrictCertificatesToNamespace to false in the autocert-config ConfigMap", subject, namespace)

	if strings.HasSuffix(subject, ".svc") && !strings.HasSuffix(subject, fmt.Sprintf(".%s.svc", namespace)) {
		return false, err
	}

	if strings.HasSuffix(subject, fmt.Sprintf(".svc.%s", clusterDomain)) && !strings.HasSuffix(subject, fmt.Sprintf(".%s.svc.%s", namespace, clusterDomain)) {
		return false, err
	}

	return true, nil
}

// mutate takes an `AdmissionReview`, determines whether it is subject to mutation, and returns
// an appropriate `AdmissionResponse` including patches or any errors that occurred.
func mutate(review *v1beta1.AdmissionReview, config *Config, provisioner Provisioner) *v1beta1.AdmissionResponse {
	ctxLog := log.WithField("uid", review.Request.UID)

	request := review.Request
	var pod corev1.Pod
	if err := json.Unmarshal(request.Object.Raw, &pod); err != nil {
		ctxLog.WithField("error", err).Error("Error unmarshalling pod")
		return &v1beta1.AdmissionResponse{
			Allowed: false,
			UID:     request.UID,
			Result: &metav1.Status{
				Message: err.Error(),
			},
		}
	}

	ctxLog = ctxLog.WithFields(log.Fields{
		"kind":         request.Kind,
		"operation":    request.Operation,
		"name":         pod.Name,
		"generateName": pod.GenerateName,
		"namespace":    request.Namespace,
		"user":         request.UserInfo,
	})

	mutationAllowed, validationErr := shouldMutate(&pod.ObjectMeta, request.Namespace, config.GetClusterDomain(), config.RestrictCertificatesToNamespace)

	if validationErr != nil {
		ctxLog.WithField("error", validationErr).Info("Validation error")
		return &v1beta1.AdmissionResponse{
			Allowed: false,
			UID:     request.UID,
			Result: &metav1.Status{
				Message: validationErr.Error(),
			},
		}
	}

	if !mutationAllowed {
		ctxLog.WithField("annotations", pod.Annotations).Info("Skipping mutation")
		return &v1beta1.AdmissionResponse{
			Allowed: true,
			UID:     request.UID,
		}
	}

	patchBytes, err := patch(&pod, request.Namespace, config, provisioner)
	if err != nil {
		ctxLog.WithField("error", err).Error("Error generating patch")
		return &v1beta1.AdmissionResponse{
			Allowed: false,
			UID:     request.UID,
			Result: &metav1.Status{
				Message: err.Error(),
			},
		}
	}

	ctxLog.WithField("patch", string(patchBytes)).Info("Generated patch")
	return &v1beta1.AdmissionResponse{
		Allowed: true,
		Patch:   patchBytes,
		UID:     request.UID,
		PatchType: func() *v1beta1.PatchType {
			pt := v1beta1.PatchTypeJSONPatch
			return &pt
		}(),
	}
}

func main() {
	if len(os.Args) != 2 {
		log.Errorf("Usage: %s <config>\n", os.Args[0])
		os.Exit(1)
	}

	config, err := loadConfig(os.Args[1])
	if err != nil {
		panic(err)
	}

	log.SetOutput(os.Stdout)
	if config.LogFormat == "json" {
		log.SetFormatter(&log.JSONFormatter{})
	}
	if config.LogFormat == "text" {
		log.SetFormatter(&log.TextFormatter{})
	}

	log.WithFields(log.Fields{
		"config": config,
	}).Info("Loaded config")

	provisionerName := os.Getenv("PROVISIONER_NAME")
	provisionerKid := os.Getenv("PROVISIONER_KID")
	log.WithFields(log.Fields{
		"provisionerName": provisionerName,
		"provisionerKid":  provisionerKid,
	}).Info("Loaded provisioner configuration")

	provisioner, err := NewProvisioner(provisionerName, provisionerKid, config.CaURL, rootCAPath, provisionerPasswordFile)
	if err != nil {
		log.Errorf("Error loading provisioner: %v", err)
		os.Exit(1)
	}
	log.WithFields(log.Fields{
		"name": provisioner.Name(),
		"kid":  provisioner.Kid(),
	}).Info("Loaded provisioner")

	namespace := os.Getenv("NAMESPACE")
	if namespace == "" {
		log.Errorf("$NAMESPACE not set")
		os.Exit(1)
	}

	name := fmt.Sprintf("autocert.%s.svc", namespace)
	token, err := provisioner.Token(name)
	if err != nil {
		log.WithField("error", err).Errorf("Error generating bootstrap token during controller startup")
		os.Exit(1)
	}
	log.WithField("name", name).Infof("Generated bootstrap token for controller")

	// make sure to cancel the renew goroutine
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	srv, err := ca.BootstrapServer(ctx, token, &http.Server{
		Addr: ":4443",
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if r.URL.Path == "/healthz" {
				log.Info("/healthz")
				fmt.Fprintf(w, "ok")
				w.WriteHeader(http.StatusOK)
				return
			}

			/*
				var name string
				if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
					name = r.TLS.PeerCertificates[0].Subject.CommonName
				}
			*/

			if r.URL.Path != "/mutate" {
				log.WithField("path", r.URL.Path).Error("Bad Request: 404 Not Found")
				http.NotFound(w, r)
				return
			}

			var body []byte
			if r.Body != nil {
				if data, err := ioutil.ReadAll(r.Body); err == nil {
					body = data
				}
			}
			if len(body) == 0 {
				log.Error("Bad Request: 400 (Empty Body)")
				http.Error(w, "Bad Request (Empty Body)", http.StatusBadRequest)
				return
			}

			contentType := r.Header.Get("Content-Type")
			if contentType != "application/json" {
				log.WithField("Content-Type", contentType).Error("Bad Request: 415 (Unsupported Media Type)")
				http.Error(w, fmt.Sprintf("Bad Request: 415 Unsupported Media Type (Expected Content-Type 'application/json' but got '%s')", contentType), http.StatusUnsupportedMediaType)
				return
			}

			var response *v1beta1.AdmissionResponse
			review := v1beta1.AdmissionReview{}
			if _, _, err := deserializer.Decode(body, nil, &review); err != nil {
				log.WithFields(log.Fields{
					"body":  body,
					"error": err,
				}).Error("Can't decode body")
				response = &v1beta1.AdmissionResponse{
					Allowed: false,
					Result: &metav1.Status{
						Message: err.Error(),
					},
				}
			} else {
				response = mutate(&review, config, provisioner)
			}

			resp, err := json.Marshal(v1beta1.AdmissionReview{
				Response: response,
			})
			if err != nil {
				log.WithFields(log.Fields{
					"uid":   review.Request.UID,
					"error": err,
				}).Info("Marshal error")
				http.Error(w, fmt.Sprintf("Marshal Error: %v", err), http.StatusInternalServerError)
			} else {
				log.WithFields(log.Fields{
					"uid":      review.Request.UID,
					"response": string(resp),
				}).Info("Returning review")
				if _, err := w.Write(resp); err != nil {
					log.WithFields(log.Fields{
						"uid":   review.Request.UID,
						"error": err,
					}).Info("Write error")
				}
			}
		}),
	}, ca.VerifyClientCertIfGiven())
	if err != nil {
		panic(err)
	}

	log.Info("Listening on :4443 ...")
	if err := srv.ListenAndServeTLS("", ""); err != nil {
		panic(err)
	}
}