From a2bd9ad3f548550bed6b3f8c5d21c30373454da9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 20 Oct 2016 11:06:15 -0700 Subject: [PATCH] Use docker container (instead of binary) for kubectl and travis cleanup (#352) This fix uses docker container for kubectl. Since Kubernetes docker image hyperkube has already been downloaded and it consists of kubectl, there is really no need to download kubectl binary again. This fix cleans up the Kubernetes related travis setup and removes unneeded scripts. This fix also fixes several mismatches of the Kubernetes version used, so that any changes in version in the future only need to update .travis.yml. Signed-off-by: Yong Tang --- .travis.yml | 16 ++- .travis/kubernetes/00_run_k8s.sh | 42 ------- .travis/kubernetes/10_setup_kubectl.sh | 23 ---- .travis/kubernetes/20_setup_k8s_services.sh | 119 ------------------ .travis/kubernetes/README.md | 19 +-- .travis/kubernetes/kill_all_containers.sh | 5 - .../{15_run_skydns.sh => run_skydns.sh} | 12 +- .travis/kubernetes/setup_k8s_services.sh | 99 +++++++++++++++ .travis/kubernetes/start_k8s_with_services.sh | 12 -- 9 files changed, 122 insertions(+), 225 deletions(-) delete mode 100755 .travis/kubernetes/00_run_k8s.sh delete mode 100755 .travis/kubernetes/10_setup_kubectl.sh delete mode 100755 .travis/kubernetes/20_setup_k8s_services.sh delete mode 100755 .travis/kubernetes/kill_all_containers.sh rename .travis/kubernetes/{15_run_skydns.sh => run_skydns.sh} (76%) create mode 100755 .travis/kubernetes/setup_k8s_services.sh delete mode 100755 .travis/kubernetes/start_k8s_with_services.sh diff --git a/.travis.yml b/.travis.yml index e5b56a196..2e82bc2da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ go: go_import_path: github.com/miekg/coredns env: - - ETCD_VERSION=2.3.1 K8S_VERSION=1.3.7 + - ETCD_VERSION=2.3.1 K8S_VERSION=1.3.7 KUBECTL="docker exec hyperkube /hyperkube kubectl" DNS_ARGUMENTS="" # In the Travis VM-based build environment, IPv6 networking is not # enabled by default. The sysctl operations below enable IPv6. @@ -30,11 +30,19 @@ before_install: before_script: - docker run -d --net=host --name=etcd quay.io/coreos/etcd:v$ETCD_VERSION - - ./.travis/kubernetes/start_k8s_with_services.sh + - docker run -d --volume=/:/rootfs:ro --volume=/sys:/sys:ro --volume=/var/lib/docker/:/var/lib/docker:rw --volume=/var/lib/kubelet/:/var/lib/kubelet:rw --volume=/var/run:/var/run:rw --net=host --pid=host --privileged --name=hyperkube gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION /hyperkube kubelet --containerized --hostname-override=127.0.0.1 --api-servers=http://localhost:8080 --config=/etc/kubernetes/manifests $DNS_ARGUMENTS --allow-privileged --v=2 + # Wait until kubectl is ready + - for i in {1..10}; do $KUBECTL version && break || sleep 5; done + - $KUBECTL version + - $KUBECTL config set-cluster test-doc --server=http://localhost:8080 + - $KUBECTL config set-context test-doc --cluster=test-doc + - $KUBECTL config use-context test-doc + # Wait until k8s is ready + - for i in {1..30}; do $KUBECTL get nodes && break || sleep 5; done + - .travis/kubernetes/setup_k8s_services.sh + - docker ps -a script: - - docker ps -a - - ./.travis/kubernetes/kubectl version - make coverage after_success: diff --git a/.travis/kubernetes/00_run_k8s.sh b/.travis/kubernetes/00_run_k8s.sh deleted file mode 100755 index 094f7d5d0..000000000 --- a/.travis/kubernetes/00_run_k8s.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e - -# Based on instructions at: http://kubernetes.io/docs/getting-started-guides/docker/ - -#K8S_VERSION=$(curl -sS https://storage.googleapis.com/kubernetes-release/release/latest.txt) -K8S_VERSION=${K8S_VERSION:-"1.3.7"} - -ARCH="amd64" - -export K8S_VERSION -export ARCH - -#RUN_SKYDNS="yes" -RUN_SKYDNS="no" - -if [ "${RUN_SKYDNS}" = "yes" ]; then - DNS_ARGUMENTS="--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" -else - DNS_ARGUMENTS="" -fi - -echo "Starting kubernetes..." - -docker run -d \ - --volume=/:/rootfs:ro \ - --volume=/sys:/sys:ro \ - --volume=/var/lib/docker/:/var/lib/docker:rw \ - --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ - --volume=/var/run:/var/run:rw \ - --net=host \ - --pid=host \ - --privileged \ - gcr.io/google_containers/hyperkube-${ARCH}:v${K8S_VERSION} \ - /hyperkube kubelet \ - --containerized \ - --hostname-override=127.0.0.1 \ - --api-servers=http://localhost:8080 \ - --config=/etc/kubernetes/manifests \ - ${DNS_ARGUMENTS} \ - --allow-privileged --v=2 diff --git a/.travis/kubernetes/10_setup_kubectl.sh b/.travis/kubernetes/10_setup_kubectl.sh deleted file mode 100755 index a7cfc1e76..000000000 --- a/.travis/kubernetes/10_setup_kubectl.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - -PWD=`pwd` -BASEDIR=`readlink -e $(dirname ${0})` - -cd ${BASEDIR} - -echo "Setting up kubectl..." - -if [ ! -e kubectl ]; then - curl -O http://storage.googleapis.com/kubernetes-release/release/v1.3.7/bin/linux/amd64/kubectl - chmod u+x kubectl -fi - -${BASEDIR}/kubectl config set-cluster test-doc --server=http://localhost:8080 -${BASEDIR}/kubectl config set-context test-doc --cluster=test-doc -${BASEDIR}/kubectl config use-context test-doc - -cd ${PWD} - -alias kubctl="${BASEDIR}/kubectl" diff --git a/.travis/kubernetes/20_setup_k8s_services.sh b/.travis/kubernetes/20_setup_k8s_services.sh deleted file mode 100755 index b2f45008d..000000000 --- a/.travis/kubernetes/20_setup_k8s_services.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -PWD=`pwd` -BASEDIR=`readlink -e $(dirname ${0})` - -cd ${BASEDIR} - -KUBECTL='./kubectl' - -wait_until_k8s_ready() { - # Wait until kubernetes is up and fully responsive - while : - do - ${KUBECTL} get nodes 2>/dev/null | grep -q '127.0.0.1' - if [ "${?}" = "0" ]; then - break - else - echo "sleeping for 5 seconds (waiting for kubernetes to start)" - sleep 5 - fi - done - echo "kubernetes nodes:" - ${KUBECTL} get nodes -} - -create_namespaces() { - for n in ${NAMESPACES}; - do - echo "Creating namespace: ${n}" - ${KUBECTL} get namespaces --no-headers 2>/dev/null | grep -q ${n} - if [ "${?}" != "0" ]; then - ${KUBECTL} create namespace ${n} - fi - done - - echo "kubernetes namespaces:" - ${KUBECTL} get namespaces -} - -# run_and_expose_service -run_and_expose_service() { - if [ "${#}" != "4" ]; then - return -1 - fi - - service="${1}" - namespace="${2}" - image="${3}" - port="${4}" - - echo " starting service '${service}' in namespace '${namespace}'" - - ${KUBECTL} get deployment --namespace=${namespace} --no-headers 2>/dev/null | grep -q ${service} - if [ "${?}" != "0" ]; then - ${KUBECTL} run ${service} --namespace=${namespace} --image=${image} - else - echo "warn: service '${service}' already running in namespace '${namespace}'" - fi - - ${KUBECTL} get service --namespace=${namespace} --no-headers 2>/dev/null | grep -q ${service} - if [ "${?}" != "0" ]; then - ${KUBECTL} expose deployment ${service} --namespace=${namespace} --port=${port} - else - echo "warn: service '${service}' already exposed in namespace '${namespace}'" - fi -} - - -#run_and_expose_rc nginx-controller nginx-rc.yml poddemo 80 -run_and_expose_rc() { - if [ "${#}" != "4" ]; then - return -1 - fi - - rc_name="${1}" - rc_file="${2}" - namespace="${3}" - port="${4}" - - echo " starting replication controller '${rc_name}' from '${rc_file}' in namespace '${namespace}'" - - ${KUBECTL} get rc --namespace=${namespace} --no-headers 2>/dev/null | grep -q ${rc_name} - if [ "${?}" != "0" ]; then - ${KUBECTL} expose -f ${rc_file} --namespace=${namespace} --port=${port} - else - echo "warn: rc '${rc_name}' already running in namespace '${namespace}'" - fi -} - -echo "Starting sample kubernetes services..." - -wait_until_k8s_ready - -NAMESPACES="demo poddemo test" -create_namespaces - -echo "" -echo "Starting services:" - -run_and_expose_service mynginx demo nginx 80 -run_and_expose_service webserver demo nginx 80 -run_and_expose_service mynginx test nginx 80 -run_and_expose_service webserver test nginx 80 - -echo "" -echo "Services exposed:" -${KUBECTL} get services --all-namespaces - -echo "" -echo "Starting replicationcontrollers:" - -run_and_expose_rc nginx-controller nginx-rc.yml poddemo 80 - -echo "" -echo "ReplicationControllers exposed:" -${KUBECTL} get rc --all-namespaces - - -cd ${PWD} diff --git a/.travis/kubernetes/README.md b/.travis/kubernetes/README.md index 9642b42a8..394cfe259 100644 --- a/.travis/kubernetes/README.md +++ b/.travis/kubernetes/README.md @@ -2,7 +2,6 @@ Requirements: docker - curl The scripts in this directory startup kubernetes with docker as the container runtime. After starting kubernetes, a couple of kubernetes services are started to allow automatic @@ -10,16 +9,10 @@ testing of CoreDNS with kubernetes. The kubernetes integration tests in `test/ku automate the launch of kubernetes and the creation of the expected sample services. To start up kubernetes and launch some sample services, -run the script `start_k8s_with_services.sh`. +run the script `setup_k8s_services.sh`. ~~~ -$ ./start_k8s_with_services.sh -~~~ - -Alternatively, the individual scripts may be run independently as needed: - -~~~ -$ ./00_run_k8s.sh && ./10_setup_kubectl.sh && ./20_setup_k8s_services.sh +$ ./setup_k8s_services.sh ~~~ After running the above scripts, kubernetes will be running on the localhost with the following services @@ -33,11 +26,3 @@ demo webserver 10.0.0.28 80/TCP 2m test mynginx 10.0.0.4 80/TCP 2m test webserver 10.0.0.39 80/TCP 2m ~~ - - -Kubernetes and all running containers can be uncerimoniously stopped by -running the `kill_all_containers.sh` script. - -~~~ -$ ./kill_all_containers.sh -~~~ diff --git a/.travis/kubernetes/kill_all_containers.sh b/.travis/kubernetes/kill_all_containers.sh deleted file mode 100755 index aa7f4255d..000000000 --- a/.travis/kubernetes/kill_all_containers.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -docker rm -f $(docker ps -a -q) -sleep 1 -docker rm -f $(docker ps -a -q) diff --git a/.travis/kubernetes/15_run_skydns.sh b/.travis/kubernetes/run_skydns.sh similarity index 76% rename from .travis/kubernetes/15_run_skydns.sh rename to .travis/kubernetes/run_skydns.sh index 266d83dcc..f7000a37f 100755 --- a/.travis/kubernetes/15_run_skydns.sh +++ b/.travis/kubernetes/run_skydns.sh @@ -4,19 +4,25 @@ PWD=`pwd` BASEDIR=`readlink -e $(dirname ${0})` - cd ${BASEDIR} -KUBECTL='./kubectl' +KUBECTL='docker exec hyperkube /hyperkube kubectl' #RUN_SKYDNS="yes" RUN_SKYDNS="no" +# DNS_ARGUMENTS needs to be passed when Kubernetes is setup. +if [ "${RUN_SKYDNS}" = "yes" ]; then + DNS_ARGUMENTS="--cluster-dns=10.0.0.10 --cluster-domain=cluster.local" +else + DNS_ARGUMENTS="" +fi + wait_until_k8s_ready() { # Wait until kubernetes is up and fully responsive while : do - ${KUBECTL} get nodes 2>/dev/null | grep -q '127.0.0.1' + ${KUBECTL} get nodes 2>/dev/null | grep -q '127.0.0.1' if [ "${?}" = "0" ]; then break else diff --git a/.travis/kubernetes/setup_k8s_services.sh b/.travis/kubernetes/setup_k8s_services.sh new file mode 100755 index 000000000..aae198a50 --- /dev/null +++ b/.travis/kubernetes/setup_k8s_services.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +set -x + +KUBECTL='docker exec hyperkube /hyperkube kubectl' + +PWD=`pwd` +cd `readlink -e $(dirname ${0})` + +create_namespaces() { + for n in ${NAMESPACES}; + do + echo "Creating namespace: ${n}" + ${KUBECTL} get namespaces --no-headers 2>/dev/null | grep -q ${n} + if [ "${?}" != "0" ]; then + ${KUBECTL} create namespace ${n} + fi + done + + echo "kubernetes namespaces:" + ${KUBECTL} get namespaces +} + +# run_and_expose_service +run_and_expose_service() { + if [ "${#}" != "4" ]; then + return -1 + fi + + service="${1}" + namespace="${2}" + image="${3}" + port="${4}" + + echo " starting service '${service}' in namespace '${namespace}'" + + ${KUBECTL} get deployment --namespace=${namespace} --no-headers 2>/dev/null | grep -q ${service} + if [ "${?}" != "0" ]; then + ${KUBECTL} run ${service} --namespace=${namespace} --image=${image} + else + echo "warn: service '${service}' already running in namespace '${namespace}'" + fi + + ${KUBECTL} get service --namespace=${namespace} --no-headers 2>/dev/null | grep -q ${service} + if [ "${?}" != "0" ]; then + ${KUBECTL} expose deployment ${service} --namespace=${namespace} --port=${port} + else + echo "warn: service '${service}' already exposed in namespace '${namespace}'" + fi +} + +#run_and_expose_rc nginx-controller nginx-rc.yml poddemo 80 +run_and_expose_rc() { + if [ "${#}" != "4" ]; then + return -1 + fi + + rc_name="${1}" + rc_file="${2}" + namespace="${3}" + port="${4}" + + echo " starting replication controller '${rc_name}' from '${rc_file}' in namespace '${namespace}'" + + ${KUBECTL} get rc --namespace=${namespace} --no-headers 2>/dev/null | grep -q ${rc_name} + if [ "${?}" != "0" ]; then + ${KUBECTL} expose -f ${rc_file} --namespace=${namespace} --port=${port} + else + echo "warn: rc '${rc_name}' already running in namespace '${namespace}'" + fi +} + +echo "Starting sample kubernetes services..." + +NAMESPACES="demo poddemo test" +create_namespaces + +echo "" +echo "Starting services:" + +run_and_expose_service mynginx demo nginx 80 +run_and_expose_service webserver demo nginx 80 +run_and_expose_service mynginx test nginx 80 +run_and_expose_service webserver test nginx 80 + +echo "" +echo "Services exposed:" +${KUBECTL} get services --all-namespaces + +echo "" +echo "Starting replicationcontrollers:" + +run_and_expose_rc nginx-controller nginx-rc.yml poddemo 80 + +echo "" +echo "ReplicationControllers exposed:" +${KUBECTL} get rc --all-namespaces + +cd ${PWD} diff --git a/.travis/kubernetes/start_k8s_with_services.sh b/.travis/kubernetes/start_k8s_with_services.sh deleted file mode 100755 index 40db49715..000000000 --- a/.travis/kubernetes/start_k8s_with_services.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -PWD=`pwd` -BASEDIR=`readlink -e $(dirname ${0})` - -cd ${BASEDIR} - -./00_run_k8s.sh && \ -./10_setup_kubectl.sh && \ -./20_setup_k8s_services.sh - -cd ${PWD}