2 ################################################################################
3 # Copyright (c) 2019 AT&T Intellectual Property. #
5 # Licensed under the Apache License, Version 2.0 (the "License"); #
6 # you may not use this file except in compliance with the License. #
7 # You may obtain a copy of the License at #
9 # http://www.apache.org/licenses/LICENSE-2.0 #
11 # Unless required by applicable law or agreed to in writing, software #
12 # distributed under the License is distributed on an "AS IS" BASIS, #
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
14 # See the License for the specific language governing permissions and #
15 # limitations under the License. #
16 ################################################################################
19 # first parameter: number of expected running pods
20 # second parameter: namespace (all-namespaces means all namespaces)
21 # third parameter: [optional] keyword
22 wait_for_pods_running () {
24 CMD="kubectl get pods --all-namespaces "
25 if [ "$NS" != "all-namespaces" ]; then
26 CMD="kubectl get pods -n $2 "
29 if [ "$#" == "3" ]; then
30 KEYWORD="${3}.*Running"
33 CMD2="$CMD | grep \"$KEYWORD\" | wc -l"
34 NUMPODS=$(eval "$CMD2")
35 echo "waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
36 while [ $NUMPODS -lt $1 ]; do
38 NUMPODS=$(eval "$CMD2")
39 echo "> waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
44 # first parameter: interface name
46 # enable ipv6 interface
47 # standard Ubuntu cloud image does not have dual interface configuration or ipv6
49 if ifconfig -a $IPv6IF; then
50 echo "" >> /etc/network/interfaces.d/50-cloud-init.cfg
51 echo "allow-hotplug ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
52 echo "iface ${IPv6IF} inet6 auto" >> /etc/network/interfaces.d/50-cloud-init.cfg
57 echo "k8s_vm_install.sh"
59 export DEBIAN_FRONTEND=noninteractive
60 echo "__host_private_ip_addr__ $(hostname)" >> /etc/hosts
67 echo "__docker_version__" > /opt/config/docker_version.txt
68 echo "__k8s_version__" > /opt/config/k8s_version.txt
69 echo "__k8s_cni_version__" > /opt/config/k8s_cni_version.txt
70 echo "__helm_version__" > /opt/config/helm_version.txt
71 echo "__host_private_ip_addr__" > /opt/config/host_private_ip_addr.txt
72 echo "__k8s_mst_floating_ip_addr__" > /opt/config/k8s_mst_floating_ip_addr.txt
73 echo "__k8s_mst_private_ip_addr__" > /opt/config/k8s_mst_private_ip_addr.txt
74 echo "__mtu__" > /opt/config/mtu.txt
75 echo "__cinder_volume_id__" > /opt/config/cinder_volume_id.txt
76 echo "__stack_name__" > /opt/config/stack_name.txt
78 # assume we are setting up AUX cluster VM if hostname contains "aux"
80 if [[ $(cat /opt/config/stack_name.txt) == *aux* ]]; then
88 modprobe -- nf_conntrack_ipv4
89 modprobe -- nf_conntrack_ipv6
90 modprobe -- nf_conntrack_proto_sctp
92 if [ ! -z "$IPV6IF" ]; then
97 #SWAPFILES=$(grep swap /etc/fstab | sed '/^[ \t]*#/ d' |cut -f1 -d' ')
98 SWAPFILES=$(grep swap /etc/fstab | sed '/^[ \t]*#/ d' | sed 's/[\t ]/ /g' | tr -s " " | cut -f1 -d' ')
99 if [ ! -z $SWAPFILES ]; then
100 for SWAPFILE in $SWAPFILES
102 if [ ! -z $SWAPFILE ]; then
103 echo "disabling swap file $SWAPFILE"
104 if [[ $SWAPFILE == UUID* ]]; then
105 UUID=$(echo $SWAPFILE | cut -f2 -d'=')
110 sed -i "\%$SWAPFILE%d" /etc/fstab
116 DOCKERV=$(cat /opt/config/docker_version.txt)
117 KUBEV=$(cat /opt/config/k8s_version.txt)
118 KUBECNIV=$(cat /opt/config/k8s_cni_version.txt)
120 KUBEVERSION="${KUBEV}-00"
121 CNIVERSION="${KUBECNIV}-00"
122 DOCKERVERSION="${DOCKERV}"
124 # adjust package version tag
125 UBUNTU_RELEASE=$(lsb_release -r | sed 's/^[a-zA-Z:\t ]\+//g')
126 if [[ ${UBUNTU_RELEASE} == 16.* ]]; then
127 echo "Installing on Ubuntu $UBUNTU_RELEASE (Xenial Xerus) host"
128 if [ ! -z "${DOCKERV}" ]; then
129 DOCKERVERSION="${DOCKERV}-0ubuntu1~16.04.5"
131 elif [[ ${UBUNTU_RELEASE} == 18.* ]]; then
132 echo "Installing on Ubuntu $UBUNTU_RELEASE (Bionic Beaver)"
133 if [ ! -z "${DOCKERV}" ]; then
134 DOCKERVERSION="${DOCKERV}-0ubuntu1~18.04.5"
137 echo "Unsupported Ubuntu release ($UBUNTU_RELEASE) detected. Exit."
142 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
143 echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
145 # tell apt to retry 3 times if failed
146 mkdir -p /etc/apt/apt.conf.d
147 echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries
149 # install low latency kernel, docker.io, and kubernetes
151 apt-get -y autoremove
152 RES=$(apt-get install -y virt-what curl jq netcat 2>&1)
153 if [[ $RES == */var/lib/dpkg/lock* ]]; then
154 echo "Fail to get dpkg lock. Wait for any other package installation"
155 echo "process to finish, then rerun this script"
159 if ! echo $(virt-what) | grep "virtualbox"; then
160 # this version of low latency kernel causes virtualbox VM to hang.
161 # install if identifying the VM not being a virtualbox VM.
162 apt-get install -y linux-image-4.15.0-45-lowlatency
165 if kubeadm version; then
166 # remove existing Kubernetes installation
167 echo "Removing existing Kubernetes installation, version $(kubeadm version)"
172 APTOPTS="--allow-downgrades --allow-change-held-packages --allow-unauthenticated --ignore-hold "
173 if [ -z ${DOCKERVERSION} ]; then
174 apt-get install -y $APTOPTS docker.io
176 apt-get install -y $APTOPTS docker.io=${DOCKERVERSION}
178 cat > /etc/docker/daemon.json <<EOF
180 "exec-opts": ["native.cgroupdriver=systemd"],
181 "log-driver": "json-file",
185 "storage-driver": "overlay2"
188 mkdir -p /etc/systemd/system/docker.service.d
189 systemctl enable docker.service
190 systemctl daemon-reload
191 systemctl restart docker
193 if [ -z ${CNIVERSION} ]; then
194 apt-get install -y $APTOPTS kubernetes-cni
196 apt-get install -y $APTOPTS kubernetes-cni=${CNIVERSION}
199 if [ -z ${KUBEVERSION} ]; then
200 apt-get install -y $APTOPTS kubeadm kubelet kubectl
202 apt-get install -y $APTOPTS kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
205 apt-mark hold docker.io kubernetes-cni kubelet kubeadm kubectl
208 # test access to k8s docker registry
209 kubeadm config images pull --kubernetes-version=${KUBEV}
213 # non-master nodes have hostnames ending with -[0-9][0-9]
214 if [ "$NODETYPE" == "master" ]; then
215 # below are steps for initializating master node, only run on the master node.
216 # minion node join will be triggered from the caller of the stack creation as ssh command.
218 # create kubenetes config file
219 if [[ ${KUBEV} == 1.13.* ]]; then
220 cat <<EOF >/root/config.yaml
221 apiVersion: kubeadm.k8s.io/v1alpha3
222 kubernetesVersion: v${KUBEV}
223 kind: ClusterConfiguration
225 feature-gates: SCTPSupport=true
227 dnsDomain: cluster.local
228 podSubnet: 10.244.0.0/16
229 serviceSubnet: 10.96.0.0/12
231 apiVersion: kubeproxy.config.k8s.io/v1alpha1
232 kind: KubeProxyConfiguration
236 elif [[ ${KUBEV} == 1.14.* ]]; then
237 cat <<EOF >/root/config.yaml
238 apiVersion: kubeadm.k8s.io/v1beta1
239 kubernetesVersion: v${KUBEV}
240 kind: ClusterConfiguration
242 feature-gates: SCTPSupport=true
244 dnsDomain: cluster.local
245 podSubnet: 10.244.0.0/16
246 serviceSubnet: 10.96.0.0/12
248 apiVersion: kubeproxy.config.k8s.io/v1alpha1
249 kind: KubeProxyConfiguration
252 elif [[ ${KUBEV} == 1.16.* ]]; then
253 cat <<EOF >/root/config.yaml
254 apiVersion: kubeadm.k8s.io/v1beta2
255 kubernetesVersion: v${KUBEV}
256 kind: ClusterConfiguration
259 feature-gates: SCTPSupport=true
261 dnsDomain: cluster.local
262 podSubnet: 10.244.0.0/16
263 serviceSubnet: 10.96.0.0/12
265 apiVersion: kubeproxy.config.k8s.io/v1alpha1
266 kind: KubeProxyConfiguration
270 echo "Unsupported Kubernetes version requested. Bail."
274 # create a RBAC file for helm (tiller)
275 cat <<EOF > /root/rbac-config.yaml
280 namespace: kube-system
282 apiVersion: rbac.authorization.k8s.io/v1
283 kind: ClusterRoleBinding
287 apiGroup: rbac.authorization.k8s.io
291 - kind: ServiceAccount
293 namespace: kube-system
297 # start cluster (make sure CIDR is enabled with the flag)
298 kubeadm init --config /root/config.yaml
300 # set up kubectl credential and config
304 cp -i /etc/kubernetes/admin.conf /root/.kube/config
305 chown root:root /root/.kube/config
307 # at this point we should be able to use kubectl
308 kubectl get pods --all-namespaces
311 if [[ ${KUBEV} == 1.16.* ]]; then
312 kubectl apply -f "https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"
314 kubectl apply -f "https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml"
317 # waiting for all 8 kube-system pods to be in running state
318 # (at this point, minions have not joined yet)
319 wait_for_pods_running 8 kube-system
321 # if running a single node cluster, need to enable master node to run pods
322 kubectl taint nodes --all node-role.kubernetes.io/master-
325 # install RBAC for Helm
326 kubectl create -f rbac-config.yaml
329 HELMV=$(cat /opt/config/helm_version.txt)
331 cd /root && rm -rf Helm && mkdir Helm && cd Helm
332 wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELMVERSION}-linux-amd64.tar.gz
333 tar -xvf helm-v${HELMVERSION}-linux-amd64.tar.gz
334 mv linux-amd64/helm /usr/local/bin/helm
337 if [[ ${KUBEV} == 1.16.* ]]; then
338 # helm init uses API extensions/v1beta1 which is depreciated by Kubernetes
339 # 1.16.0. Until upstream (helm) provides a fix, this is the work-around.
340 helm init --service-account tiller --override spec.selector.matchLabels.'name'='tiller',spec.selector.matchLabels.'app'='helm' --output yaml > helm-init.yaml
341 sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' ./helm-init.yaml > helm-init-patched.yaml
342 kubectl apply -f ./helm-init-patched.yaml
344 helm init --service-account tiller
347 export HELM_HOME="/root/.helm"
349 # waiting for tiller pod to be in running state
350 wait_for_pods_running 1 kube-system tiller-deploy
351 while ! helm version; do
352 echo "Waiting for Helm to be ready"
356 echo "Preparing a master node (lowser ID) for using local FS for PV"
357 PV_NODE_NAME=$(kubectl get nodes |grep master | cut -f1 -d' ' | sort | head -1)
358 kubectl label --overwrite nodes $PV_NODE_NAME local-storage=enable
359 if [ "$PV_NODE_NAME" == "$(hostname)" ]; then
360 mkdir -p /opt/data/dashboard-data
363 echo "Done with master node setup"
367 # add rancodev CI tool hostnames
368 if [[ ! -z "${__RUNRICENV_GERRIT_IP__}" && ! -z "${__RUNRICENV_GERRIT_HOST__}" ]]; then
369 echo "${__RUNRICENV_GERRIT_IP__} ${__RUNRICENV_GERRIT_HOST__}" >> /etc/hosts
371 if [[ ! -z "${__RUNRICENV_DOCKER_IP__}" && ! -z "${__RUNRICENV_DOCKER_HOST__}" ]]; then
372 echo "${__RUNRICENV_DOCKER_IP__} ${__RUNRICENV_DOCKER_HOST__}" >> /etc/hosts
374 if [[ ! -z "${__RUNRICENV_HELMREPO_IP__}" && ! -z "${__RUNRICENV_HELMREPO_HOST__}" ]]; then
375 echo "${__RUNRICENV_HELMREPO_IP__} ${__RUNRICENV_HELMREPO_HOST__}" >> /etc/hosts
378 if [[ "${__RUNRICENV_HELMREPO_CERT_LEN__}" -gt "100" ]]; then
379 cat <<EOF >/etc/ca-certificates/update.d/helm.crt
380 ${__RUNRICENV_HELMREPO_CERT__}
384 # add cert for accessing docker registry in Azure
385 if [[ "${__RUNRICENV_DOCKER_CERT_LEN__}" -gt "100" ]]; then
386 mkdir -p /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
387 cat <<EOF >/etc/docker/ca.crt
388 ${__RUNRICENV_DOCKER_CERT__}
390 cp /etc/docker/ca.crt /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/ca.crt
392 service docker restart
393 systemctl enable docker.service
394 docker login -u ${__RUNRICENV_DOCKER_USER__} -p ${__RUNRICENV_DOCKER_PASS__} ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
395 docker pull ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/whoami:0.0.1
398 if [ "$(uname -r)" != "4.15.0-45-lowlatency" ]; then reboot; fi