2 ################################################################################
3 # Copyright (c) 2019 AT&T Intellectual Property. #
4 # Copyright (c) 2019 Nokia. #
6 # Licensed under the Apache License, Version 2.0 (the "License"); #
7 # you may not use this file except in compliance with the License. #
8 # You may obtain a copy of the License at #
10 # http://www.apache.org/licenses/LICENSE-2.0 #
12 # Unless required by applicable law or agreed to in writing, software #
13 # distributed under the License is distributed on an "AS IS" BASIS, #
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
15 # See the License for the specific language governing permissions and #
16 # limitations under the License. #
17 ################################################################################
20 # first parameter: number of expected running pods
21 # second parameter: namespace (all-namespaces means all namespaces)
22 # third parameter: [optional] keyword
23 wait_for_pods_running () {
25 CMD="kubectl get pods --all-namespaces "
26 if [ "$NS" != "all-namespaces" ]; then
27 CMD="kubectl get pods -n $2 "
30 if [ "$#" == "3" ]; then
31 KEYWORD="${3}.*Running"
34 CMD2="$CMD | grep \"$KEYWORD\" | wc -l"
35 NUMPODS=$(eval "$CMD2")
36 echo "waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
37 while [ $NUMPODS -lt $1 ]; do
39 NUMPODS=$(eval "$CMD2")
40 echo "> waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
45 # first parameter: interface name
47 # enable ipv6 interface
48 # standard Ubuntu cloud image does not have dual interface configuration or ipv6
50 if ifconfig -a $IPv6IF; then
51 echo "" >> /etc/network/interfaces.d/50-cloud-init.cfg
52 #echo "auto ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
53 echo "allow-hotplug ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
54 echo "iface ${IPv6IF} inet6 auto" >> /etc/network/interfaces.d/50-cloud-init.cfg
56 #systemctl restart networking
61 echo "k8s_vm_install.sh"
63 export DEBIAN_FRONTEND=noninteractive
64 echo "__host_private_ip_addr__ $(hostname)" >> /etc/hosts
68 echo "__docker_version__" > /opt/config/docker_version.txt
69 echo "__k8s_version__" > /opt/config/k8s_version.txt
70 echo "__k8s_cni_version__" > /opt/config/k8s_cni_version.txt
71 echo "__helm_version__" > /opt/config/helm_version.txt
72 echo "__host_private_ip_addr__" > /opt/config/host_private_ip_addr.txt
73 echo "__k8s_mst_floating_ip_addr__" > /opt/config/k8s_mst_floating_ip_addr.txt
74 echo "__k8s_mst_private_ip_addr__" > /opt/config/k8s_mst_private_ip_addr.txt
75 echo "__mtu__" > /opt/config/mtu.txt
76 echo "__cinder_volume_id__" > /opt/config/cinder_volume_id.txt
77 echo "__stack_name__" > /opt/config/stack_name.txt
80 if [[ $(cat /opt/config/stack_name.txt) == *aux* ]]; then
88 modprobe -- nf_conntrack_ipv4
89 modprobe -- nf_conntrack_ipv6
90 modprobe -- nf_conntrack_proto_sctp
95 SWAPFILES=$(grep swap /etc/fstab | sed '/^#/ d' |cut -f1 -d' ')
96 if [ ! -z $SWAPFILES ]; then
97 for SWAPFILE in $SWAPFILES
99 if [ ! -z $SWAPFILE ]; then
100 echo "disabling swap file $SWAPFILE"
101 if [[ $SWAPFILE == UUID* ]]; then
102 UUID=$(echo $SWAPFILE | cut -f2 -d'=')
107 # edit /etc/fstab file, remove line with /swapfile
108 sed -i -e "/$SWAPFILE/d" /etc/fstab
114 # edit /etc/fstab file, remove line with /swapfile
115 #sed -i -e '/swapfile/d' /etc/fstab
118 DOCKERV=$(cat /opt/config/docker_version.txt)
119 KUBEV=$(cat /opt/config/k8s_version.txt)
120 KUBECNIV=$(cat /opt/config/k8s_cni_version.txt)
122 KUBEVERSION="${KUBEV}-00"
123 CNIVERSION="${KUBECNIV}-00"
124 DOCKERVERSION="${DOCKERV}"
125 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
126 echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
128 # install low latency kernel, docker.io, and kubernetes
130 apt-get install -y virt-what
131 if ! echo $(virt-what) | grep "virtualbox"; then
132 # this version of low latency kernel causes virtualbox VM to hand.
133 # install if identifying the VM not being a virtualbox VM.
134 apt-get install -y linux-image-4.15.0-45-lowlatency
136 apt-get install -y curl jq netcat docker.io=${DOCKERVERSION}
137 apt-get install -y kubernetes-cni=${CNIVERSION}
138 apt-get install -y --allow-unauthenticated kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
139 apt-mark hold docker.io kubernetes-cni kubelet kubeadm kubectl
142 # test access to k8s docker registry
143 kubeadm config images pull
146 # non-master nodes have hostnames ending with -[0-9][0-9]
147 if [[ $(hostname) == *-[0-9][0-9] ]]; then
148 echo "Done for non-master node"
149 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
152 # below are steps for initializating master node, only run on the master node.
153 # minion node join will be triggered from the caller of the stack creation as ssh command.
156 # create kubenetes config file
157 if [[ ${KUBEV} == 1.13.* ]]; then
158 cat <<EOF >/root/config.yaml
159 apiVersion: kubeadm.k8s.io/v1alpha3
160 kubernetesVersion: v${KUBEV}
161 kind: ClusterConfiguration
163 feature-gates: SCTPSupport=true
165 dnsDomain: cluster.local
166 podSubnet: 10.244.0.0/16
167 serviceSubnet: 10.96.0.0/12
170 apiVersion: kubeproxy.config.k8s.io/v1alpha1
171 kind: KubeProxyConfiguration
175 elif [[ ${KUBEV} == 1.14.* ]]; then
176 cat <<EOF >/root/config.yaml
177 apiVersion: kubeadm.k8s.io/v1beta1
178 kubernetesVersion: v${KUBEV}
179 kind: ClusterConfiguration
181 feature-gates: SCTPSupport=true
183 dnsDomain: cluster.local
184 podSubnet: 10.244.0.0/16
185 serviceSubnet: 10.96.0.0/12
188 apiVersion: kubeproxy.config.k8s.io/v1alpha1
189 kind: KubeProxyConfiguration
194 echo "Unsupported Kubernetes version requested. Bail."
199 # create a RBAC file for helm (tiller)
200 cat <<EOF > /root/rbac-config.yaml
205 namespace: kube-system
207 apiVersion: rbac.authorization.k8s.io/v1
208 kind: ClusterRoleBinding
212 apiGroup: rbac.authorization.k8s.io
216 - kind: ServiceAccount
218 namespace: kube-system
221 # start cluster (make sure CIDR is enabled with the flag)
222 kubeadm init --config /root/config.yaml
226 HELMV=$(cat /opt/config/helm_version.txt)
231 wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELMVERSION}-linux-amd64.tar.gz
232 tar -xvf helm-v${HELMVERSION}-linux-amd64.tar.gz
233 mv linux-amd64/helm /usr/local/bin/helm
235 # set up kubectl credential and config
239 cp -i /etc/kubernetes/admin.conf /root/.kube/config
240 chown root:root /root/.kube/config
242 # at this point we should be able to use kubectl
243 kubectl get pods --all-namespaces
246 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
249 # waiting for all 8 kube-system pods to be in running state
250 # (at this point, minions have not joined yet)
251 wait_for_pods_running 8 kube-system
253 # if running a single node cluster, need to enable master node to run pods
254 kubectl taint nodes --all node-role.kubernetes.io/master-
257 # install RBAC for Helm
258 kubectl create -f rbac-config.yaml
262 helm init --service-account tiller
263 export HELM_HOME="/root/.helm"
265 # waiting for tiller pod to be in running state
266 wait_for_pods_running 1 kube-system tiller-deploy
268 while ! helm version; do
269 echo "Waiting for Helm to be ready"
274 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
277 echo "Done with master node setup"
281 # add rancodev CI tool hostnames
282 if [[ ! -z "${__RUNRICENV_GERRIT_IP__}" && ! -z "${__RUNRICENV_GERRIT_HOST__}" ]]; then
283 echo "${__RUNRICENV_GERRIT_IP__} ${__RUNRICENV_GERRIT_HOST__}" >> /etc/hosts
285 if [[ ! -z "${__RUNRICENV_DOCKER_IP__}" && ! -z "${__RUNRICENV_DOCKER_HOST__}" ]]; then
286 echo "${__RUNRICENV_DOCKER_IP__} ${__RUNRICENV_DOCKER_HOST__}" >> /etc/hosts
288 if [[ ! -z "${__RUNRICENV_HELMREPO_IP__}" && ! -z "${__RUNRICENV_HELMREPO_HOST__}" ]]; then
289 echo "${__RUNRICENV_HELMREPO_IP__} ${__RUNRICENV_HELMREPO_HOST__}" >> /etc/hosts
292 if [ ! -z "${__RUNRICENV_HELMREPO_CERT__}" ]; then
293 cat <<EOF >/etc/ca-certificates/update.d/helm.crt
294 ${__RUNRICENV_HELMREPO_CERT__}
298 # add cert for accessing docker registry in Azure
299 if [ ! -z "${__RUNRICENV_DOCKER_CERT__}" ]; then
300 mkdir -p /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
301 cat <<EOF >/etc/docker/ca.crt
302 ${__RUNRICENV_DOCKER_CERT__}
304 cp /etc/docker/ca.crt /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/ca.crt
306 service docker restart
307 systemctl enable docker.service
308 docker login -u ${__RUNRICENV_DOCKER_USER__} -p ${__RUNRICENV_DOCKER_PASS__} ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
309 docker pull ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/whoami:0.0.1