2 ################################################################################
3 # Copyright (c) 2019 AT&T Intellectual Property. #
4 # Copyright (c) 2019 Nokia. #
6 # Licensed under the Apache License, Version 2.0 (the "License"); #
7 # you may not use this file except in compliance with the License. #
8 # You may obtain a copy of the License at #
10 # http://www.apache.org/licenses/LICENSE-2.0 #
12 # Unless required by applicable law or agreed to in writing, software #
13 # distributed under the License is distributed on an "AS IS" BASIS, #
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
15 # See the License for the specific language governing permissions and #
16 # limitations under the License. #
17 ################################################################################
20 # first parameter: number of expected running pods
21 # second parameter: namespace (all-namespaces means all namespaces)
22 # third parameter: [optional] keyword
23 wait_for_pods_running () {
25 CMD="kubectl get pods --all-namespaces "
26 if [ "$NS" != "all-namespaces" ]; then
27 CMD="kubectl get pods -n $2 "
30 if [ "$#" == "3" ]; then
31 KEYWORD="${3}.*Running"
34 CMD2="$CMD | grep \"$KEYWORD\" | wc -l"
35 NUMPODS=$(eval "$CMD2")
36 echo "waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
37 while [ $NUMPODS -lt $1 ]; do
39 NUMPODS=$(eval "$CMD2")
40 echo "> waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
45 # first parameter: interface name
47 # enable ipv6 interface
48 # standard Ubuntu cloud image does not have dual interface configuration or ipv6
50 if ifconfig -a $IPv6IF; then
51 echo "" >> /etc/network/interfaces.d/50-cloud-init.cfg
52 #echo "auto ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
53 echo "allow-hotplug ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
54 echo "iface ${IPv6IF} inet6 auto" >> /etc/network/interfaces.d/50-cloud-init.cfg
56 #systemctl restart networking
61 echo "k8s_vm_install.sh"
63 export DEBIAN_FRONTEND=noninteractive
64 echo "__host_private_ip_addr__ $(hostname)" >> /etc/hosts
68 echo "__docker_version__" > /opt/config/docker_version.txt
69 echo "__k8s_version__" > /opt/config/k8s_version.txt
70 echo "__k8s_cni_version__" > /opt/config/k8s_cni_version.txt
71 echo "__helm_version__" > /opt/config/helm_version.txt
72 echo "__host_private_ip_addr__" > /opt/config/host_private_ip_addr.txt
73 echo "__k8s_mst_floating_ip_addr__" > /opt/config/k8s_mst_floating_ip_addr.txt
74 echo "__k8s_mst_private_ip_addr__" > /opt/config/k8s_mst_private_ip_addr.txt
75 echo "__mtu__" > /opt/config/mtu.txt
76 echo "__cinder_volume_id__" > /opt/config/cinder_volume_id.txt
77 echo "__stack_name__" > /opt/config/stack_name.txt
80 if [[ $(cat /opt/config/stack_name.txt) == *aux* ]]; then
88 modprobe -- nf_conntrack_ipv4
89 modprobe -- nf_conntrack_ipv6
90 modprobe -- nf_conntrack_proto_sctp
95 SWAPFILES=$(grep swap /etc/fstab | sed '/^#/ d' |cut -f1 -d' ')
96 if [ ! -z $SWAPFILES ]; then
97 for SWAPFILE in $SWAPFILES
99 if [ ! -z $SWAPFILE ]; then
100 echo "disabling swap file $SWAPFILE"
101 if [[ $SWAPFILE == UUID* ]]; then
102 UUID=$(echo $SWAPFILE | cut -f2 -d'=')
107 # edit /etc/fstab file, remove line with /swapfile
108 sed -i -e "/$SWAPFILE/d" /etc/fstab
114 # edit /etc/fstab file, remove line with /swapfile
115 #sed -i -e '/swapfile/d' /etc/fstab
118 DOCKERV=$(cat /opt/config/docker_version.txt)
119 KUBEV=$(cat /opt/config/k8s_version.txt)
120 KUBECNIV=$(cat /opt/config/k8s_cni_version.txt)
122 KUBEVERSION="${KUBEV}-00"
123 CNIVERSION="${KUBECNIV}-00"
124 DOCKERVERSION="${DOCKERV}"
125 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
126 echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
128 # tell apt to retry 3 times if failed
129 mkdir -p /etc/apt/apt.conf.d
130 echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries
132 # install low latency kernel, docker.io, and kubernetes
134 apt-get install -y virt-what
135 if ! echo $(virt-what) | grep "virtualbox"; then
136 # this version of low latency kernel causes virtualbox VM to hand.
137 # install if identifying the VM not being a virtualbox VM.
138 apt-get install -y linux-image-4.15.0-45-lowlatency
140 if [ -z ${DOCKERVERSION} ]; then
141 apt-get install -y curl jq netcat docker.io
143 apt-get install -y curl jq netcat docker.io=${DOCKERVERSION}
145 apt-get install -y kubernetes-cni=${CNIVERSION}
146 apt-get install -y --allow-unauthenticated kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
147 apt-mark hold docker.io kubernetes-cni kubelet kubeadm kubectl
150 # test access to k8s docker registry
151 kubeadm config images pull
154 # non-master nodes have hostnames ending with -[0-9][0-9]
155 if [[ $(hostname) == *-[0-9][0-9] ]]; then
156 echo "Done for non-master node"
157 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
160 # below are steps for initializating master node, only run on the master node.
161 # minion node join will be triggered from the caller of the stack creation as ssh command.
164 # create kubenetes config file
165 if [[ ${KUBEV} == 1.13.* ]]; then
166 cat <<EOF >/root/config.yaml
167 apiVersion: kubeadm.k8s.io/v1alpha3
168 kubernetesVersion: v${KUBEV}
169 kind: ClusterConfiguration
171 feature-gates: SCTPSupport=true
173 dnsDomain: cluster.local
174 podSubnet: 10.244.0.0/16
175 serviceSubnet: 10.96.0.0/12
178 apiVersion: kubeproxy.config.k8s.io/v1alpha1
179 kind: KubeProxyConfiguration
183 elif [[ ${KUBEV} == 1.14.* ]]; then
184 cat <<EOF >/root/config.yaml
185 apiVersion: kubeadm.k8s.io/v1beta1
186 kubernetesVersion: v${KUBEV}
187 kind: ClusterConfiguration
189 feature-gates: SCTPSupport=true
191 dnsDomain: cluster.local
192 podSubnet: 10.244.0.0/16
193 serviceSubnet: 10.96.0.0/12
196 apiVersion: kubeproxy.config.k8s.io/v1alpha1
197 kind: KubeProxyConfiguration
202 echo "Unsupported Kubernetes version requested. Bail."
207 # create a RBAC file for helm (tiller)
208 cat <<EOF > /root/rbac-config.yaml
213 namespace: kube-system
215 apiVersion: rbac.authorization.k8s.io/v1
216 kind: ClusterRoleBinding
220 apiGroup: rbac.authorization.k8s.io
224 - kind: ServiceAccount
226 namespace: kube-system
229 # start cluster (make sure CIDR is enabled with the flag)
230 kubeadm init --config /root/config.yaml
234 HELMV=$(cat /opt/config/helm_version.txt)
239 wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELMVERSION}-linux-amd64.tar.gz
240 tar -xvf helm-v${HELMVERSION}-linux-amd64.tar.gz
241 mv linux-amd64/helm /usr/local/bin/helm
243 # set up kubectl credential and config
247 cp -i /etc/kubernetes/admin.conf /root/.kube/config
248 chown root:root /root/.kube/config
250 # at this point we should be able to use kubectl
251 kubectl get pods --all-namespaces
254 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
257 # waiting for all 8 kube-system pods to be in running state
258 # (at this point, minions have not joined yet)
259 wait_for_pods_running 8 kube-system
261 # if running a single node cluster, need to enable master node to run pods
262 kubectl taint nodes --all node-role.kubernetes.io/master-
265 # install RBAC for Helm
266 kubectl create -f rbac-config.yaml
270 helm init --service-account tiller
271 export HELM_HOME="/root/.helm"
273 # waiting for tiller pod to be in running state
274 wait_for_pods_running 1 kube-system tiller-deploy
276 while ! helm version; do
277 echo "Waiting for Helm to be ready"
282 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
285 echo "Done with master node setup"
289 # add rancodev CI tool hostnames
290 if [[ ! -z "${__RUNRICENV_GERRIT_IP__}" && ! -z "${__RUNRICENV_GERRIT_HOST__}" ]]; then
291 echo "${__RUNRICENV_GERRIT_IP__} ${__RUNRICENV_GERRIT_HOST__}" >> /etc/hosts
293 if [[ ! -z "${__RUNRICENV_DOCKER_IP__}" && ! -z "${__RUNRICENV_DOCKER_HOST__}" ]]; then
294 echo "${__RUNRICENV_DOCKER_IP__} ${__RUNRICENV_DOCKER_HOST__}" >> /etc/hosts
296 if [[ ! -z "${__RUNRICENV_HELMREPO_IP__}" && ! -z "${__RUNRICENV_HELMREPO_HOST__}" ]]; then
297 echo "${__RUNRICENV_HELMREPO_IP__} ${__RUNRICENV_HELMREPO_HOST__}" >> /etc/hosts
300 if [ ! -z "${__RUNRICENV_HELMREPO_CERT__}" ]; then
301 cat <<EOF >/etc/ca-certificates/update.d/helm.crt
302 ${__RUNRICENV_HELMREPO_CERT__}
306 # add cert for accessing docker registry in Azure
307 if [ ! -z "${__RUNRICENV_DOCKER_CERT__}" ]; then
308 mkdir -p /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
309 cat <<EOF >/etc/docker/ca.crt
310 ${__RUNRICENV_DOCKER_CERT__}
312 cp /etc/docker/ca.crt /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/ca.crt
314 service docker restart
315 systemctl enable docker.service
316 docker login -u ${__RUNRICENV_DOCKER_USER__} -p ${__RUNRICENV_DOCKER_PASS__} ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
317 docker pull ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/whoami:0.0.1