2 ################################################################################
3 # Copyright (c) 2019 AT&T Intellectual Property. #
4 # Copyright (c) 2019 Nokia. #
6 # Licensed under the Apache License, Version 2.0 (the "License"); #
7 # you may not use this file except in compliance with the License. #
8 # You may obtain a copy of the License at #
10 # http://www.apache.org/licenses/LICENSE-2.0 #
12 # Unless required by applicable law or agreed to in writing, software #
13 # distributed under the License is distributed on an "AS IS" BASIS, #
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
15 # See the License for the specific language governing permissions and #
16 # limitations under the License. #
17 ################################################################################
20 # first parameter: number of expected running pods
21 # second parameter: namespace (all-namespaces means all namespaces)
22 # third parameter: [optional] keyword
23 wait_for_pods_running () {
25 CMD="kubectl get pods --all-namespaces "
26 if [ "$NS" != "all-namespaces" ]; then
27 CMD="kubectl get pods -n $2 "
30 if [ "$#" == "3" ]; then
31 KEYWORD="${3}.*Running"
34 CMD2="$CMD | grep \"$KEYWORD\" | wc -l"
35 NUMPODS=$(eval "$CMD2")
36 echo "waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
37 while [ $NUMPODS -lt $1 ]; do
39 NUMPODS=$(eval "$CMD2")
40 echo "> waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
45 # first parameter: interface name
47 # enable ipv6 interface
48 # standard Ubuntu cloud image does not have dual interface configuration or ipv6
50 if ifconfig -a $IPv6IF; then
51 echo "" >> /etc/network/interfaces.d/50-cloud-init.cfg
52 #echo "auto ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
53 echo "allow-hotplug ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
54 echo "iface ${IPv6IF} inet6 auto" >> /etc/network/interfaces.d/50-cloud-init.cfg
56 #systemctl restart networking
61 echo "k8s_vm_install.sh"
63 export DEBIAN_FRONTEND=noninteractive
64 echo "__host_private_ip_addr__ $(hostname)" >> /etc/hosts
68 echo "__docker_version__" > /opt/config/docker_version.txt
69 echo "__k8s_version__" > /opt/config/k8s_version.txt
70 echo "__k8s_cni_version__" > /opt/config/k8s_cni_version.txt
71 echo "__helm_version__" > /opt/config/helm_version.txt
72 echo "__host_private_ip_addr__" > /opt/config/host_private_ip_addr.txt
73 echo "__k8s_mst_floating_ip_addr__" > /opt/config/k8s_mst_floating_ip_addr.txt
74 echo "__k8s_mst_private_ip_addr__" > /opt/config/k8s_mst_private_ip_addr.txt
75 echo "__mtu__" > /opt/config/mtu.txt
76 echo "__cinder_volume_id__" > /opt/config/cinder_volume_id.txt
77 echo "__stack_name__" > /opt/config/stack_name.txt
80 if [[ $(cat /opt/config/stack_name.txt) == *aux* ]]; then
88 modprobe -- nf_conntrack_ipv4
89 modprobe -- nf_conntrack_ipv6
90 modprobe -- nf_conntrack_proto_sctp
95 SWAPFILES=$(grep swap /etc/fstab | sed '/^#/ d' |cut -f1 -d' ')
96 if [ ! -z $SWAPFILES ]; then
97 for SWAPFILE in $SWAPFILES
99 if [ ! -z $SWAPFILE ]; then
100 echo "disabling swap file $SWAPFILE"
101 if [[ $SWAPFILE == UUID* ]]; then
102 UUID=$(echo $SWAPFILE | cut -f2 -d'=')
107 # edit /etc/fstab file, remove line with /swapfile
108 sed -i -e "/$SWAPFILE/d" /etc/fstab
114 # edit /etc/fstab file, remove line with /swapfile
115 #sed -i -e '/swapfile/d' /etc/fstab
118 DOCKERV=$(cat /opt/config/docker_version.txt)
119 KUBEV=$(cat /opt/config/k8s_version.txt)
120 KUBECNIV=$(cat /opt/config/k8s_cni_version.txt)
122 KUBEVERSION="${KUBEV}-00"
123 CNIVERSION="${KUBECNIV}-00"
124 DOCKERVERSION="${DOCKERV}-0ubuntu1.2~16.04.1"
125 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
126 echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
128 # install low latency kernel, docker.io, and kubernetes
130 apt-get install -y linux-image-4.15.0-45-lowlatency curl jq netcat docker.io=${DOCKERVERSION}
131 apt-get install -y kubernetes-cni=${CNIVERSION}
132 apt-get install -y --allow-unauthenticated kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
133 apt-mark hold kubernetes-cni kubelet kubeadm kubectl
136 # test access to k8s docker registry
137 kubeadm config images pull
140 # non-master nodes have hostnames ending with -[0-9][0-9]
141 if [[ $(hostname) == *-[0-9][0-9] ]]; then
142 echo "Done for non-master node"
143 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
146 # below are steps for initializating master node, only run on the master node.
147 # minion node join will be triggered from the caller of the stack creation as ssh command.
150 # create kubenetes config file
151 if [[ ${KUBEV} == 1.13.* ]]; then
152 cat <<EOF >/root/config.yaml
153 apiVersion: kubeadm.k8s.io/v1alpha3
154 kubernetesVersion: v${KUBEV}
155 kind: ClusterConfiguration
157 feature-gates: SCTPSupport=true
159 dnsDomain: cluster.local
160 podSubnet: 10.244.0.0/16
161 serviceSubnet: 10.96.0.0/12
164 apiVersion: kubeproxy.config.k8s.io/v1alpha1
165 kind: KubeProxyConfiguration
169 elif [[ ${KUBEV} == 1.14.* ]]; then
170 cat <<EOF >/root/config.yaml
171 apiVersion: kubeadm.k8s.io/v1beta1
172 kubernetesVersion: v${KUBEV}
173 kind: ClusterConfiguration
175 feature-gates: SCTPSupport=true
177 dnsDomain: cluster.local
178 podSubnet: 10.244.0.0/16
179 serviceSubnet: 10.96.0.0/12
182 apiVersion: kubeproxy.config.k8s.io/v1alpha1
183 kind: KubeProxyConfiguration
188 echo "Unsupported Kubernetes version requested. Bail."
193 # create a RBAC file for helm (tiller)
194 cat <<EOF > /root/rbac-config.yaml
199 namespace: kube-system
201 apiVersion: rbac.authorization.k8s.io/v1
202 kind: ClusterRoleBinding
206 apiGroup: rbac.authorization.k8s.io
210 - kind: ServiceAccount
212 namespace: kube-system
215 # start cluster (make sure CIDR is enabled with the flag)
216 kubeadm init --config /root/config.yaml
220 HELMV=$(cat /opt/config/helm_version.txt)
225 wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELMVERSION}-linux-amd64.tar.gz
226 tar -xvf helm-v${HELMVERSION}-linux-amd64.tar.gz
227 mv linux-amd64/helm /usr/local/bin/helm
229 # set up kubectl credential and config
233 cp -i /etc/kubernetes/admin.conf /root/.kube/config
234 chown root:root /root/.kube/config
236 # at this point we should be able to use kubectl
237 kubectl get pods --all-namespaces
240 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
243 # waiting for all 8 kube-system pods to be in running state
244 # (at this point, minions have not joined yet)
245 wait_for_pods_running 8 kube-system
247 # if running a single node cluster, need to enable master node to run pods
248 kubectl taint nodes --all node-role.kubernetes.io/master-
251 # install RBAC for Helm
252 kubectl create -f rbac-config.yaml
256 helm init --service-account tiller
257 export HELM_HOME="/root/.helm"
259 # waiting for tiller pod to be in running state
260 wait_for_pods_running 1 kube-system tiller-deploy
262 while ! helm version; do
263 echo "Waiting for Helm to be ready"
268 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
271 echo "Done with master node setup"
275 # add rancodev CI tool hostnames
276 if [[ ! -z "${__RUNRICENV_GERRIT_IP__}" && ! -z "${__RUNRICENV_GERRIT_HOST__}" ]]; then
277 echo "${__RUNRICENV_GERRIT_IP__} ${__RUNRICENV_GERRIT_HOST__}" >> /etc/hosts
279 if [[ ! -z "${__RUNRICENV_DOCKER_IP__}" && ! -z "${__RUNRICENV_DOCKER_HOST__}" ]]; then
280 echo "${__RUNRICENV_DOCKER_IP__} ${__RUNRICENV_DOCKER_HOST__}" >> /etc/hosts
282 if [[ ! -z "${__RUNRICENV_HELMREPO_IP__}" && ! -z "${__RUNRICENV_HELMREPO_HOST__}" ]]; then
283 echo "${__RUNRICENV_HELMREPO_IP__} ${__RUNRICENV_HELMREPO_HOST__}" >> /etc/hosts
286 if [ ! -z "${__RUNRICENV_HELMREPO_CERT__}" ]; then
287 cat <<EOF >/etc/ca-certificates/update.d/helm.crt
288 ${__RUNRICENV_HELMREPO_CERT__}
292 # add cert for accessing docker registry in Azure
293 if [ ! -z "${__RUNRICENV_DOCKER_CERT__}" ]; then
294 mkdir -p /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
295 cat <<EOF >/etc/docker/ca.crt
296 ${__RUNRICENV_DOCKER_CERT__}
298 cp /etc/docker/ca.crt /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/ca.crt
300 service docker restart
301 systemctl enable docker.service
302 docker login -u ${__RUNRICENV_DOCKER_USER__} -p ${__RUNRICENV_DOCKER_PASS__} ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
303 docker pull ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/whoami:0.0.1