2 ################################################################################
3 # Copyright (c) 2019 AT&T Intellectual Property. #
4 # Copyright (c) 2019 Nokia. #
6 # Licensed under the Apache License, Version 2.0 (the "License"); #
7 # you may not use this file except in compliance with the License. #
8 # You may obtain a copy of the License at #
10 # http://www.apache.org/licenses/LICENSE-2.0 #
12 # Unless required by applicable law or agreed to in writing, software #
13 # distributed under the License is distributed on an "AS IS" BASIS, #
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
15 # See the License for the specific language governing permissions and #
16 # limitations under the License. #
17 ################################################################################
20 # first parameter: number of expected running pods
21 # second parameter: namespace (all-namespaces means all namespaces)
22 # third parameter: [optional] keyword
23 wait_for_pods_running () {
25 CMD="kubectl get pods --all-namespaces "
26 if [ "$NS" != "all-namespaces" ]; then
27 CMD="kubectl get pods -n $2 "
30 if [ "$#" == "3" ]; then
31 KEYWORD="${3}.*Running"
34 CMD2="$CMD | grep \"$KEYWORD\" | wc -l"
35 NUMPODS=$(eval "$CMD2")
36 echo "waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
37 while [ $NUMPODS -lt $1 ]; do
39 NUMPODS=$(eval "$CMD2")
40 echo "> waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
45 # first parameter: interface name
47 # enable ipv6 interface
48 # standard Ubuntu cloud image does not have dual interface configuration or ipv6
50 if ifconfig -a $IPv6IF; then
51 echo "" >> /etc/network/interfaces.d/50-cloud-init.cfg
52 #echo "auto ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
53 echo "allow-hotplug ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
54 echo "iface ${IPv6IF} inet6 auto" >> /etc/network/interfaces.d/50-cloud-init.cfg
56 #systemctl restart networking
61 echo "k8s_vm_install.sh"
63 export DEBIAN_FRONTEND=noninteractive
64 echo "__host_private_ip_addr__ $(hostname)" >> /etc/hosts
68 echo "__docker_version__" > /opt/config/docker_version.txt
69 echo "__k8s_version__" > /opt/config/k8s_version.txt
70 echo "__k8s_cni_version__" > /opt/config/k8s_cni_version.txt
71 echo "__helm_version__" > /opt/config/helm_version.txt
72 echo "__host_private_ip_addr__" > /opt/config/host_private_ip_addr.txt
73 echo "__k8s_mst_floating_ip_addr__" > /opt/config/k8s_mst_floating_ip_addr.txt
74 echo "__k8s_mst_private_ip_addr__" > /opt/config/k8s_mst_private_ip_addr.txt
75 echo "__mtu__" > /opt/config/mtu.txt
76 echo "__cinder_volume_id__" > /opt/config/cinder_volume_id.txt
77 echo "__stack_name__" > /opt/config/stack_name.txt
80 if [[ $(cat /opt/config/stack_name.txt) == *aux* ]]; then
88 modprobe -- nf_conntrack_ipv4
89 modprobe -- nf_conntrack_ipv6
90 modprobe -- nf_conntrack_proto_sctp
95 SWAPFILES=$(grep swap /etc/fstab | sed '/^#/ d' |cut -f1 -d' ')
96 if [ ! -z $SWAPFILES ]; then
97 for SWAPFILE in $SWAPFILES
99 if [ ! -z $SWAPFILE ]; then
100 echo "disabling swap file $SWAPFILE"
101 if [[ $SWAPFILE == UUID* ]]; then
102 UUID=$(echo $SWAPFILE | cut -f2 -d'=')
107 # edit /etc/fstab file, remove line with /swapfile
108 sed -i -e "/$SWAPFILE/d" /etc/fstab
114 # edit /etc/fstab file, remove line with /swapfile
115 #sed -i -e '/swapfile/d' /etc/fstab
118 DOCKERV=$(cat /opt/config/docker_version.txt)
119 KUBEV=$(cat /opt/config/k8s_version.txt)
120 KUBECNIV=$(cat /opt/config/k8s_cni_version.txt)
122 KUBEVERSION="${KUBEV}-00"
123 CNIVERSION="${KUBECNIV}-00"
124 DOCKERVERSION="${DOCKERV}-0ubuntu1.2~16.04.1"
125 curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
126 echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
128 # install low latency kernel, docker.io, and kubernetes
130 apt-get install -y linux-image-4.15.0-45-lowlatency curl jq netcat docker.io=${DOCKERVERSION}
131 apt-get install -y kubernetes-cni=${CNIVERSION}
132 apt-get install -y --allow-unauthenticated kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
133 apt-mark hold kubernetes-cni kubelet kubeadm kubectl
136 # test access to k8s docker registry
137 kubeadm config images pull
140 # non-master nodes have hostnames ending with -[0-9][0-9]
141 if [[ $(hostname) == *-[0-9][0-9] ]]; then
142 echo "Done for non-master node"
143 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
146 # below are steps for initializating master node, only run on the master node.
147 # minion node join will be triggered from the caller of the stack creation as ssh command.
150 # create kubenetes config file
151 if [[ ${KUBEV} == 1.13.* ]]; then
152 cat <<EOF >/root/config.yaml
153 apiVersion: kubeadm.k8s.io/v1alpha3
154 kubernetesVersion: v${KUBEV}
155 kind: ClusterConfiguration
157 feature-gates: SCTPSupport=true
159 dnsDomain: cluster.local
160 podSubnet: 10.244.0.0/16
161 serviceSubnet: 10.96.0.0/12
164 apiVersion: kubeproxy.config.k8s.io/v1alpha1
165 kind: KubeProxyConfiguration
169 elif [[ ${KUBEV} == 1.14.* ]]; then
170 cat <<EOF >/root/config.yaml
171 apiVersion: kubeadm.k8s.io/v1beta1
172 kubernetesVersion: v${KUBEV}
173 kind: ClusterConfiguration
175 feature-gates: SCTPSupport=true
177 dnsDomain: cluster.local
178 podSubnet: 10.244.0.0/16
179 serviceSubnet: 10.96.0.0/12
182 apiVersion: kubeproxy.config.k8s.io/v1alpha1
183 kind: KubeProxyConfiguration
188 echo "Unsupported Kubernetes version requested. Bail."
193 # create a RBAC file for helm (tiller)
194 cat <<EOF > /root/rbac-config.yaml
199 namespace: kube-system
201 apiVersion: rbac.authorization.k8s.io/v1
202 kind: ClusterRoleBinding
206 apiGroup: rbac.authorization.k8s.io
210 - kind: ServiceAccount
212 namespace: kube-system
215 # start cluster (make sure CIDR is enabled with the flag)
216 kubeadm init --config /root/config.yaml
220 HELMV=$(cat /opt/config/helm_version.txt)
225 wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELMVERSION}-linux-amd64.tar.gz
226 tar -xvf helm-v${HELMVERSION}-linux-amd64.tar.gz
227 mv linux-amd64/helm /usr/local/bin/helm
229 # set up kubectl credential and config
233 cp -i /etc/kubernetes/admin.conf /root/.kube/config
234 chown root:root /root/.kube/config
236 # at this point we should be able to use kubectl
237 kubectl get pods --all-namespaces
240 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
243 # waiting for all 8 kube-system pods to be in running state
244 # (at this point, minions have not joined yet)
245 wait_for_pods_running 8 kube-system
247 # if running a single node cluster, need to enable master node to run pods
248 kubectl taint nodes --all node-role.kubernetes.io/master-
251 # install RBAC for Helm
252 kubectl create -f rbac-config.yaml
256 helm init --service-account tiller
257 export HELM_HOME="/root/.helm"
259 # waiting for tiller pod to be in running state
260 wait_for_pods_running 1 kube-system tiller-deploy
262 while ! helm version; do
263 echo "Waiting for Helm to be ready"
267 # install ingress controller db-less kong
268 helm install stable/kong --set ingressController.enabled=true --set postgresql.enabled=false --set env.database=off
271 echo "Starting an NC TCP server on port 29999 to indicate we are ready"
274 echo "Done with master node setup"
278 # add rancodev CI tool hostnames
279 if [[ ! -z "${__RUNRICENV_GERRIT_IP__}" && ! -z "${__RUNRICENV_GERRIT_HOST__}" ]]; then
280 echo "${__RUNRICENV_GERRIT_IP__} ${__RUNRICENV_GERRIT_HOST__}" >> /etc/hosts
282 if [[ ! -z "${__RUNRICENV_DOCKER_IP__}" && ! -z "${__RUNRICENV_DOCKER_HOST__}" ]]; then
283 echo "${__RUNRICENV_DOCKER_IP__} ${__RUNRICENV_DOCKER_HOST__}" >> /etc/hosts
285 if [[ ! -z "${__RUNRICENV_HELMREPO_IP__}" && ! -z "${__RUNRICENV_HELMREPO_HOST__}" ]]; then
286 echo "${__RUNRICENV_HELMREPO_IP__} ${__RUNRICENV_HELMREPO_HOST__}" >> /etc/hosts
289 if [ ! -z "${__RUNRICENV_HELMREPO_CERT__}" ]; then
290 cat <<EOF >/etc/ca-certificates/update.d/helm.crt
291 ${__RUNRICENV_HELMREPO_CERT__}
295 # add cert for accessing docker registry in Azure
296 if [ ! -z "${__RUNRICENV_DOCKER_CERT__}" ]; then
297 mkdir -p /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
298 cat <<EOF >/etc/docker/ca.crt
299 ${__RUNRICENV_DOCKER_CERT__}
301 cp /etc/docker/ca.crt /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/ca.crt
303 service docker restart
304 systemctl enable docker.service
305 docker login -u ${__RUNRICENV_DOCKER_USER__} -p ${__RUNRICENV_DOCKER_PASS__} ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
306 docker pull ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/whoami:0.0.1