3 ################################################################################
4 # Copyright (c) 2019 AT&T Intellectual Property. #
5 # Copyright (c) 2022 Nokia. #
7 # Licensed under the Apache License, Version 2.0 (the "License"); #
8 # you may not use this file except in compliance with the License. #
9 # You may obtain a copy of the License at #
11 # http://www.apache.org/licenses/LICENSE-2.0 #
13 # Unless required by applicable law or agreed to in writing, software #
14 # distributed under the License is distributed on an "AS IS" BASIS, #
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
16 # See the License for the specific language governing permissions and #
17 # limitations under the License. #
18 ################################################################################
22 echo "Usage: $0 [ -k <k8s version> -d <docker version> -e <helm version> -c <cni-version>" 1>&2;
24 echo "k: kubernetes version" 1>&2;
25 echo "c: kubernetes CNI version" 1>&2;
26 echo "d: docker version" 1>&2;
27 echo "e: helm version" 1>&2;
32 wait_for_pods_running () {
34 CMD="kubectl get pods --all-namespaces "
35 if [ "$NS" != "all-namespaces" ]; then
36 CMD="kubectl get pods -n $2 "
39 if [ "$#" == "3" ]; then
40 KEYWORD="${3}.*Running"
43 CMD2="$CMD | grep \"$KEYWORD\" | wc -l"
44 NUMPODS=$(eval "$CMD2")
45 echo "waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
46 while [ $NUMPODS -lt $1 ]; do
48 NUMPODS=$(eval "$CMD2")
49 echo "> waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
56 if ifconfig -a $IPv6IF; then
57 echo "" >> /etc/network/interfaces.d/50-cloud-init.cfg
58 echo "allow-hotplug ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
59 echo "iface ${IPv6IF} inet6 auto" >> /etc/network/interfaces.d/50-cloud-init.cfg
70 while getopts ":k:d:e:n:c" o; do
90 if [[ ${HELMV} == 2.* ]]; then
91 echo "helm 2 ("${HELMV}")not supported anymore"
96 export DEBIAN_FRONTEND=noninteractive
97 echo "$(hostname -I) $(hostname)" >> /etc/hosts
104 echo "" > /opt/config/docker_version.txt
105 echo "1.16.0" > /opt/config/k8s_version.txt
106 echo "0.7.5" > /opt/config/k8s_cni_version.txt
107 echo "3.14.4" > /opt/config/helm_version.txt
108 echo "$(hostname -I)" > /opt/config/host_private_ip_addr.txt
109 echo "$(curl ifconfig.co)" > /opt/config/k8s_mst_floating_ip_addr.txt
110 echo "$(hostname -I)" > /opt/config/k8s_mst_private_ip_addr.txt
111 echo "__mtu__" > /opt/config/mtu.txt
112 echo "__cinder_volume_id__" > /opt/config/cinder_volume_id.txt
113 echo "$(hostname)" > /opt/config/stack_name.txt
116 if [[ $(cat /opt/config/stack_name.txt) == *aux* ]]; then
122 modprobe -- ip_vs_wrr
124 modprobe -- nf_conntrack_ipv4
125 modprobe -- nf_conntrack_ipv6
126 modprobe -- nf_conntrack_proto_sctp
128 if [ ! -z "$IPV6IF" ]; then
129 start_ipv6_if $IPV6IF
132 SWAPFILES=$(grep swap /etc/fstab | sed '/^[ \t]*#/ d' | sed 's/[\t ]/ /g' | tr -s " " | cut -f1 -d' ')
133 if [ ! -z $SWAPFILES ]; then
134 for SWAPFILE in $SWAPFILES
136 if [ ! -z $SWAPFILE ]; then
137 echo "disabling swap file $SWAPFILE"
138 if [[ $SWAPFILE == UUID* ]]; then
139 UUID=$(echo $SWAPFILE | cut -f2 -d'=')
144 sed -i "\%$SWAPFILE%d" /etc/fstab
150 echo "### Docker version = "${DOCKERV}
151 echo "### k8s version = "${KUBEV}
152 echo "### helm version = "${HELMV}
153 echo "### k8s cni version = "${KUBECNIV}
155 #KUBEVERSION="${KUBEV}-00"
156 CNIVERSION="${KUBECNIV}-00"
157 DOCKERVERSION="${DOCKERV}"
159 UBUNTU_RELEASE=$(lsb_release -r | sed 's/^[a-zA-Z:\t ]\+//g')
160 if [[ ${UBUNTU_RELEASE} == 16.* ]]; then
161 echo "Installing on Ubuntu $UBUNTU_RELEASE (Xenial Xerus) host"
162 if [ ! -z "${DOCKERV}" ]; then
163 DOCKERVERSION="${DOCKERV}-0ubuntu1~16.04.5"
165 elif [[ ${UBUNTU_RELEASE} == 18.* ]]; then
166 echo "Installing on Ubuntu $UBUNTU_RELEASE (Bionic Beaver)"
167 if [ ! -z "${DOCKERV}" ]; then
168 DOCKERVERSION="${DOCKERV}-0ubuntu1~18.04.4"
170 elif [[ ${UBUNTU_RELEASE} == 20.* ]]; then
171 echo "Installing on Ubuntu $UBUNTU_RELEASE (Focal Fossal)"
172 if [ ! -z "${DOCKERV}" ]; then
173 DOCKERVERSION="${DOCKERV}-0ubuntu1~20.04.2" # 20.10.21-0ubuntu1~20.04.2
176 echo "Unsupported Ubuntu release ($UBUNTU_RELEASE) detected. Exit."
179 echo "docker version to use = "${DOCKERVERSION}
181 #curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
182 #echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
184 mkdir /etc/apt/keyrings
185 echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
186 curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
188 mkdir -p /etc/apt/apt.conf.d
189 echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries
192 RES=$(apt-get install -y curl jq netcat make ipset moreutils 2>&1)
193 if [[ $RES == */var/lib/dpkg/lock* ]]; then
194 echo "Fail to get dpkg lock. Wait for any other package installation"
195 echo "process to finish, then rerun this script"
199 APTOPTS="--allow-downgrades --allow-change-held-packages --allow-unauthenticated --ignore-hold "
201 for PKG in kubeadm docker.io; do
202 INSTALLED_VERSION=$(dpkg --list |grep ${PKG} |tr -s " " |cut -f3 -d ' ')
203 if [ ! -z ${INSTALLED_VERSION} ]; then
204 if [ "${PKG}" == "kubeadm" ]; then
207 apt-get -y $APTOPTS remove kubeadm kubelet kubectl kubernetes-cni
209 apt-get -y $APTOPTS remove "${PKG}"
213 apt-get -y autoremove
215 if [ -z ${DOCKERVERSION} ]; then
216 apt-get install -y $APTOPTS docker.io
218 apt-get install -y $APTOPTS docker.io=${DOCKERVERSION}
220 cat > /etc/docker/daemon.json <<EOF
222 "exec-opts": ["native.cgroupdriver=systemd"],
223 "log-driver": "json-file",
227 "storage-driver": "overlay2"
230 mkdir -p /etc/systemd/system/docker.service.d
231 systemctl enable docker.service
232 systemctl daemon-reload
233 systemctl restart docker
235 if [ -z ${CNIVERSION} ]; then
236 apt-get install -y $APTOPTS kubernetes-cni
238 apt-get install -y $APTOPTS kubernetes-cni=${CNIVERSION}
241 if [ -z ${KUBEVERSION} ]; then
242 apt-get install -y $APTOPTS kubeadm kubelet kubectl
244 apt-get install -y $APTOPTS kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
247 apt-mark hold docker.io kubernetes-cni kubelet kubeadm kubectl
250 kubeadm config images pull --kubernetes-version=${KUBEV}
254 if [ "$NODETYPE" == "master" ]; then
256 if [[ ${KUBEV} == 1.13.* ]]; then
257 cat <<EOF >/root/config.yaml
258 apiVersion: kubeadm.k8s.io/v1alpha3
259 kubernetesVersion: v${KUBEV}
260 kind: ClusterConfiguration
262 feature-gates: SCTPSupport=true
264 dnsDomain: cluster.local
265 podSubnet: 10.244.0.0/16
266 serviceSubnet: 10.96.0.0/12
268 apiVersion: kubeproxy.config.k8s.io/v1alpha1
269 kind: KubeProxyConfiguration
273 elif [[ ${KUBEV} == 1.14.* ]]; then
274 cat <<EOF >/root/config.yaml
275 apiVersion: kubeadm.k8s.io/v1beta1
276 kubernetesVersion: v${KUBEV}
277 kind: ClusterConfiguration
279 feature-gates: SCTPSupport=true
281 dnsDomain: cluster.local
282 podSubnet: 10.244.0.0/16
283 serviceSubnet: 10.96.0.0/12
285 apiVersion: kubeproxy.config.k8s.io/v1alpha1
286 kind: KubeProxyConfiguration
289 elif [[ ${KUBEV} == 1.15.* ]] || [[ ${KUBEV} == 1.16.* ]] || [[ ${KUBEV} == 1.18.* ]]; then
290 cat <<EOF >/root/config.yaml
291 apiVersion: kubeadm.k8s.io/v1beta2
292 kubernetesVersion: v${KUBEV}
293 kind: ClusterConfiguration
296 feature-gates: SCTPSupport=true
298 dnsDomain: cluster.local
299 podSubnet: 10.244.0.0/16
300 serviceSubnet: 10.96.0.0/12
302 apiVersion: kubeproxy.config.k8s.io/v1alpha1
303 kind: KubeProxyConfiguration
306 elif [[ ${KUBEV} == 1.28.* ]] ; then
307 echo "Do Nothing for now."
309 echo "Unsupported Kubernetes version requested. Bail."
313 cat <<EOF > /root/rbac-config.yaml
318 namespace: kube-system
320 apiVersion: rbac.authorization.k8s.io/v1
321 kind: ClusterRoleBinding
325 apiGroup: rbac.authorization.k8s.io
329 - kind: ServiceAccount
331 namespace: kube-system
334 if [[ ${KUBEV} == 1.28.11 ]]; then
335 kubeadm init --pod-network-cidr=10.244.0.0/16
336 mkdir -p /run/flannel
337 cat <<EOF > /run/flannel/subnet.env
338 FLANNEL_NETWORK=10.244.0.0/16
339 FLANNEL_SUBNET=10.244.0.1/24
344 kubeadm init --config /root/config.yaml
350 cp -i /etc/kubernetes/admin.conf /root/.kube/config
351 chown root:root /root/.kube/config
352 export KUBECONFIG=/root/.kube/config
353 echo "KUBECONFIG=${KUBECONFIG}" >> /etc/environment
355 kubectl get pods --all-namespaces
357 if [[ ${KUBEV} == 1.28.11 ]]; then
358 kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
360 # we refer to version 0.18.1 because later versions use namespace kube-flannel instead of kube-system TODO
361 kubectl apply -f "https://raw.githubusercontent.com/flannel-io/flannel/v0.18.1/Documentation/kube-flannel.yml"
364 if [[ ${KUBEV} == 1.28.11 ]]; then
365 wait_for_pods_running 7 kube-system
366 wait_for_pods_running 1 kube-flannel
367 kubectl taint nodes --all node-role.kubernetes.io/control-plane:NoSchedule-
369 wait_for_pods_running 8 kube-system
370 kubectl taint nodes --all node-role.kubernetes.io/master-
374 HELMV=$(cat /opt/config/helm_version.txt)
376 if [ ! -e helm-v${HELMVERSION}-linux-amd64.tar.gz ]; then
377 wget https://get.helm.sh/helm-v${HELMVERSION}-linux-amd64.tar.gz
379 cd /root && rm -rf Helm && mkdir Helm && cd Helm
380 tar -xvf ../helm-v${HELMVERSION}-linux-amd64.tar.gz
381 mv linux-amd64/helm /usr/local/bin/helm
386 # if [[ ${KUBEV} == 1.16.* ]]; then
387 # if [[ ${HELMVERSION} == 2.* ]]; then
388 # helm init --service-account tiller --override spec.selector.matchLabels.'name'='tiller',spec.selector.matchLabels.'app'='helm' --output yaml > /tmp/helm-init.yaml
389 # sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' /tmp/helm-init.yaml > /tmp/helm-init-patched.yaml
390 # kubectl apply -f /tmp/helm-init-patched.yaml
393 # if [[ ${HELMVERSION} == 2.* ]]; then
394 # helm init --service-account tiller
397 # if [[ ${HELMVERSION} == 2.* ]]; then
399 # export HELM_HOME="$(pwd)/.helm"
400 # echo "HELM_HOME=${HELM_HOME}" >> /etc/environment
403 while ! helm version; do
404 echo "Waiting for Helm to be ready"
408 echo "Preparing a master node (lower ID) for using local FS for PV"
409 PV_NODE_NAME=$(kubectl get nodes |grep master | cut -f1 -d' ' | sort | head -1)
410 kubectl label --overwrite nodes $PV_NODE_NAME local-storage=enable
411 if [ "$PV_NODE_NAME" == "$(hostname)" ]; then
412 mkdir -p /opt/data/dashboard-data
415 echo "Done with master node setup"
419 if [[ ! -z "" && ! -z "" ]]; then
420 echo " " >> /etc/hosts
422 if [[ ! -z "" && ! -z "" ]]; then
423 echo " " >> /etc/hosts
425 if [[ ! -z "" && ! -z "helm.ricinfra.local" ]]; then
426 echo " helm.ricinfra.local" >> /etc/hosts
429 if [[ "1" -gt "100" ]]; then
430 cat <<EOF >/etc/ca-certificates/update.d/helm.crt
435 if [[ "1" -gt "100" ]]; then
436 mkdir -p /etc/docker/certs.d/:
437 cat <<EOF >/etc/docker/ca.crt
440 cp /etc/docker/ca.crt /etc/docker/certs.d/:/ca.crt
442 service docker restart
443 systemctl enable docker.service
445 docker pull :/whoami:0.0.1