bin/install_k8s_and_helm.sh

   1 #!/bin/bash -x
   2 #
   3 ################################################################################
   4 #   Copyright (c) 2019 AT&T Intellectual Property.                             #
   5 #   Copyright (c) 2022 Nokia.                                                  #
   6 #                                                                              #
   7 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   8 #   you may not use this file except in compliance with the License.           #
   9 #   You may obtain a copy of the License at                                    #
  10 #                                                                              #
  11 #       http://www.apache.org/licenses/LICENSE-2.0                             #
  12 #                                                                              #
  13 #   Unless required by applicable law or agreed to in writing, software        #
  14 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  15 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  16 #   See the License for the specific language governing permissions and        #
  17 #   limitations under the License.                                             #
  18 ################################################################################
  19
  20
  21 usage() {
  22     echo "Usage: $0 [ -k <k8s version> -d <docker version> -e <helm version> -c <cni-version>" 1>&2;
  23
  24     echo "k:    kubernetes version" 1>&2;
  25     echo "c:    kubernetes CNI  version" 1>&2;
  26     echo "d:    docker version" 1>&2;
  27     echo "e:    helm version" 1>&2;
  28     exit 1;
  29 }
  30
  31
  32 wait_for_pods_running () {
  33   NS="$2"
  34   CMD="kubectl get pods --all-namespaces "
  35   if [ "$NS" != "all-namespaces" ]; then
  36     CMD="kubectl get pods -n $2 "
  37   fi
  38   KEYWORD="Running"
  39   if [ "$#" == "3" ]; then
  40     KEYWORD="${3}.*Running"
  41   fi
  42
  43   CMD2="$CMD | grep \"$KEYWORD\" | wc -l"
  44   NUMPODS=$(eval "$CMD2")
  45   echo "waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
  46   while [  $NUMPODS -lt $1 ]; do
  47     sleep 5
  48     NUMPODS=$(eval "$CMD2")
  49     echo "> waiting for $NUMPODS/$1 pods running in namespace [$NS] with keyword [$KEYWORD]"
  50   done
  51 }
  52
  53
  54 start_ipv6_if () {
  55   IPv6IF="$1"
  56   if ifconfig -a $IPv6IF; then
  57     echo "" >> /etc/network/interfaces.d/50-cloud-init.cfg
  58     echo "allow-hotplug ${IPv6IF}" >> /etc/network/interfaces.d/50-cloud-init.cfg
  59     echo "iface ${IPv6IF} inet6 auto" >> /etc/network/interfaces.d/50-cloud-init.cfg
  60     ifconfig ${IPv6IF} up
  61   fi
  62 }
  63
  64 KUBEV="1.28.11"
  65 KUBECNIV="0.7.5"
  66 HELMV="3.14.4"
  67 DOCKERV="20.10.21"
  68
  69 echo running ${0}
  70 while getopts ":k:d:e:n:c" o; do
  71     case "${o}" in
  72     e)
  73        HELMV=${OPTARG}
  74         ;;
  75     d)
  76        DOCKERV=${OPTARG}
  77         ;;
  78     k)
  79        KUBEV=${OPTARG}
  80        ;;
  81     c)
  82        KUBECNIV=${OPTARG}
  83        ;;
  84     *)
  85        usage
  86        ;;
  87     esac
  88 done
  89
  90 if [[ ${HELMV} == 2.* ]]; then
  91   echo "helm 2 ("${HELMV}")not supported anymore"
  92   exit -1
  93 fi
  94
  95 set -x
  96 export DEBIAN_FRONTEND=noninteractive
  97 echo "$(hostname -I) $(hostname)" >> /etc/hosts
  98 printenv
  99
 100 IPV6IF=""
 101
 102 rm -rf /opt/config
 103 mkdir -p /opt/config
 104 echo "" > /opt/config/docker_version.txt
 105 echo "1.16.0" > /opt/config/k8s_version.txt
 106 echo "0.7.5" > /opt/config/k8s_cni_version.txt
 107 echo "3.14.4" > /opt/config/helm_version.txt
 108 echo "$(hostname -I)" > /opt/config/host_private_ip_addr.txt
 109 echo "$(curl ifconfig.co)" > /opt/config/k8s_mst_floating_ip_addr.txt
 110 echo "$(hostname -I)" > /opt/config/k8s_mst_private_ip_addr.txt
 111 echo "__mtu__" > /opt/config/mtu.txt
 112 echo "__cinder_volume_id__" > /opt/config/cinder_volume_id.txt
 113 echo "$(hostname)" > /opt/config/stack_name.txt
 114
 115 ISAUX='false'
 116 if [[ $(cat /opt/config/stack_name.txt) == *aux* ]]; then
 117   ISAUX='true'
 118 fi
 119
 120 modprobe -- ip_vs
 121 modprobe -- ip_vs_rr
 122 modprobe -- ip_vs_wrr
 123 modprobe -- ip_vs_sh
 124 modprobe -- nf_conntrack_ipv4
 125 modprobe -- nf_conntrack_ipv6
 126 modprobe -- nf_conntrack_proto_sctp
 127
 128 if [ ! -z "$IPV6IF" ]; then
 129   start_ipv6_if $IPV6IF
 130 fi
 131
 132 SWAPFILES=$(grep swap /etc/fstab | sed '/^[ \t]*#/ d' | sed 's/[\t ]/ /g' | tr -s " " | cut -f1 -d' ')
 133 if [ ! -z $SWAPFILES ]; then
 134   for SWAPFILE in $SWAPFILES
 135   do
 136     if [ ! -z $SWAPFILE ]; then
 137       echo "disabling swap file $SWAPFILE"
 138       if [[ $SWAPFILE == UUID* ]]; then
 139         UUID=$(echo $SWAPFILE | cut -f2 -d'=')
 140         swapoff -U $UUID
 141       else
 142         swapoff $SWAPFILE
 143       fi
 144       sed -i "\%$SWAPFILE%d" /etc/fstab
 145     fi
 146   done
 147 fi
 148
 149
 150 echo "### Docker version  = "${DOCKERV}
 151 echo "### k8s version     = "${KUBEV}
 152 echo "### helm version    = "${HELMV}
 153 echo "### k8s cni version = "${KUBECNIV}
 154
 155 #KUBEVERSION="${KUBEV}-00"
 156 CNIVERSION="${KUBECNIV}-00"
 157 DOCKERVERSION="${DOCKERV}"
 158
 159 UBUNTU_RELEASE=$(lsb_release -r | sed 's/^[a-zA-Z:\t ]\+//g')
 160 if [[ ${UBUNTU_RELEASE} == 16.* ]]; then
 161   echo "Installing on Ubuntu $UBUNTU_RELEASE (Xenial Xerus) host"
 162   if [ ! -z "${DOCKERV}" ]; then
 163     DOCKERVERSION="${DOCKERV}-0ubuntu1~16.04.5"
 164   fi
 165 elif [[ ${UBUNTU_RELEASE} == 18.* ]]; then
 166   echo "Installing on Ubuntu $UBUNTU_RELEASE (Bionic Beaver)"
 167   if [ ! -z "${DOCKERV}" ]; then
 168     DOCKERVERSION="${DOCKERV}-0ubuntu1~18.04.4"
 169   fi
 170 elif [[ ${UBUNTU_RELEASE} == 20.* ]]; then
 171   echo "Installing on Ubuntu $UBUNTU_RELEASE (Focal Fossal)"
 172   if [ ! -z "${DOCKERV}" ]; then
 173     DOCKERVERSION="${DOCKERV}-0ubuntu1~20.04.2"  # 20.10.21-0ubuntu1~20.04.2
 174   fi
 175 else
 176   echo "Unsupported Ubuntu release ($UBUNTU_RELEASE) detected.  Exit."
 177 fi
 178
 179 echo "docker version to use = "${DOCKERVERSION}
 180
 181 #curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 182 #echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
 183
 184 mkdir /etc/apt/keyrings
 185 echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 186 curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
 187
 188 mkdir -p /etc/apt/apt.conf.d
 189 echo "APT::Acquire::Retries \"3\";" > /etc/apt/apt.conf.d/80-retries
 190
 191 apt-get update
 192 RES=$(apt-get install -y  curl jq netcat make ipset moreutils 2>&1)
 193 if [[ $RES == */var/lib/dpkg/lock* ]]; then
 194   echo "Fail to get dpkg lock.  Wait for any other package installation"
 195   echo "process to finish, then rerun this script"
 196   exit -1
 197 fi
 198
 199 APTOPTS="--allow-downgrades --allow-change-held-packages --allow-unauthenticated --ignore-hold "
 200
 201 for PKG in kubeadm docker.io; do
 202   INSTALLED_VERSION=$(dpkg --list |grep ${PKG} |tr -s " " |cut -f3 -d ' ')
 203   if [ ! -z ${INSTALLED_VERSION} ]; then
 204     if [ "${PKG}" == "kubeadm" ]; then
 205       kubeadm reset -f
 206       rm -rf ~/.kube
 207       apt-get -y $APTOPTS remove kubeadm kubelet kubectl kubernetes-cni
 208     else
 209       apt-get -y $APTOPTS remove "${PKG}"
 210     fi
 211   fi
 212 done
 213 apt-get -y autoremove
 214
 215 if [ -z ${DOCKERVERSION} ]; then
 216   apt-get install -y $APTOPTS docker.io
 217 else
 218   apt-get install -y $APTOPTS docker.io=${DOCKERVERSION}
 219 fi
 220 cat > /etc/docker/daemon.json <<EOF
 221 {
 222   "exec-opts": ["native.cgroupdriver=systemd"],
 223   "log-driver": "json-file",
 224   "log-opts": {
 225     "max-size": "100m"
 226   },
 227   "storage-driver": "overlay2"
 228 }
 229 EOF
 230 mkdir -p /etc/systemd/system/docker.service.d
 231 systemctl enable docker.service
 232 systemctl daemon-reload
 233 systemctl restart docker
 234
 235 if [ -z ${CNIVERSION} ]; then
 236   apt-get install -y $APTOPTS kubernetes-cni
 237 else
 238   apt-get install -y $APTOPTS kubernetes-cni=${CNIVERSION}
 239 fi
 240
 241 if [ -z ${KUBEVERSION} ]; then
 242   apt-get install -y $APTOPTS kubeadm kubelet kubectl
 243 else
 244   apt-get install -y $APTOPTS kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
 245 fi
 246
 247 apt-mark hold docker.io kubernetes-cni kubelet kubeadm kubectl
 248
 249
 250 kubeadm config images pull --kubernetes-version=${KUBEV}
 251
 252
 253 NODETYPE="master"
 254 if [ "$NODETYPE" == "master" ]; then
 255
 256   if [[ ${KUBEV} == 1.13.* ]]; then
 257     cat <<EOF >/root/config.yaml
 258 apiVersion: kubeadm.k8s.io/v1alpha3
 259 kubernetesVersion: v${KUBEV}
 260 kind: ClusterConfiguration
 261 apiServerExtraArgs:
 262   feature-gates: SCTPSupport=true
 263 networking:
 264   dnsDomain: cluster.local
 265   podSubnet: 10.244.0.0/16
 266   serviceSubnet: 10.96.0.0/12
 267 ---
 268 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 269 kind: KubeProxyConfiguration
 270 mode: ipvs
 271 EOF
 272
 273   elif [[ ${KUBEV} == 1.14.* ]]; then
 274     cat <<EOF >/root/config.yaml
 275 apiVersion: kubeadm.k8s.io/v1beta1
 276 kubernetesVersion: v${KUBEV}
 277 kind: ClusterConfiguration
 278 apiServerExtraArgs:
 279   feature-gates: SCTPSupport=true
 280 networking:
 281   dnsDomain: cluster.local
 282   podSubnet: 10.244.0.0/16
 283   serviceSubnet: 10.96.0.0/12
 284 ---
 285 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 286 kind: KubeProxyConfiguration
 287 mode: ipvs
 288 EOF
 289   elif [[ ${KUBEV} == 1.15.* ]] || [[ ${KUBEV} == 1.16.* ]] || [[ ${KUBEV} == 1.18.* ]]; then
 290     cat <<EOF >/root/config.yaml
 291 apiVersion: kubeadm.k8s.io/v1beta2
 292 kubernetesVersion: v${KUBEV}
 293 kind: ClusterConfiguration
 294 apiServer:
 295   extraArgs:
 296     feature-gates: SCTPSupport=true
 297 networking:
 298   dnsDomain: cluster.local
 299   podSubnet: 10.244.0.0/16
 300   serviceSubnet: 10.96.0.0/12
 301 ---
 302 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 303 kind: KubeProxyConfiguration
 304 mode: ipvs
 305 EOF
 306   elif [[ ${KUBEV} == 1.28.* ]] ; then
 307     echo "Do Nothing for now."
 308     else
 309     echo "Unsupported Kubernetes version requested.  Bail."
 310     exit
 311   fi
 312
 313   cat <<EOF > /root/rbac-config.yaml
 314 apiVersion: v1
 315 kind: ServiceAccount
 316 metadata:
 317   name: tiller
 318   namespace: kube-system
 319 ---
 320 apiVersion: rbac.authorization.k8s.io/v1
 321 kind: ClusterRoleBinding
 322 metadata:
 323   name: tiller
 324 roleRef:
 325   apiGroup: rbac.authorization.k8s.io
 326   kind: ClusterRole
 327   name: cluster-admin
 328 subjects:
 329   - kind: ServiceAccount
 330     name: tiller
 331     namespace: kube-system
 332 EOF
 333
 334 if [[ ${KUBEV} == 1.28.11 ]]; then
 335   kubeadm init --pod-network-cidr=10.244.0.0/16
 336   mkdir -p /run/flannel
 337 cat <<EOF > /run/flannel/subnet.env
 338 FLANNEL_NETWORK=10.244.0.0/16
 339 FLANNEL_SUBNET=10.244.0.1/24
 340 FLANNEL_MTU=1450
 341 FLANNEL_IPMASQ=true
 342 EOF
 343 else
 344   kubeadm init --config /root/config.yaml
 345 fi
 346
 347   cd /root
 348   rm -rf .kube
 349   mkdir -p .kube
 350   cp -i /etc/kubernetes/admin.conf /root/.kube/config
 351   chown root:root /root/.kube/config
 352   export KUBECONFIG=/root/.kube/config
 353   echo "KUBECONFIG=${KUBECONFIG}" >> /etc/environment
 354
 355   kubectl get pods --all-namespaces
 356
 357 if [[ ${KUBEV} == 1.28.11 ]]; then
 358   kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
 359 else
 360   # we refer to version 0.18.1 because later versions use namespace kube-flannel instead of kube-system TODO
 361   kubectl apply -f "https://raw.githubusercontent.com/flannel-io/flannel/v0.18.1/Documentation/kube-flannel.yml"
 362 fi
 363
 364 if [[ ${KUBEV} == 1.28.11 ]]; then
 365   wait_for_pods_running 7 kube-system
 366   wait_for_pods_running 1 kube-flannel
 367   kubectl taint nodes --all node-role.kubernetes.io/control-plane:NoSchedule-
 368 else
 369   wait_for_pods_running 8 kube-system
 370   kubectl taint nodes --all node-role.kubernetes.io/master-
 371 fi
 372
 373
 374   HELMV=$(cat /opt/config/helm_version.txt)
 375   HELMVERSION=${HELMV}
 376   if [ ! -e helm-v${HELMVERSION}-linux-amd64.tar.gz ]; then
 377     wget https://get.helm.sh/helm-v${HELMVERSION}-linux-amd64.tar.gz
 378   fi
 379   cd /root && rm -rf Helm && mkdir Helm && cd Helm
 380   tar -xvf ../helm-v${HELMVERSION}-linux-amd64.tar.gz
 381   mv linux-amd64/helm /usr/local/bin/helm
 382
 383   cd /root
 384
 385   rm -rf /root/.helm
 386 #  if [[ ${KUBEV} == 1.16.* ]]; then
 387 #    if [[ ${HELMVERSION} == 2.* ]]; then
 388 #       helm init --service-account tiller --override spec.selector.matchLabels.'name'='tiller',spec.selector.matchLabels.'app'='helm' --output yaml > /tmp/helm-init.yaml
 389 #       sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' /tmp/helm-init.yaml > /tmp/helm-init-patched.yaml
 390 #       kubectl apply -f /tmp/helm-init-patched.yaml
 391 #    fi
 392 #  else
 393 #    if [[ ${HELMVERSION} == 2.* ]]; then
 394 #       helm init --service-account tiller
 395 #    fi
 396 #  fi
 397 #  if [[ ${HELMVERSION} == 2.* ]]; then
 398 #     helm init -c
 399 #     export HELM_HOME="$(pwd)/.helm"
 400 #     echo "HELM_HOME=${HELM_HOME}" >> /etc/environment
 401 #  fi
 402
 403   while ! helm version; do
 404     echo "Waiting for Helm to be ready"
 405     sleep 15
 406   done
 407
 408   echo "Preparing a master node (lower ID) for using local FS for PV"
 409   PV_NODE_NAME=$(kubectl get nodes |grep master | cut -f1 -d' ' | sort | head -1)
 410   kubectl label --overwrite nodes $PV_NODE_NAME local-storage=enable
 411   if [ "$PV_NODE_NAME" == "$(hostname)" ]; then
 412     mkdir -p /opt/data/dashboard-data
 413   fi
 414
 415   echo "Done with master node setup"
 416 fi
 417
 418
 419 if [[ ! -z "" && ! -z "" ]]; then
 420   echo " " >> /etc/hosts
 421 fi
 422 if [[ ! -z "" && ! -z "" ]]; then
 423   echo " " >> /etc/hosts
 424 fi
 425 if [[ ! -z "" && ! -z "helm.ricinfra.local" ]]; then
 426   echo " helm.ricinfra.local" >> /etc/hosts
 427 fi
 428
 429 if [[ "1" -gt "100" ]]; then
 430   cat <<EOF >/etc/ca-certificates/update.d/helm.crt
 431
 432 EOF
 433 fi
 434
 435 if [[ "1" -gt "100" ]]; then
 436   mkdir -p /etc/docker/certs.d/:
 437   cat <<EOF >/etc/docker/ca.crt
 438
 439 EOF
 440   cp /etc/docker/ca.crt /etc/docker/certs.d/:/ca.crt
 441
 442   service docker restart
 443   systemctl enable docker.service
 444   docker login -u  -p  :
 445   docker pull :/whoami:0.0.1
 446 fi