infra/00-Kubernetes/bin/install-1node-k8s

   1 #!/bin/bash
   2
   3 ################################################################################
   4 #   Copyright (c) 2019 AT&T Intellectual Property.                             #
   5 #   Copyright (c) 2019 Nokia.                                                  #
   6 #                                                                              #
   7 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   8 #   you may not use this file except in compliance with the License.           #
   9 #   You may obtain a copy of the License at                                    #
  10 #                                                                              #
  11 #       http://www.apache.org/licenses/LICENSE-2.0                             #
  12 #                                                                              #
  13 #   Unless required by applicable law or agreed to in writing, software        #
  14 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  15 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  16 #   See the License for the specific language governing permissions and        #
  17 #   limitations under the License.                                             #
  18 ################################################################################
  19
  20 # The intention for this script is to stand up a dev testing k8s environment
  21 # that is ready for RIC installation for individual developer/team's API and functional
  22 # testing needs.
  23 # The integration team will maintain the synchronization of software infrastructure
  24 # stack (software, versions and configurations) between this iscript and what is
  25 # provided for the E2E validation testing.  Due to resource and other differences, this
  26 # environment is not intended for any testing related to performance, resilience,
  27 # robustness, etc.
  28
  29 # This script installs docker host, a one-node k8s cluster, and Helm for CoDev.
  30 # This script assumes that it will be executed on an Ubuntu 16.04 VM.
  31 # It is best to be run as the cloud-init script at the VM launch time, or from a
  32 # "sudo -i" shell post-launch on a newly launched VM.
  33 #
  34
  35 set -x
  36
  37 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
  38 source $DIR/../etc/k8s-1node
  39
  40
  41 if [ -z "$__RUNRICENV_GERRIT_HOST__" ]; then
  42    export __RUNRICENV_GERRIT_HOST__=$gerrithost
  43 fi
  44 if [ -z "$__RUNRICENV_GERRIT_IP__" ]; then
  45    export __RUNRICENV_GERRIT_IP__=$gerritip
  46 fi
  47 if [ -z "$__RUNRICENV_DOCKER_HOST__" ]; then
  48    export __RUNRICENV_DOCKER_HOST__=$dockerregistry
  49 fi
  50 if [ -z "$__RUNRICENV_DOCKER_IP__" ]; then
  51    export __RUNRICENV_DOCKER_IP__=$dockerip
  52 fi
  53 if [ -z "$__RUNRICENV_DOCKER_PORT__" ]; then
  54    export __RUNRICENV_DOCKER_PORT__=$dockerport
  55 fi
  56 if [ -z "$__RUNRICENV_DOCKER_USER__" ]; then
  57    export __RUNRICENV_DOCKER_USER__=$dockeruser
  58 fi
  59 if [ -z "$__RUNRICENV_DOCKER_PASS__" ]; then
  60    export __RUNRICENV_DOCKER_PASS__=$dockerpassword
  61 fi
  62 if [ -z "$__RUNRICENV_HELMREPO_HOST__" ]; then
  63    export __RUNRICENV_HELMREPO_HOST__=$helmrepo
  64 fi
  65 if [ -z "$__RUNRICENV_HELMREPO_PORT__" ]; then
  66    export __RUNRICENV_HELMREPO_PORT__=$helmport
  67 fi
  68 if [ -z "$__RUNRICENV_HELMREPO_IP__" ]; then
  69    export __RUNRICENV_HELMREPO_IP__=$helmip
  70 fi
  71 if [ -z "$__RUNRICENV_HELMREPO_USER__" ]; then
  72    export __RUNRICENV_HELMREPO_USER__=$helmuser
  73 fi
  74 if [ -z "$__RUNRICENV_HELMREPO_PASS__" ]; then
  75    export __RUNRICENV_HELMREPO_PASS__=$helmpassword
  76 fi
  77
  78
  79
  80
  81 # for RIC R0 we keep 1.13
  82 export KUBEV="1.13.3"
  83 export KUBECNIV="0.6.0"
  84 export DOCKERV="18.06.1"
  85
  86 # for new 1.14 release
  87 #export KUBEVERSION="1.14.0"
  88 #export KUBECNIVERSION="0.7.0"
  89 #export DOCKEFV="18.06.1"
  90
  91 export HELMV="2.12.3"
  92
  93 unset FIRSTBOOT
  94 unset DORESET
  95
  96 while getopts ":r" opt; do
  97   case ${opt} in
  98     r )
  99       DORESET='YES'
 100       ;;
 101     \? )
 102       echo "Usage: $0 [-r]"
 103       exit
 104       ;;
 105   esac
 106 done
 107
 108
 109 if [ ! -e /var/tmp/firstboot4setupk8s ]; then
 110   echo "First time"
 111   FIRSTBOOT='YES'
 112   touch /var/tmp/firstboot4setupk8s
 113
 114   modprobe -- ip_vs
 115   modprobe -- ip_vs_rr
 116   modprobe -- ip_vs_wrr
 117   modprobe -- ip_vs_sh
 118   modprobe -- nf_conntrack_ipv4
 119
 120   # disable swap
 121   SWAPFILES=$(grep swap /etc/fstab | sed '/^#/ d' |cut -f1 -d' ')
 122   if [ ! -z $SWAPFILES ]; then
 123     for SWAPFILE in $SWAPFILES
 124     do
 125       echo "disabling swap file $SWAPFILE"
 126       if [[ $SWAPFILE == UUID* ]]; then
 127         UUID=$(echo $SWAPFILE | cut -f2 -d'=')
 128         swapoff -U $UUID
 129       else
 130         swapoff $SWAPFILE
 131       fi
 132       # edit /etc/fstab file, remove line with /swapfile
 133       sed -i -e "/$SWAPFILE/d" /etc/fstab
 134     done
 135   fi
 136   # disable swap
 137   #swapoff /swapfile
 138   # edit /etc/fstab file, remove line with /swapfile
 139   #sed -i -e '/swapfile/d' /etc/fstab
 140
 141
 142   # add rancodev CI tool hostnames
 143   echo "${__RUNRICENV_GERRIT_IP__} ${__RUNRICENV_GERRIT_HOST__}" >> /etc/hosts
 144   echo "${__RUNRICENV_DOCKER_IP__} ${__RUNRICENV_DOCKER_HOST__}" >> /etc/hosts
 145   echo "${__RUNRICENV_HELMREPO_IP__} ${__RUNRICENV_HELMREPO_HOST__}" >> /etc/hosts
 146
 147
 148   # create kubenetes config file
 149   if [[ ${KUBEV} == 1.13.* ]]; then
 150     cat <<EOF >/root/config.yaml
 151 apiVersion: kubeadm.k8s.io/v1alpha3
 152 kubernetesVersion: v${KUBEV}
 153 kind: ClusterConfiguration
 154 apiServerExtraArgs:
 155   feature-gates: SCTPSupport=true
 156 networking:
 157   dnsDomain: cluster.local
 158   podSubnet: 10.244.0.0/16
 159   serviceSubnet: 10.96.0.0/12
 160
 161 ---
 162 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 163 kind: KubeProxyConfiguration
 164 mode: ipvs
 165 EOF
 166   elif [[ ${KUBEV} == 1.14.* ]]; then
 167     cat <<EOF >/root/config.yaml
 168 apiVersion: kubeadm.k8s.io/v1beta1
 169 kubernetesVersion: v${KUBEV}
 170 kind: ClusterConfiguration
 171 apiServerExtraArgs:
 172   feature-gates: SCTPSupport=true
 173 networking:
 174   dnsDomain: cluster.local
 175   podSubnet: 10.244.0.0/16
 176   serviceSubnet: 10.96.0.0/12
 177
 178 ---
 179 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 180 kind: KubeProxyConfiguration
 181 mode: ipvs
 182 EOF
 183   else
 184     echo "Unsupported Kubernetes version requested.  Bail."
 185     exit
 186   fi
 187
 188
 189   # create a RBAC file for helm (tiller)
 190   cat <<EOF > /root/rbac-config.yaml
 191 apiVersion: v1
 192 kind: ServiceAccount
 193 metadata:
 194   name: tiller
 195   namespace: kube-system
 196 ---
 197 apiVersion: rbac.authorization.k8s.io/v1
 198 kind: ClusterRoleBinding
 199 metadata:
 200   name: tiller
 201 roleRef:
 202   apiGroup: rbac.authorization.k8s.io
 203   kind: ClusterRole
 204   name: cluster-admin
 205 subjects:
 206   - kind: ServiceAccount
 207     name: tiller
 208     namespace: kube-system
 209 EOF
 210
 211
 212   KUBEVERSION="${KUBEV}-00"
 213   CNIVERSION="${KUBECNIV}-00"
 214   DOCKERVERSION="${DOCKERV}-0ubuntu1.2~16.04.1"
 215   curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 216   echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
 217
 218   # install low latency kernel, docker.io, and kubernetes
 219   apt-get update
 220   apt-get install -y linux-image-4.15.0-45-lowlatency docker.io=${DOCKERVERSION}
 221   apt-get install -y kubernetes-cni=${CNIVERSION}
 222   apt-get install -y --allow-unauthenticated kubeadm=${KUBEVERSION} kubelet=${KUBEVERSION} kubectl=${KUBEVERSION}
 223   apt-mark hold kubernetes-cni kubelet kubeadm kubectl
 224
 225   # install Helm
 226   HELMVERSION=${HELMV}
 227   cd /root
 228   mkdir Helm
 229   cd Helm
 230   wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELMVERSION}-linux-amd64.tar.gz
 231   tar -xvf helm-v${HELMVERSION}-linux-amd64.tar.gz
 232   mv linux-amd64/helm /usr/local/bin/helm
 233
 234
 235   # add cert for accessing docker registry in Azure
 236   mkdir -p /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
 237   cat <<EOF >/etc/docker/ca.crt
 238 -----BEGIN CERTIFICATE-----
 239 MIIEPjCCAyagAwIBAgIJAIwtTKgVAnvrMA0GCSqGSIb3DQEBCwUAMIGzMQswCQYD
 240 VQQGEwJVUzELMAkGA1UECAwCTkoxEzARBgNVBAcMCkJlZG1pbnN0ZXIxDTALBgNV
 241 BAoMBEFUJlQxETAPBgNVBAsMCFJlc2VhcmNoMTswOQYDVQQDDDIqLmRvY2tlci5y
 242 YW5jby1kZXYtdG9vbHMuZWFzdHVzLmNsb3VkYXBwLmF6dXJlLmNvbTEjMCEGCSqG
 243 SIb3DQEJARYUcmljQHJlc2VhcmNoLmF0dC5jb20wHhcNMTkwMTI0MjA0MzIzWhcN
 244 MjQwMTIzMjA0MzIzWjCBszELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5KMRMwEQYD
 245 VQQHDApCZWRtaW5zdGVyMQ0wCwYDVQQKDARBVCZUMREwDwYDVQQLDAhSZXNlYXJj
 246 aDE7MDkGA1UEAwwyKi5kb2NrZXIucmFuY28tZGV2LXRvb2xzLmVhc3R1cy5jbG91
 247 ZGFwcC5henVyZS5jb20xIzAhBgkqhkiG9w0BCQEWFHJpY0ByZXNlYXJjaC5hdHQu
 248 Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuAW1O52l9/1L+D7x
 249 Qv+782FjiELP9MWO0RCAL2JzV6Ioeo1IvuZl8jvXQVGlowanCdz1HZlMJjGm6Ybv
 250 60dVECRSMZeOxUQ0JCus6thxOhDiiCFT59m+MpdrRgHqwOzw+8B49ZwULv+lTIWt
 251 ETEQkSYTh4No9jCxnyVLKH9DyTbaW/xFK484u5f4bh7mI5uqDJapOCRvJXv8/J0E
 252 eMrkCVmk5qy0ii8I7O0oCNl61YvC5by9GCeuQhloJJc6gOjzKW8nK9JfUW8G34bC
 253 qnUj79EgwgtW/8F5SYAF5LI0USM0xXjyzlnPMbv5mikrbf0EZkZXdUreICUIzY53
 254 HRocCQIDAQABo1MwUTAdBgNVHQ4EFgQUm9NbNhZ3Zp1f50DIN4/4fvWQSNswHwYD
 255 VR0jBBgwFoAUm9NbNhZ3Zp1f50DIN4/4fvWQSNswDwYDVR0TAQH/BAUwAwEB/zAN
 256 BgkqhkiG9w0BAQsFAAOCAQEAkbuqbuMACRmzMXFKoSsMTLk/VRQDlKeubdP4lD2t
 257 Z+2dbhfbfiae9oMly7hPCDacoY0cmlBb2zZ8lgA7kVvuw0xwX8mLGYfOaNG9ENe5
 258 XxFP8MuaCySy1+v5CsNnh/WM3Oznc6MTv/0Nor2DeY0XHQtM5LWrqyKGZaVAKpMW
 259 5nHG8EPIZAOk8vj/ycg3ca3Wv3ne9/8rbrrxDJ3p4L70DOtz/JcQai10Spct4S0Z
 260 7yd4tQL+QSQCvmN7Qm9+i52bY0swYrUAhbNiEX3yJDryKjSCPirePcieGZmBRMxr
 261 7j28jxpa4g32TbWR/ZdxMYEkCVTFViTE23kZdNvahHKfdQ==
 262 -----END CERTIFICATE-----
 263 EOF
 264   cp /etc/docker/ca.crt /etc/docker/certs.d/${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/ca.crt
 265   service docker restart
 266   systemctl enable docker.service
 267   docker login -u ${__RUNRICENV_DOCKER_USER__} -p ${__RUNRICENV_DOCKER_PASS__} ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}
 268   docker pull ${__RUNRICENV_DOCKER_HOST__}:${__RUNRICENV_DOCKER_PORT__}/whoami:0.0.1
 269
 270
 271   # test access to k8s docker registry
 272   kubeadm config images pull
 273 else
 274   echo "Not first boot"
 275
 276   kubectl get pods --all-namespaces
 277 fi
 278
 279
 280 if [ -n "$DORESET" ]; then
 281   kubeadm reset
 282 fi
 283
 284 if [ -n ${DORESET+set} ] || [ -n ${FIRSTBOOT+set} ]; then
 285   # start cluster (make sure CIDR is enabled with the flag)
 286   kubeadm init --config /root/config.yaml
 287
 288   # set up kubectl credential and config
 289   cd /root
 290   rm -rf .kube
 291   mkdir -p .kube
 292   cp -i /etc/kubernetes/admin.conf /root/.kube/config
 293   chown root:root /root/.kube/config
 294
 295   # at this point we should be able to use kubectl
 296   kubectl get pods --all-namespaces
 297   # you will see the DNS pods stuck in pending state.  They are waiting for some networking to be installed.
 298
 299   # install flannel
 300   # kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
 301   kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml
 302
 303   # waiting for all kube-system pods to be in running state
 304   NUMPODS=0
 305   while [  $NUMPODS -lt 8 ]; do
 306     sleep 5
 307     OUTPUT=$(kubectl get pods --all-namespaces |grep Running)
 308     NUMPODS=$(echo "$OUTPUT" | wc -l)
 309     echo "Waiting for $NUMPODS / 8 kube-system pods reaching Running state"
 310   done
 311
 312   # if running a single node cluster, need to enable master node to run pods
 313   kubectl taint nodes --all node-role.kubernetes.io/master-
 314
 315   cd /root
 316   # install RBAC for Helm
 317   kubectl create -f rbac-config.yaml
 318
 319   rm -rf .helm
 320   helm init --service-account tiller
 321
 322
 323   cat <<EOF >/etc/ca-certificates/update.d/helm.crt
 324 -----BEGIN CERTIFICATE-----
 325 MIIESjCCAzKgAwIBAgIJAIU+AfULkw0PMA0GCSqGSIb3DQEBCwUAMIG5MQswCQYD
 326 VQQGEwJVUzETMBEGA1UECAwKTmV3IEplcnNleTETMBEGA1UEBwwKQmVkbWluc3Rl
 327 cjENMAsGA1UECgwEQVQmVDERMA8GA1UECwwIUmVzZWFyY2gxOTA3BgNVBAMMMCou
 328 aGVsbS5yYW5jby1kZXYtdG9vbHMuZWFzdHVzLmNsb3VkYXBwLmF6dXJlLmNvbTEj
 329 MCEGCSqGSIb3DQEJARYUcmljQHJlc2VhcmNoLmF0dC5jb20wHhcNMTkwMzIxMTU1
 330 MzAwWhcNMjEwMzIwMTU1MzAwWjCBuTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCk5l
 331 dyBKZXJzZXkxEzARBgNVBAcMCkJlZG1pbnN0ZXIxDTALBgNVBAoMBEFUJlQxETAP
 332 BgNVBAsMCFJlc2VhcmNoMTkwNwYDVQQDDDAqLmhlbG0ucmFuY28tZGV2LXRvb2xz
 333 LmVhc3R1cy5jbG91ZGFwcC5henVyZS5jb20xIzAhBgkqhkiG9w0BCQEWFHJpY0By
 334 ZXNlYXJjaC5hdHQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
 335 tguhSQx5Dk2w+qx2AOcFRz7IZBASEehK1Z4f5jz2KrRylGx6jjedCZASdm1b0ZEB
 336 /ZNrKht1zsWDETa7x0DF+q0Z2blff+T+6+YrJWhNxYHgZiYVi9gTuNDzpn8VVn7f
 337 +cQxcMguHo1JBDIotOLubJ4T3/oXMCPv9kRSLHcNjbEE2yTB3AqXu9dvrDXuUdeU
 338 ot6RzxhKXxRCQXPS2/FDjSV9vr9h1dv5fIkFXihpYaag0XqvXcqgncvcOJ1SsLc3
 339 DK+tyNknqG5SL8y2a7U4F7u+qGO2/3tnCO0ggYwa73hS0pQPY51EpRSckZqlfKEu
 340 Ut0s3wlEFP1VaU0RfU3aIwIDAQABo1MwUTAdBgNVHQ4EFgQUYTpoVXZPXSR/rhjr
 341 pu9PPhL7f9IwHwYDVR0jBBgwFoAUYTpoVXZPXSR/rhjrpu9PPhL7f9IwDwYDVR0T
 342 AQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAUDLbiKVIW6W9qFXLtoyO7S2e
 343 IOUSZ1F70pkfeYUqegsfFZ9njPtPqTzDfJVxYqH2V0vxxoAxXCYCpNyR6vYlYiEL
 344 R+oyxuvauW/yCoiwKBPYa4fD/PBajJnEO1EfIwZvjFLIfw4GjaX59+zDS3Zl0jT/
 345 w3uhPSsJAYXtDKLZ14btA27cM5mW4kmxVD8CRdUW0jr/cN3Hqe9uLSNWCNiDwma7
 346 RnpK7NnOgXHyhZD/nVC0nY7OzbK7VHFJatSOjyuMxgWsFGahwYNxf3AWfPwUai0K
 347 ne/fVFGZ6ifR9QdD0SuKIAEuqSyyP4BsQ92uEweU/gWKsnM6iNVmNFX8UOuU9A==
 348 -----END CERTIFICATE-----
 349 EOF
 350
 351   # waiting for tiller pod to be in running state
 352   NUMPODS=0
 353   while [ $NUMPODS -lt 1 ]; do
 354     sleep 5
 355     OUTPUT=$(kubectl get pods --all-namespaces |grep Running)
 356     NUMPODS=$(echo "$OUTPUT" | grep "tiller-deploy" | wc -l)
 357     echo "Waiting for $NUMPODS / 1 tiller-deploy pod reaching Running state"
 358   done
 359
 360   echo "All up"
 361
 362   #reboot
 363 fi