tools/k8s/bin/deploy-stack.sh

   1 #!/bin/bash
   2 ################################################################################
   3 #   Copyright (c) 2019 AT&T Intellectual Property.                             #
   4 #   Copyright (c) 2019 Nokia.                                                  #
   5 #                                                                              #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");            #
   7 #   you may not use this file except in compliance with the License.           #
   8 #   You may obtain a copy of the License at                                    #
   9 #                                                                              #
  10 #       http://www.apache.org/licenses/LICENSE-2.0                             #
  11 #                                                                              #
  12 #   Unless required by applicable law or agreed to in writing, software        #
  13 #   distributed under the License is distributed on an "AS IS" BASIS,          #
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
  15 #   See the License for the specific language governing permissions and        #
  16 #   limitations under the License.                                             #
  17 ################################################################################
  18
  19 set -e
  20
  21 stack_name="ric"
  22 full_deletion=false
  23
  24 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
  25 set -a
  26 RCS="$(find $DIR/../etc -type f -maxdepth 1)"
  27 for RC in $RCS; do
  28   echo "reading in values in $RC"
  29   source $RC
  30 done
  31 set +a
  32
  33
  34 if [ -z "$__RUNRICENV_GERRIT_HOST__" ]; then
  35    export __RUNRICENV_GERRIT_HOST__=$gerrithost
  36 fi
  37 if [ -z "$__RUNRICENV_GERRIT_IP__" ]; then
  38    export __RUNRICENV_GERRIT_IP__=$gerritip
  39 fi
  40 if [ -z "$__RUNRICENV_DOCKER_HOST__" ]; then
  41    export __RUNRICENV_DOCKER_HOST__=$dockerregistry
  42 fi
  43 if [ -z "$__RUNRICENV_DOCKER_IP__" ]; then
  44    export __RUNRICENV_DOCKER_IP__=$dockerip
  45 fi
  46 if [ -z "$__RUNRICENV_DOCKER_PORT__" ]; then
  47    export __RUNRICENV_DOCKER_PORT__=$dockerport
  48 fi
  49 if [ -z "$__RUNRICENV_DOCKER_USER__" ]; then
  50    export __RUNRICENV_DOCKER_USER__=$dockeruser
  51 fi
  52 if [ -z "$__RUNRICENV_DOCKER_PASS__" ]; then
  53    export __RUNRICENV_DOCKER_PASS__=$dockerpassword
  54 fi
  55 if [ -z "$__RUNRICENV_DOCKER_CERT__" ]; then
  56    export __RUNRICENV_DOCKER_CERT__=$dockercert
  57 fi
  58 if [ -z "$__RUNRICENV_HELMREPO_HOST__" ]; then
  59    export __RUNRICENV_HELMREPO_HOST__=$helmrepo
  60 fi
  61 if [ -z "$__RUNRICENV_HELMREPO_PORT__" ]; then
  62    export __RUNRICENV_HELMREPO_PORT__=$helmport
  63 fi
  64 if [ -z "$__RUNRICENV_HELMREPO_IP__" ]; then
  65    export __RUNRICENV_HELMREPO_IP__=$helmip
  66 fi
  67 if [ -z "$__RUNRICENV_HELMREPO_USER__" ]; then
  68    export __RUNRICENV_HELMREPO_USER__=$helmuser
  69 fi
  70 if [ -z "$__RUNRICENV_HELMREPO_PASS__" ]; then
  71    export __RUNRICENV_HELMREPO_PASS__=$helmpassword
  72 fi
  73 if [ -z "$__RUNRICENV_HELMREPO_CERT__" ]; then
  74    export __RUNRICENV_HELMREPO_CERT__=$helmcert
  75 fi
  76
  77
  78 if [ -z "$WORKSPACE" ]; then
  79     export WORKSPACE=`git rev-parse --show-toplevel`
  80 fi
  81 HEAT_DIR="$WORKSPACE/ric-infra/00-Kubernetes/heat"
  82 BIN_DIR="$WORKSPACE/ric-infra/00-Kubernetes/bin"
  83 ETC_DIR="$WORKSPACE/ric-infra/00-Kubernetes/etc"
  84 ENV_DIR="$WORKSPACE/ric-infra/00-Kubernetes/heat/env"
  85
  86
  87 cd $BIN_DIR
  88
  89
  90 openstack --version > /dev/null
  91 if [ $? -eq 0 ]; then
  92     echo "OK openstack CLI installed"
  93 else
  94     echo "Must run in an envirnment with openstack cli"
  95     exit 1
  96 fi
  97
  98 if [ -z "$OS_USERNAME" ]; then
  99     echo "Must source the Openstack RC file for the target installation tenant"
 100     exit 1
 101 fi
 102
 103
 104 usage() {
 105     echo "Usage: $0 [ -n <number of VMs {2-15}> ][ -s <stack name> ]<env> <ssh_keypair> <template>" 1>&2;
 106     echo "n:    Set the number of VMs that will be installed. " 1>&2;
 107     echo "s:    Set the name to be used for stack. This name will be used for naming of resources" 1>&2;
 108     echo "d:    Dryrun, only generating templates, no calling OpenStack API" 1>&2;
 109     echo "6:    When enabled, VMs will have an IPv6 interface." 1>&2;
 110
 111     exit 1;
 112 }
 113
 114
 115 dryrun='false'
 116 v6='false'
 117 while getopts ":n:w:s:6d" o; do
 118     case "${o}" in
 119         n)
 120             if [[ ${OPTARG} =~ ^[0-9]+$ ]];then
 121                 if [ ${OPTARG} -ge 1 -a ${OPTARG} -le 15 ]; then
 122                     vm_num=${OPTARG}
 123                 else
 124                     usage
 125                 fi
 126             else
 127                 usage
 128             fi
 129             ;;
 130         s)
 131             if [[ ! ${OPTARG} =~ ^[0-9]+$ ]];then
 132                 stack_name=${OPTARG}
 133             else
 134                 usage
 135             fi
 136             ;;
 137         w)
 138             WORKDIR_NAME=${OPTARG}
 139             ;;
 140         6)
 141             v6=true
 142             ;;
 143         d)
 144             dryrun=true
 145             ;;
 146         *)
 147             usage
 148             ;;
 149     esac
 150 done
 151 shift $((OPTIND-1))
 152
 153 if [ "$#" -lt 2 ]; then
 154    usage
 155 fi
 156
 157 ENV_FILE=$1
 158 if [ ! -f $ENV_FILE ]; then
 159     echo ENV file does not exist or was not given
 160     exit 1
 161 fi
 162 shift 1
 163
 164 SSH_KEY=$1
 165 if [ ! -s $SSH_KEY ]; then
 166     echo SSH Keypair file does not exist or was not given
 167     exit 1
 168 fi
 169 shift 1
 170
 171 if [ -z "$vm_num" ]; then
 172     TMPL_FILE=$1
 173     if [ ! -f $TMPL_FILE ]; then
 174         echo Heat template file does not exist or was not given
 175         exit 1
 176     fi
 177     shift 1
 178 fi
 179
 180 # Prints all commands to output that are executed by the terminal
 181 set -x
 182
 183 if [ -z "$WORKDIR_NAME" ]; then
 184   WORKDIR_NAME="workdir-$(date +%Y%m%d%H%M%S)"
 185 fi
 186 WORKDIR="$BIN_DIR/$WORKDIR_NAME"
 187 rm -rf "$WORKDIR"
 188 mkdir -p "$WORKDIR"
 189
 190 # get the openstack rc file env variable values in env file
 191 envsubst < $ENV_FILE > "$WORKDIR/$(basename $ENV_FILE)"
 192 ENV_FILE="$WORKDIR/$(basename $ENV_FILE)"
 193
 194 # prepare (localize) all scripts to be installed to the cluster VMs
 195 SCRIPTS=$(ls -1 $HEAT_DIR/scripts/*)
 196 for SCRIPT in $SCRIPTS; do
 197     envsubst '${__RUNRICENV_GERRIT_HOST__}
 198               ${__RUNRICENV_GERRIT_IP__}
 199               ${__RUNRICENV_DOCKER_HOST__}
 200               ${__RUNRICENV_DOCKER_IP__}
 201               ${__RUNRICENV_DOCKER_PORT__}
 202               ${__RUNRICENV_DOCKER_USER__}
 203               ${__RUNRICENV_DOCKER_PASS__}
 204               ${__RUNRICENV_DOCKER_CERT__}
 205               ${__RUNRICENV_HELMREPO_HOST__}
 206               ${__RUNRICENV_HELMREPO_PORT__}
 207               ${__RUNRICENV_HELMREPO_IP__}
 208               ${__RUNRICENV_HELMREPO_CERT__}
 209               ${__RUNRICENV_HELMREPO_USER__}
 210               ${__RUNRICENV_HELMREPO_PASS__} '< $SCRIPT > "$WORKDIR/$(basename $SCRIPT)"
 211 done
 212
 213 # generate a heat template with the specified number of VMs and IPv6 option
 214 if [ ! -z "$vm_num" ]; then
 215     CURDIR=$(pwd)
 216     if [ -z "$v6" ]; then
 217         ./gen-ric-heat-yaml.sh -n $vm_num > "$WORKDIR/k8s-${vm_num}VMs.yaml"
 218         TMPL_FILE="$WORKDIR/k8s-${vm_num}VMs.yaml"
 219     else
 220         ./gen-ric-heat-yaml.sh -6 -n $vm_num > "$WORKDIR/k8s-${vm_num}VMs-v6.yaml"
 221         TMPL_FILE="$WORKDIR/k8s-${vm_num}VMs-v6.yaml"
 222     fi
 223 fi
 224
 225 if [ "$dryrun" == "true" ]; then
 226     exit 0
 227 fi
 228
 229
 230 for n in $(seq 1 5); do
 231     echo "${n} of 5 attempts to deploy the stack $stack_name"
 232     FAILED='false'
 233     if [ ! -z "$(openstack stack list |grep -w $stack_name)" ]; then
 234         openstack stack delete $stack_name;
 235         while [ "DELETE_IN_PROGRESS" == "$(openstack stack show -c stack_status -f value $stack_name)" ]; do
 236             echo "Waiting for stack $stack_name deletion to complete"
 237             sleep 5
 238         done
 239     fi
 240
 241     # create a stack with the template and env files
 242     if ! openstack stack create -t $TMPL_FILE -e $ENV_FILE $stack_name; then
 243         FAILED='true'
 244         break
 245     fi
 246
 247     # wait for OpenStack stack creation completes
 248     while [ "CREATE_IN_PROGRESS" == "$(openstack stack show -c stack_status -f value $stack_name)" ]; do
 249         sleep 20
 250     done
 251
 252     STATUS=$(openstack stack show -c stack_status -f value $stack_name)
 253     echo $STATUS
 254     if [ "CREATE_COMPLETE" != "$STATUS" ]; then
 255         echo "OpenSatck stack creation failed"
 256         FAILED='true'
 257         break;
 258     fi
 259
 260     # wait till the Master node to become alive
 261     for i in $(seq 1 30); do
 262         sleep 30
 263         K8S_MST_IP=$(openstack stack output show $stack_name k8s_mst_vm_ip -c output_value -f value)
 264         timeout 1 ping -c 1 "$K8S_MST_IP" && break
 265     done
 266
 267     timeout 1 ping -c 1 "$K8S_MST_IP" && break
 268
 269     echo Error: OpenStack infrastructure issue: unable to reach master node "$K8S_MST_IP"
 270     FAILED='true'
 271     sleep 10
 272 done
 273
 274 if ! timeout 1 ping -c 1 "$K8S_MST_IP"; then
 275     echo "Master node not reachable, stack creation failed, exit"
 276     exit 2
 277 fi
 278
 279
 280 K8S_MASTER_HOSTNAME="${stack_name}-k8s-mst"
 281 echo "$K8S_MASTER_HOSTNAME $K8S_MST_IP" > ./ips-${stack_name}
 282 while ! nc -z $K8S_MST_IP  29999; do
 283   echo "Wait for Master node $K8S_MST_IP to be ready"
 284   sleep 5
 285 done
 286
 287 set +e
 288
 289 unset JOINCMD
 290 while [[ -z $JOINCMD ]]; do
 291   sleep 15
 292   JOINCMD=$(ssh -i $SSH_KEY ubuntu@$K8S_MST_IP  -q -o "StrictHostKeyChecking no" sudo kubeadm token create --print-join-command)
 293 done
 294
 295 for i in $(seq 1 99); do
 296   IP_NAME=k8s_$(printf "%02d" "$i")_vm_ip
 297   K8S_MINION_IP=$(openstack stack output show $stack_name $IP_NAME -c output_value -f value)
 298   if [ -z $K8S_MINION_IP ]; then
 299     break
 300   fi
 301   K8S_MINION_HOSTNAME=${stack_name}-k8s-$(printf "%02d" "$i")
 302   echo "$K8S_MINION_HOSTNAME $K8S_MINION_IP" >> ./ips-${stack_name}
 303
 304   #while ! nc -z $K8S_MINION_IP  29999; do
 305   #  echo "Wait for minion node $K8S_MINION_IP to be ready"
 306   #  sleep 5
 307   #done
 308   echo "Joining $K8S_MINION_HOSTNAME [$K8S_MINION_IP] to cluster master $K8S_MST_IP with command $JOINCMD"
 309   while ! ssh -i $SSH_KEY -q -o "StrictHostKeyChecking no" ubuntu@$K8S_MINION_IP sudo $JOINCMD; do
 310     echo "Retry join command in 10 seconds"
 311     sleep 10
 312   done
 313 done
 314
 315 export __IPS_${stack_name}__="$(cat ${WORKDIR}/ips-${stack_name})"