Add release notes
[it/dep.git] / tools / k8s / bin / deploy-stack.sh
1 #!/bin/bash
2 ################################################################################
3 #   Copyright (c) 2019 AT&T Intellectual Property.                             #
4 #   Copyright (c) 2019 Nokia.                                                  #
5 #                                                                              #
6 #   Licensed under the Apache License, Version 2.0 (the "License");            #
7 #   you may not use this file except in compliance with the License.           #
8 #   You may obtain a copy of the License at                                    #
9 #                                                                              #
10 #       http://www.apache.org/licenses/LICENSE-2.0                             #
11 #                                                                              #
12 #   Unless required by applicable law or agreed to in writing, software        #
13 #   distributed under the License is distributed on an "AS IS" BASIS,          #
14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
15 #   See the License for the specific language governing permissions and        #
16 #   limitations under the License.                                             #
17 ################################################################################
18
19 set -e 
20
21 stack_name="ric"
22 full_deletion=false
23
24 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
25 set -a
26 RCS="$(find $DIR/../etc -type f -maxdepth 1)"
27 for RC in $RCS; do
28   echo "reading in values in $RC"
29   source $RC
30 done
31 set +a
32
33
34 if [ -z "$__RUNRICENV_GERRIT_HOST__" ]; then
35    export __RUNRICENV_GERRIT_HOST__=$gerrithost
36 fi
37 if [ -z "$__RUNRICENV_GERRIT_IP__" ]; then
38    export __RUNRICENV_GERRIT_IP__=$gerritip
39 fi
40 if [ -z "$__RUNRICENV_DOCKER_HOST__" ]; then
41    export __RUNRICENV_DOCKER_HOST__=$dockerregistry
42 fi
43 if [ -z "$__RUNRICENV_DOCKER_IP__" ]; then
44    export __RUNRICENV_DOCKER_IP__=$dockerip
45 fi
46 if [ -z "$__RUNRICENV_DOCKER_PORT__" ]; then
47    export __RUNRICENV_DOCKER_PORT__=$dockerport
48 fi
49 if [ -z "$__RUNRICENV_DOCKER_USER__" ]; then
50    export __RUNRICENV_DOCKER_USER__=$dockeruser
51 fi
52 if [ -z "$__RUNRICENV_DOCKER_PASS__" ]; then
53    export __RUNRICENV_DOCKER_PASS__=$dockerpassword
54 fi
55 if [ -z "$__RUNRICENV_DOCKER_CERT__" ]; then
56    export __RUNRICENV_DOCKER_CERT__=$dockercert
57 fi
58 if [ -z "$__RUNRICENV_HELMREPO_HOST__" ]; then
59    export __RUNRICENV_HELMREPO_HOST__=$helmrepo
60 fi
61 if [ -z "$__RUNRICENV_HELMREPO_PORT__" ]; then
62    export __RUNRICENV_HELMREPO_PORT__=$helmport
63 fi
64 if [ -z "$__RUNRICENV_HELMREPO_IP__" ]; then
65    export __RUNRICENV_HELMREPO_IP__=$helmip
66 fi
67 if [ -z "$__RUNRICENV_HELMREPO_USER__" ]; then
68    export __RUNRICENV_HELMREPO_USER__=$helmuser
69 fi
70 if [ -z "$__RUNRICENV_HELMREPO_PASS__" ]; then
71    export __RUNRICENV_HELMREPO_PASS__=$helmpassword
72 fi
73 if [ -z "$__RUNRICENV_HELMREPO_CERT__" ]; then
74    export __RUNRICENV_HELMREPO_CERT__=$helmcert
75 fi
76
77
78 if [ -z "$WORKSPACE" ]; then
79     export WORKSPACE=`git rev-parse --show-toplevel`
80 fi
81 HEAT_DIR="$WORKSPACE/tools/k8s/heat"
82 BIN_DIR="$WORKSPACE/tools/k8s/bin"
83 ETC_DIR="$WORKSPACE/tools/k8s/etc"
84 ENV_DIR="$WORKSPACE/tools/k8s/heat/env"
85
86
87 cd $BIN_DIR
88
89
90 openstack --version > /dev/null
91 if [ $? -eq 0 ]; then
92     echo "OK openstack CLI installed"
93 else
94     echo "Must run in an envirnment with openstack cli"
95     exit 1
96 fi
97
98 if [ -z "$OS_USERNAME" ]; then
99     echo "Must source the Openstack RC file for the target installation tenant"
100     exit 1
101 fi
102
103
104 usage() {
105     echo "Usage: $0 [ -n <number of VMs {2-15}> ][ -s <stack name> ]<env> <ssh_keypair> <template>" 1>&2;
106     echo "n:    Set the number of VMs that will be installed. " 1>&2;
107     echo "s:    Set the name to be used for stack. This name will be used for naming of resources" 1>&2;
108     echo "d:    Dryrun, only generating templates, no calling OpenStack API" 1>&2;
109     echo "6:    When enabled, VMs will have an IPv6 interface." 1>&2;
110
111     exit 1;
112 }
113
114
115 dryrun='false'
116 v6='false'
117 while getopts ":n:w:s:6d" o; do
118     case "${o}" in
119         n)
120             if [[ ${OPTARG} =~ ^[0-9]+$ ]];then
121                 if [ ${OPTARG} -ge 1 -a ${OPTARG} -le 15 ]; then
122                     vm_num=${OPTARG}
123                 else
124                     usage
125                 fi
126             else
127                 usage
128             fi
129             ;;
130         s)
131             if [[ ! ${OPTARG} =~ ^[0-9]+$ ]];then
132                 stack_name=${OPTARG}
133             else
134                 usage
135             fi
136             ;;
137         w)
138             WORKDIR_NAME=${OPTARG}
139             ;;
140         6)
141             v6=true
142             ;;
143         d)
144             dryrun=true
145             ;;
146         *)
147             usage
148             ;;
149     esac
150 done
151 shift $((OPTIND-1))
152
153 if [ "$#" -lt 2 ]; then
154    usage
155 fi
156
157 ENV_FILE=$1
158 if [ ! -f $ENV_FILE ]; then
159     echo ENV file does not exist or was not given
160     exit 1
161 fi
162 shift 1
163
164 SSH_KEY=$1
165 if [ ! -s $SSH_KEY ]; then
166     echo SSH Keypair file does not exist or was not given
167     exit 1
168 fi
169 shift 1
170
171 if [ -z "$vm_num" ]; then
172     TMPL_FILE=$1
173     if [ ! -f $TMPL_FILE ]; then
174         echo Heat template file does not exist or was not given
175         exit 1
176     fi
177     shift 1
178 fi
179
180 # Prints all commands to output that are executed by the terminal
181 set -x
182
183 if [ -z "$WORKDIR_NAME" ]; then
184   WORKDIR_NAME="workdir-$(date +%Y%m%d%H%M%S)"
185 fi
186 WORKDIR="$BIN_DIR/$WORKDIR_NAME"
187 rm -rf "$WORKDIR"
188 mkdir -p "$WORKDIR"
189
190 # get the openstack rc file env variable values in env file
191 envsubst < $ENV_FILE > "$WORKDIR/$(basename $ENV_FILE)"
192 ENV_FILE="$WORKDIR/$(basename $ENV_FILE)"
193
194 # prepare (localize) all scripts to be installed to the cluster VMs
195 SCRIPTS=$(ls -1 $HEAT_DIR/scripts/*)
196 for SCRIPT in $SCRIPTS; do
197     envsubst '${__RUNRICENV_GERRIT_HOST__}
198               ${__RUNRICENV_GERRIT_IP__}
199               ${__RUNRICENV_DOCKER_HOST__}
200               ${__RUNRICENV_DOCKER_IP__}
201               ${__RUNRICENV_DOCKER_PORT__}
202               ${__RUNRICENV_DOCKER_USER__}
203               ${__RUNRICENV_DOCKER_PASS__}
204               ${__RUNRICENV_DOCKER_CERT__}
205               ${__RUNRICENV_HELMREPO_HOST__}
206               ${__RUNRICENV_HELMREPO_PORT__}
207               ${__RUNRICENV_HELMREPO_IP__}
208               ${__RUNRICENV_HELMREPO_CERT__}
209               ${__RUNRICENV_HELMREPO_USER__}
210               ${__RUNRICENV_HELMREPO_PASS__} '< $SCRIPT > "$WORKDIR/$(basename $SCRIPT)"
211 done
212     
213 # generate a heat template with the specified number of VMs and IPv6 option
214 if [ ! -z "$vm_num" ]; then
215     CURDIR=$(pwd)
216     if [ -z "$v6" ]; then
217         ./gen-ric-heat-yaml.sh -n $vm_num > "$WORKDIR/k8s-${vm_num}VMs.yaml"
218         TMPL_FILE="$WORKDIR/k8s-${vm_num}VMs.yaml"
219     else
220         ./gen-ric-heat-yaml.sh -6 -n $vm_num > "$WORKDIR/k8s-${vm_num}VMs-v6.yaml"
221         TMPL_FILE="$WORKDIR/k8s-${vm_num}VMs-v6.yaml"
222     fi
223 fi
224
225 if [ "$dryrun" == "true" ]; then
226     exit 0
227 fi
228
229
230 for n in $(seq 1 5); do
231     echo "${n} of 5 attempts to deploy the stack $stack_name"
232     FAILED='false'
233     if [ ! -z "$(openstack stack list |grep -w $stack_name)" ]; then
234         openstack stack delete $stack_name;
235         while [ "DELETE_IN_PROGRESS" == "$(openstack stack show -c stack_status -f value $stack_name)" ]; do
236             echo "Waiting for stack $stack_name deletion to complete"
237             sleep 5
238         done
239     fi
240
241     # create a stack with the template and env files
242     if ! openstack stack create -t $TMPL_FILE -e $ENV_FILE $stack_name; then
243         FAILED='true'
244         break
245     fi
246
247     # wait for OpenStack stack creation completes
248     while [ "CREATE_IN_PROGRESS" == "$(openstack stack show -c stack_status -f value $stack_name)" ]; do
249         sleep 20
250     done
251
252     STATUS=$(openstack stack show -c stack_status -f value $stack_name)
253     echo $STATUS
254     if [ "CREATE_COMPLETE" != "$STATUS" ]; then
255         echo "OpenSatck stack creation failed"
256         FAILED='true'
257         break;
258     fi
259
260     # wait till the Master node to become alive
261     for i in $(seq 1 30); do
262         sleep 30
263         K8S_MST_IP=$(openstack stack output show $stack_name k8s_mst_vm_ip -c output_value -f value)
264         timeout 1 ping -c 1 "$K8S_MST_IP" && break
265     done
266
267     timeout 1 ping -c 1 "$K8S_MST_IP" && break
268
269     echo Error: OpenStack infrastructure issue: unable to reach master node "$K8S_MST_IP"
270     FAILED='true'
271     sleep 10
272 done
273
274 if ! timeout 1 ping -c 1 "$K8S_MST_IP"; then
275     echo "Master node not reachable, stack creation failed, exit"
276     exit 2
277 fi
278
279
280 K8S_MASTER_HOSTNAME="${stack_name}-k8s-mst"
281 echo "$K8S_MASTER_HOSTNAME $K8S_MST_IP" > ./ips-${stack_name}
282 while ! nc -z $K8S_MST_IP  29999; do
283   echo "Wait for Master node $K8S_MST_IP to be ready"
284   sleep 5
285 done
286
287 set +e
288
289 unset JOINCMD
290 while [[ -z $JOINCMD ]]; do
291   sleep 15
292   JOINCMD=$(ssh -i $SSH_KEY ubuntu@$K8S_MST_IP  -q -o "StrictHostKeyChecking no" sudo kubeadm token create --print-join-command)
293 done
294
295 for i in $(seq 1 99); do
296   IP_NAME=k8s_$(printf "%02d" "$i")_vm_ip
297   K8S_MINION_IP=$(openstack stack output show $stack_name $IP_NAME -c output_value -f value)
298   if [ -z $K8S_MINION_IP ]; then
299     break
300   fi
301   K8S_MINION_HOSTNAME=${stack_name}-k8s-$(printf "%02d" "$i")
302   echo "$K8S_MINION_HOSTNAME $K8S_MINION_IP" >> ./ips-${stack_name}
303
304   #while ! nc -z $K8S_MINION_IP  29999; do
305   #  echo "Wait for minion node $K8S_MINION_IP to be ready"
306   #  sleep 5
307   #done
308   echo "Joining $K8S_MINION_HOSTNAME [$K8S_MINION_IP] to cluster master $K8S_MST_IP with command $JOINCMD"
309   while ! ssh -i $SSH_KEY -q -o "StrictHostKeyChecking no" ubuntu@$K8S_MINION_IP sudo $JOINCMD; do
310     echo "Retry join command in 10 seconds"
311     sleep 10
312   done
313 done
314
315 export __IPS_${stack_name}__="$(cat ${WORKDIR}/ips-${stack_name})"