python-eventlet: downversion to 0.18.4 to align with stx
[pti/rtp.git] / scripts / build_stx.sh
1 #!/bin/bash
2 #
3 # Copyright (C) 2019 Wind River Systems, Inc.
4 #
5 #  Licensed under the Apache License, Version 2.0 (the "License");
6 #  you may not use this file except in compliance with the License.
7 #  You may obtain a copy of the License at
8 #
9 #      http://www.apache.org/licenses/LICENSE-2.0
10 #
11 #  Unless required by applicable law or agreed to in writing, software
12 #  distributed under the License is distributed on an "AS IS" BASIS,
13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 #  See the License for the specific language governing permissions and
15 #  limitations under the License.
16
17 # Ensure we fail the job if any steps fail.
18 set -e -o pipefail
19
20 #########################################################################
21 # Variables
22 #########################################################################
23
24 # Only one bsp is supported now, there will be more in the future
25 SUPPORTED_BSP="intel-corei7-64 "
26
27 SRC_ORAN_BRANCH="master"
28 SRC_STX_BRANCH="master"
29 SRC_YP_BRANCH="warrior"
30
31 SRC_ORAN_URL="https://gerrit.o-ran-sc.org/r/pti/rtp"
32
33 SRC_YP_URL="\
34     git://git.yoctoproject.org/poky \
35     git://git.openembedded.org/meta-openembedded \
36     git://git.yoctoproject.org/meta-virtualization \
37     git://git.yoctoproject.org/meta-cloud-services \
38     git://git.yoctoproject.org/meta-security \
39     git://git.yoctoproject.org/meta-intel \
40     git://git.yoctoproject.org/meta-security \
41     git://git.yoctoproject.org/meta-selinux \
42     https://github.com/intel-iot-devkit/meta-iot-cloud \
43     git://git.openembedded.org/meta-python2 \
44     https://git.yoctoproject.org/git/meta-dpdk \
45     git://git.yoctoproject.org/meta-anaconda \
46 "
47
48 SUB_LAYER_META_OE="\
49     meta-oe \
50     meta-perl \
51     meta-python \
52     meta-networking \
53     meta-filesystems \
54     meta-webserver \
55     meta-initramfs \
56     meta-initramfs \
57     meta-gnome \
58 "
59
60 SUB_LAYER_META_CLOUD_SERVICES="meta-openstack"
61 SUB_LAYER_META_SECURITY="meta-security-compliance"
62
63 # For anaconda build
64 SUB_LAYER_META_OE_ANACONDA="\
65     meta-oe \
66     meta-python \
67     meta-filesystems \
68     meta-initramfs \
69     meta-networking \
70     meta-gnome \
71 "
72
73 SCRIPTS_DIR=$(dirname $(readlink -f $0))
74 TIMESTAMP=`date +"%Y%m%d_%H%M%S"`
75
76 #########################################################################
77 # Common Functions
78 #########################################################################
79
80 help_info () {
81 cat << ENDHELP
82 Usage:
83 $(basename $0) [-w WORKSPACE_DIR] [-b BSP] [-n] [-h] [-r Yes|No] [-s] [-e EXTRA_CONF]
84 where:
85     -w WORKSPACE_DIR is the path for the project
86     -b BPS is one of supported BSP: "${SUPPORTED_BSP}"
87        (default is intel-corei7-64 if not specified.)
88     -n dry-run only for bitbake
89     -h this help info
90     -e EXTRA_CONF is the pat for extra config file
91     -r whether to inherit rm_work (default is Yes)
92     -s whether to skip update the repo if already exists
93 examples:
94 ./build_stx.sh
95 ./build_stx.sh -w workspace_1234 -r no -e /path/to/extra_local.conf
96 ENDHELP
97 }
98
99 echo_step_start() {
100     [ -n "$1" ] && msg_step=$1
101     echo "#########################################################################################"
102     echo "## STEP START: ${msg_step}"
103     echo "#########################################################################################"
104 }
105
106 echo_step_end() {
107     [ -n "$1" ] && msg_step=$1
108     echo "#########################################################################################"
109     echo "## STEP END: ${msg_step}"
110     echo "#########################################################################################"
111     echo
112 }
113
114 echo_info () {
115     echo "INFO: $1"
116 }
117
118 echo_error () {
119     echo "ERROR: $1"
120 }
121
122 echo_cmd () {
123     echo
124     echo_info "$1"
125     echo "CMD: ${RUN_CMD}"
126 }
127
128 check_yn_rm_work () {
129     yn="$1"
130     case ${yn} in
131         [Yy]|[Yy]es)
132             RM_WORK="Yes"
133             ;;
134         [Nn]|[Nn]o)
135             RM_WORK="No"
136             ;;
137         *)
138             echo "Invalid arg for -r option."
139             help_info
140             exit 1
141             ;;
142     esac
143 }
144
145 check_valid_bsp () {
146     bsp="$1"
147     for b in ${SUPPORTED_BSP}; do
148         if [ "${bsp}" == "${b}" ]; then
149             BSP_VALID="${bsp}"
150             break
151         fi
152     done
153     if [ -z "${BSP_VALID}" ]; then
154         echo_error "${bsp} is not a supported BSP, the supported BSPs are: ${SUPPORTED_BSP}"
155         exit 1
156     fi
157 }
158
159
160 clone_update_repo () {
161     REPO_BRANCH=$1
162     REPO_URL=$2
163     REPO_NAME=$3
164
165     if [ -d ${REPO_NAME}/.git ]; then
166         if [ "${SKIP_UPDATE}" == "Yes" ]; then
167             echo_info "The repo ${REPO_NAME} exists, skip updating for the branch ${REPO_BRANCH}"
168         else
169             echo_info "The repo ${REPO_NAME} exists, updating for the branch ${REPO_BRANCH}"
170             cd ${REPO_NAME}
171             git checkout ${REPO_BRANCH}
172             git pull
173             cd -
174         fi
175     else
176         RUN_CMD="git clone --branch ${REPO_BRANCH} ${REPO_URL} ${REPO_NAME}"
177         echo_cmd "Cloning the source of repo '${REPO_NAME}':"
178         ${RUN_CMD}
179     fi
180 }
181
182 source_env () {
183     build_dir=$1
184     cd ${SRC_LAYER_DIR}/poky
185     set ${build_dir}
186     source ./oe-init-build-env ${build_dir}
187 }
188
189 #########################################################################
190 # Parse cmd options
191 #########################################################################
192
193 DRYRUN=""
194 EXTRA_CONF=""
195 SKIP_UPDATE="No"
196 RM_WORK="Yes"
197 BSP="intel-corei7-64"
198
199 while getopts "w:b:e:r:nsh" OPTION; do
200     case ${OPTION} in
201         w)
202             WORKSPACE=`readlink -f ${OPTARG}`
203             ;;
204         b)
205             check_valid_bsp ${OPTARG}
206             ;;
207         e)
208             EXTRA_CONF=`readlink -f ${OPTARG}`
209             ;;
210         n)
211             DRYRUN="-n"
212             ;;
213         s)
214             SKIP_UPDATE="Yes"
215             ;;
216         r)
217             check_yn_rm_work ${OPTARG}
218             ;;
219         h)
220             help_info
221             exit
222             ;;
223     esac
224 done
225
226 if [ -z ${WORKSPACE} ]; then
227     echo_info "No workspace specified, a directory 'workspace' will be created in current directory as the workspace"
228     WORKSPACE=`readlink -f workspace`
229 fi
230
231 if [ -n "${BSP_VALID}" ]; then
232     BSP="${BSP_VALID}"
233 fi
234
235 #########################################################################
236 # Functions for each step
237 #########################################################################
238 SRC_LAYER_DIR=${WORKSPACE}/src_layers
239 SRC_ORAN_DIR=${SRC_LAYER_DIR}/oran
240 PRJ_BUILD_DIR=${WORKSPACE}/prj_oran_stx
241 PRJ_BUILD_DIR_ANACONDA=${WORKSPACE}/prj_oran_stx_anaconda
242 PRJ_SHARED_DIR=${WORKSPACE}/prj_shared
243 PRJ_SHARED_DL_DIR=${WORKSPACE}/prj_shared/downloads
244 PRJ_SHARED_SS_DIR=${WORKSPACE}/prj_shared/sstate-cache
245 SRC_META_PATCHES=${SRC_ORAN_DIR}/rtp/scripts/meta-patches/src_stx
246 SRC_CONFIGS=${SRC_ORAN_DIR}/rtp/scripts/configs
247 IMG_STX=stx-image-aio
248 IMG_ANACONDA=stx-image-aio-installer
249 ISO_STX=${PRJ_BUILD_DIR}/tmp/deploy/images/${BSP}/${IMG_STX}-${BSP}.iso
250 ISO_ANACONDA=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_ANACONDA}-${BSP}.iso
251
252 prepare_workspace () {
253     msg_step="Create workspace for the build"
254     echo_step_start
255
256     mkdir -p ${PRJ_BUILD_DIR} ${SRC_ORAN_DIR} ${PRJ_BUILD_DIR_ANACONDA} ${PRJ_SHARED_DL_DIR} ${PRJ_SHARED_SS_DIR}
257
258     echo_info "The following directories are created in your workspace(${WORKSPACE}):"
259     echo_info "For all layers source: ${SRC_LAYER_DIR}"
260     echo_info "For StarlingX build project: ${PRJ_BUILD_DIR}"
261     echo_info "For anaconda (installer) build project: ${PRJ_BUILD_DIR_ANACONDA}"
262
263     echo_step_end
264 }
265
266 prepare_src () {
267     msg_step="Get the source code repos"
268     echo_step_start
269
270     # Clone the oran layer if it's not already cloned
271     # Check if the script is inside the repo
272     if cd ${SCRIPTS_DIR} && git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
273         CLONED_ORAN_REPO=`dirname ${SCRIPTS_DIR}`
274         echo_info "Use the cloned oran repo: ${CLONED_ORAN_REPO}"
275         mkdir -p ${SRC_ORAN_DIR}/rtp
276         cd ${SRC_ORAN_DIR}/rtp
277         rm -rf meta-oran meta-stx scripts
278         ln -sf ${CLONED_ORAN_REPO}/meta-oran meta-oran
279         ln -sf ${CLONED_ORAN_REPO}/meta-stx meta-stx
280         ln -sf ${CLONED_ORAN_REPO}/scripts scripts
281     else
282         echo_info "Cloning oran layer:"
283         cd ${SRC_ORAN_DIR}
284         clone_update_repo ${SRC_ORAN_BRANCH} ${SRC_ORAN_URL} rtp
285     fi
286
287     echo_info "Cloning or update Yocto layers:"
288
289     cd ${SRC_LAYER_DIR}
290     for layer_url in ${SRC_YP_URL}; do
291         layer_name=$(basename ${layer_url})
292         clone_update_repo ${SRC_YP_BRANCH} ${layer_url} ${layer_name}
293     done
294
295     echo_step_end
296
297     # Apply meta patches
298     for l in $(ls -1 ${SRC_META_PATCHES}); do
299         msg_step="Apply meta patches for ${l}"
300         echo_step_start
301         cd ${SRC_LAYER_DIR}/${l}
302
303         # backup current branch
304         local_branch=$(git rev-parse --abbrev-ref HEAD)
305         git branch -m "${local_branch}_${TIMESTAMP}"
306         git checkout ${local_branch}
307
308         for p in $(ls -1 ${SRC_META_PATCHES}/${l}); do
309             echo_info "Apllying patch: ${SRC_META_PATCHES}/${l}/${p}"
310             git am ${SRC_META_PATCHES}/${l}/${p}
311         done
312         echo_step_end
313     done
314 }
315
316 add_layer_stx_build () {
317     msg_step="Add required layers to the StarlingX build project"
318     echo_step_start
319
320     source_env ${PRJ_BUILD_DIR}
321     SRC_LAYERS=""
322     for layer_url in ${SRC_YP_URL}; do
323         layer_name=$(basename ${layer_url})
324         case ${layer_name} in
325         poky)
326             continue
327             ;;
328         meta-openembedded)
329             for sub_layer in ${SUB_LAYER_META_OE}; do
330                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
331             done
332             ;;
333         meta-cloud-services)
334             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
335             for sub_layer in ${SUB_LAYER_META_CLOUD_SERVICES}; do
336                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
337             done
338             ;;
339         meta-security)
340             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
341             for sub_layer in ${SUB_LAYER_META_SECURITY}; do
342                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
343             done
344             ;;
345         *)
346             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
347             ;;
348
349         esac
350     done
351
352     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
353
354     for src_layer in ${SRC_LAYERS}; do
355         RUN_CMD="bitbake-layers add-layer ${src_layer}"
356         echo_cmd "Add the ${src_layer} layer into the build project"
357         ${RUN_CMD}
358     done
359
360     echo_step_end
361 }
362
363 add_configs_stx_build () {
364     msg_step="Add extra configs into local.conf for StarlingX build"
365     echo_step_start
366
367     cd ${PRJ_BUILD_DIR}
368     echo_info "Adding the following extra configs into local.conf"
369     cat ${SRC_CONFIGS}/local-stx.conf | \
370         sed -e "s/@BSP@/${BSP}/" | tee -a conf/local.conf
371
372     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
373     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
374
375     if [ "${RM_WORK}" == "Yes" ]; then
376         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
377     fi
378
379
380     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
381         cat ${EXTRA_CONF} | tee -a conf/local.conf
382     fi
383     echo_step_end
384 }
385
386 setup_stx_build () {
387     echo_step_start "Setup StarlingX build project"
388
389     add_layer_stx_build
390
391     cd ${PRJ_BUILD_DIR}
392     if ! grep -q 'Configs for StarlingX' conf/local.conf; then
393         add_configs_stx_build
394     else
395         echo_info "Nothing is added into local.conf"
396     fi
397
398     echo_step_end "Setup StarlingX build project"
399 }
400
401 build_stx_image () {
402     msg_step="Build StarlingX images"
403     echo_step_start
404
405     source_env ${PRJ_BUILD_DIR}
406
407     mkdir -p logs
408
409     RUN_CMD="bitbake ${DRYRUN} ${IMG_STX}"
410     echo_cmd "Build the ${IMG_STX} image"
411     bitbake ${DRYRUN} ${IMG_STX} 2>&1|tee logs/bitbake_${IMG_STX}_${TIMESTAMP}.log
412
413     echo_step_end
414
415     echo_info "Build succeeded, you can get the image in ${ISO_STX}"
416 }
417
418 add_layer_anaconda_build () {
419     msg_step="Add required layers to the anaconda (installer) build project"
420     echo_step_start
421
422     source_env ${PRJ_BUILD_DIR_ANACONDA}
423     SRC_LAYERS=""
424     for sub_layer in ${SUB_LAYER_META_OE_ANACONDA}; do
425         SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-openembedded/${sub_layer}"
426     done
427     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-intel"
428     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-anaconda"
429     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
430
431     for src_layer in ${SRC_LAYERS}; do
432         RUN_CMD="bitbake-layers add-layer ${src_layer}"
433         echo_cmd "Add the ${src_layer} layer into the build project"
434         ${RUN_CMD}
435     done
436
437     echo_step_end
438 }
439
440 add_configs_anaconda_build () {
441     msg_step="Add extra configs into local.conf for anaconda (installer) build"
442     echo_step_start
443
444     cd ${PRJ_BUILD_DIR_ANACONDA}
445     echo_info "Adding the following extra configs into local.conf"
446     cat ${SRC_CONFIGS}/local-anaconda.conf | \
447         sed -e "s/@BSP@/${BSP}/" \
448             -e "s|@TARGET_BUILD@|${PRJ_BUILD_DIR}|" \
449             | tee -a conf/local.conf
450
451     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
452     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
453
454     if [ "${RM_WORK}" == "Yes" ]; then
455         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
456     fi
457
458     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
459         cat ${EXTRA_CONF} | tee -a conf/local.conf
460     fi
461
462     echo_step_end
463 }
464
465 setup_anaconda_build () {
466     echo_step_start "Setup anaconda (installer) build project"
467
468     add_layer_anaconda_build
469
470     cd ${PRJ_BUILD_DIR_ANACONDA}
471     if ! grep -q 'Configs for anaconda' conf/local.conf; then
472         add_configs_anaconda_build
473     else
474         echo_info "Nothing is added into local.conf"
475     fi
476
477     echo_step_end "Setup anaconda build project"
478 }
479
480 build_anaconda_image () {
481     echo_step_start "Build anaconda (installer) images"
482     source_env ${PRJ_BUILD_DIR_ANACONDA}
483
484     mkdir -p logs
485
486     if [ -f ${ISO_ANACONDA} ]; then
487         bitbake ${DRYRUN} -c clean ${IMG_ANACONDA}
488     fi
489     RUN_CMD="bitbake ${DRYRUN} ${IMG_ANACONDA}"
490     echo_cmd "Build the ${IMG_ANACONDA} image"
491     bitbake ${DRYRUN} ${IMG_ANACONDA} 2>&1|tee logs/bitbake_${IMG_ANACONDA}_${TIMESTAMP}.log
492
493     echo_step_end
494
495     echo_info "Build succeeded, you can get the image in ${ISO_ANACONDA}"
496 }
497
498 #########################################################################
499 # Main process
500 #########################################################################
501
502 prepare_workspace
503 prepare_src
504 setup_stx_build
505 setup_anaconda_build
506 build_stx_image
507 build_anaconda_image