74c6d0cd13fb61d3dd15572239d3910f89979e0d
[pti/rtp.git] / scripts / build_inf_yocto / build_inf_yocto.sh
1 #!/bin/bash
2 #
3 # Copyright (C) 2019 Wind River Systems, Inc.
4 #
5 #  Licensed under the Apache License, Version 2.0 (the "License");
6 #  you may not use this file except in compliance with the License.
7 #  You may obtain a copy of the License at
8 #
9 #      http://www.apache.org/licenses/LICENSE-2.0
10 #
11 #  Unless required by applicable law or agreed to in writing, software
12 #  distributed under the License is distributed on an "AS IS" BASIS,
13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 #  See the License for the specific language governing permissions and
15 #  limitations under the License.
16
17 # Ensure we fail the job if any steps fail.
18 set -e -o pipefail
19
20 #########################################################################
21 # Variables
22 #########################################################################
23
24 # Only one bsp is supported now, there will be more in the future
25 SUPPORTED_BSP="intel-corei7-64 "
26
27 SRC_ORAN_BRANCH="master"
28 SRC_STX_BRANCH="master"
29 SRC_YP_BRANCH="warrior"
30
31 SRC_ORAN_URL="https://gerrit.o-ran-sc.org/r/pti/rtp"
32
33 SRC_STX_URL="https://opendev.org/starlingx/meta-starlingx"
34
35 SRC_YP_URL="\
36     git://git.yoctoproject.org/poky;commit=HEAD \
37     git://git.openembedded.org/meta-openembedded;commit=HEAD \
38     git://git.openembedded.org/meta-python2;commit=HEAD \
39     git://git.yoctoproject.org/meta-virtualization;commit=HEAD \
40     git://git.yoctoproject.org/meta-cloud-services;commit=HEAD \
41     git://git.yoctoproject.org/meta-security;commit=HEAD \
42     git://git.yoctoproject.org/meta-intel;commit=HEAD \
43     git://git.yoctoproject.org/meta-selinux;commit=HEAD \
44     git://git.yoctoproject.org/meta-dpdk;commit=HEAD \
45     git://git.yoctoproject.org/meta-anaconda;commit=HEAD \
46     https://github.com/intel-iot-devkit/meta-iot-cloud.git;commit=HEAD \
47 "
48
49 SUB_LAYER_META_OE="\
50     meta-oe \
51     meta-perl \
52     meta-python \
53     meta-networking \
54     meta-filesystems \
55     meta-webserver \
56     meta-initramfs \
57     meta-initramfs \
58     meta-gnome \
59 "
60
61 SUB_LAYER_META_CLOUD_SERVICES="meta-openstack"
62 SUB_LAYER_STX="\
63     meta-stx-cloud \
64     meta-stx-distro \
65     meta-stx-flock \
66     meta-stx-integ \
67     meta-stx-virt \
68 "
69
70 # For anaconda build
71 SUB_LAYER_META_OE_ANACONDA="\
72     meta-oe \
73     meta-python \
74     meta-filesystems \
75     meta-initramfs \
76     meta-networking \
77     meta-gnome \
78 "
79
80 SCRIPTS_DIR=$(dirname $(dirname $(readlink -f $0)))
81 SCRIPTS_NAME=$(basename $0)
82 TIMESTAMP=`date +"%Y%m%d_%H%M%S"`
83
84 #########################################################################
85 # Common Functions
86 #########################################################################
87
88 help_info () {
89 cat << ENDHELP
90 Usage:
91 ${SCRIPTS_NAME} [-w WORKSPACE_DIR] [-b BSP] [-n] [-u] [-h] [-r Yes|No] [-e EXTRA_CONF]
92 where:
93     -w WORKSPACE_DIR is the path for the project
94     -b BPS is one of supported BSP: "${SUPPORTED_BSP}"
95        (default is intel-corei7-64 if not specified.)
96     -n dry-run only for bitbake
97     -u update the repo if it exists
98     -h this help info
99     -e EXTRA_CONF is the pat for extra config file
100     -r whether to inherit rm_work (default is Yes)
101 examples:
102 $0
103 $0 -w workspace_1234 -r no -e /path/to/extra_local.conf
104 ENDHELP
105 }
106
107 echo_step_start() {
108     [ -n "$1" ] && msg_step=$1
109     echo "#########################################################################################"
110     echo "## ${SCRIPTS_NAME} - STEP START: ${msg_step}"
111     echo "#########################################################################################"
112 }
113
114 echo_step_end() {
115     [ -n "$1" ] && msg_step=$1
116     echo "#########################################################################################"
117     echo "## ${SCRIPTS_NAME} - STEP END: ${msg_step}"
118     echo "#########################################################################################"
119     echo
120 }
121
122 echo_info () {
123     echo "INFO: $1"
124 }
125
126 echo_error () {
127     echo "ERROR: $1"
128 }
129
130 echo_cmd () {
131     echo
132     echo_info "$1"
133     echo "CMD: ${RUN_CMD}"
134 }
135
136 check_yn_rm_work () {
137     yn="$1"
138     case ${yn} in
139         [Yy]|[Yy]es)
140             RM_WORK="Yes"
141             ;;
142         [Nn]|[Nn]o)
143             RM_WORK="No"
144             ;;
145         *)
146             echo "Invalid arg for -r option."
147             help_info
148             exit 1
149             ;;
150     esac
151 }
152
153 check_valid_bsp () {
154     bsp="$1"
155     for b in ${SUPPORTED_BSP}; do
156         if [ "${bsp}" == "${b}" ]; then
157             BSP_VALID="${bsp}"
158             break
159         fi
160     done
161     if [ -z "${BSP_VALID}" ]; then
162         echo_error "${bsp} is not a supported BSP, the supported BSPs are: ${SUPPORTED_BSP}"
163         exit 1
164     fi
165 }
166
167
168 clone_update_repo () {
169     REPO_BRANCH=$1
170     REPO_URL=$2
171     REPO_NAME=$3
172     REPO_COMMIT=$4
173
174     if [ -d ${REPO_NAME}/.git ]; then
175         if [ "${SKIP_UPDATE}" == "Yes" ]; then
176             echo_info "The repo ${REPO_NAME} exists, skip updating for the branch ${REPO_BRANCH}"
177         else
178             echo_info "The repo ${REPO_NAME} exists, updating for the branch ${REPO_BRANCH}"
179             cd ${REPO_NAME}
180             git checkout ${REPO_BRANCH}
181             git pull
182             cd -
183         fi
184     else
185         RUN_CMD="git clone --branch ${REPO_BRANCH} ${REPO_URL} ${REPO_NAME}"
186         echo_cmd "Cloning the source of repo '${REPO_NAME}':"
187         ${RUN_CMD}
188
189         if [ -n "${REPO_COMMIT}" ]; then
190             cd ${REPO_NAME}
191             RUN_CMD="git checkout -b ${REPO_BRANCH}-${REPO_COMMIT} ${REPO_COMMIT}"
192             echo_cmd "Checkout the repo ${REPO_NAME} to specific commit: ${REPO_COMMIT}"
193             ${RUN_CMD}
194             cd -
195         fi
196     fi
197 }
198
199 source_env () {
200     build_dir=$1
201     cd ${SRC_LAYER_DIR}/poky
202     set ${build_dir}
203     source ./oe-init-build-env ${build_dir}
204 }
205
206 #########################################################################
207 # Parse cmd options
208 #########################################################################
209
210 DRYRUN=""
211 EXTRA_CONF=""
212 SKIP_UPDATE="Yes"
213 RM_WORK="Yes"
214 GET_SSTATE="No"
215 BSP="intel-corei7-64"
216
217 while getopts "w:b:e:r:unsh" OPTION; do
218     case ${OPTION} in
219         w)
220             WORKSPACE=`readlink -f ${OPTARG}`
221             ;;
222         b)
223             check_valid_bsp ${OPTARG}
224             ;;
225         e)
226             EXTRA_CONF=`readlink -f ${OPTARG}`
227             ;;
228         n)
229             DRYRUN="-n"
230             ;;
231         u)
232             SKIP_UPDATE="No"
233             ;;
234         r)
235             check_yn_rm_work ${OPTARG}
236             ;;
237         s)
238             GET_SSTATE="Yes"
239             ;;
240         h)
241             help_info
242             exit
243             ;;
244     esac
245 done
246
247 if [ -z ${WORKSPACE} ]; then
248     echo_info "No workspace specified, a directory 'workspace' will be created in current directory as the workspace"
249     WORKSPACE=`readlink -f workspace`
250 fi
251
252 if [ -n "${BSP_VALID}" ]; then
253     BSP="${BSP_VALID}"
254 fi
255
256 IMG_ARCH=${BSP}
257 if [ "${BSP}" == "intel-corei7-64"  ]; then
258     IMG_ARCH="x86-64"
259 fi
260
261 #########################################################################
262 # Functions for each step
263 #########################################################################
264 SRC_LAYER_DIR=${WORKSPACE}/src_layers
265 SRC_ORAN_DIR=${SRC_LAYER_DIR}/oran
266 SRC_STX_DIR=${SRC_LAYER_DIR}/meta-starlingx
267 PRJ_BUILD_DIR=${WORKSPACE}/prj_oran_stx
268 PRJ_BUILD_DIR_ANACONDA=${WORKSPACE}/prj_oran_inf_anaconda
269 PRJ_SHARED_DIR=${WORKSPACE}/prj_shared
270 PRJ_OUTPUT_DIR=${WORKSPACE}/prj_output
271 PRJ_SHARED_DL_DIR=${WORKSPACE}/prj_shared/downloads
272 PRJ_SHARED_SS_DIR=${WORKSPACE}/prj_shared/sstate-cache
273 SRC_SCRIPTS=${SRC_ORAN_DIR}/rtp/scripts/build_inf_yocto
274 SRC_META_PATCHES=${SRC_SCRIPTS}/meta-patches/src_stx
275 SRC_CONFIGS=${SRC_SCRIPTS}/configs
276 IMG_STX=stx-image-aio
277 IMG_ANACONDA=stx-image-aio-installer
278 IMG_INF=inf-image-aio-installer
279 ISO_STX=${PRJ_BUILD_DIR}/tmp/deploy/images/${BSP}/${IMG_STX}-${BSP}.iso
280 ISO_ANACONDA=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_ANACONDA}-${BSP}.iso
281 ISO_INF=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_INF}-${BSP}.iso
282 ISO_INF_ALIAS=${PRJ_OUTPUT_DIR}/inf-image-yocto-aio-${IMG_ARCH}.iso
283
284 SSTATE_CONTAINER_IMG=infbuilder/inf-yocto-sstate:2022.05
285
286 prepare_workspace () {
287     msg_step="Create workspace for the Yocto build"
288     echo_step_start
289
290     mkdir -p ${PRJ_BUILD_DIR} ${SRC_ORAN_DIR} ${PRJ_BUILD_DIR_ANACONDA} \
291              ${PRJ_SHARED_DL_DIR} ${PRJ_SHARED_SS_DIR} ${PRJ_OUTPUT_DIR}
292
293     echo_info "The following directories are created in your workspace(${WORKSPACE}):"
294     echo_info "For all layers source: ${SRC_LAYER_DIR}"
295     echo_info "For StarlingX build project: ${PRJ_BUILD_DIR}"
296     echo_info "For anaconda (installer) build project: ${PRJ_BUILD_DIR_ANACONDA}"
297
298     echo_step_end
299 }
300
301 # This is tend to be used for CI Jenkins build only
302 get_sstate () {
303     msg_step="Get sstate cache from dockerhub image"
304     echo_step_start
305
306     for i in {1..5}; do
307         docker pull ${SSTATE_CONTAINER_IMG}-${i}
308         docker create -ti --name inf-yocto-sstate-${i} ${SSTATE_CONTAINER_IMG}-${i} sh
309         docker cp inf-yocto-sstate-${i}:/sstate${i} ${PRJ_SHARED_SS_DIR}/sstate${i}
310         docker rm inf-yocto-sstate-${i}
311     done
312     mv ${PRJ_SHARED_SS_DIR}/sstate*/* ${PRJ_SHARED_SS_DIR}
313     #rm -rf ${PRJ_SHARED_SS_DIR}/sstate*
314
315     echo_step_end
316 }
317
318 prepare_src () {
319     msg_step="Get the source code repos"
320     echo_step_start
321
322     # Clone the oran layer if it's not already cloned
323     # Check if the script is inside the repo
324     if cd ${SCRIPTS_DIR} && git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
325         CLONED_ORAN_REPO=`dirname ${SCRIPTS_DIR}`
326         echo_info "Use the cloned oran repo: ${CLONED_ORAN_REPO}"
327         mkdir -p ${SRC_ORAN_DIR}/rtp
328         cd ${SRC_ORAN_DIR}/rtp
329         rm -rf meta-stx-oran scripts
330         ln -sf ${CLONED_ORAN_REPO}/scripts scripts
331         ln -sf ${CLONED_ORAN_REPO}/meta-stx-oran meta-stx-oran
332     else
333         echo_info "Cloning oran layer:"
334         cd ${SRC_ORAN_DIR}
335         clone_update_repo ${SRC_ORAN_BRANCH} ${SRC_ORAN_URL} rtp
336     fi
337
338     echo_info "Cloning or update Yocto layers:"
339
340     cd ${SRC_LAYER_DIR}
341     for layer_url in ${SRC_YP_URL}; do
342         layer_name=$(basename ${layer_url%%;commit*})
343         layer_commit=$(basename ${layer_url##*;commit=})
344         clone_update_repo ${SRC_YP_BRANCH} ${layer_url%%;commit*} ${layer_name} ${layer_commit}
345     done
346
347     echo_info "Cloning or update meta-starlingx layers:"
348     clone_update_repo ${SRC_STX_BRANCH} ${SRC_STX_URL} $(basename ${SRC_STX_URL})
349
350     echo_step_end
351
352 # Not any meta-patch is needed for the time being, but new ones may be needed and added
353 # sometime in the future, so just leave these code commented out here.
354 #    # Apply meta patches
355 #    for l in $(ls -1 ${SRC_META_PATCHES}); do
356 #        msg_step="Apply meta patches for ${l}"
357 #        echo_step_start
358 #        cd ${SRC_LAYER_DIR}/${l}
359 #
360 #        # backup current branch
361 #        local_branch=$(git rev-parse --abbrev-ref HEAD)
362 #        git branch -m "${local_branch}_${TIMESTAMP}"
363 #        git checkout -b ${local_branch} ${local_branch##*-}
364 #
365 #        for p in $(ls -1 ${SRC_META_PATCHES}/${l}); do
366 #            echo_info "Apllying patch: ${SRC_META_PATCHES}/${l}/${p}"
367 #            git am ${SRC_META_PATCHES}/${l}/${p}
368 #        done
369 #        echo_step_end
370 #    done
371 }
372
373 add_layer_stx_build () {
374     msg_step="Add required layers to the StarlingX build project"
375     echo_step_start
376
377     source_env ${PRJ_BUILD_DIR}
378     SRC_LAYERS=""
379     for layer_url in ${SRC_YP_URL} ${SRC_STX_URL}; do
380         layer_name=$(basename ${layer_url%%;commit*})
381         case ${layer_name} in
382         poky)
383             continue
384             ;;
385         meta-openembedded)
386             for sub_layer in ${SUB_LAYER_META_OE}; do
387                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
388             done
389             ;;
390         meta-cloud-services)
391             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
392             for sub_layer in ${SUB_LAYER_META_CLOUD_SERVICES}; do
393                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
394             done
395             ;;
396         meta-starlingx)
397             for sub_layer in ${SUB_LAYER_STX}; do
398                 SRC_LAYERS="${SRC_LAYERS} ${SRC_STX_DIR}/${sub_layer}"
399             done
400             ;;
401         *)
402             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
403             ;;
404
405         esac
406     done
407     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx-oran"
408
409
410     for src_layer in ${SRC_LAYERS}; do
411         RUN_CMD="bitbake-layers add-layer ${src_layer}"
412         echo_cmd "Add the ${src_layer} layer into the build project"
413         ${RUN_CMD}
414     done
415
416     echo_step_end
417 }
418
419 add_configs_stx_build () {
420     msg_step="Add extra configs into local.conf for StarlingX build"
421     echo_step_start
422
423     cd ${PRJ_BUILD_DIR}
424     echo_info "Adding the following extra configs into local.conf"
425     cat ${SRC_CONFIGS}/local-stx.conf | \
426         sed -e "s/@BSP@/${BSP}/" | tee -a conf/local.conf
427     cat ${SRC_CONFIGS}/local-mirrors.conf | tee -a conf/local.conf
428
429     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
430     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
431
432     if [ "${RM_WORK}" == "Yes" ]; then
433         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
434         echo "RM_WORK_EXCLUDE += '${IMG_STX}'" | tee -a conf/local.conf
435     fi
436
437
438     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
439         cat ${EXTRA_CONF} | tee -a conf/local.conf
440     fi
441     echo_step_end
442 }
443
444 setup_stx_build () {
445     echo_step_start "Setup StarlingX build project"
446
447     add_layer_stx_build
448
449     cd ${PRJ_BUILD_DIR}
450     if ! grep -q 'Configs for StarlingX' conf/local.conf; then
451         add_configs_stx_build
452     else
453         echo_info "Nothing is added into local.conf"
454     fi
455
456     echo_step_end "Setup StarlingX build project"
457 }
458
459 build_stx_image () {
460     msg_step="Build StarlingX images"
461     echo_step_start
462
463     source_env ${PRJ_BUILD_DIR}
464
465     mkdir -p logs
466
467     RUN_CMD="bitbake ${DRYRUN} ${IMG_STX}"
468     echo_cmd "Build the ${IMG_STX} image"
469     bitbake -k ${DRYRUN} ${IMG_STX} 2>&1|tee logs/bitbake_${IMG_STX}_${TIMESTAMP}.log
470
471     echo_step_end
472
473     echo_info "Build succeeded, you can get the image in ${ISO_STX}"
474 }
475
476 add_layer_anaconda_build () {
477     msg_step="Add required layers to the anaconda (installer) build project"
478     echo_step_start
479
480     source_env ${PRJ_BUILD_DIR_ANACONDA}
481     SRC_LAYERS=""
482     for sub_layer in ${SUB_LAYER_META_OE_ANACONDA}; do
483         SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-openembedded/${sub_layer}"
484     done
485     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-intel"
486     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-anaconda"
487     SRC_LAYERS="${SRC_LAYERS} ${SRC_STX_DIR}/meta-stx-distro"
488     SRC_LAYERS="${SRC_LAYERS} ${SRC_STX_DIR}/meta-stx-integ"
489     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx-oran"
490
491     for src_layer in ${SRC_LAYERS}; do
492         RUN_CMD="bitbake-layers add-layer ${src_layer}"
493         echo_cmd "Add the ${src_layer} layer into the build project"
494         ${RUN_CMD}
495     done
496
497     echo_step_end
498 }
499
500 add_configs_anaconda_build () {
501     msg_step="Add extra configs into local.conf for anaconda (installer) build"
502     echo_step_start
503
504     cd ${PRJ_BUILD_DIR_ANACONDA}
505     echo_info "Adding the following extra configs into local.conf"
506     cat ${SRC_CONFIGS}/local-anaconda.conf | \
507         sed -e "s/@BSP@/${BSP}/" \
508             -e "s|@TARGET_BUILD@|${PRJ_BUILD_DIR}|" \
509             | tee -a conf/local.conf
510     cat ${SRC_CONFIGS}/local-mirrors.conf | tee -a conf/local.conf
511
512     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
513     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
514
515     if [ "${RM_WORK}" == "Yes" ]; then
516         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
517         echo "RM_WORK_EXCLUDE += '${IMG_ANACONDA}'" | tee -a conf/local.conf
518     fi
519
520     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
521         cat ${EXTRA_CONF} | tee -a conf/local.conf
522     fi
523
524     echo_step_end
525 }
526
527 setup_anaconda_build () {
528     echo_step_start "Setup anaconda (installer) build project"
529
530     add_layer_anaconda_build
531
532     cd ${PRJ_BUILD_DIR_ANACONDA}
533     if ! grep -q 'Configs for anaconda' conf/local.conf; then
534         add_configs_anaconda_build
535     else
536         echo_info "Nothing is added into local.conf"
537     fi
538
539     echo_step_end "Setup anaconda build project"
540 }
541
542 build_anaconda_image () {
543     echo_step_start "Build anaconda (installer) images"
544     source_env ${PRJ_BUILD_DIR_ANACONDA}
545
546     mkdir -p logs
547
548     if [ -f ${ISO_ANACONDA} ]; then
549         bitbake ${DRYRUN} -c clean ${IMG_ANACONDA}
550     fi
551     RUN_CMD="bitbake ${DRYRUN} ${IMG_ANACONDA}"
552     echo_cmd "Build the ${IMG_ANACONDA} image"
553     bitbake -k ${DRYRUN} ${IMG_ANACONDA} 2>&1|tee logs/bitbake_${IMG_ANACONDA}_${TIMESTAMP}.log
554
555     if [ -z "${DRYRUN}" ]; then
556         cp -Pf ${ISO_ANACONDA} ${ISO_INF}
557         cp -f ${ISO_ANACONDA} ${ISO_INF_ALIAS}
558     fi
559
560     echo_step_end
561
562     echo_info "Build succeeded, you can get the image in ${ISO_INF_ALIAS}"
563 }
564
565 #########################################################################
566 # Main process
567 #########################################################################
568
569 prepare_workspace
570 if [ "${GET_SSTATE}" == "Yes" ]; then
571     get_sstate
572 fi
573 prepare_src
574 setup_stx_build
575 setup_anaconda_build
576 build_stx_image
577 build_anaconda_image