scripts/build_inf_yocto/build_inf_yocto.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2019 Wind River Systems, Inc.
   4 #
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16
  17 # Ensure we fail the job if any steps fail.
  18 set -e -o pipefail
  19
  20 #########################################################################
  21 # Variables
  22 #########################################################################
  23
  24 # Only one bsp is supported now, there will be more in the future
  25 SUPPORTED_BSP="intel-corei7-64 "
  26
  27 SRC_ORAN_BRANCH="master"
  28 SRC_STX_BRANCH="master"
  29 SRC_YP_BRANCH="warrior"
  30
  31 SRC_ORAN_URL="https://gerrit.o-ran-sc.org/r/pti/rtp"
  32
  33 SRC_STX_URL="https://opendev.org/starlingx/meta-starlingx"
  34
  35 SRC_YP_URL="\
  36     git://git.yoctoproject.org/poky;commit=HEAD \
  37     git://git.openembedded.org/meta-openembedded;commit=HEAD \
  38     git://git.openembedded.org/meta-python2;commit=HEAD \
  39     git://git.yoctoproject.org/meta-virtualization;commit=HEAD \
  40     git://git.yoctoproject.org/meta-cloud-services;commit=HEAD \
  41     git://git.yoctoproject.org/meta-security;commit=HEAD \
  42     git://git.yoctoproject.org/meta-intel;commit=HEAD \
  43     git://git.yoctoproject.org/meta-selinux;commit=HEAD \
  44     git://git.yoctoproject.org/meta-dpdk;commit=HEAD \
  45     git://git.yoctoproject.org/meta-anaconda;commit=HEAD \
  46     https://github.com/intel-iot-devkit/meta-iot-cloud.git;commit=HEAD \
  47 "
  48
  49 SUB_LAYER_META_OE="\
  50     meta-oe \
  51     meta-perl \
  52     meta-python \
  53     meta-networking \
  54     meta-filesystems \
  55     meta-webserver \
  56     meta-initramfs \
  57     meta-initramfs \
  58     meta-gnome \
  59 "
  60
  61 SUB_LAYER_META_CLOUD_SERVICES="meta-openstack"
  62 SUB_LAYER_STX="\
  63     meta-stx-cloud \
  64     meta-stx-distro \
  65     meta-stx-flock \
  66     meta-stx-integ \
  67     meta-stx-virt \
  68 "
  69
  70 # For anaconda build
  71 SUB_LAYER_META_OE_ANACONDA="\
  72     meta-oe \
  73     meta-python \
  74     meta-filesystems \
  75     meta-initramfs \
  76     meta-networking \
  77     meta-gnome \
  78 "
  79
  80 SCRIPTS_DIR=$(dirname $(dirname $(readlink -f $0)))
  81 TIMESTAMP=`date +"%Y%m%d_%H%M%S"`
  82
  83 #########################################################################
  84 # Common Functions
  85 #########################################################################
  86
  87 help_info () {
  88 cat << ENDHELP
  89 Usage:
  90 $(basename $0) [-w WORKSPACE_DIR] [-b BSP] [-n] [-u] [-h] [-r Yes|No] [-e EXTRA_CONF]
  91 where:
  92     -w WORKSPACE_DIR is the path for the project
  93     -b BPS is one of supported BSP: "${SUPPORTED_BSP}"
  94        (default is intel-corei7-64 if not specified.)
  95     -n dry-run only for bitbake
  96     -u update the repo if it exists
  97     -h this help info
  98     -e EXTRA_CONF is the pat for extra config file
  99     -r whether to inherit rm_work (default is Yes)
 100 examples:
 101 $0
 102 $0 -w workspace_1234 -r no -e /path/to/extra_local.conf
 103 ENDHELP
 104 }
 105
 106 echo_step_start() {
 107     [ -n "$1" ] && msg_step=$1
 108     echo "#########################################################################################"
 109     echo "## STEP START: ${msg_step}"
 110     echo "#########################################################################################"
 111 }
 112
 113 echo_step_end() {
 114     [ -n "$1" ] && msg_step=$1
 115     echo "#########################################################################################"
 116     echo "## STEP END: ${msg_step}"
 117     echo "#########################################################################################"
 118     echo
 119 }
 120
 121 echo_info () {
 122     echo "INFO: $1"
 123 }
 124
 125 echo_error () {
 126     echo "ERROR: $1"
 127 }
 128
 129 echo_cmd () {
 130     echo
 131     echo_info "$1"
 132     echo "CMD: ${RUN_CMD}"
 133 }
 134
 135 check_yn_rm_work () {
 136     yn="$1"
 137     case ${yn} in
 138         [Yy]|[Yy]es)
 139             RM_WORK="Yes"
 140             ;;
 141         [Nn]|[Nn]o)
 142             RM_WORK="No"
 143             ;;
 144         *)
 145             echo "Invalid arg for -r option."
 146             help_info
 147             exit 1
 148             ;;
 149     esac
 150 }
 151
 152 check_valid_bsp () {
 153     bsp="$1"
 154     for b in ${SUPPORTED_BSP}; do
 155         if [ "${bsp}" == "${b}" ]; then
 156             BSP_VALID="${bsp}"
 157             break
 158         fi
 159     done
 160     if [ -z "${BSP_VALID}" ]; then
 161         echo_error "${bsp} is not a supported BSP, the supported BSPs are: ${SUPPORTED_BSP}"
 162         exit 1
 163     fi
 164 }
 165
 166
 167 clone_update_repo () {
 168     REPO_BRANCH=$1
 169     REPO_URL=$2
 170     REPO_NAME=$3
 171     REPO_COMMIT=$4
 172
 173     if [ -d ${REPO_NAME}/.git ]; then
 174         if [ "${SKIP_UPDATE}" == "Yes" ]; then
 175             echo_info "The repo ${REPO_NAME} exists, skip updating for the branch ${REPO_BRANCH}"
 176         else
 177             echo_info "The repo ${REPO_NAME} exists, updating for the branch ${REPO_BRANCH}"
 178             cd ${REPO_NAME}
 179             git checkout ${REPO_BRANCH}
 180             git pull
 181             cd -
 182         fi
 183     else
 184         RUN_CMD="git clone --branch ${REPO_BRANCH} ${REPO_URL} ${REPO_NAME}"
 185         echo_cmd "Cloning the source of repo '${REPO_NAME}':"
 186         ${RUN_CMD}
 187
 188         if [ -n "${REPO_COMMIT}" ]; then
 189             cd ${REPO_NAME}
 190             RUN_CMD="git checkout -b ${REPO_BRANCH}-${REPO_COMMIT} ${REPO_COMMIT}"
 191             echo_cmd "Checkout the repo ${REPO_NAME} to specific commit: ${REPO_COMMIT}"
 192             ${RUN_CMD}
 193             cd -
 194         fi
 195     fi
 196 }
 197
 198 source_env () {
 199     build_dir=$1
 200     cd ${SRC_LAYER_DIR}/poky
 201     set ${build_dir}
 202     source ./oe-init-build-env ${build_dir}
 203 }
 204
 205 #########################################################################
 206 # Parse cmd options
 207 #########################################################################
 208
 209 DRYRUN=""
 210 EXTRA_CONF=""
 211 SKIP_UPDATE="Yes"
 212 RM_WORK="Yes"
 213 GET_SSTATE="No"
 214 BSP="intel-corei7-64"
 215
 216 while getopts "w:b:e:r:unsh" OPTION; do
 217     case ${OPTION} in
 218         w)
 219             WORKSPACE=`readlink -f ${OPTARG}`
 220             ;;
 221         b)
 222             check_valid_bsp ${OPTARG}
 223             ;;
 224         e)
 225             EXTRA_CONF=`readlink -f ${OPTARG}`
 226             ;;
 227         n)
 228             DRYRUN="-n"
 229             ;;
 230         u)
 231             SKIP_UPDATE="No"
 232             ;;
 233         r)
 234             check_yn_rm_work ${OPTARG}
 235             ;;
 236         s)
 237             GET_SSTATE="Yes"
 238             ;;
 239         h)
 240             help_info
 241             exit
 242             ;;
 243     esac
 244 done
 245
 246 if [ -z ${WORKSPACE} ]; then
 247     echo_info "No workspace specified, a directory 'workspace' will be created in current directory as the workspace"
 248     WORKSPACE=`readlink -f workspace`
 249 fi
 250
 251 if [ -n "${BSP_VALID}" ]; then
 252     BSP="${BSP_VALID}"
 253 fi
 254
 255 IMG_ARCH=${BSP}
 256 if [ "${BSP}" == "intel-corei7-64"  ]; then
 257     IMG_ARCH="x86-64"
 258 fi
 259
 260 #########################################################################
 261 # Functions for each step
 262 #########################################################################
 263 SRC_LAYER_DIR=${WORKSPACE}/src_layers
 264 SRC_ORAN_DIR=${SRC_LAYER_DIR}/oran
 265 SRC_STX_DIR=${SRC_LAYER_DIR}/meta-starlingx
 266 PRJ_BUILD_DIR=${WORKSPACE}/prj_oran_stx
 267 PRJ_BUILD_DIR_ANACONDA=${WORKSPACE}/prj_oran_inf_anaconda
 268 PRJ_SHARED_DIR=${WORKSPACE}/prj_shared
 269 PRJ_OUTPUT_DIR=${WORKSPACE}/prj_output
 270 PRJ_SHARED_DL_DIR=${WORKSPACE}/prj_shared/downloads
 271 PRJ_SHARED_SS_DIR=${WORKSPACE}/prj_shared/sstate-cache
 272 SRC_SCRIPTS=${SRC_ORAN_DIR}/rtp/scripts/build_inf_yocto
 273 SRC_META_PATCHES=${SRC_SCRIPTS}/meta-patches/src_stx
 274 SRC_CONFIGS=${SRC_SCRIPTS}/configs
 275 IMG_STX=stx-image-aio
 276 IMG_ANACONDA=stx-image-aio-installer
 277 IMG_INF=inf-image-aio-installer
 278 ISO_STX=${PRJ_BUILD_DIR}/tmp/deploy/images/${BSP}/${IMG_STX}-${BSP}.iso
 279 ISO_ANACONDA=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_ANACONDA}-${BSP}.iso
 280 ISO_INF=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_INF}-${BSP}.iso
 281 ISO_INF_ALIAS=${PRJ_OUTPUT_DIR}/inf-image-yocto-aio-${IMG_ARCH}.iso
 282
 283 SSTATE_CONTAINER_IMG=infbuilder/inf-yocto-sstate:2022.05
 284
 285 prepare_workspace () {
 286     msg_step="Create workspace for the build"
 287     echo_step_start
 288
 289     mkdir -p ${PRJ_BUILD_DIR} ${SRC_ORAN_DIR} ${PRJ_BUILD_DIR_ANACONDA} \
 290              ${PRJ_SHARED_DL_DIR} ${PRJ_SHARED_SS_DIR} ${PRJ_OUTPUT_DIR}
 291
 292     echo_info "The following directories are created in your workspace(${WORKSPACE}):"
 293     echo_info "For all layers source: ${SRC_LAYER_DIR}"
 294     echo_info "For StarlingX build project: ${PRJ_BUILD_DIR}"
 295     echo_info "For anaconda (installer) build project: ${PRJ_BUILD_DIR_ANACONDA}"
 296
 297     echo_step_end
 298 }
 299
 300 # This is tend to be used for CI Jenkins build only
 301 get_sstate () {
 302     msg_step="Get sstate cache from dockerhub image"
 303     echo_step_start
 304
 305     for i in {1..5}; do
 306         docker pull ${SSTATE_CONTAINER_IMG}-${i}
 307         docker create -ti --name inf-yocto-sstate-${i} ${SSTATE_CONTAINER_IMG}-${i} sh
 308         docker cp inf-yocto-sstate-${i}:/sstate${i} ${PRJ_SHARED_SS_DIR}/sstate${i}
 309         docker rm inf-yocto-sstate-${i}
 310     done
 311     mv ${PRJ_SHARED_SS_DIR}/sstate*/* ${PRJ_SHARED_SS_DIR}
 312     #rm -rf ${PRJ_SHARED_SS_DIR}/sstate*
 313
 314     echo_step_end
 315 }
 316
 317 prepare_src () {
 318     msg_step="Get the source code repos"
 319     echo_step_start
 320
 321     # Clone the oran layer if it's not already cloned
 322     # Check if the script is inside the repo
 323     if cd ${SCRIPTS_DIR} && git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
 324         CLONED_ORAN_REPO=`dirname ${SCRIPTS_DIR}`
 325         echo_info "Use the cloned oran repo: ${CLONED_ORAN_REPO}"
 326         mkdir -p ${SRC_ORAN_DIR}/rtp
 327         cd ${SRC_ORAN_DIR}/rtp
 328         rm -rf meta-stx-oran scripts
 329         ln -sf ${CLONED_ORAN_REPO}/scripts scripts
 330         ln -sf ${CLONED_ORAN_REPO}/meta-stx-oran meta-stx-oran
 331     else
 332         echo_info "Cloning oran layer:"
 333         cd ${SRC_ORAN_DIR}
 334         clone_update_repo ${SRC_ORAN_BRANCH} ${SRC_ORAN_URL} rtp
 335     fi
 336
 337     echo_info "Cloning or update Yocto layers:"
 338
 339     cd ${SRC_LAYER_DIR}
 340     for layer_url in ${SRC_YP_URL}; do
 341         layer_name=$(basename ${layer_url%%;commit*})
 342         layer_commit=$(basename ${layer_url##*;commit=})
 343         clone_update_repo ${SRC_YP_BRANCH} ${layer_url%%;commit*} ${layer_name} ${layer_commit}
 344     done
 345
 346     echo_info "Cloning or update meta-starlingx layers:"
 347     clone_update_repo ${SRC_STX_BRANCH} ${SRC_STX_URL} $(basename ${SRC_STX_URL})
 348
 349     echo_step_end
 350
 351 # Not andy meta-patch is needed for the time being, but new ones may be needed and added
 352 # sometime in the future, so just leave these code commented out here.
 353 #    # Apply meta patches
 354 #    for l in $(ls -1 ${SRC_META_PATCHES}); do
 355 #        msg_step="Apply meta patches for ${l}"
 356 #        echo_step_start
 357 #        cd ${SRC_LAYER_DIR}/${l}
 358 #
 359 #        # backup current branch
 360 #        local_branch=$(git rev-parse --abbrev-ref HEAD)
 361 #        git branch -m "${local_branch}_${TIMESTAMP}"
 362 #        git checkout -b ${local_branch} ${local_branch##*-}
 363 #
 364 #        for p in $(ls -1 ${SRC_META_PATCHES}/${l}); do
 365 #            echo_info "Apllying patch: ${SRC_META_PATCHES}/${l}/${p}"
 366 #            git am ${SRC_META_PATCHES}/${l}/${p}
 367 #        done
 368 #        echo_step_end
 369 #    done
 370 }
 371
 372 add_layer_stx_build () {
 373     msg_step="Add required layers to the StarlingX build project"
 374     echo_step_start
 375
 376     source_env ${PRJ_BUILD_DIR}
 377     SRC_LAYERS=""
 378     for layer_url in ${SRC_YP_URL} ${SRC_STX_URL}; do
 379         layer_name=$(basename ${layer_url%%;commit*})
 380         case ${layer_name} in
 381         poky)
 382             continue
 383             ;;
 384         meta-openembedded)
 385             for sub_layer in ${SUB_LAYER_META_OE}; do
 386                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 387             done
 388             ;;
 389         meta-cloud-services)
 390             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 391             for sub_layer in ${SUB_LAYER_META_CLOUD_SERVICES}; do
 392                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 393             done
 394             ;;
 395         meta-starlingx)
 396             for sub_layer in ${SUB_LAYER_STX}; do
 397                 SRC_LAYERS="${SRC_LAYERS} ${SRC_STX_DIR}/${sub_layer}"
 398             done
 399             ;;
 400         *)
 401             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 402             ;;
 403
 404         esac
 405     done
 406     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx-oran"
 407
 408
 409     for src_layer in ${SRC_LAYERS}; do
 410         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 411         echo_cmd "Add the ${src_layer} layer into the build project"
 412         ${RUN_CMD}
 413     done
 414
 415     echo_step_end
 416 }
 417
 418 add_configs_stx_build () {
 419     msg_step="Add extra configs into local.conf for StarlingX build"
 420     echo_step_start
 421
 422     cd ${PRJ_BUILD_DIR}
 423     echo_info "Adding the following extra configs into local.conf"
 424     cat ${SRC_CONFIGS}/local-stx.conf | \
 425         sed -e "s/@BSP@/${BSP}/" | tee -a conf/local.conf
 426     cat ${SRC_CONFIGS}/local-mirrors.conf | tee -a conf/local.conf
 427
 428     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 429     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 430
 431     if [ "${RM_WORK}" == "Yes" ]; then
 432         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 433         echo "RM_WORK_EXCLUDE += '${IMG_STX}'" | tee -a conf/local.conf
 434     fi
 435
 436
 437     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 438         cat ${EXTRA_CONF} | tee -a conf/local.conf
 439     fi
 440     echo_step_end
 441 }
 442
 443 setup_stx_build () {
 444     echo_step_start "Setup StarlingX build project"
 445
 446     add_layer_stx_build
 447
 448     cd ${PRJ_BUILD_DIR}
 449     if ! grep -q 'Configs for StarlingX' conf/local.conf; then
 450         add_configs_stx_build
 451     else
 452         echo_info "Nothing is added into local.conf"
 453     fi
 454
 455     echo_step_end "Setup StarlingX build project"
 456 }
 457
 458 build_stx_image () {
 459     msg_step="Build StarlingX images"
 460     echo_step_start
 461
 462     source_env ${PRJ_BUILD_DIR}
 463
 464     mkdir -p logs
 465
 466     RUN_CMD="bitbake ${DRYRUN} ${IMG_STX}"
 467     echo_cmd "Build the ${IMG_STX} image"
 468     bitbake ${DRYRUN} ${IMG_STX} 2>&1|tee logs/bitbake_${IMG_STX}_${TIMESTAMP}.log
 469
 470     echo_step_end
 471
 472     echo_info "Build succeeded, you can get the image in ${ISO_STX}"
 473 }
 474
 475 add_layer_anaconda_build () {
 476     msg_step="Add required layers to the anaconda (installer) build project"
 477     echo_step_start
 478
 479     source_env ${PRJ_BUILD_DIR_ANACONDA}
 480     SRC_LAYERS=""
 481     for sub_layer in ${SUB_LAYER_META_OE_ANACONDA}; do
 482         SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-openembedded/${sub_layer}"
 483     done
 484     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-intel"
 485     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-anaconda"
 486     SRC_LAYERS="${SRC_LAYERS} ${SRC_STX_DIR}/meta-stx-distro"
 487     SRC_LAYERS="${SRC_LAYERS} ${SRC_STX_DIR}/meta-stx-integ"
 488     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx-oran"
 489
 490     for src_layer in ${SRC_LAYERS}; do
 491         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 492         echo_cmd "Add the ${src_layer} layer into the build project"
 493         ${RUN_CMD}
 494     done
 495
 496     echo_step_end
 497 }
 498
 499 add_configs_anaconda_build () {
 500     msg_step="Add extra configs into local.conf for anaconda (installer) build"
 501     echo_step_start
 502
 503     cd ${PRJ_BUILD_DIR_ANACONDA}
 504     echo_info "Adding the following extra configs into local.conf"
 505     cat ${SRC_CONFIGS}/local-anaconda.conf | \
 506         sed -e "s/@BSP@/${BSP}/" \
 507             -e "s|@TARGET_BUILD@|${PRJ_BUILD_DIR}|" \
 508             | tee -a conf/local.conf
 509     cat ${SRC_CONFIGS}/local-mirrors.conf | tee -a conf/local.conf
 510
 511     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 512     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 513
 514     if [ "${RM_WORK}" == "Yes" ]; then
 515         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 516         echo "RM_WORK_EXCLUDE += '${IMG_ANACONDA}'" | tee -a conf/local.conf
 517     fi
 518
 519     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 520         cat ${EXTRA_CONF} | tee -a conf/local.conf
 521     fi
 522
 523     echo_step_end
 524 }
 525
 526 setup_anaconda_build () {
 527     echo_step_start "Setup anaconda (installer) build project"
 528
 529     add_layer_anaconda_build
 530
 531     cd ${PRJ_BUILD_DIR_ANACONDA}
 532     if ! grep -q 'Configs for anaconda' conf/local.conf; then
 533         add_configs_anaconda_build
 534     else
 535         echo_info "Nothing is added into local.conf"
 536     fi
 537
 538     echo_step_end "Setup anaconda build project"
 539 }
 540
 541 build_anaconda_image () {
 542     echo_step_start "Build anaconda (installer) images"
 543     source_env ${PRJ_BUILD_DIR_ANACONDA}
 544
 545     mkdir -p logs
 546
 547     if [ -f ${ISO_ANACONDA} ]; then
 548         bitbake ${DRYRUN} -c clean ${IMG_ANACONDA}
 549     fi
 550     RUN_CMD="bitbake ${DRYRUN} ${IMG_ANACONDA}"
 551     echo_cmd "Build the ${IMG_ANACONDA} image"
 552     bitbake ${DRYRUN} ${IMG_ANACONDA} 2>&1|tee logs/bitbake_${IMG_ANACONDA}_${TIMESTAMP}.log
 553
 554     if [ -z "${DRYRUN}" ]; then
 555         cp -Pf ${ISO_ANACONDA} ${ISO_INF}
 556         cp -f ${ISO_ANACONDA} ${ISO_INF_ALIAS}
 557     fi
 558
 559     echo_step_end
 560
 561     echo_info "Build succeeded, you can get the image in ${ISO_INF_ALIAS}"
 562 }
 563
 564 #########################################################################
 565 # Main process
 566 #########################################################################
 567
 568 prepare_workspace
 569 if [ "${GET_SSTATE}" == "Yes" ]; then
 570     get_sstate
 571 fi
 572 prepare_src
 573 setup_stx_build
 574 setup_anaconda_build
 575 build_stx_image
 576 build_anaconda_image