scripts/build_inf.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2019 Wind River Systems, Inc.
   4 #
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16
  17 # Ensure we fail the job if any steps fail.
  18 set -e -o pipefail
  19
  20 #########################################################################
  21 # Variables
  22 #########################################################################
  23
  24 # Only one bsp is supported now, there will be more in the future
  25 SUPPORTED_BSP="intel-corei7-64 "
  26
  27 SRC_ORAN_BRANCH="bronze"
  28 SRC_STX_BRANCH="master"
  29 SRC_YP_BRANCH="warrior"
  30
  31 SRC_ORAN_URL="https://gerrit.o-ran-sc.org/r/pti/rtp"
  32
  33 SRC_YP_URL="\
  34     git://git.yoctoproject.org/poky;commit=f65b24e \
  35     git://git.openembedded.org/meta-openembedded;commit=a24acf9 \
  36     git://git.yoctoproject.org/meta-virtualization;commit=343b5e2 \
  37     git://git.yoctoproject.org/meta-cloud-services;commit=f8e76d1 \
  38     git://git.yoctoproject.org/meta-security;commit=4f7be0d \
  39     git://git.yoctoproject.org/meta-intel;commit=29ee485 \
  40     git://git.yoctoproject.org/meta-selinux;commit=ebea591 \
  41     https://github.com/intel-iot-devkit/meta-iot-cloud;commit=8b6c156 \
  42     git://git.openembedded.org/meta-python2;commit=6740870 \
  43     https://git.yoctoproject.org/git/meta-dpdk;commit=c8c30c2 \
  44     git://git.yoctoproject.org/meta-anaconda;commit=82c305d \
  45 "
  46
  47 SUB_LAYER_META_OE="\
  48     meta-oe \
  49     meta-perl \
  50     meta-python \
  51     meta-networking \
  52     meta-filesystems \
  53     meta-webserver \
  54     meta-initramfs \
  55     meta-initramfs \
  56     meta-gnome \
  57 "
  58
  59 SUB_LAYER_META_CLOUD_SERVICES="meta-openstack"
  60 SUB_LAYER_META_SECURITY="meta-security-compliance"
  61
  62 # For anaconda build
  63 SUB_LAYER_META_OE_ANACONDA="\
  64     meta-oe \
  65     meta-python \
  66     meta-filesystems \
  67     meta-initramfs \
  68     meta-networking \
  69     meta-gnome \
  70 "
  71
  72 SCRIPTS_DIR=$(dirname $(readlink -f $0))
  73 TIMESTAMP=`date +"%Y%m%d_%H%M%S"`
  74
  75 #########################################################################
  76 # Common Functions
  77 #########################################################################
  78
  79 help_info () {
  80 cat << ENDHELP
  81 Usage:
  82 $(basename $0) [-w WORKSPACE_DIR] [-b BSP] [-n] [-h] [-r Yes|No] [-e EXTRA_CONF]
  83 where:
  84     -w WORKSPACE_DIR is the path for the project
  85     -b BPS is one of supported BSP: "${SUPPORTED_BSP}"
  86        (default is intel-corei7-64 if not specified.)
  87     -n dry-run only for bitbake
  88     -h this help info
  89     -e EXTRA_CONF is the pat for extra config file
  90     -r whether to inherit rm_work (default is Yes)
  91 examples:
  92 $0
  93 $0 -w workspace_1234 -r no -e /path/to/extra_local.conf
  94 ENDHELP
  95 }
  96
  97 echo_step_start() {
  98     [ -n "$1" ] && msg_step=$1
  99     echo "#########################################################################################"
 100     echo "## STEP START: ${msg_step}"
 101     echo "#########################################################################################"
 102 }
 103
 104 echo_step_end() {
 105     [ -n "$1" ] && msg_step=$1
 106     echo "#########################################################################################"
 107     echo "## STEP END: ${msg_step}"
 108     echo "#########################################################################################"
 109     echo
 110 }
 111
 112 echo_info () {
 113     echo "INFO: $1"
 114 }
 115
 116 echo_error () {
 117     echo "ERROR: $1"
 118 }
 119
 120 echo_cmd () {
 121     echo
 122     echo_info "$1"
 123     echo "CMD: ${RUN_CMD}"
 124 }
 125
 126 check_yn_rm_work () {
 127     yn="$1"
 128     case ${yn} in
 129         [Yy]|[Yy]es)
 130             RM_WORK="Yes"
 131             ;;
 132         [Nn]|[Nn]o)
 133             RM_WORK="No"
 134             ;;
 135         *)
 136             echo "Invalid arg for -r option."
 137             help_info
 138             exit 1
 139             ;;
 140     esac
 141 }
 142
 143 check_valid_bsp () {
 144     bsp="$1"
 145     for b in ${SUPPORTED_BSP}; do
 146         if [ "${bsp}" == "${b}" ]; then
 147             BSP_VALID="${bsp}"
 148             break
 149         fi
 150     done
 151     if [ -z "${BSP_VALID}" ]; then
 152         echo_error "${bsp} is not a supported BSP, the supported BSPs are: ${SUPPORTED_BSP}"
 153         exit 1
 154     fi
 155 }
 156
 157
 158 clone_update_repo () {
 159     REPO_BRANCH=$1
 160     REPO_URL=$2
 161     REPO_NAME=$3
 162     REPO_COMMIT=$4
 163
 164     if [ -d ${REPO_NAME}/.git ]; then
 165         if [ "${SKIP_UPDATE}" == "Yes" ]; then
 166             echo_info "The repo ${REPO_NAME} exists, skip updating for the branch ${REPO_BRANCH}"
 167         else
 168             echo_info "The repo ${REPO_NAME} exists, updating for the branch ${REPO_BRANCH}"
 169             cd ${REPO_NAME}
 170             git checkout ${REPO_BRANCH}
 171             git pull
 172             cd -
 173         fi
 174     else
 175         RUN_CMD="git clone --branch ${REPO_BRANCH} ${REPO_URL} ${REPO_NAME}"
 176         echo_cmd "Cloning the source of repo '${REPO_NAME}':"
 177         ${RUN_CMD}
 178
 179         if [ -n "${REPO_COMMIT}" ]; then
 180             cd ${REPO_NAME}
 181             RUN_CMD="git checkout -b ${REPO_BRANCH}-${REPO_COMMIT} ${REPO_COMMIT}"
 182             echo_cmd "Checkout the repo ${REPO_NAME} to specific commit: ${REPO_COMMIT}"
 183             ${RUN_CMD}
 184             cd -
 185         fi
 186     fi
 187 }
 188
 189 source_env () {
 190     build_dir=$1
 191     cd ${SRC_LAYER_DIR}/poky
 192     set ${build_dir}
 193     source ./oe-init-build-env ${build_dir}
 194 }
 195
 196 #########################################################################
 197 # Parse cmd options
 198 #########################################################################
 199
 200 DRYRUN=""
 201 EXTRA_CONF=""
 202 SKIP_UPDATE="Yes"
 203 RM_WORK="Yes"
 204 BSP="intel-corei7-64"
 205
 206 while getopts "w:b:e:r:nh" OPTION; do
 207     case ${OPTION} in
 208         w)
 209             WORKSPACE=`readlink -f ${OPTARG}`
 210             ;;
 211         b)
 212             check_valid_bsp ${OPTARG}
 213             ;;
 214         e)
 215             EXTRA_CONF=`readlink -f ${OPTARG}`
 216             ;;
 217         n)
 218             DRYRUN="-n"
 219             ;;
 220         r)
 221             check_yn_rm_work ${OPTARG}
 222             ;;
 223         h)
 224             help_info
 225             exit
 226             ;;
 227     esac
 228 done
 229
 230 if [ -z ${WORKSPACE} ]; then
 231     echo_info "No workspace specified, a directory 'workspace' will be created in current directory as the workspace"
 232     WORKSPACE=`readlink -f workspace`
 233 fi
 234
 235 if [ -n "${BSP_VALID}" ]; then
 236     BSP="${BSP_VALID}"
 237 fi
 238
 239 #########################################################################
 240 # Functions for each step
 241 #########################################################################
 242 SRC_LAYER_DIR=${WORKSPACE}/src_layers
 243 SRC_ORAN_DIR=${SRC_LAYER_DIR}/oran
 244 PRJ_BUILD_DIR=${WORKSPACE}/prj_oran_stx
 245 PRJ_BUILD_DIR_ANACONDA=${WORKSPACE}/prj_oran_inf_anaconda
 246 PRJ_SHARED_DIR=${WORKSPACE}/prj_shared
 247 PRJ_SHARED_DL_DIR=${WORKSPACE}/prj_shared/downloads
 248 PRJ_SHARED_SS_DIR=${WORKSPACE}/prj_shared/sstate-cache
 249 SRC_META_PATCHES=${SRC_ORAN_DIR}/rtp/scripts/meta-patches/src_stx
 250 SRC_CONFIGS=${SRC_ORAN_DIR}/rtp/scripts/configs
 251 IMG_STX=stx-image-aio
 252 IMG_ANACONDA=stx-image-aio-installer
 253 IMG_INF=inf-image-aio-installer
 254 ISO_STX=${PRJ_BUILD_DIR}/tmp/deploy/images/${BSP}/${IMG_STX}-${BSP}.iso
 255 ISO_ANACONDA=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_ANACONDA}-${BSP}.iso
 256 ISO_INF=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_INF}-${BSP}.iso
 257
 258 prepare_workspace () {
 259     msg_step="Create workspace for the build"
 260     echo_step_start
 261
 262     mkdir -p ${PRJ_BUILD_DIR} ${SRC_ORAN_DIR} ${PRJ_BUILD_DIR_ANACONDA} ${PRJ_SHARED_DL_DIR} ${PRJ_SHARED_SS_DIR}
 263
 264     echo_info "The following directories are created in your workspace(${WORKSPACE}):"
 265     echo_info "For all layers source: ${SRC_LAYER_DIR}"
 266     echo_info "For StarlingX build project: ${PRJ_BUILD_DIR}"
 267     echo_info "For anaconda (installer) build project: ${PRJ_BUILD_DIR_ANACONDA}"
 268
 269     echo_step_end
 270 }
 271
 272 prepare_src () {
 273     msg_step="Get the source code repos"
 274     echo_step_start
 275
 276     # Clone the oran layer if it's not already cloned
 277     # Check if the script is inside the repo
 278     if cd ${SCRIPTS_DIR} && git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
 279         CLONED_ORAN_REPO=`dirname ${SCRIPTS_DIR}`
 280         echo_info "Use the cloned oran repo: ${CLONED_ORAN_REPO}"
 281         mkdir -p ${SRC_ORAN_DIR}/rtp
 282         cd ${SRC_ORAN_DIR}/rtp
 283         rm -rf meta-oran meta-stx scripts
 284         ln -sf ${CLONED_ORAN_REPO}/meta-oran meta-oran
 285         ln -sf ${CLONED_ORAN_REPO}/meta-stx meta-stx
 286         ln -sf ${CLONED_ORAN_REPO}/scripts scripts
 287     else
 288         echo_info "Cloning oran layer:"
 289         cd ${SRC_ORAN_DIR}
 290         clone_update_repo ${SRC_ORAN_BRANCH} ${SRC_ORAN_URL} rtp
 291     fi
 292
 293     echo_info "Cloning or update Yocto layers:"
 294
 295     cd ${SRC_LAYER_DIR}
 296     for layer_url in ${SRC_YP_URL}; do
 297         layer_name=$(basename ${layer_url%%;commit*})
 298         layer_commit=$(basename ${layer_url##*;commit=})
 299         clone_update_repo ${SRC_YP_BRANCH} ${layer_url%%;commit*} ${layer_name} ${layer_commit}
 300     done
 301
 302     echo_step_end
 303
 304     # Apply meta patches
 305     for l in $(ls -1 ${SRC_META_PATCHES}); do
 306         msg_step="Apply meta patches for ${l}"
 307         echo_step_start
 308         cd ${SRC_LAYER_DIR}/${l}
 309
 310         # backup current branch
 311         local_branch=$(git rev-parse --abbrev-ref HEAD)
 312         git branch -m "${local_branch}_${TIMESTAMP}"
 313         git checkout -b ${local_branch} ${local_branch##*-}
 314
 315         for p in $(ls -1 ${SRC_META_PATCHES}/${l}); do
 316             echo_info "Apllying patch: ${SRC_META_PATCHES}/${l}/${p}"
 317             git am ${SRC_META_PATCHES}/${l}/${p}
 318         done
 319         echo_step_end
 320     done
 321 }
 322
 323 add_layer_stx_build () {
 324     msg_step="Add required layers to the StarlingX build project"
 325     echo_step_start
 326
 327     source_env ${PRJ_BUILD_DIR}
 328     SRC_LAYERS=""
 329     for layer_url in ${SRC_YP_URL}; do
 330         layer_name=$(basename ${layer_url%%;commit*})
 331         case ${layer_name} in
 332         poky)
 333             continue
 334             ;;
 335         meta-openembedded)
 336             for sub_layer in ${SUB_LAYER_META_OE}; do
 337                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 338             done
 339             ;;
 340         meta-cloud-services)
 341             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 342             for sub_layer in ${SUB_LAYER_META_CLOUD_SERVICES}; do
 343                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 344             done
 345             ;;
 346         meta-security)
 347             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 348             for sub_layer in ${SUB_LAYER_META_SECURITY}; do
 349                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 350             done
 351             ;;
 352         *)
 353             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 354             ;;
 355
 356         esac
 357     done
 358
 359     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 360
 361     for src_layer in ${SRC_LAYERS}; do
 362         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 363         echo_cmd "Add the ${src_layer} layer into the build project"
 364         ${RUN_CMD}
 365     done
 366
 367     echo_step_end
 368 }
 369
 370 add_configs_stx_build () {
 371     msg_step="Add extra configs into local.conf for StarlingX build"
 372     echo_step_start
 373
 374     cd ${PRJ_BUILD_DIR}
 375     echo_info "Adding the following extra configs into local.conf"
 376     cat ${SRC_CONFIGS}/local-stx.conf | \
 377         sed -e "s/@BSP@/${BSP}/" | tee -a conf/local.conf
 378     cat ${SRC_CONFIGS}/local-mirrors.conf | tee -a conf/local.conf
 379
 380     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 381     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 382
 383     if [ "${RM_WORK}" == "Yes" ]; then
 384         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 385         echo "RM_WORK_EXCLUDE += '${IMG_STX}'" | tee -a conf/local.conf
 386     fi
 387
 388
 389     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 390         cat ${EXTRA_CONF} | tee -a conf/local.conf
 391     fi
 392     echo_step_end
 393 }
 394
 395 setup_stx_build () {
 396     echo_step_start "Setup StarlingX build project"
 397
 398     add_layer_stx_build
 399
 400     cd ${PRJ_BUILD_DIR}
 401     if ! grep -q 'Configs for StarlingX' conf/local.conf; then
 402         add_configs_stx_build
 403     else
 404         echo_info "Nothing is added into local.conf"
 405     fi
 406
 407     echo_step_end "Setup StarlingX build project"
 408 }
 409
 410 build_stx_image () {
 411     msg_step="Build StarlingX images"
 412     echo_step_start
 413
 414     source_env ${PRJ_BUILD_DIR}
 415
 416     mkdir -p logs
 417
 418     RUN_CMD="bitbake ${DRYRUN} ${IMG_STX}"
 419     echo_cmd "Build the ${IMG_STX} image"
 420     bitbake ${DRYRUN} ${IMG_STX} 2>&1|tee logs/bitbake_${IMG_STX}_${TIMESTAMP}.log
 421
 422     echo_step_end
 423
 424     echo_info "Build succeeded, you can get the image in ${ISO_STX}"
 425 }
 426
 427 add_layer_anaconda_build () {
 428     msg_step="Add required layers to the anaconda (installer) build project"
 429     echo_step_start
 430
 431     source_env ${PRJ_BUILD_DIR_ANACONDA}
 432     SRC_LAYERS=""
 433     for sub_layer in ${SUB_LAYER_META_OE_ANACONDA}; do
 434         SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-openembedded/${sub_layer}"
 435     done
 436     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-intel"
 437     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-anaconda"
 438     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 439
 440     for src_layer in ${SRC_LAYERS}; do
 441         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 442         echo_cmd "Add the ${src_layer} layer into the build project"
 443         ${RUN_CMD}
 444     done
 445
 446     echo_step_end
 447 }
 448
 449 add_configs_anaconda_build () {
 450     msg_step="Add extra configs into local.conf for anaconda (installer) build"
 451     echo_step_start
 452
 453     cd ${PRJ_BUILD_DIR_ANACONDA}
 454     echo_info "Adding the following extra configs into local.conf"
 455     cat ${SRC_CONFIGS}/local-anaconda.conf | \
 456         sed -e "s/@BSP@/${BSP}/" \
 457             -e "s|@TARGET_BUILD@|${PRJ_BUILD_DIR}|" \
 458             | tee -a conf/local.conf
 459     cat ${SRC_CONFIGS}/local-mirrors.conf | tee -a conf/local.conf
 460
 461     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 462     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 463
 464     if [ "${RM_WORK}" == "Yes" ]; then
 465         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 466         echo "RM_WORK_EXCLUDE += '${IMG_ANACONDA}'" | tee -a conf/local.conf
 467     fi
 468
 469     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 470         cat ${EXTRA_CONF} | tee -a conf/local.conf
 471     fi
 472
 473     echo_step_end
 474 }
 475
 476 setup_anaconda_build () {
 477     echo_step_start "Setup anaconda (installer) build project"
 478
 479     add_layer_anaconda_build
 480
 481     cd ${PRJ_BUILD_DIR_ANACONDA}
 482     if ! grep -q 'Configs for anaconda' conf/local.conf; then
 483         add_configs_anaconda_build
 484     else
 485         echo_info "Nothing is added into local.conf"
 486     fi
 487
 488     echo_step_end "Setup anaconda build project"
 489 }
 490
 491 build_anaconda_image () {
 492     echo_step_start "Build anaconda (installer) images"
 493     source_env ${PRJ_BUILD_DIR_ANACONDA}
 494
 495     mkdir -p logs
 496
 497     if [ -f ${ISO_ANACONDA} ]; then
 498         bitbake ${DRYRUN} -c clean ${IMG_ANACONDA}
 499     fi
 500     RUN_CMD="bitbake ${DRYRUN} ${IMG_ANACONDA}"
 501     echo_cmd "Build the ${IMG_ANACONDA} image"
 502     bitbake ${DRYRUN} ${IMG_ANACONDA} 2>&1|tee logs/bitbake_${IMG_ANACONDA}_${TIMESTAMP}.log
 503
 504     if [ -z "${DRYRUN}" ]; then
 505         cp -Pf ${ISO_ANACONDA} ${ISO_INF}
 506     fi
 507
 508     echo_step_end
 509
 510     echo_info "Build succeeded, you can get the image in ${ISO_INF}"
 511 }
 512
 513 #########################################################################
 514 # Main process
 515 #########################################################################
 516
 517 prepare_workspace
 518 prepare_src
 519 setup_stx_build
 520 setup_anaconda_build
 521 build_stx_image
 522 build_anaconda_image