scripts/build_inf.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2019 Wind River Systems, Inc.
   4 #
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16
  17 # Ensure we fail the job if any steps fail.
  18 set -e -o pipefail
  19
  20 #########################################################################
  21 # Variables
  22 #########################################################################
  23
  24 # Only one bsp is supported now, there will be more in the future
  25 SUPPORTED_BSP="intel-corei7-64 "
  26
  27 SRC_ORAN_BRANCH="master"
  28 SRC_STX_BRANCH="master"
  29 SRC_YP_BRANCH="warrior"
  30
  31 SRC_ORAN_URL="https://gerrit.o-ran-sc.org/r/pti/rtp"
  32
  33 SRC_YP_URL="\
  34     git://git.yoctoproject.org/poky \
  35     git://git.openembedded.org/meta-openembedded \
  36     git://git.yoctoproject.org/meta-virtualization \
  37     git://git.yoctoproject.org/meta-cloud-services \
  38     git://git.yoctoproject.org/meta-security \
  39     git://git.yoctoproject.org/meta-intel \
  40     git://git.yoctoproject.org/meta-selinux \
  41     https://github.com/intel-iot-devkit/meta-iot-cloud \
  42     git://git.openembedded.org/meta-python2 \
  43     https://git.yoctoproject.org/git/meta-dpdk \
  44     git://git.yoctoproject.org/meta-anaconda \
  45 "
  46
  47 SUB_LAYER_META_OE="\
  48     meta-oe \
  49     meta-perl \
  50     meta-python \
  51     meta-networking \
  52     meta-filesystems \
  53     meta-webserver \
  54     meta-initramfs \
  55     meta-initramfs \
  56     meta-gnome \
  57 "
  58
  59 SUB_LAYER_META_CLOUD_SERVICES="meta-openstack"
  60 SUB_LAYER_META_SECURITY="meta-security-compliance"
  61
  62 # For anaconda build
  63 SUB_LAYER_META_OE_ANACONDA="\
  64     meta-oe \
  65     meta-python \
  66     meta-filesystems \
  67     meta-initramfs \
  68     meta-networking \
  69     meta-gnome \
  70 "
  71
  72 SCRIPTS_DIR=$(dirname $(readlink -f $0))
  73 TIMESTAMP=`date +"%Y%m%d_%H%M%S"`
  74
  75 #########################################################################
  76 # Common Functions
  77 #########################################################################
  78
  79 help_info () {
  80 cat << ENDHELP
  81 Usage:
  82 $(basename $0) [-w WORKSPACE_DIR] [-b BSP] [-n] [-h] [-r Yes|No] [-s] [-e EXTRA_CONF]
  83 where:
  84     -w WORKSPACE_DIR is the path for the project
  85     -b BPS is one of supported BSP: "${SUPPORTED_BSP}"
  86        (default is intel-corei7-64 if not specified.)
  87     -n dry-run only for bitbake
  88     -h this help info
  89     -e EXTRA_CONF is the pat for extra config file
  90     -r whether to inherit rm_work (default is Yes)
  91     -s whether to skip update the repo if already exists
  92 examples:
  93 $0
  94 $0 -w workspace_1234 -r no -e /path/to/extra_local.conf
  95 ENDHELP
  96 }
  97
  98 echo_step_start() {
  99     [ -n "$1" ] && msg_step=$1
 100     echo "#########################################################################################"
 101     echo "## STEP START: ${msg_step}"
 102     echo "#########################################################################################"
 103 }
 104
 105 echo_step_end() {
 106     [ -n "$1" ] && msg_step=$1
 107     echo "#########################################################################################"
 108     echo "## STEP END: ${msg_step}"
 109     echo "#########################################################################################"
 110     echo
 111 }
 112
 113 echo_info () {
 114     echo "INFO: $1"
 115 }
 116
 117 echo_error () {
 118     echo "ERROR: $1"
 119 }
 120
 121 echo_cmd () {
 122     echo
 123     echo_info "$1"
 124     echo "CMD: ${RUN_CMD}"
 125 }
 126
 127 check_yn_rm_work () {
 128     yn="$1"
 129     case ${yn} in
 130         [Yy]|[Yy]es)
 131             RM_WORK="Yes"
 132             ;;
 133         [Nn]|[Nn]o)
 134             RM_WORK="No"
 135             ;;
 136         *)
 137             echo "Invalid arg for -r option."
 138             help_info
 139             exit 1
 140             ;;
 141     esac
 142 }
 143
 144 check_valid_bsp () {
 145     bsp="$1"
 146     for b in ${SUPPORTED_BSP}; do
 147         if [ "${bsp}" == "${b}" ]; then
 148             BSP_VALID="${bsp}"
 149             break
 150         fi
 151     done
 152     if [ -z "${BSP_VALID}" ]; then
 153         echo_error "${bsp} is not a supported BSP, the supported BSPs are: ${SUPPORTED_BSP}"
 154         exit 1
 155     fi
 156 }
 157
 158
 159 clone_update_repo () {
 160     REPO_BRANCH=$1
 161     REPO_URL=$2
 162     REPO_NAME=$3
 163
 164     if [ -d ${REPO_NAME}/.git ]; then
 165         if [ "${SKIP_UPDATE}" == "Yes" ]; then
 166             echo_info "The repo ${REPO_NAME} exists, skip updating for the branch ${REPO_BRANCH}"
 167         else
 168             echo_info "The repo ${REPO_NAME} exists, updating for the branch ${REPO_BRANCH}"
 169             cd ${REPO_NAME}
 170             git checkout ${REPO_BRANCH}
 171             git pull
 172             cd -
 173         fi
 174     else
 175         RUN_CMD="git clone --branch ${REPO_BRANCH} ${REPO_URL} ${REPO_NAME}"
 176         echo_cmd "Cloning the source of repo '${REPO_NAME}':"
 177         ${RUN_CMD}
 178     fi
 179 }
 180
 181 source_env () {
 182     build_dir=$1
 183     cd ${SRC_LAYER_DIR}/poky
 184     set ${build_dir}
 185     source ./oe-init-build-env ${build_dir}
 186 }
 187
 188 #########################################################################
 189 # Parse cmd options
 190 #########################################################################
 191
 192 DRYRUN=""
 193 EXTRA_CONF=""
 194 SKIP_UPDATE="No"
 195 RM_WORK="Yes"
 196 BSP="intel-corei7-64"
 197
 198 while getopts "w:b:e:r:nsh" OPTION; do
 199     case ${OPTION} in
 200         w)
 201             WORKSPACE=`readlink -f ${OPTARG}`
 202             ;;
 203         b)
 204             check_valid_bsp ${OPTARG}
 205             ;;
 206         e)
 207             EXTRA_CONF=`readlink -f ${OPTARG}`
 208             ;;
 209         n)
 210             DRYRUN="-n"
 211             ;;
 212         s)
 213             SKIP_UPDATE="Yes"
 214             ;;
 215         r)
 216             check_yn_rm_work ${OPTARG}
 217             ;;
 218         h)
 219             help_info
 220             exit
 221             ;;
 222     esac
 223 done
 224
 225 if [ -z ${WORKSPACE} ]; then
 226     echo_info "No workspace specified, a directory 'workspace' will be created in current directory as the workspace"
 227     WORKSPACE=`readlink -f workspace`
 228 fi
 229
 230 if [ -n "${BSP_VALID}" ]; then
 231     BSP="${BSP_VALID}"
 232 fi
 233
 234 #########################################################################
 235 # Functions for each step
 236 #########################################################################
 237 SRC_LAYER_DIR=${WORKSPACE}/src_layers
 238 SRC_ORAN_DIR=${SRC_LAYER_DIR}/oran
 239 PRJ_BUILD_DIR=${WORKSPACE}/prj_oran_stx
 240 PRJ_BUILD_DIR_ANACONDA=${WORKSPACE}/prj_oran_inf_anaconda
 241 PRJ_SHARED_DIR=${WORKSPACE}/prj_shared
 242 PRJ_SHARED_DL_DIR=${WORKSPACE}/prj_shared/downloads
 243 PRJ_SHARED_SS_DIR=${WORKSPACE}/prj_shared/sstate-cache
 244 SRC_META_PATCHES=${SRC_ORAN_DIR}/rtp/scripts/meta-patches/src_stx
 245 SRC_CONFIGS=${SRC_ORAN_DIR}/rtp/scripts/configs
 246 IMG_STX=stx-image-aio
 247 IMG_ANACONDA=stx-image-aio-installer
 248 IMG_INF=inf-image-aio-installer
 249 ISO_STX=${PRJ_BUILD_DIR}/tmp/deploy/images/${BSP}/${IMG_STX}-${BSP}.iso
 250 ISO_ANACONDA=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_ANACONDA}-${BSP}.iso
 251 ISO_INF=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_INF}-${BSP}.iso
 252
 253 prepare_workspace () {
 254     msg_step="Create workspace for the build"
 255     echo_step_start
 256
 257     mkdir -p ${PRJ_BUILD_DIR} ${SRC_ORAN_DIR} ${PRJ_BUILD_DIR_ANACONDA} ${PRJ_SHARED_DL_DIR} ${PRJ_SHARED_SS_DIR}
 258
 259     echo_info "The following directories are created in your workspace(${WORKSPACE}):"
 260     echo_info "For all layers source: ${SRC_LAYER_DIR}"
 261     echo_info "For StarlingX build project: ${PRJ_BUILD_DIR}"
 262     echo_info "For anaconda (installer) build project: ${PRJ_BUILD_DIR_ANACONDA}"
 263
 264     echo_step_end
 265 }
 266
 267 prepare_src () {
 268     msg_step="Get the source code repos"
 269     echo_step_start
 270
 271     # Clone the oran layer if it's not already cloned
 272     # Check if the script is inside the repo
 273     if cd ${SCRIPTS_DIR} && git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
 274         CLONED_ORAN_REPO=`dirname ${SCRIPTS_DIR}`
 275         echo_info "Use the cloned oran repo: ${CLONED_ORAN_REPO}"
 276         mkdir -p ${SRC_ORAN_DIR}/rtp
 277         cd ${SRC_ORAN_DIR}/rtp
 278         rm -rf meta-oran meta-stx scripts
 279         ln -sf ${CLONED_ORAN_REPO}/meta-oran meta-oran
 280         ln -sf ${CLONED_ORAN_REPO}/meta-stx meta-stx
 281         ln -sf ${CLONED_ORAN_REPO}/scripts scripts
 282     else
 283         echo_info "Cloning oran layer:"
 284         cd ${SRC_ORAN_DIR}
 285         clone_update_repo ${SRC_ORAN_BRANCH} ${SRC_ORAN_URL} rtp
 286     fi
 287
 288     echo_info "Cloning or update Yocto layers:"
 289
 290     cd ${SRC_LAYER_DIR}
 291     for layer_url in ${SRC_YP_URL}; do
 292         layer_name=$(basename ${layer_url})
 293         clone_update_repo ${SRC_YP_BRANCH} ${layer_url} ${layer_name}
 294     done
 295
 296     echo_step_end
 297
 298     # Apply meta patches
 299     for l in $(ls -1 ${SRC_META_PATCHES}); do
 300         msg_step="Apply meta patches for ${l}"
 301         echo_step_start
 302         cd ${SRC_LAYER_DIR}/${l}
 303
 304         # backup current branch
 305         local_branch=$(git rev-parse --abbrev-ref HEAD)
 306         git branch -m "${local_branch}_${TIMESTAMP}"
 307         git checkout ${local_branch}
 308
 309         for p in $(ls -1 ${SRC_META_PATCHES}/${l}); do
 310             echo_info "Apllying patch: ${SRC_META_PATCHES}/${l}/${p}"
 311             git am ${SRC_META_PATCHES}/${l}/${p}
 312         done
 313         echo_step_end
 314     done
 315 }
 316
 317 add_layer_stx_build () {
 318     msg_step="Add required layers to the StarlingX build project"
 319     echo_step_start
 320
 321     source_env ${PRJ_BUILD_DIR}
 322     SRC_LAYERS=""
 323     for layer_url in ${SRC_YP_URL}; do
 324         layer_name=$(basename ${layer_url})
 325         case ${layer_name} in
 326         poky)
 327             continue
 328             ;;
 329         meta-openembedded)
 330             for sub_layer in ${SUB_LAYER_META_OE}; do
 331                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 332             done
 333             ;;
 334         meta-cloud-services)
 335             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 336             for sub_layer in ${SUB_LAYER_META_CLOUD_SERVICES}; do
 337                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 338             done
 339             ;;
 340         meta-security)
 341             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 342             for sub_layer in ${SUB_LAYER_META_SECURITY}; do
 343                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 344             done
 345             ;;
 346         *)
 347             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 348             ;;
 349
 350         esac
 351     done
 352
 353     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 354
 355     for src_layer in ${SRC_LAYERS}; do
 356         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 357         echo_cmd "Add the ${src_layer} layer into the build project"
 358         ${RUN_CMD}
 359     done
 360
 361     echo_step_end
 362 }
 363
 364 add_configs_stx_build () {
 365     msg_step="Add extra configs into local.conf for StarlingX build"
 366     echo_step_start
 367
 368     cd ${PRJ_BUILD_DIR}
 369     echo_info "Adding the following extra configs into local.conf"
 370     cat ${SRC_CONFIGS}/local-stx.conf | \
 371         sed -e "s/@BSP@/${BSP}/" | tee -a conf/local.conf
 372
 373     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 374     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 375
 376     if [ "${RM_WORK}" == "Yes" ]; then
 377         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 378     fi
 379
 380
 381     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 382         cat ${EXTRA_CONF} | tee -a conf/local.conf
 383     fi
 384     echo_step_end
 385 }
 386
 387 setup_stx_build () {
 388     echo_step_start "Setup StarlingX build project"
 389
 390     add_layer_stx_build
 391
 392     cd ${PRJ_BUILD_DIR}
 393     if ! grep -q 'Configs for StarlingX' conf/local.conf; then
 394         add_configs_stx_build
 395     else
 396         echo_info "Nothing is added into local.conf"
 397     fi
 398
 399     echo_step_end "Setup StarlingX build project"
 400 }
 401
 402 build_stx_image () {
 403     msg_step="Build StarlingX images"
 404     echo_step_start
 405
 406     source_env ${PRJ_BUILD_DIR}
 407
 408     mkdir -p logs
 409
 410     RUN_CMD="bitbake ${DRYRUN} ${IMG_STX}"
 411     echo_cmd "Build the ${IMG_STX} image"
 412     bitbake ${DRYRUN} ${IMG_STX} 2>&1|tee logs/bitbake_${IMG_STX}_${TIMESTAMP}.log
 413
 414     echo_step_end
 415
 416     echo_info "Build succeeded, you can get the image in ${ISO_STX}"
 417 }
 418
 419 add_layer_anaconda_build () {
 420     msg_step="Add required layers to the anaconda (installer) build project"
 421     echo_step_start
 422
 423     source_env ${PRJ_BUILD_DIR_ANACONDA}
 424     SRC_LAYERS=""
 425     for sub_layer in ${SUB_LAYER_META_OE_ANACONDA}; do
 426         SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-openembedded/${sub_layer}"
 427     done
 428     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-intel"
 429     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-anaconda"
 430     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 431
 432     for src_layer in ${SRC_LAYERS}; do
 433         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 434         echo_cmd "Add the ${src_layer} layer into the build project"
 435         ${RUN_CMD}
 436     done
 437
 438     echo_step_end
 439 }
 440
 441 add_configs_anaconda_build () {
 442     msg_step="Add extra configs into local.conf for anaconda (installer) build"
 443     echo_step_start
 444
 445     cd ${PRJ_BUILD_DIR_ANACONDA}
 446     echo_info "Adding the following extra configs into local.conf"
 447     cat ${SRC_CONFIGS}/local-anaconda.conf | \
 448         sed -e "s/@BSP@/${BSP}/" \
 449             -e "s|@TARGET_BUILD@|${PRJ_BUILD_DIR}|" \
 450             | tee -a conf/local.conf
 451
 452     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 453     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 454
 455     if [ "${RM_WORK}" == "Yes" ]; then
 456         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 457     fi
 458
 459     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 460         cat ${EXTRA_CONF} | tee -a conf/local.conf
 461     fi
 462
 463     echo_step_end
 464 }
 465
 466 setup_anaconda_build () {
 467     echo_step_start "Setup anaconda (installer) build project"
 468
 469     add_layer_anaconda_build
 470
 471     cd ${PRJ_BUILD_DIR_ANACONDA}
 472     if ! grep -q 'Configs for anaconda' conf/local.conf; then
 473         add_configs_anaconda_build
 474     else
 475         echo_info "Nothing is added into local.conf"
 476     fi
 477
 478     echo_step_end "Setup anaconda build project"
 479 }
 480
 481 build_anaconda_image () {
 482     echo_step_start "Build anaconda (installer) images"
 483     source_env ${PRJ_BUILD_DIR_ANACONDA}
 484
 485     mkdir -p logs
 486
 487     if [ -f ${ISO_ANACONDA} ]; then
 488         bitbake ${DRYRUN} -c clean ${IMG_ANACONDA}
 489     fi
 490     RUN_CMD="bitbake ${DRYRUN} ${IMG_ANACONDA}"
 491     echo_cmd "Build the ${IMG_ANACONDA} image"
 492     bitbake ${DRYRUN} ${IMG_ANACONDA} 2>&1|tee logs/bitbake_${IMG_ANACONDA}_${TIMESTAMP}.log
 493
 494     if [ -z "${DRYRUN}" ]; then
 495         cp -Pf ${ISO_ANACONDA} ${ISO_INF}
 496     fi
 497
 498     echo_step_end
 499
 500     echo_info "Build succeeded, you can get the image in ${ISO_INF}"
 501 }
 502
 503 #########################################################################
 504 # Main process
 505 #########################################################################
 506
 507 prepare_workspace
 508 prepare_src
 509 setup_stx_build
 510 setup_anaconda_build
 511 build_stx_image
 512 build_anaconda_image