scripts/build_stx.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2019 Wind River Systems, Inc.
   4 #
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16
  17 # Ensure we fail the job if any steps fail.
  18 set -e -o pipefail
  19
  20 #########################################################################
  21 # Variables
  22 #########################################################################
  23
  24 # Only one bsp is supported now, there will be more in the future
  25 SUPPORTED_BSP="intel-corei7-64 "
  26
  27 SRC_ORAN_BRANCH="master"
  28 SRC_STX_BRANCH="master"
  29 SRC_YP_BRANCH="warrior"
  30
  31 SRC_ORAN_URL="https://gerrit.o-ran-sc.org/r/pti/rtp"
  32
  33 SRC_YP_URL="\
  34     git://git.yoctoproject.org/poky \
  35     git://git.openembedded.org/meta-openembedded \
  36     git://git.yoctoproject.org/meta-virtualization \
  37     git://git.yoctoproject.org/meta-cloud-services \
  38     git://git.yoctoproject.org/meta-security \
  39     git://git.yoctoproject.org/meta-intel \
  40     git://git.yoctoproject.org/meta-selinux \
  41     https://github.com/intel-iot-devkit/meta-iot-cloud \
  42     git://git.openembedded.org/meta-python2 \
  43     https://git.yoctoproject.org/git/meta-dpdk \
  44     git://git.yoctoproject.org/meta-anaconda \
  45 "
  46
  47 SUB_LAYER_META_OE="\
  48     meta-oe \
  49     meta-perl \
  50     meta-python \
  51     meta-networking \
  52     meta-filesystems \
  53     meta-webserver \
  54     meta-initramfs \
  55     meta-initramfs \
  56     meta-gnome \
  57 "
  58
  59 SUB_LAYER_META_CLOUD_SERVICES="meta-openstack"
  60 SUB_LAYER_META_SECURITY="meta-security-compliance"
  61
  62 # For anaconda build
  63 SUB_LAYER_META_OE_ANACONDA="\
  64     meta-oe \
  65     meta-python \
  66     meta-filesystems \
  67     meta-initramfs \
  68     meta-networking \
  69     meta-gnome \
  70 "
  71
  72 SCRIPTS_DIR=$(dirname $(readlink -f $0))
  73 TIMESTAMP=`date +"%Y%m%d_%H%M%S"`
  74
  75 #########################################################################
  76 # Common Functions
  77 #########################################################################
  78
  79 help_info () {
  80 cat << ENDHELP
  81 Usage:
  82 $(basename $0) [-w WORKSPACE_DIR] [-b BSP] [-n] [-h] [-r Yes|No] [-s] [-e EXTRA_CONF]
  83 where:
  84     -w WORKSPACE_DIR is the path for the project
  85     -b BPS is one of supported BSP: "${SUPPORTED_BSP}"
  86        (default is intel-corei7-64 if not specified.)
  87     -n dry-run only for bitbake
  88     -h this help info
  89     -e EXTRA_CONF is the pat for extra config file
  90     -r whether to inherit rm_work (default is Yes)
  91     -s whether to skip update the repo if already exists
  92 examples:
  93 ./build_stx.sh
  94 ./build_stx.sh -w workspace_1234 -r no -e /path/to/extra_local.conf
  95 ENDHELP
  96 }
  97
  98 echo_step_start() {
  99     [ -n "$1" ] && msg_step=$1
 100     echo "#########################################################################################"
 101     echo "## STEP START: ${msg_step}"
 102     echo "#########################################################################################"
 103 }
 104
 105 echo_step_end() {
 106     [ -n "$1" ] && msg_step=$1
 107     echo "#########################################################################################"
 108     echo "## STEP END: ${msg_step}"
 109     echo "#########################################################################################"
 110     echo
 111 }
 112
 113 echo_info () {
 114     echo "INFO: $1"
 115 }
 116
 117 echo_error () {
 118     echo "ERROR: $1"
 119 }
 120
 121 echo_cmd () {
 122     echo
 123     echo_info "$1"
 124     echo "CMD: ${RUN_CMD}"
 125 }
 126
 127 check_yn_rm_work () {
 128     yn="$1"
 129     case ${yn} in
 130         [Yy]|[Yy]es)
 131             RM_WORK="Yes"
 132             ;;
 133         [Nn]|[Nn]o)
 134             RM_WORK="No"
 135             ;;
 136         *)
 137             echo "Invalid arg for -r option."
 138             help_info
 139             exit 1
 140             ;;
 141     esac
 142 }
 143
 144 check_valid_bsp () {
 145     bsp="$1"
 146     for b in ${SUPPORTED_BSP}; do
 147         if [ "${bsp}" == "${b}" ]; then
 148             BSP_VALID="${bsp}"
 149             break
 150         fi
 151     done
 152     if [ -z "${BSP_VALID}" ]; then
 153         echo_error "${bsp} is not a supported BSP, the supported BSPs are: ${SUPPORTED_BSP}"
 154         exit 1
 155     fi
 156 }
 157
 158
 159 clone_update_repo () {
 160     REPO_BRANCH=$1
 161     REPO_URL=$2
 162     REPO_NAME=$3
 163
 164     if [ -d ${REPO_NAME}/.git ]; then
 165         if [ "${SKIP_UPDATE}" == "Yes" ]; then
 166             echo_info "The repo ${REPO_NAME} exists, skip updating for the branch ${REPO_BRANCH}"
 167         else
 168             echo_info "The repo ${REPO_NAME} exists, updating for the branch ${REPO_BRANCH}"
 169             cd ${REPO_NAME}
 170             git checkout ${REPO_BRANCH}
 171             git pull
 172             cd -
 173         fi
 174     else
 175         RUN_CMD="git clone --branch ${REPO_BRANCH} ${REPO_URL} ${REPO_NAME}"
 176         echo_cmd "Cloning the source of repo '${REPO_NAME}':"
 177         ${RUN_CMD}
 178     fi
 179 }
 180
 181 source_env () {
 182     build_dir=$1
 183     cd ${SRC_LAYER_DIR}/poky
 184     set ${build_dir}
 185     source ./oe-init-build-env ${build_dir}
 186 }
 187
 188 #########################################################################
 189 # Parse cmd options
 190 #########################################################################
 191
 192 DRYRUN=""
 193 EXTRA_CONF=""
 194 SKIP_UPDATE="No"
 195 RM_WORK="Yes"
 196 BSP="intel-corei7-64"
 197
 198 while getopts "w:b:e:r:nsh" OPTION; do
 199     case ${OPTION} in
 200         w)
 201             WORKSPACE=`readlink -f ${OPTARG}`
 202             ;;
 203         b)
 204             check_valid_bsp ${OPTARG}
 205             ;;
 206         e)
 207             EXTRA_CONF=`readlink -f ${OPTARG}`
 208             ;;
 209         n)
 210             DRYRUN="-n"
 211             ;;
 212         s)
 213             SKIP_UPDATE="Yes"
 214             ;;
 215         r)
 216             check_yn_rm_work ${OPTARG}
 217             ;;
 218         h)
 219             help_info
 220             exit
 221             ;;
 222     esac
 223 done
 224
 225 if [ -z ${WORKSPACE} ]; then
 226     echo_info "No workspace specified, a directory 'workspace' will be created in current directory as the workspace"
 227     WORKSPACE=`readlink -f workspace`
 228 fi
 229
 230 if [ -n "${BSP_VALID}" ]; then
 231     BSP="${BSP_VALID}"
 232 fi
 233
 234 #########################################################################
 235 # Functions for each step
 236 #########################################################################
 237 SRC_LAYER_DIR=${WORKSPACE}/src_layers
 238 SRC_ORAN_DIR=${SRC_LAYER_DIR}/oran
 239 PRJ_BUILD_DIR=${WORKSPACE}/prj_oran_stx
 240 PRJ_BUILD_DIR_ANACONDA=${WORKSPACE}/prj_oran_stx_anaconda
 241 PRJ_SHARED_DIR=${WORKSPACE}/prj_shared
 242 PRJ_SHARED_DL_DIR=${WORKSPACE}/prj_shared/downloads
 243 PRJ_SHARED_SS_DIR=${WORKSPACE}/prj_shared/sstate-cache
 244 SRC_META_PATCHES=${SRC_ORAN_DIR}/rtp/scripts/meta-patches/src_stx
 245 SRC_CONFIGS=${SRC_ORAN_DIR}/rtp/scripts/configs
 246 IMG_STX=stx-image-aio
 247 IMG_ANACONDA=stx-image-aio-installer
 248 ISO_STX=${PRJ_BUILD_DIR}/tmp/deploy/images/${BSP}/${IMG_STX}-${BSP}.iso
 249 ISO_ANACONDA=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_ANACONDA}-${BSP}.iso
 250
 251 prepare_workspace () {
 252     msg_step="Create workspace for the build"
 253     echo_step_start
 254
 255     mkdir -p ${PRJ_BUILD_DIR} ${SRC_ORAN_DIR} ${PRJ_BUILD_DIR_ANACONDA} ${PRJ_SHARED_DL_DIR} ${PRJ_SHARED_SS_DIR}
 256
 257     echo_info "The following directories are created in your workspace(${WORKSPACE}):"
 258     echo_info "For all layers source: ${SRC_LAYER_DIR}"
 259     echo_info "For StarlingX build project: ${PRJ_BUILD_DIR}"
 260     echo_info "For anaconda (installer) build project: ${PRJ_BUILD_DIR_ANACONDA}"
 261
 262     echo_step_end
 263 }
 264
 265 prepare_src () {
 266     msg_step="Get the source code repos"
 267     echo_step_start
 268
 269     # Clone the oran layer if it's not already cloned
 270     # Check if the script is inside the repo
 271     if cd ${SCRIPTS_DIR} && git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
 272         CLONED_ORAN_REPO=`dirname ${SCRIPTS_DIR}`
 273         echo_info "Use the cloned oran repo: ${CLONED_ORAN_REPO}"
 274         mkdir -p ${SRC_ORAN_DIR}/rtp
 275         cd ${SRC_ORAN_DIR}/rtp
 276         rm -rf meta-oran meta-stx scripts
 277         ln -sf ${CLONED_ORAN_REPO}/meta-oran meta-oran
 278         ln -sf ${CLONED_ORAN_REPO}/meta-stx meta-stx
 279         ln -sf ${CLONED_ORAN_REPO}/scripts scripts
 280     else
 281         echo_info "Cloning oran layer:"
 282         cd ${SRC_ORAN_DIR}
 283         clone_update_repo ${SRC_ORAN_BRANCH} ${SRC_ORAN_URL} rtp
 284     fi
 285
 286     echo_info "Cloning or update Yocto layers:"
 287
 288     cd ${SRC_LAYER_DIR}
 289     for layer_url in ${SRC_YP_URL}; do
 290         layer_name=$(basename ${layer_url})
 291         clone_update_repo ${SRC_YP_BRANCH} ${layer_url} ${layer_name}
 292     done
 293
 294     echo_step_end
 295
 296     # Apply meta patches
 297     for l in $(ls -1 ${SRC_META_PATCHES}); do
 298         msg_step="Apply meta patches for ${l}"
 299         echo_step_start
 300         cd ${SRC_LAYER_DIR}/${l}
 301
 302         # backup current branch
 303         local_branch=$(git rev-parse --abbrev-ref HEAD)
 304         git branch -m "${local_branch}_${TIMESTAMP}"
 305         git checkout ${local_branch}
 306
 307         for p in $(ls -1 ${SRC_META_PATCHES}/${l}); do
 308             echo_info "Apllying patch: ${SRC_META_PATCHES}/${l}/${p}"
 309             git am ${SRC_META_PATCHES}/${l}/${p}
 310         done
 311         echo_step_end
 312     done
 313 }
 314
 315 add_layer_stx_build () {
 316     msg_step="Add required layers to the StarlingX build project"
 317     echo_step_start
 318
 319     source_env ${PRJ_BUILD_DIR}
 320     SRC_LAYERS=""
 321     for layer_url in ${SRC_YP_URL}; do
 322         layer_name=$(basename ${layer_url})
 323         case ${layer_name} in
 324         poky)
 325             continue
 326             ;;
 327         meta-openembedded)
 328             for sub_layer in ${SUB_LAYER_META_OE}; do
 329                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 330             done
 331             ;;
 332         meta-cloud-services)
 333             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 334             for sub_layer in ${SUB_LAYER_META_CLOUD_SERVICES}; do
 335                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 336             done
 337             ;;
 338         meta-security)
 339             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 340             for sub_layer in ${SUB_LAYER_META_SECURITY}; do
 341                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 342             done
 343             ;;
 344         *)
 345             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 346             ;;
 347
 348         esac
 349     done
 350
 351     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 352
 353     for src_layer in ${SRC_LAYERS}; do
 354         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 355         echo_cmd "Add the ${src_layer} layer into the build project"
 356         ${RUN_CMD}
 357     done
 358
 359     echo_step_end
 360 }
 361
 362 add_configs_stx_build () {
 363     msg_step="Add extra configs into local.conf for StarlingX build"
 364     echo_step_start
 365
 366     cd ${PRJ_BUILD_DIR}
 367     echo_info "Adding the following extra configs into local.conf"
 368     cat ${SRC_CONFIGS}/local-stx.conf | \
 369         sed -e "s/@BSP@/${BSP}/" | tee -a conf/local.conf
 370
 371     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 372     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 373
 374     if [ "${RM_WORK}" == "Yes" ]; then
 375         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 376     fi
 377
 378
 379     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 380         cat ${EXTRA_CONF} | tee -a conf/local.conf
 381     fi
 382     echo_step_end
 383 }
 384
 385 setup_stx_build () {
 386     echo_step_start "Setup StarlingX build project"
 387
 388     add_layer_stx_build
 389
 390     cd ${PRJ_BUILD_DIR}
 391     if ! grep -q 'Configs for StarlingX' conf/local.conf; then
 392         add_configs_stx_build
 393     else
 394         echo_info "Nothing is added into local.conf"
 395     fi
 396
 397     echo_step_end "Setup StarlingX build project"
 398 }
 399
 400 build_stx_image () {
 401     msg_step="Build StarlingX images"
 402     echo_step_start
 403
 404     source_env ${PRJ_BUILD_DIR}
 405
 406     mkdir -p logs
 407
 408     RUN_CMD="bitbake ${DRYRUN} ${IMG_STX}"
 409     echo_cmd "Build the ${IMG_STX} image"
 410     bitbake ${DRYRUN} ${IMG_STX} 2>&1|tee logs/bitbake_${IMG_STX}_${TIMESTAMP}.log
 411
 412     echo_step_end
 413
 414     echo_info "Build succeeded, you can get the image in ${ISO_STX}"
 415 }
 416
 417 add_layer_anaconda_build () {
 418     msg_step="Add required layers to the anaconda (installer) build project"
 419     echo_step_start
 420
 421     source_env ${PRJ_BUILD_DIR_ANACONDA}
 422     SRC_LAYERS=""
 423     for sub_layer in ${SUB_LAYER_META_OE_ANACONDA}; do
 424         SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-openembedded/${sub_layer}"
 425     done
 426     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-intel"
 427     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-anaconda"
 428     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 429
 430     for src_layer in ${SRC_LAYERS}; do
 431         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 432         echo_cmd "Add the ${src_layer} layer into the build project"
 433         ${RUN_CMD}
 434     done
 435
 436     echo_step_end
 437 }
 438
 439 add_configs_anaconda_build () {
 440     msg_step="Add extra configs into local.conf for anaconda (installer) build"
 441     echo_step_start
 442
 443     cd ${PRJ_BUILD_DIR_ANACONDA}
 444     echo_info "Adding the following extra configs into local.conf"
 445     cat ${SRC_CONFIGS}/local-anaconda.conf | \
 446         sed -e "s/@BSP@/${BSP}/" \
 447             -e "s|@TARGET_BUILD@|${PRJ_BUILD_DIR}|" \
 448             | tee -a conf/local.conf
 449
 450     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 451     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 452
 453     if [ "${RM_WORK}" == "Yes" ]; then
 454         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 455     fi
 456
 457     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 458         cat ${EXTRA_CONF} | tee -a conf/local.conf
 459     fi
 460
 461     echo_step_end
 462 }
 463
 464 setup_anaconda_build () {
 465     echo_step_start "Setup anaconda (installer) build project"
 466
 467     add_layer_anaconda_build
 468
 469     cd ${PRJ_BUILD_DIR_ANACONDA}
 470     if ! grep -q 'Configs for anaconda' conf/local.conf; then
 471         add_configs_anaconda_build
 472     else
 473         echo_info "Nothing is added into local.conf"
 474     fi
 475
 476     echo_step_end "Setup anaconda build project"
 477 }
 478
 479 build_anaconda_image () {
 480     echo_step_start "Build anaconda (installer) images"
 481     source_env ${PRJ_BUILD_DIR_ANACONDA}
 482
 483     mkdir -p logs
 484
 485     if [ -f ${ISO_ANACONDA} ]; then
 486         bitbake ${DRYRUN} -c clean ${IMG_ANACONDA}
 487     fi
 488     RUN_CMD="bitbake ${DRYRUN} ${IMG_ANACONDA}"
 489     echo_cmd "Build the ${IMG_ANACONDA} image"
 490     bitbake ${DRYRUN} ${IMG_ANACONDA} 2>&1|tee logs/bitbake_${IMG_ANACONDA}_${TIMESTAMP}.log
 491
 492     echo_step_end
 493
 494     echo_info "Build succeeded, you can get the image in ${ISO_ANACONDA}"
 495 }
 496
 497 #########################################################################
 498 # Main process
 499 #########################################################################
 500
 501 prepare_workspace
 502 prepare_src
 503 setup_stx_build
 504 setup_anaconda_build
 505 build_stx_image
 506 build_anaconda_image