scripts/build_stx.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2019 Wind River Systems, Inc.
   4 #
   5 #  Licensed under the Apache License, Version 2.0 (the "License");
   6 #  you may not use this file except in compliance with the License.
   7 #  You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #  Unless required by applicable law or agreed to in writing, software
  12 #  distributed under the License is distributed on an "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #  See the License for the specific language governing permissions and
  15 #  limitations under the License.
  16
  17 # Ensure we fail the job if any steps fail.
  18 set -e -o pipefail
  19
  20 #########################################################################
  21 # Variables
  22 #########################################################################
  23
  24 # Only one bsp is supported now, there will be more in the future
  25 SUPPORTED_BSP="intel-corei7-64 "
  26
  27 SRC_ORAN_BRANCH="master"
  28 SRC_STX_BRANCH="master"
  29 SRC_YP_BRANCH="warrior"
  30
  31 SRC_ORAN_URL="https://gerrit.o-ran-sc.org/r/pti/rtp"
  32
  33 SRC_YP_URL="\
  34     git://git.yoctoproject.org/poky \
  35     git://git.openembedded.org/meta-openembedded \
  36     git://git.yoctoproject.org/meta-virtualization \
  37     git://git.yoctoproject.org/meta-cloud-services \
  38     git://git.yoctoproject.org/meta-security \
  39     git://git.yoctoproject.org/meta-intel \
  40     git://git.yoctoproject.org/meta-security \
  41     git://git.yoctoproject.org/meta-selinux \
  42     https://github.com/intel-iot-devkit/meta-iot-cloud \
  43     git://git.openembedded.org/meta-python2 \
  44     https://git.yoctoproject.org/git/meta-dpdk \
  45     git://git.yoctoproject.org/meta-anaconda \
  46 "
  47
  48 SUB_LAYER_META_OE="\
  49     meta-oe \
  50     meta-perl \
  51     meta-python \
  52     meta-networking \
  53     meta-filesystems \
  54     meta-webserver \
  55     meta-initramfs \
  56     meta-initramfs \
  57     meta-gnome \
  58 "
  59
  60 SUB_LAYER_META_CLOUD_SERVICES="meta-openstack"
  61 SUB_LAYER_META_SECURITY="meta-security-compliance"
  62
  63 # For anaconda build
  64 SUB_LAYER_META_OE_ANACONDA="\
  65     meta-oe \
  66     meta-python \
  67     meta-filesystems \
  68     meta-initramfs \
  69     meta-networking \
  70     meta-gnome \
  71 "
  72
  73 SCRIPTS_DIR=$(dirname $(readlink -f $0))
  74 TIMESTAMP=`date +"%Y%m%d_%H%M%S"`
  75
  76 #########################################################################
  77 # Common Functions
  78 #########################################################################
  79
  80 help_info () {
  81 cat << ENDHELP
  82 Usage:
  83 $(basename $0) [-w WORKSPACE_DIR] [-b BSP] [-n] [-h] [-r Yes|No] [-s] [-e EXTRA_CONF]
  84 where:
  85     -w WORKSPACE_DIR is the path for the project
  86     -b BPS is one of supported BSP: "${SUPPORTED_BSP}"
  87        (default is intel-corei7-64 if not specified.)
  88     -n dry-run only for bitbake
  89     -h this help info
  90     -e EXTRA_CONF is the pat for extra config file
  91     -r whether to inherit rm_work (default is Yes)
  92     -s whether to skip update the repo if already exists
  93 examples:
  94 ./build_stx.sh
  95 ./build_stx.sh -w workspace_1234 -r no -e /path/to/extra_local.conf
  96 ENDHELP
  97 }
  98
  99 echo_step_start() {
 100     [ -n "$1" ] && msg_step=$1
 101     echo "#########################################################################################"
 102     echo "## STEP START: ${msg_step}"
 103     echo "#########################################################################################"
 104 }
 105
 106 echo_step_end() {
 107     [ -n "$1" ] && msg_step=$1
 108     echo "#########################################################################################"
 109     echo "## STEP END: ${msg_step}"
 110     echo "#########################################################################################"
 111     echo
 112 }
 113
 114 echo_info () {
 115     echo "INFO: $1"
 116 }
 117
 118 echo_error () {
 119     echo "ERROR: $1"
 120 }
 121
 122 echo_cmd () {
 123     echo
 124     echo_info "$1"
 125     echo "CMD: ${RUN_CMD}"
 126 }
 127
 128 check_yn_rm_work () {
 129     yn="$1"
 130     case ${yn} in
 131         [Yy]|[Yy]es)
 132             RM_WORK="Yes"
 133             ;;
 134         [Nn]|[Nn]o)
 135             RM_WORK="No"
 136             ;;
 137         *)
 138             echo "Invalid arg for -r option."
 139             help_info
 140             exit 1
 141             ;;
 142     esac
 143 }
 144
 145 check_valid_bsp () {
 146     bsp="$1"
 147     for b in ${SUPPORTED_BSP}; do
 148         if [ "${bsp}" == "${b}" ]; then
 149             BSP_VALID="${bsp}"
 150             break
 151         fi
 152     done
 153     if [ -z "${BSP_VALID}" ]; then
 154         echo_error "${bsp} is not a supported BSP, the supported BSPs are: ${SUPPORTED_BSP}"
 155         exit 1
 156     fi
 157 }
 158
 159
 160 clone_update_repo () {
 161     REPO_BRANCH=$1
 162     REPO_URL=$2
 163     REPO_NAME=$3
 164
 165     if [ -d ${REPO_NAME}/.git ]; then
 166         if [ "${SKIP_UPDATE}" == "Yes" ]; then
 167             echo_info "The repo ${REPO_NAME} exists, skip updating for the branch ${REPO_BRANCH}"
 168         else
 169             echo_info "The repo ${REPO_NAME} exists, updating for the branch ${REPO_BRANCH}"
 170             cd ${REPO_NAME}
 171             git checkout ${REPO_BRANCH}
 172             git pull
 173             cd -
 174         fi
 175     else
 176         RUN_CMD="git clone --branch ${REPO_BRANCH} ${REPO_URL} ${REPO_NAME}"
 177         echo_cmd "Cloning the source of repo '${REPO_NAME}':"
 178         ${RUN_CMD}
 179     fi
 180 }
 181
 182 source_env () {
 183     build_dir=$1
 184     cd ${SRC_LAYER_DIR}/poky
 185     set ${build_dir}
 186     source ./oe-init-build-env ${build_dir}
 187 }
 188
 189 #########################################################################
 190 # Parse cmd options
 191 #########################################################################
 192
 193 DRYRUN=""
 194 EXTRA_CONF=""
 195 SKIP_UPDATE="No"
 196 RM_WORK="Yes"
 197 BSP="intel-corei7-64"
 198
 199 while getopts "w:b:e:r:nsh" OPTION; do
 200     case ${OPTION} in
 201         w)
 202             WORKSPACE=`readlink -f ${OPTARG}`
 203             ;;
 204         b)
 205             check_valid_bsp ${OPTARG}
 206             ;;
 207         e)
 208             EXTRA_CONF=`readlink -f ${OPTARG}`
 209             ;;
 210         n)
 211             DRYRUN="-n"
 212             ;;
 213         s)
 214             SKIP_UPDATE="Yes"
 215             ;;
 216         r)
 217             check_yn_rm_work ${OPTARG}
 218             ;;
 219         h)
 220             help_info
 221             exit
 222             ;;
 223     esac
 224 done
 225
 226 if [ -z ${WORKSPACE} ]; then
 227     echo_info "No workspace specified, a directory 'workspace' will be created in current directory as the workspace"
 228     WORKSPACE=`readlink -f workspace`
 229 fi
 230
 231 if [ -n "${BSP_VALID}" ]; then
 232     BSP="${BSP_VALID}"
 233 fi
 234
 235 #########################################################################
 236 # Functions for each step
 237 #########################################################################
 238 SRC_LAYER_DIR=${WORKSPACE}/src_layers
 239 SRC_ORAN_DIR=${SRC_LAYER_DIR}/oran
 240 PRJ_BUILD_DIR=${WORKSPACE}/prj_oran_stx
 241 PRJ_BUILD_DIR_ANACONDA=${WORKSPACE}/prj_oran_stx_anaconda
 242 PRJ_SHARED_DIR=${WORKSPACE}/prj_shared
 243 PRJ_SHARED_DL_DIR=${WORKSPACE}/prj_shared/downloads
 244 PRJ_SHARED_SS_DIR=${WORKSPACE}/prj_shared/sstate-cache
 245 SRC_META_PATCHES=${SRC_ORAN_DIR}/rtp/scripts/meta-patches/src_stx
 246 SRC_CONFIGS=${SRC_ORAN_DIR}/rtp/scripts/configs
 247 IMG_STX=stx-image-aio
 248 IMG_ANACONDA=stx-image-aio-installer
 249 ISO_STX=${PRJ_BUILD_DIR}/tmp/deploy/images/${BSP}/${IMG_STX}-${BSP}.iso
 250 ISO_ANACONDA=${PRJ_BUILD_DIR_ANACONDA}/tmp-glibc/deploy/images/${BSP}/${IMG_ANACONDA}-${BSP}.iso
 251
 252 prepare_workspace () {
 253     msg_step="Create workspace for the build"
 254     echo_step_start
 255
 256     mkdir -p ${PRJ_BUILD_DIR} ${SRC_ORAN_DIR} ${PRJ_BUILD_DIR_ANACONDA} ${PRJ_SHARED_DL_DIR} ${PRJ_SHARED_SS_DIR}
 257
 258     echo_info "The following directories are created in your workspace(${WORKSPACE}):"
 259     echo_info "For all layers source: ${SRC_LAYER_DIR}"
 260     echo_info "For StarlingX build project: ${PRJ_BUILD_DIR}"
 261     echo_info "For anaconda (installer) build project: ${PRJ_BUILD_DIR_ANACONDA}"
 262
 263     echo_step_end
 264 }
 265
 266 prepare_src () {
 267     msg_step="Get the source code repos"
 268     echo_step_start
 269
 270     # Clone the oran layer if it's not already cloned
 271     # Check if the script is inside the repo
 272     if cd ${SCRIPTS_DIR} && git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
 273         CLONED_ORAN_REPO=`dirname ${SCRIPTS_DIR}`
 274         echo_info "Use the cloned oran repo: ${CLONED_ORAN_REPO}"
 275         mkdir -p ${SRC_ORAN_DIR}/rtp
 276         cd ${SRC_ORAN_DIR}/rtp
 277         rm -rf meta-oran meta-stx scripts
 278         ln -sf ${CLONED_ORAN_REPO}/meta-oran meta-oran
 279         ln -sf ${CLONED_ORAN_REPO}/meta-stx meta-stx
 280         ln -sf ${CLONED_ORAN_REPO}/scripts scripts
 281     else
 282         echo_info "Cloning oran layer:"
 283         cd ${SRC_ORAN_DIR}
 284         clone_update_repo ${SRC_ORAN_BRANCH} ${SRC_ORAN_URL} rtp
 285     fi
 286
 287     echo_info "Cloning or update Yocto layers:"
 288
 289     cd ${SRC_LAYER_DIR}
 290     for layer_url in ${SRC_YP_URL}; do
 291         layer_name=$(basename ${layer_url})
 292         clone_update_repo ${SRC_YP_BRANCH} ${layer_url} ${layer_name}
 293     done
 294
 295     echo_step_end
 296
 297     # Apply meta patches
 298     for l in $(ls -1 ${SRC_META_PATCHES}); do
 299         msg_step="Apply meta patches for ${l}"
 300         echo_step_start
 301         cd ${SRC_LAYER_DIR}/${l}
 302
 303         # backup current branch
 304         local_branch=$(git rev-parse --abbrev-ref HEAD)
 305         git branch -m "${local_branch}_${TIMESTAMP}"
 306         git checkout ${local_branch}
 307
 308         for p in $(ls -1 ${SRC_META_PATCHES}/${l}); do
 309             echo_info "Apllying patch: ${SRC_META_PATCHES}/${l}/${p}"
 310             git am ${SRC_META_PATCHES}/${l}/${p}
 311         done
 312         echo_step_end
 313     done
 314 }
 315
 316 add_layer_stx_build () {
 317     msg_step="Add required layers to the StarlingX build project"
 318     echo_step_start
 319
 320     source_env ${PRJ_BUILD_DIR}
 321     SRC_LAYERS=""
 322     for layer_url in ${SRC_YP_URL}; do
 323         layer_name=$(basename ${layer_url})
 324         case ${layer_name} in
 325         poky)
 326             continue
 327             ;;
 328         meta-openembedded)
 329             for sub_layer in ${SUB_LAYER_META_OE}; do
 330                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 331             done
 332             ;;
 333         meta-cloud-services)
 334             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 335             for sub_layer in ${SUB_LAYER_META_CLOUD_SERVICES}; do
 336                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 337             done
 338             ;;
 339         meta-security)
 340             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 341             for sub_layer in ${SUB_LAYER_META_SECURITY}; do
 342                 SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}/${sub_layer}"
 343             done
 344             ;;
 345         *)
 346             SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/${layer_name}"
 347             ;;
 348
 349         esac
 350     done
 351
 352     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 353
 354     for src_layer in ${SRC_LAYERS}; do
 355         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 356         echo_cmd "Add the ${src_layer} layer into the build project"
 357         ${RUN_CMD}
 358     done
 359
 360     echo_step_end
 361 }
 362
 363 add_configs_stx_build () {
 364     msg_step="Add extra configs into local.conf for StarlingX build"
 365     echo_step_start
 366
 367     cd ${PRJ_BUILD_DIR}
 368     echo_info "Adding the following extra configs into local.conf"
 369     cat ${SRC_CONFIGS}/local-stx.conf | \
 370         sed -e "s/@BSP@/${BSP}/" | tee -a conf/local.conf
 371
 372     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 373     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 374
 375     if [ "${RM_WORK}" == "Yes" ]; then
 376         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 377     fi
 378
 379
 380     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 381         cat ${EXTRA_CONF} | tee -a conf/local.conf
 382     fi
 383     echo_step_end
 384 }
 385
 386 setup_stx_build () {
 387     echo_step_start "Setup StarlingX build project"
 388
 389     add_layer_stx_build
 390
 391     cd ${PRJ_BUILD_DIR}
 392     if ! grep -q 'Configs for StarlingX' conf/local.conf; then
 393         add_configs_stx_build
 394     else
 395         echo_info "Nothing is added into local.conf"
 396     fi
 397
 398     echo_step_end "Setup StarlingX build project"
 399 }
 400
 401 build_stx_image () {
 402     msg_step="Build StarlingX images"
 403     echo_step_start
 404
 405     source_env ${PRJ_BUILD_DIR}
 406
 407     mkdir -p logs
 408
 409     RUN_CMD="bitbake ${DRYRUN} ${IMG_STX}"
 410     echo_cmd "Build the ${IMG_STX} image"
 411     bitbake ${DRYRUN} ${IMG_STX} 2>&1|tee logs/bitbake_${IMG_STX}_${TIMESTAMP}.log
 412
 413     echo_step_end
 414
 415     echo_info "Build succeeded, you can get the image in ${ISO_STX}"
 416 }
 417
 418 add_layer_anaconda_build () {
 419     msg_step="Add required layers to the anaconda (installer) build project"
 420     echo_step_start
 421
 422     source_env ${PRJ_BUILD_DIR_ANACONDA}
 423     SRC_LAYERS=""
 424     for sub_layer in ${SUB_LAYER_META_OE_ANACONDA}; do
 425         SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-openembedded/${sub_layer}"
 426     done
 427     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-intel"
 428     SRC_LAYERS="${SRC_LAYERS} ${SRC_LAYER_DIR}/meta-anaconda"
 429     SRC_LAYERS="${SRC_LAYERS} ${SRC_ORAN_DIR}/rtp/meta-stx"
 430
 431     for src_layer in ${SRC_LAYERS}; do
 432         RUN_CMD="bitbake-layers add-layer ${src_layer}"
 433         echo_cmd "Add the ${src_layer} layer into the build project"
 434         ${RUN_CMD}
 435     done
 436
 437     echo_step_end
 438 }
 439
 440 add_configs_anaconda_build () {
 441     msg_step="Add extra configs into local.conf for anaconda (installer) build"
 442     echo_step_start
 443
 444     cd ${PRJ_BUILD_DIR_ANACONDA}
 445     echo_info "Adding the following extra configs into local.conf"
 446     cat ${SRC_CONFIGS}/local-anaconda.conf | \
 447         sed -e "s/@BSP@/${BSP}/" \
 448             -e "s|@TARGET_BUILD@|${PRJ_BUILD_DIR}|" \
 449             | tee -a conf/local.conf
 450
 451     echo "DL_DIR = '${PRJ_SHARED_DL_DIR}'" | tee -a conf/local.conf
 452     echo "SSTATE_DIR = '${PRJ_SHARED_SS_DIR}'" | tee -a conf/local.conf
 453
 454     if [ "${RM_WORK}" == "Yes" ]; then
 455         echo "INHERIT += 'rm_work'" | tee -a conf/local.conf
 456     fi
 457
 458     if [ "${EXTRA_CONF}" != "" ] && [ -f "${EXTRA_CONF}" ]; then
 459         cat ${EXTRA_CONF} | tee -a conf/local.conf
 460     fi
 461
 462     echo_step_end
 463 }
 464
 465 setup_anaconda_build () {
 466     echo_step_start "Setup anaconda (installer) build project"
 467
 468     add_layer_anaconda_build
 469
 470     cd ${PRJ_BUILD_DIR_ANACONDA}
 471     if ! grep -q 'Configs for anaconda' conf/local.conf; then
 472         add_configs_anaconda_build
 473     else
 474         echo_info "Nothing is added into local.conf"
 475     fi
 476
 477     echo_step_end "Setup anaconda build project"
 478 }
 479
 480 build_anaconda_image () {
 481     echo_step_start "Build anaconda (installer) images"
 482     source_env ${PRJ_BUILD_DIR_ANACONDA}
 483
 484     mkdir -p logs
 485
 486     if [ -f ${ISO_ANACONDA} ]; then
 487         bitbake ${DRYRUN} -c clean ${IMG_ANACONDA}
 488     fi
 489     RUN_CMD="bitbake ${DRYRUN} ${IMG_ANACONDA}"
 490     echo_cmd "Build the ${IMG_ANACONDA} image"
 491     bitbake ${DRYRUN} ${IMG_ANACONDA} 2>&1|tee logs/bitbake_${IMG_ANACONDA}_${TIMESTAMP}.log
 492
 493     echo_step_end
 494
 495     echo_info "Build succeeded, you can get the image in ${ISO_ANACONDA}"
 496 }
 497
 498 #########################################################################
 499 # Main process
 500 #########################################################################
 501
 502 prepare_workspace
 503 prepare_src
 504 setup_stx_build
 505 setup_anaconda_build
 506 build_stx_image
 507 build_anaconda_image