From ffd5888d23fca4a950a9cfb97c697a8929d9ce53 Mon Sep 17 00:00:00 2001 From: Jackie Huang Date: Tue, 11 Oct 2022 13:58:36 +0800 Subject: [PATCH] build_inf_prepare_jenkins.sh: split steps Issue-ID: INF-284 Signed-off-by: Jackie Huang Change-Id: Icf78669fa0ab39333a62bbffe1a1c604296aa2dd --- scripts/build_inf_centos/build_inf_centos.sh | 27 +- .../build_inf_centos/build_inf_prepare_jenkins.sh | 510 +++++++++++---------- 2 files changed, 296 insertions(+), 241 deletions(-) diff --git a/scripts/build_inf_centos/build_inf_centos.sh b/scripts/build_inf_centos/build_inf_centos.sh index 399157b..8a44fc6 100755 --- a/scripts/build_inf_centos/build_inf_centos.sh +++ b/scripts/build_inf_centos/build_inf_centos.sh @@ -227,6 +227,9 @@ repo_init_sync () { msg_step="Init the repo and sync" echo_step_start + # Avoid the colorization prompt + git config --global color.ui false + cd ${MY_REPO_ROOT_DIR} STX_MANIFEST="default.xml" if [ "$LAYER" != "" ]; then @@ -246,15 +249,19 @@ get_mirror_src () { msg_step="Get src mirror from dockerhub image" echo_step_start - docker pull ${MIRROR_SRC_STX} - docker create -ti --name inf-src-stx ${MIRROR_SRC_STX} sh - docker cp inf-src-stx:/stx-${STX_VER}.tar.bz2 ${MY_REPO_ROOT_DIR} - docker rm inf-src-stx - - cd ${MY_REPO_ROOT_DIR} - tar xf stx-${STX_VER}.tar.bz2 - mv stx-${STX_VER}/* stx-${STX_VER}/.repo . - rm -rf stx-${STX_VER} stx-${STX_VER}.tar.bz2 + if [ -d ${MY_REPO_ROOT_DIR}/.repo ]; then + echo_info "The src repos already exists, skipping" + else + docker pull ${MIRROR_SRC_STX} + docker create -ti --name inf-src-stx ${MIRROR_SRC_STX} sh + docker cp inf-src-stx:/stx-${STX_VER}.tar.bz2 ${MY_REPO_ROOT_DIR} + docker rm inf-src-stx + + cd ${MY_REPO_ROOT_DIR} + tar xf stx-${STX_VER}.tar.bz2 + mv stx-${STX_VER}/* stx-${STX_VER}/.repo . + rm -rf stx-${STX_VER} stx-${STX_VER}.tar.bz2 + fi echo_step_end } @@ -265,7 +272,7 @@ get_mirror_pkg () { docker pull ${MIRROR_CONTAINER_IMG} docker create -ti --name inf-centos-mirror ${MIRROR_CONTAINER_IMG} sh - docker cp inf-centos-mirror:/mirror_stx-${STX_VER} ${STX_MIRROR_DIR} + docker cp inf-centos-mirror:/mirror_stx-${STX_VER}/. ${STX_MIRROR_DIR} docker rm inf-centos-mirror echo_step_end diff --git a/scripts/build_inf_centos/build_inf_prepare_jenkins.sh b/scripts/build_inf_centos/build_inf_prepare_jenkins.sh index a9b7b91..d3f97e6 100755 --- a/scripts/build_inf_centos/build_inf_prepare_jenkins.sh +++ b/scripts/build_inf_centos/build_inf_prepare_jenkins.sh @@ -22,11 +22,16 @@ set -e -o pipefail # Variables ######################################################################### WORKSPACE="" -MIRROR_VER=stx-6.0 -MIRROR_CONTAINER_IMG=infbuilder/inf-centos-mirror:2022.05-${MIRROR_VER} +SCRIPTS_DIR=$(dirname $(readlink -f $0)) SCRIPTS_NAME=$(basename $0) +TOOLS_DIR="${SCRIPTS_DIR}/tools" +STEP_STATUS_DIR="${SCRIPTS_DIR}/step_status" + +BUILD_GROUP="jenkins" +WGET_OPENDEV="wget --no-check-certificate" + ######################################################################### # Common Functions ######################################################################### @@ -75,246 +80,289 @@ while getopts "w:h" OPTION; do esac done -get_mirror () { - msg_step="Get rpm mirror from dockerhub image" +######################################################################### +# Functions +######################################################################### +LOCALDISK="${WORKSPACE}/localdisk" +MIRROR_DIR="${WORKSPACE}/mirror" + +prepare_env () { + msg_step="Prepare build directories" + echo_step_start + + mkdir -p ${LOCALDISK}/loadbuild/mock-cache + mkdir -p ${LOCALDISK}/loadbuild/mock + mkdir -p ${LOCALDISK}/designer + mkdir -p ${LOCALDISK}/loadbuild + mkdir -p ${STEP_STATUS_DIR} + + sudo chmod 775 ${LOCALDISK}/loadbuild/mock + sudo chown root:mock ${LOCALDISK}/loadbuild/mock + sudo chmod 775 ${LOCALDISK}/loadbuild/mock-cache + sudo chown root:mock ${LOCALDISK}/loadbuild/mock-cache + + TMP_OPT_DIR=${SCRIPTS_DIR}/opt + mkdir -p ${TMP_OPT_DIR} + + echo_step_end +} + +install_pkgs () { + msg_step="Install/downlaod/config required dependencies by mirror/build processes." echo_step_start - docker pull ${MIRROR_CONTAINER_IMG} - docker create -ti --name inf-centos-mirror ${MIRROR_CONTAINER_IMG} sh - docker cp inf-centos-mirror:/mirror_${MIRROR_VER} ${MIRROR_DIR} - docker rm inf-centos-mirror + echo_info "Install required packages" + sudo yum install -y \ + anaconda \ + anaconda-runtime \ + autoconf-archive \ + autogen \ + automake \ + bc \ + bind \ + bind-utils \ + bison \ + cpanminus \ + createrepo \ + createrepo_c \ + deltarpm \ + expat-devel \ + flex \ + isomd5sum \ + gcc \ + gettext \ + libguestfs-tools \ + libtool \ + libxml2 \ + lighttpd \ + lighttpd-fastcgi \ + lighttpd-mod_geoip \ + net-tools \ + mkisofs \ + mongodb \ + mongodb-server \ + pax \ + perl-CPAN \ + python-deltarpm \ + python-pep8 \ + python-pip \ + python-psutil \ + python2-psutil \ + python36-psutil \ + python36-requests \ + python3-devel \ + python-sphinx \ + python-subunit \ + python-virtualenv \ + python-yaml \ + python2-ruamel-yaml \ + postgresql \ + qemu-kvm \ + quilt \ + rpm-build \ + rpm-sign \ + rpm-python \ + squashfs-tools \ + sudo \ + systemd \ + syslinux \ + udisks2 \ + vim-enhanced \ + wget + + echo_info "Install required cpan modules" + # cpan modules, installing with cpanminus to avoid stupid questions since cpan is whack + sudo cpanm --notest Fatal + sudo cpanm --notest XML::SAX + sudo cpanm --notest XML::SAX::Expat + sudo cpanm --notest XML::Parser + sudo cpanm --notest XML::Simple + + echo_info "Install repo tool" + sudo wget https://storage.googleapis.com/git-repo-downloads/repo -O /usr/local/bin/repo + sudo chmod a+x /usr/local/bin/repo + + echo_info "Install go and setting paths" + export GOPATH="/usr/local/go" + export PATH="${GOPATH}/bin:${PATH}" + sudo yum install -y golang + sudo mkdir -p ${GOPATH}/bin + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sudo -E sh + + echo_info "Install pip packages" + # Install required python modules globally; versions are in the constraints file. + # Be careful not to replace modules provided by RPMs as it may break + # other system packages. Look for warnings similar to "Uninstalling a + # distutils installed project has been deprecated" from pip. + sudo pip install -c ${TOOLS_DIR}/toCOPY/builder-constraints.txt \ + testrepository \ + fixtures \ + pbr \ + git-review \ + python-subunit \ + junitxml \ + testtools + + # This image requires a set of scripts and helpers + # for working correctly, in this section they are + # copied inside the image. + sudo cp ${TOOLS_DIR}/toCOPY/finishSetup.sh /usr/local/bin + sudo cp ${TOOLS_DIR}/toCOPY/populate_downloads.sh /usr/local/bin + sudo cp ${TOOLS_DIR}/toCOPY/generate-local-repo.sh /usr/local/bin + sudo cp ${TOOLS_DIR}/toCOPY/generate-centos-repo.sh /usr/local/bin + sudo cp ${TOOLS_DIR}/toCOPY/lst_utils.sh /usr/local/bin + + sudo chmod a+x /usr/local/bin/* echo_step_end } +get_tools_repo () { + msg_step="Clone the tools repo" + echo_step_start + + cd ${SCRIPTS_DIR} + if [ -d tools/.git ]; then + echo_info "The 'tools' repo already exists, skipping" + else + git clone https://opendev.org/starlingx/tools.git + fi + + echo_step_end +} + +config_mock () { + msg_step="mock custumizations" + echo_step_start + # forcing chroots since a couple of packages naughtily insist on network access and + # we dont have nspawn and networks happy together. + sudo groupadd -f -g 9001 mockbuild + sudo useradd -s /sbin/nologin -u 9001 -g 9001 mockbuild || echo "User mockbuild already exists." + [ -L /var/lib/mock ] || { + sudo rmdir /var/lib/mock + sudo ln -s ${LOCALDISK}/loadbuild/mock /var/lib/mock + } + + [ -L /var/cache/mock ] || { + sudo mv /var/cache/mock/* ${LOCALDISK}/loadbuild/mock-cache/ + sudo rmdir /var/cache/mock + sudo ln -s ${LOCALDISK}/loadbuild/mock-cache /var/cache/mock + } + echo "config_opts['use_nspawn'] = False" | sudo tee -a /etc/mock/site-defaults.cfg + echo "config_opts['rpmbuild_networking'] = True" | sudo tee -a /etc/mock/site-defaults.cfg + echo | sudo tee -a /etc/mock/site-defaults.cfg + + echo_step_end +} + +install_mock_overlay () { + msg_step="Inherited tools for mock stuff" + echo_step_start + + # we at least need the mock_cache_unlock tool + # they install into /usr/bin + cp -rf ${TOOLS_DIR}/toCOPY/mock_overlay ${TMP_OPT_DIR}/mock_overlay + cd ${TMP_OPT_DIR}/mock_overlay + make + sudo make install + + echo_step_end +} + +config_yum () { + msg_step="configs for yum and rpm" + echo_step_start + + # Customizations for mirror creation + sudo rm -f /etc/yum.repos.d/* + sudo cp -f ${TOOLS_DIR}/centos-mirror-tools/yum.repos.d/* /etc/yum.repos.d/ + sudo cp -f ${TOOLS_DIR}/centos-mirror-tools/rpm-gpg-keys/* /etc/pki/rpm-gpg/ + + # Import GPG keys + sudo rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY* + + # Try to continue a yum command even if a StarlingX repo is unavailable. + sudo yum-config-manager --setopt=StarlingX\*.skip_if_unavailable=1 --save + + echo_step_end +} + +create_py27_venv () { + msg_step="Create a sane py27 virtualenv" + echo_step_start + + virtualenv ${TMP_OPT_DIR}/py27 + source ${TMP_OPT_DIR}/py27/bin/activate + pip install -c ${TOOLS_DIR}/toCOPY/builder-opt-py27-constraints.txt tox + sudo ln -s ${TMP_OPT_DIR}/py27/bin/tox /usr/bin + + echo_step_end +} + +setup_lighttpd () { + msg_step="Setup for lighttpd" + echo_step_start + + sudo mkdir -p /www + sudo chown ${USER}:${BUILD_GROUP} /www + mkdir -p /www/run + mkdir -p /www/logs + mkdir -p /www/home + mkdir -p /www/root/htdocs/localdisk + mkdir -p /www/root/htdocs/$(dirname ${WORKSPACE}) + ln -s ${LOCALDISK}/loadbuild /www/root/htdocs/localdisk/loadbuild + ln -s ${LOCALDISK}/designer /www/root/htdocs/localdisk/designer + ln -s ${MIRROR_DIR}/CentOS /www/root/htdocs/CentOS + ln -s ${WORKSPACE} /www/root/htdocs/${WORKSPACE} + + # lighthttpd setup + # chmod for /var/log/lighttpd fixes a centos issue + # in place sed for server root since it's expanded soon thereafter + # echo "server.bind = \"localhost\"" >> /etc/lighttpd/lighttpd.conf && \ + sudo mkdir -p /var/log/lighttpd + sudo chmod a+rwx /var/log/lighttpd/ + sudo sed -i -e 's%^var\.log_root.*$%var.log_root = "/www/logs"%g' \ + -e 's%^var\.server_root.*$%var.server_root = "/www/root"%g' \ + -e 's%^var\.home_dir.*$%var.home_dir = "/www/home"%g' \ + -e 's%^var\.state_dir.*$%var.state_dir = "/www/run"%g' \ + -e "s/server.port/#server.port/g" \ + -e "s/server.use-ipv6/#server.use-ipv6/g" \ + -e "s/server.username/#server.username/g" \ + -e "s/server.groupname/#server.groupname/g" \ + -e "s/server.bind/#server.bind/g" \ + -e "s/server.document-root/#server.document-root/g" \ + -e "s/server.dirlisting/#server.dirlisting/g" \ + /etc/lighttpd/lighttpd.conf + + echo "server.port = 8088" | sudo tee -a /etc/lighttpd/lighttpd.conf + echo "server.use-ipv6 = \"disable\"" | sudo tee -a /etc/lighttpd/lighttpd.conf + echo "server.username = \"$USER\"" | sudo tee -a /etc/lighttpd/lighttpd.conf + echo "server.groupname = \"$BUILD_GROUP\"" | sudo tee -a /etc/lighttpd/lighttpd.conf + echo "server.bind = \"localhost\"" | sudo tee -a /etc/lighttpd/lighttpd.conf + echo "server.document-root = \"/www/root/htdocs\"" | sudo tee -a /etc/lighttpd/lighttpd.conf + + sudo sed -i "s/dir-listing.activate/#dir-listing.activate/g" \ + /etc/lighttpd/conf.d/dirlisting.conf + + echo "dir-listing.activate = \"enable\"" | sudo tee -a /etc/lighttpd/conf.d/dirlisting.conf + + sudo /usr/sbin/lighttpd -f /etc/lighttpd/lighttpd.conf + + echo_step_end +} ######################################################################### # Main process ######################################################################### -msg_step="Prepare build directories" -echo_step_start - -export BUILD_GROUP="jenkins" -export WGET_OPENDEV="wget --no-check-certificate" -export LOCALDISK="${WORKSPACE}/localdisk" -export MIRROR_DIR="${WORKSPACE}/mirror" -export TOOLS_DIR="${WORKSPACE}/tools" - -mkdir -p ${LOCALDISK}/loadbuild/mock-cache -mkdir -p ${LOCALDISK}/loadbuild/mock -mkdir -p ${LOCALDISK}/designer -mkdir -p ${LOCALDISK}/loadbuild - -sudo chmod 775 ${LOCALDISK}/loadbuild/mock -sudo chown root:mock ${LOCALDISK}/loadbuild/mock -sudo chmod 775 ${LOCALDISK}/loadbuild/mock-cache -sudo chown root:mock ${LOCALDISK}/loadbuild/mock-cache - -echo_step_end - -msg_step="Install/downlaod/config required dependencies by mirror/build processes." -echo_step_start - -echo_info "Install required packages" -sudo yum install -y \ - anaconda \ - anaconda-runtime \ - autoconf-archive \ - autogen \ - automake \ - bc \ - bind \ - bind-utils \ - bison \ - cpanminus \ - createrepo \ - createrepo_c \ - deltarpm \ - expat-devel \ - flex \ - isomd5sum \ - gcc \ - gettext \ - libguestfs-tools \ - libtool \ - libxml2 \ - lighttpd \ - lighttpd-fastcgi \ - lighttpd-mod_geoip \ - net-tools \ - mkisofs \ - mongodb \ - mongodb-server \ - pax \ - perl-CPAN \ - python-deltarpm \ - python-pep8 \ - python-pip \ - python-psutil \ - python2-psutil \ - python36-psutil \ - python36-requests \ - python3-devel \ - python-sphinx \ - python-subunit \ - python-virtualenv \ - python-yaml \ - python2-ruamel-yaml \ - postgresql \ - qemu-kvm \ - quilt \ - rpm-build \ - rpm-sign \ - rpm-python \ - squashfs-tools \ - sudo \ - systemd \ - syslinux \ - udisks2 \ - vim-enhanced \ - wget - - - -echo_info "Clone the tools repo" -cd ${WORKSPACE} -git clone https://opendev.org/starlingx/tools.git - -echo_info "mock custumizations" -# forcing chroots since a couple of packages naughtily insist on network access and -# we dont have nspawn and networks happy together. -set -x -sudo groupadd -g 9001 mockbuild -sudo useradd -s /sbin/nologin -u 9001 -g 9001 mockbuild -sudo rmdir /var/lib/mock -sudo ln -s ${LOCALDISK}/loadbuild/mock /var/lib/mock -sudo mv /var/cache/mock/* ${LOCALDISK}/loadbuild/mock-cache/ -sudo rmdir /var/cache/mock -sudo ln -s ${LOCALDISK}/loadbuild/mock-cache /var/cache/mock -echo "config_opts['use_nspawn'] = False" | sudo tee -a /etc/mock/site-defaults.cfg -echo "config_opts['rpmbuild_networking'] = True" | sudo tee -a /etc/mock/site-defaults.cfg -echo | sudo tee -a /etc/mock/site-defaults.cfg -set +x - -echo_info "Install required cpan modules" -# cpan modules, installing with cpanminus to avoid stupid questions since cpan is whack -sudo cpanm --notest Fatal -sudo cpanm --notest XML::SAX -sudo cpanm --notest XML::SAX::Expat -sudo cpanm --notest XML::Parser -sudo cpanm --notest XML::Simple - -echo_info "Install repo tool" -sudo wget https://storage.googleapis.com/git-repo-downloads/repo -O /usr/local/bin/repo -sudo chmod a+x /usr/local/bin/repo - -echo_info "Install go and setting paths" -export GOPATH="/usr/local/go" -export PATH="${GOPATH}/bin:${PATH}" -sudo yum install -y golang -sudo mkdir -p ${GOPATH}/bin -curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sudo -E sh - -echo_info "Install pip packages" -# Install required python modules globally; versions are in the constraints file. -# Be careful not to replace modules provided by RPMs as it may break -# other system packages. Look for warnings similar to "Uninstalling a -# distutils installed project has been deprecated" from pip. -sudo pip install -c ${TOOLS_DIR}/toCOPY/builder-constraints.txt \ - testrepository \ - fixtures \ - pbr \ - git-review \ - python-subunit \ - junitxml \ - testtools - - -echo_info "Inherited tools for mock stuff" -# we at least need the mock_cache_unlock tool -# they install into /usr/bin -set -x -mkdir -p ${WORKSPACE}/opt -cp -rf ${TOOLS_DIR}/toCOPY/mock_overlay ${WORKSPACE}/opt/mock_overlay -cd ${WORKSPACE}/opt/mock_overlay -make -sudo make install - -# This image requires a set of scripts and helpers -# for working correctly, in this section they are -# copied inside the image. -sudo cp ${TOOLS_DIR}/toCOPY/finishSetup.sh /usr/local/bin -sudo cp ${TOOLS_DIR}/toCOPY/populate_downloads.sh /usr/local/bin -sudo cp ${TOOLS_DIR}/toCOPY/generate-local-repo.sh /usr/local/bin -sudo cp ${TOOLS_DIR}/toCOPY/generate-centos-repo.sh /usr/local/bin -sudo cp ${TOOLS_DIR}/toCOPY/lst_utils.sh /usr/local/bin # centos locales are broken. this needs to be run after the last yum install/update sudo localedef -i en_US -f UTF-8 en_US.UTF-8 -echo_info "Setup for lighttpd" -sudo mkdir -p /www -sudo chown ${USER}:${BUILD_GROUP} /www -mkdir -p /www/run -mkdir -p /www/logs -mkdir -p /www/home -mkdir -p /www/root/htdocs/localdisk -mkdir -p /www/root/htdocs/$(dirname ${WORKSPACE}) -ln -s ${LOCALDISK}/loadbuild /www/root/htdocs/localdisk/loadbuild -ln -s ${LOCALDISK}/designer /www/root/htdocs/localdisk/designer -ln -s ${MIRROR_DIR}/CentOS /www/root/htdocs/CentOS -ln -s ${WORKSPACE} /www/root/htdocs/${WORKSPACE} - -# lighthttpd setup -# chmod for /var/log/lighttpd fixes a centos issue -# in place sed for server root since it's expanded soon thereafter -# echo "server.bind = \"localhost\"" >> /etc/lighttpd/lighttpd.conf && \ -sudo mkdir -p /var/log/lighttpd -sudo chmod a+rwx /var/log/lighttpd/ -sudo sed -i -e 's%^var\.log_root.*$%var.log_root = "/www/logs"%g' \ - -e 's%^var\.server_root.*$%var.server_root = "/www/root"%g' \ - -e 's%^var\.home_dir.*$%var.home_dir = "/www/home"%g' \ - -e 's%^var\.state_dir.*$%var.state_dir = "/www/run"%g' \ - -e "s/server.port/#server.port/g" \ - -e "s/server.use-ipv6/#server.use-ipv6/g" \ - -e "s/server.username/#server.username/g" \ - -e "s/server.groupname/#server.groupname/g" \ - -e "s/server.bind/#server.bind/g" \ - -e "s/server.document-root/#server.document-root/g" \ - -e "s/server.dirlisting/#server.dirlisting/g" \ - /etc/lighttpd/lighttpd.conf - -echo "server.port = 8088" | sudo tee -a /etc/lighttpd/lighttpd.conf -echo "server.use-ipv6 = \"disable\"" | sudo tee -a /etc/lighttpd/lighttpd.conf -echo "server.username = \"$USER\"" | sudo tee -a /etc/lighttpd/lighttpd.conf -echo "server.groupname = \"$BUILD_GROUP\"" | sudo tee -a /etc/lighttpd/lighttpd.conf -echo "server.bind = \"localhost\"" | sudo tee -a /etc/lighttpd/lighttpd.conf -echo "server.document-root = \"/www/root/htdocs\"" | sudo tee -a /etc/lighttpd/lighttpd.conf - -sudo sed -i "s/dir-listing.activate/#dir-listing.activate/g" \ - /etc/lighttpd/conf.d/dirlisting.conf - -echo "dir-listing.activate = \"enable\"" | sudo tee -a /etc/lighttpd/conf.d/dirlisting.conf - -sudo /usr/sbin/lighttpd -f /etc/lighttpd/lighttpd.conf - -sudo chmod a+x /usr/local/bin/* - -# Customizations for mirror creation -sudo rm -f /etc/yum.repos.d/* -sudo cp -f ${TOOLS_DIR}/centos-mirror-tools/yum.repos.d/* /etc/yum.repos.d/ -sudo cp -f ${TOOLS_DIR}/centos-mirror-tools/rpm-gpg-keys/* /etc/pki/rpm-gpg/ - -# Import GPG keys -sudo rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY* - -# Try to continue a yum command even if a StarlingX repo is unavailable. -sudo yum-config-manager --setopt=StarlingX\*.skip_if_unavailable=1 --save - -# Create a sane py27 virtualenv -virtualenv ${WORKSPACE}/opt/py27 -source ${WORKSPACE}/opt/py27/bin/activate -pip install -c ${TOOLS_DIR}/toCOPY/builder-opt-py27-constraints.txt tox -sudo ln -s ${WORKSPACE}/opt/py27/bin/tox /usr/bin - -echo_step_end +prepare_env +get_tools_repo +install_pkgs +config_mock +install_mock_overlay +setup_lighttpd +config_yum +create_py27_venv -- 2.16.6