scripts/build_inf_centos/stx_build_prepare_centos.sh

   1 #!/bin/sh
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 #
  15
  16 #
  17 # Common env
  18 #
  19 # username to build
  20 export MYUNAME=stxbuilder
  21 export MYUID=1001
  22 export MY_EMAIL="oran.inf@windriver.com"
  23 export WGET_OPENDEV="wget --no-check-certificate"
  24 export LOCALDISK="/localdisk"
  25 export MIRROR_DIR="/import/mirrors"
  26
  27 groupadd cgts
  28 echo "mock:x:751:root" >> /etc/group
  29 echo "mockbuild:x:9001:" >> /etc/group
  30
  31 useradd -r -u $MYUID -g cgts -m $MYUNAME
  32 echo "Li69nux*"|sudo passwd --stdin ${MYUNAME}
  33
  34 gpasswd -a $MYUNAME mock
  35
  36 mkdir -p ${LOCALDISK}/loadbuild/mock-cache
  37 mkdir -p ${LOCALDISK}/loadbuild/mock
  38 mkdir -p ${LOCALDISK}/designer
  39 mkdir -p ${MIRROR_DIR}/CentOS
  40
  41 chmod 775 ${LOCALDISK}/loadbuild/mock
  42 chown root:mock ${LOCALDISK}/loadbuild/mock
  43 chmod 775 ${LOCALDISK}/loadbuild/mock-cache
  44 chown root:mock ${LOCALDISK}/loadbuild/mock-cache
  45
  46
  47 # Proxy configuration
  48 export http_proxy="http://147.11.252.42:9090"
  49 export https_proxy="http://147.11.252.42:9090"
  50 export ftp_proxy="http://147.11.252.42:9090"
  51
  52 echo "proxy=$http_proxy" >> /etc/yum.conf && \
  53     echo -e "export http_proxy=$http_proxy\nexport https_proxy=$https_proxy\n\
  54 export ftp_proxy=$ftp_proxy" >> /root/.bashrc
  55
  56
  57 # CentOS & EPEL URLs that match the base image
  58 # Override these with --build-arg if you have a mirror
  59 CENTOS_7_8_URL=https://vault.centos.org/centos/7.8.2003
  60 CENTOS_7_9_URL=http://mirror.centos.org/centos-7/7.9.2009
  61 EPEL_7_8_URL=https://archives.fedoraproject.org/pub/archive/epel/7.2020-04-20
  62
  63 # Lock down centos & epel repos
  64 rm -f /etc/yum.repos.d/*
  65 cd /etc/yum.repos.d/
  66 ${WGET_OPENDEV} https://opendev.org/starlingx/tools/raw/branch/master/toCOPY/yum.repos.d/centos-7.9.repo
  67 ${WGET_OPENDEV} https://opendev.org/starlingx/tools/raw/branch/master/toCOPY/yum.repos.d/centos.repo
  68 ${WGET_OPENDEV} https://opendev.org/starlingx/tools/raw/branch/master/toCOPY/yum.repos.d/epel.repo
  69
  70 cd /etc/pki/rpm-gpg/
  71 ${WGET_OPENDEV} https://opendev.org/starlingx/tools/raw/branch/master/centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-EPEL-7
  72
  73 cd -
  74 rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY* && \
  75     echo "http_caching=packages" >> /etc/yum.conf && \
  76     echo "skip_missing_names_on_install=0" >>/etc/yum.conf && \
  77     # yum variables must be in lower case ; \
  78     echo "$CENTOS_7_8_URL" >/etc/yum/vars/centos_7_8_url && \
  79     echo "$EPEL_7_8_URL" >/etc/yum/vars/epel_7_8_url && \
  80     echo "$CENTOS_7_9_URL" >/etc/yum/vars/centos_7_9_url && \
  81     # disable fastestmirror plugin because we are not using mirrors ; \
  82     # FIXME: use a mirrorlist URL for centos/vault/epel archives. I couldn't find one.
  83     sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf && \
  84     echo "[main]" >> /etc/yum/pluginconf.d/subscription-manager.conf && \
  85     echo "enabled=0" >> /etc/yum/pluginconf.d/subscription-manager.conf && \
  86     yum clean all && \
  87     yum makecache && \
  88     yum install -y deltarpm
  89
  90
  91 # root CA cert expired on October 1st, 2021
  92 yum update -y --enablerepo=centos-7.9-updates ca-certificates
  93
  94 # Download required dependencies by mirror/build processes.
  95 yum install -y \
  96         anaconda \
  97         anaconda-runtime \
  98         autoconf-archive \
  99         autogen \
 100         automake \
 101         bc \
 102         bind \
 103         bind-utils \
 104         bison \
 105         cpanminus \
 106         createrepo \
 107         createrepo_c \
 108         deltarpm \
 109         docker-client \
 110         expat-devel \
 111         flex \
 112         isomd5sum \
 113         gcc \
 114         gettext \
 115         git \
 116         libguestfs-tools \
 117         libtool \
 118         libxml2 \
 119         lighttpd \
 120         lighttpd-fastcgi \
 121         lighttpd-mod_geoip \
 122         net-tools \
 123         mkisofs \
 124         mongodb \
 125         mongodb-server \
 126         pax \
 127         perl-CPAN \
 128         python-deltarpm \
 129         python-pep8 \
 130         python-pip \
 131         python-psutil \
 132         python2-psutil \
 133         python36-psutil \
 134         python36-requests \
 135         python3-devel \
 136         python-sphinx \
 137         python-subunit \
 138         python-virtualenv \
 139         python-yaml \
 140         python2-ruamel-yaml \
 141         postgresql \
 142         qemu-kvm \
 143         quilt \
 144         rpm-build \
 145         rpm-sign \
 146         rpm-python \
 147         squashfs-tools \
 148         sudo \
 149         systemd \
 150         syslinux \
 151         udisks2 \
 152         vim-enhanced \
 153         wget
 154
 155 # clone the tools repo
 156 cd ~
 157 git clone https://opendev.org/starlingx/tools.git
 158
 159 # Finally install a locked down version of mock
 160 yum install -y https://mirrors.xlhy1.com/centos/7/updates/x86_64/Packages/python36-rpm-4.11.3-7.el7.x86_64.rpm
 161 yum install -y \
 162     http://mirror.starlingx.cengn.ca/mirror/centos/epel/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/mock-1.4.16-1.el7.noarch.rpm \
 163     http://mirror.starlingx.cengn.ca/mirror/centos/epel/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/mock-core-configs-31.6-1.el7.noarch.rpm
 164
 165 # mock custumizations
 166 # forcing chroots since a couple of packages naughtily insist on network access and
 167 # we dont have nspawn and networks happy together.
 168 useradd -s /sbin/nologin -u 9001 -g 9001 mockbuild && \
 169     rmdir /var/lib/mock && \
 170     ln -s ${LOCALDISK}/loadbuild/mock /var/lib/mock && \
 171     rmdir /var/cache/mock && \
 172     ln -s ${LOCALDISK}/loadbuild/mock-cache /var/cache/mock && \
 173     echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg && \
 174     echo "config_opts['rpmbuild_networking'] = True" >> /etc/mock/site-defaults.cfg && \
 175     echo  >> /etc/mock/site-defaults.cfg
 176
 177 # cpan modules, installing with cpanminus to avoid stupid questions since cpan is whack
 178 cpanm --notest Fatal && \
 179     cpanm --notest XML::SAX  && \
 180     cpanm --notest XML::SAX::Expat && \
 181     cpanm --notest XML::Parser && \
 182     cpanm --notest XML::Simple
 183
 184 # Install repo tool
 185 curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo && \
 186     chmod a+x /usr/local/bin/repo
 187
 188 # installing go and setting paths
 189 export GOPATH="/usr/local/go"
 190 export PATH="${GOPATH}/bin:${PATH}"
 191 yum install -y golang && \
 192     mkdir -p ${GOPATH}/bin && \
 193     curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
 194
 195 # Uprev git, repo
 196 yum install -y dh-autoreconf curl-devel expat-devel gettext-devel openssl-devel perl-devel zlib-devel asciidoc xmlto docbook2X && \
 197     cd /tmp && \
 198     wget https://github.com/git/git/archive/v2.29.2.tar.gz -O git-2.29.2.tar.gz && \
 199     tar xzvf git-2.29.2.tar.gz && \
 200     cd git-2.29.2 && \
 201     make configure && \
 202     ./configure --prefix=/usr/local && \
 203     make all doc && \
 204     make install install-doc && \
 205     cd /tmp && \
 206     rm -rf git-2.29.2.tar.gz git-2.29.2
 207
 208 # Systemd Enablement
 209 #(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
 210 #    rm -f /lib/systemd/system/multi-user.target.wants/*;\
 211 #    rm -f /etc/systemd/system/*.wants/*;\
 212 #    rm -f /lib/systemd/system/local-fs.target.wants/*; \
 213 #    rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
 214 #    rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
 215 #    rm -f /lib/systemd/system/basic.target.wants/*;\
 216 #    rm -f /lib/systemd/system/anaconda.target.wants/*
 217
 218 # pip installs
 219 # Install required python modules globally; versions are in the constraints file.
 220 # Be careful not to replace modules provided by RPMs as it may break
 221 # other system packages. Look for warnings similar to "Uninstalling a
 222 # distutils installed project has been deprecated" from pip.
 223 pip install -c ~/tools/toCOPY/builder-constraints.txt \
 224         testrepository \
 225         fixtures \
 226         pbr \
 227         git-review \
 228         python-subunit \
 229         junitxml \
 230         testtools
 231
 232 # Create a sane py27 virtualenv
 233 virtualenv /opt/py27 && \
 234     source /opt/py27/bin/activate && \
 235     pip install -c ~/tools/toCOPY/builder-opt-py27-constraints.txt \
 236             tox \
 237         && \
 238     for prog in tox ; do \
 239         ln -s /opt/py27/bin/$prog /usr/bin ; \
 240     done
 241
 242 # Inherited  tools for mock stuff
 243 # we at least need the mock_cache_unlock tool
 244 # they install into /usr/bin
 245 cp -rf ~/tools/toCOPY/mock_overlay /opt/mock_overlay
 246 cd /opt/mock_overlay && \
 247     make && \
 248     make install
 249
 250 # This image requires a set of scripts and helpers
 251 # for working correctly, in this section they are
 252 # copied inside the image.
 253 cp ~/tools/toCOPY/finishSetup.sh /usr/local/bin
 254 cp ~/tools/toCOPY/populate_downloads.sh /usr/local/bin
 255 cp ~/tools/toCOPY/generate-local-repo.sh /usr/local/bin
 256 cp ~/tools/toCOPY/generate-centos-repo.sh /usr/local/bin
 257 cp ~/tools/toCOPY/lst_utils.sh /usr/local/bin
 258 cp ~/tools/toCOPY/.inputrc /home/$MYUNAME/
 259 chown $MYUNAME:cgts /home/$MYUNAME/.inputrc
 260
 261 # Thes are included for backward compatibility, and
 262 # should be removed after a reasonable time.
 263 cp ~/tools/toCOPY/generate-cgcs-tis-repo /usr/local/bin
 264 cp ~/tools/toCOPY/generate-cgcs-centos-repo.sh /usr/local/bin
 265
 266 #  ENV setup
 267 echo "# Load stx-builder configuration" >> /etc/profile.d/stx-builder-conf.sh && \
 268     echo "if [[ -r \${HOME}/buildrc ]]; then" >> /etc/profile.d/stx-builder-conf.sh && \
 269     echo "    source \${HOME}/buildrc" >> /etc/profile.d/stx-builder-conf.sh && \
 270     echo "    export PROJECT SRC_BUILD_ENVIRONMENT MYPROJECTNAME MYUNAME" >> /etc/profile.d/stx-builder-conf.sh && \
 271     echo "    export MY_BUILD_CFG MY_BUILD_CFG_RT MY_BUILD_CFG_STD MY_BUILD_DIR MY_BUILD_ENVIRONMENT MY_BUILD_ENVIRONMENT_FILE MY_BUILD_ENVIRONMENT_FILE_RT MY_BUILD_ENVIRONMENT_FILE_STD MY_DEBUG_BUILD_CFG_RT MY_DEBUG_BUILD_CFG_STD MY_LOCAL_DISK MY_MOCK_ROOT MY_REPO MY_REPO_ROOT_DIR MY_SRC_RPM_BUILD_DIR MY_RELEASE MY_WORKSPACE LAYER" >> /etc/profile.d/stx-builder-conf.sh && \
 272     echo "fi" >> /etc/profile.d/stx-builder-conf.sh && \
 273     echo "export FORMAL_BUILD=0" >> /etc/profile.d/stx-builder-conf.sh && \
 274     echo "export PATH=\$MY_REPO/build-tools:\$PATH" >> /etc/profile.d/stx-builder-conf.sh
 275
 276 # centos locales are broken. this needs to be run after the last yum install/update
 277 localedef -i en_US -f UTF-8 en_US.UTF-8
 278
 279 # setup
 280 mkdir -p /www/run && \
 281     mkdir -p /www/logs && \
 282     mkdir -p /www/home && \
 283     mkdir -p /www/root/htdocs/localdisk && \
 284     chown -R $MYUID:cgts /www && \
 285     ln -s ${LOCALDISK}/loadbuild /www/root/htdocs/localdisk/loadbuild && \
 286     ln -s ${MIRROR_DIR}/CentOS /www/root/htdocs/CentOS && \
 287     ln -s ${LOCALDISK}/designer /www/root/htdocs/localdisk/designer
 288
 289 # lighthttpd setup
 290 # chmod for /var/log/lighttpd fixes a centos issue
 291 # in place sed for server root since it's expanded soon thereafter
 292 #     echo "server.bind = \"localhost\"" >> /etc/lighttpd/lighttpd.conf && \
 293 echo "$MYUNAME ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers && \
 294     mkdir -p  /var/log/lighttpd  && \
 295     chmod a+rwx /var/log/lighttpd/ && \
 296     sed -i 's%^var\.log_root.*$%var.log_root = "/www/logs"%g' /etc/lighttpd/lighttpd.conf  && \
 297     sed -i 's%^var\.server_root.*$%var.server_root = "/www/root"%g' /etc/lighttpd/lighttpd.conf  && \
 298     sed -i 's%^var\.home_dir.*$%var.home_dir = "/www/home"%g' /etc/lighttpd/lighttpd.conf  && \
 299     sed -i 's%^var\.state_dir.*$%var.state_dir = "/www/run"%g' /etc/lighttpd/lighttpd.conf  && \
 300     sed -i "s/server.port/#server.port/g" /etc/lighttpd/lighttpd.conf  && \
 301     sed -i "s/server.use-ipv6/#server.use-ipv6/g" /etc/lighttpd/lighttpd.conf && \
 302     sed -i "s/server.username/#server.username/g" /etc/lighttpd/lighttpd.conf && \
 303     sed -i "s/server.groupname/#server.groupname/g" /etc/lighttpd/lighttpd.conf && \
 304     sed -i "s/server.bind/#server.bind/g" /etc/lighttpd/lighttpd.conf && \
 305     sed -i "s/server.document-root/#server.document-root/g" /etc/lighttpd/lighttpd.conf && \
 306     sed -i "s/server.dirlisting/#server.dirlisting/g" /etc/lighttpd/lighttpd.conf && \
 307     echo "server.port = 8088" >> /etc/lighttpd/lighttpd.conf && \
 308     echo "server.use-ipv6 = \"disable\"" >> /etc/lighttpd/lighttpd.conf && \
 309     echo "server.username = \"$MYUNAME\"" >> /etc/lighttpd/lighttpd.conf && \
 310     echo "server.groupname = \"cgts\"" >> /etc/lighttpd/lighttpd.conf && \
 311     echo "server.bind = \"localhost\"" >> /etc/lighttpd/lighttpd.conf && \
 312     echo "server.document-root   = \"/www/root/htdocs\"" >> /etc/lighttpd/lighttpd.conf && \
 313     sed -i "s/dir-listing.activate/#dir-listing.activate/g" /etc/lighttpd/conf.d/dirlisting.conf && \
 314     echo "dir-listing.activate = \"enable\"" >> /etc/lighttpd/conf.d/dirlisting.conf
 315
 316 #systemctl enable lighttpd
 317 #systemctl start lighttpd
 318 /usr/sbin/lighttpd  -f /etc/lighttpd/lighttpd.conf
 319
 320
 321 echo "export PATH=/usr/local/bin:${LOCALDISK}/designer/$MYUNAME/bin:\$PATH" >> /home/$MYUNAME/.bashrc
 322 chmod a+x /usr/local/bin/*
 323
 324 # Genrate a git configuration file in order to save an extra step
 325 # for end users, this file is required by "repo" tool.
 326 cd /home/$MYUNAME/
 327 runuser -u $MYUNAME -- git config --global user.email $MY_EMAIL && \
 328     runuser -u $MYUNAME -- git config --global user.name $MYUNAME && \
 329     runuser -u $MYUNAME -- git config --global color.ui false
 330
 331 # Customizations for mirror creation
 332 rm -f /etc/yum.repos.d/*
 333 cp -f ~/tools/centos-mirror-tools/yum.repos.d/* /etc/yum.repos.d/
 334 cp -f ~/tools/centos-mirror-tools/rpm-gpg-keys/* /etc/pki/rpm-gpg/
 335
 336 # Import GPG keys
 337 rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY*
 338
 339 # Try to continue a yum command even if a StarlingX repo is unavailable.
 340 yum-config-manager --setopt=StarlingX\*.skip_if_unavailable=1 --save