gerrit.o-ran-sc Code Review - pti/rtp.git/blob

   1 From 516085675edd2a3330b4ddba10c4b613ee91955b Mon Sep 17 00:00:00 2001
   2 From: Jackie Huang <jackie.huang@windriver.com>
   3 Date: Wed, 26 Jul 2023 14:55:29 +0800
   4 Subject: [PATCH 3/4] build-tools: add supoort for arm64
   5
   6 * Check the host arch.
   7 * use ARCH to replace the hardcoded arch names.
   8 * Support arch specific pkg list files and yaml files,
   9   and the arch specific one (with suffix _arm64) will
  10   be used if it exists.
  11
  12 e.g.
  13 debian-image.inc
  14 debian-image_arm64.inc
  15
  16 debian_iso_image.inc
  17 debian_iso_image_arm64.inc
  18
  19 debian_pkg_dirs
  20 debian_pkg_dirs_arm64
  21
  22 base-bullseye.yaml
  23 base-bullseye_arm64.yaml
  24
  25 base-initramfs-bullseye.yaml
  26 base-initramfs-bullseye_arm64.yaml
  27
  28 os-std.lst
  29 os-std_arm64.lst
  30
  31 Test Plan:
  32 PASS: update packages list for arm64 in repos:
  33       - starlingx/tools
  34       - starlingx/kernel
  35       - starlingx/integ
  36       - starlingx/virt
  37       - starlingx/utilities
  38 PASS: downloader -s -b on x86-64 host
  39 PASS: downloader -s -b on arm64 host
  40 PASS: build-pkgs on x86-64 host
  41 PASS: build-image on x86-64 host
  42 PASS: build-pkgs on arm64 host
  43 PASS: build-image on arm64 host
  44 PASS: Deploy AIO-SX on x86-64 target
  45 PASS: Deploy AIO-SX on arm64 target
  46
  47 Story: 2010739
  48 Task: 48013
  49
  50 Change-Id: I9e381f3f04f6747c68d40011c9eda419219c2311
  51 Signed-off-by: Jackie Huang <jackie.huang@windriver.com>
  52 ---
  53  build-tools/stx/aptly_deb_usage.py | 12 ++++--
  54  build-tools/stx/build-image        | 65 ++++++++++++++++++++++++------
  55  build-tools/stx/build-pkgs         |  8 +++-
  56  build-tools/stx/discovery.py       | 42 ++++++++++++++++---
  57  build-tools/stx/downloader         |  2 +-
  58  build-tools/stx/repo_manage.py     |  9 ++++-
  59  6 files changed, 111 insertions(+), 27 deletions(-)
  60
  61 diff --git a/build-tools/stx/aptly_deb_usage.py b/build-tools/stx/aptly_deb_usage.py
  62 index f1d91c2..d10e6a2 100755
  63 --- a/build-tools/stx/aptly_deb_usage.py
  64 +++ b/build-tools/stx/aptly_deb_usage.py
  65 @@ -25,6 +25,7 @@ from debian import debian_support
  66  import os
  67  import time
  68  from typing import Optional
  69 +import platform
  70
  71  PREFIX_LOCAL = 'deb-local-'
  72  PREFIX_REMOTE = 'deb-remote-'
  73 @@ -34,6 +35,11 @@ SIGN_PASSWD = 'starlingx'
  74  DEFAULT_TIMEOUT_COUNT = 1
  75  STX_DIST = os.environ.get('STX_DIST')
  76
  77 +STX_ARCH = 'amd64'
  78 +host_arch = platform.machine()
  79 +if host_arch == 'aarch64':
  80 +    STX_ARCH = "arm64"
  81 +
  82  # Class used to manage aptly data base, it can:
  83  #     create_remote: Create a repository link to a remote mirror
  84  #     deploy_remote: Sync and deploy a remote mirror
  85 @@ -326,7 +332,7 @@ class Deb_aptly():
  86                      self.logger.warning('Drop failed publication %s : %s', publish_name, task_state)
  87                      return None
  88          task = self.aptly.publish.publish(source_kind='local', sources=[{'Name': repo_name}],
  89 -                                          architectures=['amd64', 'source'], prefix=publish_name,
  90 +                                          architectures=[STX_ARCH, 'source'], prefix=publish_name,
  91                                            distribution=None, sign_skip=True)
  92          task_state = self.__wait_for_task(task, 10)
  93          if task_state != 'SUCCEEDED':
  94 @@ -366,8 +372,8 @@ class Deb_aptly():
  95              extra_param['distribution'] = mirror.distribution
  96              extra_param['origin'] = None
  97          else:
  98 -            # Only support binary_amd64 and source packages
  99 -            extra_param['architectures'] = ['amd64', 'source']
 100 +            # Only support binary_amd64/arm64 and source packages
 101 +            extra_param['architectures'] = [STX_ARCH, 'source']
 102              extra_param['distribution'] = None
 103              extra_param['origin'] = self.origin
 104
 105 diff --git a/build-tools/stx/build-image b/build-tools/stx/build-image
 106 index 8536c32..ea04fa2 100755
 107 --- a/build-tools/stx/build-image
 108 +++ b/build-tools/stx/build-image
 109 @@ -28,11 +28,17 @@ import sys
 110  import time
 111  import utils
 112  import yaml
 113 +import platform
 114
 115  STX_DEFAULT_DISTRO = discovery.STX_DEFAULT_DISTRO
 116  ALL_LAYERS = discovery.get_all_layers(distro=STX_DEFAULT_DISTRO)
 117  ALL_BUILD_TYPES = discovery.get_all_build_types(distro=STX_DEFAULT_DISTRO)
 118
 119 +STX_ARCH = 'amd64'
 120 +host_arch = platform.machine()
 121 +if host_arch == 'aarch64':
 122 +    STX_ARCH = "arm64"
 123 +
 124  LAT_ROOT = '/localdisk'
 125  REPO_ALL = 'deb-merge-all'
 126  REPO_BINARY = 'deb-local-binary'
 127 @@ -48,8 +54,8 @@ IMAGE_LAYERS_PATH = os.path.join(
 128  )
 129  img_pkgs = []
 130  kernel_type = 'std'
 131 -stx_std_kernel = 'linux-image-5.10.0-6-amd64-unsigned'
 132 -stx_rt_kernel =  'linux-rt-image-5.10.0-6-rt-amd64-unsigned'
 133 +stx_std_kernel = 'linux-image-5.10.0-6-%s-unsigned' % STX_ARCH
 134 +stx_rt_kernel =  'linux-rt-image-5.10.0-6-rt-%s-unsigned' % STX_ARCH
 135  WAIT_TIME_BEFORE_CHECKING_LOG = 2
 136  # The max timeout value to wait LAT to output the build log
 137  MAX_WAIT_LAT_TIME = 300
 138 @@ -242,7 +248,7 @@ def update_rt_kernel_in_initramfs_yaml(initramfs_yaml):
 139
 140      # Updated the name of kernel module
 141      for layer in ALL_LAYERS:
 142 -        pkg_dirs = discovery.package_dir_list(distro=STX_DEFAULT_DISTRO, layer=layer, build_type='rt')
 143 +        pkg_dirs = discovery.package_dir_list(distro=STX_DEFAULT_DISTRO, layer=layer, build_type='rt', arch=STX_ARCH)
 144          if not pkg_dirs:
 145              continue
 146          for pkg_dir in pkg_dirs:
 147 @@ -305,7 +311,7 @@ def add_lat_packages(img_yaml, packages):
 148      yaml_doc['packages'].extend(packages)
 149
 150      for build_type in ALL_BUILD_TYPES:
 151 -        pkgs = discovery.package_iso_list(distro=STX_DEFAULT_DISTRO, layer="all", build_type=build_type)
 152 +        pkgs = discovery.package_iso_list(distro=STX_DEFAULT_DISTRO, layer="all", build_type=build_type, arch=STX_ARCH)
 153          yaml_doc['packages'].extend(pkgs)
 154
 155      yaml_doc['packages'] = list(set(yaml_doc['packages']))
 156 @@ -319,8 +325,16 @@ def add_lat_packages(img_yaml, packages):
 157
 158
 159  def check_base_os_binaries(repomgr):
 160 -    base_bins_list = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'),
 161 +    base_bins_list_default = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'),
 162                                    'cgcs-root/build-tools/stx/debian-image.inc')
 163 +    base_bins_list_arch = os.path.join(os.environ.get('MY_REPO_ROOT_DIR'),
 164 +                                  'cgcs-root/build-tools/stx',
 165 +                                  'debian-image_%s.inc' % STX_ARCH)
 166 +    if os.path.exists(base_bins_list_arch):
 167 +        base_bins_list = base_bins_list_arch
 168 +    else:
 169 +        base_bins_list = base_bins_list_default
 170 +
 171      if not os.path.exists(base_bins_list):
 172          logger.error(' '.join(['Base OS packages list', base_bins_list,
 173                                 'does not exist']))
 174 @@ -338,8 +352,15 @@ def check_base_os_binaries(repomgr):
 175
 176
 177  def check_stx_binaries(repomgr, btype='std'):
 178 -    stx_bins_list = ''.join([PKG_LIST_DIR, '/debian/distro/os-', btype,
 179 -                             '.lst'])
 180 +    stx_bins_list_default = ''.join([PKG_LIST_DIR, '/debian/distro/os-',
 181 +                                     btype, '.lst'])
 182 +    stx_bins_list_arch = ''.join([PKG_LIST_DIR, '/debian/distro/os-',
 183 +                                  btype, '_', STX_ARCH, '.lst'])
 184 +    if os.path.exists(stx_bins_list_arch):
 185 +        stx_bins_list = stx_bins_list_arch
 186 +    else:
 187 +        stx_bins_list = stx_bins_list_default
 188 +
 189      if not os.path.exists(stx_bins_list):
 190          logger.warning(' '.join(['STX binary packages list', stx_bins_list,
 191                                  'does not exist']))
 192 @@ -660,16 +681,30 @@ if __name__ == "__main__":
 193          logger.error("Fail to get prepared to build image")
 194          sys.exit(1)
 195
 196 -    base_yaml = os.path.join(PKG_LIST_DIR, 'debian/common/base-bullseye.yaml')
 197 -    base_initramfs_yaml = os.path.join(PKG_LIST_DIR, 'debian/common/base-initramfs-bullseye.yaml')
 198 +    base_yaml_default = os.path.join(PKG_LIST_DIR, 'debian/common/base-bullseye.yaml')
 199 +    base_yaml_arch = os.path.join(PKG_LIST_DIR, 'debian/common/base-bullseye_%s.yaml' % STX_ARCH)
 200 +    base_initramfs_yaml_default = os.path.join(PKG_LIST_DIR, 'debian/common/base-initramfs-bullseye.yaml')
 201 +    base_initramfs_yaml_arch = os.path.join(PKG_LIST_DIR, 'debian/common/base-initramfs-bullseye_%s.yaml' % STX_ARCH)
 202 +
 203      os.environ["WORKSPACE_DIR"] = LAT_ROOT
 204      lat_yaml = os.path.join(LAT_ROOT, "lat.yaml")
 205      lat_initramfs_yaml = os.path.join(LAT_ROOT, "lat-initramfs.yaml")
 206
 207 -    for yaml_file in (base_yaml, base_initramfs_yaml):
 208 -        if not os.path.exists(yaml_file):
 209 -            logger.error(' '.join(['Base yaml file', yaml_file, 'does not exist']))
 210 -            sys.exit(1)
 211 +    if os.path.exists(base_yaml_arch):
 212 +        base_yaml = base_yaml_arch
 213 +    elif os.path.exists(base_yaml_default):
 214 +        base_yaml = base_yaml_default
 215 +    else:
 216 +        logger.error(' '.join(['Base yaml file', base_yaml_default, 'does not exist']))
 217 +        sys.exit(1)
 218 +
 219 +    if os.path.exists(base_initramfs_yaml_arch):
 220 +        base_initramfs_yaml = base_initramfs_yaml_arch
 221 +    elif os.path.exists(base_initramfs_yaml_default):
 222 +        base_initramfs_yaml = base_initramfs_yaml_default
 223 +    else:
 224 +        logger.error(' '.join(['Base yaml file', base_initramfs_yaml_default, 'does not exist']))
 225 +        sys.exit(1)
 226
 227      if not os.path.exists(LAT_ROOT):
 228          os.makedirs(LAT_ROOT)
 229 @@ -758,6 +793,10 @@ if __name__ == "__main__":
 230                      logger.info("build-image successfully done, check the output in %s", LAT_ROOT)
 231                      ret = 0
 232                      break
 233 +                if STX_ARCH == "arm64" and "INFO: Create ISO Image: Succeeded" in line:
 234 +                    logger.info("build-image successfully done, check the output in %s", LAT_ROOT)
 235 +                    ret = 0
 236 +                    break
 237      # stop latd
 238      stop_latd()
 239
 240 diff --git a/build-tools/stx/build-pkgs b/build-tools/stx/build-pkgs
 241 index a8feaa9..9dacef3 100755
 242 --- a/build-tools/stx/build-pkgs
 243 +++ b/build-tools/stx/build-pkgs
 244 @@ -37,7 +37,7 @@ import tempfile
 245  import time
 246  import utils
 247  import yaml
 248 -
 249 +import platform
 250
 251  BUILDER_URL = os.environ.get('BUILDER_URL')
 252  REPOMGR_URL = os.environ.get('REPOMGR_URL')
 253 @@ -49,6 +49,10 @@ USER = os.environ.get('MYUNAME')
 254  PROJECT = os.environ.get('PROJECT')
 255  DISTRIBUTION = os.environ.get('DEBIAN_DISTRIBUTION')
 256  STX_ARCH = 'amd64'
 257 +host_arch = platform.machine()
 258 +if host_arch == 'aarch64':
 259 +    STX_ARCH = "arm64"
 260 +
 261  STX_META_NAME = 'stx-meta'
 262  STX_META_PKG = 'stx-meta_1.0.orig.tar.gz'
 263  # Different reasons can lead to package build failure
 264 @@ -1658,7 +1662,7 @@ class BuildController():
 265              logger.error('Failed to specify build_type')
 266              return
 267
 268 -        pkg_dirs = discovery.package_dir_list(distro=self.attrs['distro'], layer=layer, build_type=build_type)
 269 +        pkg_dirs = discovery.package_dir_list(distro=self.attrs['distro'], layer=layer, build_type=build_type, arch=STX_ARCH)
 270          layer_pkg_dirs = pkg_dirs
 271          word = "all"
 272          if packages:
 273 diff --git a/build-tools/stx/discovery.py b/build-tools/stx/discovery.py
 274 index 3de7d48..d16e520 100644
 275 --- a/build-tools/stx/discovery.py
 276 +++ b/build-tools/stx/discovery.py
 277 @@ -17,6 +17,7 @@ import os
 278  import re
 279  import glob
 280  import yaml
 281 +import platform
 282
 283  from git_utils import git_list
 284  from repo_utils import repo_root
 285 @@ -30,6 +31,11 @@ STX_DEFAULT_DISTRO_LIST = [ "debian", "centos" ]
 286  STX_DEFAULT_BUILD_TYPE = "std"
 287  STX_DEFAULT_BUILD_TYPE_LIST = [STX_DEFAULT_BUILD_TYPE]
 288
 289 +STX_ARCH = "amd64"
 290 +STX_SUPPORTED_ARCH = ["amd64", "arm64"]
 291 +host_arch = platform.machine()
 292 +if host_arch == 'aarch64':
 293 +    STX_ARCH = "arm64"
 294
 295  def get_all_distros():
 296      distro_lst = STX_DEFAULT_DISTRO_LIST
 297 @@ -126,6 +132,14 @@ def get_layer_build_types (layer, distro="debian", skip_non_buildable=True):
 298      for proj_dir in project_dir_list_all:
 299          for pkg_dir_file in glob.glob("%s/%s%s" % (proj_dir, distro, "_pkg_dirs_*")):
 300              bt = os.path.basename(pkg_dir_file).split("_pkg_dirs_")[1]
 301 +            # cleanup arch specific suffix
 302 +            if bt in STX_SUPPORTED_ARCH:
 303 +                continue
 304 +            else:
 305 +                for arch in STX_SUPPORTED_ARCH:
 306 +                    arch_suffix = "_" + arch
 307 +                    if bt.endswith(arch_suffix):
 308 +                        bt = bt.replace(arch_suffix, "")
 309              if not bt in bt_lst:
 310                  bt_lst.append(bt)
 311      return sort_build_type_list(bt_lst, layer)
 312 @@ -137,6 +151,14 @@ def get_all_build_types (distro="debian", skip_non_buildable=True):
 313      for proj_dir in project_dir_list_all:
 314          for pkg_dir_file in glob.glob("%s/%s%s" % (proj_dir, distro, "_pkg_dirs_*")):
 315              bt = os.path.basename(pkg_dir_file).split("_pkg_dirs_")[1]
 316 +            # cleanup arch specific suffix
 317 +            if bt in STX_SUPPORTED_ARCH:
 318 +                continue
 319 +            else:
 320 +                for arch in STX_SUPPORTED_ARCH:
 321 +                    arch_suffix = "_" + arch
 322 +                    if bt.endswith(arch_suffix):
 323 +                        bt = bt.replace(arch_suffix, "")
 324              if not bt in bt_lst:
 325                  bt_lst.append(bt)
 326      return sorted(bt_lst)
 327 @@ -186,32 +208,40 @@ def package_dir_list_handler(entry, proj_dir):
 328      return [ path ]
 329
 330
 331 -def package_iso_list (distro="debian", layer="all", build_type="std", skip_non_buildable=True):
 332 +def package_iso_list (distro="debian", layer="all", build_type="std", arch=STX_ARCH, skip_non_buildable=True):
 333      pkg_iso_list = []
 334      if layer is None:
 335          layer = "all"
 336      for proj_dir in project_dir_list(distro=distro, layer=layer, skip_non_buildable=skip_non_buildable):
 337 -        iso_file = os.path.join(proj_dir, "%s%s%s%s" % (distro, "_iso_image_", build_type, ".inc"))
 338 +        iso_file = os.path.join(proj_dir, "%s_%s_%s_%s%s" % (distro, "iso_image", build_type, arch, ".inc"))
 339 +        if not os.path.isfile(iso_file):
 340 +            iso_file = os.path.join(proj_dir, "%s_%s_%s%s" % (distro, "iso_image", build_type, ".inc"))
 341          if not os.path.isfile(iso_file):
 342              if build_type == "std":
 343                  # It's permitted to omit the "_std" suffix from the file name
 344 -                iso_file = os.path.join(proj_dir, "%s%s" % (distro, "_iso_image.inc"))
 345 +                iso_file = os.path.join(proj_dir, "%s_%s_%s%s" % (distro, "iso_image", arch, ".inc"))
 346 +                if not os.path.isfile(iso_file):
 347 +                    iso_file = os.path.join(proj_dir, "%s_%s" % (distro, "iso_image.inc"))
 348          if not os.path.isfile(iso_file):
 349              continue
 350          pkg_iso_list.extend(bc_safe_fetch(iso_file))
 351      return pkg_iso_list
 352
 353
 354 -def package_dir_list (distro="debian", layer="all", build_type="std", skip_non_buildable=True):
 355 +def package_dir_list (distro="debian", layer="all", build_type="std", arch=STX_ARCH, skip_non_buildable=True):
 356      pkg_dir_list = []
 357      if layer is None:
 358          layer = "all"
 359      for proj_dir in project_dir_list(distro=distro, layer=layer, skip_non_buildable=skip_non_buildable):
 360 -        pkg_file = os.path.join(proj_dir, "%s%s%s" % (distro, "_pkg_dirs_", build_type))
 361 +        pkg_file = os.path.join(proj_dir, "%s_%s_%s_%s" % (distro, "pkg_dirs", build_type, arch))
 362 +        if not os.path.isfile(pkg_file):
 363 +            pkg_file = os.path.join(proj_dir, "%s_%s_%s" % (distro, "pkg_dirs", build_type))
 364          if not os.path.isfile(pkg_file):
 365              if build_type == "std":
 366                  # It's permitted to omit the "_std" suffix from the file name
 367 -                pkg_file = os.path.join(proj_dir, "%s%s" % (distro, "_pkg_dirs"))
 368 +                pkg_file = os.path.join(proj_dir, "%s_%s_%s" % (distro, "pkg_dirs", arch))
 369 +                if not os.path.isfile(pkg_file):
 370 +                    pkg_file = os.path.join(proj_dir, "%s_%s" % (distro, "pkg_dirs"))
 371          if not os.path.isfile(pkg_file):
 372              continue
 373          pkg_dir_list.extend(bc_safe_fetch(pkg_file, package_dir_list_handler, proj_dir))
 374 diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader
 375 index 2d98707..47093ea 100755
 376 --- a/build-tools/stx/downloader
 377 +++ b/build-tools/stx/downloader
 378 @@ -635,7 +635,7 @@ class SrcDownloader(BaseDownloader):
 379                      logger.warning(' '.join([build_type, 'is not a valid build_type for distro', distro, 'of layer', layer]))
 380                      continue
 381
 382 -                pkg_dirs.extend(discovery.package_dir_list(distro=distro, layer=layer, build_type=build_type))
 383 +                pkg_dirs.extend(discovery.package_dir_list(distro=distro, layer=layer, build_type=build_type, arch=STX_ARCH))
 384
 385          if not len(pkg_dirs):
 386              logger.info("No source packages found")
 387 diff --git a/build-tools/stx/repo_manage.py b/build-tools/stx/repo_manage.py
 388 index dbe0cc3..bf68b70 100755
 389 --- a/build-tools/stx/repo_manage.py
 390 +++ b/build-tools/stx/repo_manage.py
 391 @@ -31,6 +31,7 @@ import shutil
 392  from threading import Lock
 393  import urllib.request
 394  import utils
 395 +import platform
 396
 397  REPOMGR_URL = os.environ.get('REPOMGR_URL')
 398  REPOMGR_ORIGIN = os.environ.get('REPOMGR_ORIGIN')
 399 @@ -38,6 +39,10 @@ REPOMGR_DEPLOY_URL = os.environ.get('REPOMGR_DEPLOY_URL')
 400
 401  APTFETCH_JOBS = 20
 402
 403 +STX_ARCH = 'amd64'
 404 +host_arch = platform.machine()
 405 +if host_arch == 'aarch64':
 406 +    STX_ARCH = "arm64"
 407
 408  class AptFetch():
 409      '''
 410 @@ -497,7 +502,7 @@ class RepoMgr():
 411      # kwargs:url: URL of the upstream repo (http://deb.debian.org/debian)
 412      # kwargs:distribution: the distribution of the repo (bullseye)
 413      # kwargs:component: component of the repo (main)
 414 -    # kwargs:architecture: architecture of the repo, "all" is always enabled. (amd64)
 415 +    # kwargs:architecture: architecture of the repo, "all" is always enabled. (amd64 or arm64)
 416      # kwargs:with_sources: include source packages, default is False.
 417      # Output: None
 418      def mirror(self, repo_name, **kwargs):
 419 @@ -923,7 +928,7 @@ def subcmd_mirror(subparsers):
 420      mirror_parser.add_argument('--component', '-c', help='component name', required=False,
 421                                 default='nginx')
 422      mirror_parser.add_argument('--architectures', '-a', help='architectures', required=False,
 423 -                               default='amd64')
 424 +                               default=STX_ARCH)
 425      mirror_parser.add_argument('--with-sources', '-s', help='include source packages',
 426                                 action='store_true')
 427      mirror_parser.set_defaults(handle=_handleMirror)
 428 --
 429 2.30.2
 430