-From 7518b647ea0085c231a903c7a3e4351b2b2966c4 Mon Sep 17 00:00:00 2001
+From 74f44fc8cf8f6b94a57ca8ff7370c62320cb5af5 Mon Sep 17 00:00:00 2001
From: Jackie Huang <jackie.huang@windriver.com>
Date: Wed, 26 Jul 2023 17:23:55 +0800
Subject: [PATCH 1/4] debrepack: add arm64 support for meta data
1 file changed, 93 insertions(+), 21 deletions(-)
diff --git a/build-tools/stx/debrepack.py b/build-tools/stx/debrepack.py
-index 524b38d..85f593b 100755
+index c4fb570..3cf6410 100755
--- a/build-tools/stx/debrepack.py
+++ b/build-tools/stx/debrepack.py
-@@ -28,6 +28,7 @@ import sys
+@@ -30,6 +30,7 @@ import tempfile
import utils
- from utils import run_shell_cmd, get_download_url
+ from utils import run_shell_cmd, run_shell_cmd_full, get_download_url
import yaml
+import platform
RELEASENOTES = " ".join([os.environ.get('PROJECT'), os.environ.get('MY_RELEASE'), "distribution"])
-@@ -58,7 +59,21 @@ if STX_MIRROR_STRATEGY is None:
+@@ -60,7 +61,21 @@ if STX_MIRROR_STRATEGY is None:
STX_MIRROR_STRATEGY = "stx_mirror_first"
BTYPE = "@KERNEL_TYPE@"
class DownloadProgress():
def __init__(self):
-@@ -296,6 +311,16 @@ class Parser():
+@@ -348,6 +363,16 @@ class Parser():
self.pkginfo["debfolder"] = os.path.join(local_debian)
def get_gitrevcount_srcdir(self, gitrevcount_obj):
src_dir = str(gitrevcount_obj.get("SRC_DIR", ""))
if src_dir:
-@@ -463,8 +488,14 @@ class Parser():
+@@ -515,8 +540,14 @@ class Parser():
self.logger.info("Overwrite the debian folder by %s", metadata)
run_shell_cmd('cp -r %s/* %s' % (metadata, deb_folder), self.logger)
return True
format_ver, format_type = self.set_deb_format()
-@@ -497,11 +528,16 @@ class Parser():
+@@ -549,11 +580,16 @@ class Parser():
run_shell_cmd('cp -rL %s %s' % (src_file, self.pkginfo["srcdir"]),
self.logger)
dl_path = os.path.join(self.pkginfo["packdir"], dl_file)
if not os.path.exists(dl_path):
self.logger.error("No such file %s in local mirror", dl_file)
-@@ -537,8 +573,15 @@ class Parser():
+@@ -589,8 +625,15 @@ class Parser():
def apply_src_patches(self):
format_ver, format_type = self.set_deb_format()
return True
f = open(series)
-@@ -547,7 +590,13 @@ class Parser():
+@@ -599,7 +642,13 @@ class Parser():
f.close()
patches_folder = os.path.join(self.pkginfo["srcdir"], "debian/patches")
+
if not os.path.isdir(patches_folder):
os.mkdir(patches_folder)
- os.mknod(series_file)
-@@ -587,9 +636,16 @@ class Parser():
+ os.mknod(series_file, 0o666)
+@@ -639,9 +688,16 @@ class Parser():
def apply_deb_patches(self):
f = open(series)
patches = f.readlines()
patches_src = os.path.dirname(series)
-@@ -609,8 +665,12 @@ class Parser():
+@@ -661,8 +717,12 @@ class Parser():
return True
def extract_tarball(self):
tarball_file = os.path.join(self.pkginfo["packdir"], tarball_name)
cmd, _, _ = tar_cmd(tarball_name, self.logger)
-@@ -715,9 +775,15 @@ class Parser():
+@@ -770,9 +830,15 @@ class Parser():
pwd = os.getcwd()
os.chdir(saveto)
url = dl_file_info['url']
if "sha256sum" in dl_file_info:
check_cmd = "sha256sum"
-@@ -739,16 +805,21 @@ class Parser():
- if not checksum(dl_file, check_sum, check_cmd, self.logger):
+@@ -795,16 +861,21 @@ class Parser():
raise Exception(f'Fail to download {dl_file}')
+ rel_used_dl_files.append(dl_file)
- if "dl_path" in self.meta_data:
- dl_file = self.meta_data["dl_path"]["name"]
if not checksum(dl_file, check_sum, check_cmd, self.logger):
(dl_url, alt_dl_url) = get_download_url(url, self.strategy)
if alt_dl_url:
-@@ -813,6 +884,7 @@ class Parser():
+@@ -935,6 +1006,7 @@ class Parser():
os.mkdir(self.pkginfo["packdir"])
self.set_build_type()
logfile = os.path.join(self.pkginfo["packdir"], self.pkginfo["pkgname"] + ".log")
if os.path.exists(logfile):
-@@ -832,7 +904,7 @@ class Parser():
+@@ -954,7 +1026,7 @@ class Parser():
if "dl_hook" in self.meta_data:
self.run_dl_hook()
-From c3a2332443a87b1a5c9477838cae5314413eebe6 Mon Sep 17 00:00:00 2001
+From 8c3f0091bf119df68661b227ac7edcc93e2efe8f Mon Sep 17 00:00:00 2001
From: Jackie Huang <jackie.huang@windriver.com>
Date: Wed, 26 Jul 2023 13:06:48 +0800
Subject: [PATCH 2/4] downloader: add support for arm64
Signed-off-by: Jackie Huang <jackie.huang@windriver.com>
---
build-tools/stx/debdownloader | 5 +++++
- build-tools/stx/downloader | 37 +++++++++++++++++++++++++++--------
- 2 files changed, 34 insertions(+), 8 deletions(-)
+ build-tools/stx/downloader | 38 +++++++++++++++++++++++++++--------
+ 2 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/build-tools/stx/debdownloader b/build-tools/stx/debdownloader
index 6988cc7..a1ca92e 100755
if debs_downloaded and pname_all in debs_downloaded:
logger.info(''.join([name, '-', version,
diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader
-index 14d97f7..2d98707 100755
+index fc2f4ed..4341fe2 100755
--- a/build-tools/stx/downloader
+++ b/build-tools/stx/downloader
-@@ -27,8 +27,14 @@ import shutil
- import signal
+@@ -29,13 +29,20 @@ import signal
+ import subprocess
import sys
import utils
+import platform
+ # make ourself nice
+ pid = os.getpid()
+ os.setpriority(os.PRIO_PROCESS, 0, 15)
+ subprocess.run(['ionice', '-c', '3', '-p', str(pid)])
+
-DEFAULT_ARCH = 'amd64'
+STX_ARCH = 'amd64'
+host_arch = platform.machine()
+ STX_ARCH = "arm64"
+
+DEB_DISTRO = 'bullseye'
++
REPO_BIN = 'deb-local-binary'
mirror_root = os.environ.get('STX_MIRROR')
stx_src_mirror = os.path.join(mirror_root, 'sources')
-@@ -115,7 +121,11 @@ def get_all_stx_pkgs():
+@@ -122,7 +129,11 @@ def get_all_stx_pkgs():
for r in files:
# Find all types of package dirs?
if r in types_pkg_dirs:
pkgs.update(get_pkgs_from_list(root, pkgs_file))
return pkgs
-@@ -153,24 +163,34 @@ def get_all_binary_list(distro=STX_DEFAULT_DISTRO, layers=None, build_types=None
+@@ -160,24 +171,34 @@ def get_all_binary_list(distro=STX_DEFAULT_DISTRO, layers=None, build_types=None
logger.warning(' '.join([build_type, 'is not a valid build_type for distro', distro, 'of layer', layer]))
continue
logger.info(
f"Binary lists for layer `common`: "
-@@ -727,13 +747,14 @@ if __name__ == "__main__":
- args.download_source = True
-
+@@ -771,15 +792,16 @@ if __name__ == "__main__":
+ shutil.rmtree(dl_list_dir)
+ os.makedirs(dl_list_dir, exist_ok=True)
if args.download_binary:
+
all_binary_lists = get_all_binary_list(distro=distro, layers=layers, build_types=build_types)
-- binary_dl = DebDownloader(DEFAULT_ARCH, stx_bin_mirror, clean_mirror, all_binary_lists)
-+ binary_dl = DebDownloader(STX_ARCH, stx_bin_mirror, clean_mirror, all_binary_lists)
+ dl_list_file_bin = '%s/binaries.txt' % dl_list_dir
+- binary_dl = DebDownloader(DEFAULT_ARCH, stx_bin_mirror, dl_list_file_bin, clean_mirror, all_binary_lists)
++ binary_dl = DebDownloader(STX_ARCH, stx_bin_mirror, dl_list_file_bin, clean_mirror, all_binary_lists)
if not binary_dl.create_binary_repo():
sys.exit(1)
if args.download_source:
-- source_dl = SrcDownloader(DEFAULT_ARCH, stx_src_mirror, clean_mirror)
-+ source_dl = SrcDownloader(STX_ARCH, stx_src_mirror, clean_mirror)
+ dl_list_file_src = '%s/sources.txt' % dl_list_dir
+- source_dl = SrcDownloader(DEFAULT_ARCH, stx_src_mirror, dl_list_file_src, clean_mirror)
++ source_dl = SrcDownloader(STX_ARCH, stx_src_mirror, dl_list_file_src, clean_mirror)
dl_register_signal_handler()
if binary_dl:
-From 516085675edd2a3330b4ddba10c4b613ee91955b Mon Sep 17 00:00:00 2001
+From 5ba18d573d5c38e58bb2ae8e0bebec0e902797b1 Mon Sep 17 00:00:00 2001
From: Jackie Huang <jackie.huang@windriver.com>
Date: Wed, 26 Jul 2023 14:55:29 +0800
Subject: [PATCH 3/4] build-tools: add supoort for arm64
---
build-tools/stx/aptly_deb_usage.py | 12 ++++--
build-tools/stx/build-image | 65 ++++++++++++++++++++++++------
- build-tools/stx/build-pkgs | 8 +++-
+ build-tools/stx/build-pkgs | 7 +++-
build-tools/stx/discovery.py | 42 ++++++++++++++++---
build-tools/stx/downloader | 2 +-
build-tools/stx/repo_manage.py | 9 ++++-
- 6 files changed, 111 insertions(+), 27 deletions(-)
+ 6 files changed, 111 insertions(+), 26 deletions(-)
diff --git a/build-tools/stx/aptly_deb_usage.py b/build-tools/stx/aptly_deb_usage.py
index f1d91c2..d10e6a2 100755
extra_param['origin'] = self.origin
diff --git a/build-tools/stx/build-image b/build-tools/stx/build-image
-index 8536c32..ea04fa2 100755
+index 7b60f3f..9c5f803 100755
--- a/build-tools/stx/build-image
+++ b/build-tools/stx/build-image
-@@ -28,11 +28,17 @@ import sys
+@@ -29,6 +29,7 @@ import sys
import time
import utils
import yaml
+import platform
- STX_DEFAULT_DISTRO = discovery.STX_DEFAULT_DISTRO
+ # make ourself nice
+ pid = os.getpid()
+@@ -39,6 +40,11 @@ STX_DEFAULT_DISTRO = discovery.STX_DEFAULT_DISTRO
ALL_LAYERS = discovery.get_all_layers(distro=STX_DEFAULT_DISTRO)
ALL_BUILD_TYPES = discovery.get_all_build_types(distro=STX_DEFAULT_DISTRO)
LAT_ROOT = '/localdisk'
REPO_ALL = 'deb-merge-all'
REPO_BINARY = 'deb-local-binary'
-@@ -48,8 +54,8 @@ IMAGE_LAYERS_PATH = os.path.join(
+@@ -54,8 +60,8 @@ IMAGE_LAYERS_PATH = os.path.join(
)
img_pkgs = []
kernel_type = 'std'
WAIT_TIME_BEFORE_CHECKING_LOG = 2
# The max timeout value to wait LAT to output the build log
MAX_WAIT_LAT_TIME = 300
-@@ -242,7 +248,7 @@ def update_rt_kernel_in_initramfs_yaml(initramfs_yaml):
+@@ -248,7 +254,7 @@ def update_rt_kernel_in_initramfs_yaml(initramfs_yaml):
# Updated the name of kernel module
for layer in ALL_LAYERS:
if not pkg_dirs:
continue
for pkg_dir in pkg_dirs:
-@@ -305,7 +311,7 @@ def add_lat_packages(img_yaml, packages):
+@@ -311,7 +317,7 @@ def add_lat_packages(img_yaml, packages):
yaml_doc['packages'].extend(packages)
for build_type in ALL_BUILD_TYPES:
yaml_doc['packages'].extend(pkgs)
yaml_doc['packages'] = list(set(yaml_doc['packages']))
-@@ -319,8 +325,16 @@ def add_lat_packages(img_yaml, packages):
+@@ -325,8 +331,16 @@ def add_lat_packages(img_yaml, packages):
def check_base_os_binaries(repomgr):
if not os.path.exists(base_bins_list):
logger.error(' '.join(['Base OS packages list', base_bins_list,
'does not exist']))
-@@ -338,8 +352,15 @@ def check_base_os_binaries(repomgr):
+@@ -344,8 +358,15 @@ def check_base_os_binaries(repomgr):
def check_stx_binaries(repomgr, btype='std'):
if not os.path.exists(stx_bins_list):
logger.warning(' '.join(['STX binary packages list', stx_bins_list,
'does not exist']))
-@@ -660,16 +681,30 @@ if __name__ == "__main__":
+@@ -749,16 +770,30 @@ if __name__ == "__main__":
logger.error("Fail to get prepared to build image")
sys.exit(1)
if not os.path.exists(LAT_ROOT):
os.makedirs(LAT_ROOT)
-@@ -758,6 +793,10 @@ if __name__ == "__main__":
+@@ -847,6 +882,10 @@ if __name__ == "__main__":
logger.info("build-image successfully done, check the output in %s", LAT_ROOT)
ret = 0
break
stop_latd()
diff --git a/build-tools/stx/build-pkgs b/build-tools/stx/build-pkgs
-index a8feaa9..9dacef3 100755
+index ec812a4..a7cbe5b 100755
--- a/build-tools/stx/build-pkgs
+++ b/build-tools/stx/build-pkgs
-@@ -37,7 +37,7 @@ import tempfile
+@@ -37,6 +37,7 @@ import tempfile
import time
import utils
import yaml
--
+import platform
- BUILDER_URL = os.environ.get('BUILDER_URL')
- REPOMGR_URL = os.environ.get('REPOMGR_URL')
-@@ -49,6 +49,10 @@ USER = os.environ.get('MYUNAME')
+ # make ourself nice
+ pid = os.getpid()
+@@ -53,6 +54,10 @@ USER = os.environ.get('MYUNAME')
PROJECT = os.environ.get('PROJECT')
DISTRIBUTION = os.environ.get('DEBIAN_DISTRIBUTION')
STX_ARCH = 'amd64'
+
STX_META_NAME = 'stx-meta'
STX_META_PKG = 'stx-meta_1.0.orig.tar.gz'
- # Different reasons can lead to package build failure
-@@ -1658,7 +1662,7 @@ class BuildController():
+
+@@ -1715,7 +1720,7 @@ class BuildController():
logger.error('Failed to specify build_type')
return
word = "all"
if packages:
diff --git a/build-tools/stx/discovery.py b/build-tools/stx/discovery.py
-index 3de7d48..d16e520 100644
+index dc4b35f..d94d295 100644
--- a/build-tools/stx/discovery.py
+++ b/build-tools/stx/discovery.py
@@ -17,6 +17,7 @@ import os
from git_utils import git_list
from repo_utils import repo_root
-@@ -30,6 +31,11 @@ STX_DEFAULT_DISTRO_LIST = [ "debian", "centos" ]
+@@ -30,6 +31,11 @@ STX_DEFAULT_DISTRO_LIST = [ "debian" ]
STX_DEFAULT_BUILD_TYPE = "std"
STX_DEFAULT_BUILD_TYPE_LIST = [STX_DEFAULT_BUILD_TYPE]
continue
pkg_dir_list.extend(bc_safe_fetch(pkg_file, package_dir_list_handler, proj_dir))
diff --git a/build-tools/stx/downloader b/build-tools/stx/downloader
-index 2d98707..47093ea 100755
+index 4341fe2..bca05ca 100755
--- a/build-tools/stx/downloader
+++ b/build-tools/stx/downloader
-@@ -635,7 +635,7 @@ class SrcDownloader(BaseDownloader):
+@@ -672,7 +672,7 @@ class SrcDownloader(BaseDownloader):
logger.warning(' '.join([build_type, 'is not a valid build_type for distro', distro, 'of layer', layer]))
continue
-From c0f2e1c04e0a3ceb7a9aff4651186dfb68dc3f16 Mon Sep 17 00:00:00 2001
+From c9f81b1bdc69fb020c91a0d99e15a50d52767cd0 Mon Sep 17 00:00:00 2001
From: Jackie Huang <jackie.huang@windriver.com>
Date: Thu, 27 Jul 2023 23:46:36 +0800
Subject: [PATCH 4/4] Add debian-image_arm64.inc
Code Review / pti / rtp.git / commitdiff