Code Review / pti / rtp.git / blob
? search:


3526ba12030826991b25352390347c1c1db5b3e6
[pti/rtp.git] /
1 From 7518b647ea0085c231a903c7a3e4351b2b2966c4 Mon Sep 17 00:00:00 2001
2 From: Jackie Huang <jackie.huang@windriver.com>
3 Date: Wed, 26 Jul 2023 17:23:55 +0800
4 Subject: [PATCH 1/4] debrepack: add arm64 support for meta data
5 MIME-Version: 1.0
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
8
9 Add set_build_arch (similar to set_build_type) so
10 @BUILD_ARCH@ can be used as placeholder in meta data
11 files to avoid hardcoded arch names.
12
13 The items in dl_path/dl_files in meta_data.yaml may
14 have different values and the patches in the patches
15 or deb_patches directory may be different for arm64
16 arch, this change will allow multiple dl_path/dl_files
17 items exist in meta_data.yaml, and multiple series files
18 in patches or deb_patches directories exist, and the
19 arch specific ones will be used as first priority if
20 it exists.
21
22 The arch specific ones will have sufix like _arm64,
23 e.g. in meta_data.yaml
24
25 dl_files:
26   pkg-file-amd64.tar.gz:
27     topdir: pkg-amd64
28     url: https://path/to/pkg-file-amd64.tar.gz
29     sha256sum: <sha256sum value>
30 dl_files_arm64:
31   pkg-file-arm64.tar.gz:
32     topdir: pkg-arm64
33     url: https://path/to/pkg-file-arm64.tar.gz
34     sha256sum: <sha256sum value>
35
36 dl_path:
37   name: pkg-name-amd64.tar.gz
38   url: https://path/to/pkg-name-amd64.tar.gz
39   sha256sum: <sha256sum value>
40 dl_path_arm64:
41   name: pkg-name-arm64.tar.gz
42   url: https://path/to/pkg-name-arm64.tar.gz
43   sha256sum: <sha256sum value>
44
45 e.g. for patches/deb_patches
46
47 deb_patches
48 ├── 0001-deb-patch1.patch
49 ├── 0002-deb-patch2.patch
50 ├── 0003-arm64-deb-patch1.patch
51 ├── series
52 └── series_arm64
53
54 patches
55 ├── 0001-patch1.patch
56 ├── 0002-patch2.patch
57 ├── 0002-arm64-patch1.patch
58 ├── series
59 └── series_arm64
60
61 Test Plan:
62 PASS: update meta data for packages in repos:
63       - starlingx/integ
64       - starlingx/kernel
65 PASS: downloader -s on x86-64 host
66 PASS: downloader -s on arm64 host
67 PASS: build-pkgs -p <pkg> on x86-64 host
68 PASS: build-pkgs -p <pkg> on arm64 host
69 PASS: build-pkgs on x86-64 host
70 PASS: build-image on x86-64 host
71 PASS: build-pkgs on arm64 host
72 PASS: build-image on arm64 host
73 PASS: Deploy AIO-SX on x86-64 target
74 PASS: Deploy AIO-SX on arm64 target
75
76 Story: 2010739
77 Task: 48013
78
79 Change-Id: I42063364c416e40794d85af41e7a22366a84faa1
80 Signed-off-by: Jackie Huang <jackie.huang@windriver.com>
81 ---
82  build-tools/stx/debrepack.py | 114 ++++++++++++++++++++++++++++-------
83  1 file changed, 93 insertions(+), 21 deletions(-)
84
85 diff --git a/build-tools/stx/debrepack.py b/build-tools/stx/debrepack.py
86 index 524b38d..85f593b 100755
87 --- a/build-tools/stx/debrepack.py
88 +++ b/build-tools/stx/debrepack.py
89 @@ -28,6 +28,7 @@ import sys
90  import utils
91  from utils import run_shell_cmd, get_download_url
92  import yaml
93 +import platform
94  
95  
96  RELEASENOTES = " ".join([os.environ.get('PROJECT'), os.environ.get('MY_RELEASE'), "distribution"])
97 @@ -58,7 +59,21 @@ if STX_MIRROR_STRATEGY is None:
98          STX_MIRROR_STRATEGY = "stx_mirror_first"
99  
100  BTYPE = "@KERNEL_TYPE@"
101 +BARCH = "@BUILD_ARCH@"
102  
103 +STX_ARCH = "amd64"
104 +host_arch = platform.machine()
105 +if host_arch == 'aarch64':
106 +    STX_ARCH = "arm64"
107 +
108 +DL_FILES_DEFAULT = "dl_files"
109 +DL_FILES_ARCH = DL_FILES_DEFAULT + "_" + STX_ARCH
110 +
111 +DL_PATH_DEFAULT = "dl_path"
112 +DL_PATH_ARCH = DL_PATH_DEFAULT + "_" + STX_ARCH
113 +
114 +SERIES_DEFAULT = "series"
115 +SERIES_ARCH = SERIES_DEFAULT + "_" + STX_ARCH
116  
117  class DownloadProgress():
118      def __init__(self):
119 @@ -296,6 +311,16 @@ class Parser():
120  
121          self.pkginfo["debfolder"] = os.path.join(local_debian)
122  
123 +    def set_build_arch(self):
124 +
125 +        local_debian = os.path.join(self.pkginfo["packdir"], "local_debian")
126 +
127 +        # clean @BUILD_ARCH@
128 +        sed_cmd = 'sed -i s#%s#%s#g %s'
129 +        for root, _, files in os.walk(local_debian):
130 +            for name in files:
131 +                run_shell_cmd(sed_cmd % (BARCH, STX_ARCH, os.path.join(root, name)), self.logger)
132 +
133      def get_gitrevcount_srcdir(self, gitrevcount_obj):
134          src_dir = str(gitrevcount_obj.get("SRC_DIR", ""))
135          if src_dir:
136 @@ -463,8 +488,14 @@ class Parser():
137          self.logger.info("Overwrite the debian folder by %s", metadata)
138          run_shell_cmd('cp -r %s/* %s' % (metadata, deb_folder), self.logger)
139  
140 -        series = os.path.join(metadata, "patches/series")
141 -        if not os.path.isfile(series):
142 +        series_default = os.path.join(metadata, "patches", SERIES_DEFAULT)
143 +        series_arch = os.path.join(metadata, "patches", SERIES_ARCH)
144 +
145 +        if os.path.isfile(series_arch):
146 +            series = series_arch
147 +        elif os.path.isfile(series_default):
148 +            series = series_default
149 +        else:
150              return True
151  
152          format_ver, format_type = self.set_deb_format()
153 @@ -497,11 +528,16 @@ class Parser():
154                  run_shell_cmd('cp -rL %s %s' % (src_file, self.pkginfo["srcdir"]),
155                                self.logger)
156  
157 -        if "dl_files" in self.meta_data:
158 +        dl_files = DL_FILES_DEFAULT
159 +        # Arch specific dl_files have higher priority
160 +        if DL_FILES_ARCH in self.meta_data:
161 +            dl_files = DL_FILES_ARCH
162 +
163 +        if dl_files in self.meta_data:
164              pwd = os.getcwd()
165              os.chdir(self.pkginfo["packdir"])
166 -            for dl_file in self.meta_data['dl_files']:
167 -                dir_name = self.meta_data['dl_files'][dl_file]['topdir']
168 +            for dl_file in self.meta_data[dl_files]:
169 +                dir_name = self.meta_data[dl_files][dl_file]['topdir']
170                  dl_path = os.path.join(self.pkginfo["packdir"], dl_file)
171                  if not os.path.exists(dl_path):
172                      self.logger.error("No such file %s in local mirror", dl_file)
173 @@ -537,8 +573,15 @@ class Parser():
174      def apply_src_patches(self):
175  
176          format_ver, format_type = self.set_deb_format()
177 -        series = os.path.join(self.pkginfo["debfolder"], "patches/series")
178 -        if not os.path.isfile(series):
179 +
180 +        series_default = os.path.join(self.pkginfo["debfolder"], "patches", SERIES_DEFAULT)
181 +        series_arch = os.path.join(self.pkginfo["debfolder"], "patches", SERIES_ARCH)
182 +
183 +        if os.path.isfile(series_arch):
184 +            series = series_arch
185 +        elif os.path.isfile(series_default):
186 +            series = series_default
187 +        else:
188              return True
189  
190          f = open(series)
191 @@ -547,7 +590,13 @@ class Parser():
192          f.close()
193  
194          patches_folder = os.path.join(self.pkginfo["srcdir"], "debian/patches")
195 -        series_file = os.path.join(self.pkginfo["srcdir"], "debian/patches/series")
196 +        series_file_default = os.path.join(patches_folder, SERIES_DEFAULT)
197 +        series_file_arch = os.path.join(patches_folder, SERIES_ARCH)
198 +
199 +        series_file = series_file_default
200 +        if os.path.isfile(series_file_arch):
201 +            series_file = series_file_arch
202 +
203          if not os.path.isdir(patches_folder):
204              os.mkdir(patches_folder)
205              os.mknod(series_file)
206 @@ -587,9 +636,16 @@ class Parser():
207  
208      def apply_deb_patches(self):
209  
210 -        series = os.path.join(self.pkginfo["debfolder"], "deb_patches/series")
211 -        if not os.path.isfile(series):
212 +        series_default = os.path.join(self.pkginfo["debfolder"], "deb_patches", SERIES_DEFAULT)
213 +        series_arch = os.path.join(self.pkginfo["debfolder"], "deb_patches", SERIES_ARCH)
214 +
215 +        if os.path.isfile(series_arch):
216 +            series = series_arch
217 +        elif os.path.isfile(series_default):
218 +            series = series_default
219 +        else:
220              return True
221 +
222          f = open(series)
223          patches = f.readlines()
224          patches_src = os.path.dirname(series)
225 @@ -609,8 +665,12 @@ class Parser():
226          return True
227  
228      def extract_tarball(self):
229 +        # Arch specific dl_path have higher priority
230 +        dl_path = DL_PATH_DEFAULT
231 +        if DL_PATH_ARCH in self.meta_data:
232 +            dl_path = DL_PATH_ARCH
233  
234 -        tarball_name = self.meta_data["dl_path"]["name"]
235 +        tarball_name = self.meta_data[dl_path]["name"]
236          tarball_file = os.path.join(self.pkginfo["packdir"], tarball_name)
237  
238          cmd, _, _ = tar_cmd(tarball_name, self.logger)
239 @@ -715,9 +775,15 @@ class Parser():
240  
241          pwd = os.getcwd()
242          os.chdir(saveto)
243 -        if "dl_files" in self.meta_data:
244 -            for dl_file in self.meta_data['dl_files']:
245 -                dl_file_info = self.meta_data['dl_files'][dl_file]
246 +
247 +        dl_files = DL_FILES_DEFAULT
248 +        # Arch specific dl_files have higher priority
249 +        if DL_FILES_ARCH in self.meta_data:
250 +            dl_files = DL_FILES_ARCH
251 +
252 +        if dl_files in self.meta_data:
253 +            for dl_file in self.meta_data[dl_files]:
254 +                dl_file_info = self.meta_data[dl_files][dl_file]
255                  url = dl_file_info['url']
256                  if "sha256sum" in dl_file_info:
257                      check_cmd = "sha256sum"
258 @@ -739,16 +805,21 @@ class Parser():
259                      if not checksum(dl_file, check_sum, check_cmd, self.logger):
260                          raise Exception(f'Fail to download {dl_file}')
261  
262 -        if "dl_path" in self.meta_data:
263 -            dl_file = self.meta_data["dl_path"]["name"]
264 -            url = self.meta_data["dl_path"]["url"]
265 -            if "sha256sum" in self.meta_data["dl_path"]:
266 +        dl_path = DL_PATH_DEFAULT
267 +        # Arch specific dl_path have higher priority
268 +        if DL_PATH_ARCH in self.meta_data:
269 +            dl_path = DL_PATH_ARCH
270 +
271 +        if dl_path in self.meta_data:
272 +            dl_file = self.meta_data[dl_path]["name"]
273 +            url = self.meta_data[dl_path]["url"]
274 +            if "sha256sum" in self.meta_data[dl_path]:
275                  check_cmd = "sha256sum"
276 -                check_sum = self.meta_data["dl_path"]['sha256sum']
277 +                check_sum = self.meta_data[dl_path]['sha256sum']
278              else:
279                  self.logger.warning(f"{dl_file} missing sha256sum")
280                  check_cmd = "md5sum"
281 -                check_sum = self.meta_data["dl_path"]['md5sum']
282 +                check_sum = self.meta_data[dl_path]['md5sum']
283              if not checksum(dl_file, check_sum, check_cmd, self.logger):
284                  (dl_url, alt_dl_url) = get_download_url(url, self.strategy)
285                  if alt_dl_url:
286 @@ -813,6 +884,7 @@ class Parser():
287          os.mkdir(self.pkginfo["packdir"])
288  
289          self.set_build_type()
290 +        self.set_build_arch()
291  
292          logfile = os.path.join(self.pkginfo["packdir"], self.pkginfo["pkgname"] + ".log")
293          if os.path.exists(logfile):
294 @@ -832,7 +904,7 @@ class Parser():
295  
296          if "dl_hook" in self.meta_data:
297              self.run_dl_hook()
298 -        elif "dl_path" in self.meta_data:
299 +        elif (DL_PATH_DEFAULT or DL_PATH_ARCH) in self.meta_data:
300              self.extract_tarball()
301          elif "src_path" in self.meta_data:
302              self.create_src_package()
303 -- 
304 2.30.2
305
Performance Tuned Infrastructure