openh264_build.py (16039B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 import glob 6 import os 7 import re 8 import subprocess 9 import sys 10 11 # load modules from parent dir 12 sys.path.insert(1, os.path.dirname(sys.path[0])) 13 14 # import the guts 15 import mozharness 16 from mozharness.base.log import DEBUG, ERROR, FATAL 17 from mozharness.base.transfer import TransferMixin 18 from mozharness.base.vcs.vcsbase import VCSScript 19 from mozharness.mozilla.tooltool import TooltoolMixin 20 21 external_tools_path = os.path.join( 22 os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 23 "external_tools", 24 ) 25 26 27 class OpenH264Build(TransferMixin, VCSScript, TooltoolMixin): 28 all_actions = [ 29 "clobber", 30 "get-tooltool", 31 "checkout-sources", 32 "build", 33 "test", 34 "package", 35 "dump-symbols", 36 ] 37 38 default_actions = [ 39 "get-tooltool", 40 "checkout-sources", 41 "build", 42 "package", 43 "dump-symbols", 44 ] 45 46 config_options = [ 47 [ 48 ["--repo"], 49 { 50 "dest": "repo", 51 "help": "OpenH264 repository to use", 52 "default": "https://github.com/dminor/openh264.git", 53 }, 54 ], 55 [ 56 ["--rev"], 57 {"dest": "revision", "help": "revision to checkout", "default": "master"}, 58 ], 59 [ 60 ["--debug"], 61 { 62 "dest": "debug_build", 63 "action": "store_true", 64 "help": "Do a debug build", 65 }, 66 ], 67 [ 68 ["--arch"], 69 { 70 "dest": "arch", 71 "help": "Arch type to use (x64, x86, arm, or aarch64)", 72 }, 73 ], 74 [ 75 ["--os"], 76 { 77 "dest": "operating_system", 78 "help": "Specify the operating system to build for", 79 }, 80 ], 81 [ 82 ["--branch"], 83 { 84 "dest": "branch", 85 "help": "dummy option", 86 }, 87 ], 88 ] 89 90 def __init__( 91 self, 92 require_config_file=False, 93 config={}, 94 all_actions=all_actions, 95 default_actions=default_actions, 96 ): 97 # Default configuration 98 default_config = { 99 "debug_build": False, 100 "upload_ssh_key": "~/.ssh/ffxbld_rsa", 101 "upload_ssh_user": "ffxbld", 102 "upload_ssh_host": "upload.ffxbld.productdelivery.prod.mozaws.net", 103 "upload_path_base": "/tmp/openh264", 104 } 105 default_config.update(config) 106 107 VCSScript.__init__( 108 self, 109 config_options=self.config_options, 110 require_config_file=require_config_file, 111 config=default_config, 112 all_actions=all_actions, 113 default_actions=default_actions, 114 ) 115 116 def query_abs_dirs(self): 117 if self.abs_dirs: 118 return self.abs_dirs 119 dirs = super().query_abs_dirs() 120 dirs["abs_upload_dir"] = os.path.join(dirs["abs_work_dir"], "upload") 121 self.abs_dirs = dirs 122 return self.abs_dirs 123 124 def get_tooltool(self): 125 c = self.config 126 if not c.get("tooltool_manifest_file"): 127 self.info("Skipping tooltool fetching since no tooltool manifest") 128 return 129 dirs = self.query_abs_dirs() 130 self.mkdir_p(dirs["abs_work_dir"]) 131 manifest = os.path.join( 132 dirs["abs_src_dir"], 133 "testing", 134 "mozharness", 135 "configs", 136 "openh264", 137 "tooltool-manifests", 138 c["tooltool_manifest_file"], 139 ) 140 self.info("Getting tooltool files from manifest (%s)" % manifest) 141 try: 142 self.tooltool_fetch( 143 manifest=manifest, 144 output_dir=os.path.join(dirs["abs_work_dir"]), 145 cache=c.get("tooltool_cache"), 146 ) 147 except KeyError: 148 self.error("missing a required key.") 149 150 def query_package_name(self): 151 if self.config["arch"] in ("x64", "aarch64"): 152 bits = "64" 153 else: 154 bits = "32" 155 version = self.config["revision"] 156 157 if sys.platform in ("linux2", "linux"): 158 if self.config.get("operating_system") == "android": 159 return "openh264-android-{arch}-{version}.zip".format( 160 version=version, arch=self.config["arch"] 161 ) 162 elif self.config.get("operating_system") == "darwin": 163 suffix = "" 164 if self.config["arch"] != "x64": 165 suffix = "-" + self.config["arch"] 166 return f"openh264-macosx{bits}{suffix}-{version}.zip" 167 elif self.config["arch"] == "aarch64": 168 return f"openh264-linux64-aarch64-{version}.zip" 169 else: 170 return f"openh264-linux{bits}-{version}.zip" 171 elif sys.platform == "win32": 172 if self.config["arch"] == "aarch64": 173 return f"openh264-win64-aarch64-{version}.zip" 174 else: 175 return f"openh264-win{bits}-{version}.zip" 176 self.fatal("can't determine platform") 177 178 def query_make_params(self): 179 retval = [] 180 if self.config["debug_build"]: 181 retval.append("BUILDTYPE=Debug") 182 183 if self.config["arch"] in ("x64", "aarch64"): 184 retval.append("ENABLE64BIT=Yes") 185 else: 186 retval.append("ENABLE64BIT=No") 187 188 if self.config["arch"] == "x86": 189 retval.append("ARCH=x86") 190 elif self.config["arch"] == "x64": 191 retval.append("ARCH=x86_64") 192 elif self.config["arch"] == "aarch64": 193 retval.append("ARCH=arm64") 194 else: 195 self.fatal("Unknown arch: {}".format(self.config["arch"])) 196 197 if "operating_system" in self.config: 198 retval.append("OS=%s" % self.config["operating_system"]) 199 if self.config["operating_system"] == "android": 200 retval.append("TARGET=invalid") 201 retval.append("NDKLEVEL=%s" % self.config["min_sdk"]) 202 retval.append("NDKROOT=%s/android-ndk" % os.environ["MOZ_FETCHES_DIR"]) 203 retval.append("NDK_TOOLCHAIN_VERSION=clang") 204 if self.config["operating_system"] == "darwin": 205 retval.append("OS=darwin") 206 207 if self._is_windows(): 208 retval.append("OS=msvc") 209 retval.append("CC=clang-cl") 210 retval.append("CXX=clang-cl") 211 if self.config["arch"] == "aarch64": 212 retval.append("CXX_LINK_O=-nologo --target=aarch64-windows-msvc -Fe$@") 213 else: 214 retval.append("CC=clang") 215 retval.append("CXX=clang++") 216 217 return retval 218 219 def query_upload_ssh_key(self): 220 return self.config["upload_ssh_key"] 221 222 def query_upload_ssh_host(self): 223 return self.config["upload_ssh_host"] 224 225 def query_upload_ssh_user(self): 226 return self.config["upload_ssh_user"] 227 228 def query_upload_ssh_path(self): 229 return "%s/%s" % (self.config["upload_path_base"], self.config["revision"]) 230 231 def run_make(self, target, capture_output=False): 232 make = ( 233 f"{os.environ['MOZ_FETCHES_DIR']}/mozmake/mozmake" 234 if sys.platform == "win32" 235 else "make" 236 ) 237 cmd = [make, target] + self.query_make_params() 238 dirs = self.query_abs_dirs() 239 repo_dir = os.path.join(dirs["abs_work_dir"], "openh264") 240 env = None 241 if self.config.get("partial_env"): 242 env = self.query_env(self.config["partial_env"]) 243 kwargs = dict(cwd=repo_dir, env=env) 244 if capture_output: 245 return self.get_output_from_command(cmd, **kwargs) 246 else: 247 return self.run_command(cmd, **kwargs) 248 249 def _git_checkout(self, repo, repo_dir, rev): 250 try: 251 subprocess.run( 252 ["git", "clone", "-q", "--no-checkout", repo, repo_dir], check=True 253 ) 254 subprocess.run( 255 ["git", "checkout", "-q", "-f", f"{rev}^0"], check=True, cwd=repo_dir 256 ) 257 except Exception: 258 self.rmtree(repo_dir) 259 raise 260 return True 261 262 def checkout_sources(self): 263 repo = self.config["repo"] 264 rev = self.config["revision"] 265 266 dirs = self.query_abs_dirs() 267 repo_dir = os.path.join(dirs["abs_work_dir"], "openh264") 268 269 if self._is_windows(): 270 # We don't have git on our windows builders, so download a zip 271 # package instead. 272 path = repo.replace(".git", "/archive/") + rev + ".zip" 273 self.download_file(path) 274 self.unzip(rev + ".zip", dirs["abs_work_dir"]) 275 self.move( 276 os.path.join(dirs["abs_work_dir"], "openh264-" + rev), 277 os.path.join(dirs["abs_work_dir"], "openh264"), 278 ) 279 280 # Retrieve in-tree version of gmp-api 281 self.copytree( 282 os.path.join(dirs["abs_src_dir"], "dom", "media", "gmp", "gmp-api"), 283 os.path.join(repo_dir, "gmp-api"), 284 ) 285 286 # We need gas-preprocessor.pl for arm64 builds 287 if self.config["arch"] == "aarch64": 288 openh264_dir = os.path.join(dirs["abs_work_dir"], "openh264") 289 self.download_file( 290 ( 291 "https://raw.githubusercontent.com/libav/" 292 "gas-preprocessor/c2bc63c96678d9739509e58" 293 "7aa30c94bdc0e636d/gas-preprocessor.pl" 294 ), 295 parent_dir=openh264_dir, 296 ) 297 self.chmod(os.path.join(openh264_dir, "gas-preprocessor.pl"), 744) 298 299 # gas-preprocessor.pl expects cpp to exist 300 # os.symlink is not available on Windows until we switch to 301 # Python 3. 302 os.system( 303 "ln -s %s %s" 304 % ( 305 os.path.join( 306 os.environ["MOZ_FETCHES_DIR"], "clang", "bin", "clang.exe" 307 ), 308 os.path.join(openh264_dir, "cpp"), 309 ) 310 ) 311 return 0 312 313 self.retry( 314 self._git_checkout, 315 error_level=FATAL, 316 error_message="Automation Error: couldn't clone repo", 317 args=(repo, repo_dir, rev), 318 ) 319 320 # Checkout gmp-api 321 # TODO: Nothing here updates it yet, or enforces versions! 322 if not os.path.exists(os.path.join(repo_dir, "gmp-api")): 323 retval = self.run_make("gmp-bootstrap") 324 if retval != 0: 325 self.fatal("couldn't bootstrap gmp") 326 else: 327 self.info("skipping gmp bootstrap - we have it locally") 328 329 # Checkout gtest 330 # TODO: Requires svn! 331 if not os.path.exists(os.path.join(repo_dir, "gtest")): 332 retval = self.run_make("gtest-bootstrap") 333 if retval != 0: 334 self.fatal("couldn't bootstrap gtest") 335 else: 336 self.info("skipping gtest bootstrap - we have it locally") 337 338 return retval 339 340 def build(self): 341 retval = self.run_make("plugin") 342 if retval != 0: 343 self.fatal("couldn't build plugin") 344 345 def package(self): 346 dirs = self.query_abs_dirs() 347 srcdir = os.path.join(dirs["abs_work_dir"], "openh264") 348 package_name = self.query_package_name() 349 package_file = os.path.join(dirs["abs_work_dir"], package_name) 350 if os.path.exists(package_file): 351 os.unlink(package_file) 352 to_package = [] 353 for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*")): 354 if not re.search( 355 r"(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)", f 356 ): 357 # Don't package unnecessary zip bloat 358 # Blocks things like libgmpopenh264.2.dylib and libgmpopenh264.so.1 359 self.log(f"Skipping packaging of {f}") 360 continue 361 to_package.append(os.path.basename(f)) 362 self.log("Packaging files %s" % to_package) 363 cmd = ["zip", package_file] + to_package 364 retval = self.run_command(cmd, cwd=srcdir) 365 if retval != 0: 366 self.fatal("couldn't make package") 367 self.copy_to_upload_dir( 368 package_file, dest=os.path.join(srcdir, "artifacts", package_name) 369 ) 370 371 # Taskcluster expects this path to exist, but we don't use it 372 # because our builds are private. 373 path = os.path.join( 374 self.query_abs_dirs()["abs_work_dir"], "..", "public", "build" 375 ) 376 self.mkdir_p(path) 377 378 def dump_symbols(self): 379 dirs = self.query_abs_dirs() 380 c = self.config 381 srcdir = os.path.join(dirs["abs_work_dir"], "openh264") 382 package_name = self.run_make("echo-plugin-name", capture_output=True) 383 if not package_name: 384 self.fatal("failure running make") 385 zip_package_name = self.query_package_name() 386 if not zip_package_name[-4:] == ".zip": 387 self.fatal("Unexpected zip_package_name") 388 symbol_package_name = f"{zip_package_name[:-4]}.symbols.zip" 389 symbol_zip_path = os.path.join(srcdir, "artifacts", symbol_package_name) 390 repo_dir = os.path.join(dirs["abs_work_dir"], "openh264") 391 env = None 392 if self.config.get("partial_env"): 393 env = self.query_env(self.config["partial_env"]) 394 kwargs = dict(cwd=repo_dir, env=env) 395 dump_syms = os.path.join(dirs["abs_work_dir"], c["dump_syms_binary"]) 396 self.chmod(dump_syms, 0o755) 397 python = self.query_exe("python3") 398 cmd = [ 399 python, 400 os.path.join(external_tools_path, "packagesymbols.py"), 401 "--symbol-zip", 402 symbol_zip_path, 403 dump_syms, 404 os.path.join(srcdir, package_name), 405 ] 406 self.run_command(cmd, **kwargs) 407 408 def test(self): 409 retval = self.run_make("test") 410 if retval != 0: 411 self.fatal("test failures") 412 413 def copy_to_upload_dir( 414 self, 415 target, 416 dest=None, 417 log_level=DEBUG, 418 error_level=ERROR, 419 compress=False, 420 upload_dir=None, 421 ): 422 """Copy target file to upload_dir/dest. 423 424 Potentially update a manifest in the future if we go that route. 425 426 Currently only copies a single file; would be nice to allow for 427 recursive copying; that would probably done by creating a helper 428 _copy_file_to_upload_dir(). 429 """ 430 dest_filename_given = dest is not None 431 if upload_dir is None: 432 upload_dir = self.query_abs_dirs()["abs_upload_dir"] 433 if dest is None: 434 dest = os.path.basename(target) 435 if dest.endswith("/"): 436 dest_file = os.path.basename(target) 437 dest_dir = os.path.join(upload_dir, dest) 438 dest_filename_given = False 439 else: 440 dest_file = os.path.basename(dest) 441 dest_dir = os.path.join(upload_dir, os.path.dirname(dest)) 442 if compress and not dest_filename_given: 443 dest_file += ".gz" 444 dest = os.path.join(dest_dir, dest_file) 445 if not os.path.exists(target): 446 self.log("%s doesn't exist!" % target, level=error_level) 447 return None 448 self.mkdir_p(dest_dir) 449 self.copyfile(target, dest, log_level=log_level, compress=compress) 450 if os.path.exists(dest): 451 return dest 452 else: 453 self.log("%s doesn't exist after copy!" % dest, level=error_level) 454 return None 455 456 457 # main {{{1 458 if __name__ == "__main__": 459 myScript = OpenH264Build() 460 myScript.run_and_exit()