test_packager.py (23046B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import os 6 import unittest 7 8 import mozunit 9 from buildconfig import topobjdir 10 from mozunit import MockedOpen 11 12 import mozpack.path as mozpath 13 from mozbuild.preprocessor import Preprocessor 14 from mozpack.chrome.manifest import ( 15 ManifestBinaryComponent, 16 ManifestContent, 17 ManifestResource, 18 ) 19 from mozpack.errors import ErrorMessage, errors 20 from mozpack.files import GeneratedFile 21 from mozpack.packager import ( 22 CallDeque, 23 Component, 24 SimpleManifestSink, 25 SimplePackager, 26 preprocess_manifest, 27 ) 28 29 MANIFEST = """ 30 bar/* 31 [foo] 32 foo/* 33 -foo/bar 34 chrome.manifest 35 [zot destdir="destdir"] 36 foo/zot 37 ; comment 38 #ifdef baz 39 [baz] 40 baz@SUFFIX@ 41 #endif 42 """ 43 44 45 class TestPreprocessManifest(unittest.TestCase): 46 MANIFEST_PATH = mozpath.join("$OBJDIR", "manifest") 47 48 EXPECTED_LOG = [ 49 ((MANIFEST_PATH, 2), "add", "", "bar/*"), 50 ((MANIFEST_PATH, 4), "add", "foo", "foo/*"), 51 ((MANIFEST_PATH, 5), "remove", "foo", "foo/bar"), 52 ((MANIFEST_PATH, 6), "add", "foo", "chrome.manifest"), 53 ((MANIFEST_PATH, 8), "add", 'zot destdir="destdir"', "foo/zot"), 54 ] 55 56 def setUp(self): 57 class MockSink: 58 def __init__(self): 59 self.log = [] 60 61 def add(self, component, path): 62 self._log(errors.get_context(), "add", repr(component), path) 63 64 def remove(self, component, path): 65 self._log(errors.get_context(), "remove", repr(component), path) 66 67 def _log(self, *args): 68 self.log.append(args) 69 70 self.sink = MockSink() 71 self.cwd = os.getcwd() 72 os.chdir(topobjdir) 73 74 def tearDown(self): 75 os.chdir(self.cwd) 76 77 def test_preprocess_manifest(self): 78 with MockedOpen({"manifest": MANIFEST}): 79 preprocess_manifest(self.sink, "manifest") 80 self.assertEqual(self.sink.log, self.EXPECTED_LOG) 81 82 def test_preprocess_manifest_missing_define(self): 83 with MockedOpen({"manifest": MANIFEST}): 84 self.assertRaises( 85 Preprocessor.Error, 86 preprocess_manifest, 87 self.sink, 88 "manifest", 89 {"baz": 1}, 90 ) 91 92 def test_preprocess_manifest_defines(self): 93 with MockedOpen({"manifest": MANIFEST}): 94 preprocess_manifest(self.sink, "manifest", {"baz": 1, "SUFFIX": ".exe"}) 95 self.assertEqual( 96 self.sink.log, 97 self.EXPECTED_LOG + [((self.MANIFEST_PATH, 12), "add", "baz", "baz.exe")], 98 ) 99 100 101 class MockFinder: 102 def __init__(self, files): 103 self.files = files 104 self.log = [] 105 106 def find(self, path): 107 self.log.append(path) 108 for f in sorted(self.files): 109 if mozpath.match(f, path): 110 yield f, self.files[f] 111 112 def __iter__(self): 113 return self.find("") 114 115 116 class MockFormatter: 117 def __init__(self): 118 self.log = [] 119 120 def add_base(self, *args): 121 self._log(errors.get_context(), "add_base", *args) 122 123 def add_manifest(self, *args): 124 self._log(errors.get_context(), "add_manifest", *args) 125 126 def add_interfaces(self, *args): 127 self._log(errors.get_context(), "add_interfaces", *args) 128 129 def add(self, *args): 130 self._log(errors.get_context(), "add", *args) 131 132 def _log(self, *args): 133 self.log.append(args) 134 135 136 class TestSimplePackager(unittest.TestCase): 137 def test_simple_packager(self): 138 class GeneratedFileWithPath(GeneratedFile): 139 def __init__(self, path, content): 140 GeneratedFile.__init__(self, content) 141 self.path = path 142 143 formatter = MockFormatter() 144 packager = SimplePackager(formatter) 145 curdir = os.path.abspath(os.curdir) 146 file = GeneratedFileWithPath( 147 os.path.join(curdir, "foo", "bar.manifest"), 148 b"resource bar bar/\ncontent bar bar/", 149 ) 150 with errors.context("manifest", 1): 151 packager.add("foo/bar.manifest", file) 152 153 file = GeneratedFileWithPath( 154 os.path.join(curdir, "foo", "baz.manifest"), b"resource baz baz/" 155 ) 156 with errors.context("manifest", 2): 157 packager.add("bar/baz.manifest", file) 158 159 with errors.context("manifest", 3): 160 packager.add( 161 "qux/qux.manifest", 162 GeneratedFile( 163 b"".join([ 164 b"resource qux qux/\n", 165 b"binary-component qux.so\n", 166 ]) 167 ), 168 ) 169 bar_xpt = GeneratedFile(b"bar.xpt") 170 qux_xpt = GeneratedFile(b"qux.xpt") 171 foo_html = GeneratedFile(b"foo_html") 172 bar_html = GeneratedFile(b"bar_html") 173 with errors.context("manifest", 4): 174 packager.add("foo/bar.xpt", bar_xpt) 175 with errors.context("manifest", 5): 176 packager.add("foo/bar/foo.html", foo_html) 177 packager.add("foo/bar/bar.html", bar_html) 178 179 file = GeneratedFileWithPath( 180 os.path.join(curdir, "foo.manifest"), 181 b"".join([ 182 b"manifest foo/bar.manifest\n", 183 b"manifest bar/baz.manifest\n", 184 ]), 185 ) 186 with errors.context("manifest", 6): 187 packager.add("foo.manifest", file) 188 with errors.context("manifest", 7): 189 packager.add("foo/qux.xpt", qux_xpt) 190 191 file = GeneratedFileWithPath( 192 os.path.join(curdir, "addon", "chrome.manifest"), b"resource hoge hoge/" 193 ) 194 with errors.context("manifest", 8): 195 packager.add("addon/chrome.manifest", file) 196 197 install_rdf = GeneratedFile(b"<RDF></RDF>") 198 with errors.context("manifest", 9): 199 packager.add("addon/install.rdf", install_rdf) 200 201 with errors.context("manifest", 10): 202 packager.add("addon2/install.rdf", install_rdf) 203 packager.add( 204 "addon2/chrome.manifest", GeneratedFile(b"binary-component addon2.so") 205 ) 206 207 with errors.context("manifest", 11): 208 packager.add("addon3/install.rdf", install_rdf) 209 packager.add( 210 "addon3/chrome.manifest", 211 GeneratedFile(b"manifest components/components.manifest"), 212 ) 213 packager.add( 214 "addon3/components/components.manifest", 215 GeneratedFile(b"binary-component addon3.so"), 216 ) 217 218 with errors.context("manifest", 12): 219 install_rdf_addon4 = GeneratedFile( 220 b"<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>" 221 ) 222 packager.add("addon4/install.rdf", install_rdf_addon4) 223 224 with errors.context("manifest", 13): 225 install_rdf_addon5 = GeneratedFile( 226 b"<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>" 227 ) 228 packager.add("addon5/install.rdf", install_rdf_addon5) 229 230 with errors.context("manifest", 14): 231 install_rdf_addon6 = GeneratedFile( 232 b"<RDF>\n<... em:unpack=true>\n<...>\n</RDF>" 233 ) 234 packager.add("addon6/install.rdf", install_rdf_addon6) 235 236 with errors.context("manifest", 15): 237 install_rdf_addon7 = GeneratedFile( 238 b"<RDF>\n<... em:unpack=false>\n<...>\n</RDF>" 239 ) 240 packager.add("addon7/install.rdf", install_rdf_addon7) 241 242 with errors.context("manifest", 16): 243 install_rdf_addon8 = GeneratedFile( 244 b'<RDF>\n<... em:unpack="true">\n<...>\n</RDF>' 245 ) 246 packager.add("addon8/install.rdf", install_rdf_addon8) 247 248 with errors.context("manifest", 17): 249 install_rdf_addon9 = GeneratedFile( 250 b'<RDF>\n<... em:unpack="false">\n<...>\n</RDF>' 251 ) 252 packager.add("addon9/install.rdf", install_rdf_addon9) 253 254 with errors.context("manifest", 18): 255 install_rdf_addon10 = GeneratedFile( 256 b"<RDF>\n<... em:unpack='true'>\n<...>\n</RDF>" 257 ) 258 packager.add("addon10/install.rdf", install_rdf_addon10) 259 260 with errors.context("manifest", 19): 261 install_rdf_addon11 = GeneratedFile( 262 b"<RDF>\n<... em:unpack='false'>\n<...>\n</RDF>" 263 ) 264 packager.add("addon11/install.rdf", install_rdf_addon11) 265 266 we_manifest = GeneratedFile( 267 b'{"manifest_version": 2, "name": "Test WebExtension", "version": "1.0"}' 268 ) 269 # hybrid and hybrid2 are both bootstrapped extensions with 270 # embedded webextensions, they differ in the order in which 271 # the manifests are added to the packager. 272 with errors.context("manifest", 20): 273 packager.add("hybrid/install.rdf", install_rdf) 274 275 with errors.context("manifest", 21): 276 packager.add("hybrid/webextension/manifest.json", we_manifest) 277 278 with errors.context("manifest", 22): 279 packager.add("hybrid2/webextension/manifest.json", we_manifest) 280 281 with errors.context("manifest", 23): 282 packager.add("hybrid2/install.rdf", install_rdf) 283 284 with errors.context("manifest", 24): 285 packager.add("webextension/manifest.json", we_manifest) 286 287 non_we_manifest = GeneratedFile(b'{"not a webextension": true}') 288 with errors.context("manifest", 25): 289 packager.add("nonwebextension/manifest.json", non_we_manifest) 290 291 self.assertEqual(formatter.log, []) 292 293 with errors.context("dummy", 1): 294 packager.close() 295 self.maxDiff = None 296 # The formatter is expected to reorder the manifest entries so that 297 # chrome entries appear before the others. 298 self.assertEqual( 299 formatter.log, 300 [ 301 (("dummy", 1), "add_base", "", False), 302 (("dummy", 1), "add_base", "addon", True), 303 (("dummy", 1), "add_base", "addon10", "unpacked"), 304 (("dummy", 1), "add_base", "addon11", True), 305 (("dummy", 1), "add_base", "addon2", "unpacked"), 306 (("dummy", 1), "add_base", "addon3", "unpacked"), 307 (("dummy", 1), "add_base", "addon4", "unpacked"), 308 (("dummy", 1), "add_base", "addon5", True), 309 (("dummy", 1), "add_base", "addon6", "unpacked"), 310 (("dummy", 1), "add_base", "addon7", True), 311 (("dummy", 1), "add_base", "addon8", "unpacked"), 312 (("dummy", 1), "add_base", "addon9", True), 313 (("dummy", 1), "add_base", "hybrid", True), 314 (("dummy", 1), "add_base", "hybrid2", True), 315 (("dummy", 1), "add_base", "qux", False), 316 (("dummy", 1), "add_base", "webextension", True), 317 ( 318 (os.path.join(curdir, "foo", "bar.manifest"), 2), 319 "add_manifest", 320 ManifestContent("foo", "bar", "bar/"), 321 ), 322 ( 323 (os.path.join(curdir, "foo", "bar.manifest"), 1), 324 "add_manifest", 325 ManifestResource("foo", "bar", "bar/"), 326 ), 327 ( 328 ("bar/baz.manifest", 1), 329 "add_manifest", 330 ManifestResource("bar", "baz", "baz/"), 331 ), 332 ( 333 ("qux/qux.manifest", 1), 334 "add_manifest", 335 ManifestResource("qux", "qux", "qux/"), 336 ), 337 ( 338 ("qux/qux.manifest", 2), 339 "add_manifest", 340 ManifestBinaryComponent("qux", "qux.so"), 341 ), 342 (("manifest", 4), "add_interfaces", "foo/bar.xpt", bar_xpt), 343 (("manifest", 7), "add_interfaces", "foo/qux.xpt", qux_xpt), 344 ( 345 (os.path.join(curdir, "addon", "chrome.manifest"), 1), 346 "add_manifest", 347 ManifestResource("addon", "hoge", "hoge/"), 348 ), 349 ( 350 ("addon2/chrome.manifest", 1), 351 "add_manifest", 352 ManifestBinaryComponent("addon2", "addon2.so"), 353 ), 354 ( 355 ("addon3/components/components.manifest", 1), 356 "add_manifest", 357 ManifestBinaryComponent("addon3/components", "addon3.so"), 358 ), 359 (("manifest", 5), "add", "foo/bar/foo.html", foo_html), 360 (("manifest", 5), "add", "foo/bar/bar.html", bar_html), 361 (("manifest", 9), "add", "addon/install.rdf", install_rdf), 362 (("manifest", 10), "add", "addon2/install.rdf", install_rdf), 363 (("manifest", 11), "add", "addon3/install.rdf", install_rdf), 364 (("manifest", 12), "add", "addon4/install.rdf", install_rdf_addon4), 365 (("manifest", 13), "add", "addon5/install.rdf", install_rdf_addon5), 366 (("manifest", 14), "add", "addon6/install.rdf", install_rdf_addon6), 367 (("manifest", 15), "add", "addon7/install.rdf", install_rdf_addon7), 368 (("manifest", 16), "add", "addon8/install.rdf", install_rdf_addon8), 369 (("manifest", 17), "add", "addon9/install.rdf", install_rdf_addon9), 370 (("manifest", 18), "add", "addon10/install.rdf", install_rdf_addon10), 371 (("manifest", 19), "add", "addon11/install.rdf", install_rdf_addon11), 372 (("manifest", 20), "add", "hybrid/install.rdf", install_rdf), 373 ( 374 ("manifest", 21), 375 "add", 376 "hybrid/webextension/manifest.json", 377 we_manifest, 378 ), 379 ( 380 ("manifest", 22), 381 "add", 382 "hybrid2/webextension/manifest.json", 383 we_manifest, 384 ), 385 (("manifest", 23), "add", "hybrid2/install.rdf", install_rdf), 386 (("manifest", 24), "add", "webextension/manifest.json", we_manifest), 387 ( 388 ("manifest", 25), 389 "add", 390 "nonwebextension/manifest.json", 391 non_we_manifest, 392 ), 393 ], 394 ) 395 396 self.assertEqual( 397 packager.get_bases(), 398 set([ 399 "", 400 "addon", 401 "addon2", 402 "addon3", 403 "addon4", 404 "addon5", 405 "addon6", 406 "addon7", 407 "addon8", 408 "addon9", 409 "addon10", 410 "addon11", 411 "qux", 412 "hybrid", 413 "hybrid2", 414 "webextension", 415 ]), 416 ) 417 self.assertEqual(packager.get_bases(addons=False), set(["", "qux"])) 418 419 def test_simple_packager_manifest_consistency(self): 420 formatter = MockFormatter() 421 # bar/ is detected as an addon because of install.rdf, but top-level 422 # includes a manifest inside bar/. 423 packager = SimplePackager(formatter) 424 packager.add( 425 "base.manifest", 426 GeneratedFile(b"manifest foo/bar.manifest\nmanifest bar/baz.manifest\n"), 427 ) 428 packager.add("foo/bar.manifest", GeneratedFile(b"resource bar bar")) 429 packager.add("bar/baz.manifest", GeneratedFile(b"resource baz baz")) 430 packager.add("bar/install.rdf", GeneratedFile(b"")) 431 432 with self.assertRaises(ErrorMessage) as e: 433 packager.close() 434 435 self.assertEqual( 436 str(e.exception), 437 'error: "bar/baz.manifest" is included from "base.manifest", ' 438 'which is outside "bar"', 439 ) 440 441 # bar/ is detected as a separate base because of chrome.manifest that 442 # is included nowhere, but top-level includes another manifest inside 443 # bar/. 444 packager = SimplePackager(formatter) 445 packager.add( 446 "base.manifest", 447 GeneratedFile(b"manifest foo/bar.manifest\nmanifest bar/baz.manifest\n"), 448 ) 449 packager.add("foo/bar.manifest", GeneratedFile(b"resource bar bar")) 450 packager.add("bar/baz.manifest", GeneratedFile(b"resource baz baz")) 451 packager.add("bar/chrome.manifest", GeneratedFile(b"resource baz baz")) 452 453 with self.assertRaises(ErrorMessage) as e: 454 packager.close() 455 456 self.assertEqual( 457 str(e.exception), 458 'error: "bar/baz.manifest" is included from "base.manifest", ' 459 'which is outside "bar"', 460 ) 461 462 # bar/ is detected as a separate base because of chrome.manifest that 463 # is included nowhere, but chrome.manifest includes baz.manifest from 464 # the same directory. This shouldn't error out. 465 packager = SimplePackager(formatter) 466 packager.add("base.manifest", GeneratedFile(b"manifest foo/bar.manifest\n")) 467 packager.add("foo/bar.manifest", GeneratedFile(b"resource bar bar")) 468 packager.add("bar/baz.manifest", GeneratedFile(b"resource baz baz")) 469 packager.add("bar/chrome.manifest", GeneratedFile(b"manifest baz.manifest")) 470 packager.close() 471 472 473 class TestSimpleManifestSink(unittest.TestCase): 474 def test_simple_manifest_parser(self): 475 formatter = MockFormatter() 476 foobar = GeneratedFile(b"foobar") 477 foobaz = GeneratedFile(b"foobaz") 478 fooqux = GeneratedFile(b"fooqux") 479 foozot = GeneratedFile(b"foozot") 480 finder = MockFinder({ 481 "bin/foo/bar": foobar, 482 "bin/foo/baz": foobaz, 483 "bin/foo/qux": fooqux, 484 "bin/foo/zot": foozot, 485 "bin/foo/chrome.manifest": GeneratedFile(b"resource foo foo/"), 486 "bin/chrome.manifest": GeneratedFile(b"manifest foo/chrome.manifest"), 487 }) 488 parser = SimpleManifestSink(finder, formatter) 489 component0 = Component("component0") 490 component1 = Component("component1") 491 component2 = Component("component2", destdir="destdir") 492 parser.add(component0, "bin/foo/b*") 493 parser.add(component1, "bin/foo/qux") 494 parser.add(component1, "bin/foo/chrome.manifest") 495 parser.add(component2, "bin/foo/zot") 496 self.assertRaises(ErrorMessage, parser.add, "component1", "bin/bar") 497 498 self.assertEqual(formatter.log, []) 499 parser.close() 500 self.assertEqual( 501 formatter.log, 502 [ 503 (None, "add_base", "", False), 504 ( 505 ("foo/chrome.manifest", 1), 506 "add_manifest", 507 ManifestResource("foo", "foo", "foo/"), 508 ), 509 (None, "add", "foo/bar", foobar), 510 (None, "add", "foo/baz", foobaz), 511 (None, "add", "foo/qux", fooqux), 512 (None, "add", "destdir/foo/zot", foozot), 513 ], 514 ) 515 516 self.assertEqual( 517 finder.log, 518 [ 519 "bin/foo/b*", 520 "bin/foo/qux", 521 "bin/foo/chrome.manifest", 522 "bin/foo/zot", 523 "bin/bar", 524 "bin/chrome.manifest", 525 ], 526 ) 527 528 529 class TestCallDeque(unittest.TestCase): 530 def test_call_deque(self): 531 class Logger: 532 def __init__(self): 533 self._log = [] 534 535 def log(self, str): 536 self._log.append(str) 537 538 @staticmethod 539 def staticlog(logger, str): 540 logger.log(str) 541 542 def do_log(logger, str): 543 logger.log(str) 544 545 logger = Logger() 546 d = CallDeque() 547 d.append(logger.log, "foo") 548 d.append(logger.log, "bar") 549 d.append(logger.staticlog, logger, "baz") 550 d.append(do_log, logger, "qux") 551 self.assertEqual(logger._log, []) 552 d.execute() 553 self.assertEqual(logger._log, ["foo", "bar", "baz", "qux"]) 554 555 556 class TestComponent(unittest.TestCase): 557 def do_split(self, string, name, options): 558 n, o = Component._split_component_and_options(string) 559 self.assertEqual(name, n) 560 self.assertEqual(options, o) 561 562 def test_component_split_component_and_options(self): 563 self.do_split("component", "component", {}) 564 self.do_split("trailingspace ", "trailingspace", {}) 565 self.do_split(" leadingspace", "leadingspace", {}) 566 self.do_split(" trim ", "trim", {}) 567 self.do_split(' trim key="value"', "trim", {"key": "value"}) 568 self.do_split(' trim empty=""', "trim", {"empty": ""}) 569 self.do_split(' trim space=" "', "trim", {"space": " "}) 570 self.do_split( 571 'component key="value" key2="second" ', 572 "component", 573 {"key": "value", "key2": "second"}, 574 ) 575 self.do_split( 576 'trim key=" value with spaces " key2="spaces again"', 577 "trim", 578 {"key": " value with spaces ", "key2": "spaces again"}, 579 ) 580 581 def do_split_error(self, string): 582 self.assertRaises(ValueError, Component._split_component_and_options, string) 583 584 def test_component_split_component_and_options_errors(self): 585 self.do_split_error('"component') 586 self.do_split_error('comp"onent') 587 self.do_split_error('component"') 588 self.do_split_error('"component"') 589 self.do_split_error("=component") 590 self.do_split_error("comp=onent") 591 self.do_split_error("component=") 592 self.do_split_error('key="val"') 593 self.do_split_error("component key=") 594 self.do_split_error('component key="val') 595 self.do_split_error('component key=val"') 596 self.do_split_error('component key="val" x') 597 self.do_split_error('component x key="val"') 598 self.do_split_error('component key1="val" x key2="val"') 599 600 def do_from_string(self, string, name, destdir=""): 601 component = Component.from_string(string) 602 self.assertEqual(name, component.name) 603 self.assertEqual(destdir, component.destdir) 604 605 def test_component_from_string(self): 606 self.do_from_string("component", "component") 607 self.do_from_string("component-with-hyphen", "component-with-hyphen") 608 self.do_from_string('component destdir="foo/bar"', "component", "foo/bar") 609 self.do_from_string('component destdir="bar spc"', "component", "bar spc") 610 self.assertRaises(ErrorMessage, Component.from_string, "") 611 self.assertRaises(ErrorMessage, Component.from_string, "component novalue=") 612 self.assertRaises( 613 ErrorMessage, Component.from_string, "component badoption=badvalue" 614 ) 615 616 617 if __name__ == "__main__": 618 mozunit.main()