commit b2b8db285933dcc780fbdba9eb6cc5e82fb3f172 parent e93300dce09e802a3e1aa420126e9cf134c8130d Author: Alex Hochheiden <ahochheiden@mozilla.com> Date: Wed, 7 Jan 2026 17:17:26 +0000 Bug 2006716 - Reformat entire repo with `ruff-format` (`black` replacement) r=ahal,sylvestre,linter-reviewers,perftest-reviewers,mozperftest-reviewers,geckoview-reviewers,extension-reviewers,media-playback-reviewers,webrtc-reviewers,releng-reviewers,sessionstore-reviewers,win-reviewers,dom-storage-reviewers,translations-reviewers,omc-reviewers,android-reviewers,home-newtab-reviewers,toolkit-telemetry-reviewers,firefox-style-system-reviewers,layout-reviewers,dom-worker-reviewers,geckodriver-reviewers,Sasha,emz,fchasen,sthompson,dholbert,sparky,padenot,emilio,Gijs,mconley,tcampbell,mjf,emcminn,willdurand,gstoll,jgraham,asuth # ignore-this-changeset Differential Revision: https://phabricator.services.mozilla.com/D276894 Diffstat:
746 files changed, 10995 insertions(+), 12594 deletions(-)
diff --git a/accessible/tests/browser/windows/a11y_setup.py b/accessible/tests/browser/windows/a11y_setup.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -"""Python environment for Windows a11y browser tests. -""" +"""Python environment for Windows a11y browser tests.""" import ctypes import os diff --git a/browser/components/asrouter/content-src/schemas/make-schemas.py b/browser/components/asrouter/content-src/schemas/make-schemas.py @@ -262,13 +262,11 @@ def bundle_schema(schema_def: SchemaDefinition): # patch_schema mutates the given schema, so we read a new copy in for # each bundle operation. - defs.update( - { - name: dfn - for name, dfn in common_schema["$defs"].items() - if dfn_filter(name) - } - ) + defs.update({ + name: dfn + for name, dfn in common_schema["$defs"].items() + if dfn_filter(name) + }) # Ensure all bundled schemas have an $id so that $refs inside the # bundled schema work correctly (i.e, they will reference the subschema diff --git a/browser/components/backup/tests/marionette/test_backup.py b/browser/components/backup/tests/marionette/test_backup.py @@ -21,19 +21,17 @@ class BackupTest(MarionetteTestCase): # We need to force the service to be enabled because it's disabled # by default for Marionette. Also "browser.backup.log" has to be set # to true before Firefox starts in order for it to be displayed. - self.marionette.enforce_gecko_prefs( - { - "browser.backup.enabled": True, - "browser.backup.log": True, - "browser.backup.archive.enabled": True, - "browser.backup.restore.enabled": True, - "browser.backup.archive.overridePlatformCheck": True, - "browser.backup.restore.overridePlatformCheck": True, - # Necessary to test Session Restore from backup, which relies on - # the crash restore mechanism. - "browser.sessionstore.resume_from_crash": True, - } - ) + self.marionette.enforce_gecko_prefs({ + "browser.backup.enabled": True, + "browser.backup.log": True, + "browser.backup.archive.enabled": True, + "browser.backup.restore.enabled": True, + "browser.backup.archive.overridePlatformCheck": True, + "browser.backup.restore.overridePlatformCheck": True, + # Necessary to test Session Restore from backup, which relies on + # the crash restore mechanism. + "browser.sessionstore.resume_from_crash": True, + }) self.marionette.set_context("chrome") diff --git a/browser/components/search/test/marionette/telemetry/test_ping_submitted.py b/browser/components/search/test/marionette/telemetry/test_ping_submitted.py @@ -12,13 +12,11 @@ class TestPingSubmitted(MarionetteTestCase): self.marionette.set_context(self.marionette.CONTEXT_CHROME) - self.marionette.enforce_gecko_prefs( - { - "datareporting.healthreport.uploadEnabled": True, - "telemetry.fog.test.localhost_port": 3000, - "browser.search.log": True, - } - ) + self.marionette.enforce_gecko_prefs({ + "datareporting.healthreport.uploadEnabled": True, + "telemetry.fog.test.localhost_port": 3000, + "browser.search.log": True, + }) # The categorization ping is submitted on startup. If anything delays # its initialization, turning the preference on and immediately # attaching a categorization event could result in the ping being diff --git a/browser/components/search/test/marionette/test_engines_on_restart.py b/browser/components/search/test/marionette/test_engines_on_restart.py @@ -10,11 +10,9 @@ from marionette_harness.marionette_test import MarionetteTestCase class TestEnginesOnRestart(MarionetteTestCase): def setUp(self): super().setUp() - self.marionette.enforce_gecko_prefs( - { - "browser.search.log": True, - } - ) + self.marionette.enforce_gecko_prefs({ + "browser.search.log": True, + }) def get_default_search_engine(self): """Retrieve the identifier of the default search engine.""" diff --git a/browser/components/sessionstore/test/marionette/session_store_test_case.py b/browser/components/sessionstore/test/marionette/session_store_test_case.py @@ -15,23 +15,21 @@ def inline(doc): # Each list element represents a window of tabs loaded at # some testing URL -DEFAULT_WINDOWS = set( - [ - # Window 1. Note the comma after the inline call - - # this is Python's way of declaring a 1 item tuple. - (inline("""<div">Lorem</div>"""),), - # Window 2 - ( - inline("""<div">ipsum</div>"""), - inline("""<div">dolor</div>"""), - ), - # Window 3 - ( - inline("""<div">sit</div>"""), - inline("""<div">amet</div>"""), - ), - ] -) +DEFAULT_WINDOWS = set([ + # Window 1. Note the comma after the inline call - + # this is Python's way of declaring a 1 item tuple. + (inline("""<div">Lorem</div>"""),), + # Window 2 + ( + inline("""<div">ipsum</div>"""), + inline("""<div">dolor</div>"""), + ), + # Window 3 + ( + inline("""<div">sit</div>"""), + inline("""<div">amet</div>"""), + ), +]) class SessionStoreTestCase(WindowManagerMixin, MarionetteTestCase): @@ -53,37 +51,33 @@ class SessionStoreTestCase(WindowManagerMixin, MarionetteTestCase): self.test_windows = test_windows - self.private_windows = set( - [ - ( - inline("""<div">consectetur</div>"""), - inline("""<div">ipsum</div>"""), - ), - ( - inline("""<div">adipiscing</div>"""), - inline("""<div">consectetur</div>"""), - ), - ] - ) - - self.marionette.enforce_gecko_prefs( - { - # Set browser restore previous session pref, - # depending on what the test requires. - "browser.startup.page": startup_page, - # Make the content load right away instead of waiting for - # the user to click on the background tabs - "browser.sessionstore.restore_on_demand": restore_on_demand, - # Avoid race conditions by having the content process never - # send us session updates unless the parent has explicitly asked - # for them via the TabStateFlusher. - "browser.sessionstore.debug.no_auto_updates": no_auto_updates, - # Whether to enable the register application restart mechanism. - "toolkit.winRegisterApplicationRestart": win_register_restart, - # Whether to enable taskbar tabs for this test - "browser.taskbarTabs.enabled": taskbartabs_enable, - } - ) + self.private_windows = set([ + ( + inline("""<div">consectetur</div>"""), + inline("""<div">ipsum</div>"""), + ), + ( + inline("""<div">adipiscing</div>"""), + inline("""<div">consectetur</div>"""), + ), + ]) + + self.marionette.enforce_gecko_prefs({ + # Set browser restore previous session pref, + # depending on what the test requires. + "browser.startup.page": startup_page, + # Make the content load right away instead of waiting for + # the user to click on the background tabs + "browser.sessionstore.restore_on_demand": restore_on_demand, + # Avoid race conditions by having the content process never + # send us session updates unless the parent has explicitly asked + # for them via the TabStateFlusher. + "browser.sessionstore.debug.no_auto_updates": no_auto_updates, + # Whether to enable the register application restart mechanism. + "toolkit.winRegisterApplicationRestart": win_register_restart, + # Whether to enable taskbar tabs for this test + "browser.taskbarTabs.enabled": taskbartabs_enable, + }) self.all_windows = self.test_windows.copy() self.open_windows(self.test_windows) diff --git a/browser/components/sessionstore/test/marionette/test_log_files.py b/browser/components/sessionstore/test/marionette/test_log_files.py @@ -16,12 +16,10 @@ def inline(doc): class TestSessionRestoreLogging(WindowManagerMixin, MarionetteTestCase): def setUp(self): super().setUp() - self.marionette.enforce_gecko_prefs( - { - "browser.sessionstore.loglevel": "Debug", - "browser.sessionstore.log.appender.file.logOnSuccess": True, - } - ) + self.marionette.enforce_gecko_prefs({ + "browser.sessionstore.loglevel": "Debug", + "browser.sessionstore.log.appender.file.logOnSuccess": True, + }) def tearDown(self): try: @@ -101,11 +99,9 @@ class TestSessionRestoreLogging(WindowManagerMixin, MarionetteTestCase): self.marionette.start_session() def test_errors_flush_to_disk(self): - self.marionette.enforce_gecko_prefs( - { - "browser.sessionstore.log.appender.file.logOnSuccess": False, - } - ) + self.marionette.enforce_gecko_prefs({ + "browser.sessionstore.log.appender.file.logOnSuccess": False, + }) self.marionette.quit() sessionFile = self.getSessionFilePath() self.assertTrue( @@ -114,7 +110,7 @@ class TestSessionRestoreLogging(WindowManagerMixin, MarionetteTestCase): ) # replace the contents with nonsense so we get a not-readable error on startup with open(sessionFile, "wb") as f: - f.write(b"\x00\xFF\xABgarbageDATA") + f.write(b"\x00\xff\xabgarbageDATA") self.marionette.start_session() self.marionette.set_context("chrome") diff --git a/browser/components/sessionstore/test/marionette/test_persist_closed_tabs_restore_manually.py b/browser/components/sessionstore/test/marionette/test_persist_closed_tabs_restore_manually.py @@ -33,15 +33,13 @@ class TestSessionRestoreClosedTabs(SessionStoreTestCase): startup_page=1, include_private=False, restore_on_demand=True, - test_windows=set( - [ - # Window 1 - ( - inline("lorem ipsom"), - inline("dolor"), - ), - ] - ), + test_windows=set([ + # Window 1 + ( + inline("lorem ipsom"), + inline("dolor"), + ), + ]), ) def test_restore(self): diff --git a/browser/components/sessionstore/test/marionette/test_restore_manually.py b/browser/components/sessionstore/test/marionette/test_restore_manually.py @@ -26,17 +26,15 @@ class TestSessionRestoreManually(SessionStoreTestCase): startup_page=1, include_private=False, restore_on_demand=True, - test_windows=set( - [ - # Window 1 - ( - inline("lorem ipsom"), - inline("dolor"), - ), - # Window 2 - (inline("sit"),), - ] - ), + test_windows=set([ + # Window 1 + ( + inline("lorem ipsom"), + inline("dolor"), + ), + # Window 2 + (inline("sit"),), + ]), ) def test_restore(self): diff --git a/browser/components/sessionstore/test/marionette/test_restore_manually_with_pinned_tabs.py b/browser/components/sessionstore/test/marionette/test_restore_manually_with_pinned_tabs.py @@ -23,16 +23,14 @@ class TestSessionRestoreWithPinnedTabs(SessionStoreTestCase): startup_page=1, include_private=False, restore_on_demand=True, - test_windows=set( - [ - # Window 1 - ( - inline("""<div">ipsum</div>"""), - inline("""<div">dolor</div>"""), - inline("""<div">amet</div>"""), - ), - ] - ), + test_windows=set([ + # Window 1 + ( + inline("""<div">ipsum</div>"""), + inline("""<div">dolor</div>"""), + inline("""<div">amet</div>"""), + ), + ]), ) def test_no_restore_with_quit(self): diff --git a/browser/components/sessionstore/test/marionette/test_restore_manually_with_tab_groups.py b/browser/components/sessionstore/test/marionette/test_restore_manually_with_tab_groups.py @@ -23,17 +23,15 @@ class TestSessionRestoreWithTabGroups(SessionStoreTestCase): startup_page=1, include_private=False, restore_on_demand=True, - test_windows=set( - [ - ( - inline("""<div">lorem</div>"""), - inline("""<div">ipsum</div>"""), - inline("""<div">dolor</div>"""), - inline("""<div">sit</div>"""), - inline("""<div">amet</div>"""), - ), - ] - ), + test_windows=set([ + ( + inline("""<div">lorem</div>"""), + inline("""<div">ipsum</div>"""), + inline("""<div">dolor</div>"""), + inline("""<div">sit</div>"""), + inline("""<div">amet</div>"""), + ), + ]), ) def test_no_restore_with_quit(self): diff --git a/browser/components/sessionstore/test/marionette/test_restore_sidebar.py b/browser/components/sessionstore/test/marionette/test_restore_sidebar.py @@ -25,14 +25,12 @@ class TestSessionRestore(SessionStoreTestCase): startup_page=1, include_private=False, restore_on_demand=True, - test_windows=set( - [ - ( - inline("lorem ipsom"), - inline("dolor"), - ), - ] - ), + test_windows=set([ + ( + inline("lorem ipsom"), + inline("dolor"), + ), + ]), ) def test_restore_sidebar_open(self): diff --git a/browser/components/sessionstore/test/marionette/test_restore_sidebar_automatic.py b/browser/components/sessionstore/test/marionette/test_restore_sidebar_automatic.py @@ -25,14 +25,12 @@ class TestSessionRestore(SessionStoreTestCase): startup_page=3, include_private=False, restore_on_demand=False, - test_windows=set( - [ - ( - inline("lorem ipsom"), - inline("dolor"), - ), - ] - ), + test_windows=set([ + ( + inline("lorem ipsom"), + inline("dolor"), + ), + ]), ) def test_restore(self): diff --git a/browser/components/sessionstore/test/marionette/test_tabgroups_restore.py b/browser/components/sessionstore/test/marionette/test_tabgroups_restore.py @@ -19,13 +19,11 @@ def inline(doc): # Each list element represents a window of tabs loaded at # some testing URL -DEFAULT_WINDOWS = set( - [ - # Window 1. Note the comma after the inline call - - # this is Python's way of declaring a 1 item tuple. - (inline("""<div">Lorem</div>"""), inline("""<div">Ipsum</div>""")), - ] -) +DEFAULT_WINDOWS = set([ + # Window 1. Note the comma after the inline call - + # this is Python's way of declaring a 1 item tuple. + (inline("""<div">Lorem</div>"""), inline("""<div">Ipsum</div>""")), +]) class TestAutoRestoreWithTabGroups(SessionStoreTestCase): diff --git a/browser/components/sessionstore/test/marionette/test_taskbartab_restore.py b/browser/components/sessionstore/test/marionette/test_taskbartab_restore.py @@ -24,15 +24,13 @@ class TestManualRestoreWithTaskbarTabs(SessionStoreTestCase): include_private=False, restore_on_demand=False, taskbartabs_enable=True, - test_windows=set( - [ - # Window 1 - ( - inline("lorem ipsom"), - inline("dolor"), - ), - ] - ), + test_windows=set([ + # Window 1 + ( + inline("lorem ipsom"), + inline("dolor"), + ), + ]), ) """ @@ -94,15 +92,13 @@ class TestAutoRestoreWithTaskbarTabs(SessionStoreTestCase): include_private=False, restore_on_demand=False, taskbartabs_enable=True, - test_windows=set( - [ - # Window 1 - ( - inline("lorem ipsom"), - inline("dolor"), - ), - ] - ), + test_windows=set([ + # Window 1 + ( + inline("lorem ipsom"), + inline("dolor"), + ), + ]), ) """ diff --git a/browser/components/sessionstore/test/marionette/test_taskbartab_sessionstate.py b/browser/components/sessionstore/test/marionette/test_taskbartab_sessionstate.py @@ -22,15 +22,13 @@ class TestTaskbarTabSessionState(SessionStoreTestCase): include_private=False, restore_on_demand=False, taskbartabs_enable=True, - test_windows=set( - [ - # Window 1 - ( - inline("lorem ipsom"), - inline("dolor"), - ), - ] - ), + test_windows=set([ + # Window 1 + ( + inline("lorem ipsom"), + inline("dolor"), + ), + ]), ) """ diff --git a/browser/components/sidebar/tests/marionette/test_default_launcher_visible.py b/browser/components/sidebar/tests/marionette/test_default_launcher_visible.py @@ -15,7 +15,6 @@ initial_prefs = { class TestDefaultLauncherVisible(MarionetteTestCase): - def setUp(self): MarionetteTestCase.setUp(self) @@ -106,12 +105,10 @@ class TestDefaultLauncherVisible(MarionetteTestCase): ) # Mimic an update which enables sidebar.revamp for the first time - self.restart_with_prefs( - { - "sidebar.revamp": True, - "browser.startup.page": 3, - } - ) + self.restart_with_prefs({ + "sidebar.revamp": True, + "browser.startup.page": 3, + }) self.assertTrue( self.is_button_visible(), @@ -212,12 +209,10 @@ class TestDefaultLauncherVisible(MarionetteTestCase): ) # This mocks the enrollment in which Nimbus sets the following prefs - self.marionette.set_prefs( - { - "sidebar.revamp": True, - "sidebar.revamp.defaultLauncherVisible": False, - } - ) + self.marionette.set_prefs({ + "sidebar.revamp": True, + "sidebar.revamp.defaultLauncherVisible": False, + }) # We expect enabling the pref to add the button to the toolbar Wait(self.marionette).until( @@ -244,13 +239,11 @@ class TestDefaultLauncherVisible(MarionetteTestCase): def test_vertical_tabs_default_hidden(self): # Verify that starting with verticalTabs enabled and default visibility false results in a visible # launcher with the vertical tabstrip - self.restart_with_prefs( - { - "sidebar.revamp": True, - "sidebar.verticalTabs": True, - "sidebar.visibility": "always-show", - } - ) + self.restart_with_prefs({ + "sidebar.revamp": True, + "sidebar.verticalTabs": True, + "sidebar.visibility": "always-show", + }) Wait(self.marionette).until( lambda _: self.is_launcher_visible(), diff --git a/browser/components/sidebar/tests/marionette/test_initialize_vertical_tabs.py b/browser/components/sidebar/tests/marionette/test_initialize_vertical_tabs.py @@ -97,14 +97,12 @@ class TestInitializeVerticalTabs(MarionetteTestCase): def test_vertical_widgets_in_area(self): # A clean startup in verticalTabs mode; we should get all the defaults - self.restart_with_prefs( - { - "sidebar.revamp": True, - "sidebar.verticalTabs": True, - customization_pref: None, - snapshot_pref: None, - } - ) + self.restart_with_prefs({ + "sidebar.revamp": True, + "sidebar.verticalTabs": True, + customization_pref: None, + snapshot_pref: None, + }) horiz_tab_ids = self.get_area_widgets("AREA_TABSTRIP") vertical_tab_ids = self.get_area_widgets("AREA_VERTICAL_TABSTRIP") @@ -153,13 +151,11 @@ class TestInitializeVerticalTabs(MarionetteTestCase): "sidebar.revamp": True, "sidebar.verticalTabs": False, } - self.restart_with_prefs( - { - **fixture_prefs, - customization_pref: None, - snapshot_pref: None, - } - ) + self.restart_with_prefs({ + **fixture_prefs, + customization_pref: None, + snapshot_pref: None, + }) # Add a widget at the start of the horizontal tabstrip # This is synchronous and should result in updating the UI and the saved state in uiCustomization pref @@ -222,13 +218,11 @@ class TestInitializeVerticalTabs(MarionetteTestCase): "sidebar.verticalTabs": True, "sidebar.visibility": "hide-sidebar", } - self.restart_with_prefs( - { - **fixture_prefs, - customization_pref: None, - snapshot_pref: None, - } - ) + self.restart_with_prefs({ + **fixture_prefs, + customization_pref: None, + snapshot_pref: None, + }) pref_value = self.marionette.execute_script( """ @@ -265,12 +259,10 @@ class TestInitializeVerticalTabs(MarionetteTestCase): def test_hide_drag_to_pin_promo_if_horizontal_tabs_pinned(self): # Pin a tab using the horizontal tabstrip. - self.restart_with_prefs( - { - "sidebar.revamp": False, - "sidebar.verticalTabs": False, - } - ) + self.restart_with_prefs({ + "sidebar.revamp": False, + "sidebar.verticalTabs": False, + }) self.marionette.execute_async_script( """ let resolve = arguments[0]; diff --git a/browser/extensions/newtab/mach_commands.py b/browser/extensions/newtab/mach_commands.py @@ -104,9 +104,13 @@ def watch(command_context): processes = [] try: - p1 = subprocess.Popen( - ["./mach", "npm", "run", "watchmc", "--prefix=browser/extensions/newtab"] - ) + p1 = subprocess.Popen([ + "./mach", + "npm", + "run", + "watchmc", + "--prefix=browser/extensions/newtab", + ]) p2 = subprocess.Popen(["./mach", "watch"]) processes.extend([p1, p2]) print("Watching subprocesses started. Press Ctrl-C to terminate them.") @@ -145,9 +149,13 @@ def update_locales(command_context): # Step 1: We download the latest reckoning of strings from firefox-l10n print("Cloning the latest HEAD of firefox-l10n repository") with tempfile.TemporaryDirectory() as clone_dir: - subprocess.check_call( - ["git", "clone", "--depth=1", FIREFOX_L10N_REPO, clone_dir] - ) + subprocess.check_call([ + "git", + "clone", + "--depth=1", + FIREFOX_L10N_REPO, + clone_dir, + ]) # Step 2: Get some metadata about what we just pulled down - # specifically, the revision. revision = subprocess.check_output( @@ -831,9 +839,13 @@ def bundle(command_context): proc = None try: - proc = subprocess.Popen( - ["./mach", "npm", "run", "bundle", "--prefix=browser/extensions/newtab"] - ) + proc = subprocess.Popen([ + "./mach", + "npm", + "run", + "bundle", + "--prefix=browser/extensions/newtab", + ]) print("Bundling newtab started. Press Ctrl-C to terminate.") proc.wait() except KeyboardInterrupt: @@ -871,9 +883,12 @@ def install(command_context): proc = None try: - proc = subprocess.Popen( - ["./mach", "npm", "install", "--prefix=browser/extensions/newtab"] - ) + proc = subprocess.Popen([ + "./mach", + "npm", + "install", + "--prefix=browser/extensions/newtab", + ]) print( "Installing node dependencies for newtab started. Press Ctrl-C to terminate." ) diff --git a/build/RunCbindgen.py b/build/RunCbindgen.py @@ -71,19 +71,17 @@ def generate_metadata(output, cargo_config): def generate(output, metadata_path, cbindgen_crate_path, *in_tree_dependencies): - stdout, returncode = _run_process( - [ - buildconfig.substs["CBINDGEN"], - buildconfig.topsrcdir, - "--lockfile", - CARGO_LOCK, - "--crate", - _get_crate_name(cbindgen_crate_path), - "--metadata", - metadata_path, - "--cpp-compat", - ] - ) + stdout, returncode = _run_process([ + buildconfig.substs["CBINDGEN"], + buildconfig.topsrcdir, + "--lockfile", + CARGO_LOCK, + "--crate", + _get_crate_name(cbindgen_crate_path), + "--metadata", + metadata_path, + "--cpp-compat", + ]) if returncode != 0: return returncode diff --git a/build/appini_header.py b/build/appini_header.py @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Parses a given application.ini file and outputs the corresponding - StaticXREAppData structure as a C++ header file""" +StaticXREAppData structure as a C++ header file""" import configparser import sys diff --git a/build/gen_symverscript.py b/build/gen_symverscript.py @@ -11,11 +11,9 @@ from mozbuild.preprocessor import Preprocessor def main(output, input_file, version): pp = Preprocessor() - pp.context.update( - { - "VERSION": version, - } - ) + pp.context.update({ + "VERSION": version, + }) pp.out = output pp.do_include(input_file) diff --git a/build/gen_test_packages_manifest.py b/build/gen_test_packages_manifest.py @@ -62,7 +62,7 @@ def parse_args(): required=True, action="store", dest="tests_common", - help='Name of the "common" archive, a package to be used by all ' "harnesses.", + help='Name of the "common" archive, a package to be used by all harnesses.', ) parser.add_argument( "--jsshell", diff --git a/build/gn_processor.py b/build/gn_processor.py @@ -609,13 +609,11 @@ def write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs): conditions.add(cond) for cond in sorted(conditions): - common_attrs = find_common_attrs( - [ - attrs - for args, attrs in configs - if all((args.get(k) or "") == v for k, v in cond) - ] - ) + common_attrs = find_common_attrs([ + attrs + for args, attrs in configs + if all((args.get(k) or "") == v for k, v in cond) + ]) if any(common_attrs.values()): if cond: mb.write_condition(dict(cond)) @@ -726,31 +724,25 @@ def generate_gn_config( srcdir = build_root_dir / target_dir input_variables = input_variables.copy() - input_variables.update( - { - f"{moz_build_flag}": True, - "concurrent_links": 1, - "action_pool_depth": 1, - } - ) + input_variables.update({ + f"{moz_build_flag}": True, + "concurrent_links": 1, + "action_pool_depth": 1, + }) if input_variables["target_os"] == "win": - input_variables.update( - { - "visual_studio_path": "/", - "visual_studio_version": 2015, - "wdk_path": "/", - "windows_sdk_version": "n/a", - } - ) + input_variables.update({ + "visual_studio_path": "/", + "visual_studio_version": 2015, + "wdk_path": "/", + "windows_sdk_version": "n/a", + }) if input_variables["target_os"] == "mac": - input_variables.update( - { - "mac_sdk_path": "/", - } - ) + input_variables.update({ + "mac_sdk_path": "/", + }) - gn_args = f'--args={" ".join([f"{k}={str_for_arg(v)}" for k, v in input_variables.items()])}' + gn_args = f"--args={' '.join([f'{k}={str_for_arg(v)}' for k, v in input_variables.items()])}" with tempfile.TemporaryDirectory() as tempdir: # On Mac, `tempdir` starts with /var which is a symlink to /private/var. # We resolve the symlinks in `tempdir` here so later usage with diff --git a/build/test/python/test_android_gradle_build.py b/build/test/python/test_android_gradle_build.py @@ -186,13 +186,13 @@ def assert_all_task_statuses(objdir, acceptable_statuses, always_executed_tasks= actual_status = task.get("status") if task_name in always_executed_tasks: - assert ( - actual_status == "EXECUTED" - ), f"Task {task_name} should always execute, got '{actual_status}'" + assert actual_status == "EXECUTED", ( + f"Task {task_name} should always execute, got '{actual_status}'" + ) else: - assert ( - actual_status in acceptable_statuses - ), f"Task {task_name} had status '{actual_status}', expected one of {acceptable_statuses}" + assert actual_status in acceptable_statuses, ( + f"Task {task_name} had status '{actual_status}', expected one of {acceptable_statuses}" + ) def assert_ordered_task_outcomes(objdir, ordered_expected_task_statuses): @@ -219,18 +219,18 @@ def assert_ordered_task_outcomes(objdir, ordered_expected_task_statuses): assert not missing_tasks, f"Tasks not found in build metrics: {missing_tasks}" # Check order matches expectation - assert ( - task_order == expected_order - ), f"Task execution order mismatch. Expected: {expected_order}, Got: {task_order}" + assert task_order == expected_order, ( + f"Task execution order mismatch. Expected: {expected_order}, Got: {task_order}" + ) # Check statuses for each task task_lookup = {task.get("path"): task for task in metrics_tasks} for task_name, expected_status in ordered_expected_task_statuses: task_info = task_lookup[task_name] actual_status = task_info.get("status") - assert ( - actual_status == expected_status - ), f"Task {task_name} had status '{actual_status}', expected '{expected_status}'" + assert actual_status == expected_status, ( + f"Task {task_name} had status '{actual_status}', expected '{expected_status}'" + ) def test_artifact_build(objdir, mozconfig, run_mach): diff --git a/build/variables.py b/build/variables.py @@ -97,7 +97,7 @@ def source_repo_header(output): changeset = get_hg_changeset(buildconfig.topsrcdir) if not changeset: raise Exception( - "could not resolve changeset; " "try setting MOZ_SOURCE_CHANGESET" + "could not resolve changeset; try setting MOZ_SOURCE_CHANGESET" ) if changeset: diff --git a/client.py b/client.py @@ -32,9 +32,13 @@ def do_hg_pull(dir, repository, hg): if repository is not None: cmd.append(repository) check_call_noisy(cmd) - check_call( - [hg, "parent", "-R", fulldir, "--template=Updated to revision {node}.\n"] - ) + check_call([ + hg, + "parent", + "-R", + fulldir, + "--template=Updated to revision {node}.\n", + ]) def do_hg_replace(dir, repository, tag, exclusions, hg): diff --git a/config/check_macroassembler_style.py b/config/check_macroassembler_style.py @@ -27,12 +27,23 @@ import sys architecture_independent = set(["generic"]) all_unsupported_architectures_names = set(["mips64", "mips_shared"]) -all_architecture_names = set( - ["x86", "x64", "arm", "arm64", "loong64", "riscv64", "wasm32"] -) -all_shared_architecture_names = set( - ["x86_shared", "arm", "arm64", "loong64", "riscv64", "wasm32"] -) +all_architecture_names = set([ + "x86", + "x64", + "arm", + "arm64", + "loong64", + "riscv64", + "wasm32", +]) +all_shared_architecture_names = set([ + "x86_shared", + "arm", + "arm64", + "loong64", + "riscv64", + "wasm32", +]) reBeforeArg = r"(?<=[(,\s])" reArgType = r"(?P<type>[\w\s:*&<>]+)" @@ -101,14 +112,12 @@ def get_normalized_signatures(signature, fileAnnot=None): return signatures -file_suffixes = set( - [ - a.replace("_", "-") - for a in all_architecture_names.union(all_shared_architecture_names).union( - all_unsupported_architectures_names - ) - ] -) +file_suffixes = set([ + a.replace("_", "-") + for a in all_architecture_names.union(all_shared_architecture_names).union( + all_unsupported_architectures_names + ) +]) def get_file_annotation(filename): diff --git a/config/check_spidermonkey_style.py b/config/check_spidermonkey_style.py @@ -54,63 +54,61 @@ ignored_js_src_dirs = [ ] # We ignore #includes of these files, because they don't follow the usual rules. -included_inclnames_to_ignore = set( - [ - "ffi.h", # generated in ctypes/libffi/ - "devtools/Instruments.h", # we ignore devtools/ in general - "diplomat_runtime.hpp", # ICU4X - "double-conversion/double-conversion.h", # strange MFBT case - "frontend/ReservedWordsGenerated.h", # generated in $OBJDIR - "gc/StatsPhasesGenerated.h", # generated in $OBJDIR - "gc/StatsPhasesGenerated.inc", # generated in $OBJDIR - "icu4x/Calendar.hpp", # ICU4X - "icu4x/Date.hpp", # ICU4X - "icu4x/GraphemeClusterSegmenter.hpp", # ICU4X - "icu4x/IsoDate.hpp", # ICU4X - "icu4x/Locale.hpp", # ICU4X - "icu4x/SentenceSegmenter.hpp", # ICU4X - "icu4x/WordSegmenter.hpp", # ICU4X - "jit/ABIFunctionTypeGenerated.h", # generated in $OBJDIR" - "jit/AtomicOperationsGenerated.h", # generated in $OBJDIR - "jit/CacheIROpsGenerated.h", # generated in $OBJDIR - "jit/CacheIRAOTGenerated.h", # generated in $OBJDIR - "jit/LIROpsGenerated.h", # generated in $OBJDIR - "jit/MIROpsGenerated.h", # generated in $OBJDIR - "js/PrefsGenerated.h", # generated in $OBJDIR - "js/ProfilingCategoryList.h", # comes from mozglue/baseprofiler - "mozilla/glue/Debug.h", # comes from mozglue/misc, shadowed by <mozilla/Debug.h> - "jscustomallocator.h", # provided by embedders; allowed to be missing - "js-config.h", # generated in $OBJDIR - "fdlibm.h", # fdlibm - "FuzzerDefs.h", # included without a path - "FuzzingInterface.h", # included without a path - "mozmemory.h", # included without a path - "mozmemory_stall.h", # included without a path - "pratom.h", # NSPR - "prcvar.h", # NSPR - "prerror.h", # NSPR - "prinit.h", # NSPR - "prio.h", # NSPR - "private/pprio.h", # NSPR - "prlink.h", # NSPR - "prlock.h", # NSPR - "prprf.h", # NSPR - "prthread.h", # NSPR - "prtypes.h", # NSPR - "selfhosted.out.h", # generated in $OBJDIR - "shellmoduleloader.out.h", # generated in $OBJDIR - "unicode/locid.h", # ICU - "unicode/uchar.h", # ICU - "unicode/uniset.h", # ICU - "unicode/unistr.h", # ICU - "unicode/utypes.h", # ICU - "vtune/VTuneWrapper.h", # VTune - "wasm/WasmBuiltinModuleGenerated.h", # generated in $OBJDIR" - "zydis/ZydisAPI.h", # Zydis - "xsum/xsum.h", # xsum - "fmt/format.h", # {fmt} main header - ] -) +included_inclnames_to_ignore = set([ + "ffi.h", # generated in ctypes/libffi/ + "devtools/Instruments.h", # we ignore devtools/ in general + "diplomat_runtime.hpp", # ICU4X + "double-conversion/double-conversion.h", # strange MFBT case + "frontend/ReservedWordsGenerated.h", # generated in $OBJDIR + "gc/StatsPhasesGenerated.h", # generated in $OBJDIR + "gc/StatsPhasesGenerated.inc", # generated in $OBJDIR + "icu4x/Calendar.hpp", # ICU4X + "icu4x/Date.hpp", # ICU4X + "icu4x/GraphemeClusterSegmenter.hpp", # ICU4X + "icu4x/IsoDate.hpp", # ICU4X + "icu4x/Locale.hpp", # ICU4X + "icu4x/SentenceSegmenter.hpp", # ICU4X + "icu4x/WordSegmenter.hpp", # ICU4X + "jit/ABIFunctionTypeGenerated.h", # generated in $OBJDIR" + "jit/AtomicOperationsGenerated.h", # generated in $OBJDIR + "jit/CacheIROpsGenerated.h", # generated in $OBJDIR + "jit/CacheIRAOTGenerated.h", # generated in $OBJDIR + "jit/LIROpsGenerated.h", # generated in $OBJDIR + "jit/MIROpsGenerated.h", # generated in $OBJDIR + "js/PrefsGenerated.h", # generated in $OBJDIR + "js/ProfilingCategoryList.h", # comes from mozglue/baseprofiler + "mozilla/glue/Debug.h", # comes from mozglue/misc, shadowed by <mozilla/Debug.h> + "jscustomallocator.h", # provided by embedders; allowed to be missing + "js-config.h", # generated in $OBJDIR + "fdlibm.h", # fdlibm + "FuzzerDefs.h", # included without a path + "FuzzingInterface.h", # included without a path + "mozmemory.h", # included without a path + "mozmemory_stall.h", # included without a path + "pratom.h", # NSPR + "prcvar.h", # NSPR + "prerror.h", # NSPR + "prinit.h", # NSPR + "prio.h", # NSPR + "private/pprio.h", # NSPR + "prlink.h", # NSPR + "prlock.h", # NSPR + "prprf.h", # NSPR + "prthread.h", # NSPR + "prtypes.h", # NSPR + "selfhosted.out.h", # generated in $OBJDIR + "shellmoduleloader.out.h", # generated in $OBJDIR + "unicode/locid.h", # ICU + "unicode/uchar.h", # ICU + "unicode/uniset.h", # ICU + "unicode/unistr.h", # ICU + "unicode/utypes.h", # ICU + "vtune/VTuneWrapper.h", # VTune + "wasm/WasmBuiltinModuleGenerated.h", # generated in $OBJDIR" + "zydis/ZydisAPI.h", # Zydis + "xsum/xsum.h", # xsum + "fmt/format.h", # {fmt} main header +]) # JSAPI functions should be included through headers from js/public instead of # using the old, catch-all jsapi.h file. @@ -126,19 +124,17 @@ deprecated_inclnames_in_header_excludes = { # These files have additional constraints on where they are #included, so we # ignore #includes of them when checking #include ordering. -oddly_ordered_inclnames = set( - [ - "ctypes/typedefs.h", # Included multiple times in the body of ctypes/CTypes.h - # Included in the body of frontend/TokenStream.h - "frontend/ReservedWordsGenerated.h", - "gc/StatsPhasesGenerated.h", # Included in the body of gc/Statistics.h - "gc/StatsPhasesGenerated.inc", # Included in the body of gc/Statistics.cpp - "psapi.h", # Must be included after "util/WindowsWrapper.h" on Windows - "machine/endian.h", # Must be included after <sys/types.h> on BSD - "process.h", # Windows-specific - "util/WindowsWrapper.h", # Must precede other system headers(?) - ] -) +oddly_ordered_inclnames = set([ + "ctypes/typedefs.h", # Included multiple times in the body of ctypes/CTypes.h + # Included in the body of frontend/TokenStream.h + "frontend/ReservedWordsGenerated.h", + "gc/StatsPhasesGenerated.h", # Included in the body of gc/Statistics.h + "gc/StatsPhasesGenerated.inc", # Included in the body of gc/Statistics.cpp + "psapi.h", # Must be included after "util/WindowsWrapper.h" on Windows + "machine/endian.h", # Must be included after <sys/types.h> on BSD + "process.h", # Windows-specific + "util/WindowsWrapper.h", # Must precede other system headers(?) +]) # System headers which shouldn't be included directly, but instead use the # designated wrapper. @@ -217,9 +213,7 @@ js/src/tests/style/BadIncludesOrder-inl.h:28:29: error: -> tests/style/HeaderCycleB1-inl.h -> tests/style/HeaderCycleB4-inl.h -""".splitlines( - True -) +""".splitlines(True) actual_output = [] @@ -381,9 +375,7 @@ def check_style(enable_fixup): def module_name(name): """Strip the trailing .cpp, .h, or -inl.h from a filename.""" - return ( - name.replace("-inl.h", "").replace(".h", "").replace(".cpp", "") - ) # NOQA: E501 + return name.replace("-inl.h", "").replace(".h", "").replace(".cpp", "") # NOQA: E501 def is_module_header(enclosing_inclname, header_inclname): diff --git a/config/external/ffi/subst_header.py b/config/external/ffi/subst_header.py @@ -12,14 +12,12 @@ from mozbuild.preprocessor import Preprocessor def main(output, input_file, *defines): pp = Preprocessor() - pp.context.update( - { - "FFI_EXEC_TRAMPOLINE_TABLE": "0", - "HAVE_LONG_DOUBLE": "0", - "TARGET": buildconfig.substs["FFI_TARGET"], - "VERSION": "", - } - ) + pp.context.update({ + "FFI_EXEC_TRAMPOLINE_TABLE": "0", + "HAVE_LONG_DOUBLE": "0", + "TARGET": buildconfig.substs["FFI_TARGET"], + "VERSION": "", + }) for d in defines: pp.context.update({d: "1"}) pp.do_filter("substitution") diff --git a/config/mozunit/mozunit/mozunit.py b/config/mozunit/mozunit/mozunit.py @@ -312,22 +312,20 @@ def main(*args, **kwargs): args.append("--color=yes") module = __import__("__main__") - args.extend( - [ - "--rootdir", - str(topsrcdir), - "-c", - os.path.join(here, "pytest.ini"), - "-vv", - "--tb=short", - "-p", - "mozlog.pytest_mozlog.plugin", - "-p", - "mozunit.pytest_plugin", - "-p", - "no:cacheprovider", - "-rsx", # show reasons for skip / xfail - module.__file__, - ] - ) + args.extend([ + "--rootdir", + str(topsrcdir), + "-c", + os.path.join(here, "pytest.ini"), + "-vv", + "--tb=short", + "-p", + "mozlog.pytest_mozlog.plugin", + "-p", + "mozunit.pytest_plugin", + "-p", + "no:cacheprovider", + "-rsx", # show reasons for skip / xfail + module.__file__, + ]) sys.exit(pytest.main(args)) diff --git a/config/tests/unit-nsinstall.py b/config/tests/unit-nsinstall.py @@ -139,7 +139,7 @@ class TestNsinstall(unittest.TestCase): "Test that nsinstall handles non-ASCII files" filename = "\u2325\u3452\u2415\u5081" testfile = self.touch(filename) - testdir = self.mkdirs("\u4241\u1D04\u1414") + testdir = self.mkdirs("\u4241\u1d04\u1414") self.assertEqual( nsinstall([testfile.encode("utf-8"), testdir.encode("utf-8")]), 0 ) @@ -155,7 +155,7 @@ class TestNsinstall(unittest.TestCase): "Test that nsinstall as a subprocess handles non-ASCII files" filename = "\u2325\u3452\u2415\u5081" testfile = self.touch(filename) - testdir = self.mkdirs("\u4241\u1D04\u1414") + testdir = self.mkdirs("\u4241\u1d04\u1414") p = subprocess.Popen([sys.executable, NSINSTALL_PATH, testfile, testdir]) rv = p.wait() diff --git a/config/tests/unitMozZipFile.py b/config/tests/unitMozZipFile.py @@ -207,6 +207,7 @@ for first in allfiles: class TestExtensiveDeflated(TestExtensiveStored): "Test all that has been tested with ZIP_STORED with DEFLATED, too." + compression = zipfile.ZIP_DEFLATED diff --git a/configure.py b/configure.py @@ -241,7 +241,7 @@ def config_status(config, execute=True): fh.write("%s = " % k) pprint.pprint(v, stream=fh, indent=4) fh.write( - "__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']" + "__all__ = ['topobjdir', 'topsrcdir', 'defines', 'substs', 'mozconfig']" ) if execute: diff --git a/dom/base/usecounters.py b/dom/base/usecounters.py @@ -383,42 +383,38 @@ def parse_use_counters(): ) name = f"{counter['interface_name']}_{counter['attribute_name']}".lower() attr = f"{counter['interface_name']}.{counter['attribute_name']}" - page.append( - (f"{enum_root}_getter", f"{name}_getter", f"Whether a page got {attr}.") - ) - page.append( - (f"{enum_root}_setter", f"{name}_setter", f"Whether a page set {attr}.") - ) - doc.append( - ( - f"{enum_root}_getter", - f"{name}_getter", - f"Whether a document got {attr}.", - ) - ) - doc.append( - ( - f"{enum_root}_setter", - f"{name}_setter", - f"Whether a document set {attr}.", - ) - ) + page.append(( + f"{enum_root}_getter", + f"{name}_getter", + f"Whether a page got {attr}.", + )) + page.append(( + f"{enum_root}_setter", + f"{name}_setter", + f"Whether a page set {attr}.", + )) + doc.append(( + f"{enum_root}_getter", + f"{name}_getter", + f"Whether a document got {attr}.", + )) + doc.append(( + f"{enum_root}_setter", + f"{name}_setter", + f"Whether a document set {attr}.", + )) elif counter["type"] == "custom": enum_name = f"eUseCounter_custom_{counter['name']}" - page.append( - ( - enum_name, - to_snake_case(counter["name"]), - f"Whether a page {counter['desc']}.", - ) - ) - doc.append( - ( - enum_name, - to_snake_case(counter["name"]), - f"Whether a document {counter['desc']}.", - ) - ) + page.append(( + enum_name, + to_snake_case(counter["name"]), + f"Whether a page {counter['desc']}.", + )) + doc.append(( + enum_name, + to_snake_case(counter["name"]), + f"Whether a document {counter['desc']}.", + )) else: print(f"Found unexpected use counter type {counter['type']}. Returning 1.") return 1 @@ -434,84 +430,72 @@ def parse_use_counters(): enum_name = f"{counter['interface_name']}_{counter['method_name']}" name = f"{counter['interface_name']}_{counter['method_name']}".lower() method = f"called {counter['interface_name']}.{counter['method_name']}" - dedicated.append( - (enum_name, name, f"Whether a dedicated worker called {method}.") - ) - shared.append( - (enum_name, name, f"Whether a shared worker called {method}.") - ) - service.append( - (enum_name, name, f"Whether a service worker called {method}.") - ) + dedicated.append(( + enum_name, + name, + f"Whether a dedicated worker called {method}.", + )) + shared.append(( + enum_name, + name, + f"Whether a shared worker called {method}.", + )) + service.append(( + enum_name, + name, + f"Whether a service worker called {method}.", + )) elif counter["type"] == "attribute": enum_root = f"{counter['interface_name']}_{counter['attribute_name']}" name = f"{counter['interface_name']}_{counter['attribute_name']}".lower() attr = f"{counter['interface_name']}.{counter['attribute_name']}" - dedicated.append( - ( - f"{enum_root}_getter", - f"{name}_getter", - f"Whether a dedicated worker got {attr}.", - ) - ) - dedicated.append( - ( - f"{enum_root}_setter", - f"{name}_setter", - f"Whether a dedicated worker set {attr}.", - ) - ) - shared.append( - ( - f"{enum_root}_getter", - f"{name}_getter", - f"Whether a shared worker got {attr}.", - ) - ) - shared.append( - ( - f"{enum_root}_setter", - f"{name}_setter", - f"Whether a shared worker set {attr}.", - ) - ) - service.append( - ( - f"{enum_root}_getter", - f"{name}_getter", - f"Whether a service worker got {attr}.", - ) - ) - service.append( - ( - f"{enum_root}_setter", - f"{name}_setter", - f"Whether a service worker set {attr}.", - ) - ) + dedicated.append(( + f"{enum_root}_getter", + f"{name}_getter", + f"Whether a dedicated worker got {attr}.", + )) + dedicated.append(( + f"{enum_root}_setter", + f"{name}_setter", + f"Whether a dedicated worker set {attr}.", + )) + shared.append(( + f"{enum_root}_getter", + f"{name}_getter", + f"Whether a shared worker got {attr}.", + )) + shared.append(( + f"{enum_root}_setter", + f"{name}_setter", + f"Whether a shared worker set {attr}.", + )) + service.append(( + f"{enum_root}_getter", + f"{name}_getter", + f"Whether a service worker got {attr}.", + )) + service.append(( + f"{enum_root}_setter", + f"{name}_setter", + f"Whether a service worker set {attr}.", + )) elif counter["type"] == "custom": enum_name = f"Custom_{counter['name']}" - dedicated.append( - ( - enum_name, - to_snake_case(counter["name"]), - f"Whether a dedicated worker {counter['desc']}.", - ) - ) - shared.append( - ( - enum_name, - to_snake_case(counter["name"]), - f"Whether a shared worker {counter['desc']}.", - ) - ) - service.append( - ( - enum_name, - to_snake_case(counter["name"]), - f"Whether a service worker {counter['desc']}.", - ) - ) + dedicated.append(( + enum_name, + to_snake_case(counter["name"]), + f"Whether a dedicated worker {counter['desc']}.", + )) + shared.append(( + enum_name, + to_snake_case(counter["name"]), + f"Whether a shared worker {counter['desc']}.", + )) + service.append(( + enum_name, + to_snake_case(counter["name"]), + f"Whether a service worker {counter['desc']}.", + )) else: print( f"Found unexpected worker use counter type {counter['type']}. Returning 1." @@ -564,16 +548,16 @@ def parse_use_counters(): method = method[3:] # remove the moz prefix enum_name = f"eUseCounter_property_{method}" - css_page.append( - (enum_name, prop_name, f"Whether a page used the CSS property {prop.name}.") - ) - css_doc.append( - ( - enum_name, - prop_name, - f"Whether a document used the CSS property {prop.name}.", - ) - ) + css_page.append(( + enum_name, + prop_name, + f"Whether a page used the CSS property {prop.name}.", + )) + css_doc.append(( + enum_name, + prop_name, + f"Whether a document used the CSS property {prop.name}.", + )) # Counted unknown properties: AKA - stuff that doesn't exist, but we want # to count uses of anyway. @@ -600,20 +584,16 @@ def parse_use_counters(): for prop in unknown_properties: enum_name = f"eUseCounter_unknown_property_{to_camel_case(prop)}" prop_name = to_snake_case(prop) - css_page.append( - ( - enum_name, - prop_name, - f"Whether a page used the (unknown, counted) CSS property {prop}.", - ) - ) - css_doc.append( - ( - enum_name, - prop_name, - f"Whether a document used the (unknown, counted) CSS property {prop}.", - ) - ) + css_page.append(( + enum_name, + prop_name, + f"Whether a page used the (unknown, counted) CSS property {prop}.", + )) + css_doc.append(( + enum_name, + prop_name, + f"Whether a document used the (unknown, counted) CSS property {prop}.", + )) return (page, doc, dedicated, shared, service, ops_page, ops_doc, css_page, css_doc) diff --git a/dom/bindings/Configuration.py b/dom/bindings/Configuration.py @@ -837,8 +837,7 @@ class Descriptor(DescriptorProvider): ) if len(m.signatures()) != 1: raise TypeError( - "We don't support overloaded " - "legacycaller.\n%s" % m.location + "We don't support overloaded legacycaller.\n%s" % m.location ) addOperation("LegacyCaller", m) diff --git a/dom/bindings/parser/tests/test_attr_sequence_type.py b/dom/bindings/parser/tests/test_attr_sequence_type.py @@ -54,7 +54,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Attribute type must not be a union with a nullable sequence " "member type", + "Attribute type must not be a union with a nullable sequence member type", ) parser.reset() diff --git a/dom/bindings/parser/tests/test_constructor.py b/dom/bindings/parser/tests/test_constructor.py @@ -446,7 +446,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Can't have both a HTMLConstructor and a throwing constructor " "operation", + "Can't have both a HTMLConstructor and a throwing constructor operation", ) # Test HTMLConstructor and [ChromeOnly] constructor operation @@ -486,7 +486,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Can't have both a throwing chromeonly constructor and a " "HTMLConstructor", + "Can't have both a throwing chromeonly constructor and a HTMLConstructor", ) parser = parser.reset() @@ -507,7 +507,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Can't have both a HTMLConstructor and a chromeonly constructor " "operation", + "Can't have both a HTMLConstructor and a chromeonly constructor operation", ) parser = parser.reset() @@ -549,8 +549,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Can't have a constructor operation on a [LegacyNoInterfaceObject] " - "interface", + "Can't have a constructor operation on a [LegacyNoInterfaceObject] interface", ) parser = parser.reset() diff --git a/dom/bindings/parser/tests/test_dictionary.py b/dom/bindings/parser/tests/test_dictionary.py @@ -120,7 +120,7 @@ def WebIDLTest(parser, harness): threw = True harness.ok( - threw, "Should not allow name duplication in a dictionary and " "its ancestor" + threw, "Should not allow name duplication in a dictionary and its ancestor" ) # More reset @@ -324,7 +324,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Union arg containing dictionary followed by optional arg must " "be optional", + "Union arg containing dictionary followed by optional arg must be optional", ) parser = parser.reset() @@ -404,7 +404,7 @@ def WebIDLTest(parser, harness): harness.ok(threw, "Required dictionary arg must not be nullable") harness.ok( "nullable" in str(threw), - "Must have the expected exception for required nullable " "dictionary arg", + "Must have the expected exception for required nullable dictionary arg", ) parser = parser.reset() @@ -605,7 +605,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Member type must not be a Dictionary that " "inherits from its Dictionary.", + "Member type must not be a Dictionary that inherits from its Dictionary.", ) parser = parser.reset() diff --git a/dom/bindings/parser/tests/test_distinguishability.py b/dom/bindings/parser/tests/test_distinguishability.py @@ -58,11 +58,11 @@ def WebIDLTest(parser, harness): harness.ok( not nullableUnionType.isDistinguishableFrom(nullableIfaceType), - "Nullable type not distinguishable from union with nullable " "member type", + "Nullable type not distinguishable from union with nullable member type", ) harness.ok( not nullableIfaceType.isDistinguishableFrom(nullableUnionType), - "Union with nullable member type not distinguishable from " "nullable type", + "Union with nullable member type not distinguishable from nullable type", ) parser = parser.reset() diff --git a/dom/bindings/parser/tests/test_empty_sequence_default_value.py b/dom/bindings/parser/tests/test_empty_sequence_default_value.py @@ -50,5 +50,5 @@ def WebIDLTest(parser, harness): harness.ok( isinstance(results[0].members[0].defaultValue, WebIDL.IDLEmptySequenceValue), - "Should have IDLEmptySequenceValue as default value of " "dictionary member", + "Should have IDLEmptySequenceValue as default value of dictionary member", ) diff --git a/dom/bindings/parser/tests/test_global_extended_attr.py b/dom/bindings/parser/tests/test_global_extended_attr.py @@ -42,7 +42,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Should have thrown for [Global] used on an interface with a " "named setter", + "Should have thrown for [Global] used on an interface with a named setter", ) parser = parser.reset() @@ -63,7 +63,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Should have thrown for [Global] used on an interface with a " "named deleter", + "Should have thrown for [Global] used on an interface with a named deleter", ) parser = parser.reset() @@ -128,7 +128,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Should have thrown for [Global] used on an interface with a " "descendant", + "Should have thrown for [Global] used on an interface with a descendant", ) parser = parser.reset() diff --git a/dom/bindings/parser/tests/test_interface.py b/dom/bindings/parser/tests/test_interface.py @@ -263,8 +263,7 @@ def WebIDLTest(parser, harness): threw = True harness.ok( threw, - "Should not allow a name collision between partial interface " - "and other object", + "Should not allow a name collision between partial interface and other object", ) parser = parser.reset() @@ -284,7 +283,7 @@ def WebIDLTest(parser, harness): except WebIDL.WebIDLError: threw = True harness.ok( - threw, "Should not allow a name collision between interface " "and other object" + threw, "Should not allow a name collision between interface and other object" ) parser = parser.reset() @@ -303,8 +302,7 @@ def WebIDLTest(parser, harness): threw = True harness.ok( threw, - "Should not allow a name collision between external interface " - "and other object", + "Should not allow a name collision between external interface and other object", ) parser = parser.reset() @@ -323,7 +321,7 @@ def WebIDLTest(parser, harness): threw = True harness.ok( threw, - "Should not allow a name collision between external interface " "and interface", + "Should not allow a name collision between external interface and interface", ) parser = parser.reset() @@ -336,7 +334,7 @@ def WebIDLTest(parser, harness): results = parser.finish() harness.ok( len(results) == 1 and isinstance(results[0], WebIDL.IDLExternalInterface), - "Should allow name collisions between external interface " "declarations", + "Should allow name collisions between external interface declarations", ) parser = parser.reset() diff --git a/dom/bindings/parser/tests/test_interfacemixin.py b/dom/bindings/parser/tests/test_interfacemixin.py @@ -219,7 +219,7 @@ def WebIDLTest(parser, harness): threw = True harness.ok( threw, - "Should not allow a name collision between interface mixin " "and other object", + "Should not allow a name collision between interface mixin and other object", ) parser = parser.reset() @@ -523,12 +523,12 @@ def WebIDLTest(parser, harness): harness.check( attr.exposureSet, set(["Window", "Worker"]), - "Should expose on all globals where including interfaces are " "exposed", + "Should expose on all globals where including interfaces are exposed", ) base = results[3] attr = base.members[0] harness.check( attr.exposureSet, set(["Window", "Worker"]), - "Should expose on all globals where including interfaces are " "exposed", + "Should expose on all globals where including interfaces are exposed", ) diff --git a/dom/bindings/parser/tests/test_optional_constraints.py b/dom/bindings/parser/tests/test_optional_constraints.py @@ -18,8 +18,7 @@ def WebIDLTest(parser, harness): harness.ok( not threw, - "Should not have thrown on non-optional argument following " - "optional argument.", + "Should not have thrown on non-optional argument following optional argument.", ) parser = parser.reset() diff --git a/dom/bindings/parser/tests/test_promise.py b/dom/bindings/parser/tests/test_promise.py @@ -98,9 +98,7 @@ def WebIDLTest(parser, harness): ) parser.finish() - harness.ok( - True, "Should allow overloads which only have Promise and return " "types." - ) + harness.ok(True, "Should allow overloads which only have Promise and return types.") parser = parser.reset() threw = False diff --git a/dom/bindings/parser/tests/test_toJSON.py b/dom/bindings/parser/tests/test_toJSON.py @@ -167,7 +167,7 @@ def WebIDLTest(parser, harness): ) doTest( - "dictionary Foo { %s foo; }; " "interface Test { Foo toJSON(); }; " % type, + "dictionary Foo { %s foo; }; interface Test { Foo toJSON(); }; " % type, False, "dictionary containing only JSON type (%s) should be a JSON type" % type, ) @@ -233,7 +233,7 @@ def WebIDLTest(parser, harness): ) doTest( - "interface Foo : InterfaceWithToJSON { };" "interface Test { Foo toJSON(); };", + "interface Foo : InterfaceWithToJSON { };interface Test { Foo toJSON(); };", False, "inherited interface with toJSON should be a JSON type", ) @@ -252,7 +252,7 @@ def WebIDLTest(parser, harness): ) doTest( - "dictionary Foo { %s foo; }; " "interface Test { Foo toJSON(); }; " % type, + "dictionary Foo { %s foo; }; interface Test { Foo toJSON(); }; " % type, True, "Dictionary containing a non-JSON type (%s) should not be a JSON type" % type, @@ -299,14 +299,13 @@ def WebIDLTest(parser, harness): ) doTest( - "dictionary Foo { long foo; any bar; };" "interface Test { Foo toJSON(); };", + "dictionary Foo { long foo; any bar; };interface Test { Foo toJSON(); };", True, "dictionary containing a non-JSON type should not be a JSON type", ) doTest( - "interface Foo : InterfaceWithoutToJSON { }; " - "interface Test { Foo toJSON(); };", + "interface Foo : InterfaceWithoutToJSON { }; interface Test { Foo toJSON(); };", True, "interface without toJSON should not be a JSON type", ) diff --git a/dom/bindings/parser/tests/test_typedef.py b/dom/bindings/parser/tests/test_typedef.py @@ -74,8 +74,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Should have thrown on nullable inside nullable const typedef " - "after interface.", + "Should have thrown on nullable inside nullable const typedef after interface.", ) parser = parser.reset() diff --git a/dom/bindings/parser/tests/test_union.py b/dom/bindings/parser/tests/test_union.py @@ -174,16 +174,13 @@ def WebIDLTest(parser, harness): parser = parser.reset() for invalid in invalidUnionTypes: - interface = ( - testPre - + string.Template( - """ + interface = testPre + string.Template( + """ interface TestUnion { undefined method(${type} arg); }; """ - ).substitute(type=invalid) - ) + ).substitute(type=invalid) threw = False try: diff --git a/dom/bindings/parser/tests/test_union_nullable.py b/dom/bindings/parser/tests/test_union_nullable.py @@ -36,7 +36,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "A nullable union type with a nullable member type should have " "thrown.", + "A nullable union type with a nullable member type should have thrown.", ) parser.reset() @@ -59,5 +59,5 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "A nullable union type with a nullable member type should have " "thrown.", + "A nullable union type with a nullable member type should have thrown.", ) diff --git a/dom/bindings/parser/tests/test_variadic_constraints.py b/dom/bindings/parser/tests/test_variadic_constraints.py @@ -18,7 +18,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Should have thrown on variadic argument followed by required " "argument.", + "Should have thrown on variadic argument followed by required argument.", ) parser = parser.reset() @@ -37,7 +37,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Should have thrown on variadic argument followed by optional " "argument.", + "Should have thrown on variadic argument followed by optional argument.", ) parser = parser.reset() @@ -57,7 +57,7 @@ def WebIDLTest(parser, harness): harness.ok( threw, - "Should have thrown on variadic argument explicitly flagged as " "optional.", + "Should have thrown on variadic argument explicitly flagged as optional.", ) parser = parser.reset() diff --git a/dom/canvas/test/reftest/colors/generate_color_canvas_reftests.py b/dom/canvas/test/reftest/colors/generate_color_canvas_reftests.py @@ -332,7 +332,7 @@ assert encode_url_v("e_color", "color(srgb 0 0 0)") == "color(srgb,,,0,,,0 def url_from_config(kvs: Config) -> str: - parts = [f"{k}={encode_url_v(k,v)}" for k, v in kvs.items()] + parts = [f"{k}={encode_url_v(k, v)}" for k, v in kvs.items()] url = "color_canvas.html?" + "&".join(parts) return url @@ -405,7 +405,7 @@ REFTEST_LIST_PREAMBLE = f"""\ {GENERATED_FILE_LINE} {GENERATED_FILE_LINE} -# Generated by `{' '.join(GENERATED_BY_ARGS)}`. +# Generated by `{" ".join(GENERATED_BY_ARGS)}`. # - defaults pref(webgl.drawing_buffer_color_space,true) @@ -414,9 +414,9 @@ defaults pref(webgl.drawing_buffer_color_space,true) # - # Ensure not white-screening: -!= {url_from_config({})+'='} about:blank +!= {url_from_config({}) + "="} about:blank # Ensure differing results with different args: -!= {url_from_config({'e_color':'color(srgb 1 0 0)'})} {url_from_config({'e_color':'color(srgb 0 1 0)'})} +!= {url_from_config({"e_color": "color(srgb 1 0 0)"})} {url_from_config({"e_color": "color(srgb 0 1 0)"})} {GENERATED_FILE_LINE} # - diff --git a/dom/canvas/test/webgl-conf/generate-wrappers-and-manifest.py b/dom/canvas/test/webgl-conf/generate-wrappers-and-manifest.py @@ -32,14 +32,12 @@ EXTRA_SUPPORT_FILES = [ "mochi-single.html", ] -ACCEPTABLE_ERRATA_KEYS = set( - [ - "fail-if", - "prefs", - "skip-if", - "tags", - ] -) +ACCEPTABLE_ERRATA_KEYS = set([ + "fail-if", + "prefs", + "skip-if", + "tags", +]) def ChooseSubsuite(name): @@ -475,9 +473,9 @@ def LoadTOML(path): if line[0] == "[": assert line[-1] == "]", f"{path}:{lineNum}" curSectionName = line[1:-1].strip('"') - assert ( - curSectionName not in ret - ), f"Line {lineNum}: Duplicate section: {line}" + assert curSectionName not in ret, ( + f"Line {lineNum}: Duplicate section: {line}" + ) curSectionMap = {} ret[curSectionName] = (lineNum, curSectionMap) continue @@ -506,9 +504,9 @@ def LoadErrata(): continue elif sectionName != "DEFAULT": path = sectionName.replace("/", os.sep) - assert os.path.exists( - path - ), f"Errata line {sectionLineNum}: Invalid file: {sectionName}" + assert os.path.exists(path), ( + f"Errata line {sectionLineNum}: Invalid file: {sectionName}" + ) for key, (lineNum, val) in sectionMap.items(): assert key in ACCEPTABLE_ERRATA_KEYS, f"Line {lineNum}: {key}" diff --git a/dom/canvas/test/webgl-conf/import.py b/dom/canvas/test/webgl-conf/import.py @@ -13,9 +13,9 @@ from pathlib import Path REL_PATH = "/dom/canvas/test/webgl-conf" REPO_DIR = Path.cwd() DIR_IN_GECKO = Path(__file__).parent -assert not REPO_DIR.samefile( - DIR_IN_GECKO -), "Run this script from the source git checkout." +assert not REPO_DIR.samefile(DIR_IN_GECKO), ( + "Run this script from the source git checkout." +) assert DIR_IN_GECKO.as_posix().endswith(REL_PATH) # Be paranoid with rm -rf. gecko_base_dir = DIR_IN_GECKO.as_posix()[: -len(REL_PATH)] diff --git a/dom/media/driftcontrol/plot.py b/dom/media/driftcontrol/plot.py @@ -106,12 +106,10 @@ MOZ_LOG_FILE=/tmp/driftcontrol.csv \ absAvgError = [abs(e) for e in avgError] slow_offset = [e / slowConvergenceSecs - slowHysteresis for e in absAvgError] fast_offset = [e / adjustmentInterval for e in absAvgError] - low_offset, high_offset = zip( - *[ - (s, f) if e >= 0 else (-f, -s) - for (e, s, f) in zip(avgError, slow_offset, fast_offset) - ] - ) + low_offset, high_offset = zip(*[ + (s, f) if e >= 0 else (-f, -s) + for (e, s, f) in zip(avgError, slow_offset, fast_offset) + ]) fig2 = figure(x_range=fig1.x_range) fig2.varea( diff --git a/dom/media/test/reftest/gen_combos.py b/dom/media/test/reftest/gen_combos.py @@ -69,83 +69,75 @@ WEBM = cross_combine([{"ext": "webm"}], keyed_combiner("vcodec", WEBM_CODECS)) # - -FORMAT_LIST = set( - [ +FORMAT_LIST = set([ + "yuv420p", + "yuv420p10", + # 'yuv420p12', + # 'yuv420p16be', + # 'yuv420p16le', + "gbrp", +]) + +if "--all" in ARGS: + FORMAT_LIST |= set([ "yuv420p", "yuv420p10", - # 'yuv420p12', - # 'yuv420p16be', - # 'yuv420p16le', + "yuv420p12", + "yuv420p16be", + "yuv420p16le", + "yuv422p", + "yuv422p10", + "yuv422p12", + "yuv422p16be", + "yuv422p16le", + "yuv444p", + "yuv444p10", + "yuv444p12", + "yuv444p16be", + "yuv444p16le", + "yuv411p", + "yuv410p", + "yuyv422", + "uyvy422", + "rgb24", + "bgr24", + "rgb8", + "bgr8", + "rgb444be", + "rgb444le", + "bgr444be", + "bgr444le", + # 'nv12', # Encoding not different than yuv420p? + # 'nv21', # Encoding not different than yuv420p? "gbrp", - ] -) - -if "--all" in ARGS: - FORMAT_LIST |= set( - [ - "yuv420p", - "yuv420p10", - "yuv420p12", - "yuv420p16be", - "yuv420p16le", - "yuv422p", - "yuv422p10", - "yuv422p12", - "yuv422p16be", - "yuv422p16le", - "yuv444p", - "yuv444p10", - "yuv444p12", - "yuv444p16be", - "yuv444p16le", - "yuv411p", - "yuv410p", - "yuyv422", - "uyvy422", - "rgb24", - "bgr24", - "rgb8", - "bgr8", - "rgb444be", - "rgb444le", - "bgr444be", - "bgr444le", - # 'nv12', # Encoding not different than yuv420p? - # 'nv21', # Encoding not different than yuv420p? - "gbrp", - "gbrp9be", - "gbrp9le", - "gbrp10be", - "gbrp10le", - "gbrp12be", - "gbrp12le", - "gbrp14be", - "gbrp14le", - "gbrp16be", - "gbrp16le", - ] - ) + "gbrp9be", + "gbrp9le", + "gbrp10be", + "gbrp10le", + "gbrp12be", + "gbrp12le", + "gbrp14be", + "gbrp14le", + "gbrp16be", + "gbrp16le", + ]) FORMATS = keyed_combiner("format", list(FORMAT_LIST)) RANGE = keyed_combiner("range", ["tv", "pc"]) -CSPACE_LIST = set( - [ - "bt709", - # 'bt2020', - ] -) +CSPACE_LIST = set([ + "bt709", + # 'bt2020', +]) if "--all" in ARGS: - CSPACE_LIST |= set( - [ - "bt709", - "bt2020", - "bt601-6-525", # aka smpte170m NTSC - "bt601-6-625", # aka bt470bg PAL - ] - ) + CSPACE_LIST |= set([ + "bt709", + "bt2020", + "bt601-6-525", # aka smpte170m NTSC + "bt601-6-625", # aka bt470bg PAL + ]) CSPACE_LIST = list(CSPACE_LIST) # - @@ -164,17 +156,15 @@ print(f"{len(COMBOS)} combinations...") todo = [] for c in COMBOS: - dst_name = ".".join( - [ - SRC_PATH.name, - c["src_cspace"], - c["dst_cspace"], - c["range"], - c["format"], - c["vcodec"], - c["ext"], - ] - ) + dst_name = ".".join([ + SRC_PATH.name, + c["src_cspace"], + c["dst_cspace"], + c["range"], + c["format"], + c["vcodec"], + c["ext"], + ]) src_cspace = c["src_cspace"] diff --git a/dom/media/tools/checkGmpBalrog.py b/dom/media/tools/checkGmpBalrog.py @@ -47,15 +47,13 @@ def fetch_balrog_xml( for channel in channels: results[channel] = {} for target in targets: - balrog_url = url.format_map( - { - "url_base": url_base, - "buildid": buildid, - "channel": channel, - "version": version, - "target": target, - } - ) + balrog_url = url.format_map({ + "url_base": url_base, + "buildid": buildid, + "channel": channel, + "version": version, + "target": target, + }) response = requests.get(balrog_url) response.raise_for_status() diff --git a/dom/media/tools/generateGmpJson.py b/dom/media/tools/generateGmpJson.py @@ -32,9 +32,14 @@ def fetch_url_for_cdms(cdms, urlParams): "{} expected https scheme '{}'".format(cdm["target"], redirectUrl) ) - sanitizedUrl = urlunparse( - (parsedUrl.scheme, parsedUrl.netloc, parsedUrl.path, None, None, None) - ) + sanitizedUrl = urlunparse(( + parsedUrl.scheme, + parsedUrl.netloc, + parsedUrl.path, + None, + None, + None, + )) # Note that here we modify the returned URL from the # component update service because it returns a preferred diff --git a/dom/media/webrtc/third_party_build/filter_git_changes.py b/dom/media/webrtc/third_party_build/filter_git_changes.py @@ -47,9 +47,9 @@ def filter_git_changes(github_path, commit_sha, diff_filter): # Convert the directory exclude list to a regex string and filter # out the excluded directory paths (note the lack of trailing '$' # in the regex). - regex_excludes = "|".join( - [f"^(M|A|D|R\\d\\d\\d)\t{incl}" for incl in exclude_dir_list] - ) + regex_excludes = "|".join([ + f"^(M|A|D|R\\d\\d\\d)\t{incl}" for incl in exclude_dir_list + ]) files_not_excluded = [ path for path in changed_files if not re.findall(regex_excludes, path) ] diff --git a/dom/media/webrtc/third_party_build/write_default_config.py b/dom/media/webrtc/third_party_build/write_default_config.py @@ -220,6 +220,6 @@ if __name__ == "__main__": run_shell( f"{'git commit -m' if repo_type == RepoType.GIT else 'hg commit --message'} " f'"Bug {args.bug_number} - ' - f'updated default_config_env for v{args.milestone+1}"' + f'updated default_config_env for v{args.milestone + 1}"' f" {args.output_path}" ) diff --git a/dom/origin-trials/gen-keys.py b/dom/origin-trials/gen-keys.py @@ -17,12 +17,12 @@ def public_key_to_string(file, name): ) key = decoder.decode(substrate) ident = key[0][0] - assert ident[0] == univ.ObjectIdentifier( - "1.2.840.10045.2.1" - ), "should be an ECPublicKey" - assert ident[1] == univ.ObjectIdentifier( - "1.2.840.10045.3.1.7" - ), "should be a EcdsaP256 key" + assert ident[0] == univ.ObjectIdentifier("1.2.840.10045.2.1"), ( + "should be an ECPublicKey" + ) + assert ident[1] == univ.ObjectIdentifier("1.2.840.10045.3.1.7"), ( + "should be a EcdsaP256 key" + ) bits = key[0][1] assert isinstance(bits, univ.BitString), "Should be a bit string" assert len(bits) == 520, "Should be 520 bits (65 bytes)" diff --git a/dom/quota/scripts/qm-try-analysis/qm_try_analysis/analyze.py b/dom/quota/scripts/qm-try-analysis/qm_try_analysis/analyze.py @@ -63,7 +63,7 @@ def analyze_qm_failures(output_to, workdir): sys.exit(2) if output_to == "qmstacks_until_<lasteventtime>.txt": - output_to = path.join(workdir, f'qmstacks_until_{run["lasteventtime"]}.txt') + output_to = path.join(workdir, f"qmstacks_until_{run['lasteventtime']}.txt") elif output_to.exists(): error( f'The output file "{output_to}" already exists. This script would override it.' diff --git a/dom/quota/scripts/qm-try-analysis/qm_try_analysis/fn_anchors.py b/dom/quota/scripts/qm-try-analysis/qm_try_analysis/fn_anchors.py @@ -14,12 +14,10 @@ cached_functions = {} def getMetricsJson(src_url): if src_url.startswith("http"): info(f"Fetching source for function extraction: {src_url}") - metrics = subprocess.check_output( - [ - path.join(path.dirname(path.realpath(__file__)), "fetch_fn_names.sh"), - src_url, - ] - ) + metrics = subprocess.check_output([ + path.join(path.dirname(path.realpath(__file__)), "fetch_fn_names.sh"), + src_url, + ]) else: warning(f"Skip fetching source: {src_url}") metrics = "" @@ -37,13 +35,11 @@ def getSpaceFunctionsRecursive(metrics_space): and metrics_space["name"] and metrics_space["name"] != "<anonymous>" ): - functions.append( - { - "name": metrics_space["name"], - "start_line": int(metrics_space["start_line"]), - "end_line": int(metrics_space["end_line"]), - } - ) + functions.append({ + "name": metrics_space["name"], + "start_line": int(metrics_space["start_line"]), + "end_line": int(metrics_space["end_line"]), + }) for space in metrics_space["spaces"]: functions += getSpaceFunctionsRecursive(space) return functions diff --git a/dom/quota/scripts/qm-try-analysis/qm_try_analysis/report.py b/dom/quota/scripts/qm-try-analysis/qm_try_analysis/report.py @@ -191,9 +191,9 @@ def report_qm_failures(key, stacksfile, open_modified, workdir): continue search_string = " ".join(filter(None, match.groups())) - search_results = bugzilla_client.search_bugs( - [{"product": "Core", "summary": search_string}] - )["bugs"] + search_results = bugzilla_client.search_bugs([ + {"product": "Core", "summary": search_string} + ])["bugs"] if bug_id := reduce(search_results, by=anchor): info(f'Found bug {BUGZILLA_BUG_URL + str(bug_id)} for anchor "{anchor}".') diff --git a/dom/quota/scripts/qm-try-analysis/qm_try_analysis/stackanalysis.py b/dom/quota/scripts/qm-try-analysis/qm_try_analysis/stackanalysis.py @@ -227,16 +227,14 @@ def collectRawStacks(rows): "frames": [], } - stack["frames"].append( - { - "location": row["location"], - "source_file": row["source_file"], - "source_line": row["source_line"], - "seq": row["seq"], - "severity": row["severity"], - "result": row["result"], - } - ) + stack["frames"].append({ + "location": row["location"], + "source_file": row["source_file"], + "source_line": row["source_line"], + "seq": row["seq"], + "severity": row["severity"], + "result": row["result"], + }) first = False return raw_stacks diff --git a/dom/security/sanitizer/update-safe-default-configuration.py b/dom/security/sanitizer/update-safe-default-configuration.py @@ -114,44 +114,44 @@ namespace mozilla::dom::sanitizer {{ constexpr nsStaticAtom* kDefaultHTMLElements[] = {{ // clang-format off -{ xhtml_elements_body } +{xhtml_elements_body} // clang-format on }}; constexpr nsStaticAtom* kDefaultMathMLElements[] = {{ // clang-format off -{ mathml_elements_body } +{mathml_elements_body} // clang-format on }}; constexpr nsStaticAtom* kDefaultSVGElements[] = {{ // clang-format off -{ svg_elements_body } +{svg_elements_body} // clang-format on }}; constexpr nsStaticAtom* kDefaultAttributes[] = {{ // clang-format off -{ attributes_body } +{attributes_body} // clang-format on }}; // Data is encoded as: element, attributes..., nullptr constexpr nsStaticAtom* kHTMLElementWithAttributes[] = {{ // clang-format off -{ xhtml_element_with_attributes_body } +{xhtml_element_with_attributes_body} // clang-format on }}; constexpr nsStaticAtom* kMathMLElementWithAttributes[] = {{ // clang-format off -{ mathml_element_with_attributes_body } +{mathml_element_with_attributes_body} // clang-format on }}; constexpr nsStaticAtom* kSVGElementWithAttributes[] = {{ // clang-format off -{ svg_element_with_attributes_body } +{svg_element_with_attributes_body} // clang-format on }}; diff --git a/dom/workers/test/marionette/test_service_workers_disabled.py b/dom/workers/test/marionette/test_service_workers_disabled.py @@ -23,11 +23,9 @@ class ServiceWorkersDisabledTestCase(MarionetteServiceWorkerTestCase): def test_service_workers_disabled_at_startup(self): # self.marionette.set_pref sets preferences after startup. Using it # here causes intermittent failures. - self.marionette.instance.profile.set_preferences( - { - "dom.serviceWorkers.enabled": False, - } - ) + self.marionette.instance.profile.set_preferences({ + "dom.serviceWorkers.enabled": False, + }) self.marionette.restart() diff --git a/gfx/gl/GLConsts.py b/gfx/gl/GLConsts.py @@ -50,17 +50,13 @@ HEADER = b""" * * To generate this file, see tutorial in \'GLConsts.py\'. */ -"""[ - 1: -] +"""[1:] FOOTER = b""" #endif // GLCONSTS_H_ // clang-format on -"""[ - 1: -] +"""[1:] # - diff --git a/intl/icu_sources_data.py b/intl/icu_sources_data.py @@ -211,9 +211,7 @@ def try_run(name, command, cwd=None, **kwargs): except subprocess.CalledProcessError: print( """Error running "{}" in directory {} - See output in {}""".format( - " ".join(command), cwd, f.name - ), + See output in {}""".format(" ".join(command), cwd, f.name), file=sys.stderr, ) return False @@ -232,21 +230,19 @@ def update_data_file(topsrcdir): configure = mozpath.join(topsrcdir, "intl/icu/source/configure") env = dict(os.environ) # bug 1262101 - these should be shared with the moz.build files - env.update( - { - "CPPFLAGS": ( - "-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 " - + "-DU_HIDE_OBSOLETE_UTF_OLD_H=1 " - + "-DUCONFIG_NO_LEGACY_CONVERSION " - + "-DUCONFIG_NO_TRANSLITERATION " - + "-DUCONFIG_NO_REGULAR_EXPRESSIONS " - + "-DUCONFIG_NO_BREAK_ITERATION " - + "-DUCONFIG_NO_IDNA " - + "-DUCONFIG_NO_MF2 " - + "-DU_CHARSET_IS_UTF8 " - ) - } - ) + env.update({ + "CPPFLAGS": ( + "-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 " + + "-DU_HIDE_OBSOLETE_UTF_OLD_H=1 " + + "-DUCONFIG_NO_LEGACY_CONVERSION " + + "-DUCONFIG_NO_TRANSLITERATION " + + "-DUCONFIG_NO_REGULAR_EXPRESSIONS " + + "-DUCONFIG_NO_BREAK_ITERATION " + + "-DUCONFIG_NO_IDNA " + + "-DUCONFIG_NO_MF2 " + + "-DU_CHARSET_IS_UTF8 " + ) + }) # Exclude data that we currently don't need. # diff --git a/ipc/ipdl/ipdl.py b/ipc/ipdl/ipdl.py @@ -31,7 +31,7 @@ class WorkerPool: allsyncmessages, alljsonobjs, *, - processes=None + processes=None, ): if processes is None: processes = os.cpu_count() or 1 diff --git a/ipc/pull-chromium.py b/ipc/pull-chromium.py @@ -36,16 +36,14 @@ os.path.exists(chromiumsrc) and rmtree(chromiumsrc) def doexport(svnpath): localpath = os.path.join(chromiumsrc, svnpath) os.makedirs(os.path.dirname(localpath)) - check_call( - [ - "svn", - "export", - "-r", - "BASE", - os.path.join(chromiumtree, "src", svnpath), - localpath, - ] - ) + check_call([ + "svn", + "export", + "-r", + "BASE", + os.path.join(chromiumtree, "src", svnpath), + localpath, + ]) doexport("base") diff --git a/js/src/builtin/embedjs.py b/js/src/builtin/embedjs.py @@ -101,13 +101,11 @@ def embed( ) args = cppflags + ["-D%(k)s=%(v)s" % {"k": k, "v": env[k]} for k in env] preprocessed = preprocess(cxx, preprocessorOption, combinedSources, args) - processed = "\n".join( - [ - line - for line in preprocessed.splitlines() - if (line.strip() and not line.startswith("#")) - ] - ) + processed = "\n".join([ + line + for line in preprocessed.splitlines() + if (line.strip() and not line.startswith("#")) + ]) js_out.write(processed) import zlib diff --git a/js/src/builtin/intl/make_intl_data.py b/js/src/builtin/intl/make_intl_data.py @@ -4,40 +4,40 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" Usage: - make_intl_data.py langtags [cldr_common.zip] - make_intl_data.py tzdata - make_intl_data.py currency - make_intl_data.py units - make_intl_data.py numbering +"""Usage: +make_intl_data.py langtags [cldr_common.zip] +make_intl_data.py tzdata +make_intl_data.py currency +make_intl_data.py units +make_intl_data.py numbering - Target "langtags": - This script extracts information about 1) mappings between deprecated and - current Unicode BCP 47 locale identifiers, and 2) deprecated and current - BCP 47 Unicode extension value from CLDR, and converts it to C++ mapping - code in intl/components/LocaleGenerated.cpp. The code is used in - intl/components/Locale.cpp. +Target "langtags": +This script extracts information about 1) mappings between deprecated and +current Unicode BCP 47 locale identifiers, and 2) deprecated and current +BCP 47 Unicode extension value from CLDR, and converts it to C++ mapping +code in intl/components/LocaleGenerated.cpp. The code is used in +intl/components/Locale.cpp. - Target "tzdata": - This script computes which time zone informations are not up-to-date in ICU - and provides the necessary mappings to workaround this problem. - https://ssl.icu-project.org/trac/ticket/12044 +Target "tzdata": +This script computes which time zone informations are not up-to-date in ICU +and provides the necessary mappings to workaround this problem. +https://ssl.icu-project.org/trac/ticket/12044 - Target "currency": - Generates the mapping from currency codes to decimal digits used for them. +Target "currency": +Generates the mapping from currency codes to decimal digits used for them. - Target "units": - Generate source and test files using the list of so-called "sanctioned unit - identifiers" and verifies that the ICU data filter includes these units. +Target "units": +Generate source and test files using the list of so-called "sanctioned unit +identifiers" and verifies that the ICU data filter includes these units. - Target "numbering": - Generate source and test files using the list of numbering systems with - simple digit mappings and verifies that it's in sync with ICU/CLDR. +Target "numbering": +Generate source and test files using the list of numbering systems with +simple digit mappings and verifies that it's in sync with ICU/CLDR. """ import io @@ -121,9 +121,7 @@ bool mozilla::intl::Locale::{fn_name}({type_name} {name}) {{ println( """ -}""".lstrip( - "\n" - ) +}""".lstrip("\n") ) @@ -157,18 +155,14 @@ def writeMappingsBinarySearchBody( println( f""" if ({source_name}.Length() == {length}) {{ -""".rstrip( - "\n" - ) +""".rstrip("\n") ) else: trailing_return = False println( """ { -""".rstrip( - "\n" - ) +""".rstrip("\n") ) # The subtags need to be sorted for binary search to work. @@ -187,17 +181,13 @@ def writeMappingsBinarySearchBody( return true; }} return false; -""".strip( - "\n" - ) +""".strip("\n") ) else: println( f""" return {equals(subtags[0])}; -""".strip( - "\n" - ) +""".strip("\n") ) elif len(subtags) <= 4: if type(mappings) is dict: @@ -208,17 +198,13 @@ def writeMappingsBinarySearchBody( {target_name}.Set("{mappings[subtag]}"); return true; }} -""".strip( - "\n" - ) +""".strip("\n") ) println( """ return false; -""".strip( - "\n" - ) +""".strip("\n") ) else: cond = (equals(subtag) for subtag in subtags) @@ -226,9 +212,7 @@ def writeMappingsBinarySearchBody( println( f""" return {cond}; -""".strip( - "\n" - ) +""".strip("\n") ) else: write_array(subtags, source_name + "s", length, True) @@ -255,9 +239,7 @@ def writeMappingsBinarySearchBody( println( """ } -""".strip( - "\n" - ) +""".strip("\n") ) if trailing_return: @@ -310,16 +292,12 @@ void mozilla::intl::Locale::PerformComplexLanguageMappings() { println( f""" - {if_kind} ({cond}) {{""".strip( - "\n" - ) + {if_kind} ({cond}) {{""".strip("\n") ) println( f""" - SetLanguage("{language}");""".strip( - "\n" - ) + SetLanguage("{language}");""".strip("\n") ) if script is not None: @@ -327,32 +305,24 @@ void mozilla::intl::Locale::PerformComplexLanguageMappings() { f""" if (Script().Missing()) {{ SetScript("{script}"); - }}""".strip( - "\n" - ) + }}""".strip("\n") ) if region is not None: println( f""" if (Region().Missing()) {{ SetRegion("{region}"); - }}""".strip( - "\n" - ) + }}""".strip("\n") ) println( """ - }""".strip( - "\n" - ) + }""".strip("\n") ) println( """ } -""".strip( - "\n" - ) +""".strip("\n") ) @@ -406,14 +376,12 @@ void mozilla::intl::Locale::PerformComplexRegionMappings() { println( f""" - {if_kind} ({cond}) {{""".strip( - "\n" - ) + {if_kind} ({cond}) {{""".strip("\n") ) - replacement_regions = sorted( - {region for (_, _, region) in non_default_replacements} - ) + replacement_regions = sorted({ + region for (_, _, region) in non_default_replacements + }) first_case = True for replacement_region in replacement_regions: @@ -441,9 +409,7 @@ void mozilla::intl::Locale::PerformComplexRegionMappings() { f""" {if_kind} ({cond}) {{ SetRegion("{replacement_region}"); - }}""".rstrip().strip( - "\n" - ) + }}""".rstrip().strip("\n") ) println( @@ -451,17 +417,13 @@ void mozilla::intl::Locale::PerformComplexRegionMappings() { else {{ SetRegion("{default}"); }} - }}""".rstrip().strip( - "\n" - ) + }}""".rstrip().strip("\n") ) println( """ } -""".strip( - "\n" - ) +""".strip("\n") ) @@ -531,9 +493,7 @@ bool mozilla::intl::Locale::PerformVariantMappings() { if ({no_replacements}) {{ removeVariantAt(i); }} -""".strip( - "\n" - ) +""".strip("\n") ) for deprecated_variant, (type, replacement) in sorted( @@ -543,26 +503,20 @@ bool mozilla::intl::Locale::PerformVariantMappings() { f""" else if (variant.Span() == mozilla::MakeStringSpan("{deprecated_variant}")) {{ removeVariantAt(i); -""".strip( - "\n" - ) +""".strip("\n") ) if type == "language": println( f""" SetLanguage("{replacement}"); -""".strip( - "\n" - ) +""".strip("\n") ) elif type == "region": println( f""" SetRegion("{replacement}"); -""".strip( - "\n" - ) +""".strip("\n") ) else: assert type == "variant" @@ -571,17 +525,13 @@ bool mozilla::intl::Locale::PerformVariantMappings() { if (!insertVariantSortedIfNotPresent(mozilla::MakeStringSpan("{replacement}"))) {{ return false; }} -""".strip( - "\n" - ) +""".strip("\n") ) println( """ } -""".strip( - "\n" - ) +""".strip("\n") ) println( @@ -592,9 +542,7 @@ bool mozilla::intl::Locale::PerformVariantMappings() { } return true; } -""".strip( - "\n" - ) +""".strip("\n") ) @@ -732,9 +680,7 @@ bool mozilla::intl::Locale::UpdateLegacyMappings() { mRegion.Set(mozilla::MakeStringSpan("")); } } -""".rstrip().lstrip( - "\n" - ) +""".rstrip().lstrip("\n") ) # Finally handle all remaining cases. @@ -784,9 +730,7 @@ bool mozilla::intl::Locale::UpdateLegacyMappings() { println( f""" else if ({cond}) {{ -""".rstrip().lstrip( - "\n" - ) +""".rstrip().lstrip("\n") ) mappings = legacy_mappings_by_language[langs[0]] @@ -820,9 +764,7 @@ bool mozilla::intl::Locale::UpdateLegacyMappings() { println( f""" {" " * i}{maybe_else}if (auto* {variant} = findVariant(mozilla::MakeStringSpan("{variant}"))) {{ -""".rstrip().lstrip( - "\n" - ) +""".rstrip().lstrip("\n") ) indent = " " * len_variants @@ -832,26 +774,20 @@ bool mozilla::intl::Locale::UpdateLegacyMappings() { {indent}removeVariant{"s" if len_variants > 1 else ""}({", ".join(sorted_variants)}); {indent}SetLanguage("{r_language}"); {indent}{"return true;" if not chain_if else ""} -""".rstrip().lstrip( - "\n" - ) +""".rstrip().lstrip("\n") ) for i in range(len_variants, 0, -1): println( f""" {" " * (i - 1)}}} -""".rstrip().lstrip( - "\n" - ) +""".rstrip().lstrip("\n") ) println( """ } -""".rstrip().lstrip( - "\n" - ) +""".rstrip().lstrip("\n") ) println( @@ -962,9 +898,9 @@ def readSupplementalData(core_file): # See UTS35, §Annex C, Definitions - 1. Multimap interpretation. def language_id_to_multimap(language_id): match = re_unicode_language_id.match(language_id) - assert ( - match is not None - ), f"{language_id} invalid Unicode BCP 47 locale identifier" + assert match is not None, ( + f"{language_id} invalid Unicode BCP 47 locale identifier" + ) canonical_language_id = bcp47_canonical( *match.group("language", "script", "region", "variants") @@ -1011,9 +947,9 @@ def readSupplementalData(core_file): language_id_to_multimap("und-" + r) for r in replacements ] - assert ( - type not in territory_exception_rules - ), f"Duplicate alias rule: {type}" + assert type not in territory_exception_rules, ( + f"Duplicate alias rule: {type}" + ) territory_exception_rules[type] = replacement_list @@ -1213,9 +1149,9 @@ def readSupplementalData(core_file): assert r_language is not None, "Can't remove a language subtag" # We don't yet support this case. - assert ( - r_variants is None - ), f"Unhandled variant replacement in language alias: {replacement}" + assert r_variants is None, ( + f"Unhandled variant replacement in language alias: {replacement}" + ) if replacement == (Any, None, None, None): language_mappings[language] = r_language @@ -1223,9 +1159,9 @@ def readSupplementalData(core_file): complex_language_mappings[language] = replacement[:-1] elif script is not None: # We don't support removing script subtags. - assert ( - r_script is not None - ), f"Can't remove a script subtag: {replacement}" + assert r_script is not None, ( + f"Can't remove a script subtag: {replacement}" + ) # We only support one-to-one script mappings for now. assert replacement == ( @@ -1238,9 +1174,9 @@ def readSupplementalData(core_file): script_mappings[script] = r_script elif region is not None: # We don't support removing region subtags. - assert ( - r_region is not None - ), f"Can't remove a region subtag: {replacement}" + assert r_region is not None, ( + f"Can't remove a region subtag: {replacement}" + ) # We only support one-to-one region mappings for now. assert replacement == ( @@ -1262,9 +1198,9 @@ def readSupplementalData(core_file): assert len(variants.split("-")) == 1 # We only support one-to-one variant mappings for now. - assert ( - multi_map_size(replacement) <= 1 - ), f"Unhandled replacement in variant alias: {replacement}" + assert multi_map_size(replacement) <= 1, ( + f"Unhandled replacement in variant alias: {replacement}" + ) if r_language is not None: variant_mappings[variants] = ("language", r_language) @@ -1311,21 +1247,21 @@ def readSupplementalData(core_file): for likely_subtag in tree.iterfind(".//likelySubtag"): from_tag = bcp47_id(likely_subtag.get("from")) from_match = re_unicode_language_id.match(from_tag) - assert ( - from_match is not None - ), f"{from_tag} invalid Unicode BCP 47 locale identifier" - assert ( - from_match.group("variants") is None - ), f"unexpected variant subtags in {from_tag}" + assert from_match is not None, ( + f"{from_tag} invalid Unicode BCP 47 locale identifier" + ) + assert from_match.group("variants") is None, ( + f"unexpected variant subtags in {from_tag}" + ) to_tag = bcp47_id(likely_subtag.get("to")) to_match = re_unicode_language_id.match(to_tag) - assert ( - to_match is not None - ), f"{to_tag} invalid Unicode BCP 47 locale identifier" - assert ( - to_match.group("variants") is None - ), f"unexpected variant subtags in {to_tag}" + assert to_match is not None, ( + f"{to_tag} invalid Unicode BCP 47 locale identifier" + ) + assert to_match.group("variants") is None, ( + f"unexpected variant subtags in {to_tag}" + ) from_canonical = bcp47_canonical( *from_match.group("language", "script", "region", "variants") @@ -1472,9 +1408,9 @@ def readUnicodeExtensions(core_file): continue # All other names should match the 'type' production. - assert ( - typeRE.match(name) is not None - ), f"{name} matches the 'type' production" + assert typeRE.match(name) is not None, ( + f"{name} matches the 'type' production" + ) # <https://unicode.org/reports/tr35/#Unicode_Locale_Extension_Data_Files>: # @@ -1548,9 +1484,9 @@ def readUnicodeExtensions(core_file): tree = ET.parse(file) for alias in tree.iterfind(".//subdivisionAlias"): type = alias.get("type") - assert ( - typeRE.match(type) is not None - ), f"{type} matches the 'type' production" + assert typeRE.match(type) is not None, ( + f"{type} matches the 'type' production" + ) # Take the first replacement when multiple ones are present. replacement = alias.get("replacement").split(" ")[0].lower() @@ -1560,9 +1496,9 @@ def readUnicodeExtensions(core_file): replacement += "zzzz" # Assert the replacement is syntactically correct. - assert ( - typeRE.match(replacement) is not None - ), f"replacement {replacement} matches the 'type' production" + assert typeRE.match(replacement) is not None, ( + f"replacement {replacement} matches the 'type' production" + ) # 'subdivisionAlias' applies to 'rg' and 'sd' keys. mapping["u"].setdefault("rg", {})[type] = replacement @@ -1851,9 +1787,9 @@ def writeCLDRLanguageTagLikelySubtagsTest(println, data, url): region = region_mappings[region] else: # Assume no complex region mappings are needed for now. - assert ( - region not in complex_region_mappings - ), f"unexpected region with complex mappings: {region}" + assert region not in complex_region_mappings, ( + f"unexpected region with complex mappings: {region}" + ) return (language, script, region) @@ -3228,9 +3164,7 @@ static inline bool Is{extension}Type(mozilla::Span<const char> type, const char return type.size() == (Length - 1) && memcmp(type.data(), str, Length - 1) == 0; }} -""".rstrip( - "\n" - ) +""".rstrip("\n") ) linear_search_max_length = 4 @@ -3276,9 +3210,7 @@ static inline const char* Search{extension}Replacement( }} return nullptr; }} -""".rstrip( - "\n" - ) +""".rstrip("\n") ) println( @@ -3337,9 +3269,7 @@ const char* mozilla::intl::Locale::Replace{extension}ExtensionType( cond = (" ||\n" + " " * (2 + len(if_kind) + 2)).join(cond) println( f""" - {if_kind} ({cond}) {{""".strip( - "\n" - ) + {if_kind} ({cond}) {{""".strip("\n") ) first_key = False @@ -3355,9 +3285,7 @@ const char* mozilla::intl::Locale::Replace{extension}ExtensionType( println( f""" return Search{extension}Replacement(types, aliases, type); -""".strip( - "\n" - ) +""".strip("\n") ) else: for type, replacement in replacements: @@ -3365,25 +3293,19 @@ const char* mozilla::intl::Locale::Replace{extension}ExtensionType( f""" if (Is{extension}Type(type, "{type}")) {{ return "{replacement}"; - }}""".strip( - "\n" - ) + }}""".strip("\n") ) println( """ - }""".lstrip( - "\n" - ) + }""".lstrip("\n") ) println( """ return nullptr; } -""".strip( - "\n" - ) +""".strip("\n") ) @@ -3626,9 +3548,7 @@ inline constexpr SimpleMeasureUnit simpleMeasureUnits[] = { } // namespace mozilla::intl #endif -""".strip( - "\n" - ) +""".strip("\n") ) writeUnitTestFiles(all_units, sanctioned_units) @@ -3667,9 +3587,7 @@ def writeUnitTestFiles(all_units, sanctioned_units): println( """ if (typeof reportCompare === "function") -{}reportCompare(true, true);""".format( - " " * indent - ) +{}reportCompare(true, true);""".format(" " * indent) ) write_test( @@ -3999,16 +3917,16 @@ def updateNumberingSystems(topsrcdir, args): # Assert ICU includes support for all required numbering systems. If this assertion fails, # something is broken in ICU. - assert all_numbering_systems_simple_digits.issuperset( - numbering_systems - ), f"{numbering_systems.difference(all_numbering_systems_simple_digits)}" + assert all_numbering_systems_simple_digits.issuperset(numbering_systems), ( + f"{numbering_systems.difference(all_numbering_systems_simple_digits)}" + ) # Assert the spec requires support for all numbering systems with simple digit mappings. If # this assertion fails, file a PR at <https://github.com/tc39/ecma402> to include any new # numbering systems. - assert all_numbering_systems_simple_digits.issubset( - numbering_systems - ), f"{all_numbering_systems_simple_digits.difference(numbering_systems)}" + assert all_numbering_systems_simple_digits.issubset(numbering_systems), ( + f"{all_numbering_systems_simple_digits.difference(numbering_systems)}" + ) writeNumberingSystemFiles(all_numbering_systems) @@ -4077,8 +3995,7 @@ if __name__ == "__main__": metavar="URL", default="https://www.six-group.com/dam/download/financial-information/data-center/iso-currrency/lists/list-one.xml", # NOQA: E501 type=EnsureHttps, - help="Download url for the currency & funds code list (default: " - "%(default)s)", + help="Download url for the currency & funds code list (default: %(default)s)", ) parser_currency.add_argument( "--out", diff --git a/js/src/devtools/automation/autospider.py b/js/src/devtools/automation/autospider.py @@ -30,8 +30,7 @@ def directories(pathmodule, cwd, fixup=lambda s: s): source = pathmodule.abspath(pathmodule.join(js_src, "..", "..")) mozbuild = pathmodule.abspath( # os.path.expanduser does not work on Windows. - env.get("MOZBUILD_STATE_PATH") - or pathmodule.join(Path.home(), ".mozbuild") + env.get("MOZBUILD_STATE_PATH") or pathmodule.join(Path.home(), ".mozbuild") ) fetches = pathmodule.abspath(env.get("MOZ_FETCHES_DIR", mozbuild)) return Dirs(scripts, js_src, source, fetches) @@ -148,8 +147,7 @@ parser.add_argument( type=str, metavar="TESTSUITE", default="", - help="comma-separated set of test suites to remove from the variant's default " - "set", + help="comma-separated set of test suites to remove from the variant's default set", ) parser.add_argument( "--build-only", @@ -591,12 +589,10 @@ if "jittest" in test_suites: "--timeout=300", "--jitflags=all", ] - results.append( - ( - "mach jit-test", - run_mach_command(["jit-test", "--", *auto_args, *extra_args["jit-test"]]), - ) - ) + results.append(( + "mach jit-test", + run_mach_command(["jit-test", "--", *auto_args, *extra_args["jit-test"]]), + )) if "jsapitests" in test_suites: st = run_jsapitests([]) if st == 0: @@ -606,23 +602,17 @@ if "jstests" in test_suites: auto_args = [] if AUTOMATION: auto_args = ["--no-progress", "--format=automation", "--timeout=300"] - results.append( - ( - "mach jstests", - run_mach_command(["jstests", "--", *auto_args, *extra_args["jstests"]]), - ) - ) + results.append(( + "mach jstests", + run_mach_command(["jstests", "--", *auto_args, *extra_args["jstests"]]), + )) if "gdb" in test_suites: test_script = os.path.join(DIR.js_src, "gdb", "run-tests.py") auto_args = ["-s", "-o", "--no-progress"] if AUTOMATION else [] - results.append( - ( - "gdb", - run_test_command( - [PYTHON, test_script, *auto_args, *extra_args["gdb"], OBJDIR] - ), - ) - ) + results.append(( + "gdb", + run_test_command([PYTHON, test_script, *auto_args, *extra_args["gdb"], OBJDIR]), + )) # FIXME bug 1291449: This would be unnecessary if we could run msan with -mllvm # -msan-keep-going, but in clang 3.8 it causes a hang during compilation. @@ -691,15 +681,13 @@ if args.variant == "wasi": # Generate stacks from minidumps. if use_minidump: - run_mach_command( - [ - "python", - "--virtualenv=build", - os.path.join(DIR.source, "testing/mozbase/mozcrash/mozcrash/mozcrash.py"), - os.getenv("TMPDIR", "/tmp"), - os.path.join(OBJDIR, "dist/crashreporter-symbols"), - ] - ) + run_mach_command([ + "python", + "--virtualenv=build", + os.path.join(DIR.source, "testing/mozbase/mozcrash/mozcrash/mozcrash.py"), + os.getenv("TMPDIR", "/tmp"), + os.path.join(OBJDIR, "dist/crashreporter-symbols"), + ]) for name, st in results: print("exit status %d for '%s'" % (st, name)) diff --git a/js/src/devtools/gc/gc-test.py b/js/src/devtools/gc/gc-test.py @@ -150,7 +150,7 @@ if __name__ == "__main__": "--baseline", metavar="JSON_PATH", dest="baseline_path", - help="json file with baseline values to " "compare against", + help="json file with baseline values to compare against", ) (OPTIONS, args) = op.parse_args() diff --git a/js/src/devtools/rootAnalysis/analyze.py b/js/src/devtools/rootAnalysis/analyze.py @@ -358,7 +358,7 @@ parser.add_argument( "--expect-file", type=str, nargs="?", - help="deprecated option, temporarily still present for backwards " "compatibility", + help="deprecated option, temporarily still present for backwards compatibility", ) parser.add_argument( "--verbose", @@ -434,12 +434,13 @@ for step in steps: # Trim the {curly brackets} off of the output keys. data[name[1:-1]] = outfiles[i] num_outputs += 1 - assert ( - len(outfiles) == num_outputs - ), 'step "%s": mismatched number of output files (%d) and params (%d)' % ( - step, - num_outputs, - len(outfiles), + assert len(outfiles) == num_outputs, ( + 'step "%s": mismatched number of output files (%d) and params (%d)' + % ( + step, + num_outputs, + len(outfiles), + ) ) # NOQA: E501 if args.step: diff --git a/js/src/devtools/rootAnalysis/explain.py b/js/src/devtools/rootAnalysis/explain.py @@ -158,11 +158,13 @@ num_refs = 0 num_missing = 0 try: - with open(args.rootingHazards) as rootingHazards, open( - args.hazards, "w" - ) as hazards, open(args.extra, "w") as extra, open(args.refs, "w") as refs, open( - args.html, "w" - ) as html: + with ( + open(args.rootingHazards) as rootingHazards, + open(args.hazards, "w") as hazards, + open(args.extra, "w") as extra, + open(args.refs, "w") as refs, + open(args.html, "w") as html, + ): current_gcFunction = None hazardousGCFunctions = set() diff --git a/js/src/devtools/rootAnalysis/mach_commands.py b/js/src/devtools/rootAnalysis/mach_commands.py @@ -159,8 +159,9 @@ CLOBBER_CHOICES = {"objdir", "work", "shell", "all"} "what", default=["objdir", "work"], nargs="*", - help="Target to clobber, must be one of {{{}}} (default " - "objdir and work).".format(", ".join(CLOBBER_CHOICES)), + help="Target to clobber, must be one of {{{}}} (default objdir and work).".format( + ", ".join(CLOBBER_CHOICES) + ), ) def clobber(command_context, what, **kwargs): from mozbuild.controller.clobber import Clobberer @@ -344,15 +345,13 @@ def gather_hazard_data(command_context, **kwargs): ) fh.write(data) - buildscript = " ".join( - [ - command_context.topsrcdir + "/mach hazards compile", - *kwargs.get("what", []), - "--job-size=3.0", # Conservatively estimate 3GB/process - "--project=" + project, - "--haz-objdir=" + objdir, - ] - ) + buildscript = " ".join([ + command_context.topsrcdir + "/mach hazards compile", + *kwargs.get("what", []), + "--job-size=3.0", # Conservatively estimate 3GB/process + "--project=" + project, + "--haz-objdir=" + objdir, + ]) args = [ os.path.join(script_dir(command_context), "run_complete"), "--foreground", diff --git a/js/src/devtools/rootAnalysis/run-test.py b/js/src/devtools/rootAnalysis/run-test.py @@ -84,9 +84,11 @@ if not cfg.sixgill_plugin: cfg.sixgill, "usr", "libexec", "sixgill", "gcc", "xgill.so" ) -subprocess.check_call( - [cfg.js, "-e", 'if (!getBuildConfiguration("has-ctypes")) quit(1)'] -) +subprocess.check_call([ + cfg.js, + "-e", + 'if (!getBuildConfiguration("has-ctypes")) quit(1)', +]) def binpath(prog): diff --git a/js/src/devtools/rootAnalysis/t/graph/test.py b/js/src/devtools/rootAnalysis/t/graph/test.py @@ -42,13 +42,11 @@ for src, dst in [ assert callgraph.calleeGraph[node[src]][node[dst]] funcInfo = test.load_funcInfo() -rroots = set( - [ - callgraph.mangledToUnmangled[f] - for f in funcInfo - if funcInfo[f].get("recursive_root") - ] -) +rroots = set([ + callgraph.mangledToUnmangled[f] + for f in funcInfo + if funcInfo[f].get("recursive_root") +]) assert len(set([node[1], node[2]]) & rroots) == 1 assert len(set([node[4], node[5]]) & rroots) == 1 assert len(rroots) == 4, f"rroots = {rroots}" # n1, n4, f, self_recursive diff --git a/js/src/frontend/align_stack_comment.py b/js/src/frontend/align_stack_comment.py @@ -4,13 +4,13 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. -""" Usage: align_stack_comment.py FILE +"""Usage: align_stack_comment.py FILE - This script aligns the stack transition comment in BytecodeEmitter and - its helper classes. +This script aligns the stack transition comment in BytecodeEmitter and +its helper classes. - The stack transition comment looks like the following: - // [stack] VAL1 VAL2 VAL3 +The stack transition comment looks like the following: + // [stack] VAL1 VAL2 VAL3 """ import re diff --git a/js/src/gdb/run-tests.py b/js/src/gdb/run-tests.py @@ -252,14 +252,12 @@ def run_tests(tests, summary): # python 3.3 fixed a bug with concurrently writing .pyc files. # https://bugs.python.org/issue13146 embedded_version = ( - subprocess.check_output( - [ - OPTIONS.gdb_executable, - "--batch", - "--ex", - "python import sys; print(sys.hexversion)", - ] - ) + subprocess.check_output([ + OPTIONS.gdb_executable, + "--batch", + "--ex", + "python import sys; print(sys.hexversion)", + ]) .decode("ascii") .strip() ) diff --git a/js/src/jit-test/jit_test.py b/js/src/jit-test/jit_test.py @@ -107,13 +107,13 @@ def main(argv): "--failed-only", dest="failed_only", action="store_true", - help="if --show-output is given, only print output for" " failed tests", + help="if --show-output is given, only print output for failed tests", ) op.add_argument( "--no-show-failed", dest="no_show_failed", action="store_true", - help="don't print output for failed tests" " (no-op with --show-output)", + help="don't print output for failed tests (no-op with --show-output)", ) op.add_argument( "-x", @@ -327,14 +327,14 @@ def main(argv): action="store", type=str, default="/data/local/tmp/test_root", - help="The remote directory to use as test root" " (e.g. %(default)s)", + help="The remote directory to use as test root (e.g. %(default)s)", ) op.add_argument( "--localLib", dest="local_lib", action="store", type=str, - help="The location of libraries to push -- preferably" " stripped", + help="The location of libraries to push -- preferably stripped", ) op.add_argument( "--repeat", type=int, default=1, help="Repeat tests the given number of times." @@ -536,8 +536,7 @@ def main(argv): elif options.debugger: if job_count > 1: print( - "Multiple tests match command line" - " arguments, debugger can only run one" + "Multiple tests match command line arguments, debugger can only run one" ) jobs = list(job_list) @@ -589,7 +588,7 @@ def main(argv): except OSError: if not os.path.exists(prefix[0]): print( - "JS shell argument: file does not exist:" f" '{prefix[0]}'", + f"JS shell argument: file does not exist: '{prefix[0]}'", file=sys.stderr, ) sys.exit(1) diff --git a/js/src/jit/GenerateABIFunctionType.py b/js/src/jit/GenerateABIFunctionType.py @@ -106,7 +106,7 @@ def arm32_soft_fp_args(func_type): contents += f"MakeInt64(stack_pointer[{stackOffset}], stack_pointer[{stackOffset + 1}])" stackOffset += 2 else: - contents += f"MakeInt64(a{intRegIndex}, a{intRegIndex+1})" + contents += f"MakeInt64(a{intRegIndex}, a{intRegIndex + 1})" intRegIndex += 2 elif arg == "Float32": if intRegIndex == numIntArgRegs: @@ -162,7 +162,7 @@ def arm32_hard_fp_args(func_type): contents += f"MakeInt64(stack_pointer[{stackOffset}], stack_pointer[{stackOffset + 1}])" stackOffset += 2 else: - contents += f"MakeInt64(a{intRegIndex}, a{intRegIndex+1})" + contents += f"MakeInt64(a{intRegIndex}, a{intRegIndex + 1})" intRegIndex += 2 elif arg == "Float32": if floatRegIndex == numFloatArgRegs: diff --git a/js/src/shell/js-gdb.py b/js/src/shell/js-gdb.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" GDB Python customization auto-loader for JS shell binary """ +"""GDB Python customization auto-loader for JS shell binary""" # This script will be installed into $objdir/dist/bin. Add $objdir to gdb's # source search path and load in the Gecko+JS init file. diff --git a/js/src/tests/compare_bench.py b/js/src/tests/compare_bench.py @@ -1,6 +1,5 @@ #!/usr/bin/env python2.4 -"""usage: %progname candidate_path baseline_path -""" +"""usage: %progname candidate_path baseline_path""" import json import optparse diff --git a/js/src/tests/jstests.py b/js/src/tests/jstests.py @@ -112,7 +112,7 @@ def parse_args(): op.add_argument( "--xul-info", dest="xul_info_src", - help="config data for xulRuntime" " (avoids search for config/autoconf.mk)", + help="config data for xulRuntime (avoids search for config/autoconf.mk)", ) harness_og = op.add_argument_group("Harness Controls", "Control how tests are run.") @@ -121,19 +121,19 @@ def parse_args(): "--worker-count", type=int, default=max(1, get_cpu_count()), - help="Number of tests to run in parallel" " (default %(default)s)", + help="Number of tests to run in parallel (default %(default)s)", ) harness_og.add_argument( "-t", "--timeout", type=float, default=150.0, - help="Set maximum time a test is allows to run" " (in seconds).", + help="Set maximum time a test is allows to run (in seconds).", ) harness_og.add_argument( "--show-slow", action="store_true", - help="Show tests taking longer than a minimum time" " (in seconds).", + help="Show tests taking longer than a minimum time (in seconds).", ) harness_og.add_argument( "--slow-test-threshold", @@ -168,12 +168,12 @@ def parse_args(): harness_og.add_argument( "--tbpl", action="store_true", - help="Runs each test in all configurations tbpl" " tests.", + help="Runs each test in all configurations tbpl tests.", ) harness_og.add_argument( "--tbpl-debug", action="store_true", - help="Runs each test in some faster configurations" " tbpl tests.", + help="Runs each test in some faster configurations tbpl tests.", ) harness_og.add_argument( "-g", "--debug", action="store_true", help="Run a test in debugger." @@ -187,7 +187,7 @@ def parse_args(): harness_og.add_argument( "--passthrough", action="store_true", - help="Run tests with stdin/stdout attached to" " caller.", + help="Run tests with stdin/stdout attached to caller.", ) harness_og.add_argument( "--test-reflect-stringify", @@ -244,14 +244,14 @@ def parse_args(): action="store", type=str, default="/data/local/tmp/test_root", - help="The remote directory to use as test root" " (e.g. %(default)s)", + help="The remote directory to use as test root (e.g. %(default)s)", ) harness_og.add_argument( "--localLib", dest="local_lib", action="store", type=str, - help="The location of libraries to push -- preferably" " stripped", + help="The location of libraries to push -- preferably stripped", ) harness_og.add_argument( "--no-xdr", @@ -319,7 +319,7 @@ def parse_args(): input_og.add_argument( "--no-extensions", action="store_true", - help="Run only tests conforming to the ECMAScript 5" " standard.", + help="Run only tests conforming to the ECMAScript 5 standard.", ) input_og.add_argument( "--repeat", type=int, default=1, help="Repeat tests the given number of times." @@ -336,23 +336,23 @@ def parse_args(): "-o", "--show-output", action="store_true", - help="Print each test's output to the file given by" " --output-file.", + help="Print each test's output to the file given by --output-file.", ) output_og.add_argument( "-F", "--failed-only", action="store_true", - help="If a --show-* option is given, only print" " output for failed tests.", + help="If a --show-* option is given, only print output for failed tests.", ) output_og.add_argument( "--no-show-failed", action="store_true", - help="Don't print output for failed tests" " (no-op with --show-output).", + help="Don't print output for failed tests (no-op with --show-output).", ) output_og.add_argument( "-O", "--output-file", - help="Write all output to the given file" " (default: stdout).", + help="Write all output to the given file (default: stdout).", ) output_og.add_argument( "--failure-file", help="Write all not-passed tests to the given file." @@ -375,7 +375,7 @@ def parse_args(): dest="format", default="none", choices=["automation", "none"], - help="Output format. Either automation or none" " (default %(default)s).", + help="Output format. Either automation or none (default %(default)s).", ) output_og.add_argument( "--log-wptreport", @@ -458,9 +458,9 @@ def parse_args(): # requested tests set. if options.test_file: for test_file in options.test_file: - requested_paths |= set( - [line.strip() for line in open(test_file).readlines()] - ) + requested_paths |= set([ + line.strip() for line in open(test_file).readlines() + ]) excluded_paths = set(options.excluded_paths) @@ -556,17 +556,15 @@ def load_wpt_tests(xul_tester, requested_paths, excluded_paths, update_manifest= ) kwargs = vars(wptcommandline.create_parser().parse_args([])) - kwargs.update( - { - "config": os.path.join( - manifest_root, "_tests", "web-platform", "wptrunner.local.ini" - ), - "gecko_e10s": False, - "product": "firefox", - "verify": False, - "wasm": xul_tester.test("wasmIsSupported()"), - } - ) + kwargs.update({ + "config": os.path.join( + manifest_root, "_tests", "web-platform", "wptrunner.local.ini" + ), + "gecko_e10s": False, + "product": "firefox", + "verify": False, + "wasm": xul_tester.test("wasmIsSupported()"), + }) wptcommandline.set_from_config(kwargs) def filter_jsshell_tests(it): @@ -793,16 +791,17 @@ def main(): if options.debug: if test_count > 1: print( - "Multiple tests match command line arguments," - " debugger can only run one" + "Multiple tests match command line arguments, debugger can only run one" ) for tc in test_gen: print(f" {tc.path}") return 2 - with changedir(test_dir), change_env( - test_environment - ), TemporaryDirectory() as tempdir: + with ( + changedir(test_dir), + change_env(test_environment), + TemporaryDirectory() as tempdir, + ): cmd = next(test_gen).get_command(prefix, tempdir) if options.show_cmd: print(list2cmdline(cmd)) @@ -844,9 +843,11 @@ def main(): return 0 if results.all_passed() else 1 - with changedir(test_dir), change_env( - test_environment - ), TemporaryDirectory() as tempdir: + with ( + changedir(test_dir), + change_env(test_environment), + TemporaryDirectory() as tempdir, + ): results = ResultsSink("jstests", options, test_count) try: for out in run_all_tests(test_gen, prefix, tempdir, results.pb, options): diff --git a/js/src/tests/lib/jittests.py b/js/src/tests/lib/jittests.py @@ -288,12 +288,12 @@ class JitTest: else: test.expect_status = status except ValueError: - print("warning: couldn't parse exit status" f" {value}") + print(f"warning: couldn't parse exit status {value}") elif name == "thread-count": try: test.jitflags.append(f"--thread-count={int(value, 0)}") except ValueError: - print("warning: couldn't parse thread-count" f" {value}") + print(f"warning: couldn't parse thread-count {value}") elif name == "include": test.other_lib_includes.append(value) elif name == "local-include": @@ -310,8 +310,7 @@ class JitTest: print("warning: couldn't parse skip-variant-if") else: print( - f"{path}: warning: unrecognized |jit-test| attribute" - f" {part}" + f"{path}: warning: unrecognized |jit-test| attribute {part}" ) elif name == "slow": test.slow = True @@ -339,9 +338,9 @@ class JitTest: # works, and not meant as a way to accept temporary # failing tests. These tests should either be fixed or # skipped. - assert ( - "self-test" in path - ), f"{path}: has an unexpected crash annotation." + assert "self-test" in path, ( + f"{path}: has an unexpected crash annotation." + ) test.expect_crash = True elif name.startswith("--"): # // |jit-test| --ion-gvn=off; --no-sse4 @@ -352,9 +351,7 @@ class JitTest: # // |jit-test| -P pref(=value)? test.jitflags.append("--setpref=" + prefAndValue[1]) else: - print( - f"{path}: warning: unrecognized |jit-test| attribute" f" {part}" - ) + print(f"{path}: warning: unrecognized |jit-test| attribute {part}") if options.valgrind_all: test.valgrind = True diff --git a/js/src/tests/lib/manifest.py b/js/src/tests/lib/manifest.py @@ -42,9 +42,7 @@ var xulRuntime = {{ shell: true }}; var release_or_beta = getBuildConfiguration('release_or_beta'); var isDebugBuild={str(self.isdebug).lower()}; var Android={str(self.os == "Android").lower()}; -""".replace( - "\n", "" - ) +""".replace("\n", "") @classmethod def create(cls, jsdir): @@ -283,18 +281,16 @@ def _build_manifest_script_entry(script_name, test): properties = [] if test.terms: # Remove jsreftest internal terms. - terms = " ".join( - [ - term - for term in test.terms.split() - if not ( - term in {"module", "async", "test262-raw"} - or term.startswith("error:") - or term.startswith("ignore-flag(") - or term.startswith("shell-option(") - ) - ] - ) + terms = " ".join([ + term + for term in test.terms.split() + if not ( + term in {"module", "async", "test262-raw"} + or term.startswith("error:") + or term.startswith("ignore-flag(") + or term.startswith("shell-option(") + ) + ]) if terms: line.append(terms) if test.error: @@ -512,7 +508,7 @@ def _parse_external_manifest(filename, relpath): if not matches: matches = include_re.match(line) if not matches: - print("warning: unrecognized line in jstests.list:" f" {line}") + print(f"warning: unrecognized line in jstests.list: {line}") continue include_file = matches.group("path") @@ -533,13 +529,11 @@ def _parse_external_manifest(filename, relpath): assert path.endswith("jstests.list") path = path[: -len("jstests.list")] - entries.append( - { - "path": path, - "terms": matches.group("terms"), - "comment": comment.strip(), - } - ) + entries.append({ + "path": path, + "terms": matches.group("terms"), + "comment": comment.strip(), + }) # if one directory name is a prefix of another, we want the shorter one # first @@ -566,16 +560,14 @@ def _apply_external_manifests(filename, testcase, entries, xul_tester): def _is_test_file(path_from_root, basename, filename, path_options): # Any file whose basename matches something in this set is ignored. - EXCLUDED = set( - ( - "browser.js", - "shell.js", - "template.js", - "user.js", - "js-test-driver-begin.js", - "js-test-driver-end.js", - ) - ) + EXCLUDED = set(( + "browser.js", + "shell.js", + "template.js", + "user.js", + "js-test-driver-begin.js", + "js-test-driver-end.js", + )) # Skip js files in the root test directory. if not path_from_root: diff --git a/js/src/tests/lib/results.py b/js/src/tests/lib/results.py @@ -121,14 +121,12 @@ class TestResult: test_output += "expected %s, found %s" % (expected, test.status) if test.message: test_output += ' (with message: "%s")' % (test.message,) - subtests.append( - { - "test": output.test.wpt.id, - "subtest": test.name, - "status": test.status, - "expected": expected, - } - ) + subtests.append({ + "test": output.test.wpt.id, + "subtest": test.name, + "status": test.status, + "expected": expected, + }) results.append(test_result) stdout.append(test_output) @@ -172,8 +170,7 @@ class TestResult: results.append((cls.PASS, msg)) else: m = re.match( - "--- NOTE: IN THIS TESTCASE, WE EXPECT EXIT CODE" - " ((?:-|\\d)+) ---", + "--- NOTE: IN THIS TESTCASE, WE EXPECT EXIT CODE ((?:-|\\d)+) ---", line, ) if m: diff --git a/js/src/tests/lib/wptreport.py b/js/src/tests/lib/wptreport.py @@ -26,23 +26,19 @@ class WptreportHandler: """ Produce the "suite_start" message at the present time. """ - self.formatter.suite_start( - { - "time": time(), - "run_info": {}, - } - ) + self.formatter.suite_start({ + "time": time(), + "run_info": {}, + }) def suite_end(self): """ Produce the "suite_end" message at the present time and write the results to the file path given in the constructor. """ - result = self.formatter.suite_end( - { - "time": time(), - } - ) + result = self.formatter.suite_end({ + "time": time(), + }) with open(self.out, "w") as fp: fp.write(result) @@ -65,21 +61,17 @@ class WptreportHandler: end_time = time() start_time = end_time - duration - self.formatter.test_start( - { - "test": testname, - "time": start_time, - } - ) + self.formatter.test_start({ + "test": testname, + "time": start_time, + }) for subtest in result["subtests"]: self.formatter.test_status(subtest) - self.formatter.test_end( - { - "test": testname, - "time": end_time, - "status": result["status"], - "expected": result["expected"], - } - ) + self.formatter.test_end({ + "test": testname, + "time": end_time, + "status": result["status"], + "expected": result["expected"], + }) diff --git a/js/src/tests/non262/String/make-normalize-generateddata-input.py b/js/src/tests/non262/String/make-normalize-generateddata-input.py @@ -1,10 +1,10 @@ #!/usr/bin/python -B -""" Usage: make-normalize-generateddata-input.py PATH_TO_MOZILLA_CENTRAL +"""Usage: make-normalize-generateddata-input.py PATH_TO_MOZILLA_CENTRAL - This script generates test input data for String.prototype.normalize - from intl/icu/source/data/unidata/NormalizationTest.txt - to js/src/tests/non262/String/normalize-generateddata-input.js +This script generates test input data for String.prototype.normalize +from intl/icu/source/data/unidata/NormalizationTest.txt +to js/src/tests/non262/String/normalize-generateddata-input.js """ import re diff --git a/js/src/tests/parsemark.py b/js/src/tests/parsemark.py @@ -134,20 +134,20 @@ def main(): metavar="COUNT", type=int, default=50, - help="timed data runs that count towards the average" " [%default]", + help="timed data runs that count towards the average [%default]", ) parser.add_option( "-s", "--shell", metavar="PATH", - help="explicit shell location; when omitted, will look" " in likely places", + help="explicit shell location; when omitted, will look in likely places", ) parser.add_option( "-b", "--baseline", metavar="JSON_PATH", dest="baseline_path", - help="json file with baseline values to " "compare against", + help="json file with baseline values to compare against", ) parser.add_option( "--mode", diff --git a/js/src/tests/test/run.py b/js/src/tests/test/run.py @@ -51,28 +51,35 @@ class TestExport(unittest.TestCase): subprocess.check_call(["git", "-C", cloneDir, "checkout", "-b", branch]) # Make changes on the new branch # Remove test/language/export/escaped-from.js - subprocess.check_call( - ["git", "-C", cloneDir, "rm", "test/language/export/escaped-from.js"] - ) + subprocess.check_call([ + "git", + "-C", + cloneDir, + "rm", + "test/language/export/escaped-from.js", + ]) # Rename test/language/export/escaped-default.js - subprocess.check_call( - [ - "git", - "-C", - cloneDir, - "mv", - "test/language/export/escaped-default.js", - "test/language/export/escaped-foobarbaz.js", - ] - ) + subprocess.check_call([ + "git", + "-C", + cloneDir, + "mv", + "test/language/export/escaped-default.js", + "test/language/export/escaped-foobarbaz.js", + ]) # Copy fixtures files fixturesDir = os.path.join(testDir, "fixtures", "import", "files") shutil.copytree(fixturesDir, os.path.join(cloneDir, "test", "temp42")) # Stage and Commit changes subprocess.check_call(["git", "-C", cloneDir, "add", "."]) - subprocess.check_call( - ["git", "-C", cloneDir, "commit", "-m", '"local foo"'] - ) + subprocess.check_call([ + "git", + "-C", + cloneDir, + "commit", + "-m", + '"local foo"', + ]) # Run import script print( diff --git a/js/src/tests/test262-export.py b/js/src/tests/test262-export.py @@ -17,16 +17,14 @@ import yaml # Skip all common files used to support tests for jstests # These files are listed in the README.txt -SUPPORT_FILES = set( - [ - "browser.js", - "shell.js", - "template.js", - "user.js", - "js-test-driver-begin.js", - "js-test-driver-end.js", - ] -) +SUPPORT_FILES = set([ + "browser.js", + "shell.js", + "template.js", + "user.js", + "js-test-driver-begin.js", + "js-test-driver-end.js", +]) # Run once per subdirectory diff --git a/js/src/tests/test262-update.py b/js/src/tests/test262-update.py @@ -14,16 +14,14 @@ from itertools import chain from operator import itemgetter # Skip all tests which use features not supported in SpiderMonkey. -UNSUPPORTED_FEATURES = set( - [ - "tail-call-optimization", - "Intl.Locale-info", # Bug 1693576 - "source-phase-imports", - "source-phase-imports-module-source", - "import-defer", - "nonextensible-applies-to-private", # Bug 1991478 - ] -) +UNSUPPORTED_FEATURES = set([ + "tail-call-optimization", + "Intl.Locale-info", # Bug 1693576 + "source-phase-imports", + "source-phase-imports-module-source", + "import-defer", + "nonextensible-applies-to-private", # Bug 1991478 +]) FEATURE_CHECK_NEEDED = { "Atomics": "!this.hasOwnProperty('Atomics')", "SharedArrayBuffer": "!this.hasOwnProperty('SharedArrayBuffer')", @@ -318,7 +316,7 @@ def convertTestFile(test262parser, testSource, testName, includeSet, strictTests or isAsync or isNegative or testName.split(os.path.sep)[0] == "harness" - ), ("Missing async attribute in: %s" % testName) + ), "Missing async attribute in: %s" % testName # When the "module" attribute is set, the source code is module code. isModule = "module" in testRec @@ -341,38 +339,29 @@ def convertTestFile(test262parser, testSource, testName, includeSet, strictTests else: releaseOrBeta = [f for f in testRec["features"] if f in RELEASE_OR_BETA] if releaseOrBeta: - refTestSkipIf.append( - ( - "release_or_beta", - "%s is not released yet" % ",".join(releaseOrBeta), - ) - ) + refTestSkipIf.append(( + "release_or_beta", + "%s is not released yet" % ",".join(releaseOrBeta), + )) featureCheckNeeded = [ f for f in testRec["features"] if f in FEATURE_CHECK_NEEDED ] if featureCheckNeeded: - refTestSkipIf.append( - ( - "||".join( - [FEATURE_CHECK_NEEDED[f] for f in featureCheckNeeded] - ), - "%s is not enabled unconditionally" - % ",".join(featureCheckNeeded), - ) - ) + refTestSkipIf.append(( + "||".join([FEATURE_CHECK_NEEDED[f] for f in featureCheckNeeded]), + "%s is not enabled unconditionally" % ",".join(featureCheckNeeded), + )) if ( "Atomics" in testRec["features"] and "SharedArrayBuffer" in testRec["features"] ): - refTestSkipIf.append( - ( - "(this.hasOwnProperty('getBuildConfiguration')" - "&&getBuildConfiguration('arm64-simulator'))", - "ARM64 Simulator cannot emulate atomics", - ) - ) + refTestSkipIf.append(( + "(this.hasOwnProperty('getBuildConfiguration')" + "&&getBuildConfiguration('arm64-simulator'))", + "ARM64 Simulator cannot emulate atomics", + )) shellOptions = { SHELL_OPTIONS[f] for f in testRec["features"] if f in SHELL_OPTIONS @@ -577,12 +566,10 @@ def process_test262(test262Dir, test262OutDir, strictTests, externManifests): writeTestFile(test262OutDir, newFileName, newSource) if externRefTest is not None: - externManifests.append( - { - "name": newFileName, - "reftest": externRefTest, - } - ) + externManifests.append({ + "name": newFileName, + "reftest": externRefTest, + }) # Remove "sm/non262.js" because it overwrites our test harness with stub # functions. @@ -873,13 +860,23 @@ def update_test262(args): return fetch_local_changes(inDir, outDir, srcDir, strictTests) if revision == "HEAD": - subprocess.check_call( - ["git", "clone", "--depth=1", "--branch=%s" % branch, url, inDir] - ) + subprocess.check_call([ + "git", + "clone", + "--depth=1", + "--branch=%s" % branch, + url, + inDir, + ]) else: - subprocess.check_call( - ["git", "clone", "--single-branch", "--branch=%s" % branch, url, inDir] - ) + subprocess.check_call([ + "git", + "clone", + "--single-branch", + "--branch=%s" % branch, + url, + inDir, + ]) subprocess.check_call(["git", "-C", inDir, "reset", "--hard", revision]) # If a PR number is provided, fetches only the new and modified files diff --git a/js/src/util/make_unicode.py b/js/src/util/make_unicode.py @@ -571,15 +571,13 @@ def process_special_casing(special_casing, table, index): ) # Ensure all case mapping contexts are known (see Unicode 9.0, §3.13 Default Case Algorithms). - assert set( - [ - "After_I", - "After_Soft_Dotted", - "Final_Sigma", - "More_Above", - "Not_Before_Dot", - ] - ).issuperset( + assert set([ + "After_I", + "After_Soft_Dotted", + "Final_Sigma", + "More_Above", + "Not_Before_Dot", + ]).issuperset( set( filter( partial(is_not, None), @@ -1442,15 +1440,20 @@ def update_unicode(args): pat_version = re.compile(r"# %s-(?P<version>\d+\.\d+\.\d+).txt" % fname) return pat_version.match(f.readline()).group("version") - with open( - os.path.join(base_path, "UnicodeData.txt"), encoding="utf-8" - ) as unicode_data, open( - os.path.join(base_path, "CaseFolding.txt"), encoding="utf-8" - ) as case_folding, open( - os.path.join(base_path, "DerivedCoreProperties.txt"), encoding="utf-8" - ) as derived_core_properties, open( - os.path.join(base_path, "SpecialCasing.txt"), encoding="utf-8" - ) as special_casing: + with ( + open( + os.path.join(base_path, "UnicodeData.txt"), encoding="utf-8" + ) as unicode_data, + open( + os.path.join(base_path, "CaseFolding.txt"), encoding="utf-8" + ) as case_folding, + open( + os.path.join(base_path, "DerivedCoreProperties.txt"), encoding="utf-8" + ) as derived_core_properties, + open( + os.path.join(base_path, "SpecialCasing.txt"), encoding="utf-8" + ) as special_casing, + ): unicode_version = version_from_file( derived_core_properties, "DerivedCoreProperties" ) diff --git a/js/src/vm/jsopcode.py b/js/src/vm/jsopcode.py @@ -149,12 +149,12 @@ def find_by_name(list, name): def add_to_index(index, opcode): types = find_by_name(index, opcode.category_name) if types is None: - raise Exception("Category is not listed in index: " f"{opcode.category_name}") + raise Exception(f"Category is not listed in index: {opcode.category_name}") opcodes = find_by_name(types, opcode.type_name) if opcodes is None: if opcode.type_name: raise Exception( - f"Type is not listed in {opcode.category_name}: " f"{opcode.type_name}" + f"Type is not listed in {opcode.category_name}: {opcode.type_name}" ) types.append((opcode.type_name, [opcode])) return @@ -302,7 +302,7 @@ def get_opcodes(dir): opcode.sort_key = opcode.op if opcode.category_name == "": - raise Exception("Category is not specified for " f"{opcode.op}") + raise Exception(f"Category is not specified for {opcode.op}") add_to_index(index, opcode) else: if group_head.length != opcode.length: diff --git a/js/src/vm/make_opcode_doc.py b/js/src/vm/make_opcode_doc.py @@ -4,14 +4,14 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. -""" Usage: python make_opcode_doc.py +"""Usage: python make_opcode_doc.py - This script generates SpiderMonkey bytecode documentation - from js/src/vm/Opcodes.h. +This script generates SpiderMonkey bytecode documentation +from js/src/vm/Opcodes.h. - Output is written to stdout and should be pasted into the following - MDN page: - https://developer.mozilla.org/en-US/docs/SpiderMonkey/Internals/Bytecode +Output is written to stdout and should be pasted into the following +MDN page: +https://developer.mozilla.org/en-US/docs/SpiderMonkey/Internals/Bytecode """ import os diff --git a/js/src/wasm/GenerateBuiltinModules.py b/js/src/wasm/GenerateBuiltinModules.py @@ -122,7 +122,7 @@ def main(c_out, yaml_path): if "inline_op" in op: inlineOp = f"BuiltinInlineOp::{op['inline_op']}" contents += ( - f" M({op['op']}, \"{op['export']}\", " + f' M({op["op"]}, "{op["export"]}", ' f"{sa['name']}, {sa['type']}, {cppBool(sa['needs_thunk'])}, {op['entry']}, {cppBool(op['uses_memory'])}, {inlineOp}, {i})\\\n" ) contents += "\n" diff --git a/layout/tools/reftest/reftest/__init__.py b/layout/tools/reftest/reftest/__init__.py @@ -125,9 +125,9 @@ class ReftestManifest: for annotation in annotations: key, condition = self.translate_condition_for_mozinfo(annotation) - test_dict[key] = "\n".join( - [t for t in [test_dict.get(key, ""), condition] if t] - ) + test_dict[key] = "\n".join([ + t for t in [test_dict.get(key, ""), condition] if t + ]) self.tests.append(test_dict) diff --git a/layout/tools/reftest/reftestcommandline.py b/layout/tools/reftest/reftestcommandline.py @@ -152,8 +152,7 @@ class ReftestArgumentsParser(argparse.ArgumentParser): dest="ignoreWindowSize", action="store_true", default=False, - help="ignore the window size, which may cause spurious " - "failures and passes", + help="ignore the window size, which may cause spurious failures and passes", ) self.add_argument( @@ -188,7 +187,7 @@ class ReftestArgumentsParser(argparse.ArgumentParser): default=[], dest="environment", metavar="NAME=VALUE", - help="sets the given variable in the application's " "environment", + help="sets the given variable in the application's environment", ) self.add_argument( @@ -368,13 +367,11 @@ class ReftestArgumentsParser(argparse.ArgumentParser): return moznetwork.get_ip() def set_default_suite(self, options): - manifests = OrderedDict( - [ - ("reftest.list", "reftest"), - ("crashtests.list", "crashtest"), - ("jstests.list", "jstestbrowser"), - ] - ) + manifests = OrderedDict([ + ("reftest.list", "reftest"), + ("crashtests.list", "crashtest"), + ("jstests.list", "jstestbrowser"), + ]) for test_path in options.tests: file_name = os.path.basename(test_path) diff --git a/layout/tools/reftest/remotereftest.py b/layout/tools/reftest/remotereftest.py @@ -329,7 +329,7 @@ class RemoteReftest(RefTest): options, server=options.remoteWebServer, port=options.httpPort, - **kwargs + **kwargs, ) profileDir = profile.profile prefs = {} @@ -407,7 +407,7 @@ class RemoteReftest(RefTest): valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, - **profileArgs + **profileArgs, ): if cmdargs is None: cmdargs = [] diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py @@ -5,6 +5,7 @@ """ Runs the reftest test harness. """ + import json import os import platform @@ -763,12 +764,10 @@ class RefTest: print("REFTEST INFO | Result summary:") for summaryObj, (text, categories) in zip(summaryObjects, summaryLines): - details = ", ".join( - [ - "%d %s" % (summaryObj[attribute], description) - for (attribute, description) in categories - ] - ) + details = ", ".join([ + "%d %s" % (summaryObj[attribute], description) + for (attribute, description) in categories + ]) print( "REFTEST INFO | " + text diff --git a/layout/tools/reftest/selftest/conftest.py b/layout/tools/reftest/selftest/conftest.py @@ -51,40 +51,32 @@ def get_reftest(setup_test_harness, binary, parser): build = parser.build_obj options = vars(parser.parse_args([])) - options.update( - { - "app": binary, - "focusFilterMode": "non-needs-focus", - "suite": "reftest", - } - ) + options.update({ + "app": binary, + "focusFilterMode": "non-needs-focus", + "suite": "reftest", + }) if not os.path.isdir(build.bindir): package_root = os.path.dirname(harness_root) - options.update( - { - "extraProfileFiles": [os.path.join(package_root, "bin", "plugins")], - "reftestExtensionPath": os.path.join(harness_root, "reftest"), - "sandboxReadWhitelist": [here, os.environ["PYTHON_TEST_TMP"]], - "utilityPath": os.path.join(package_root, "bin"), - "specialPowersExtensionPath": os.path.join( - harness_root, "specialpowers" - ), - } - ) + options.update({ + "extraProfileFiles": [os.path.join(package_root, "bin", "plugins")], + "reftestExtensionPath": os.path.join(harness_root, "reftest"), + "sandboxReadWhitelist": [here, os.environ["PYTHON_TEST_TMP"]], + "utilityPath": os.path.join(package_root, "bin"), + "specialPowersExtensionPath": os.path.join(harness_root, "specialpowers"), + }) if "MOZ_FETCHES_DIR" in os.environ: options["sandboxReadWhitelist"].append(os.environ["MOZ_FETCHES_DIR"]) else: - options.update( - { - "extraProfileFiles": [os.path.join(build.topobjdir, "dist", "plugins")], - "sandboxReadWhitelist": [build.topobjdir, build.topsrcdir], - "specialPowersExtensionPath": os.path.join( - build.distdir, "xpi-stage", "specialpowers" - ), - } - ) + options.update({ + "extraProfileFiles": [os.path.join(build.topobjdir, "dist", "plugins")], + "sandboxReadWhitelist": [build.topobjdir, build.topsrcdir], + "specialPowersExtensionPath": os.path.join( + build.distdir, "xpi-stage", "specialpowers" + ), + }) def inner(**opts): options.update(opts) diff --git a/memory/replace/dmd/block_analyzer.py b/memory/replace/dmd/block_analyzer.py @@ -117,8 +117,7 @@ parser.add_argument( "-c", "--chain-reports", action="store_true", - help="if only one block is found to hold onto the object, report " - "the next one, too", + help="if only one block is found to hold onto the object, report the next one, too", ) diff --git a/mobile/android/android-components/automation/publish_to_maven_local_if_modified.py b/mobile/android/android-components/automation/publish_to_maven_local_if_modified.py @@ -124,9 +124,11 @@ else: shell=True, ) else: - run_cmd_checked( - ["./gradlew", "publishToMavenLocal", f"-Plocal={time.time_ns()}"] - ) + run_cmd_checked([ + "./gradlew", + "publishToMavenLocal", + f"-Plocal={time.time_ns()}", + ]) with open(LAST_CONTENTS_HASH_FILE, "w") as f: f.write(contents_hash) f.write("\n") diff --git a/mobile/android/android-components/plugins/dependencies/src/main/java/appservices_version_bump.py b/mobile/android/android-components/plugins/dependencies/src/main/java/appservices_version_bump.py @@ -167,8 +167,7 @@ def update_application_services(revision): json = get_as_nightly_json(f"revision.{revision}") target_as_version = json["version"] log.info( - f"Target app-services {as_channel.capitalize()} version " - f"is {target_as_version}" + f"Target app-services {as_channel.capitalize()} version is {target_as_version}" ) if compare_as_versions(current_as_version, target_as_version) >= 0: diff --git a/mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/experimentintegration/generate_smoke_tests.py b/mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/experimentintegration/generate_smoke_tests.py @@ -42,7 +42,7 @@ def search_for_smoke_tests(tests_name): locations.append(count) for location in locations: - test_names.append(f"{class_name}#{code[location+3].strip('()')}") + test_names.append(f"{class_name}#{code[location + 3].strip('()')}") return test_names diff --git a/mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/syncintegration/conftest.py b/mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/syncintegration/conftest.py @@ -156,11 +156,11 @@ def gradlewbuild(fxa_account, monkeypatch, gradlewbuild_log): def pytest_addoption(parser): parser.addoption( "--firefox", - help="path to firefox binary (defaults to " "downloading latest nightly build)", + help="path to firefox binary (defaults to downloading latest nightly build)", ) parser.addoption( "--tps", - help="path to tps add-on (defaults to " "downloading latest nightly build)", + help="path to tps add-on (defaults to downloading latest nightly build)", ) diff --git a/mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/syncintegration/tps.py b/mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/syncintegration/tps.py @@ -21,13 +21,11 @@ class TPS: self.firefox_log.write(line + "\n") def run(self, test, phase="phase1", ignore_unused_engines=True): - self.profile.set_preferences( - { - "testing.tps.testFile": os.path.abspath(test), - "testing.tps.testPhase": phase, - "testing.tps.ignoreUnusedEngines": ignore_unused_engines, - } - ) + self.profile.set_preferences({ + "testing.tps.testFile": os.path.abspath(test), + "testing.tps.testPhase": phase, + "testing.tps.ignoreUnusedEngines": ignore_unused_engines, + }) args = ["-marionette"] process_args = {"processOutputLine": [self._log]} self.logger.info("Running: {} {}".format(self.firefox, " ".join(args))) diff --git a/mobile/android/fenix/tools/data_renewal_request.py b/mobile/android/fenix/tools/data_renewal_request.py @@ -29,19 +29,19 @@ for row in csv_reader: if row["keep(Y/N)"] == "n": continue total_count += 1 - output_string += f'` {row["name"]}`\n' + output_string += f"` {row['name']}`\n" output_string += "1) Provide a link to the initial Data Collection Review Request for this collection.\n" - output_string += f' - {eval(row["data_reviews"])[0]}\n' + output_string += f" - {eval(row['data_reviews'])[0]}\n" output_string += "\n" output_string += "2) When will this collection now expire?\n" if len(row["new expiry version"]) == 0: output_string += f" - {updated_version}\n" else: - output_string += f' - {row["new expiry version"]}\n' + output_string += f" - {row['new expiry version']}\n" output_string += "\n" output_string += "3) Why was the initial period of collection insufficient?\n" - output_string += f' - {row["reason to extend"]}\n' + output_string += f" - {row['reason to extend']}\n" output_string += "\n" output_string += "———\n" diff --git a/mobile/android/fenix/tools/update-glean-tags.py b/mobile/android/fenix/tools/update-glean-tags.py @@ -5,6 +5,7 @@ Scrapes GitHub labels for Fenix and generates a set of glean tags for use in met See https://mozilla.github.io/glean/book/reference/yaml/tags.html """ + import urllib from pathlib import Path diff --git a/mobile/android/focus-android/tools/data_renewal_request.py b/mobile/android/focus-android/tools/data_renewal_request.py @@ -29,19 +29,19 @@ for row in csv_reader: if row["keep(Y/N)"] == "n": continue total_count += 1 - output_string += f'` {row["name"]}`\n' + output_string += f"` {row['name']}`\n" output_string += "1) Provide a link to the initial Data Collection Review Request for this collection.\n" - output_string += f' - {eval(row["data_reviews"])[0]}\n' + output_string += f" - {eval(row['data_reviews'])[0]}\n" output_string += "\n" output_string += "2) When will this collection now expire?\n" if len(row["new expiry version"]) == 0: output_string += f" - {updated_version}\n" else: - output_string += f' - {row["new expiry version"]}\n' + output_string += f" - {row['new expiry version']}\n" output_string += "\n" output_string += "3) Why was the initial period of collection insufficient?\n" - output_string += f' - {row["reason to extend"]}\n' + output_string += f" - {row['reason to extend']}\n" output_string += "\n" output_string += "———\n" diff --git a/mobile/android/focus-android/tools/update-glean-tags.py b/mobile/android/focus-android/tools/update-glean-tags.py @@ -5,6 +5,7 @@ Scrapes GitHub labels for Focus and generates a set of glean tags for use in met See https://mozilla.github.io/glean/book/reference/yaml/tags.html """ + import urllib from pathlib import Path diff --git a/mobile/android/gradle/plugins/apilint/apidoc-plugin/src/test/resources/apidoc_test.py b/mobile/android/gradle/plugins/apilint/apidoc-plugin/src/test/resources/apidoc_test.py @@ -20,47 +20,41 @@ args = parser.parse_args() output = args.out_dir + "/api.txt" -sp.check_call( - [ - args.javadoc, - "-doclet", - "org.mozilla.doclet.ApiDoclet", - "-docletpath", - args.doclet_jar, - "-subpackages", - "org.mozilla.test", - "-sourcepath", - args.java_root, - "-root-dir", - args.java_root, - "-skip-class-regex", - "TestSkippedClass$:^org.mozilla.test.TestClass.TestSkippedClass2$", - "-output", - output, - ] -) - -result = sp.call( - [ - "python3", - "../apilint/src/main/resources/diff.py", - "--existing", - args.expected, - "--local", - output, - ] -) - -result_map = sp.call( - [ - "python3", - "../apilint/src/main/resources/diff.py", - "--existing", - args.expected_map, - "--local", - output + ".map", - ] -) +sp.check_call([ + args.javadoc, + "-doclet", + "org.mozilla.doclet.ApiDoclet", + "-docletpath", + args.doclet_jar, + "-subpackages", + "org.mozilla.test", + "-sourcepath", + args.java_root, + "-root-dir", + args.java_root, + "-skip-class-regex", + "TestSkippedClass$:^org.mozilla.test.TestClass.TestSkippedClass2$", + "-output", + output, +]) + +result = sp.call([ + "python3", + "../apilint/src/main/resources/diff.py", + "--existing", + args.expected, + "--local", + output, +]) + +result_map = sp.call([ + "python3", + "../apilint/src/main/resources/diff.py", + "--existing", + args.expected_map, + "--local", + output + ".map", +]) # result == 0 from `diff` means that the files are identical if result != 0 or result_map != 0: diff --git a/mobile/android/gradle/plugins/apilint/apilint/src/main/resources/apilint.py b/mobile/android/gradle/plugins/apilint/apilint/src/main/resources/apilint.py @@ -449,9 +449,12 @@ class Class: self.name = self.fullname[self.fullname.rindex(".") + 1 :] def __hash__(self): - return hash( - (self.raw, tuple(self.ctors), tuple(self.fields), tuple(self.methods)) - ) + return hash(( + self.raw, + tuple(self.ctors), + tuple(self.fields), + tuple(self.methods), + )) def __repr__(self): return self.raw diff --git a/mobile/android/gradle/plugins/apilint/apilint/src/main/resources/changelog-check.py b/mobile/android/gradle/plugins/apilint/apilint/src/main/resources/changelog-check.py @@ -73,16 +73,14 @@ if __name__ == "__main__": if args.result_json is None: return if info is not None: - results["failures"].append( - { - "column": info["column"], - "file": args.changelog_file.name, - "line": info["line"], - "msg": info["message"], - "rule": info["rule"], - "error": True, - } - ) + results["failures"].append({ + "column": info["column"], + "file": args.changelog_file.name, + "line": info["line"], + "msg": info["message"], + "rule": info["rule"], + "error": True, + }) args.result_json.seek(0) args.result_json.truncate(0) json.dump(results, args.result_json) @@ -90,16 +88,14 @@ if __name__ == "__main__": try: (lineNumber, expectedApiVersion) = findApiVersion(args.changelog_file) except MissingApiVersionError: - dumpJsonError( - { - "column": 0, - "line": 1, - "message": "The api changelog file does not have a version pin. " - "Please update the file and add the following line: " - f"[api-version]: {currentApiVersion}", - "rule": "missing_api_version", - } - ) + dumpJsonError({ + "column": 0, + "line": 1, + "message": "The api changelog file does not have a version pin. " + "Please update the file and add the following line: " + f"[api-version]: {currentApiVersion}", + "rule": "missing_api_version", + }) print( "ERROR: The api changelog file does not have a version pin. Please update" ) @@ -115,16 +111,14 @@ if __name__ == "__main__": sys.exit(11) if currentApiVersion != expectedApiVersion: - dumpJsonError( - { - "column": 14, - "line": lineNumber, - "message": "The api changelog file is out of date. Please update the " - "file and modify the [api-version] line as follows: " - f"[api-version]: {currentApiVersion}", - "rule": "wrong_api_version", - } - ) + dumpJsonError({ + "column": 14, + "line": lineNumber, + "message": "The api changelog file is out of date. Please update the " + "file and modify the [api-version] line as follows: " + f"[api-version]: {currentApiVersion}", + "rule": "wrong_api_version", + }) print("ERROR: The api changelog file is out of date. Please update the file at") print("") print(args.changelog_file.name) diff --git a/mobile/android/gradle/plugins/apilint/apilint/src/test/resources/apilint_test.py b/mobile/android/gradle/plugins/apilint/apilint/src/test/resources/apilint_test.py @@ -37,16 +37,14 @@ for t in tests: after_api = test_base + ".txt" if check_compat: - sp.call( - [ - "python3", - "src/main/resources/diff.py", - "--existing", - before_api, - "--local", - after_api, - ] - ) + sp.call([ + "python3", + "src/main/resources/diff.py", + "--existing", + before_api, + "--local", + after_api, + ]) json_file = "{}/{}-result.json".format(args.build_dir, t["test"]) test = [ diff --git a/mobile/android/mach_commands.py b/mobile/android/mach_commands.py @@ -569,15 +569,13 @@ def gradle(command_context, args, verbose=False, gradle_path=None, topsrcdir=Non env = os.environ.copy() - env.update( - { - "GRADLE_OPTS": "-Dfile.encoding=utf-8", - "JAVA_HOME": java_home, - "JAVA_TOOL_OPTIONS": "-Dfile.encoding=utf-8", - # Let Gradle get the right Python path on Windows - "GRADLE_MACH_PYTHON": sys.executable, - } - ) + env.update({ + "GRADLE_OPTS": "-Dfile.encoding=utf-8", + "JAVA_HOME": java_home, + "JAVA_TOOL_OPTIONS": "-Dfile.encoding=utf-8", + # Let Gradle get the right Python path on Windows + "GRADLE_MACH_PYTHON": sys.executable, + }) # Set ANDROID_SDK_ROOT if --with-android-sdk was set. # See https://bugzilla.mozilla.org/show_bug.cgi?id=1576471 android_sdk_root = command_context.substs.get("ANDROID_SDK_ROOT", "") diff --git a/modules/libpref/init/generate_static_pref_list.py b/modules/libpref/init/generate_static_pref_list.py @@ -34,23 +34,21 @@ VALID_BOOL_TYPES = { } VALID_TYPES = VALID_BOOL_TYPES.copy() -VALID_TYPES.update( - { - "int32_t": "int32_t", - "uint32_t": "uint32_t", - "float": "float", - # These ones are defined in StaticPrefsBase.h. - "RelaxedAtomicInt32": "int32_t", - "RelaxedAtomicUint32": "uint32_t", - "ReleaseAcquireAtomicInt32": "int32_t", - "ReleaseAcquireAtomicUint32": "uint32_t", - "SequentiallyConsistentAtomicInt32": "int32_t", - "SequentiallyConsistentAtomicUint32": "uint32_t", - "AtomicFloat": "float", - "String": None, - "DataMutexString": "nsACString", - } -) +VALID_TYPES.update({ + "int32_t": "int32_t", + "uint32_t": "uint32_t", + "float": "float", + # These ones are defined in StaticPrefsBase.h. + "RelaxedAtomicInt32": "int32_t", + "RelaxedAtomicUint32": "uint32_t", + "ReleaseAcquireAtomicInt32": "int32_t", + "ReleaseAcquireAtomicUint32": "uint32_t", + "SequentiallyConsistentAtomicInt32": "int32_t", + "SequentiallyConsistentAtomicUint32": "uint32_t", + "AtomicFloat": "float", + "String": None, + "DataMutexString": "nsACString", +}) # Map non-atomic C++ types to equivalent Rust types. RUST_TYPES = { diff --git a/modules/libpref/test/test_generate_static_pref_list.py b/modules/libpref/test/test_generate_static_pref_list.py @@ -108,18 +108,14 @@ good_input = """ # The corresponding code for good_input. good = {} -good[ - "static_pref_list_all_h" -] = """\ +good["static_pref_list_all_h"] = """\ // This file was generated by generate_static_pref_list.py from (string input). DO NOT EDIT. #include "mozilla/StaticPrefList_my.h" #include "mozilla/StaticPrefList_my_dashed.h" """ -good[ - "static_prefs_all_h" -] = """\ +good["static_prefs_all_h"] = """\ // This file was generated by generate_static_pref_list.py from (string input). DO NOT EDIT. #include "mozilla/StaticPrefs_my.h" @@ -217,9 +213,7 @@ good["static_prefs_group_h"] = { """ } -good[ - "static_prefs_c_getters_cpp" -] = """\ +good["static_prefs_c_getters_cpp"] = """\ // This file was generated by generate_static_pref_list.py from (string input). DO NOT EDIT. extern "C" uint32_t StaticPrefs_my_uint() { @@ -240,9 +234,7 @@ extern "C" void StaticPrefs_my_datamutex_string_rust(nsACString *result) { } """ -good[ - "static_prefs_rs" -] = """\ +good["static_prefs_rs"] = """\ // This file was generated by generate_static_pref_list.py from (string input). DO NOT EDIT. pub use nsstring::nsCString; diff --git a/moz.configure b/moz.configure @@ -131,8 +131,7 @@ set_define("FMT_USE_LOCALE", 0) option( "--enable-debug", nargs="?", - help="Enable building with developer debug info " - "(using the given compiler flags)", + help="Enable building with developer debug info (using the given compiler flags)", ) diff --git a/mozglue/baseprofiler/build/generate_profiling_categories.py b/mozglue/baseprofiler/build/generate_profiling_categories.py @@ -145,9 +145,9 @@ def generate_macro_header(c_out, yaml_path): color = category["color"] assert isinstance(color, str) subcategories = category.get("subcategories", None) - assert ( - isinstance(subcategories, list) and len(subcategories) > 0 - ), f"At least one subcategory expected as default in {name}." + assert isinstance(subcategories, list) and len(subcategories) > 0, ( + f"At least one subcategory expected as default in {name}." + ) category_items.append( generate_category_macro(name, label, color, subcategories) @@ -270,9 +270,9 @@ def generate_rust_enums(c_out, yaml_path): # This will be used as our main enum field and sub category enum. cat_label = "".join(filter(str.isalnum, cat_label)) cat_subcategories = category.get("subcategories", None) - assert ( - isinstance(cat_subcategories, list) and len(cat_subcategories) > 0 - ), f"At least one subcategory expected as default in {cat_name}." + assert isinstance(cat_subcategories, list) and len(cat_subcategories) > 0, ( + f"At least one subcategory expected as default in {cat_name}." + ) # Create a new enum for this sub category and append it to the enums list. category_enum = RustEnum(cat_label) @@ -300,9 +300,9 @@ def generate_rust_enums(c_out, yaml_path): ) profiling_category_pair_value += 1 - assert ( - category_enum.default_category is not None - ), "There must be a default subcategory with the same name." + assert category_enum.default_category is not None, ( + "There must be a default subcategory with the same name." + ) # Append the main enums. profiling_category_pair_enum.append_optional_tuple_field(cat_label) diff --git a/netwerk/test/marionette/test_purge_http_cache_at_shutdown.py b/netwerk/test/marionette/test_purge_http_cache_at_shutdown.py @@ -12,13 +12,11 @@ from marionette_harness import MarionetteTestCase class PurgeHTTPCacheAtShutdownTestCase(MarionetteTestCase): def setUp(self): super().setUp() - self.marionette.enforce_gecko_prefs( - { - "privacy.sanitize.sanitizeOnShutdown": True, - "privacy.clearOnShutdown.cache": True, - "network.cache.shutdown_purge_in_background_task": True, - } - ) + self.marionette.enforce_gecko_prefs({ + "privacy.sanitize.sanitizeOnShutdown": True, + "privacy.clearOnShutdown.cache": True, + "network.cache.shutdown_purge_in_background_task": True, + }) self.profile_path = Path(self.marionette.profile_path) self.cache_path = self.profile_path.joinpath("cache2") diff --git a/netwerk/test/perf/hooks_throttling.py b/netwerk/test/perf/hooks_throttling.py @@ -5,6 +5,7 @@ Drives the throttling feature when the test calls our controlled server. """ + import http.client import json import os diff --git a/python/gdb-symbol-server/symbols.py b/python/gdb-symbol-server/symbols.py @@ -63,9 +63,10 @@ def try_fetch_symbols(filename, build_id, destination): u = urlopen(url) if u.getcode() != 200: return None - with open(debug_file, "wb") as f, gzip.GzipFile( - fileobj=io.BytesIO(u.read()), mode="r" - ) as z: + with ( + open(debug_file, "wb") as f, + gzip.GzipFile(fileobj=io.BytesIO(u.read()), mode="r") as z, + ): shutil.copyfileobj(z, f) print(f"Fetched symbols from {url}") return debug_file diff --git a/python/mach/mach/commands/commandinfo.py b/python/mach/mach/commands/commandinfo.py @@ -203,9 +203,9 @@ def run_completion(command_context, args): def _zsh_describe(value, description=None): value = '"' + value.replace(":", "\\:") if description: - description = subprocess.list2cmdline( - [re.sub(r'(["\'#&;`|*?~<>^()\[\]{}$\\\x0A\xFF])', r"\\\1", description)] - ).lstrip('"') + description = subprocess.list2cmdline([ + re.sub(r'(["\'#&;`|*?~<>^()\[\]{}$\\\x0A\xFF])', r"\\\1", description) + ]).lstrip('"') if description.endswith('"') and not description.endswith(r"\""): description = description[:-1] @@ -247,14 +247,12 @@ def completion_bash(command_context, outfile): if options: case_options.append( - "\n".join( - [ - f" ({cmd.name})", - ' opts="${{opts}} {}"'.format(" ".join(options)), - " ;;", - "", - ] - ) + "\n".join([ + f" ({cmd.name})", + ' opts="${{opts}} {}"'.format(" ".join(options)), + " ;;", + "", + ]) ) # Build case statement for subcommand options. @@ -266,14 +264,12 @@ def completion_bash(command_context, outfile): if options: case_options.append( - "\n".join( - [ - f' ("{sub.name} {sub.subcommand}")', - ' opts="${{opts}} {}"'.format(" ".join(options)), - " ;;", - "", - ] - ) + "\n".join([ + f' ("{sub.name} {sub.subcommand}")', + ' opts="${{opts}} {}"'.format(" ".join(options)), + " ;;", + "", + ]) ) # Build case statement for subcommands. @@ -281,25 +277,21 @@ def completion_bash(command_context, outfile): if subcommands: comsubs = " ".join([h.subcommand for h in cmd.subcommands]) case_commands_subcommands.append( - "\n".join( - [ - f" ({cmd.name})", - f' comsubs=" {comsubs} "', - " ;;", - "", - ] - ) + "\n".join([ + f" ({cmd.name})", + f' comsubs=" {comsubs} "', + " ;;", + "", + ]) ) case_subcommands.append( - "\n".join( - [ - f" ({cmd.name})", - ' subs="${{subs}} {}"'.format(" ".join(subcommands)), - " ;;", - "", - ] - ) + "\n".join([ + f" ({cmd.name})", + ' subs="${{subs}} {}"'.format(" ".join(subcommands)), + " ;;", + "", + ]) ) globalopts = [ @@ -345,14 +337,12 @@ def completion_zsh(command_context, outfile): if options: case_options.append( - "\n".join( - [ - f" ({cmd.name})", - " opts+=({})".format(" ".join(options)), - " ;;", - "", - ] - ) + "\n".join([ + f" ({cmd.name})", + " opts+=({})".format(" ".join(options)), + " ;;", + "", + ]) ) # Build case statement for subcommand options. @@ -364,14 +354,12 @@ def completion_zsh(command_context, outfile): if options: case_options.append( - "\n".join( - [ - f" ({sub.name} {sub.subcommand})", - " opts+=({})".format(" ".join(options)), - " ;;", - "", - ] - ) + "\n".join([ + f" ({sub.name} {sub.subcommand})", + " opts+=({})".format(" ".join(options)), + " ;;", + "", + ]) ) # Build case statement for subcommands. @@ -386,14 +374,12 @@ def completion_zsh(command_context, outfile): ) case_subcommands.append( - "\n".join( - [ - f" ({cmd.name})", - " subs+=({})".format(" ".join(subcommands)), - " ;;", - "", - ] - ) + "\n".join([ + f" ({cmd.name})", + " subs+=({})".format(" ".join(subcommands)), + " ;;", + "", + ]) ) globalopts = [] @@ -436,9 +422,8 @@ def completion_fish(command_context, outfile): globalopts = [] for opt_strs, description in global_options(command_context).items(): - comp = ( - "complete -c mach -n '__fish_mach_complete_no_command' " - "-d '{}'".format(description.replace("'", "\\'")) + comp = "complete -c mach -n '__fish_mach_complete_no_command' -d '{}'".format( + description.replace("'", "\\'") ) comp = _append_opt_strs(comp, opt_strs) globalopts.append(comp) diff --git a/python/mach/mach/decorators.py b/python/mach/mach/decorators.py @@ -76,7 +76,7 @@ class _MachCommand: self.hidden = hidden if ok_if_tests_disabled and category != "testing": raise ValueError( - "ok_if_tests_disabled should only be set for " "`testing` mach commands" + "ok_if_tests_disabled should only be set for `testing` mach commands" ) self.ok_if_tests_disabled = ok_if_tests_disabled diff --git a/python/mach/mach/dispatcher.py b/python/mach/mach/dispatcher.py @@ -145,9 +145,10 @@ class CommandAction(argparse.Action): # If there are sub-commands, parse the intent out immediately. if handler.subcommand_handlers and args: # mach <command> help <subcommand> - if set(args[: args.index("--")] if "--" in args else args).intersection( - ("help", "--help") - ): + if set(args[: args.index("--")] if "--" in args else args).intersection(( + "help", + "--help", + )): self._handle_subcommand_help(parser, handler, args) sys.exit(0) # mach <command> <subcommand> ... diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py @@ -528,7 +528,7 @@ def get_argument_parser(context=None, action=CommandAction, topsrcdir=None): parser = ArgumentParser( add_help=False, - usage="%(prog)s [global arguments] " "command [command arguments]", + usage="%(prog)s [global arguments] command [command arguments]", ) # WARNING!!! If you add a global argument here, also add it to the diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py @@ -33,7 +33,7 @@ class MachRegistrar: if not handler.category: raise MachError( - "Cannot register a mach command without a " "category: %s" % name + "Cannot register a mach command without a category: %s" % name ) if handler.category not in self.categories: diff --git a/python/mach/mach/sentry.py b/python/mach/mach/sentry.py @@ -214,9 +214,9 @@ def _is_unmodified_mach_core(topsrcdir: Path): repo = _get_repository_object(topsrcdir) try: files = set(repo.get_outgoing_files()) | set(repo.get_changed_files()) - _is_unmodified_mach_core_result = not any( - [file for file in files if file == "mach" or file.endswith(".py")] - ) + _is_unmodified_mach_core_result = not any([ + file for file in files if file == "mach" or file.endswith(".py") + ]) except MissingUpstreamRepo: # If we don't know the upstream state, we don't know if the mach files # have been unmodified. diff --git a/python/mach/mach/site.py b/python/mach/mach/site.py @@ -342,9 +342,9 @@ class MachSiteManager: # yet, and the system isn't guaranteed to have the packages we need. For example, # "./mach bootstrap" can't have any dependencies. # So, all external dependencies of Mach's must be optional. - assert ( - not requirements.pypi_requirements - ), "Mach pip package requirements must be optional." + assert not requirements.pypi_requirements, ( + "Mach pip package requirements must be optional." + ) # external_python is the Python interpreter that invoked Mach for this process. external_python = ExternalPythonSite(sys.executable) @@ -607,9 +607,9 @@ class CommandSiteManager: should be created """ active_metadata = MozSiteMetadata.from_runtime() - assert ( - active_metadata - ), "A Mach-managed site must be active before doing work with command sites" + assert active_metadata, ( + "A Mach-managed site must be active before doing work with command sites" + ) mach_site_packages_source = active_metadata.mach_site_packages_source pip_restricted_site = site_name in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS @@ -1044,12 +1044,10 @@ class PythonVirtualenv: constraints_path = os.path.join(tempdir, "site-constraints.txt") with open(constraints_path, "w") as file: file.write( - "\n".join( - [ - f"{name}=={version}" - for name, version in existing_packages.items() - ] - ) + "\n".join([ + f"{name}=={version}" + for name, version in existing_packages.items() + ]) ) self.pip_install(["--constraint", constraints_path] + pip_args) @@ -1536,9 +1534,11 @@ def _create_venv_with_pthfile( if populate_with_pip: for requirements_txt_file in requirements.requirements_txt_files: - target_venv.pip_install( - ["--requirement", requirements_txt_file.path, "--require-hashes"] - ) + target_venv.pip_install([ + "--requirement", + requirements_txt_file.path, + "--require-hashes", + ]) if requirements.pypi_requirements: requirements_list = [ str(req.requirement) for req in requirements.pypi_requirements diff --git a/python/mach/mach/test/script_site_activation.py b/python/mach/mach/test/script_site_activation.py @@ -54,13 +54,11 @@ def main(): ) command_site.activate() command_sys_path = sys.path.copy() - print( - [ - initial_sys_path, - mach_sys_path, - command_sys_path, - ] - ) + print([ + initial_sys_path, + mach_sys_path, + command_sys_path, + ]) if __name__ == "__main__": diff --git a/python/mach/mach/test/test_site.py b/python/mach/mach/test/test_site.py @@ -142,9 +142,9 @@ def test_requirements_txt_install_requires_hashes( pytest.fail("Expected CalledProcessError to be raised due to missing hashes") except subprocess.CalledProcessError as e: error_output = e.stderr if e.stderr else "" - assert ( - "hash" in error_output.lower() - ), f"Expected hash error in stderr, got: {error_output}" + assert "hash" in error_output.lower(), ( + f"Expected hash error in stderr, got: {error_output}" + ) def test_requirements_txt_installs_with_hashes(run_create_venv_with_pthfile): diff --git a/python/mach/mach/test/test_site_activation.py b/python/mach/mach/test/test_site_activation.py @@ -63,14 +63,12 @@ def test_new_package_metadta_is_found(): pass with tempfile.TemporaryDirectory() as venv_dir: - subprocess.check_call( - [ - sys.executable, - "-m", - "venv", - venv_dir, - ] - ) + subprocess.check_call([ + sys.executable, + "-m", + "venv", + venv_dir, + ]) venv = PythonVirtualenv(venv_dir) venv.pip_install([f"{pkg}=={version}"]) diff --git a/python/mach/mach/util.py b/python/mach/mach/util.py @@ -121,4 +121,4 @@ def strtobool(value: str): if value in false_vals: return 0 - raise ValueError(f'Expected one of: {", ".join(true_vals + false_vals)}') + raise ValueError(f"Expected one of: {', '.join(true_vals + false_vals)}") diff --git a/python/mozboot/bin/bootstrap.py b/python/mozboot/bin/bootstrap.py @@ -444,7 +444,7 @@ def main(args): "--no-system-changes", dest="no_system_changes", action="store_true", - help="Only executes actions that leave the system " "configuration alone.", + help="Only executes actions that leave the system configuration alone.", ) options, leftover = parser.parse_args(args) diff --git a/python/mozboot/mozboot/base.py b/python/mozboot/mozboot/base.py @@ -667,18 +667,16 @@ class BaseBootstrapper: rustup_init.chmod(mode | stat.S_IRWXU) print("Ok") print("Running rustup-init...") - subprocess.check_call( - [ - str(rustup_init), - "-y", - "--default-toolchain", - "stable", - "--default-host", - platform, - "--component", - "rustfmt", - ] - ) + subprocess.check_call([ + str(rustup_init), + "-y", + "--default-toolchain", + "stable", + "--default-host", + platform, + "--component", + "rustfmt", + ]) cargo_home, cargo_bin = self.cargo_home() self.print_rust_path_advice(RUST_INSTALL_COMPLETE, cargo_home, cargo_bin) finally: diff --git a/python/mozboot/mozboot/bootstrap.py b/python/mozboot/mozboot/bootstrap.py @@ -58,15 +58,13 @@ Please choose the version of Firefox you want to build (see note above): %s Your choice: """ -APPLICATIONS = OrderedDict( - [ - ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"), - ("Firefox for Desktop", "browser"), - ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"), - ("GeckoView/Firefox for Android", "mobile_android"), - ("SpiderMonkey JavaScript engine", "js"), - ] -) +APPLICATIONS = OrderedDict([ + ("Firefox for Desktop Artifact Mode", "browser_artifact_mode"), + ("Firefox for Desktop", "browser"), + ("GeckoView/Firefox for Android Artifact Mode", "mobile_android_artifact_mode"), + ("GeckoView/Firefox for Android", "mobile_android"), + ("SpiderMonkey JavaScript engine", "js"), +]) FINISHED = """ Your system should be ready to build %s! @@ -320,7 +318,7 @@ class Bootstrapper: cls = WindowsBootstrapper if cls is None: raise NotImplementedError( - "Bootstrap support is not yet available " "for your OS." + "Bootstrap support is not yet available for your OS." ) self.instance = cls(**args) @@ -718,12 +716,10 @@ def current_firefox_checkout(env, hg: Optional[Path] = None): Returns one of None, ``git``, or ``hg``. """ - HG_ROOT_REVISIONS = set( - [ - # From mozilla-unified. - "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29" - ] - ) + HG_ROOT_REVISIONS = set([ + # From mozilla-unified. + "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29" + ]) path = Path.cwd() while path: diff --git a/python/mozboot/mozboot/linux_common.py b/python/mozboot/mozboot/linux_common.py @@ -56,17 +56,15 @@ class LinuxBootstrapper(MobileAndroidBootstrapper): pass def install_system_packages(self): - self.install_packages( - [ - "bash", - "findutils", # contains xargs - "gzip", - "make", - "perl", - "tar", - "unzip", - ] - ) + self.install_packages([ + "bash", + "findutils", # contains xargs + "gzip", + "make", + "perl", + "tar", + "unzip", + ]) # Optional packages try: self.install_packages(["watchman"]) diff --git a/python/mozboot/mozboot/mozconfig.py b/python/mozboot/mozboot/mozconfig.py @@ -117,7 +117,7 @@ def find_mozconfig(topsrcdir: Union[str, Path], env=os.environ): if not env_path.is_file(): raise MozconfigFindException( - "MOZCONFIG environment variable refers to a " f"non-file: {env_path}" + f"MOZCONFIG environment variable refers to a non-file: {env_path}" ) srcdir_paths = [topsrcdir / p for p in DEFAULT_TOPSRCDIR_PATHS] diff --git a/python/mozboot/mozboot/osx.py b/python/mozboot/mozboot/osx.py @@ -215,18 +215,22 @@ class OSXBootstrapper(OSXAndroidBootstrapper, BaseBootstrapper): # Ensure that we can access old versions of packages. if b"homebrew/cask-versions" not in known_taps: - subprocess.check_output( - [to_optional_str(self.brew), "tap", "homebrew/cask-versions"] - ) + subprocess.check_output([ + to_optional_str(self.brew), + "tap", + "homebrew/cask-versions", + ]) # "caskroom/versions" has been renamed to "homebrew/cask-versions", so # it is safe to remove the old tap. Removing the old tap is necessary # to avoid the error "Cask [name of cask] exists in multiple taps". # See https://bugzilla.mozilla.org/show_bug.cgi?id=1544981 if b"caskroom/versions" in known_taps: - subprocess.check_output( - [to_optional_str(self.brew), "untap", "caskroom/versions"] - ) + subprocess.check_output([ + to_optional_str(self.brew), + "untap", + "caskroom/versions", + ]) self._ensure_homebrew_packages(casks, is_for_cask=True) diff --git a/python/mozbuild/mozbuild/action/check_binary.py b/python/mozbuild/mozbuild/action/check_binary.py @@ -208,43 +208,39 @@ def check_mozglue_order(binary): def check_networking(binary): retcode = 0 - networking_functions = set( - [ - # socketpair is not concerning; it is restricted to AF_UNIX - "recv", - "send", - # We would be concerned by recvmsg and sendmsg; but we believe - # they are okay as documented in 1376621#c23 - "gethostbyname", - "gethostbyaddr", - "gethostent", - "sethostent", - "endhostent", - "gethostent_r", - "gethostbyname2", - "gethostbyaddr_r", - "gethostbyname_r", - "gethostbyname2_r", - "getservent", - "getservbyname", - "getservbyport", - "setservent", - "getprotoent", - "getprotobyname", - "getprotobynumber", - "setprotoent", - "endprotoent", - ] - ) + networking_functions = set([ + # socketpair is not concerning; it is restricted to AF_UNIX + "recv", + "send", + # We would be concerned by recvmsg and sendmsg; but we believe + # they are okay as documented in 1376621#c23 + "gethostbyname", + "gethostbyaddr", + "gethostent", + "sethostent", + "endhostent", + "gethostent_r", + "gethostbyname2", + "gethostbyaddr_r", + "gethostbyname_r", + "gethostbyname2_r", + "getservent", + "getservbyname", + "getservbyport", + "setservent", + "getprotoent", + "getprotobyname", + "getprotobynumber", + "setprotoent", + "endprotoent", + ]) # These are used by the crash monitor & crash monitor client to talk with # the main process on Linux and macOS. - socket_functions = set( - [ - "connect", - "accept", - "listen", - ] - ) + socket_functions = set([ + "connect", + "accept", + "listen", + ]) if PLATFORM == "WINNT": networking_functions |= socket_functions diff --git a/python/mozbuild/mozbuild/action/exe_7z_archive.py b/python/mozbuild/mozbuild/action/exe_7z_archive.py @@ -45,23 +45,21 @@ def archive_exe(pkg_dir, tagfile, sfx_package, package, use_upx): except BuildEnvironmentNotFoundException: # configure hasn't been run, just use the default sevenz = "7z" - subprocess.check_call( - [ - sevenz, - "a", - "-r", - "-t7z", - mozpath.join(tmpdir, "app.7z"), - "-mx", - "-m0=BCJ2", - "-m1=LZMA:d25", - "-m2=LZMA:d19", - "-m3=LZMA:d19", - "-mb0:1", - "-mb0s1:2", - "-mb0s2:3", - ] - ) + subprocess.check_call([ + sevenz, + "a", + "-r", + "-t7z", + mozpath.join(tmpdir, "app.7z"), + "-mx", + "-m0=BCJ2", + "-m1=LZMA:d25", + "-m2=LZMA:d19", + "-m3=LZMA:d19", + "-mb0:1", + "-mb0s1:2", + "-mb0s2:3", + ]) with open(package, "wb") as o: for i in [final_sfx, tagfile, mozpath.join(tmpdir, "app.7z")]: diff --git a/python/mozbuild/mozbuild/action/fat_aar.py b/python/mozbuild/mozbuild/action/fat_aar.py @@ -34,21 +34,19 @@ def _download_zip(distdir, arch): } dest = mozpath.join(distdir, "input", arch) - subprocess.check_call( - [ - sys.executable, - mozpath.join(buildconfig.topsrcdir, "mach"), - "artifact", - "install", - "--job", - jobs[arch], - "--distdir", - dest, - "--no-tests", - "--no-process", - "--maven-zip", - ] - ) + subprocess.check_call([ + sys.executable, + mozpath.join(buildconfig.topsrcdir, "mach"), + "artifact", + "install", + "--job", + jobs[arch], + "--distdir", + dest, + "--no-tests", + "--no-process", + "--maven-zip", + ]) return mozpath.join(dest, "target.maven.zip") diff --git a/python/mozbuild/mozbuild/action/langpack_manifest.py b/python/mozbuild/mozbuild/action/langpack_manifest.py @@ -64,13 +64,13 @@ def get_build_date(): ### def get_dt_from_hg(path): with mozversioncontrol.get_repository_object(path=path) as repo: - phase = repo._run("log", "-r", ".", "-T" "{phase}") + phase = repo._run("log", "-r", ".", "-T{phase}") if phase.strip() != "public": return get_build_date() repo_url = repo._run("paths", "default") repo_url = repo_url.strip().replace("ssh://", "https://") repo_url = repo_url.replace("hg://", "https://") - cs = repo._run("log", "-r", ".", "-T" "{node}") + cs = repo._run("log", "-r", ".", "-T{node}") url = pushlog_api_url.format(repo_url, cs) session = requests.Session() @@ -354,15 +354,13 @@ def parse_chrome_manifest(path, base_path, chrome_entries): entry_path = os.path.join( os.path.relpath(os.path.dirname(path), base_path), entry.relpath ) - chrome_entries.append( - { - "type": "locale", - "alias": entry.name, - "locale": entry.id, - "platforms": convert_entry_flags_to_platform_codes(entry.flags), - "path": mozpath.normsep(entry_path), - } - ) + chrome_entries.append({ + "type": "locale", + "alias": entry.name, + "locale": entry.id, + "platforms": convert_entry_flags_to_platform_codes(entry.flags), + "path": mozpath.normsep(entry_path), + }) else: raise Exception(f"Unknown type {entry.name}") diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py @@ -709,13 +709,11 @@ ARCHIVE_FILES = { } if buildconfig.substs.get("MOZ_CODE_COVERAGE"): - ARCHIVE_FILES["common"].append( - { - "source": buildconfig.topsrcdir, - "base": "python/mozbuild/", - "patterns": ["mozpack/**", "mozbuild/codecoverage/**"], - } - ) + ARCHIVE_FILES["common"].append({ + "source": buildconfig.topsrcdir, + "base": "python/mozbuild/", + "patterns": ["mozpack/**", "mozbuild/codecoverage/**"], + }) if ( diff --git a/python/mozbuild/mozbuild/action/tooltool.py b/python/mozbuild/mozbuild/action/tooltool.py @@ -132,7 +132,7 @@ def retry( action_name = getattr(action, "__name__", action) if log_args and (args or kwargs): log_attempt_args = ( - "retry: calling %s with args: %s," " kwargs: %s, attempt #%d", + "retry: calling %s with args: %s, kwargs: %s, attempt #%d", action_name, args, kwargs, @@ -330,24 +330,22 @@ def normalize_string( port, content_hash, ): - return "\n".join( - [ - normalize_header_attr(header) - # The blank lines are important. They follow what the Node Hawk lib does. - for header in [ - "hawk." + str(HAWK_VER) + "." + mac_type, - timestamp, - nonce, - method or "", - name or "", - host, - port, - content_hash or "", - "", # for ext which is empty in this case - "", # Add trailing new line. - ] + return "\n".join([ + normalize_header_attr(header) + # The blank lines are important. They follow what the Node Hawk lib does. + for header in [ + "hawk." + str(HAWK_VER) + "." + mac_type, + timestamp, + nonce, + method or "", + name or "", + host, + port, + content_hash or "", + "", # for ext which is empty in this case + "", # Add trailing new line. ] - ) + ]) def calculate_mac( @@ -888,9 +886,10 @@ def fetch_file(base_urls, file_record, grabchunk=1024 * 4, auth_file=None, regio # Well, the file doesn't exist locally. Let's fetch it. try: - with request(url, auth_file) as f, builtins.open( - temp_path, mode="wb" - ) as out: + with ( + request(url, auth_file) as f, + builtins.open(temp_path, mode="wb") as out, + ): k = True size = 0 while k: @@ -984,9 +983,11 @@ class TarFile(tarfile.TarFile): shutil.copy(source, targetpath) self.chown(member, targetpath, numeric_owner) else: - deferred_links.setdefault(source, []).append( - (member, targetpath, numeric_owner) - ) + deferred_links.setdefault(source, []).append(( + member, + targetpath, + numeric_owner, + )) return extract(member, path, set_attrs, numeric_owner=numeric_owner, **kwargs) @@ -1637,8 +1638,7 @@ Supported commands are: default=False, dest="unpack", action="store_true", - help="Request unpacking this file after fetch." - " This is helpful with tarballs.", + help="Request unpacking this file after fetch. This is helpful with tarballs.", ) parser.add_option( "--version", @@ -1661,8 +1661,7 @@ Supported commands are: "--url", dest="base_url", action="append", - help="RelengAPI URL ending with /tooltool/; default " - "is appropriate for Mozilla", + help="RelengAPI URL ending with /tooltool/; default is appropriate for Mozilla", ) parser.add_option( "-c", "--cache-folder", dest="cache_folder", help="Local cache folder" @@ -1678,7 +1677,7 @@ Supported commands are: parser.add_option( "-r", "--region", - help="Preferred AWS region for upload or fetch; " "example: --region=us-west-2", + help="Preferred AWS region for upload or fetch; example: --region=us-west-2", ) parser.add_option( "--message", diff --git a/python/mozbuild/mozbuild/action/zip.py b/python/mozbuild/mozbuild/action/zip.py @@ -25,7 +25,7 @@ def main(args): "-C", metavar="DIR", default=".", - help="Change to given directory before considering " "other paths", + help="Change to given directory before considering other paths", ) parser.add_argument("--strip", action="store_true", help="Strip executables") parser.add_argument( diff --git a/python/mozbuild/mozbuild/android_version_code.py b/python/mozbuild/mozbuild/android_version_code.py @@ -27,8 +27,7 @@ def android_version_code_v0(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): return base + min_sdk + 3 else: raise ValueError( - "Don't know how to compute android:versionCode " - "for CPU arch %s" % cpu_arch + "Don't know how to compute android:versionCode for CPU arch %s" % cpu_arch ) @@ -129,8 +128,7 @@ def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): pass else: raise ValueError( - "Don't know how to compute android:versionCode " - "for CPU arch %s" % cpu_arch + "Don't know how to compute android:versionCode for CPU arch %s" % cpu_arch ) # 'p' bit is 1 for 64-bit architectures. @@ -140,8 +138,7 @@ def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): pass else: raise ValueError( - "Don't know how to compute android:versionCode " - "for CPU arch %s" % cpu_arch + "Don't know how to compute android:versionCode for CPU arch %s" % cpu_arch ) # 'g' bit is currently always 1, but may depend on `min_sdk` in the future. diff --git a/python/mozbuild/mozbuild/artifact_cache.py b/python/mozbuild/mozbuild/artifact_cache.py @@ -20,7 +20,6 @@ None of the instances (or the underlying caches) are safe for concurrent use. A future need, perhaps. """ - import binascii import hashlib import logging diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py @@ -31,7 +31,6 @@ environment. ``mach artifact`` ensures these modules are available, but other consumers will need to arrange this themselves. """ - import collections import functools import glob @@ -277,8 +276,7 @@ class ArtifactJob: ) if self._tests_re and not tests_artifact: raise ValueError( - f'Expected tests archive matching "{self._tests_re}", but ' - "found none!" + f'Expected tests archive matching "{self._tests_re}", but found none!' ) if self._maven_zip_re and not maven_zip_artifact: raise ValueError( @@ -746,15 +744,13 @@ class MacArtifactJob(ArtifactJob): @property def _extra_archives(self): extra_archives = super()._extra_archives - extra_archives.update( - { - ".update_framework_artifacts.zip": { - "description": "Update-related macOS Framework Artifacts", - "src_prefix": "", - "dest_prefix": "update_framework_artifacts", - }, - } - ) + extra_archives.update({ + ".update_framework_artifacts.zip": { + "description": "Update-related macOS Framework Artifacts", + "src_prefix": "", + "dest_prefix": "update_framework_artifacts", + }, + }) return extra_archives @property @@ -919,12 +915,10 @@ class UnfilteredProjectPackageArtifactJob(ArtifactJob): """ # Can't yet handle `AndroidArtifactJob` uniformly, since the `product` is "mobile". - package_re = "|".join( - [ - f"({cls.package_re})" - for cls in (LinuxArtifactJob, MacArtifactJob, WinArtifactJob) - ] - ) + package_re = "|".join([ + f"({cls.package_re})" + for cls in (LinuxArtifactJob, MacArtifactJob, WinArtifactJob) + ]) job_configuration = GeckoJobConfiguration @property diff --git a/python/mozbuild/mozbuild/backend/base.py b/python/mozbuild/mozbuild/backend/base.py @@ -305,12 +305,10 @@ class BuildBackend(LoggingMixin): in the current environment.""" pp = Preprocessor() srcdir = mozpath.dirname(obj.input_path) - pp.context.update( - { - k: " ".join(v) if isinstance(v, list) else v - for k, v in obj.config.substs.items() - } - ) + pp.context.update({ + k: " ".join(v) if isinstance(v, list) else v + for k, v in obj.config.substs.items() + }) pp.context.update( top_srcdir=obj.topsrcdir, topobjdir=obj.topobjdir, diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py @@ -162,9 +162,9 @@ class CommonBackend(BuildBackend): return False elif isinstance(obj, SandboxedWasmLibrary): - self._handle_generated_sources( - [mozpath.join(obj.relobjdir, f"{obj.basename}.h")] - ) + self._handle_generated_sources([ + mozpath.join(obj.relobjdir, f"{obj.basename}.h") + ]) return False elif isinstance(obj, (Sources, HostSources)): @@ -203,9 +203,9 @@ class CommonBackend(BuildBackend): for f in files: basename = FinalTargetPreprocessedFiles.get_obj_basename(f) relpath = mozpath.join(obj.install_target, path, basename) - self._handle_generated_sources( - [ObjDirPath(obj._context, "!/" + relpath).full_path] - ) + self._handle_generated_sources([ + ObjDirPath(obj._context, "!/" + relpath).full_path + ]) return False else: diff --git a/python/mozbuild/mozbuild/backend/configenvironment.py b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -150,12 +150,10 @@ class ConfigEnvironment: self.bin_suffix = self.substs.get("BIN_SUFFIX", "") global_defines = [name for name in self.defines] - self.substs["ACDEFINES"] = " ".join( - [ - "-D%s=%s" % (name, shell_quote(self.defines[name]).replace("$", "$$")) - for name in sorted(global_defines) - ] - ) + self.substs["ACDEFINES"] = " ".join([ + "-D%s=%s" % (name, shell_quote(self.defines[name]).replace("$", "$$")) + for name in sorted(global_defines) + ]) def serialize(name, obj): if isinstance(obj, str): @@ -165,13 +163,11 @@ class ConfigEnvironment: raise Exception("Unhandled type %s for %s", type(obj), str(name)) self.substs["ALLSUBSTS"] = "\n".join( - sorted( - [ - "%s = %s" % (name, serialize(name, self.substs[name])) - for name in self.substs - if self.substs[name] - ] - ) + sorted([ + "%s = %s" % (name, serialize(name, self.substs[name])) + for name in self.substs + if self.substs[name] + ]) ) self.substs["ALLEMPTYSUBSTS"] = "\n".join( sorted(["%s =" % name for name in self.substs if not self.substs[name]]) @@ -334,13 +330,10 @@ class PartialConfigEnvironment: defines = config["defines"].copy() global_defines = [name for name in config["defines"]] - acdefines = " ".join( - [ - "-D%s=%s" - % (name, shell_quote(config["defines"][name]).replace("$", "$$")) - for name in sorted(global_defines) - ] - ) + acdefines = " ".join([ + "-D%s=%s" % (name, shell_quote(config["defines"][name]).replace("$", "$$")) + for name in sorted(global_defines) + ]) substs["ACDEFINES"] = acdefines all_defines = OrderedDict() diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/mozbuild/backend/cpp_eclipse.py @@ -210,18 +210,16 @@ class CppEclipseBackend(CommonBackend): self._write_noindex() try: - subprocess.check_call( - [ - "eclipse", - "-application", - "-nosplash", - "org.eclipse.cdt.managedbuilder.core.headlessbuild", - "-data", - self._workspace_dir, - "-importAll", - self._project_dir, - ] - ) + subprocess.check_call([ + "eclipse", + "-application", + "-nosplash", + "org.eclipse.cdt.managedbuilder.core.headlessbuild", + "-data", + self._workspace_dir, + "-importAll", + self._project_dir, + ]) except OSError as e: # Remove the workspace directory so we re-generate it and # try to import again when the backend is invoked again. diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py @@ -86,9 +86,11 @@ class FasterMakeBackend(MakeBackend, PartialBackend): f, ) ) - self._l10n_dependencies[dep_target].append( - (merge, f.full_path, src) - ) + self._l10n_dependencies[dep_target].append(( + merge, + f.full_path, + src, + )) src = merge else: src = f.full_path @@ -220,12 +222,10 @@ class FasterMakeBackend(MakeBackend, PartialBackend): rule = mk.create_rule([merge]).add_dependencies( [ref_file, l10n_file] + python_deps ) - rule.add_commands( - [ - "$(PYTHON3) -m moz.l10n.bin.build_file " - f"--source {ref_file} --l10n {l10n_file} --target {merge}" - ] - ) + rule.add_commands([ + "$(PYTHON3) -m moz.l10n.bin.build_file " + f"--source {ref_file} --l10n {l10n_file} --target {merge}" + ]) # Add a dummy rule for the l10n file since it might not exist. mk.create_rule([l10n_file]) diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -181,10 +181,9 @@ class BackendMakeFile: self.fh.write("NONRECURSIVE_TARGETS += export\n") self.fh.write("NONRECURSIVE_TARGETS_export += xpidl\n") self.fh.write( - "NONRECURSIVE_TARGETS_export_xpidl_DIRECTORY = " - "$(DEPTH)/xpcom/xpidl\n" + "NONRECURSIVE_TARGETS_export_xpidl_DIRECTORY = $(DEPTH)/xpcom/xpidl\n" ) - self.fh.write("NONRECURSIVE_TARGETS_export_xpidl_TARGETS += " "export\n") + self.fh.write("NONRECURSIVE_TARGETS_export_xpidl_TARGETS += export\n") return self.fh.close() @@ -1617,9 +1616,12 @@ class RecursiveMakeBackend(MakeBackend): else: install_manifest.add_pattern_link(f.srcdir, f, dest_dir) elif isinstance(f, AbsolutePath): - if not f.full_path.lower().endswith( - (".dll", ".pdb", ".so", ".dylib") - ): + if not f.full_path.lower().endswith(( + ".dll", + ".pdb", + ".so", + ".dylib", + )): raise Exception( "Absolute paths installed to FINAL_TARGET_FILES must" " only be shared libraries or associated debug" @@ -1761,16 +1763,16 @@ class RecursiveMakeBackend(MakeBackend): mozpath.join("$(DEPTH)", top_level), make_quote(shell_quote("manifest %s" % path)), ] - rule.add_commands( - ["$(call py_action,buildlist %s,%s)" % (path, " ".join(args))] - ) + rule.add_commands([ + "$(call py_action,buildlist %s,%s)" % (path, " ".join(args)) + ]) args = [ mozpath.join("$(DEPTH)", obj.path), make_quote(shell_quote(str(obj.entry))), ] - rule.add_commands( - ["$(call py_action,buildlist %s,%s)" % (obj.entry.path, " ".join(args))] - ) + rule.add_commands([ + "$(call py_action,buildlist %s,%s)" % (obj.entry.path, " ".join(args)) + ]) fragment.dump(backend_file.fh, removal_guard=False) self._no_skip["misc"].add(obj.relsrcdir) @@ -1853,22 +1855,17 @@ class RecursiveMakeBackend(MakeBackend): basename = os.path.basename(source) sorted_nonstatic_ipdl_basenames.append(basename) rule = mk.create_rule([basename]) - rule.add_dependencies( - [ - source, - "backend.mk", - "Makefile", - "$(DEPTH)/config/autoconf.mk", - "$(topsrcdir)/config/config.mk", - ] - ) - rule.add_commands( - [ - "$(RM) $@", - "$(call py_action,preprocessor $@,$(DEFINES) $(ACDEFINES) " - "$< -o $@)", - ] - ) + rule.add_dependencies([ + source, + "backend.mk", + "Makefile", + "$(DEPTH)/config/autoconf.mk", + "$(topsrcdir)/config/config.mk", + ]) + rule.add_commands([ + "$(RM) $@", + "$(call py_action,preprocessor $@,$(DEFINES) $(ACDEFINES) $< -o $@)", + ]) mk.add_statement( "ALL_IPDLSRCS := %s %s" @@ -1943,25 +1940,20 @@ class RecursiveMakeBackend(MakeBackend): for source in sorted(webidls.all_preprocessed_sources()): basename = os.path.basename(source) rule = mk.create_rule([basename]) - rule.add_dependencies( - [ - source, - "backend.mk", - "Makefile", - "$(DEPTH)/config/autoconf.mk", - "$(topsrcdir)/config/config.mk", - ] - ) - rule.add_commands( - [ - # Remove the file before writing so bindings that go from - # static to preprocessed don't end up writing to a symlink, - # which would modify content in the source directory. - "$(RM) $@", - "$(call py_action,preprocessor $@,$(DEFINES) $(ACDEFINES) " - "$< -o $@)", - ] - ) + rule.add_dependencies([ + source, + "backend.mk", + "Makefile", + "$(DEPTH)/config/autoconf.mk", + "$(topsrcdir)/config/config.mk", + ]) + rule.add_commands([ + # Remove the file before writing so bindings that go from + # static to preprocessed don't end up writing to a symlink, + # which would modify content in the source directory. + "$(RM) $@", + "$(call py_action,preprocessor $@,$(DEFINES) $(ACDEFINES) $< -o $@)", + ]) self._add_unified_build_rules( mk, diff --git a/python/mozbuild/mozbuild/code_analysis/mach_commands.py b/python/mozbuild/mozbuild/code_analysis/mach_commands.py @@ -582,8 +582,7 @@ def _get_clang_tidy_command( @StaticAnalysisSubCommand( "static-analysis", "autotest", - "Run the auto-test suite in order to determine that" - " the analysis did not regress.", + "Run the auto-test suite in order to determine that the analysis did not regress.", ) @CommandArgument( "--dump-results", diff --git a/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py b/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py @@ -74,9 +74,9 @@ class LcovRecord: for fn_name, count in self.function_exec_counts.items() if fn_name in self.functions.values() } - self.covered_function_count = len( - [c for c in self.function_exec_counts.values() if c] - ) + self.covered_function_count = len([ + c for c in self.function_exec_counts.values() if c + ]) self.line_count = len(self.lines) self.covered_line_count = len([c for c, _ in self.lines.values() if c]) self.branch_count = len(self.branches) @@ -177,9 +177,9 @@ class RecordRewriter: def rewrite_record(self, record, pp_info): # Rewrite the lines in the given record according to preprocessor info # and split to additional records when pp_info has included file info. - self._current_pp_info = dict( - [(tuple([int(l) for l in k.split(",")]), v) for k, v in pp_info.items()] - ) + self._current_pp_info = dict([ + (tuple([int(l) for l in k.split(",")]), v) for k, v in pp_info.items() + ]) self._ranges = sorted(self._current_pp_info.keys()) self._additions = {} self._rewrite_lines(record) diff --git a/python/mozbuild/mozbuild/compilation/database.py b/python/mozbuild/mozbuild/compilation/database.py @@ -129,13 +129,11 @@ class CompileDBBackend(CommonBackend): per_source_flags = self._per_source_flags.get(filename) if per_source_flags is not None: c.extend(per_source_flags) - db.append( - { - "directory": directory, - "command": shell_quote(*c), - "file": mozpath.join(directory, filename), - } - ) + db.append({ + "directory": directory, + "command": shell_quote(*c), + "file": mozpath.join(directory, filename), + }) import json diff --git a/python/mozbuild/mozbuild/config_status.py b/python/mozbuild/mozbuild/config_status.py @@ -106,17 +106,15 @@ def config_status( """ if "CONFIG_FILES" in os.environ: - raise Exception( - "Using the CONFIG_FILES environment variable is not " "supported." - ) + raise Exception("Using the CONFIG_FILES environment variable is not supported.") if "CONFIG_HEADERS" in os.environ: raise Exception( - "Using the CONFIG_HEADERS environment variable is not " "supported." + "Using the CONFIG_HEADERS environment variable is not supported." ) if not os.path.isabs(topsrcdir): raise Exception( - "topsrcdir must be defined as an absolute directory: " "%s" % topsrcdir + "topsrcdir must be defined as an absolute directory: %s" % topsrcdir ) default_backends = ["RecursiveMake"] diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py @@ -330,22 +330,20 @@ class ConfigureSandbox(dict): # Expose a limited set of functions from os.path OS = ReadOnlyNamespace( - path=ReadOnlyNamespace( - **{ - k: getattr(mozpath, k, getattr(os.path, k)) - for k in ( - "abspath", - "basename", - "dirname", - "isabs", - "join", - "normcase", - "normpath", - "realpath", - "relpath", - ) - } - ) + path=ReadOnlyNamespace(**{ + k: getattr(mozpath, k, getattr(os.path, k)) + for k in ( + "abspath", + "basename", + "dirname", + "isabs", + "join", + "normcase", + "normpath", + "realpath", + "relpath", + ) + }) ) def __init__( @@ -708,7 +706,7 @@ class ConfigureSandbox(dict): raise ConfigureError("Option must not contain an '='") if name not in self._options: raise ConfigureError( - "'%s' is not a known option. " "Maybe it's declared too late?" % arg + "'%s' is not a known option. Maybe it's declared too late?" % arg ) arg = self._options[name] self._seen.add(arg) @@ -833,8 +831,7 @@ class ConfigureSandbox(dict): for c in conditions: if c != when: raise ConfigureError( - "@depends function needs the same `when` " - "as options it depends on" + "@depends function needs the same `when` as options it depends on" ) def decorator(func): @@ -1088,7 +1085,7 @@ class ConfigureSandbox(dict): raise TypeError("Unexpected type: '%s'" % type(name).__name__) if name in data: raise ConfigureError( - "Cannot add '%s' to configuration: Key already " "exists" % name + "Cannot add '%s' to configuration: Key already exists" % name ) value = self._resolve(value) if value is not None: @@ -1108,9 +1105,10 @@ class ConfigureSandbox(dict): """ when = self._normalize_when(when, "set_config") - self._execution_queue.append( - (self._resolve_and_set, (self._config, name, value, when)) - ) + self._execution_queue.append(( + self._resolve_and_set, + (self._config, name, value, when), + )) def set_define_impl(self, name, value, when=None): """Implementation of set_define(). @@ -1123,9 +1121,10 @@ class ConfigureSandbox(dict): when = self._normalize_when(when, "set_define") defines = self._config.setdefault("DEFINES", {}) - self._execution_queue.append( - (self._resolve_and_set, (defines, name, value, when)) - ) + self._execution_queue.append(( + self._resolve_and_set, + (defines, name, value, when), + )) def imply_option_impl(self, option, value, reason=None, when=None): """Implementation of imply_option(). diff --git a/python/mozbuild/mozbuild/configure/constants.py b/python/mozbuild/mozbuild/configure/constants.py @@ -111,30 +111,28 @@ class Abi(EnumString): # The order of those checks matter -CPU_preprocessor_checks = OrderedDict( - ( - ("x86", "__i386__ || _M_IX86"), - ("x86_64", "__x86_64__ || _M_X64"), - ("arm", "__arm__ || _M_ARM"), - ("aarch64", "__aarch64__ || _M_ARM64"), - ("ia64", "__ia64__"), - ("s390x", "__s390x__"), - ("s390", "__s390__"), - ("ppc64", "__powerpc64__"), - ("ppc", "__powerpc__"), - ("Alpha", "__alpha__"), - ("hppa", "__hppa__"), - ("sparc64", "__sparc__ && __arch64__"), - ("sparc", "__sparc__"), - ("m68k", "__m68k__"), - ("mips64", "__mips64"), - ("mips32", "__mips__"), - ("riscv64", "__riscv && __riscv_xlen == 64"), - ("loongarch64", "__loongarch64"), - ("sh4", "__sh__"), - ("wasm32", "__wasm32__"), - ) -) +CPU_preprocessor_checks = OrderedDict(( + ("x86", "__i386__ || _M_IX86"), + ("x86_64", "__x86_64__ || _M_X64"), + ("arm", "__arm__ || _M_ARM"), + ("aarch64", "__aarch64__ || _M_ARM64"), + ("ia64", "__ia64__"), + ("s390x", "__s390x__"), + ("s390", "__s390__"), + ("ppc64", "__powerpc64__"), + ("ppc", "__powerpc__"), + ("Alpha", "__alpha__"), + ("hppa", "__hppa__"), + ("sparc64", "__sparc__ && __arch64__"), + ("sparc", "__sparc__"), + ("m68k", "__m68k__"), + ("mips64", "__mips64"), + ("mips32", "__mips__"), + ("riscv64", "__riscv && __riscv_xlen == 64"), + ("loongarch64", "__loongarch64"), + ("sh4", "__sh__"), + ("wasm32", "__wasm32__"), +)) assert sorted(CPU_preprocessor_checks.keys()) == sorted(CPU.POSSIBLE_VALUES) diff --git a/python/mozbuild/mozbuild/configure/lint.py b/python/mozbuild/mozbuild/configure/lint.py @@ -237,8 +237,7 @@ class LintSandbox(ConfigureSandbox): if name.startswith(f"--{prefix}-"): frame = self._pretty_current_frame() e = ConfigureError( - "{} should be used instead of " - "{} with default={}".format( + "{} should be used instead of {} with default={}".format( name.replace(f"--{prefix}-", f"--{replacement}-"), name, default, diff --git a/python/mozbuild/mozbuild/configure/options.py b/python/mozbuild/mozbuild/configure/options.py @@ -220,8 +220,7 @@ class Option: ): if not name and not env: raise InvalidOptionError( - "At least an option name or an environment variable name must " - "be given" + "At least an option name or an environment variable name must be given" ) if name: if not isinstance(name, str): diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py @@ -55,10 +55,9 @@ Preferences. """.strip() -INSTALL_TESTS_CLOBBER = "".join( - [ - TextWrapper().fill(line) + "\n" - for line in """ +INSTALL_TESTS_CLOBBER = "".join([ + TextWrapper().fill(line) + "\n" + for line in """ The build system was unable to install tests because the CLOBBER file has \ been updated. This means if you edited any test files, your changes may not \ be picked up until a full/clobber build is performed. @@ -73,8 +72,7 @@ and proceed with running tests. To do this run: $ touch {clobber_file} """.splitlines() - ] -) +]) CLOBBER_REQUESTED_MESSAGE = """ =================== @@ -1153,7 +1151,7 @@ class BuildDriver(MozbuildObject): monitor.start() if directory is not None and not what: - print("Can only use -C/--directory with an explicit target " "name.") + print("Can only use -C/--directory with an explicit target name.") return 1 if directory is not None: @@ -1259,14 +1257,12 @@ class BuildDriver(MozbuildObject): status = None - if not config_rc and any( - [ - self.backend_out_of_date( - mozpath.join(self.topobjdir, "backend.%sBackend" % backend) - ) - for backend in all_backends - ] - ): + if not config_rc and any([ + self.backend_out_of_date( + mozpath.join(self.topobjdir, "backend.%sBackend" % backend) + ) + for backend in all_backends + ]): print("Build configuration changed. Regenerating backend.") args = [ config.substs["PYTHON3"], @@ -1428,9 +1424,10 @@ class BuildDriver(MozbuildObject): ) if os.path.exists(pathToThirdparty): - with open(pathToThirdparty, encoding="utf-8", newline="\n") as f, open( - pathToGenerated, encoding="utf-8", newline="\n" - ) as g: + with ( + open(pathToThirdparty, encoding="utf-8", newline="\n") as f, + open(pathToGenerated, encoding="utf-8", newline="\n") as g, + ): # Normalize the path (no trailing /) LOCAL_SUPPRESS_DIRS = tuple( [line.strip("\n/") for line in f] @@ -1490,7 +1487,7 @@ class BuildDriver(MozbuildObject): logging.WARNING, "compiler_warning", warning, - "warning: {normpath}:{line}:{column} [{flag}] " "{message}", + "warning: {normpath}:{line}:{column} [{flag}] {message}", ) else: self.log( @@ -1532,9 +1529,9 @@ class BuildDriver(MozbuildObject): self.notify("Build complete" if not status else "Build failed") if status: - if what and any( - [target for target in what if target not in ("faster", "binaries")] - ): + if what and any([ + target for target in what if target not in ("faster", "binaries") + ]): print( "Hey! Builds initiated with `mach build " "$A_SPECIFIC_TARGET` may not always work, even if the " @@ -1548,7 +1545,7 @@ class BuildDriver(MozbuildObject): # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) - print("To view a profile of the build, run |mach " "resource-usage|.") + print("To view a profile of the build, run |mach resource-usage|.") long_build = monitor.elapsed > 1200 @@ -1840,13 +1837,11 @@ class BuildDriver(MozbuildObject): return `True` if the clobber was required but not completed, and return `False` if the clobber was not required and not completed. """ - auto_clobber = any( - [ - env.get("AUTOCLOBBER", False), - (mozconfig["env"] or {}).get("added", {}).get("AUTOCLOBBER", False), - "AUTOCLOBBER=1" in (mozconfig["make_extra"] or []), - ] - ) + auto_clobber = any([ + env.get("AUTOCLOBBER", False), + (mozconfig["env"] or {}).get("added", {}).get("AUTOCLOBBER", False), + "AUTOCLOBBER=1" in (mozconfig["make_extra"] or []), + ]) from mozbuild.base import BuildEnvironmentNotFoundException substs = dict() diff --git a/python/mozbuild/mozbuild/controller/clobber.py b/python/mozbuild/mozbuild/controller/clobber.py @@ -13,10 +13,9 @@ from textwrap import TextWrapper from mozfile.mozfile import remove as mozfileremove from mozpack import path as mozpath -CLOBBER_MESSAGE = "".join( - [ - TextWrapper().fill(line) + "\n" - for line in """ +CLOBBER_MESSAGE = "".join([ + TextWrapper().fill(line) + "\n" + for line in """ The CLOBBER file has been updated, indicating that an incremental build since \ your last build will probably not work. A full/clobber build is required. @@ -38,8 +37,7 @@ Well, are ya? -- you can ignore this clobber requirement by running: $ touch {clobber_file} """.splitlines() - ] -) +]) class Clobberer: @@ -141,9 +139,9 @@ class Clobberer: """ # Determine where cargo build artifacts are stored RUST_TARGET_VARS = ("RUST_HOST_TARGET", "RUST_TARGET") - rust_targets = set( - [self.substs[x] for x in RUST_TARGET_VARS if x in self.substs] - ) + rust_targets = set([ + self.substs[x] for x in RUST_TARGET_VARS if x in self.substs + ]) rust_build_kind = "release" if self.substs.get("MOZ_DEBUG_RUST"): rust_build_kind = "debug" diff --git a/python/mozbuild/mozbuild/faster_daemon.py b/python/mozbuild/mozbuild/faster_daemon.py @@ -71,11 +71,9 @@ class Daemon: defines = dict(self.config_environment.acdefines) # These additions work around warts in the build system: see # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93 - defines.update( - { - "AB_CD": "en-US", - } - ) + defines.update({ + "AB_CD": "en-US", + }) return defines @mozbuild.util.memoized_property @@ -143,14 +141,10 @@ class Daemon: data = self.client.getSubscription("topsrcdir") if data: for dat in data: - files |= set( - [ - mozpath.normpath( - mozpath.join(self.config_environment.topsrcdir, f) - ) - for f in dat.get("files", []) - ] - ) + files |= set([ + mozpath.normpath(mozpath.join(self.config_environment.topsrcdir, f)) + for f in dat.get("files", []) + ]) return files diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py @@ -495,13 +495,11 @@ class LinkFlags(BaseCompileFlags): # TODO: This is pretty convoluted, and isn't really a per-context thing, # configure would be a better place to aggregate these. - if all( - [ - self._context.config.substs.get("OS_ARCH") == "WINNT", - self._context.config.substs.get("CC_TYPE") == "clang-cl", - not self._context.config.substs.get("MOZ_DEBUG"), - ] - ): + if all([ + self._context.config.substs.get("OS_ARCH") == "WINNT", + self._context.config.substs.get("CC_TYPE") == "clang-cl", + not self._context.config.substs.get("MOZ_DEBUG"), + ]): if self._context.config.substs.get("MOZ_OPTIMIZE"): flags.append("-OPT:REF,ICF") @@ -540,7 +538,7 @@ class TargetCompileFlags(BaseCompileFlags): def __setitem__(self, key, value): if key not in self._known_keys: raise ValueError( - "Invalid value. `%s` is not a compile flags " "category." % key + "Invalid value. `%s` is not a compile flags category." % key ) if key in self and self[key] is None: raise ValueError( @@ -1190,9 +1188,12 @@ SchedulingComponents = ContextDerivedTypedRecord( ("exclusive", TypedList(str, StrictOrderingOnAppendList)), ) -GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory( - {"script": str, "inputs": list, "force": bool, "flags": list} -) +GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({ + "script": str, + "inputs": list, + "force": bool, + "flags": list, +}) class Files(SubContext): @@ -2359,17 +2360,15 @@ VARIABLES = { """, ), "GYP_DIRS": ( - StrictOrderingOnAppendListWithFlagsFactory( - { - "variables": dict, - "input": str, - "sandbox_vars": dict, - "no_chromium": bool, - "no_unified": bool, - "non_unified_sources": StrictOrderingOnAppendList, - "action_overrides": dict, - } - ), + StrictOrderingOnAppendListWithFlagsFactory({ + "variables": dict, + "input": str, + "sandbox_vars": dict, + "no_chromium": bool, + "no_unified": bool, + "non_unified_sources": StrictOrderingOnAppendList, + "action_overrides": dict, + }), list, """Defines a list of object directories handled by gyp configurations. diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py @@ -1399,20 +1399,18 @@ class GeneratedFile(ContextDerived): self.required_during_compile = [ f for f in self.outputs - if f.endswith( - ( - ".asm", - ".c", - ".cpp", - ".inc", - ".m", - ".mm", - ".def", - ".s", - ".S", - "symverscript", - ) - ) + if f.endswith(( + ".asm", + ".c", + ".cpp", + ".inc", + ".m", + ".mm", + ".def", + ".s", + ".S", + "symverscript", + )) ] else: self.required_during_compile = required_during_compile diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py @@ -670,13 +670,11 @@ class TreeMetadataEmitter(LoggingMixin): if program: check_unique_binary(program, kind) self._binaries[program] = cls(context, program) - self._linkage.append( - ( - context, - self._binaries[program], - kind.replace("PROGRAM", "USE_LIBS"), - ) - ) + self._linkage.append(( + context, + self._binaries[program], + kind.replace("PROGRAM", "USE_LIBS"), + )) add_program(self._binaries[program], kind) all_rust_programs = [] @@ -711,13 +709,11 @@ class TreeMetadataEmitter(LoggingMixin): check_unique_binary(program, kind) self._binaries[program] = cls(context, program, cargo_file) - self._linkage.append( - ( - context, - self._binaries[program], - kind.replace("RUST_PROGRAMS", "USE_LIBS"), - ) - ) + self._linkage.append(( + context, + self._binaries[program], + kind.replace("RUST_PROGRAMS", "USE_LIBS"), + )) add_program(self._binaries[program], kind) for kind, cls in [ @@ -736,17 +732,11 @@ class TreeMetadataEmitter(LoggingMixin): self._binaries[program] = cls( context, program, is_unit_test=kind == "CPP_UNIT_TESTS" ) - self._linkage.append( - ( - context, - self._binaries[program], - ( - "HOST_USE_LIBS" - if kind == "HOST_SIMPLE_PROGRAMS" - else "USE_LIBS" - ), - ) - ) + self._linkage.append(( + context, + self._binaries[program], + ("HOST_USE_LIBS" if kind == "HOST_SIMPLE_PROGRAMS" else "USE_LIBS"), + )) add_program(self._binaries[program], kind) host_libname = context.get("HOST_LIBRARY_NAME") @@ -795,19 +785,17 @@ class TreeMetadataEmitter(LoggingMixin): if final_lib: if static_lib: raise SandboxValidationError( - "FINAL_LIBRARY implies FORCE_STATIC_LIB. " - "Please remove the latter.", + "FINAL_LIBRARY implies FORCE_STATIC_LIB. Please remove the latter.", context, ) if shared_lib: raise SandboxValidationError( - "FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. " - "Please remove one.", + "FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. Please remove one.", context, ) if is_framework: raise SandboxValidationError( - "FINAL_LIBRARY conflicts with IS_FRAMEWORK. " "Please remove one.", + "FINAL_LIBRARY conflicts with IS_FRAMEWORK. Please remove one.", context, ) static_args["link_into"] = final_lib @@ -817,7 +805,7 @@ class TreeMetadataEmitter(LoggingMixin): if is_framework: if soname: raise SandboxValidationError( - "IS_FRAMEWORK conflicts with SONAME. " "Please remove one.", + "IS_FRAMEWORK conflicts with SONAME. Please remove one.", context, ) shared_lib = True @@ -961,7 +949,7 @@ class TreeMetadataEmitter(LoggingMixin): if lib_defines: if not libname: raise SandboxValidationError( - "LIBRARY_DEFINES needs a " "LIBRARY_NAME to take effect", + "LIBRARY_DEFINES needs a LIBRARY_NAME to take effect", context, ) lib.lib_defines.update(lib_defines) @@ -1040,8 +1028,7 @@ class TreeMetadataEmitter(LoggingMixin): for f in context_srcs: if f in seen_sources: raise SandboxValidationError( - "Source file should only " - "be added to %s once: %s" % (symbol, f), + "Source file should only be added to %s once: %s" % (symbol, f), context, ) seen_sources.add(f) @@ -1058,8 +1045,7 @@ class TreeMetadataEmitter(LoggingMixin): if isinstance(f, SourcePath) and not os.path.exists(full_path): raise SandboxValidationError( - "File listed in %s does not " - "exist: '%s'" % (symbol, full_path), + "File listed in %s does not exist: '%s'" % (symbol, full_path), context, ) @@ -1089,7 +1075,7 @@ class TreeMetadataEmitter(LoggingMixin): if no_pgo: if no_pgo_sources: raise SandboxValidationError( - "NO_PGO and SOURCES[...].no_pgo " "cannot be set at the same time", + "NO_PGO and SOURCES[...].no_pgo cannot be set at the same time", context, ) passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo @@ -1261,16 +1247,14 @@ class TreeMetadataEmitter(LoggingMixin): and context["DELAYLOAD_DLLS"] ): if context.config.substs.get("CC_TYPE") != "clang": - context["LDFLAGS"].extend( - [("-DELAYLOAD:%s" % dll) for dll in context["DELAYLOAD_DLLS"]] - ) + context["LDFLAGS"].extend([ + ("-DELAYLOAD:%s" % dll) for dll in context["DELAYLOAD_DLLS"] + ]) else: - context["LDFLAGS"].extend( - [ - ("-Wl,-Xlink=-DELAYLOAD:%s" % dll) - for dll in context["DELAYLOAD_DLLS"] - ] - ) + context["LDFLAGS"].extend([ + ("-Wl,-Xlink=-DELAYLOAD:%s" % dll) + for dll in context["DELAYLOAD_DLLS"] + ]) context["OS_LIBS"].append("delayimp") for v in ["CMFLAGS", "CMMFLAGS"]: @@ -1448,12 +1432,10 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._handle_linkables(context, passthru, generated_files): yield obj - generated_files.update( - [ - "%s%s" % (k, self.config.substs.get("BIN_SUFFIX", "")) - for k in self._binaries.keys() - ] - ) + generated_files.update([ + "%s%s" % (k, self.config.substs.get("BIN_SUFFIX", "")) + for k in self._binaries.keys() + ]) processed_moz_src_files = None if "MOZ_SRC_FILES" in context: @@ -1587,7 +1569,7 @@ class TreeMetadataEmitter(LoggingMixin): context.get("DIST_SUBDIR") or context.get("XPI_NAME") ): raise SandboxValidationError( - "RESOURCES_FILES cannot be used with DIST_SUBDIR or " "XPI_NAME.", + "RESOURCES_FILES cannot be used with DIST_SUBDIR or XPI_NAME.", context, ) @@ -1673,14 +1655,14 @@ class TreeMetadataEmitter(LoggingMixin): if not xpidl_module: if context["XPIDL_SOURCES"]: raise SandboxValidationError( - "XPIDL_MODULE must be defined if " "XPIDL_SOURCES is defined.", + "XPIDL_MODULE must be defined if XPIDL_SOURCES is defined.", context, ) return if not context["XPIDL_SOURCES"]: raise SandboxValidationError( - "XPIDL_MODULE cannot be defined " "unless there are XPIDL_SOURCES", + "XPIDL_MODULE cannot be defined unless there are XPIDL_SOURCES", context, ) @@ -1695,7 +1677,7 @@ class TreeMetadataEmitter(LoggingMixin): for idl in context["XPIDL_SOURCES"]: if not os.path.exists(idl.full_path): raise SandboxValidationError( - "File %s from XPIDL_SOURCES " "does not exist" % idl.full_path, + "File %s from XPIDL_SOURCES does not exist" % idl.full_path, context, ) @@ -1839,15 +1821,13 @@ class TreeMetadataEmitter(LoggingMixin): obj.installs[source] = (dest, False) obj.external_installs |= install_info.external_installs for install_path in install_info.deferred_installs: - if all( - [ - "*" not in install_path, - not os.path.isfile( - mozpath.join(context.config.topsrcdir, install_path[2:]) - ), - install_path not in install_info.external_installs, - ] - ): + if all([ + "*" not in install_path, + not os.path.isfile( + mozpath.join(context.config.topsrcdir, install_path[2:]) + ), + install_path not in install_info.external_installs, + ]): raise SandboxValidationError( "Error processing test " "manifest %s: entry in support-files not present " @@ -1934,8 +1914,7 @@ class TreeMetadataEmitter(LoggingMixin): jar_manifests = context.get("JAR_MANIFESTS", []) if len(jar_manifests) > 1: raise SandboxValidationError( - "While JAR_MANIFESTS is a list, " - "it is currently limited to one value.", + "While JAR_MANIFESTS is a list, it is currently limited to one value.", context, ) diff --git a/python/mozbuild/mozbuild/frontend/gyp_reader.py b/python/mozbuild/mozbuild/frontend/gyp_reader.py @@ -431,12 +431,10 @@ class GypProcessor: if config.substs["CC_TYPE"] == "clang-cl": # This isn't actually used anywhere in this generator, but it's needed # to override the registry detection of VC++ in gyp. - os.environ.update( - { - "GYP_MSVS_OVERRIDE_PATH": "fake_path", - "GYP_MSVS_VERSION": config.substs["MSVS_VERSION"], - } - ) + os.environ.update({ + "GYP_MSVS_OVERRIDE_PATH": "fake_path", + "GYP_MSVS_VERSION": config.substs["MSVS_VERSION"], + }) params = { "parallel": False, diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py @@ -875,8 +875,7 @@ class BuildReader: def summary(self): return ExecutionSummary( - "Finished reading {file_count:d} moz.build files in " - "{execution_time:.2f}s", + "Finished reading {file_count:d} moz.build files in {execution_time:.2f}s", file_count=self._file_count, execution_time=self._execution_time, ) @@ -1217,7 +1216,7 @@ class BuildReader: for v in ("input", "variables"): if not getattr(gyp_dir, v): raise SandboxValidationError( - "Missing value for " 'GYP_DIRS["%s"].%s' % (target_dir, v), + 'Missing value for GYP_DIRS["%s"].%s' % (target_dir, v), context, ) diff --git a/python/mozbuild/mozbuild/frontend/sandbox.py b/python/mozbuild/mozbuild/frontend/sandbox.py @@ -101,20 +101,18 @@ class Sandbox(dict): """ # The default set of builtins. - BUILTINS = ReadOnlyDict( - { - # Only real Python built-ins should go here. - "None": None, - "False": False, - "True": True, - "sorted": alphabetical_sorted, - "int": int, - "len": len, - "range": range, - "set": set, - "tuple": tuple, - } - ) + BUILTINS = ReadOnlyDict({ + # Only real Python built-ins should go here. + "None": None, + "False": False, + "True": True, + "sorted": alphabetical_sorted, + "int": int, + "len": len, + "range": range, + "set": set, + "tuple": tuple, + }) def __init__(self, context, finder=default_finder): """Initialize a Sandbox ready for execution.""" diff --git a/python/mozbuild/mozbuild/lockfiles/generate_python_lockfiles.py b/python/mozbuild/mozbuild/lockfiles/generate_python_lockfiles.py @@ -28,7 +28,6 @@ class MissingUVError(Exception): class GeneratePythonLockfiles(MozbuildObject): - def __init__(self, *args, **kwargs) -> None: super().__init__(*args, virtualenv_name="uv", **kwargs) diff --git a/python/mozbuild/mozbuild/lockfiles/site_dependency_extractor.py b/python/mozbuild/mozbuild/lockfiles/site_dependency_extractor.py @@ -31,7 +31,6 @@ class DependencyParseError(Exception): class SiteDependencyExtractor: - def __init__(self, site_name: str, sites_dir: Path, topsrcdir: Path) -> None: self.site_file = sites_dir / f"{site_name}.txt" if not self.site_file.is_file(): diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py @@ -127,36 +127,34 @@ def _cargo_config_yaml_schema(): else: raise ValueError - return Schema( - { - # The name of the command (not checked for now, but maybe - # later) - Required("command"): All(str, starts_with_cargo), - # Whether `make` should stop immediately in case - # of error returned by the command. Default: False - "continue_on_error": Boolean, - # Whether this command requires pre_export and export build - # targets to have run. Defaults to bool(cargo_build_flags). - "requires_export": Boolean, - # Build flags to use. If this variable is not - # defined here, the build flags are generated automatically and are - # the same as for `cargo build`. See available substitutions at the - # end. - "cargo_build_flags": [str], - # Extra build flags to use. These flags are added - # after the cargo_build_flags both when they are provided or - # automatically generated. See available substitutions at the end. - "cargo_extra_flags": [str], - # Available substitutions for `cargo_*_flags`: - # * {arch}: architecture target - # * {crate}: current crate name - # * {directory}: Directory of the current crate within the source tree - # * {features}: Rust features (for `--features`) - # * {manifest}: full path of `Cargo.toml` file - # * {target}: `--lib` for library, `--bin CRATE` for executables - # * {topsrcdir}: Top directory of sources - } - ) + return Schema({ + # The name of the command (not checked for now, but maybe + # later) + Required("command"): All(str, starts_with_cargo), + # Whether `make` should stop immediately in case + # of error returned by the command. Default: False + "continue_on_error": Boolean, + # Whether this command requires pre_export and export build + # targets to have run. Defaults to bool(cargo_build_flags). + "requires_export": Boolean, + # Build flags to use. If this variable is not + # defined here, the build flags are generated automatically and are + # the same as for `cargo build`. See available substitutions at the + # end. + "cargo_build_flags": [str], + # Extra build flags to use. These flags are added + # after the cargo_build_flags both when they are provided or + # automatically generated. See available substitutions at the end. + "cargo_extra_flags": [str], + # Available substitutions for `cargo_*_flags`: + # * {arch}: architecture target + # * {crate}: current crate name + # * {directory}: Directory of the current crate within the source tree + # * {features}: Rust features (for `--features`) + # * {manifest}: full path of `Cargo.toml` file + # * {target}: `--lib` for library, `--bin CRATE` for executables + # * {topsrcdir}: Top directory of sources + }) @Command( @@ -487,8 +485,9 @@ CLOBBER_CHOICES = {"objdir", "python", "gradle", "artifacts"} "what", default=["objdir", "python"], nargs="*", - help="Target to clobber, must be one of {{{}}} (default " - "objdir and python).".format(", ".join(CLOBBER_CHOICES)), + help="Target to clobber, must be one of {{{}}} (default objdir and python).".format( + ", ".join(CLOBBER_CHOICES) + ), ) @CommandArgument("--full", action="store_true", help="Perform a full clobber") def clobber(command_context, what, full=False): @@ -717,16 +716,14 @@ def handle_log_file(command_context, log_file): start_time = created command_context.log_manager.terminal_handler.formatter.start_time = created if "line" in params: - record = logging.makeLogRecord( - { - "created": created, - "name": command_context._logger.name, - "levelno": logging.INFO, - "msg": "{line}", - "params": params, - "action": action, - } - ) + record = logging.makeLogRecord({ + "created": created, + "name": command_context._logger.name, + "levelno": logging.INFO, + "msg": "{line}", + "params": params, + "action": action, + }) command_context._logger.handle(record) @@ -1582,8 +1579,7 @@ def _get_android_run_parser(): "--no-wait", action="store_true", default=False, - help="Do not wait for application to start before returning " - "(default: False)", + help="Do not wait for application to start before returning (default: False)", ) group.add_argument( "--enable-fission", @@ -1736,8 +1732,7 @@ def _get_desktop_run_parser(): group.add_argument( "--temp-profile", action="store_true", - help="Run the program using a new temporary profile created inside " - "the objdir.", + help="Run the program using a new temporary profile created inside the objdir.", ) group.add_argument( "--macos-open", @@ -3097,8 +3092,7 @@ def repackage_msi( "--unsigned", default=False, action="store_true", - help="Support `Add-AppxPackage ... -AllowUnsigned` on Windows 11." - "(Default: false)", + help="Support `Add-AppxPackage ... -AllowUnsigned` on Windows 11.(Default: false)", ) def repackage_msix( command_context, @@ -3541,8 +3535,7 @@ def repackage_snap_install(command_context, snap_file, snap_name, sudo=None): logging.ERROR, "repackage-snap-install-no-sudo", {}, - "Couldn't find a command to run snap as root; please use the" - " --sudo option", + "Couldn't find a command to run snap as root; please use the --sudo option", ) if not snap_file: diff --git a/python/mozbuild/mozbuild/repackaging/flatpak.py b/python/mozbuild/mozbuild/repackaging/flatpak.py @@ -150,19 +150,17 @@ def repackage_flatpak( application_ini_data = application_ini_data_from_directory(str(lib_dir)) variables = get_build_variables(application_ini_data, arch, version) - variables.update( - { - "FREEDESKTOP_VERSION": FREEDESKTOP_VERSION, - "FIREFOX_BASEAPP_CHANNEL": FIREFOX_BASEAPP_CHANNEL, - "FLATPAK_BRANCH": flatpak_branch, - "DATE": variables["TIMESTAMP"].strftime("%Y-%m-%d"), - # Override PKG_NAME since we use branches for beta vs release - "PKG_NAME": product, - "DBusActivatable": "false", - # Override Icon to match the flatpak's name - "Icon": flatpak_name, - } - ) + variables.update({ + "FREEDESKTOP_VERSION": FREEDESKTOP_VERSION, + "FIREFOX_BASEAPP_CHANNEL": FIREFOX_BASEAPP_CHANNEL, + "FLATPAK_BRANCH": flatpak_branch, + "DATE": variables["TIMESTAMP"].strftime("%Y-%m-%d"), + # Override PKG_NAME since we use branches for beta vs release + "PKG_NAME": product, + "DBusActivatable": "false", + # Override Icon to match the flatpak's name + "Icon": flatpak_name, + }) _render_flatpak_templates(template_dir, build_dir, variables) from fluent.runtime.fallback import FluentLocalization, FluentResourceLoader diff --git a/python/mozbuild/mozbuild/repackaging/msix.py b/python/mozbuild/mozbuild/repackaging/msix.py @@ -737,9 +737,7 @@ def repackage_msix( if not makeappx: makeappx = find_sdk_tool("makeappx.exe", log=log) if not makeappx: - raise ValueError( - "makeappx is required; " "set MAKEAPPX or WINDOWSSDKDIR or PATH" - ) + raise ValueError("makeappx is required; set MAKEAPPX or WINDOWSSDKDIR or PATH") # `makeappx.exe` supports both slash and hyphen style arguments; `makemsix` # supports only hyphen style. `makeappx.exe` allows to overwrite and to @@ -776,7 +774,7 @@ def repackage_msix( def _sign_msix_win(output, force, log, verbose): powershell_exe = find_sdk_tool("powershell.exe", log=log) if not powershell_exe: - raise ValueError("powershell is required; " "set POWERSHELL or PATH") + raise ValueError("powershell is required; set POWERSHELL or PATH") def powershell(argstring, check=True): "Invoke `powershell.exe`. Arguments are given as a string to allow consumer to quote." @@ -789,9 +787,7 @@ def _sign_msix_win(output, force, log, verbose): signtool = find_sdk_tool("signtool.exe", log=log) if not signtool: - raise ValueError( - "signtool is required; " "set SIGNTOOL or WINDOWSSDKDIR or PATH" - ) + raise ValueError("signtool is required; set SIGNTOOL or WINDOWSSDKDIR or PATH") # Our first order of business is to find, or generate, a (self-signed) # certificate. @@ -979,12 +975,12 @@ def _sign_msix_posix(output, force, log, verbose): makeappx = find_sdk_tool("makeappx", log=log) if not makeappx: - raise ValueError("makeappx is required; " "set MAKEAPPX or PATH") + raise ValueError("makeappx is required; set MAKEAPPX or PATH") openssl = find_sdk_tool("openssl", log=log) if not openssl: - raise ValueError("openssl is required; " "set OPENSSL or PATH") + raise ValueError("openssl is required; set OPENSSL or PATH") if "sign" not in subprocess.run( makeappx, check=False, capture_output=True diff --git a/python/mozbuild/mozbuild/repackaging/snap.py b/python/mozbuild/mozbuild/repackaging/snap.py @@ -118,16 +118,14 @@ def repackage_snap( def unpack_tarball(package, destdir): os.makedirs(destdir, exist_ok=True) - subprocess.check_call( - [ - "tar", - "-C", - destdir, - "-xvf", - package, - "--strip-components=1", - ] - ) + subprocess.check_call([ + "tar", + "-C", + destdir, + "-xvf", + package, + "--strip-components=1", + ]) def missing_connections(app_name): diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py @@ -17,22 +17,18 @@ from sphinx.util.docutils import ReferenceRole def function_reference(f, attr, args, doc): lines = [] - lines.extend( - [ - f, - "-" * len(f), - "", - ] - ) + lines.extend([ + f, + "-" * len(f), + "", + ]) docstring = prepare_docstring(doc) - lines.extend( - [ - docstring[0], - "", - ] - ) + lines.extend([ + docstring[0], + "", + ]) arg_types = [] @@ -46,12 +42,10 @@ def function_reference(f, attr, args, doc): arg_s = "(%s)" % ", ".join(arg_types) - lines.extend( - [ - ":Arguments: %s" % arg_s, - "", - ] - ) + lines.extend([ + ":Arguments: %s" % arg_s, + "", + ]) lines.extend(docstring[1:]) lines.append("") @@ -68,20 +62,16 @@ def variable_reference(v, st_type, in_type, doc): docstring = prepare_docstring(doc) - lines.extend( - [ - docstring[0], - "", - ] - ) + lines.extend([ + docstring[0], + "", + ]) - lines.extend( - [ - ":Storage Type: ``%s``" % st_type.__name__, - ":Input Type: ``%s``" % in_type.__name__, - "", - ] - ) + lines.extend([ + ":Storage Type: ``%s``" % st_type.__name__, + ":Input Type: ``%s``" % in_type.__name__, + "", + ]) lines.extend(docstring[1:]) lines.append("") @@ -98,14 +88,12 @@ def special_reference(v, func, typ, doc): docstring = prepare_docstring(doc) - lines.extend( - [ - docstring[0], - "", - ":Type: ``%s``" % typ.__name__, - "", - ] - ) + lines.extend([ + docstring[0], + "", + ":Type: ``%s``" % typ.__name__, + "", + ]) lines.extend(docstring[1:]) lines.append("") @@ -116,25 +104,21 @@ def special_reference(v, func, typ, doc): def format_module(m): lines = [] - lines.extend( - [ - ".. note::", - " moz.build files' implementation includes a ``Path`` class.", - ] - ) + lines.extend([ + ".. note::", + " moz.build files' implementation includes a ``Path`` class.", + ]) path_docstring_minus_summary = prepare_docstring(m.Path.__doc__)[2:] lines.extend([" " + line for line in path_docstring_minus_summary]) for subcontext, cls in sorted(m.SUBCONTEXTS.items()): - lines.extend( - [ - ".. _mozbuild_subcontext_%s:" % subcontext, - "", - "Sub-Context: %s" % subcontext, - "=============" + "=" * len(subcontext), - "", - ] - ) + lines.extend([ + ".. _mozbuild_subcontext_%s:" % subcontext, + "", + "Sub-Context: %s" % subcontext, + "=============" + "=" * len(subcontext), + "", + ]) lines.extend(prepare_docstring(cls.__doc__)) if lines[-1]: lines.append("") @@ -142,35 +126,29 @@ def format_module(m): for k, v in sorted(cls.VARIABLES.items()): lines.extend(variable_reference(k, *v)) - lines.extend( - [ - "Variables", - "=========", - "", - ] - ) + lines.extend([ + "Variables", + "=========", + "", + ]) for v in sorted(m.VARIABLES): lines.extend(variable_reference(v, *m.VARIABLES[v])) - lines.extend( - [ - "Functions", - "=========", - "", - ] - ) + lines.extend([ + "Functions", + "=========", + "", + ]) for func in sorted(m.FUNCTIONS): lines.extend(function_reference(func, *m.FUNCTIONS[func])) - lines.extend( - [ - "Special Variables", - "=================", - "", - ] - ) + lines.extend([ + "Special Variables", + "=================", + "", + ]) for v in sorted(m.SPECIAL_VARIABLES): lines.extend(special_reference(v, *m.SPECIAL_VARIABLES[v])) diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/mozbuild/mozbuild/test/action/test_buildlist.py @@ -47,7 +47,7 @@ class TestBuildList(unittest.TestCase): self.assertEqual(line, l.pop(0)) self.assertTrue( len(l) == 0, - f"not enough lines in file! (expected '{l}'," f" got '{lines}'", + f"not enough lines in file! (expected '{l}', got '{lines}'", ) def test_basic(self): diff --git a/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py b/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py @@ -80,12 +80,10 @@ class TestPartial(unittest.TestCase): self.assertTrue(os.path.exists(path)) def _assert_deps(self, env, deps): - deps = sorted( - [ - "$(wildcard %s)" % (mozpath.join(env.topobjdir, "config.statusd", d)) - for d in deps - ] - ) + deps = sorted([ + "$(wildcard %s)" % (mozpath.join(env.topobjdir, "config.statusd", d)) + for d in deps + ]) self.assertEqual(sorted(env.get_dependencies()), deps) def test_dependencies(self): diff --git a/python/mozbuild/mozbuild/test/backend/test_test_manifest.py b/python/mozbuild/mozbuild/test/backend/test_test_manifest.py @@ -120,14 +120,12 @@ class TestTestManifestBackend(BackendTester): self.assertEqual( sources, - set( - [ - mozpath.join(env.topsrcdir, "mochitest.toml"), - mozpath.join(env.topsrcdir, "mochitest-common.toml"), - mozpath.join(env.topsrcdir, "moz.build"), - status_path, - ] - ), + set([ + mozpath.join(env.topsrcdir, "mochitest.toml"), + mozpath.join(env.topsrcdir, "mochitest-common.toml"), + mozpath.join(env.topsrcdir, "moz.build"), + status_path, + ]), ) diff --git a/python/mozbuild/mozbuild/test/common.py b/python/mozbuild/mozbuild/test/common.py @@ -50,7 +50,7 @@ class MockConfig: "DLL_PREFIX": "lib", "DLL_SUFFIX": ".so", }, - **extra_substs + **extra_substs, ) self.defines = self.substs diff --git a/python/mozbuild/mozbuild/test/configure/test_bootstrap.py b/python/mozbuild/mozbuild/test/configure/test_bootstrap.py @@ -16,7 +16,7 @@ from mozbuild.util import ReadOnlyNamespace class IndexSearch: def should_replace_task(self, task, *args): - return f'fake-task-id-for-{task["index"][0]}' + return f"fake-task-id-for-{task['index'][0]}" class TestBootstrap(BaseConfigureTest): diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py @@ -535,8 +535,7 @@ class TestChecksConfigure(unittest.TestCase): """ DEBUG: a: Looking for known-a ERROR: Paths provided to find_program must be a list of strings, not %r - """ - % mozpath.dirname(self.OTHER_A) + """ % mozpath.dirname(self.OTHER_A) ), ) @@ -1144,9 +1143,10 @@ class TestChecksConfigure(unittest.TestCase): }, ) - with MockedOpen( - {"default-key": "default-id default-key\n", "key": "fake-id fake-key\n"} - ): + with MockedOpen({ + "default-key": "default-id default-key\n", + "key": "fake-id fake-key\n", + }): config, output, status = self.get_result( "id_and_secret_keyfile('Bing API', default='default-key')", args=["--with-bing-api-keyfile=key"], diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py @@ -40,9 +40,9 @@ class TestConfigure(unittest.TestCase): return config def moz_configure(self, source): - return MockedOpen( - {os.path.join(test_data_path, "moz.configure"): textwrap.dedent(source)} - ) + return MockedOpen({ + os.path.join(test_data_path, "moz.configure"): textwrap.dedent(source) + }) def test_defaults(self): config = self.get_config() @@ -269,16 +269,18 @@ class TestConfigure(unittest.TestCase): def test_returned_choices(self): for val in ("a", "b", "c"): - config = self.get_config( - ["--enable-values=alpha", "--returned-choices=%s" % val] - ) + config = self.get_config([ + "--enable-values=alpha", + "--returned-choices=%s" % val, + ]) self.assertIn("CHOICES", config) self.assertEqual(PositiveOptionValue((val,)), config["CHOICES"]) for val in ("0", "1", "2"): - config = self.get_config( - ["--enable-values=numeric", "--returned-choices=%s" % val] - ) + config = self.get_config([ + "--enable-values=numeric", + "--returned-choices=%s" % val, + ]) self.assertIn("CHOICES", config) self.assertEqual(PositiveOptionValue((val,)), config["CHOICES"]) @@ -1374,7 +1376,7 @@ class TestConfigure(unittest.TestCase): self.assertEqual( str(e.exception), - "@depends function needs the same `when` as " "options it depends on", + "@depends function needs the same `when` as options it depends on", ) with self.moz_configure( @@ -1394,7 +1396,7 @@ class TestConfigure(unittest.TestCase): self.assertEqual( str(e.exception), - "@depends function needs the same `when` as " "options it depends on", + "@depends function needs the same `when` as options it depends on", ) with self.moz_configure( @@ -1485,10 +1487,9 @@ class TestConfigure(unittest.TestCase): self.assertEqual(str(e.exception), "Unexpected type: 'int'") def test_include_when(self): - with MockedOpen( - { - os.path.join(test_data_path, "moz.configure"): textwrap.dedent( - """ + with MockedOpen({ + os.path.join(test_data_path, "moz.configure"): textwrap.dedent( + """ option('--with-foo', help='Foo') include('always.configure', when=True) @@ -1499,27 +1500,27 @@ class TestConfigure(unittest.TestCase): set_config('BAR', bar) set_config('QUX', qux) """ - ), - os.path.join(test_data_path, "always.configure"): textwrap.dedent( - """ + ), + os.path.join(test_data_path, "always.configure"): textwrap.dedent( + """ option('--with-bar', help='Bar') @depends('--with-bar') def bar(x): if x: return 'bar' """ - ), - os.path.join(test_data_path, "never.configure"): textwrap.dedent( - """ + ), + os.path.join(test_data_path, "never.configure"): textwrap.dedent( + """ option('--with-qux', help='Qux') @depends('--with-qux') def qux(x): if x: return 'qux' """ - ), - os.path.join(test_data_path, "foo.configure"): textwrap.dedent( - """ + ), + os.path.join(test_data_path, "foo.configure"): textwrap.dedent( + """ option('--with-foo-really', help='Really foo') @depends('--with-foo-really') def foo(x): @@ -1528,14 +1529,13 @@ class TestConfigure(unittest.TestCase): include('foo2.configure', when='--with-foo-really') """ - ), - os.path.join(test_data_path, "foo2.configure"): textwrap.dedent( - """ + ), + os.path.join(test_data_path, "foo2.configure"): textwrap.dedent( + """ set_config('FOO2', True) """ - ), - } - ): + ), + }): config = self.get_config() self.assertEqual(config, {}) @@ -1586,7 +1586,7 @@ class TestConfigure(unittest.TestCase): self.get_config() self.assertIn( - "Cannot assign `foo` because it is neither a @depends nor a " "@template", + "Cannot assign `foo` because it is neither a @depends nor a @template", str(e.exception), ) @@ -1615,7 +1615,7 @@ class TestConfigure(unittest.TestCase): self.assertEqual( str(e.exception), - "'--with-foo' is not a known option. Maybe it's " "declared too late?", + "'--with-foo' is not a known option. Maybe it's declared too late?", ) with self.assertRaises(ConfigureError) as e: @@ -1642,7 +1642,7 @@ class TestConfigure(unittest.TestCase): self.assertEqual( str(e.exception), - "Cannot use object of type 'int' as argument " "to @depends", + "Cannot use object of type 'int' as argument to @depends", ) with self.assertRaises(ConfigureError) as e: @@ -1947,7 +1947,7 @@ class TestConfigure(unittest.TestCase): self.assertEqual( str(e.exception), - "@depends function needs the same `when` as " "options it depends on", + "@depends function needs the same `when` as options it depends on", ) with self.moz_configure( @@ -2084,9 +2084,9 @@ class TestConfigure(unittest.TestCase): ("", NegativeOptionValue()), ("--baz=baz", PositiveOptionValue(("baz",))), ): - config = self.get_config( - [x for x in (foo_opt, bar_opt, baz_opt) if x] - ) + config = self.get_config([ + x for x in (foo_opt, bar_opt, baz_opt) if x + ]) self.assertEqual( config, { diff --git a/python/mozbuild/mozbuild/test/configure/test_lint.py b/python/mozbuild/mozbuild/test/configure/test_lint.py @@ -47,9 +47,9 @@ class TestLint(unittest.TestCase): sandbox.run(mozpath.join(test_data_path, "moz.configure")) def moz_configure(self, source): - return MockedOpen( - {os.path.join(test_data_path, "moz.configure"): textwrap.dedent(source)} - ) + return MockedOpen({ + os.path.join(test_data_path, "moz.configure"): textwrap.dedent(source) + }) def assertRaisesFromLine(self, exc_type, line): return AssertRaisesFromLine( @@ -341,7 +341,7 @@ class TestLint(unittest.TestCase): self.lint_test() self.assertEqual( str(e.exception), - "--disable-foo should be used instead of " "--enable-foo with default=True", + "--disable-foo should be used instead of --enable-foo with default=True", ) def test_default_disable(self): @@ -361,8 +361,7 @@ class TestLint(unittest.TestCase): self.lint_test() self.assertEqual( str(e.exception), - "--enable-foo should be used instead of " - "--disable-foo with default=False", + "--enable-foo should be used instead of --disable-foo with default=False", ) def test_default_with(self): @@ -382,7 +381,7 @@ class TestLint(unittest.TestCase): self.lint_test() self.assertEqual( str(e.exception), - "--without-foo should be used instead of " "--with-foo with default=True", + "--without-foo should be used instead of --with-foo with default=True", ) def test_default_without(self): @@ -402,7 +401,7 @@ class TestLint(unittest.TestCase): self.lint_test() self.assertEqual( str(e.exception), - "--with-foo should be used instead of " "--without-foo with default=False", + "--with-foo should be used instead of --without-foo with default=False", ) def test_default_func(self): @@ -427,8 +426,7 @@ class TestLint(unittest.TestCase): self.lint_test() self.assertEqual( str(e.exception), - '`help` should contain "{Enable|Disable}" because of ' - "non-constant default", + '`help` should contain "{Enable|Disable}" because of non-constant default', ) def test_dual_help(self): diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py @@ -62,14 +62,12 @@ SUPPORTS_CXX20 = { @memoize def GCC_BASE(version): version = Version(version) - return FakeCompiler( - { - "__GNUC__": version.major, - "__GNUC_MINOR__": version.minor, - "__GNUC_PATCHLEVEL__": version.patch, - "__STDC__": 1, - } - ) + return FakeCompiler({ + "__GNUC__": version.major, + "__GNUC_MINOR__": version.minor, + "__GNUC_PATCHLEVEL__": version.patch, + "__STDC__": 1, + }) @memoize @@ -145,14 +143,12 @@ GCC_PLATFORM_X86_64_WIN = FakeCompiler(GCC_PLATFORM_X86_64, GCC_PLATFORM_WIN) @memoize def CLANG_BASE(version): version = Version(version) - return FakeCompiler( - { - "__clang__": 1, - "__clang_major__": version.major, - "__clang_minor__": version.minor, - "__clang_patchlevel__": version.patch, - } - ) + return FakeCompiler({ + "__clang__": 1, + "__clang_major__": version.major, + "__clang_minor__": version.minor, + "__clang_patchlevel__": version.patch, + }) @memoize @@ -239,17 +235,15 @@ CLANG_PLATFORM_X86_64_WIN = CLANG_PLATFORM(GCC_PLATFORM_X86_64_WIN) @memoize def VS(version): version = Version(version) - return FakeCompiler( - { - None: { - "_MSC_VER": "%02d%02d" % (version.major, version.minor), - "_MSC_FULL_VER": "%02d%02d%05d" - % (version.major, version.minor, version.patch), - "_MT": "1", - }, - "*.cpp": DEFAULT_CXX_97, - } - ) + return FakeCompiler({ + None: { + "_MSC_VER": "%02d%02d" % (version.major, version.minor), + "_MSC_FULL_VER": "%02d%02d%05d" + % (version.major, version.minor, version.patch), + "_MT": "1", + }, + "*.cpp": DEFAULT_CXX_97, + }) VS_2017u8 = VS("19.15.26726") @@ -703,12 +697,10 @@ class LinuxToolchainTest(BaseToolchainTest): def test_absolute_path(self): paths = dict(self.PATHS) - paths.update( - { - "/opt/clang/bin/clang": paths["/usr/bin/clang"], - "/opt/clang/bin/clang++": paths["/usr/bin/clang++"], - } - ) + paths.update({ + "/opt/clang/bin/clang": paths["/usr/bin/clang"], + "/opt/clang/bin/clang++": paths["/usr/bin/clang++"], + }) result = { "c_compiler": self.DEFAULT_CLANG_RESULT + {"compiler": "/opt/clang/bin/clang"}, @@ -725,12 +717,10 @@ class LinuxToolchainTest(BaseToolchainTest): def test_atypical_name(self): paths = dict(self.PATHS) - paths.update( - { - "/usr/bin/afl-clang-fast": paths["/usr/bin/clang"], - "/usr/bin/afl-clang-fast++": paths["/usr/bin/clang++"], - } - ) + paths.update({ + "/usr/bin/afl-clang-fast": paths["/usr/bin/clang"], + "/usr/bin/afl-clang-fast++": paths["/usr/bin/clang++"], + }) self.do_toolchain_test( paths, { @@ -884,13 +874,11 @@ class OSXToolchainTest(BaseToolchainTest): GCC_10_RESULT = LinuxToolchainTest.GCC_10_RESULT GXX_10_RESULT = LinuxToolchainTest.GXX_10_RESULT SYSROOT_FLAGS = { - "flags": PrependFlags( - [ - "-isysroot", - xcrun("", ("--show-sdk-path",))[1], - "-mmacosx-version-min=10.15", - ] - ) + "flags": PrependFlags([ + "-isysroot", + xcrun("", ("--show-sdk-path",))[1], + "-mmacosx-version-min=10.15", + ]) } def test_clang(self): @@ -1362,12 +1350,10 @@ class LinuxCrossCompileToolchainTest(BaseToolchainTest): }, ) - paths.update( - { - "%s-gcc" % toolchain_prefix: DEFAULT_GCC + self.PLATFORMS[target], - "%s-g++" % toolchain_prefix: DEFAULT_GXX + self.PLATFORMS[target], - } - ) + paths.update({ + "%s-gcc" % toolchain_prefix: DEFAULT_GCC + self.PLATFORMS[target], + "%s-g++" % toolchain_prefix: DEFAULT_GXX + self.PLATFORMS[target], + }) self.do_toolchain_test( paths, { @@ -1494,12 +1480,10 @@ class LinuxCrossCompileToolchainTest(BaseToolchainTest): def test_cross_atypical_clang(self): paths = dict(self.PATHS) - paths.update( - { - "/usr/bin/afl-clang-fast": paths["/usr/bin/clang"], - "/usr/bin/afl-clang-fast++": paths["/usr/bin/clang++"], - } - ) + paths.update({ + "/usr/bin/afl-clang-fast": paths["/usr/bin/clang"], + "/usr/bin/afl-clang-fast++": paths["/usr/bin/clang++"], + }) afl_clang_result = self.DEFAULT_CLANG_RESULT + { "compiler": "/usr/bin/afl-clang-fast" } @@ -1522,12 +1506,10 @@ class LinuxCrossCompileToolchainTest(BaseToolchainTest): class OSXCrossToolchainTest(BaseToolchainTest): TARGET = "i686-apple-darwin11.2.0" PATHS = dict(LinuxToolchainTest.PATHS) - PATHS.update( - { - "/usr/bin/clang": CLANG_17 + CLANG_PLATFORM_X86_64_LINUX, - "/usr/bin/clang++": CLANGXX_17 + CLANG_PLATFORM_X86_64_LINUX, - } - ) + PATHS.update({ + "/usr/bin/clang": CLANG_17 + CLANG_PLATFORM_X86_64_LINUX, + "/usr/bin/clang++": CLANGXX_17 + CLANG_PLATFORM_X86_64_LINUX, + }) DEFAULT_CLANG_RESULT = CompilerResult( flags=[], version="17.0.0", diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py @@ -98,9 +98,10 @@ class TestCompilerPreprocessor(unittest.TestCase): self.assertEqual(pp.out.getvalue(), '1 . 2 . c "D"') def test_normalization(self): - pp = CompilerPreprocessor( - {"__has_attribute(bar)": 1, '__has_warning("-Wc++98-foo")': 1} - ) + pp = CompilerPreprocessor({ + "__has_attribute(bar)": 1, + '__has_warning("-Wc++98-foo")': 1, + }) pp.out = StringIO() input = StringIO( dedent( @@ -268,15 +269,13 @@ class TestFakeCompiler(unittest.TestCase): compiler = FakeCompiler({"A": "1", "B": "2"}) self.assertEqual(compiler(None, ["-E", "file"]), (0, "1 2 C", "")) - compiler = FakeCompiler( - { - None: {"A": "1", "B": "2"}, - "-foo": {"C": "foo"}, - "-bar": {"B": "bar", "C": "bar"}, - "-qux": {"B": False}, - "*.c": {"B": "42"}, - } - ) + compiler = FakeCompiler({ + None: {"A": "1", "B": "2"}, + "-foo": {"C": "foo"}, + "-bar": {"B": "bar", "C": "bar"}, + "-qux": {"B": False}, + "*.c": {"B": "42"}, + }) self.assertEqual(compiler(None, ["-E", "file"]), (0, "1 2 C", "")) self.assertEqual(compiler(None, ["-E", "-foo", "file"]), (0, "1 2 foo", "")) self.assertEqual( diff --git a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py @@ -38,7 +38,7 @@ class TestToolkitMozConfigure(BaseConfigureTest): ) self.assertEqual( - "--enable-application=browser " "MOZ_VTUNE=1", + "--enable-application=browser MOZ_VTUNE=1", get_value_for(["--enable-application=browser", "MOZ_VTUNE=1"]), ) diff --git a/python/mozbuild/mozbuild/test/configure/test_util.py b/python/mozbuild/mozbuild/test/configure/test_util.py @@ -59,7 +59,7 @@ class TestConfigureOutputHandler(unittest.TestCase): logger.debug("qux") self.assertEqual(out.getvalue(), "baz\n") - self.assertEqual(err.getvalue(), "ERROR:foo\n" "WARNING:bar\n") + self.assertEqual(err.getvalue(), "ERROR:foo\nWARNING:bar\n") def test_continuation(self): out = StringIO() @@ -75,7 +75,7 @@ class TestConfigureOutputHandler(unittest.TestCase): logger.info("yes") logger.info("qux") - self.assertEqual(out.getvalue(), "foo\n" "checking bar... yes\n" "qux\n") + self.assertEqual(out.getvalue(), "foo\nchecking bar... yes\nqux\n") out.seek(0) out.truncate() @@ -88,7 +88,7 @@ class TestConfigureOutputHandler(unittest.TestCase): self.assertEqual( out.getvalue(), - "foo\n" "checking bar... \n" "WARNING:hoge\n" " ... no\n" "qux\n", + "foo\nchecking bar... \nWARNING:hoge\n ... no\nqux\n", ) out.seek(0) @@ -103,12 +103,7 @@ class TestConfigureOutputHandler(unittest.TestCase): self.assertEqual( out.getvalue(), - "foo\n" - "checking bar... \n" - "WARNING:hoge\n" - "WARNING:fuga\n" - " ... no\n" - "qux\n", + "foo\nchecking bar... \nWARNING:hoge\nWARNING:fuga\n ... no\nqux\n", ) out.seek(0) @@ -127,9 +122,9 @@ class TestConfigureOutputHandler(unittest.TestCase): logger.info("no") logger.info("qux") - self.assertEqual(out.getvalue(), "foo\n" "checking bar... no\n" "qux\n") + self.assertEqual(out.getvalue(), "foo\nchecking bar... no\nqux\n") - self.assertEqual(err.getvalue(), "WARNING:hoge\n" "WARNING:fuga\n") + self.assertEqual(err.getvalue(), "WARNING:hoge\nWARNING:fuga\n") def test_queue_debug(self): out = StringIO() @@ -146,7 +141,7 @@ class TestConfigureOutputHandler(unittest.TestCase): logger.info("yes") logger.info("qux") - self.assertEqual(out.getvalue(), "checking bar... yes\n" "qux\n") + self.assertEqual(out.getvalue(), "checking bar... yes\nqux\n") out.seek(0) out.truncate() @@ -158,7 +153,7 @@ class TestConfigureOutputHandler(unittest.TestCase): logger.error("fail") self.assertEqual( - out.getvalue(), "checking bar... no\n" "DEBUG:do foo\n" "ERROR:fail\n" + out.getvalue(), "checking bar... no\nDEBUG:do foo\nERROR:fail\n" ) out.seek(0) @@ -219,7 +214,7 @@ class TestConfigureOutputHandler(unittest.TestCase): self.assertIs(caught, e) self.assertEqual( - out.getvalue(), "checking bar... no\n" "DEBUG:do foo\n" "DEBUG:do bar\n" + out.getvalue(), "checking bar... no\nDEBUG:do foo\nDEBUG:do bar\n" ) def test_queue_debug_reentrant(self): @@ -245,7 +240,7 @@ class TestConfigureOutputHandler(unittest.TestCase): self.assertEqual( out.getvalue(), - "outer info\n" "inner info\n" "DEBUG| outer debug\n" "DEBUG| inner debug\n", + "outer info\ninner info\nDEBUG| outer debug\nDEBUG| inner debug\n", ) out.seek(0) @@ -265,7 +260,7 @@ class TestConfigureOutputHandler(unittest.TestCase): self.assertEqual( out.getvalue(), - "outer info\n" "inner info\n" "DEBUG| outer debug\n" "DEBUG| inner debug\n", + "outer info\ninner info\nDEBUG| outer debug\nDEBUG| inner debug\n", ) out.seek(0) @@ -406,7 +401,7 @@ class TestLogSubprocessOutput(unittest.TestCase): self.assertEqual(status, 0) quote_char = "'" if getpreferredencoding().lower() == "utf-8": - quote_char = "\u00B4" + quote_char = "\u00b4" self.assertEqual(out.getvalue().strip(), quote_char) diff --git a/python/mozbuild/mozbuild/test/frontend/test_context.py b/python/mozbuild/mozbuild/test/frontend/test_context.py @@ -29,13 +29,11 @@ from mozbuild.util import StrictOrderingOnAppendListWithFlagsFactory class TestContext(unittest.TestCase): def test_defaults(self): - test = Context( - { - "foo": (int, int, ""), - "bar": (bool, bool, ""), - "baz": (dict, dict, ""), - } - ) + test = Context({ + "foo": (int, int, ""), + "bar": (bool, bool, ""), + "baz": (dict, dict, ""), + }) self.assertEqual(list(test), []) @@ -57,12 +55,10 @@ class TestContext(unittest.TestCase): self.assertEqual(set(test.keys()), {"foo", "bar", "baz"}) def test_type_check(self): - test = Context( - { - "foo": (int, int, ""), - "baz": (dict, list, ""), - } - ) + test = Context({ + "foo": (int, int, ""), + "baz": (dict, list, ""), + }) test["foo"] = 5 @@ -81,13 +77,11 @@ class TestContext(unittest.TestCase): self.assertEqual(test["baz"], {"a": 1, "b": 2}) def test_update(self): - test = Context( - { - "foo": (int, int, ""), - "bar": (bool, bool, ""), - "baz": (dict, list, ""), - } - ) + test = Context({ + "foo": (int, int, ""), + "bar": (bool, bool, ""), + "baz": (dict, list, ""), + }) self.assertEqual(list(test), []) @@ -548,11 +542,9 @@ class TestPaths(unittest.TestCase): MyListWithFlags = ContextDerivedTypedListWithItems( Path, - StrictOrderingOnAppendListWithFlagsFactory( - { - "foo": bool, - } - ), + StrictOrderingOnAppendListWithFlagsFactory({ + "foo": bool, + }), ) l = MyListWithFlags(ctxt1) l += paths diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -1008,7 +1008,7 @@ class TestEmitterBasic(unittest.TestCase): with self.assertRaisesRegex( SandboxValidationError, "entry in generated-files not present elsewhere" ): - self.read_topsrcdir(reader), + (self.read_topsrcdir(reader),) def test_test_manifest_parent_support_files_dir(self): """support-files referencing a file in a parent directory works.""" @@ -1058,9 +1058,12 @@ class TestEmitterBasic(unittest.TestCase): mozpath.relpath(p, ipdl_collection.topsrcdir) for p in ipdl_collection.all_regular_sources() ) - expected = set( - ["bar/bar.ipdl", "bar/bar2.ipdlh", "foo/foo.ipdl", "foo/foo2.ipdlh"] - ) + expected = set([ + "bar/bar.ipdl", + "bar/bar2.ipdlh", + "foo/foo.ipdl", + "foo/foo2.ipdlh", + ]) self.assertEqual(ipdls, expected) diff --git a/python/mozbuild/mozbuild/test/frontend/test_namespaces.py b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py @@ -64,11 +64,9 @@ VARIABLES = { "HOGEHOGE": ( ContextDerivedTypedListWithItems( Piyo, - StrictOrderingOnAppendListWithFlagsFactory( - { - "foo": bool, - } - ), + StrictOrderingOnAppendListWithFlagsFactory({ + "foo": bool, + }), ), list, None, diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py @@ -289,9 +289,9 @@ class TestBuildReader(unittest.TestCase): ) # Missing moz.build from missing intermediate directory. - paths = reader._find_relevant_mozbuilds( - ["d1/no-intermediate-moz-build/child/file"] - ) + paths = reader._find_relevant_mozbuilds([ + "d1/no-intermediate-moz-build/child/file" + ]) self.assertEqual( paths, { @@ -304,9 +304,9 @@ class TestBuildReader(unittest.TestCase): ) # Lots of empty directories. - paths = reader._find_relevant_mozbuilds( - ["d1/parent-is-far/dir1/dir2/dir3/file"] - ) + paths = reader._find_relevant_mozbuilds([ + "d1/parent-is-far/dir1/dir2/dir3/file" + ]) self.assertEqual( paths, { @@ -319,9 +319,10 @@ class TestBuildReader(unittest.TestCase): ) # Lots of levels. - paths = reader._find_relevant_mozbuilds( - ["d1/every-level/a/file", "d1/every-level/b/file"] - ) + paths = reader._find_relevant_mozbuilds([ + "d1/every-level/a/file", + "d1/every-level/b/file", + ]) self.assertEqual( paths, { @@ -354,9 +355,11 @@ class TestBuildReader(unittest.TestCase): def test_read_relevant_mozbuilds(self): reader = self.reader("reader-relevant-mozbuild") - paths, contexts = reader.read_relevant_mozbuilds( - ["d1/every-level/a/file", "d1/every-level/b/file", "d2/file"] - ) + paths, contexts = reader.read_relevant_mozbuilds([ + "d1/every-level/a/file", + "d1/every-level/b/file", + "d2/file", + ]) self.assertEqual(len(paths), 3) self.assertEqual(len(contexts), 6) @@ -387,13 +390,11 @@ class TestBuildReader(unittest.TestCase): def test_files_bug_component_static(self): reader = self.reader("files-info") - v = reader.files_info( - [ - "bug_component/static/foo", - "bug_component/static/bar", - "bug_component/static/foo/baz", - ] - ) + v = reader.files_info([ + "bug_component/static/foo", + "bug_component/static/bar", + "bug_component/static/foo/baz", + ]) self.assertEqual(len(v), 3) self.assertEqual( v["bug_component/static/foo"]["BUG_COMPONENT"], @@ -420,13 +421,11 @@ class TestBuildReader(unittest.TestCase): def test_files_bug_component_different_matchers(self): reader = self.reader("files-info") - v = reader.files_info( - [ - "bug_component/different-matchers/foo.jsm", - "bug_component/different-matchers/bar.cpp", - "bug_component/different-matchers/baz.misc", - ] - ) + v = reader.files_info([ + "bug_component/different-matchers/foo.jsm", + "bug_component/different-matchers/bar.cpp", + "bug_component/different-matchers/baz.misc", + ]) self.assertEqual(len(v), 3) js_flags = v["bug_component/different-matchers/foo.jsm"] @@ -445,14 +444,12 @@ class TestBuildReader(unittest.TestCase): def test_files_bug_component_final(self): reader = self.reader("files-info") - v = reader.files_info( - [ - "bug_component/final/foo", - "bug_component/final/Makefile.in", - "bug_component/final/subcomponent/Makefile.in", - "bug_component/final/subcomponent/bar", - ] - ) + v = reader.files_info([ + "bug_component/final/foo", + "bug_component/final/Makefile.in", + "bug_component/final/subcomponent/Makefile.in", + "bug_component/final/subcomponent/bar", + ]) self.assertEqual( v["bug_component/final/foo"]["BUG_COMPONENT"], @@ -479,17 +476,15 @@ class TestBuildReader(unittest.TestCase): def test_schedules(self): reader = self.reader("schedules") - info = reader.files_info( - [ - "win.and.osx", - "somefile", - "foo.win", - "foo.osx", - "subd/aa.py", - "subd/yaml.py", - "subd/win.js", - ] - ) + info = reader.files_info([ + "win.and.osx", + "somefile", + "foo.win", + "foo.osx", + "subd/aa.py", + "subd/yaml.py", + "subd/win.js", + ]) # default: all exclusive, no inclusive self.assertEqual(info["somefile"]["SCHEDULES"].inclusive, []) self.assertEqual( diff --git a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py @@ -25,11 +25,9 @@ test_data_path = mozpath.join(test_data_path, "data") class TestSandbox(unittest.TestCase): def sandbox(self): return Sandbox( - Context( - { - "DIRS": (list, list, None), - } - ) + Context({ + "DIRS": (list, list, None), + }) ) def test_exec_source_success(self): @@ -498,11 +496,9 @@ def Template(): def foo(a, b): return type(a), type(b) - FUNCTIONS.update( - { - "foo": (lambda self: foo, (Foo, int), ""), - } - ) + FUNCTIONS.update({ + "foo": (lambda self: foo, (Foo, int), ""), + }) try: sandbox = self.sandbox() diff --git a/python/mozbuild/mozbuild/test/repackaging/test_utils.py b/python/mozbuild/mozbuild/test/repackaging/test_utils.py @@ -329,7 +329,10 @@ def test_copy_plain_config(monkeypatch): def test_render_templates(): - with tempfile.TemporaryDirectory() as template_dir, tempfile.TemporaryDirectory() as source_dir: + with ( + tempfile.TemporaryDirectory() as template_dir, + tempfile.TemporaryDirectory() as source_dir, + ): with open(os.path.join(template_dir, "debian_file1.in"), "w") as f: f.write("${some_build_variable}") @@ -611,7 +614,10 @@ def test_prepare_langpack_files(monkeypatch, languages, remoting_name, expected) monkeypatch.setattr(utils.shutil, "copy", _mock_copy) - with tempfile.TemporaryDirectory() as xpi_dir, tempfile.TemporaryDirectory() as output_dir: + with ( + tempfile.TemporaryDirectory() as xpi_dir, + tempfile.TemporaryDirectory() as output_dir, + ): for language in languages: path = os.path.join(xpi_dir, f"{language}.langpack.xpi") with zipfile.ZipFile(path, "w") as zip_file: diff --git a/python/mozbuild/mozbuild/test/test_confvars.py b/python/mozbuild/mozbuild/test/test_confvars.py @@ -13,7 +13,6 @@ def TemporaryConfVars(): class TestContext(unittest.TestCase): - def loads(self, *lines): with NamedTemporaryFile("wt", delete=False) as ntf: ntf.writelines(lines) @@ -64,7 +63,7 @@ class TestContext(unittest.TestCase): ) def test_parse_quoted_assignment(self): - confvars = self.loads("a='b'\n" "b=' c'\n" 'c=" \'c"\n') + confvars = self.loads("a='b'\nb=' c'\nc=\" 'c\"\n") self.assertEqual(confvars, {"a": "b", "b": " c", "c": " 'c"}) def test_parse_invalid_assignment(self): diff --git a/python/mozbuild/mozbuild/test/test_legacy_test.py b/python/mozbuild/mozbuild/test/test_legacy_test.py @@ -22,9 +22,9 @@ def test_extra_legacy_tests(): for src, fd in repo.get_tracked_files_finder().find("**/moz.build"): if src in allowlist: continue - assert ( - b"LegacyTest(" not in fd.read() - ), f"LegacyTest used in {src}, please refrain and use another test kind." + assert b"LegacyTest(" not in fd.read(), ( + f"LegacyTest used in {src}, please refrain and use another test kind." + ) if __name__ == "__main__": diff --git a/python/mozbuild/mozbuild/test/test_line_endings.py b/python/mozbuild/mozbuild/test/test_line_endings.py @@ -26,17 +26,17 @@ class TestLineEndings(unittest.TestCase): self.f.flush() def testMac(self): - self.createFile([b"\x0D"] * 3) + self.createFile([b"\x0d"] * 3) self.pp.do_include(self.f.name) self.assertEqual(self.pp.out.getvalue(), "a\nb\nc\n") def testUnix(self): - self.createFile([b"\x0A"] * 3) + self.createFile([b"\x0a"] * 3) self.pp.do_include(self.f.name) self.assertEqual(self.pp.out.getvalue(), "a\nb\nc\n") def testWindows(self): - self.createFile([b"\x0D\x0A"] * 3) + self.createFile([b"\x0d\x0a"] * 3) self.pp.do_include(self.f.name) self.assertEqual(self.pp.out.getvalue(), "a\nb\nc\n") diff --git a/python/mozbuild/mozbuild/test/test_manifest.py b/python/mozbuild/mozbuild/test/test_manifest.py @@ -43,11 +43,10 @@ class TestManifest(unittest.TestCase): "bugzilla": {"component": "Graphics", "product": "Core"}, } - self.process_test_vectors( - [ - ( - simple_dict, - b""" + self.process_test_vectors([ + ( + simple_dict, + b""" --- schema: 1 origin: @@ -63,10 +62,10 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - ( - simple_dict, - b""" + ), + ( + simple_dict, + b""" schema: 1 origin: name: cairo @@ -81,32 +80,30 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - ] - ) + ), + ]) # =========================================================================================== def test_updatebot(self): - self.process_test_vectors( - [ - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - }, + self.process_test_vectors([ + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", }, - b""" + "bugzilla": {"component": "Graphics", "product": "Core"}, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + }, + }, + b""" --- schema: 1 origin: @@ -125,26 +122,26 @@ updatebot: maintainer-phab: tjr maintainer-bz: a@example.com """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "001122334455", + "url": "https://www.cairographics.org/", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", }, - b""" + }, + b""" --- schema: 1 origin: @@ -163,27 +160,27 @@ updatebot: maintainer-phab: tjr maintainer-bz: a@example.com """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "updatebot": { - "try-preset": "foo", - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "001122334455", + "url": "https://www.cairographics.org/", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "updatebot": { + "try-preset": "foo", + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", }, - b""" + }, + b""" --- schema: 1 origin: @@ -203,32 +200,32 @@ updatebot: maintainer-phab: tjr maintainer-bz: a@example.com """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "fuzzy-query": "!linux64", - "tasks": [{"type": "commit-alert"}], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", }, - b""" + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + }, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "fuzzy-query": "!linux64", + "tasks": [{"type": "commit-alert"}], + }, + }, + b""" --- schema: 1 origin: @@ -253,11 +250,11 @@ updatebot: tasks: - type: commit-alert """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -283,32 +280,32 @@ updatebot: tasks: - type: commit-alert """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "fuzzy-paths": ["dir1/", "dir2"], - "tasks": [{"type": "commit-alert"}], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", }, - b""" + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "fuzzy-paths": ["dir1/", "dir2"], + "tasks": [{"type": "commit-alert"}], + }, + }, + b""" --- schema: 1 origin: @@ -335,32 +332,32 @@ updatebot: tasks: - type: commit-alert """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "fuzzy-paths": ["dir1/"], - "tasks": [{"type": "commit-alert"}], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + }, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "fuzzy-paths": ["dir1/"], + "tasks": [{"type": "commit-alert"}], }, - b""" + }, + b""" --- schema: 1 origin: @@ -385,43 +382,43 @@ updatebot: tasks: - type: commit-alert """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - "tracking": "commit", - "flavor": "rust", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "tasks": [ - {"type": "commit-alert", "frequency": "release"}, - { - "type": "vendoring", - "enabled": False, - "cc": ["b@example.com"], - "needinfo": ["c@example.com"], - "frequency": "1 weeks", - "platform": "windows", - }, - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + "tracking": "commit", + "flavor": "rust", + }, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "tasks": [ + {"type": "commit-alert", "frequency": "release"}, + { + "type": "vendoring", + "enabled": False, + "cc": ["b@example.com"], + "needinfo": ["c@example.com"], + "frequency": "1 weeks", + "platform": "windows", + }, + ], }, - b""" + }, + b""" --- schema: 1 origin: @@ -454,43 +451,43 @@ updatebot: frequency: 1 weeks platform: windows """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - "tracking": "tag", - "flavor": "rust", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "tasks": [ - {"type": "commit-alert", "frequency": "release"}, - { - "type": "vendoring", - "enabled": False, - "cc": ["b@example.com"], - "needinfo": ["c@example.com"], - "frequency": "1 weeks, 4 commits", - "platform": "windows", - }, - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", }, - b""" + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + "tracking": "tag", + "flavor": "rust", + }, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "tasks": [ + {"type": "commit-alert", "frequency": "release"}, + { + "type": "vendoring", + "enabled": False, + "cc": ["b@example.com"], + "needinfo": ["c@example.com"], + "frequency": "1 weeks, 4 commits", + "platform": "windows", + }, + ], + }, + }, + b""" --- schema: 1 origin: @@ -523,11 +520,11 @@ updatebot: frequency: 1 weeks, 4 commits platform: windows """.strip(), - ), - # ------------------------------------------------- - ( - "exception", # rust flavor cannot use update-actions - b""" + ), + # ------------------------------------------------- + ( + "exception", # rust flavor cannot use update-actions + b""" --- schema: 1 origin: @@ -564,46 +561,46 @@ updatebot: frequency: 1 weeks, 4 commits platform: windows """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "tasks": [ - { - "type": "vendoring", - "enabled": False, - "cc": ["b@example.com", "c@example.com"], - "needinfo": ["d@example.com", "e@example.com"], - "frequency": "every", - }, - { - "type": "commit-alert", - "filter": "none", - "source-extensions": [".c", ".cpp"], - "frequency": "2 weeks", - "platform": "linux", - }, - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", }, - b""" + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "tasks": [ + { + "type": "vendoring", + "enabled": False, + "cc": ["b@example.com", "c@example.com"], + "needinfo": ["d@example.com", "e@example.com"], + "frequency": "every", + }, + { + "type": "commit-alert", + "filter": "none", + "source-extensions": [".c", ".cpp"], + "frequency": "2 weeks", + "platform": "linux", + }, + ], + }, + }, + b""" --- schema: 1 origin: @@ -642,46 +639,46 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "tasks": [ - { - "type": "vendoring", - "enabled": False, - "cc": ["b@example.com", "c@example.com"], - "needinfo": ["d@example.com", "e@example.com"], - "frequency": "every", - }, - { - "type": "commit-alert", - "filter": "none", - "source-extensions": [".c", ".cpp"], - "frequency": "2 commits", - "platform": "linux", - }, - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + }, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "tasks": [ + { + "type": "vendoring", + "enabled": False, + "cc": ["b@example.com", "c@example.com"], + "needinfo": ["d@example.com", "e@example.com"], + "frequency": "every", + }, + { + "type": "commit-alert", + "filter": "none", + "source-extensions": [".c", ".cpp"], + "frequency": "2 commits", + "platform": "linux", + }, + ], }, - b""" + }, + b""" --- schema: 1 origin: @@ -720,47 +717,47 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "AA001122334455", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - }, - "updatebot": { - "maintainer-phab": "tjr", - "maintainer-bz": "a@example.com", - "tasks": [ - { - "type": "vendoring", - "enabled": False, - "cc": ["b@example.com", "c@example.com"], - "needinfo": ["d@example.com", "e@example.com"], - "frequency": "every", - "blocking": "1234", - }, - { - "type": "commit-alert", - "filter": "none", - "source-extensions": [".c", ".cpp"], - "frequency": "2 commits", - "platform": "linux", - }, - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "AA001122334455", + "url": "https://www.cairographics.org/", }, - b""" + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + }, + "updatebot": { + "maintainer-phab": "tjr", + "maintainer-bz": "a@example.com", + "tasks": [ + { + "type": "vendoring", + "enabled": False, + "cc": ["b@example.com", "c@example.com"], + "needinfo": ["d@example.com", "e@example.com"], + "frequency": "every", + "blocking": "1234", + }, + { + "type": "commit-alert", + "filter": "none", + "source-extensions": [".c", ".cpp"], + "frequency": "2 commits", + "platform": "linux", + }, + ], + }, + }, + b""" --- schema: 1 origin: @@ -800,11 +797,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -845,30 +842,28 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "description": "2D Graphics Library", - "url": "https://www.cairographics.org/", - "release": "version 1.6.4", - "revision": "AA001122334455", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - "flavor": "individual-files", - "individual-files": [ - {"upstream": "foo", "destination": "bar"} - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "description": "2D Graphics Library", + "url": "https://www.cairographics.org/", + "release": "version 1.6.4", + "revision": "AA001122334455", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + "flavor": "individual-files", + "individual-files": [{"upstream": "foo", "destination": "bar"}], }, - b""" + }, + b""" --- schema: 1 origin: @@ -891,33 +886,31 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "description": "2D Graphics Library", - "url": "https://www.cairographics.org/", - "release": "version 1.6.4", - "revision": "AA001122334455", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - "flavor": "individual-files", - "individual-files": [ - {"upstream": "foo", "destination": "bar"} - ], - "update-actions": [ - {"action": "move-file", "from": "foo", "to": "bar"} - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "description": "2D Graphics Library", + "url": "https://www.cairographics.org/", + "release": "version 1.6.4", + "revision": "AA001122334455", }, - b""" + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + "flavor": "individual-files", + "individual-files": [{"upstream": "foo", "destination": "bar"}], + "update-actions": [ + {"action": "move-file", "from": "foo", "to": "bar"} + ], + }, + }, + b""" --- schema: 1 origin: @@ -944,33 +937,33 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - { - "schema": "1", - "origin": { - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "description": "2D Graphics Library", - "url": "https://www.cairographics.org/", - "release": "version 1.6.4", - "revision": "AA001122334455", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, - "vendoring": { - "url": "https://example.com", - "source-hosting": "gitlab", - "flavor": "individual-files", - "individual-files-default-destination": "bar", - "individual-files-default-upstream": "foo", - "individual-files-list": ["bar", "foo"], - "update-actions": [ - {"action": "move-file", "from": "foo", "to": "bar"} - ], - }, + ), + # ------------------------------------------------- + ( + { + "schema": "1", + "origin": { + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "description": "2D Graphics Library", + "url": "https://www.cairographics.org/", + "release": "version 1.6.4", + "revision": "AA001122334455", + }, + "bugzilla": {"component": "Graphics", "product": "Core"}, + "vendoring": { + "url": "https://example.com", + "source-hosting": "gitlab", + "flavor": "individual-files", + "individual-files-default-destination": "bar", + "individual-files-default-upstream": "foo", + "individual-files-list": ["bar", "foo"], + "update-actions": [ + {"action": "move-file", "from": "foo", "to": "bar"} + ], }, - b""" + }, + b""" --- schema: 1 origin: @@ -999,11 +992,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", # can't have both types of indidivudal-files list - b""" + ), + # ------------------------------------------------- + ( + "exception", # can't have both types of indidivudal-files list + b""" --- schema: 1 origin: @@ -1032,11 +1025,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", # can't have indidivudal-files-default-upstream - b""" + ), + # ------------------------------------------------- + ( + "exception", # can't have indidivudal-files-default-upstream + b""" --- schema: 1 origin: @@ -1064,11 +1057,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", # must have indidivudal-files-default-upstream - b""" + ), + # ------------------------------------------------- + ( + "exception", # must have indidivudal-files-default-upstream + b""" --- schema: 1 origin: @@ -1096,11 +1089,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1124,11 +1117,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1148,11 +1141,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1175,11 +1168,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1201,11 +1194,11 @@ bugzilla: product: Core component: Graphics """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1245,11 +1238,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1289,11 +1282,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1332,11 +1325,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1375,11 +1368,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1419,11 +1412,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1463,11 +1456,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1506,11 +1499,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1540,11 +1533,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1576,11 +1569,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1605,11 +1598,11 @@ updatebot: - type: vendoring filter: none """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1633,11 +1626,11 @@ updatebot: tasks: - type: foo """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1664,11 +1657,11 @@ updatebot: - .c - .cpp """.strip(), - ), - # ------------------------------------------------- - ( - "exception", - b""" + ), + # ------------------------------------------------- + ( + "exception", + b""" --- schema: 1 origin: @@ -1693,10 +1686,10 @@ updatebot: - type: commit-alert filter: hogwash """.strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1729,10 +1722,10 @@ updatebot: source-extensions: - .c - .cpp""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1765,10 +1758,10 @@ updatebot: source-extensions: - .c - .cpp""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1801,10 +1794,10 @@ updatebot: source-extensions: - .c - .cpp""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1837,10 +1830,10 @@ updatebot: source-extensions: - .c - .cpp""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1865,9 +1858,8 @@ updatebot: - type: commit-alert frequency: 0 weeks """.strip(), - ), - ] - ) + ), + ]) # =========================================================================================== def test_malformed(self): @@ -1898,22 +1890,21 @@ updatebot: load_moz_yaml(tf.name, require_license_file=False) def test_revision(self): - self.process_test_vectors( - [ - ( - { - "schema": "1", - "origin": { - "description": "2D Graphics Library", - "license": ["MPL-1.1", "LGPL-2.1"], - "name": "cairo", - "release": "version 1.6.4", - "revision": "v1.6.37", - "url": "https://www.cairographics.org/", - }, - "bugzilla": {"component": "Graphics", "product": "Core"}, + self.process_test_vectors([ + ( + { + "schema": "1", + "origin": { + "description": "2D Graphics Library", + "license": ["MPL-1.1", "LGPL-2.1"], + "name": "cairo", + "release": "version 1.6.4", + "revision": "v1.6.37", + "url": "https://www.cairographics.org/", }, - b""" + "bugzilla": {"component": "Graphics", "product": "Core"}, + }, + b""" --- schema: 1 origin: @@ -1928,10 +1919,10 @@ origin: bugzilla: product: Core component: Graphics""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1946,10 +1937,10 @@ origin: bugzilla: product: Core component: Graphics""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1964,10 +1955,10 @@ origin: bugzilla: product: Core component: Graphics""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -1982,10 +1973,10 @@ origin: bugzilla: product: Core component: Graphics""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -2000,10 +1991,10 @@ origin: bugzilla: product: Core component: Graphics""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -2018,10 +2009,10 @@ origin: bugzilla: product: Core component: Graphics""".strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -2046,10 +2037,10 @@ vendoring: args: ['hi'] pattern: 'hi' """.strip(), - ), - ( - "exception", - b""" + ), + ( + "exception", + b""" --- schema: 1 origin: @@ -2072,9 +2063,8 @@ vendoring: cwd: '{cwd}' args: ['hi'] """.strip(), - ), - ] - ) + ), + ]) if __name__ == "__main__": diff --git a/python/mozbuild/mozbuild/test/test_preprocessor.py b/python/mozbuild/mozbuild/test/test_preprocessor.py @@ -34,15 +34,13 @@ class TestPreprocessor(unittest.TestCase): self.do_include_compare(content_lines, ["PASS"]) def test_conditional_if_0(self): - self.do_include_pass( - [ - "#if 0", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#if 0", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_no_marker(self): lines = [ @@ -80,99 +78,83 @@ class TestPreprocessor(unittest.TestCase): ) def test_conditional_if_0_elif_1(self): - self.do_include_pass( - [ - "#if 0", - "#elif 1", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if 0", + "#elif 1", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_conditional_if_1(self): - self.do_include_pass( - [ - "#if 1", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if 1", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_conditional_if_0_or_1(self): - self.do_include_pass( - [ - "#if 0 || 1", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if 0 || 1", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_conditional_if_1_elif_1_else(self): - self.do_include_pass( - [ - "#if 1", - "PASS", - "#elif 1", - "FAIL", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if 1", + "PASS", + "#elif 1", + "FAIL", + "#else", + "FAIL", + "#endif", + ]) def test_conditional_if_1_if_1(self): - self.do_include_pass( - [ - "#if 1", - "#if 1", - "PASS", - "#else", - "FAIL", - "#endif", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if 1", + "#if 1", + "PASS", + "#else", + "FAIL", + "#endif", + "#else", + "FAIL", + "#endif", + ]) def test_conditional_not_0(self): - self.do_include_pass( - [ - "#if !0", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if !0", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_conditional_not_0_and_1(self): - self.do_include_pass( - [ - "#if !0 && !1", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#if !0 && !1", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_conditional_not_1(self): - self.do_include_pass( - [ - "#if !1", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#if !1", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_conditional_not_emptyval(self): self.do_include_compare( @@ -193,36 +175,30 @@ class TestPreprocessor(unittest.TestCase): ) def test_conditional_not_nullval(self): - self.do_include_pass( - [ - "#define NULLVAL 0", - "#if !NULLVAL", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#define NULLVAL 0", + "#if !NULLVAL", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_indentation(self): - self.do_include_pass( - [ - " #define NULLVAL 0", - " #if !NULLVAL", - "PASS", - " #else", - "FAIL", - " #endif", - ] - ) + self.do_include_pass([ + " #define NULLVAL 0", + " #if !NULLVAL", + "PASS", + " #else", + "FAIL", + " #endif", + ]) def test_expand(self): - self.do_include_pass( - [ - "#define ASVAR AS", - "#expand P__ASVAR__S", - ] - ) + self.do_include_pass([ + "#define ASVAR AS", + "#expand P__ASVAR__S", + ]) def test_undef_defined(self): self.do_include_compare( @@ -330,14 +306,12 @@ class TestPreprocessor(unittest.TestCase): ) def test_filter_substitution(self): - self.do_include_pass( - [ - "#define VAR ASS", - "#filter substitution", - "P@VAR@", - "#unfilter substitution", - ] - ) + self.do_include_pass([ + "#define VAR ASS", + "#filter substitution", + "P@VAR@", + "#unfilter substitution", + ]) def test_error(self): with MockedOpen({"f": "#error spit this message out\n"}): @@ -361,265 +335,223 @@ class TestPreprocessor(unittest.TestCase): full = os.path.join(tmpdir, "javascript_line.js.in") with open(full, "w") as fh: fh.write( - "\n".join( - [ - "// Line 1", - "#if 0", - "// line 3", - "#endif", - "// line 5", - "# comment", - "// line 7", - "// line 8", - "// line 9", - "# another comment", - "// line 11", - "#define LINE 1", - "// line 13, given line number overwritten with 2", - "", - ] - ) + "\n".join([ + "// Line 1", + "#if 0", + "// line 3", + "#endif", + "// line 5", + "# comment", + "// line 7", + "// line 8", + "// line 9", + "# another comment", + "// line 11", + "#define LINE 1", + "// line 13, given line number overwritten with 2", + "", + ]) ) self.pp.do_include(full) - out = "\n".join( - [ - "// Line 1", - '//@line 5 "CWDjavascript_line.js.in"', - "// line 5", - '//@line 7 "CWDjavascript_line.js.in"', - "// line 7", - "// line 8", - "// line 9", - '//@line 11 "CWDjavascript_line.js.in"', - "// line 11", - '//@line 2 "CWDjavascript_line.js.in"', - "// line 13, given line number overwritten with 2", - "", - ] - ) + out = "\n".join([ + "// Line 1", + '//@line 5 "CWDjavascript_line.js.in"', + "// line 5", + '//@line 7 "CWDjavascript_line.js.in"', + "// line 7", + "// line 8", + "// line 9", + '//@line 11 "CWDjavascript_line.js.in"', + "// line 11", + '//@line 2 "CWDjavascript_line.js.in"', + "// line 13, given line number overwritten with 2", + "", + ]) out = out.replace("CWD", tmpdir + os.path.sep) self.assertEqual(self.pp.out.getvalue(), out) finally: shutil.rmtree(tmpdir) def test_literal(self): - self.do_include_pass( - [ - "#literal PASS", - ] - ) + self.do_include_pass([ + "#literal PASS", + ]) def test_var_directory(self): - self.do_include_pass( - [ - "#ifdef DIRECTORY", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#ifdef DIRECTORY", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_var_file(self): - self.do_include_pass( - [ - "#ifdef FILE", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#ifdef FILE", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_var_if_0(self): - self.do_include_pass( - [ - "#define VAR 0", - "#if VAR", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#define VAR 0", + "#if VAR", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_var_if_0_elifdef(self): - self.do_include_pass( - [ - "#if 0", - "#elifdef FILE", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if 0", + "#elifdef FILE", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_var_if_0_elifndef(self): - self.do_include_pass( - [ - "#if 0", - "#elifndef VAR", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if 0", + "#elifndef VAR", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_var_ifdef_0(self): - self.do_include_pass( - [ - "#define VAR 0", - "#ifdef VAR", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#define VAR 0", + "#ifdef VAR", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_var_ifdef_1_or_undef(self): - self.do_include_pass( - [ - "#define FOO 1", - "#if defined(FOO) || defined(BAR)", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#define FOO 1", + "#if defined(FOO) || defined(BAR)", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_var_ifdef_undef(self): - self.do_include_pass( - [ - "#define VAR 0", - "#undef VAR", - "#ifdef VAR", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#define VAR 0", + "#undef VAR", + "#ifdef VAR", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_var_ifndef_0(self): - self.do_include_pass( - [ - "#define VAR 0", - "#ifndef VAR", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#define VAR 0", + "#ifndef VAR", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_var_ifndef_0_and_undef(self): - self.do_include_pass( - [ - "#define FOO 0", - "#if !defined(FOO) && !defined(BAR)", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#define FOO 0", + "#if !defined(FOO) && !defined(BAR)", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_var_ifndef_undef(self): - self.do_include_pass( - [ - "#define VAR 0", - "#undef VAR", - "#ifndef VAR", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#define VAR 0", + "#undef VAR", + "#ifndef VAR", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_var_line(self): - self.do_include_pass( - [ - "#ifdef LINE", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#ifdef LINE", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_filterDefine(self): - self.do_include_pass( - [ - "#filter substitution", - "#define VAR AS", - "#define VAR2 P@VAR@", - "@VAR2@S", - ] - ) + self.do_include_pass([ + "#filter substitution", + "#define VAR AS", + "#define VAR2 P@VAR@", + "@VAR2@S", + ]) def test_number_value_equals(self): - self.do_include_pass( - [ - "#define FOO 1000", - "#if FOO == 1000", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#define FOO 1000", + "#if FOO == 1000", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_default_defines(self): self.pp.handleCommandLine(["-DFOO"]) - self.do_include_pass( - [ - "#if FOO == 1", - "PASS", - "#else", - "FAIL", - ] - ) + self.do_include_pass([ + "#if FOO == 1", + "PASS", + "#else", + "FAIL", + ]) def test_number_value_equals_defines(self): self.pp.handleCommandLine(["-DFOO=1000"]) - self.do_include_pass( - [ - "#if FOO == 1000", - "PASS", - "#else", - "FAIL", - ] - ) + self.do_include_pass([ + "#if FOO == 1000", + "PASS", + "#else", + "FAIL", + ]) def test_octal_value_equals(self): - self.do_include_pass( - [ - "#define FOO 0100", - "#if FOO == 0100", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#define FOO 0100", + "#if FOO == 0100", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_octal_value_equals_defines(self): self.pp.handleCommandLine(["-DFOO=0100"]) - self.do_include_pass( - [ - "#if FOO == 0100", - "PASS", - "#else", - "FAIL", - "#endif", - ] - ) + self.do_include_pass([ + "#if FOO == 0100", + "PASS", + "#else", + "FAIL", + "#endif", + ]) def test_value_quoted_expansion(self): """ @@ -647,27 +579,23 @@ class TestPreprocessor(unittest.TestCase): def test_number_value_not_equals_quoted_defines(self): self.pp.handleCommandLine(['-DFOO="1000"']) - self.do_include_pass( - [ - "#if FOO == 1000", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#if FOO == 1000", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_octal_value_not_equals_quoted_defines(self): self.pp.handleCommandLine(['-DFOO="0100"']) - self.do_include_pass( - [ - "#if FOO == 0100", - "FAIL", - "#else", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#if FOO == 0100", + "FAIL", + "#else", + "PASS", + "#endif", + ]) def test_undefined_variable(self): with MockedOpen({"f": "#filter substitution\n@foo@"}): @@ -677,29 +605,23 @@ class TestPreprocessor(unittest.TestCase): def test_include(self): files = { - "foo/test": "\n".join( - [ - "#define foo foobarbaz", - "#include @inc@", - "@bar@", - "", - ] - ), - "bar": "\n".join( - [ - "#define bar barfoobaz", - "@foo@", - "", - ] - ), - "f": "\n".join( - [ - "#filter substitution", - "#define inc ../bar", - "#include foo/test", - "", - ] - ), + "foo/test": "\n".join([ + "#define foo foobarbaz", + "#include @inc@", + "@bar@", + "", + ]), + "bar": "\n".join([ + "#define bar barfoobaz", + "@foo@", + "", + ]), + "f": "\n".join([ + "#filter substitution", + "#define inc ../bar", + "#include foo/test", + "", + ]), } with MockedOpen(files): @@ -708,41 +630,33 @@ class TestPreprocessor(unittest.TestCase): def test_include_line(self): files = { - "srcdir/test.js": "\n".join( - [ - "#define foo foobarbaz", - "#include @inc@", - "@bar@", - "", - ] - ), - "srcdir/bar.js": "\n".join( - [ - "#define bar barfoobaz", - "@foo@", - "", - ] - ), - "srcdir/foo.js": "\n".join( - [ - "bazfoobar", - "#include bar.js", - "bazbarfoo", - "", - ] - ), + "srcdir/test.js": "\n".join([ + "#define foo foobarbaz", + "#include @inc@", + "@bar@", + "", + ]), + "srcdir/bar.js": "\n".join([ + "#define bar barfoobaz", + "@foo@", + "", + ]), + "srcdir/foo.js": "\n".join([ + "bazfoobar", + "#include bar.js", + "bazbarfoo", + "", + ]), "objdir/baz.js": "baz\n", - "srcdir/f.js": "\n".join( - [ - "#include foo.js", - "#filter substitution", - "#define inc bar.js", - "#include test.js", - "#include ../objdir/baz.js", - "fin", - "", - ] - ), + "srcdir/f.js": "\n".join([ + "#include foo.js", + "#filter substitution", + "#define inc bar.js", + "#include test.js", + "#include ../objdir/baz.js", + "fin", + "", + ]), } preprocessed = ( @@ -819,13 +733,11 @@ class TestPreprocessor(unittest.TestCase): self.assertEqual(e.exception.key, "INVALID_VAR") # Trailing whitespaces, while not nice, shouldn't be an error. - self.do_include_pass( - [ - "#ifndef FOO ", - "PASS", - "#endif", - ] - ) + self.do_include_pass([ + "#ifndef FOO ", + "PASS", + "#endif", + ]) if __name__ == "__main__": diff --git a/python/mozbuild/mozbuild/test/test_pythonutil.py b/python/mozbuild/mozbuild/test/test_pythonutil.py @@ -12,12 +12,10 @@ from mozbuild.pythonutil import iter_modules_in_path def test_iter_modules_in_path(): tests_path = os.path.normcase(os.path.dirname(__file__)) paths = list(iter_modules_in_path(tests_path)) - assert set(paths) == set( - [ - os.path.join(os.path.abspath(tests_path), "__init__.py"), - os.path.join(os.path.abspath(tests_path), "test_pythonutil.py"), - ] - ) + assert set(paths) == set([ + os.path.join(os.path.abspath(tests_path), "__init__.py"), + os.path.join(os.path.abspath(tests_path), "test_pythonutil.py"), + ]) if __name__ == "__main__": diff --git a/python/mozbuild/mozbuild/test/test_telemetry_settings.py b/python/mozbuild/mozbuild/test/test_telemetry_settings.py @@ -94,17 +94,15 @@ bar = 1 def _initialize_telemetry(settings, is_employee, contributor_prompt_response=None): - with mock.patch( - "mach.telemetry.resolve_is_employee", return_value=is_employee - ), mock.patch( - "mach.telemetry.prompt_telemetry_message_contributor", - return_value=contributor_prompt_response, - ) as prompt_mock, mock.patch( - "subprocess.run", return_value=Mock(returncode=0) - ), mock.patch( - "mach.config.ConfigSettings" - ), mock.patch( - "mach.telemetry.record_is_employee_telemetry_setting" + with ( + mock.patch("mach.telemetry.resolve_is_employee", return_value=is_employee), + mock.patch( + "mach.telemetry.prompt_telemetry_message_contributor", + return_value=contributor_prompt_response, + ) as prompt_mock, + mock.patch("subprocess.run", return_value=Mock(returncode=0)), + mock.patch("mach.config.ConfigSettings"), + mock.patch("mach.telemetry.record_is_employee_telemetry_setting"), ): initialize_telemetry_setting(settings, "", "") return prompt_mock.call_count == 1 @@ -144,16 +142,17 @@ def test_initialize_noop_when_telemetry_disabled_env(monkeypatch): def test_initialize_when_request_error_falls_back_to_vcs(settings, monkeypatch): monkeypatch.delenv("MOZ_AUTOMATION", raising=False) - with mock.patch( - "requests.get", - side_effect=requests.exceptions.RequestException("Unlucky"), - ), mock.patch( - "mach.telemetry.resolve_is_employee_by_vcs", return_value=True - ), mock.patch( - "mach.telemetry.prompt_telemetry_message_contributor" - ) as prompt_mock, mock.patch( - "mach.telemetry.record_telemetry_settings" - ) as record_mock: + with ( + mock.patch( + "requests.get", + side_effect=requests.exceptions.RequestException("Unlucky"), + ), + mock.patch("mach.telemetry.resolve_is_employee_by_vcs", return_value=True), + mock.patch( + "mach.telemetry.prompt_telemetry_message_contributor" + ) as prompt_mock, + mock.patch("mach.telemetry.record_telemetry_settings") as record_mock, + ): initialize_telemetry_setting(settings, "", "") assert prompt_mock.call_count == 0 assert record_mock.call_count == 1 @@ -165,13 +164,16 @@ def test_resolve_is_employee(tmpdir, monkeypatch): monkeypatch.delenv("MOZ_AUTOMATION", raising=False) def mock_and_run(is_employee_bugzilla, is_employee_vcs): - with mock.patch( - "mach.telemetry.resolve_is_employee_by_credentials", - return_value=is_employee_bugzilla, - ), mock.patch( - "mach.telemetry.resolve_is_employee_by_vcs", return_value=is_employee_vcs - ), mock.patch( - "mach.telemetry.record_is_employee_telemetry_setting" + with ( + mock.patch( + "mach.telemetry.resolve_is_employee_by_credentials", + return_value=is_employee_bugzilla, + ), + mock.patch( + "mach.telemetry.resolve_is_employee_by_vcs", + return_value=is_employee_vcs, + ), + mock.patch("mach.telemetry.record_is_employee_telemetry_setting"), ): fake_settings = Mock() fake_settings.mach_telemetry.is_employee = None @@ -191,14 +193,16 @@ def test_resolve_is_employee_no_cache_when_unknown(tmpdir, monkeypatch): """Test that cache is not updated when employee status cannot be determined.""" monkeypatch.delenv("MOZ_AUTOMATION", raising=False) - with mock.patch( - "mach.telemetry.resolve_is_employee_by_credentials", - return_value=None, - ), mock.patch( - "mach.telemetry.resolve_is_employee_by_vcs", return_value=None - ), mock.patch( - "mach.telemetry.record_is_employee_telemetry_setting" - ) as record_mock: + with ( + mock.patch( + "mach.telemetry.resolve_is_employee_by_credentials", + return_value=None, + ), + mock.patch("mach.telemetry.resolve_is_employee_by_vcs", return_value=None), + mock.patch( + "mach.telemetry.record_is_employee_telemetry_setting" + ) as record_mock, + ): fake_settings = Mock() fake_settings.mach_telemetry.is_employee = None result = resolve_is_employee(None, str(tmpdir), fake_settings) diff --git a/python/mozbuild/mozbuild/test/test_util.py b/python/mozbuild/mozbuild/test/test_util.py @@ -447,12 +447,10 @@ class TestStrictOrderingOnAppendListWithAction(unittest.TestCase): class TestStrictOrderingOnAppendListWithFlagsFactory(unittest.TestCase): def test_strict_ordering_on_append_list_with_flags_factory(self): - cls = StrictOrderingOnAppendListWithFlagsFactory( - { - "foo": bool, - "bar": int, - } - ) + cls = StrictOrderingOnAppendListWithFlagsFactory({ + "foo": bool, + "bar": int, + }) l = cls() l += ["a", "b"] diff --git a/python/mozbuild/mozbuild/test/test_vendor.py b/python/mozbuild/mozbuild/test/test_vendor.py @@ -32,16 +32,14 @@ def test_up_to_date_vendor(): # Verify that re-vendoring did not cause file changes. # Note that we don't want hg-ignored generated files # to bust the diff, so we exclude them (pycache, egg-info). - subprocess.check_call( - [ - "diff", - "-r", - os.path.join(topsrcdir, os.path.join("third_party", "python")), - os.path.join(work_dir, os.path.join("third_party", "python")), - "--exclude=__pycache__", - "--strip-trailing-cr", - ] - ) + subprocess.check_call([ + "diff", + "-r", + os.path.join(topsrcdir, os.path.join("third_party", "python")), + os.path.join(work_dir, os.path.join("third_party", "python")), + "--exclude=__pycache__", + "--strip-trailing-cr", + ]) if __name__ == "__main__": diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py @@ -152,9 +152,10 @@ class SupportFilesConverter: full = mozpath.normpath( mozpath.join(manifest_dir, mozpath.basename(pattern)) ) - info.installs.append( - (full, mozpath.join(install_root, pattern[1:])) - ) + info.installs.append(( + full, + mozpath.join(install_root, pattern[1:]), + )) else: full = mozpath.normpath(mozpath.join(manifest_dir, pattern)) dest_path = mozpath.join(out_dir, pattern) diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py @@ -411,11 +411,9 @@ class List(list): def __setitem__(self, key, val): if isinstance(key, slice): if not isinstance(val, list): - raise ValueError( - "List can only be sliced with other list " "instances." - ) + raise ValueError("List can only be sliced with other list instances.") if key.step: - raise ValueError("List cannot be sliced with a nonzero step " "value") + raise ValueError("List cannot be sliced with a nonzero step value") return super().__setitem__(key, val) return super().__setitem__(key, val) diff --git a/python/mozbuild/mozbuild/vendor/host_git.py b/python/mozbuild/mozbuild/vendor/host_git.py @@ -13,9 +13,12 @@ class GitHost(BaseHost): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.workdir = tempfile.TemporaryDirectory(suffix="." + self.repo_url.netloc) - subprocess.check_call( - ["git", "clone", self.repo_url.geturl(), self.workdir.name] - ) + subprocess.check_call([ + "git", + "clone", + self.repo_url.geturl(), + self.workdir.name, + ]) def upstream_commit(self, revision): sha = subprocess.check_output( diff --git a/python/mozbuild/mozbuild/vendor/host_github.py b/python/mozbuild/mozbuild/vendor/host_github.py @@ -22,9 +22,11 @@ class GitHubHost(BaseHost): return (info["sha"], info["commit"]["committer"]["date"]) def upstream_snapshot(self, revision): - return "/".join( - [self.manifest["vendoring"]["url"], "archive", revision + ".tar.gz"] - ) + return "/".join([ + self.manifest["vendoring"]["url"], + "archive", + revision + ".tar.gz", + ]) def upstream_path_to_file(self, revision, filepath): repo = self.repo_url.path[1:] diff --git a/python/mozbuild/mozbuild/vendor/host_gitlab.py b/python/mozbuild/mozbuild/vendor/host_gitlab.py @@ -21,6 +21,9 @@ class GitLabHost(BaseHost): return (info["id"], info["committed_date"]) def upstream_snapshot(self, revision): - return "/".join( - [self.manifest["vendoring"]["url"], "-", "archive", revision + ".tar.gz"] - ) + return "/".join([ + self.manifest["vendoring"]["url"], + "-", + "archive", + revision + ".tar.gz", + ]) diff --git a/python/mozbuild/mozbuild/vendor/host_googlesource.py b/python/mozbuild/mozbuild/vendor/host_googlesource.py @@ -12,9 +12,11 @@ from mozbuild.vendor.host_base import BaseHost class GoogleSourceHost(BaseHost): def upstream_commit(self, revision): """Query for a git commit and timestamp.""" - url = "/".join( - [self.manifest["vendoring"]["url"], "+", revision + "?format=JSON"] - ) + url = "/".join([ + self.manifest["vendoring"]["url"], + "+", + revision + "?format=JSON", + ]) req = requests.get(url) req.raise_for_status() try: @@ -29,9 +31,11 @@ class GoogleSourceHost(BaseHost): return (info["commit"], info["committer"]["time"]) def upstream_snapshot(self, revision): - return "/".join( - [self.manifest["vendoring"]["url"], "+archive", revision + ".tar.gz"] - ) + return "/".join([ + self.manifest["vendoring"]["url"], + "+archive", + revision + ".tar.gz", + ]) def upstream_path_to_file(self, revision, filepath): return ( diff --git a/python/mozbuild/mozbuild/vendor/mach_commands.py b/python/mozbuild/mozbuild/vendor/mach_commands.py @@ -178,9 +178,7 @@ def check_modified_files(command_context): {files} Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`. -""".format( - files="\n".join(sorted(modified)) - ), +""".format(files="\n".join(sorted(modified))), ) sys.exit(1) diff --git a/python/mozbuild/mozbuild/vendor/moz_yaml.py b/python/mozbuild/mozbuild/vendor/moz_yaml.py @@ -152,93 +152,91 @@ def _schema_1(): ], ) - return Schema( - { - Required("schema"): "1", - Required("bugzilla"): { - Required("product"): All(str, Length(min=1)), - Required("component"): All(str, Length(min=1)), - }, - "origin": { - Required("name"): All(str, Length(min=1)), - Required("description"): All(str, Length(min=1)), - "notes": All(str, Length(min=1)), - Required("url"): FqdnUrl(), - Required("license"): Msg(License(), msg="Unsupported License"), - "license-file": All(str, Length(min=1)), - Required("release"): All(str, Length(min=1)), - # The following regex defines a valid git reference - # The first group [^ ~^:?*[\]] matches 0 or more times anything - # that isn't a Space, ~, ^, :, ?, *, or ] - # The second group [^ ~^:?*[\]\.]+ matches 1 or more times - # anything that isn't a Space, ~, ^, :, ?, *, [, ], or . - "revision": Match(r"^[^ ~^:?*[\]]*[^ ~^:?*[\]\.]+$"), - }, - "updatebot": { - Required("maintainer-phab"): All(str, Length(min=1)), - Required("maintainer-bz"): All(str, Length(min=1)), - "try-preset": All(str, Length(min=1)), - "fuzzy-query": All(str, Length(min=1)), - "fuzzy-paths": All([str], Length(min=1)), - "tasks": All( - UpdatebotTasks(), - [ - { - Required("type"): In( - ["vendoring", "commit-alert"], - msg="Invalid type specified in tasks", - ), - "branch": All(str, Length(min=1)), - "enabled": Boolean(), - "cc": Unique([str]), - "needinfo": Unique([str]), - "filter": In( - ["none", "security", "source-extensions"], - msg="Invalid filter value specified in tasks", - ), - "source-extensions": Unique([str]), - "blocking": Match(r"^[0-9]+$"), - "frequency": Match( - r"^(every|release|[1-9][0-9]* weeks?|[1-9][0-9]* commits?|" - + r"[1-9][0-9]* weeks?, ?[1-9][0-9]* commits?)$" - ), - "platform": Match(r"^(windows|linux)$"), - } - ], - ), - }, - "vendoring": { - Required("url"): FqdnUrl(), - Required("source-hosting"): All( - str, - Length(min=1), - In(VALID_SOURCE_HOSTS, msg="Unsupported Source Hosting"), - ), - "source-host-path": str, - "tracking": Match(r"^(commit|tag)$"), - "release-artifact": All(str, Length(min=1)), - "flavor": Match(r"^(regular|rust|individual-files)$"), - "skip-vendoring-steps": Unique([str]), - "vendor-directory": All(str, Length(min=1)), - "patches": Unique([str]), - "keep": Unique([str]), - "exclude": Unique([str]), - "include": Unique([str]), - "generated": Unique([str]), - "individual-files": [ + return Schema({ + Required("schema"): "1", + Required("bugzilla"): { + Required("product"): All(str, Length(min=1)), + Required("component"): All(str, Length(min=1)), + }, + "origin": { + Required("name"): All(str, Length(min=1)), + Required("description"): All(str, Length(min=1)), + "notes": All(str, Length(min=1)), + Required("url"): FqdnUrl(), + Required("license"): Msg(License(), msg="Unsupported License"), + "license-file": All(str, Length(min=1)), + Required("release"): All(str, Length(min=1)), + # The following regex defines a valid git reference + # The first group [^ ~^:?*[\]] matches 0 or more times anything + # that isn't a Space, ~, ^, :, ?, *, or ] + # The second group [^ ~^:?*[\]\.]+ matches 1 or more times + # anything that isn't a Space, ~, ^, :, ?, *, [, ], or . + "revision": Match(r"^[^ ~^:?*[\]]*[^ ~^:?*[\]\.]+$"), + }, + "updatebot": { + Required("maintainer-phab"): All(str, Length(min=1)), + Required("maintainer-bz"): All(str, Length(min=1)), + "try-preset": All(str, Length(min=1)), + "fuzzy-query": All(str, Length(min=1)), + "fuzzy-paths": All([str], Length(min=1)), + "tasks": All( + UpdatebotTasks(), + [ { - Required("upstream"): All(str, Length(min=1)), - Required("destination"): All(str, Length(min=1)), + Required("type"): In( + ["vendoring", "commit-alert"], + msg="Invalid type specified in tasks", + ), + "branch": All(str, Length(min=1)), + "enabled": Boolean(), + "cc": Unique([str]), + "needinfo": Unique([str]), + "filter": In( + ["none", "security", "source-extensions"], + msg="Invalid filter value specified in tasks", + ), + "source-extensions": Unique([str]), + "blocking": Match(r"^[0-9]+$"), + "frequency": Match( + r"^(every|release|[1-9][0-9]* weeks?|[1-9][0-9]* commits?|" + + r"[1-9][0-9]* weeks?, ?[1-9][0-9]* commits?)$" + ), + "platform": Match(r"^(windows|linux)$"), } ], - "individual-files-default-upstream": str, - "individual-files-default-destination": All(str, Length(min=1)), - "individual-files-list": Unique([str]), - "update-actions": actions_schema, - "post-patch-actions": actions_schema, - }, - } - ) + ), + }, + "vendoring": { + Required("url"): FqdnUrl(), + Required("source-hosting"): All( + str, + Length(min=1), + In(VALID_SOURCE_HOSTS, msg="Unsupported Source Hosting"), + ), + "source-host-path": str, + "tracking": Match(r"^(commit|tag)$"), + "release-artifact": All(str, Length(min=1)), + "flavor": Match(r"^(regular|rust|individual-files)$"), + "skip-vendoring-steps": Unique([str]), + "vendor-directory": All(str, Length(min=1)), + "patches": Unique([str]), + "keep": Unique([str]), + "exclude": Unique([str]), + "include": Unique([str]), + "generated": Unique([str]), + "individual-files": [ + { + Required("upstream"): All(str, Length(min=1)), + Required("destination"): All(str, Length(min=1)), + } + ], + "individual-files-default-upstream": str, + "individual-files-default-destination": All(str, Length(min=1)), + "individual-files-list": Unique([str]), + "update-actions": actions_schema, + "post-patch-actions": actions_schema, + }, + }) def _schema_1_additional(filename, manifest, require_license_file=True): diff --git a/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py b/python/mozbuild/mozbuild/vendor/rewrite_mozbuild.py @@ -434,23 +434,21 @@ def mozbuild_file_to_source_assignments(normalized_mozbuild_filename, assignment ) ) ] - source_assignment_nodes.extend( - [ - node - for node in ast.walk(root) - if isinstance(node, ast.Assign) - and ( - ( - isinstance(node.targets[0], ast.Name) - and node.targets[0].id == "EXPORTS" - ) - or ( - isinstance(node.targets[0], ast.Attribute) - and get_attribute_label(node.targets[0]).startswith("EXPORTS") - ) + source_assignment_nodes.extend([ + node + for node in ast.walk(root) + if isinstance(node, ast.Assign) + and ( + ( + isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "EXPORTS" ) - ] - ) + or ( + isinstance(node.targets[0], ast.Attribute) + and get_attribute_label(node.targets[0]).startswith("EXPORTS") + ) + ) + ]) # Get the source-assignment-location for the node: assignment_index = 1 @@ -473,11 +471,12 @@ def mozbuild_file_to_source_assignments(normalized_mozbuild_filename, assignment if source_assignment_location in source_assignments: source_assignment_location = node_to_readable_file_location(code, a) - assert ( - source_assignment_location not in source_assignments - ), "In %s, two assignments have the same key ('%s')" % ( - normalized_mozbuild_filename, - source_assignment_location, + assert source_assignment_location not in source_assignments, ( + "In %s, two assignments have the same key ('%s')" + % ( + normalized_mozbuild_filename, + source_assignment_location, + ) ) source_assignments[source_assignment_location] = normalized_source_filename_list assignment_index += 1 @@ -562,9 +561,9 @@ def get_mozbuild_file_search_order( ordered_list = [] if all_mozbuild_filenames_normalized is None: - assert os.path.isfile( - ".arcconfig" - ), "We do not seem to be running from the gecko root" + assert os.path.isfile(".arcconfig"), ( + "We do not seem to be running from the gecko root" + ) # The first time around, this variable name is incorrect. # It's actually the full path+filename, not a directory. @@ -642,12 +641,10 @@ def filenames_directory_is_in_filename_list( f("foo/bar/a.c", ["foo/b.c", "foo/bar/c.c"]) -> true f("foo/bar/a.c", ["foo/b.c", "foo/bar/baz/d.c"]) -> false """ - path_list = set( - [ - os.path.dirname(f).replace(os.path.sep, "/") - for f in list_of_normalized_filenames - ] - ) + path_list = set([ + os.path.dirname(f).replace(os.path.sep, "/") + for f in list_of_normalized_filenames + ]) return os.path.dirname(filename_normalized).replace(os.path.sep, "/") in path_list @@ -1050,9 +1047,9 @@ def add_file_to_moz_build_file( normalized_filename_to_add = original_normalized_filename_to_add continue - assert ( - len(possible_assignments) > 0 - ), "Could not find a single possible source assignment" + assert len(possible_assignments) > 0, ( + "Could not find a single possible source assignment" + ) if len(possible_assignments) > 1: best_guess, _ = guess_best_assignment( possible_assignments, normalized_filename_to_add diff --git a/python/mozbuild/mozbuild/vendor/vendor_manifest.py b/python/mozbuild/mozbuild/vendor/vendor_manifest.py @@ -663,18 +663,13 @@ class VendorManifest(MozbuildObject): f.write(("".join(yaml)).encode("utf-8")) def spurious_check(self, revision, ignore_modified): - changed_files = set( - [ - os.path.abspath(f) - for f in self.repository.get_changed_files(mode="staged") - ] - ) - generated_files = set( - [ - self.get_full_path(f) - for f in self.manifest["vendoring"].get("generated", []) - ] - ) + changed_files = set([ + os.path.abspath(f) for f in self.repository.get_changed_files(mode="staged") + ]) + generated_files = set([ + self.get_full_path(f) + for f in self.manifest["vendoring"].get("generated", []) + ]) changed_files = set(changed_files) - generated_files if not changed_files: self.logInfo({"r": revision}, "Upstream {r} hasn't modified files locally.") diff --git a/python/mozbuild/mozbuild/vendor/vendor_python.py b/python/mozbuild/mozbuild/vendor/vendor_python.py @@ -146,23 +146,21 @@ class VendorPython(MozbuildObject): with TemporaryDirectory() as tmp: # use requirements.txt to download archived source distributions of all # packages - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "download", - "-r", - str(requirements_path), - "--no-deps", - "--dest", - tmp, - "--abi", - "none", - "--platform", - "any", - ] - ) + subprocess.check_call([ + sys.executable, + "-m", + "pip", + "download", + "-r", + str(requirements_path), + "--no-deps", + "--dest", + tmp, + "--abi", + "none", + "--platform", + "any", + ]) _purge_vendor_dir(vendor_dir) self._extract(tmp, vendor_dir, keep_extra_files) diff --git a/python/mozbuild/mozbuild/vendor/vendor_rust.py b/python/mozbuild/mozbuild/vendor/vendor_rust.py @@ -163,20 +163,18 @@ class VendorRust(MozbuildObject): self._issues = [] def serialize_issues_json(self): - return json.dumps( - { - "Cargo.lock": [ - { - "path": "Cargo.lock", - "column": None, - "line": None, - "level": "error" if level == logging.ERROR else "warning", - "message": msg, - } - for (level, msg) in self._issues - ] - } - ) + return json.dumps({ + "Cargo.lock": [ + { + "path": "Cargo.lock", + "column": None, + "line": None, + "level": "error" if level == logging.ERROR else "warning", + "message": msg, + } + for (level, msg) in self._issues + ] + }) def generate_diff_stream(self): return self.repository.diff_stream() @@ -268,9 +266,7 @@ class VendorRust(MozbuildObject): {files} Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`. -""".format( - files="\n".join(sorted(modified)) - ), +""".format(files="\n".join(sorted(modified))), ) return modified @@ -706,13 +702,11 @@ license file's hash. for name, packages in grouped.items(): # Allow to have crates of the same name when one depends on the other. - num = len( - [ - p - for p in packages - if all(d.split()[0] != name for d in p.get("dependencies", [])) - ] - ) + num = len([ + p + for p in packages + if all(d.split()[0] != name for d in p.get("dependencies", [])) + ]) if num > 1: self.log( logging.ERROR, @@ -728,12 +722,10 @@ license file's hash. # Only emit warnings for cargo-vet for now. env = os.environ.copy() - env["PATH"] = os.pathsep.join( - ( - str(Path(cargo).parent), - os.environ["PATH"], - ) - ) + env["PATH"] = os.pathsep.join(( + str(Path(cargo).parent), + os.environ["PATH"], + )) flags = ["--output-format=json"] if "MOZ_AUTOMATION" in os.environ: flags.append("--locked") diff --git a/python/mozbuild/mozpack/archive.py b/python/mozbuild/mozpack/archive.py @@ -79,7 +79,7 @@ def create_tar_from_files(fp, files): # would be a glaring security hole if the archive were # uncompressed as root. if ti.mode & (stat.S_ISUID | stat.S_ISGID): - raise ValueError("cannot add file with setuid or setgid set: " "%s" % f) + raise ValueError("cannot add file with setuid or setgid set: %s" % f) # Set uid, gid, username, and group as deterministic values. ti.uid = 0 diff --git a/python/mozbuild/mozpack/chrome/manifest.py b/python/mozbuild/mozpack/chrome/manifest.py @@ -339,16 +339,14 @@ class ManifestContract(ManifestEntry): # All manifest classes by their type name. -MANIFESTS_TYPES = dict( - [ - (c.type, c) - for c in globals().values() - if type(c) is type - and issubclass(c, ManifestEntry) - and hasattr(c, "type") - and c.type - ] -) +MANIFESTS_TYPES = dict([ + (c.type, c) + for c in globals().values() + if type(c) is type + and issubclass(c, ManifestEntry) + and hasattr(c, "type") + and c.type +]) MANIFEST_RE = re.compile(r"^#.*$") diff --git a/python/mozbuild/mozpack/dmg.py b/python/mozbuild/mozpack/dmg.py @@ -52,15 +52,13 @@ def generate_hfs_file( size = int(output.split()[0]) / 1000 # Get in MB size = int(size * 1.02) # Bump the used size slightly larger. # Setup a proper file sized out with zero's - subprocess.check_call( - [ - "dd", - "if=/dev/zero", - f"of={hfs}", - "bs=1M", - f"count={size}", - ] - ) + subprocess.check_call([ + "dd", + "if=/dev/zero", + f"of={hfs}", + "bs=1M", + f"count={size}", + ]) subprocess.check_call([mkfshfs_tool, "-v", volume_name, hfs]) @@ -106,23 +104,26 @@ def create_dmg_from_staged( if attribution_sentinel: while len(attribution_sentinel) < 1024: attribution_sentinel += "\t" - subprocess.check_call( - [ - hfs_tool, - hfs, - "setattr", - f"{volume_name}.app", - "com.apple.application-instance", - attribution_sentinel, - ] - ) + subprocess.check_call([ + hfs_tool, + hfs, + "setattr", + f"{volume_name}.app", + "com.apple.application-instance", + attribution_sentinel, + ]) subprocess.check_call(["cp", hfs, str(Path(output_dmg).parent)]) dmg_cmd.append(attribution_sentinel) if compression == "lzma": - dmg_cmd.extend( - ["--compression", "lzma", "--level", "5", "--run-sectors", "2048"] - ) + dmg_cmd.extend([ + "--compression", + "lzma", + "--level", + "5", + "--run-sectors", + "2048", + ]) subprocess.check_call( dmg_cmd, @@ -135,35 +136,31 @@ def create_dmg_from_staged( format = "ULMO" hybrid = tmpdir / "hybrid.dmg" - subprocess.check_call( - [ - "hdiutil", - "makehybrid", - "-hfs", - "-hfs-volume-name", - volume_name, - "-hfs-openfolder", - stagedir, - "-ov", - stagedir, - "-o", - hybrid, - ] - ) - subprocess.check_call( - [ - "hdiutil", - "convert", - "-format", - format, - "-imagekey", - "bzip2-level=9", - "-ov", - hybrid, - "-o", - output_dmg, - ] - ) + subprocess.check_call([ + "hdiutil", + "makehybrid", + "-hfs", + "-hfs-volume-name", + volume_name, + "-hfs-openfolder", + stagedir, + "-ov", + stagedir, + "-o", + hybrid, + ]) + subprocess.check_call([ + "hdiutil", + "convert", + "-format", + format, + "-imagekey", + "bzip2-level=9", + "-ov", + hybrid, + "-o", + output_dmg, + ]) def create_dmg( diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py @@ -1253,13 +1253,11 @@ class MercurialRevisionFinder(BaseFinder): # Immediately populate the list of files in the repo since nearly every # operation requires this list. - out = self._client.rawcommand( - [ - b"files", - b"--rev", - self._rev.encode(), - ] - ) + out = self._client.rawcommand([ + b"files", + b"--rev", + self._rev.encode(), + ]) for relpath in out.splitlines(): # Mercurial may use \ as path separator on Windows. So use # normpath(). diff --git a/python/mozbuild/mozpack/mozjar.py b/python/mozbuild/mozpack/mozjar.py @@ -197,18 +197,16 @@ class JarCdirEnd(JarStruct): """ MAGIC = 0x06054B50 - STRUCT = OrderedDict( - [ - ("disk_num", "uint16"), - ("cdir_disk", "uint16"), - ("disk_entries", "uint16"), - ("cdir_entries", "uint16"), - ("cdir_size", "uint32"), - ("cdir_offset", "uint32"), - ("comment_size", "uint16"), - ("comment", "comment_size"), - ] - ) + STRUCT = OrderedDict([ + ("disk_num", "uint16"), + ("cdir_disk", "uint16"), + ("disk_entries", "uint16"), + ("cdir_entries", "uint16"), + ("cdir_size", "uint32"), + ("cdir_offset", "uint32"), + ("comment_size", "uint16"), + ("comment", "comment_size"), + ]) CDIR_END_SIZE = JarCdirEnd().size @@ -220,29 +218,27 @@ class JarCdirEntry(JarStruct): """ MAGIC = 0x02014B50 - STRUCT = OrderedDict( - [ - ("creator_version", "uint16"), - ("min_version", "uint16"), - ("general_flag", "uint16"), - ("compression", "uint16"), - ("lastmod_time", "uint16"), - ("lastmod_date", "uint16"), - ("crc32", "uint32"), - ("compressed_size", "uint32"), - ("uncompressed_size", "uint32"), - ("filename_size", "uint16"), - ("extrafield_size", "uint16"), - ("filecomment_size", "uint16"), - ("disknum", "uint16"), - ("internal_attr", "uint16"), - ("external_attr", "uint32"), - ("offset", "uint32"), - ("filename", "filename_size"), - ("extrafield", "extrafield_size"), - ("filecomment", "filecomment_size"), - ] - ) + STRUCT = OrderedDict([ + ("creator_version", "uint16"), + ("min_version", "uint16"), + ("general_flag", "uint16"), + ("compression", "uint16"), + ("lastmod_time", "uint16"), + ("lastmod_date", "uint16"), + ("crc32", "uint32"), + ("compressed_size", "uint32"), + ("uncompressed_size", "uint32"), + ("filename_size", "uint16"), + ("extrafield_size", "uint16"), + ("filecomment_size", "uint16"), + ("disknum", "uint16"), + ("internal_attr", "uint16"), + ("external_attr", "uint32"), + ("offset", "uint32"), + ("filename", "filename_size"), + ("extrafield", "extrafield_size"), + ("filecomment", "filecomment_size"), + ]) class JarLocalFileHeader(JarStruct): @@ -251,22 +247,20 @@ class JarLocalFileHeader(JarStruct): """ MAGIC = 0x04034B50 - STRUCT = OrderedDict( - [ - ("min_version", "uint16"), - ("general_flag", "uint16"), - ("compression", "uint16"), - ("lastmod_time", "uint16"), - ("lastmod_date", "uint16"), - ("crc32", "uint32"), - ("compressed_size", "uint32"), - ("uncompressed_size", "uint32"), - ("filename_size", "uint16"), - ("extra_field_size", "uint16"), - ("filename", "filename_size"), - ("extra_field", "extra_field_size"), - ] - ) + STRUCT = OrderedDict([ + ("min_version", "uint16"), + ("general_flag", "uint16"), + ("compression", "uint16"), + ("lastmod_time", "uint16"), + ("lastmod_date", "uint16"), + ("crc32", "uint32"), + ("compressed_size", "uint32"), + ("uncompressed_size", "uint32"), + ("filename_size", "uint16"), + ("extra_field_size", "uint16"), + ("filename", "filename_size"), + ("extra_field", "extra_field_size"), + ]) class JarFileReader: diff --git a/python/mozbuild/mozpack/pkg.py b/python/mozbuild/mozpack/pkg.py @@ -130,11 +130,11 @@ def create_payload(destination: Path, root_path: Path, cpio_tool: str): break yield chunk - with tmp_payload_path.open("rb") as f_in, destination.open( - "wb" - ) as f_out, concurrent.futures.ThreadPoolExecutor( - max_workers=cpu_count() - ) as executor: + with ( + tmp_payload_path.open("rb") as f_in, + destination.open("wb") as f_out, + concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count()) as executor, + ): f_out.write(b"pbzx") f_out.write(struct.pack(">Q", PBZX_CHUNK_SIZE)) chunks = chunker(f_in, PBZX_CHUNK_SIZE) @@ -165,17 +165,15 @@ def create_bom(bom_path: Path, root_path: Path, mkbom_tool: Path): mkbom_tool: Path, mkbom tool Path """ print(f"Creating BOM file from {root_path} to {bom_path}") - subprocess.check_call( - [ - mkbom_tool, - "-u", - "0", - "-g", - "80", - str(root_path), - str(bom_path), - ] - ) + subprocess.check_call([ + mkbom_tool, + "-u", + "0", + "-g", + "80", + str(root_path), + str(bom_path), + ]) print(f"Created BOM File size: {bom_path.stat().st_size // 1024}kb") @@ -256,14 +254,12 @@ def create_pkg( root_path.mkdir(parents=True, exist_ok=True) # Copy files over - subprocess.check_call( - [ - "cp", - "-R", - str(source_app), - str(root_path), - ] - ) + subprocess.check_call([ + "cp", + "-R", + str(source_app), + str(root_path), + ]) # Count all files (innards + itself) file_count = len(list(source_app.glob("**/*"))) + 1 diff --git a/python/mozbuild/mozpack/test/test_chrome_manifest.py b/python/mozbuild/mozpack/test/test_chrome_manifest.py @@ -108,12 +108,10 @@ class TestManifest(unittest.TestCase): "chrome://browser/content/viewSourceOverlay.xul", ), ] - with mozunit.MockedOpen( - { - "manifest": "\n".join(manifest), - "other/manifest": "\n".join(other_manifest), - } - ): + with mozunit.MockedOpen({ + "manifest": "\n".join(manifest), + "other/manifest": "\n".join(other_manifest), + }): # Ensure we have tests for all types of manifests. self.assertEqual( set(type(e) for e in expected_result), set(MANIFESTS_TYPES.values()) diff --git a/python/mozbuild/mozpack/test/test_files.py b/python/mozbuild/mozpack/test/test_files.py @@ -829,12 +829,12 @@ class TestManifestFile(TestWithTmpDir): # because they will be automatically included by virtue of being an # argument to a method of |bar|. bar_xpt = GeneratedFile( - b"\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A" - + b"\x01\x02\x00\x02\x00\x00\x00\x7B\x00\x00\x00\x24\x00\x00\x00\x5C" + b"\x58\x50\x43\x4f\x4d\x0a\x54\x79\x70\x65\x4c\x69\x62\x0d\x0a\x1a" + + b"\x01\x02\x00\x02\x00\x00\x00\x7b\x00\x00\x00\x24\x00\x00\x00\x5c" + b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - + b"\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x5F" - + b"\x70\xDA\x76\x51\x9C\x48\x58\xB7\x1E\xE3\xC9\x23\x33\xE2\xD6\x00" - + b"\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x0D\x00\x66\x6F\x6F\x00" + + b"\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x5f" + + b"\x70\xda\x76\x51\x9c\x48\x58\xb7\x1e\xe3\xc9\x23\x33\xe2\xd6\x00" + + b"\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x0d\x00\x66\x6f\x6f\x00" + b"\x62\x61\x72\x00\x62\x61\x72\x00\x00\x00\x00\x01\x00\x00\x00\x00" + b"\x09\x01\x80\x92\x00\x01\x80\x06\x00\x00\x80" ) @@ -845,11 +845,11 @@ bar_xpt = GeneratedFile( # void foo(); # }; foo_xpt = GeneratedFile( - b"\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A" + b"\x58\x50\x43\x4f\x4d\x0a\x54\x79\x70\x65\x4c\x69\x62\x0d\x0a\x1a" + b"\x01\x02\x00\x01\x00\x00\x00\x57\x00\x00\x00\x24\x00\x00\x00\x40" - + b"\x80\x00\x00\x32\x71\xBE\xBC\x92\x7E\x4B\xEF\x93\x5E\x44\xE0\xAA" - + b"\xF3\xC1\xE5\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00" - + b"\x66\x6F\x6F\x00\x66\x6F\x6F\x00\x00\x00\x00\x01\x00\x00\x00\x00" + + b"\x80\x00\x00\x32\x71\xbe\xbc\x92\x7e\x4b\xef\x93\x5e\x44\xe0\xaa" + + b"\xf3\xc1\xe5\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00" + + b"\x66\x6f\x6f\x00\x66\x6f\x6f\x00\x00\x00\x00\x01\x00\x00\x00\x00" + b"\x05\x00\x80\x06\x00\x00\x00" ) @@ -859,11 +859,11 @@ foo_xpt = GeneratedFile( # void foo(); # }; foo2_xpt = GeneratedFile( - b"\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A" + b"\x58\x50\x43\x4f\x4d\x0a\x54\x79\x70\x65\x4c\x69\x62\x0d\x0a\x1a" + b"\x01\x02\x00\x01\x00\x00\x00\x57\x00\x00\x00\x24\x00\x00\x00\x40" - + b"\x80\x00\x00\x70\x57\xF2\xAA\xFD\xC2\x45\x59\xAB\xDE\x08\xD9\x39" - + b"\xF7\xE8\x0D\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00" - + b"\x66\x6F\x6F\x00\x66\x6F\x6F\x00\x00\x00\x00\x01\x00\x00\x00\x00" + + b"\x80\x00\x00\x70\x57\xf2\xaa\xfd\xc2\x45\x59\xab\xde\x08\xd9\x39" + + b"\xf7\xe8\x0d\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00" + + b"\x66\x6f\x6f\x00\x66\x6f\x6f\x00\x00\x00\x00\x01\x00\x00\x00\x00" + b"\x05\x00\x80\x06\x00\x00\x00" ) @@ -1282,12 +1282,10 @@ class TestComposedFinder(MatchTestTemplate, TestWithTmpDir): ensureParentDir(self.tmppath("a/foo/qux/hoge")) open(self.tmppath("a/foo/qux/hoge"), "wb").write(b"hoge") open(self.tmppath("a/foo/qux/bar"), "wb").write(b"not the right content") - self.finder = ComposedFinder( - { - "": FileFinder(self.tmppath("a")), - "foo/qux": FileFinder(self.tmppath("b")), - } - ) + self.finder = ComposedFinder({ + "": FileFinder(self.tmppath("a")), + "foo/qux": FileFinder(self.tmppath("b")), + }) self.do_match_test() self.assertIsNone(self.finder.get("does-not-exist")) diff --git a/python/mozbuild/mozpack/test/test_mozjar.py b/python/mozbuild/mozpack/test/test_mozjar.py @@ -29,17 +29,15 @@ test_data_path = mozpath.join(test_data_path, "data") class TestJarStruct(unittest.TestCase): class Foo(JarStruct): MAGIC = 0x01020304 - STRUCT = OrderedDict( - [ - ("foo", "uint32"), - ("bar", "uint16"), - ("qux", "uint16"), - ("length", "uint16"), - ("length2", "uint16"), - ("string", "length"), - ("string2", "length2"), - ] - ) + STRUCT = OrderedDict([ + ("foo", "uint32"), + ("bar", "uint16"), + ("qux", "uint16"), + ("length", "uint16"), + ("length2", "uint16"), + ("string", "length"), + ("string2", "length2"), + ]) def test_jar_struct(self): foo = TestJarStruct.Foo() @@ -304,30 +302,26 @@ class TestPreload(unittest.TestCase): class TestJarLog(unittest.TestCase): def test_jarlog(self): s = StringIO( - "\n".join( - [ - "bar/baz.jar first", - "bar/baz.jar second", - "bar/baz.jar third", - "bar/baz.jar second", - "bar/baz.jar second", - "omni.ja stuff", - "bar/baz.jar first", - "omni.ja other/stuff", - "omni.ja stuff", - "bar/baz.jar third", - ] - ) + "\n".join([ + "bar/baz.jar first", + "bar/baz.jar second", + "bar/baz.jar third", + "bar/baz.jar second", + "bar/baz.jar second", + "omni.ja stuff", + "bar/baz.jar first", + "omni.ja other/stuff", + "omni.ja stuff", + "bar/baz.jar third", + ]) ) log = JarLog(fileobj=s) self.assertEqual( set(log.keys()), - set( - [ - "bar/baz.jar", - "omni.ja", - ] - ), + set([ + "bar/baz.jar", + "omni.ja", + ]), ) self.assertEqual( log["bar/baz.jar"], diff --git a/python/mozbuild/mozpack/test/test_packager.py b/python/mozbuild/mozpack/test/test_packager.py @@ -160,12 +160,10 @@ class TestSimplePackager(unittest.TestCase): packager.add( "qux/qux.manifest", GeneratedFile( - b"".join( - [ - b"resource qux qux/\n", - b"binary-component qux.so\n", - ] - ) + b"".join([ + b"resource qux qux/\n", + b"binary-component qux.so\n", + ]) ), ) bar_xpt = GeneratedFile(b"bar.xpt") @@ -180,12 +178,10 @@ class TestSimplePackager(unittest.TestCase): file = GeneratedFileWithPath( os.path.join(curdir, "foo.manifest"), - b"".join( - [ - b"manifest foo/bar.manifest\n", - b"manifest bar/baz.manifest\n", - ] - ), + b"".join([ + b"manifest foo/bar.manifest\n", + b"manifest bar/baz.manifest\n", + ]), ) with errors.context("manifest", 6): packager.add("foo.manifest", file) @@ -399,26 +395,24 @@ class TestSimplePackager(unittest.TestCase): self.assertEqual( packager.get_bases(), - set( - [ - "", - "addon", - "addon2", - "addon3", - "addon4", - "addon5", - "addon6", - "addon7", - "addon8", - "addon9", - "addon10", - "addon11", - "qux", - "hybrid", - "hybrid2", - "webextension", - ] - ), + set([ + "", + "addon", + "addon2", + "addon3", + "addon4", + "addon5", + "addon6", + "addon7", + "addon8", + "addon9", + "addon10", + "addon11", + "qux", + "hybrid", + "hybrid2", + "webextension", + ]), ) self.assertEqual(packager.get_bases(addons=False), set(["", "qux"])) @@ -429,9 +423,7 @@ class TestSimplePackager(unittest.TestCase): packager = SimplePackager(formatter) packager.add( "base.manifest", - GeneratedFile( - b"manifest foo/bar.manifest\n" b"manifest bar/baz.manifest\n" - ), + GeneratedFile(b"manifest foo/bar.manifest\nmanifest bar/baz.manifest\n"), ) packager.add("foo/bar.manifest", GeneratedFile(b"resource bar bar")) packager.add("bar/baz.manifest", GeneratedFile(b"resource baz baz")) @@ -452,9 +444,7 @@ class TestSimplePackager(unittest.TestCase): packager = SimplePackager(formatter) packager.add( "base.manifest", - GeneratedFile( - b"manifest foo/bar.manifest\n" b"manifest bar/baz.manifest\n" - ), + GeneratedFile(b"manifest foo/bar.manifest\nmanifest bar/baz.manifest\n"), ) packager.add("foo/bar.manifest", GeneratedFile(b"resource bar bar")) packager.add("bar/baz.manifest", GeneratedFile(b"resource baz baz")) @@ -487,16 +477,14 @@ class TestSimpleManifestSink(unittest.TestCase): foobaz = GeneratedFile(b"foobaz") fooqux = GeneratedFile(b"fooqux") foozot = GeneratedFile(b"foozot") - finder = MockFinder( - { - "bin/foo/bar": foobar, - "bin/foo/baz": foobaz, - "bin/foo/qux": fooqux, - "bin/foo/zot": foozot, - "bin/foo/chrome.manifest": GeneratedFile(b"resource foo foo/"), - "bin/chrome.manifest": GeneratedFile(b"manifest foo/chrome.manifest"), - } - ) + finder = MockFinder({ + "bin/foo/bar": foobar, + "bin/foo/baz": foobaz, + "bin/foo/qux": fooqux, + "bin/foo/zot": foozot, + "bin/foo/chrome.manifest": GeneratedFile(b"resource foo foo/"), + "bin/chrome.manifest": GeneratedFile(b"manifest foo/chrome.manifest"), + }) parser = SimpleManifestSink(finder, formatter) component0 = Component("component0") component1 = Component("component1") diff --git a/python/mozbuild/mozpack/test/test_packager_formats.py b/python/mozbuild/mozpack/test/test_packager_formats.py @@ -107,27 +107,23 @@ RESULT_FLAT = { } for addon in ("addon0", "addon1", "app/chrome/addons/addon2"): - RESULT_FLAT.update( - { - mozpath.join(addon, p): f - for p, f in { - "chrome.manifest": [ - "manifest chrome/chrome.manifest", - "manifest components/components.manifest", - ], - "chrome/chrome.manifest": [ - "content %s foo/bar/" % mozpath.basename(addon) - ], - "chrome/foo/bar/baz": FILES[mozpath.join(addon, "chrome/foo/bar/baz")], - "components/components.manifest": [ - "interfaces bar.xpt", - "interfaces foo.xpt", - ], - "components/bar.xpt": bar_xpt, - "components/foo.xpt": foo2_xpt, - }.items() - } - ) + RESULT_FLAT.update({ + mozpath.join(addon, p): f + for p, f in { + "chrome.manifest": [ + "manifest chrome/chrome.manifest", + "manifest components/components.manifest", + ], + "chrome/chrome.manifest": ["content %s foo/bar/" % mozpath.basename(addon)], + "chrome/foo/bar/baz": FILES[mozpath.join(addon, "chrome/foo/bar/baz")], + "components/components.manifest": [ + "interfaces bar.xpt", + "interfaces foo.xpt", + ], + "components/bar.xpt": bar_xpt, + "components/foo.xpt": foo2_xpt, + }.items() + }) RESULT_JAR = { p: RESULT_FLAT[p] @@ -149,42 +145,40 @@ RESULT_JAR = { ) } -RESULT_JAR.update( - { - "chrome/f/f.manifest": [ - "content oo jar:oo.jar!/", - "content bar jar:oo.jar!/bar/", - "resource foo resource://bar/", - ], - "chrome/f/oo.jar": { - "bar/baz": FILES["chrome/f/oo/bar/baz"], - "baz": FILES["chrome/f/oo/baz"], - "qux": FILES["chrome/f/oo/qux"], - }, - "app/chrome/chrome.manifest": [ - "content content jar:foo.jar!/", - ], - "app/chrome/foo.jar": { - "foo": FILES["app/chrome/foo/foo"], - }, - "addon0/chrome/chrome.manifest": [ - "content addon0 jar:foo.jar!/bar/", - ], - "addon0/chrome/foo.jar": { - "bar/baz": FILES["addon0/chrome/foo/bar/baz"], - }, - "addon1.xpi": { - mozpath.relpath(p, "addon1"): f - for p, f in RESULT_FLAT.items() - if p.startswith("addon1/") - }, - "app/chrome/addons/addon2.xpi": { - mozpath.relpath(p, "app/chrome/addons/addon2"): f - for p, f in RESULT_FLAT.items() - if p.startswith("app/chrome/addons/addon2/") - }, - } -) +RESULT_JAR.update({ + "chrome/f/f.manifest": [ + "content oo jar:oo.jar!/", + "content bar jar:oo.jar!/bar/", + "resource foo resource://bar/", + ], + "chrome/f/oo.jar": { + "bar/baz": FILES["chrome/f/oo/bar/baz"], + "baz": FILES["chrome/f/oo/baz"], + "qux": FILES["chrome/f/oo/qux"], + }, + "app/chrome/chrome.manifest": [ + "content content jar:foo.jar!/", + ], + "app/chrome/foo.jar": { + "foo": FILES["app/chrome/foo/foo"], + }, + "addon0/chrome/chrome.manifest": [ + "content addon0 jar:foo.jar!/bar/", + ], + "addon0/chrome/foo.jar": { + "bar/baz": FILES["addon0/chrome/foo/bar/baz"], + }, + "addon1.xpi": { + mozpath.relpath(p, "addon1"): f + for p, f in RESULT_FLAT.items() + if p.startswith("addon1/") + }, + "app/chrome/addons/addon2.xpi": { + mozpath.relpath(p, "app/chrome/addons/addon2"): f + for p, f in RESULT_FLAT.items() + if p.startswith("app/chrome/addons/addon2/") + }, +}) RESULT_OMNIJAR = { p: RESULT_FLAT[p] @@ -196,55 +190,51 @@ RESULT_OMNIJAR = { RESULT_OMNIJAR.update({p: RESULT_JAR[p] for p in RESULT_JAR if p.startswith("addon")}) -RESULT_OMNIJAR.update( - { - "omni.foo": { - "components/components.manifest": [ - "interfaces bar.xpt", - "interfaces foo.xpt", - ], - }, - "chrome.manifest": [ - "manifest components/components.manifest", - ], +RESULT_OMNIJAR.update({ + "omni.foo": { "components/components.manifest": [ - "binary-component foo.so", + "interfaces bar.xpt", + "interfaces foo.xpt", ], - "app/omni.foo": { - p: RESULT_FLAT["app/" + p] - for p in chain( - ( - "chrome.manifest", - "chrome/chrome.manifest", - "chrome/foo/foo", - "components/components.manifest", - "components/foo.js", - ), - ( - mozpath.relpath(p, "app") - for p in RESULT_FLAT.keys() - if p.startswith("app/chrome/addons/addon2/") - ), - ) - }, - } -) - -RESULT_OMNIJAR["omni.foo"].update( - { - p: RESULT_FLAT[p] - for p in ( - "chrome.manifest", - "chrome/chrome.manifest", - "chrome/f/f.manifest", - "chrome/f/oo/bar/baz", - "chrome/f/oo/baz", - "chrome/f/oo/qux", - "components/foo.xpt", - "components/bar.xpt", + }, + "chrome.manifest": [ + "manifest components/components.manifest", + ], + "components/components.manifest": [ + "binary-component foo.so", + ], + "app/omni.foo": { + p: RESULT_FLAT["app/" + p] + for p in chain( + ( + "chrome.manifest", + "chrome/chrome.manifest", + "chrome/foo/foo", + "components/components.manifest", + "components/foo.js", + ), + ( + mozpath.relpath(p, "app") + for p in RESULT_FLAT.keys() + if p.startswith("app/chrome/addons/addon2/") + ), ) - } -) + }, +}) + +RESULT_OMNIJAR["omni.foo"].update({ + p: RESULT_FLAT[p] + for p in ( + "chrome.manifest", + "chrome/chrome.manifest", + "chrome/f/f.manifest", + "chrome/f/oo/bar/baz", + "chrome/f/oo/baz", + "chrome/f/oo/qux", + "components/foo.xpt", + "components/bar.xpt", + ) +}) RESULT_OMNIJAR_WITH_SUBPATH = { k.replace("omni.foo", "bar/omni.foo"): v for k, v in RESULT_OMNIJAR.items() @@ -453,7 +443,7 @@ class TestFormatters(TestErrors, unittest.TestCase): self.assertEqual( str(e.exception), - 'error: "content foo foo/" overrides ' '"content foo foo/unix"', + 'error: "content foo foo/" overrides "content foo foo/unix"', ) # Chrome with the same name and same flags overrides the previous @@ -475,7 +465,7 @@ class TestFormatters(TestErrors, unittest.TestCase): self.assertEqual( str(e.exception), - 'error: "content bar bar/unix" overrides ' '"content bar bar/win os=WINNT"', + 'error: "content bar bar/unix" overrides "content bar bar/win os=WINNT"', ) # Adding something more specific still works. diff --git a/python/mozbuild/mozpack/test/test_packager_l10n.py b/python/mozbuild/mozpack/test/test_packager_l10n.py @@ -25,41 +25,39 @@ class TestL10NRepack(unittest.TestCase): dict_cc = GeneratedFile(b"dict_cc") barbaz = GeneratedFile(b"barbaz") lst = GeneratedFile(b"foo\nbar") - app_finder = MockFinder( - { - "bar/foo": foo, - "chrome/foo/foobar": foobar, - "chrome/qux/qux.properties": qux, - "chrome/qux/baz/baz.properties": baz, - "chrome/chrome.manifest": ManifestFile( - "chrome", - [ - ManifestContent("chrome", "foo", "foo/"), - ManifestLocale("chrome", "qux", "en-US", "qux/"), - ], - ), - "chrome.manifest": ManifestFile( - "", [Manifest("", "chrome/chrome.manifest")] - ), - "dict/aa": dict_aa, - "app/chrome/bar/barbaz.dtd": barbaz, - "app/chrome/chrome.manifest": ManifestFile( - "app/chrome", [ManifestLocale("app/chrome", "bar", "en-US", "bar/")] - ), - "app/chrome.manifest": ManifestFile( - "app", [Manifest("app", "chrome/chrome.manifest")] - ), - "app/dict/bb": dict_bb, - "app/dict/cc": dict_cc, - "app/chrome/bar/search/foo.xml": foo, - "app/chrome/bar/search/bar.xml": bar, - "app/chrome/bar/search/lst.txt": lst, - "META-INF/foo": foo, # Stripped. - "inner/META-INF/foo": foo, # Not stripped. - "app/META-INF/foo": foo, # Stripped. - "app/inner/META-INF/foo": foo, # Not stripped. - } - ) + app_finder = MockFinder({ + "bar/foo": foo, + "chrome/foo/foobar": foobar, + "chrome/qux/qux.properties": qux, + "chrome/qux/baz/baz.properties": baz, + "chrome/chrome.manifest": ManifestFile( + "chrome", + [ + ManifestContent("chrome", "foo", "foo/"), + ManifestLocale("chrome", "qux", "en-US", "qux/"), + ], + ), + "chrome.manifest": ManifestFile( + "", [Manifest("", "chrome/chrome.manifest")] + ), + "dict/aa": dict_aa, + "app/chrome/bar/barbaz.dtd": barbaz, + "app/chrome/chrome.manifest": ManifestFile( + "app/chrome", [ManifestLocale("app/chrome", "bar", "en-US", "bar/")] + ), + "app/chrome.manifest": ManifestFile( + "app", [Manifest("app", "chrome/chrome.manifest")] + ), + "app/dict/bb": dict_bb, + "app/dict/cc": dict_cc, + "app/chrome/bar/search/foo.xml": foo, + "app/chrome/bar/search/bar.xml": bar, + "app/chrome/bar/search/lst.txt": lst, + "META-INF/foo": foo, # Stripped. + "inner/META-INF/foo": foo, # Not stripped. + "app/META-INF/foo": foo, # Stripped. + "app/inner/META-INF/foo": foo, # Not stripped. + }) app_finder.jarlogs = {} app_finder.base = "app" foo_l10n = GeneratedFile(b"foo_l10n") @@ -67,35 +65,33 @@ class TestL10NRepack(unittest.TestCase): baz_l10n = GeneratedFile(b"baz_l10n") barbaz_l10n = GeneratedFile(b"barbaz_l10n") lst_l10n = GeneratedFile(b"foo\nqux") - l10n_finder = MockFinder( - { - "chrome/qux-l10n/qux.properties": qux_l10n, - "chrome/qux-l10n/baz/baz.properties": baz_l10n, - "chrome/chrome.manifest": ManifestFile( - "chrome", - [ - ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"), - ], - ), - "chrome.manifest": ManifestFile( - "", [Manifest("", "chrome/chrome.manifest")] - ), - "dict/bb": dict_bb, - "dict/cc": dict_cc, - "app/chrome/bar-l10n/barbaz.dtd": barbaz_l10n, - "app/chrome/chrome.manifest": ManifestFile( - "app/chrome", - [ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")], - ), - "app/chrome.manifest": ManifestFile( - "app", [Manifest("app", "chrome/chrome.manifest")] - ), - "app/dict/aa": dict_aa, - "app/chrome/bar-l10n/search/foo.xml": foo_l10n, - "app/chrome/bar-l10n/search/qux.xml": qux_l10n, - "app/chrome/bar-l10n/search/lst.txt": lst_l10n, - } - ) + l10n_finder = MockFinder({ + "chrome/qux-l10n/qux.properties": qux_l10n, + "chrome/qux-l10n/baz/baz.properties": baz_l10n, + "chrome/chrome.manifest": ManifestFile( + "chrome", + [ + ManifestLocale("chrome", "qux", "x-test", "qux-l10n/"), + ], + ), + "chrome.manifest": ManifestFile( + "", [Manifest("", "chrome/chrome.manifest")] + ), + "dict/bb": dict_bb, + "dict/cc": dict_cc, + "app/chrome/bar-l10n/barbaz.dtd": barbaz_l10n, + "app/chrome/chrome.manifest": ManifestFile( + "app/chrome", + [ManifestLocale("app/chrome", "bar", "x-test", "bar-l10n/")], + ), + "app/chrome.manifest": ManifestFile( + "app", [Manifest("app", "chrome/chrome.manifest")] + ), + "app/dict/aa": dict_aa, + "app/chrome/bar-l10n/search/foo.xml": foo_l10n, + "app/chrome/bar-l10n/search/qux.xml": qux_l10n, + "app/chrome/bar-l10n/search/lst.txt": lst_l10n, + }) l10n_finder.base = "l10n" copier = FileRegistry() formatter = l10n.FlatFormatter(copier) diff --git a/python/mozbuild/mozpack/test/test_path.py b/python/mozbuild/mozpack/test/test_path.py @@ -56,9 +56,11 @@ class TestPath(unittest.TestCase): def test_commonprefix(self): self.assertEqual( - commonprefix( - [self.SEP.join(("foo", "bar", "baz")), "foo/qux", "foo/baz/qux"] - ), + commonprefix([ + self.SEP.join(("foo", "bar", "baz")), + "foo/qux", + "foo/baz/qux", + ]), "foo/", ) self.assertEqual( diff --git a/python/mozbuild/mozpack/test/test_unify.py b/python/mozbuild/mozpack/test/test_unify.py @@ -47,18 +47,18 @@ class TestUnifiedFinder(TestUnified): sorted=["test"], ) self.assertEqual( - sorted( - [(f, c.open().read().decode("utf-8")) for f, c in finder.find("foo")] - ), + sorted([ + (f, c.open().read().decode("utf-8")) for f, c in finder.find("foo") + ]), [("foo/bar", "foobar"), ("foo/baz", "foobaz")], ) self.assertRaises(ErrorMessage, any, finder.find("bar")) self.assertRaises(ErrorMessage, any, finder.find("baz")) self.assertRaises(ErrorMessage, any, finder.find("qux")) self.assertEqual( - sorted( - [(f, c.open().read().decode("utf-8")) for f, c in finder.find("test")] - ), + sorted([ + (f, c.open().read().decode("utf-8")) for f, c in finder.find("test") + ]), [("test/bar", "a\nb\nc\n"), ("test/foo", "a\nb\nc\n")], ) @@ -74,12 +74,10 @@ class TestUnifiedBuildFinder(TestUnified): self.create_one("a", "chrome/chrome.manifest", "a\nb\nc\n") self.create_one("b", "chrome/chrome.manifest", "b\nc\na\n") self.assertEqual( - sorted( - [ - (f, c.open().read().decode("utf-8")) - for f, c in finder.find("**/chrome.manifest") - ] - ), + sorted([ + (f, c.open().read().decode("utf-8")) + for f, c in finder.find("**/chrome.manifest") + ]), [("chrome.manifest", "a\nb\nc\n"), ("chrome/chrome.manifest", "a\nb\nc\n")], ) @@ -87,59 +85,51 @@ class TestUnifiedBuildFinder(TestUnified): self.create_one( "a", "chrome/browser/foo/buildconfig.html", - "\n".join( - [ - "<html>", - " <body>", - " <div>", - " <h1>Build Configuration</h1>", - " <div>foo</div>", - " </div>", - " </body>", - "</html>", - ] - ), + "\n".join([ + "<html>", + " <body>", + " <div>", + " <h1>Build Configuration</h1>", + " <div>foo</div>", + " </div>", + " </body>", + "</html>", + ]), ) self.create_one( "b", "chrome/browser/foo/buildconfig.html", - "\n".join( - [ - "<html>", - " <body>", - " <div>", - " <h1>Build Configuration</h1>", - " <div>bar</div>", - " </div>", - " </body>", - "</html>", - ] - ), + "\n".join([ + "<html>", + " <body>", + " <div>", + " <h1>Build Configuration</h1>", + " <div>bar</div>", + " </div>", + " </body>", + "</html>", + ]), ) self.assertEqual( - sorted( - [ - (f, c.open().read().decode("utf-8")) - for f, c in finder.find("**/buildconfig.html") - ] - ), + sorted([ + (f, c.open().read().decode("utf-8")) + for f, c in finder.find("**/buildconfig.html") + ]), [ ( "chrome/browser/foo/buildconfig.html", - "\n".join( - [ - "<html>", - " <body>", - " <div>", - " <h1>Build Configuration</h1>", - " <div>foo</div>", - " <hr> </hr>", - " <div>bar</div>", - " </div>", - " </body>", - "</html>", - ] - ), + "\n".join([ + "<html>", + " <body>", + " <div>", + " <h1>Build Configuration</h1>", + " <div>foo</div>", + " <hr> </hr>", + " <div>bar</div>", + " </div>", + " </body>", + "</html>", + ]), ) ], ) @@ -171,23 +161,19 @@ class TestUnifiedBuildFinder(TestUnified): target_tag = "<{em}targetPlatform>{platform}</{em}targetPlatform>" target_attr = '{em}targetPlatform="{platform}" ' - rdf_tag = "".join( - [ - '<{RDF}Description {em}bar="bar" {em}qux="qux">', - "<{em}foo>foo</{em}foo>", - "{targets}", - "<{em}baz>baz</{em}baz>", - "</{RDF}Description>", - ] - ) - rdf_attr = "".join( - [ - '<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">', - "{targets}", - "<{em}foo>foo</{em}foo><{em}baz>baz</{em}baz>", - "</{RDF}Description>", - ] - ) + rdf_tag = "".join([ + '<{RDF}Description {em}bar="bar" {em}qux="qux">', + "<{em}foo>foo</{em}foo>", + "{targets}", + "<{em}baz>baz</{em}baz>", + "</{RDF}Description>", + ]) + rdf_attr = "".join([ + '<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">', + "{targets}", + "<{em}foo>foo</{em}foo><{em}baz>baz</{em}baz>", + "</{RDF}Description>", + ]) for descr_ns, target_ns in (("RDF:", ""), ("", "em:"), ("RDF:", "em:")): # First we need to infuse the above strings with our namespaces and @@ -236,12 +222,10 @@ class TestUnifiedBuildFinder(TestUnified): results.append((filename, result)) self.assertEqual( - sorted( - [ - (f, c.open().read().decode("utf-8")) - for f, c in finder.find("**/install.rdf") - ] - ), + sorted([ + (f, c.open().read().decode("utf-8")) + for f, c in finder.find("**/install.rdf") + ]), results, ) diff --git a/python/mozlint/mozlint/errors.py b/python/mozlint/mozlint/errors.py @@ -29,5 +29,5 @@ class LintersNotConfigured(LintException): def __init__(self): LintException.__init__( self, - "No linters registered! Use `LintRoller.read` " "to register a linter.", + "No linters registered! Use `LintRoller.read` to register a linter.", ) diff --git a/python/mozlint/mozlint/formatters/stylish.py b/python/mozlint/mozlint/formatters/stylish.py @@ -29,9 +29,7 @@ class StylishFormatter: fmt = """ {c1}{lineno}{column} {c2}{level}{normal} {message} {c1}{rule}({linter}){source}{normal} -{diff}""".lstrip( - "\n" - ) +{diff}""".lstrip("\n") fmt_summary = ( "{t.bold}{c}\u2716 {problem} ({error}, {warning}{failure}, {fixed}){t.normal}" ) diff --git a/python/mozlint/mozlint/formatters/summary.py b/python/mozlint/mozlint/formatters/summary.py @@ -33,12 +33,12 @@ class SummaryFormatter: ] abspath = mozpath.join(commonprefix, *parts) - summary[abspath][0] += len( - [r for r in result.issues[path] if r.level == "error"] - ) - summary[abspath][1] += len( - [r for r in result.issues[path] if r.level == "warning"] - ) + summary[abspath][0] += len([ + r for r in result.issues[path] if r.level == "error" + ]) + summary[abspath][1] += len([ + r for r in result.issues[path] if r.level == "warning" + ]) summary[abspath][1] += result.suppressed_warnings[path] msg = [] diff --git a/python/mozlint/mozlint/parser.py b/python/mozlint/mozlint/parser.py @@ -39,7 +39,7 @@ class Parser: if missing_attrs: raise LinterParseError( relpath, - "Missing required attribute(s): " "{}".format(",".join(missing_attrs)), + "Missing required attribute(s): {}".format(",".join(missing_attrs)), ) if linter["type"] not in supported_types: @@ -54,7 +54,7 @@ class Parser: ): raise LinterParseError( relpath, - f"The {attr} directive must be a " "list of strings!", + f"The {attr} directive must be a list of strings!", ) invalid_paths = set() for path in linter[attr]: diff --git a/python/mozlint/mozlint/pathutils.py b/python/mozlint/mozlint/pathutils.py @@ -196,7 +196,6 @@ def filterpaths( # First handle include/exclude directives # that exist (i.e don't have globs) for inc in include: - # If the include directive is a file and we're specifically linting # it, keep it. if inc.isfile and path.path == inc.path: diff --git a/python/mozlint/mozlint/roller.py b/python/mozlint/mozlint/roller.py @@ -399,12 +399,9 @@ class LintRoller: # Make sure all paths are absolute. Join `paths` to cwd and `vcs_paths` to root. paths = set(map(os.path.abspath, paths)) - vcs_paths = set( - [ - os.path.join(self.root, p) if not os.path.isabs(p) else p - for p in vcs_paths - ] - ) + vcs_paths = set([ + os.path.join(self.root, p) if not os.path.isabs(p) else p for p in vcs_paths + ]) num_procs = num_procs or cpu_count() jobs = list(self._generate_jobs(paths, vcs_paths, num_procs)) diff --git a/python/mozlint/test/test_cli.py b/python/mozlint/test/test_cli.py @@ -156,15 +156,13 @@ def test_cli_run_with_stdin_filename(run, filedir, capfd, monkeypatch, tmp_path) monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"foobar\n"))) tmpfile = tmp_path / "temp" - run( - [ - "-l", - "string", - f"--stdin-filename={filedir}/foobar.py", - "--dump-stdin-file", - str(tmpfile), - ] - ) + run([ + "-l", + "string", + f"--stdin-filename={filedir}/foobar.py", + "--dump-stdin-file", + str(tmpfile), + ]) out, err = capfd.readouterr() assert out == "" assert tmpfile.read_text() == "foobar\n" diff --git a/python/mozlint/test/test_editor.py b/python/mozlint/test/test_editor.py @@ -28,23 +28,21 @@ def capture_commands(monkeypatch): @pytest.fixture def result(): result = ResultSummary("/fake/root") - result.issues["foo.py"].extend( - [ - Issue( - linter="no-foobar", - path="foo.py", - lineno=1, - message="Oh no!", - ), - Issue( - linter="no-foobar", - path="foo.py", - lineno=3, - column=10, - message="To Yuma!", - ), - ] - ) + result.issues["foo.py"].extend([ + Issue( + linter="no-foobar", + path="foo.py", + lineno=1, + message="Oh no!", + ), + Issue( + linter="no-foobar", + path="foo.py", + lineno=3, + column=10, + message="To Yuma!", + ), + ]) return result diff --git a/python/mozlint/test/test_formatters.py b/python/mozlint/test/test_formatters.py @@ -67,18 +67,14 @@ TEST-UNEXPECTED-WARNING | /fake/root/d/e/f.txt:4:2 | oh no bar (bar-not-allowed) {abc}:4:10: baz error: oh no baz {abc}:5: foo-diff error: oh no foo-diff {def}:4:2: bar-not-allowed warning: oh no bar -""".format( - **NORMALISED_PATHS - ).strip(), +""".format(**NORMALISED_PATHS).strip(), }, "summary": { "kwargs": {}, "format": """ {root}/a: 3 errors {root}/d: 0 errors, 1 warning -""".format( - **NORMALISED_PATHS - ).strip(), +""".format(**NORMALISED_PATHS).strip(), }, } diff --git a/python/mozperftest/mozperftest/environment.py b/python/mozperftest/mozperftest/environment.py @@ -19,9 +19,9 @@ class MachEnvironment(MachLogger): def __init__(self, mach_cmd, flavor="desktop-browser", hooks=None, **kwargs): MachLogger.__init__(self, mach_cmd) self._mach_cmd = mach_cmd - self._mach_args = dict( - [(self._normalize(key), value) for key, value in kwargs.items()] - ) + self._mach_args = dict([ + (self._normalize(key), value) for key, value in kwargs.items() + ]) self.layers = [] if flavor not in FLAVORS: raise NotImplementedError(flavor) diff --git a/python/mozperftest/mozperftest/metrics/common.py b/python/mozperftest/mozperftest/metrics/common.py @@ -245,9 +245,9 @@ class MetricsStorage: for data_type, data_info in results.items(): newresults = [] for res in data_info: - if any([met["name"] in res["subtest"] for met in metrics]) and not any( - [met in res["subtest"] for met in exclude] - ): + if any([met["name"] in res["subtest"] for met in metrics]) and not any([ + met in res["subtest"] for met in exclude + ]): res["transformer"] = self.ptnb_config[data_type][ "custom_transformer" ] diff --git a/python/mozperftest/mozperftest/metrics/notebookupload.py b/python/mozperftest/mozperftest/metrics/notebookupload.py @@ -16,42 +16,40 @@ class Notebook(Layer): activated = False arguments = COMMON_ARGS - arguments.update( - { - "analysis": { - "nargs": "*", - "default": [], - "help": "List of analyses to run in Iodide.", - }, - "analyze-strings": { - "action": "store_true", - "default": False, - "help": ( - "If set, strings won't be filtered out of the results to analyze in Iodide." - ), - }, - "no-server": { - "action": "store_true", - "default": False, - "help": "If set, the data won't be opened in Iodide.", - }, - "compare-to": { - "nargs": "*", - "default": [], - "help": ( - "Compare the results from this test to the historical data in the folder(s) " - "specified through this option. Only JSON data can be processed for the " - "moment. Each folder containing those JSONs is considered as a distinct " - "data point to compare with the newest run." - ), - }, - "stats": { - "action": "store_true", - "default": False, - "help": "If set, browsertime statistics will be reported.", - }, - } - ) + arguments.update({ + "analysis": { + "nargs": "*", + "default": [], + "help": "List of analyses to run in Iodide.", + }, + "analyze-strings": { + "action": "store_true", + "default": False, + "help": ( + "If set, strings won't be filtered out of the results to analyze in Iodide." + ), + }, + "no-server": { + "action": "store_true", + "default": False, + "help": "If set, the data won't be opened in Iodide.", + }, + "compare-to": { + "nargs": "*", + "default": [], + "help": ( + "Compare the results from this test to the historical data in the folder(s) " + "specified through this option. Only JSON data can be processed for the " + "moment. Each folder containing those JSONs is considered as a distinct " + "data point to compare with the newest run." + ), + }, + "stats": { + "action": "store_true", + "default": False, + "help": "If set, browsertime statistics will be reported.", + }, + }) def run(self, metadata): exclusions = None @@ -73,12 +71,10 @@ class Notebook(Layer): raise Exception(f"{dirpath} is not a directory") # TODO: Handle more than just JSON data. for jsonfile in dirpath.rglob("*.json"): - metadata.add_result( - { - "results": str(jsonfile.resolve()), - "name": jsonfile.parent.name, - } - ) + metadata.add_result({ + "results": str(jsonfile.resolve()), + "name": jsonfile.parent.name, + }) results = filtered_metrics( metadata, diff --git a/python/mozperftest/mozperftest/metrics/perfboard/grafana.py b/python/mozperftest/mozperftest/metrics/perfboard/grafana.py @@ -26,9 +26,9 @@ class Grafana: def get_dashboard(self, title): existing = self.client.search.search_dashboards(tag="component") - existing = dict( - [(dashboard["title"].lower(), dashboard["uid"]) for dashboard in existing] - ) + existing = dict([ + (dashboard["title"].lower(), dashboard["uid"]) for dashboard in existing + ]) if title in existing: return self.client.dashboard.get_dashboard(existing[title]) self.layer.debug(f"Creating dashboard {title}") diff --git a/python/mozperftest/mozperftest/metrics/perfboard/influx.py b/python/mozperftest/mozperftest/metrics/perfboard/influx.py @@ -17,49 +17,47 @@ class Influx(Layer): name = "perfboard" activated = False arguments = COMMON_ARGS - arguments.update( - { - "dashboard": { - "type": str, - "default": None, - "help": "Name of the dashboard - defaults to the script" - " `component` metadata. When not set, falls back to" - " `perftest`", - }, - "influx-host": { - "type": str, - "default": "perfboard.dev.mozaws.net", - }, - "influx-user": { - "type": str, - "default": "admin", - }, - "influx-port": { - "type": int, - "default": 8086, - }, - "influx-password": { - "type": str, - "default": None, - }, - "influx-db": { - "type": str, - "default": "perf", - }, - "grafana-host": { - "type": str, - "default": "perfboard.dev.mozaws.net", - }, - "grafana-key": { - "type": str, - "default": None, - }, - "grafana-port": { - "type": int, - "default": 3000, - }, - } - ) + arguments.update({ + "dashboard": { + "type": str, + "default": None, + "help": "Name of the dashboard - defaults to the script" + " `component` metadata. When not set, falls back to" + " `perftest`", + }, + "influx-host": { + "type": str, + "default": "perfboard.dev.mozaws.net", + }, + "influx-user": { + "type": str, + "default": "admin", + }, + "influx-port": { + "type": int, + "default": 8086, + }, + "influx-password": { + "type": str, + "default": None, + }, + "influx-db": { + "type": str, + "default": "perf", + }, + "grafana-host": { + "type": str, + "default": "perfboard.dev.mozaws.net", + }, + "grafana-key": { + "type": str, + "default": None, + }, + "grafana-port": { + "type": int, + "default": 3000, + }, + }) def _setup(self): venv = self.mach_cmd.virtualenv_manager @@ -157,9 +155,9 @@ class Influx(Layer): metric_name = line["subtest"] short_name = metric_name.split(".")[-1] short_name = short_name.lower() - if metrics and not any( - [m.lower().startswith(short_name.lower()) for m in metrics] - ): + if metrics and not any([ + m.lower().startswith(short_name.lower()) for m in metrics + ]): continue values = [v["value"] for v in line["data"]] data[short_name].extend(values) diff --git a/python/mozperftest/mozperftest/metrics/perfherder.py b/python/mozperftest/mozperftest/metrics/perfherder.py @@ -31,23 +31,21 @@ class Perfherder(Layer): activated = False arguments = COMMON_ARGS - arguments.update( - { - "stats": { - "action": "store_true", - "default": False, - "help": "If set, browsertime statistics will be reported.", - }, - "timestamp": { - "type": float, - "default": None, - "help": ( - "Timestamp to use for the perfherder data. Can be the " - "current date or a past date if needed." - ), - }, - } - ) + arguments.update({ + "stats": { + "action": "store_true", + "default": False, + "help": "If set, browsertime statistics will be reported.", + }, + "timestamp": { + "type": float, + "default": None, + "help": ( + "Timestamp to use for the perfherder data. Can be the " + "current date or a past date if needed." + ), + }, + }) def run(self, metadata): """Processes the given results into a perfherder-formatted data blob. diff --git a/python/mozperftest/mozperftest/metrics/utils.py b/python/mozperftest/mozperftest/metrics/utils.py @@ -19,9 +19,14 @@ with pathlib.Path(PARENT, "schemas", "intermediate-results-schema.json").open() # These are the properties we know about in the schema. # If anything other than these is present, then we will # fail validation. -KNOWN_PERFHERDER_PROPS = set( - ["name", "value", "unit", "lowerIsBetter", "shouldAlert", "alertThreshold"] -) +KNOWN_PERFHERDER_PROPS = set([ + "name", + "value", + "unit", + "lowerIsBetter", + "shouldAlert", + "alertThreshold", +]) KNOWN_SUITE_PROPS = set( set(["results", "transformer", "transformer-options", "extraOptions", "framework"]) | KNOWN_PERFHERDER_PROPS @@ -125,7 +130,7 @@ def metric_fields(value): raise ValueError(f"Unexpected metrics definition {field}") if sfield[0] not in KNOWN_SUITE_PROPS: raise ValueError( - f"Unknown field '{sfield[0]}', should be in " f"{KNOWN_SUITE_PROPS}" + f"Unknown field '{sfield[0]}', should be in {KNOWN_SUITE_PROPS}" ) sfield = [sfield[0], sfield[2]] diff --git a/python/mozperftest/mozperftest/metrics/visualmetrics.py b/python/mozperftest/mozperftest/metrics/visualmetrics.py @@ -87,14 +87,12 @@ class VisualMetrics(Layer): self.info(f"Treated {treated} videos.") if len(self.metrics) > 0: - metadata.add_result( - { - "name": metadata.script["name"] + "-vm", - "framework": {"name": "mozperftest"}, - "transformer": "mozperftest.metrics.visualmetrics:VisualData", - "results": list(self.metrics.values()), - } - ) + metadata.add_result({ + "name": metadata.script["name"] + "-vm", + "framework": {"name": "mozperftest"}, + "transformer": "mozperftest.metrics.visualmetrics:VisualData", + "results": list(self.metrics.values()), + }) # we also extend --perfherder-metrics and --console-metrics if they # are activated diff --git a/python/mozperftest/mozperftest/runner.py b/python/mozperftest/mozperftest/runner.py @@ -14,6 +14,7 @@ This runner can be executed in two different ways: When the module is executed directly, if the --on-try option is used, it will fetch arguments from Tascluster's parameters. """ + import json import logging import os diff --git a/python/mozperftest/mozperftest/system/geckoprofiler.py b/python/mozperftest/mozperftest/system/geckoprofiler.py @@ -80,9 +80,9 @@ class GeckoProfilerController: config_content = f"""env: MOZ_PROFILER_STARTUP: 1 - MOZ_PROFILER_STARTUP_INTERVAL: {geckoprofiler_opts['interval']} - MOZ_PROFILER_STARTUP_FEATURES: {geckoprofiler_opts['features']} - MOZ_PROFILER_STARTUP_FILTERS: {geckoprofiler_opts['filters']} + MOZ_PROFILER_STARTUP_INTERVAL: {geckoprofiler_opts["interval"]} + MOZ_PROFILER_STARTUP_FEATURES: {geckoprofiler_opts["features"]} + MOZ_PROFILER_STARTUP_FILTERS: {geckoprofiler_opts["filters"]} """.encode() with tempfile.NamedTemporaryFile(delete=False) as config_file: diff --git a/python/mozperftest/mozperftest/system/proxy.py b/python/mozperftest/mozperftest/system/proxy.py @@ -160,9 +160,9 @@ class ProxyRunner(Layer): if metadata.flavor == "mobile-browser": command.extend(["--tool=%s" % "mitmproxy-android"]) command.extend(["--binary=android"]) - command.extend( - [f"--app={get_pretty_app_name(self.get_arg('android-app-name'))}"] - ) + command.extend([ + f"--app={get_pretty_app_name(self.get_arg('android-app-name'))}" + ]) else: command.extend(["--tool=%s" % "mitmproxy"]) # XXX See bug 1712337, we need a single point where we can get the binary used from diff --git a/python/mozperftest/mozperftest/test/alert.py b/python/mozperftest/mozperftest/test/alert.py @@ -462,12 +462,10 @@ class AlertTestRunner(Layer): text=False, ) if test not in self.perfherder_data: - failed_commands.append( - { - "cmd": cmd, - "test": test, - } - ) + failed_commands.append({ + "cmd": cmd, + "test": test, + }) # Output results in a more readable manner for test, perfherder_data in self.perfherder_data.items(): diff --git a/python/mozperftest/mozperftest/test/androidlog.py b/python/mozperftest/mozperftest/test/androidlog.py @@ -50,13 +50,11 @@ class AndroidLog(Layer): "transform-subtest-name": self.get_arg("subtest-name"), } - metadata.add_result( - { - "results": str(self._get_logcat()), - "transformer": "LogCatTimeTransformer", - "transformer-options": options, - "name": "LogCat", - } - ) + metadata.add_result({ + "results": str(self._get_logcat()), + "transformer": "LogCatTimeTransformer", + "transformer-options": options, + "name": "LogCat", + }) return metadata diff --git a/python/mozperftest/mozperftest/test/browsertime/runner.py b/python/mozperftest/mozperftest/test/browsertime/runner.py @@ -353,9 +353,10 @@ class BrowsertimeRunner(NodeRunner): existing = self.get_arg("browsertime-existing-results") if existing: - metadata.add_result( - {"results": existing, "name": self._test_script["name"]} - ) + metadata.add_result({ + "results": existing, + "name": self._test_script["name"], + }) return metadata cycles = self.get_arg("cycles", 1) @@ -464,8 +465,9 @@ class BrowsertimeRunner(NodeRunner): if exit_code != 0: raise NodeException(exit_code) - metadata.add_result( - {"results": str(result_dir), "name": self._test_script["name"]} - ) + metadata.add_result({ + "results": str(result_dir), + "name": self._test_script["name"], + }) return metadata diff --git a/python/mozperftest/mozperftest/test/browsertime/visualtools.py b/python/mozperftest/mozperftest/test/browsertime/visualtools.py @@ -1,8 +1,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" Collects visualmetrics dependencies. -""" +"""Collects visualmetrics dependencies.""" + import contextlib import os import subprocess diff --git a/python/mozperftest/mozperftest/test/mochitest.py b/python/mozperftest/mozperftest/test/mochitest.py @@ -138,13 +138,11 @@ class Mochitest(Layer): if self.get_arg("gecko-profile"): gecko_profile_args.append("--profiler") - gecko_profile_args.extend( - [ - f"--setenv=MOZ_PROFILER_STARTUP_FEATURES={gecko_profile_features}", - f"--setenv=MOZ_PROFILER_STARTUP_FILTERS={gecko_profile_threads}", - f"--setenv=MOZ_PROFILER_STARTUP_ENTRIES={gecko_profile_entries}", - ] - ) + gecko_profile_args.extend([ + f"--setenv=MOZ_PROFILER_STARTUP_FEATURES={gecko_profile_features}", + f"--setenv=MOZ_PROFILER_STARTUP_FILTERS={gecko_profile_threads}", + f"--setenv=MOZ_PROFILER_STARTUP_ENTRIES={gecko_profile_entries}", + ]) if gecko_profile_interval: gecko_profile_args.append( f"--setenv=MOZ_PROFILER_STARTUP_INTERVAL={gecko_profile_interval}" @@ -192,23 +190,19 @@ class Mochitest(Layer): if not ON_TRY: os.environ["MOZ_HOST_BIN"] = self.mach_cmd.bindir - mochitest_android_args.extend( - [ - f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}", - ] - ) + mochitest_android_args.extend([ + f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}", + ]) else: os.environ["MOZ_HOST_BIN"] = str( Path(os.getenv("MOZ_FETCHES_DIR"), "hostutils") ) - mochitest_android_args.extend( - [ - f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}", - f"--remote-webserver={os.environ['HOST_IP']}", - "--http-port=8854", - "--ssl-port=4454", - ] - ) + mochitest_android_args.extend([ + f"--setenv=MOZ_HOST_BIN={os.environ['MOZ_HOST_BIN']}", + f"--remote-webserver={os.environ['HOST_IP']}", + "--http-port=8854", + "--ssl-port=4454", + ]) return mochitest_android_args @@ -240,8 +234,7 @@ class Mochitest(Layer): ) if not manifest_flavor: raise MissingMochitestInformation( - "Mochitest flavor needs to be provided" - "(e.g. plain, browser-chrome, ...)" + "Mochitest flavor needs to be provided(e.g. plain, browser-chrome, ...)" ) manifest_path = Path(test.parent, manifest_name) @@ -307,7 +300,6 @@ class Mochitest(Layer): results = [] cycles = self.get_arg("cycles", 1) for cycle in range(1, cycles + 1): - metadata.run_hook( "before_cycle", metadata, self.env, cycle, metadata.script ) @@ -358,13 +350,11 @@ class Mochitest(Layer): if len(results) == 0: raise NoPerfMetricsError("mochitest") - metadata.add_result( - { - "name": test_name, - "framework": {"name": "mozperftest"}, - "transformer": "mozperftest.test.mochitest:MochitestData", - "results": results, - } - ) + metadata.add_result({ + "name": test_name, + "framework": {"name": "mozperftest"}, + "transformer": "mozperftest.test.mochitest:MochitestData", + "results": results, + }) return metadata diff --git a/python/mozperftest/mozperftest/test/shellscript.py b/python/mozperftest/mozperftest/test/shellscript.py @@ -200,14 +200,12 @@ class ShellScriptRunner(Layer): shutil.copytree(testing_dir, output_dir) self.env.set_arg("output", output_dir) - metadata.add_result( - { - "name": test["name"], - "framework": {"name": "mozperftest"}, - "transformer": "mozperftest.test.shellscript:ShellScriptData", - "shouldAlert": True, - "results": self.parse_metrics(), - } - ) + metadata.add_result({ + "name": test["name"], + "framework": {"name": "mozperftest"}, + "transformer": "mozperftest.test.shellscript:ShellScriptData", + "shouldAlert": True, + "results": self.parse_metrics(), + }) return metadata diff --git a/python/mozperftest/mozperftest/test/webpagetest.py b/python/mozperftest/mozperftest/test/webpagetest.py @@ -358,22 +358,20 @@ class WebPageTest(Layer): def add_wpt_run_to_metadata(self, wbt_run, metadata, website): requested_values = self.extract_desired_values_from_wpt_run(wbt_run) if requested_values is not None: - metadata.add_result( - { - "name": ("WebPageTest:" + re.match(r"(^.\w+)", website)[0]), - "framework": {"name": "mozperftest"}, - "transformer": "mozperftest.test.webpagetest:WebPageTestData", - "shouldAlert": True, - "results": [ - { - "values": [metric_value], - "name": metric_name, - "shouldAlert": True, - } - for metric_name, metric_value in requested_values.items() - ], - } - ) + metadata.add_result({ + "name": ("WebPageTest:" + re.match(r"(^.\w+)", website)[0]), + "framework": {"name": "mozperftest"}, + "transformer": "mozperftest.test.webpagetest:WebPageTestData", + "shouldAlert": True, + "results": [ + { + "values": [metric_value], + "name": metric_name, + "shouldAlert": True, + } + for metric_name, metric_value in requested_values.items() + ], + }) def extract_desired_values_from_wpt_run(self, wpt_run): view_types = ["firstView"] diff --git a/python/mozperftest/mozperftest/test/xpcshell.py b/python/mozperftest/mozperftest/test/xpcshell.py @@ -166,17 +166,15 @@ class XPCShell(Layer): if len(results.items()) == 0: raise NoPerfMetricsError("xpcshell") - metadata.add_result( - { - "name": test.name, - "framework": {"name": "mozperftest"}, - "transformer": "mozperftest.test.xpcshell:XPCShellData", - "results": [ - {"values": measures, "name": subtest} - for subtest, measures in results.items() - ], - } - ) + metadata.add_result({ + "name": test.name, + "framework": {"name": "mozperftest"}, + "transformer": "mozperftest.test.xpcshell:XPCShellData", + "results": [ + {"values": measures, "name": subtest} + for subtest, measures in results.items() + ], + }) return metadata diff --git a/python/mozperftest/mozperftest/tests/test_alert.py b/python/mozperftest/mozperftest/tests/test_alert.py @@ -41,13 +41,11 @@ def test_alert_basic_command(alert_json, expected_command): alert_layer = get_alert_layer(test.layers) line_handler = alert_layer.create_line_handler("a-test") - with mock.patch( - "mozperftest.test.alert.requests.get" - ) as mocked_request, mock.patch( - "mozperftest.test.alert.mozprocess" - ) as mocked_mozprocess, ( - MOCK_DATA_DIR / alert_json - ).open() as f: + with ( + mock.patch("mozperftest.test.alert.requests.get") as mocked_request, + mock.patch("mozperftest.test.alert.mozprocess") as mocked_mozprocess, + (MOCK_DATA_DIR / alert_json).open() as f, + ): mocked_response = mock.MagicMock() mocked_response.configure_mock(status_code=200) mocked_response.json.return_value = json.load(f) @@ -90,13 +88,11 @@ def test_alert_basic_command_test_specification(alert_json, tests, expected_comm alert_layer = get_alert_layer(test.layers) line_handler = alert_layer.create_line_handler("a-test") - with mock.patch( - "mozperftest.test.alert.requests.get" - ) as mocked_request, mock.patch( - "mozperftest.test.alert.mozprocess" - ) as mocked_mozprocess, ( - MOCK_DATA_DIR / alert_json - ).open() as f: + with ( + mock.patch("mozperftest.test.alert.requests.get") as mocked_request, + mock.patch("mozperftest.test.alert.mozprocess") as mocked_mozprocess, + (MOCK_DATA_DIR / alert_json).open() as f, + ): mocked_response = mock.MagicMock() mocked_response.configure_mock(status_code=200) mocked_response.json.return_value = json.load(f) @@ -294,15 +290,12 @@ def test_alert_exact_command( mach_cmd, metadata, env = get_running_env(**args) test = env.layers[TEST] - with mock.patch( - "mozperftest.test.alert.requests.get" - ) as mocked_request, mock.patch( - "mozperftest.test.alert.mozprocess" - ) as mocked_mozprocess, ( - MOCK_DATA_DIR / alert_json - ).open() as alert_file, ( - MOCK_DATA_DIR / task_info_json - ).open() as task_file: + with ( + mock.patch("mozperftest.test.alert.requests.get") as mocked_request, + mock.patch("mozperftest.test.alert.mozprocess") as mocked_mozprocess, + (MOCK_DATA_DIR / alert_json).open() as alert_file, + (MOCK_DATA_DIR / task_info_json).open() as task_file, + ): mocked_alert_response = mock.MagicMock() mocked_alert_response.configure_mock(status_code=200) mocked_alert_response.json.return_value = json.load(alert_file) @@ -333,13 +326,11 @@ def test_alert_basic_command_failed(): alert_layer = get_alert_layer(test.layers) alert_layer.create_line_handler("a-test") - with mock.patch( - "mozperftest.test.alert.requests.get" - ) as mocked_request, mock.patch( - "mozperftest.test.alert.mozprocess" - ) as mocked_mozprocess, ( - MOCK_DATA_DIR / alert_json - ).open() as f: + with ( + mock.patch("mozperftest.test.alert.requests.get") as mocked_request, + mock.patch("mozperftest.test.alert.mozprocess") as mocked_mozprocess, + (MOCK_DATA_DIR / alert_json).open() as f, + ): mocked_response = mock.MagicMock() mocked_response.configure_mock(status_code=200) mocked_response.json.return_value = json.load(f) diff --git a/python/mozperftest/mozperftest/tests/test_argparser.py b/python/mozperftest/mozperftest/tests/test_argparser.py @@ -31,18 +31,20 @@ def test_argparser_defaults(): def test_options(): assert Options.args["--proxy"]["help"] == "Activates the proxy layer" assert Options.args["--no-browsertime"]["help"] == ( - "Deactivates the " "browsertime layer" + "Deactivates the browsertime layer" ) def test_layer_option(): parser = PerftestArgumentParser() - assert parser.parse_args(["--notebook-metrics"]) == parser.parse_args( - ["--notebook-metrics", "--notebook"] - ) - assert parser.parse_known_args(["--notebook-metrics"]) == parser.parse_known_args( - ["--notebook-metrics", "--notebook"] - ) + assert parser.parse_args(["--notebook-metrics"]) == parser.parse_args([ + "--notebook-metrics", + "--notebook", + ]) + assert parser.parse_known_args(["--notebook-metrics"]) == parser.parse_known_args([ + "--notebook-metrics", + "--notebook", + ]) def test_bad_test_date(): diff --git a/python/mozperftest/mozperftest/tests/test_browsertime.py b/python/mozperftest/mozperftest/tests/test_browsertime.py @@ -150,10 +150,15 @@ def test_browsertime_no_reinstall(): tests=[EXAMPLE_TEST], ) - with mock.patch( - "mozperftest.test.browsertime.runner.pathlib.Path.open", - build_mock_open([BTIME_PKG_DEP, BTIME_PKG_NO_INSTALL]), - ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload): + with ( + mock.patch( + "mozperftest.test.browsertime.runner.pathlib.Path.open", + build_mock_open([BTIME_PKG_DEP, BTIME_PKG_NO_INSTALL]), + ), + mock.patch( + "mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload + ), + ): browser = env.layers[TEST] btime_layer = browser.layers[0] assert not btime_layer._should_install() @@ -172,10 +177,15 @@ def test_browsertime_should_reinstall(): tests=[EXAMPLE_TEST], ) - with mock.patch( - "mozperftest.test.browsertime.runner.pathlib.Path.open", - build_mock_open([BTIME_PKG_DEP, BTIME_PKG_REINSTALL]), - ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload): + with ( + mock.patch( + "mozperftest.test.browsertime.runner.pathlib.Path.open", + build_mock_open([BTIME_PKG_DEP, BTIME_PKG_REINSTALL]), + ), + mock.patch( + "mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload + ), + ): browser = env.layers[TEST] btime_layer = browser.layers[0] assert btime_layer._should_install() @@ -290,9 +300,9 @@ def test_add_options(): "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages" ) def test_install_url(*mocked): - url = "https://here/tarball/" + "".join( - [random.choice(string.hexdigits[:-6]) for c in range(40)] - ) + url = "https://here/tarball/" + "".join([ + random.choice(string.hexdigits[:-6]) for c in range(40) + ]) mach, metadata, env = get_running_env( browsertime_install_url=url, tests=[EXAMPLE_TEST], diff --git a/python/mozperftest/mozperftest/tests/test_change_detector.py b/python/mozperftest/mozperftest/tests/test_change_detector.py @@ -50,49 +50,43 @@ async def test_change_detector_basic(kwargs=None, return_value=({}, {})): @pytest.mark.asyncio async def test_change_detector_with_task_name(): - await test_change_detector_basic( - { - "task_names": ["test-platform/opt-browsertime-test"], + await test_change_detector_basic({ + "task_names": ["test-platform/opt-browsertime-test"], + "new_test_name": None, + "platform": None, + "new_platform": None, + "base_branch": "try", + "new_branch": "try", + "base_revision": "99", + "new_revision": "99", + }) + + +@pytest.mark.asyncio +async def test_change_detector_option_failure(): + with pytest.raises(Exception): + await test_change_detector_basic({ + "test_name": None, "new_test_name": None, - "platform": None, + "platform": "test-platform/opt", "new_platform": None, "base_branch": "try", "new_branch": "try", "base_revision": "99", "new_revision": "99", - } - ) - - -@pytest.mark.asyncio -async def test_change_detector_option_failure(): - with pytest.raises(Exception): - await test_change_detector_basic( - { - "test_name": None, - "new_test_name": None, - "platform": "test-platform/opt", - "new_platform": None, - "base_branch": "try", - "new_branch": "try", - "base_revision": "99", - "new_revision": "99", - } - ) + }) with pytest.raises(Exception): - await test_change_detector_basic( - { - "test_name": "browsertime-test", - "new_test_name": None, - "platform": None, - "new_platform": None, - "base_branch": "try", - "new_branch": "try", - "base_revision": "99", - "new_revision": "99", - } - ) + await test_change_detector_basic({ + "test_name": "browsertime-test", + "new_test_name": None, + "platform": None, + "new_platform": None, + "base_branch": "try", + "new_branch": "try", + "base_revision": "99", + "new_revision": "99", + }) @pytest.mark.asyncio diff --git a/python/mozperftest/mozperftest/tests/test_functionaltestrunner.py b/python/mozperftest/mozperftest/tests/test_functionaltestrunner.py @@ -8,12 +8,13 @@ from mozperftest.test.functionaltestrunner import ( def test_functionaltestrunner_pass(): - with mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, mock.patch( - "mozperftest.test.functionaltestrunner.load_class_from_path" - ) as load_class_path_mock, mock.patch( - "mozperftest.test.functionaltestrunner.FunctionalTestProcessor" - ), mock.patch( - "mozperftest.test.functionaltestrunner.mozlog" + with ( + mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, + mock.patch( + "mozperftest.test.functionaltestrunner.load_class_from_path" + ) as load_class_path_mock, + mock.patch("mozperftest.test.functionaltestrunner.FunctionalTestProcessor"), + mock.patch("mozperftest.test.functionaltestrunner.mozlog"), ): test_mock = mock.MagicMock() test_mock.test.return_value = 0 diff --git a/python/mozperftest/mozperftest/tests/test_influx.py b/python/mozperftest/mozperftest/tests/test_influx.py @@ -62,20 +62,18 @@ def mocks(): responses.add( responses.GET, re.compile(secrets), - body=json.dumps( - { - "secret": { - "influx_host": "influxdb", - "influx_port": 0, - "influx_user": "admin", - "influx_password": "pass", - "influx_db": "db", - "grafana_key": "xxx", - "grafana_host": "grafana", - "grafana_port": 0, - } + body=json.dumps({ + "secret": { + "influx_host": "influxdb", + "influx_port": 0, + "influx_user": "admin", + "influx_password": "pass", + "influx_db": "db", + "grafana_key": "xxx", + "grafana_host": "grafana", + "grafana_port": 0, } - ), + }), status=200, ) diff --git a/python/mozperftest/mozperftest/tests/test_mach_commands.py b/python/mozperftest/mozperftest/tests/test_mach_commands.py @@ -224,13 +224,13 @@ def test_help_help_selected(*kwargs): @mock.patch("mozperftest.utils.run_python_script") @mock.patch("mozperftest.utils.install_package") def test_side_by_side(mock1, mock2, mock3, patched_mozperftest_tools): - with mock.patch( - "mozperftest.utils.create_path", return_value="fake_path" - ) as _, mock.patch( - "mozperftest.runner._create_artifacts_dir", return_value="fake_path" - ) as _, mock.patch( - "mozperftest.runner._save_params", return_value="fake_path" - ) as _: + with ( + mock.patch("mozperftest.utils.create_path", return_value="fake_path") as _, + mock.patch( + "mozperftest.runner._create_artifacts_dir", return_value="fake_path" + ) as _, + mock.patch("mozperftest.runner._save_params", return_value="fake_path") as _, + ): with _get_tools_command() as (cmd, command_context), silence(command_context): cmd(command_context) patched_mozperftest_tools.run.assert_called() @@ -241,17 +241,20 @@ def test_side_by_side(mock1, mock2, mock3, patched_mozperftest_tools): @mock.patch("mozperftest.utils.run_python_script") @mock.patch("mozperftest.utils.install_package") def test_change_detector(mock1, mock2, mock3, patched_mozperftest_tools): - with mock.patch( - "mozperftest.utils.create_path", return_value="fake_path" - ) as _, mock.patch( - "mozperftest.runner._create_artifacts_dir", return_value="fake_path" - ) as _, mock.patch( - "mozperftest.runner._save_params", return_value="fake_path" - ) as _: - with _get_tools_command(tool="change-detector") as ( - cmd, - command_context, - ), silence(command_context): + with ( + mock.patch("mozperftest.utils.create_path", return_value="fake_path") as _, + mock.patch( + "mozperftest.runner._create_artifacts_dir", return_value="fake_path" + ) as _, + mock.patch("mozperftest.runner._save_params", return_value="fake_path") as _, + ): + with ( + _get_tools_command(tool="change-detector") as ( + cmd, + command_context, + ), + silence(command_context), + ): cmd(command_context) patched_mozperftest_tools.run.assert_called() diff --git a/python/mozperftest/mozperftest/tests/test_mochitest.py b/python/mozperftest/mozperftest/tests/test_mochitest.py @@ -33,12 +33,15 @@ def test_mochitest_metrics(*mocked): sys = env.layers[SYSTEM] mochitest = env.layers[TEST] - with mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, mock.patch( - "mozperftest.test.functionaltestrunner.load_class_from_path" - ) as load_class_path_mock, mock.patch( - "mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__" - ) as formatter_mock, mock.patch( - "mozperftest.test.mochitest.install_requirements_file" + with ( + mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, + mock.patch( + "mozperftest.test.functionaltestrunner.load_class_from_path" + ) as load_class_path_mock, + mock.patch( + "mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__" + ) as formatter_mock, + mock.patch("mozperftest.test.mochitest.install_requirements_file"), ): formatter_mock.return_value = lambda x: x @@ -86,12 +89,15 @@ def test_mochitest_android_metrics(*mocked): sys = env.layers[SYSTEM] mochitest = env.layers[TEST] - with mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, mock.patch( - "mozperftest.test.functionaltestrunner.load_class_from_path" - ) as load_class_path_mock, mock.patch( - "mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__" - ) as formatter_mock, mock.patch( - "mozperftest.test.mochitest.install_requirements_file" + with ( + mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, + mock.patch( + "mozperftest.test.functionaltestrunner.load_class_from_path" + ) as load_class_path_mock, + mock.patch( + "mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__" + ) as formatter_mock, + mock.patch("mozperftest.test.mochitest.install_requirements_file"), ): formatter_mock.return_value = lambda x: x @@ -139,12 +145,15 @@ def test_mochitest_multi_metrics(multimetrics_output): sys = env.layers[SYSTEM] mochitest = env.layers[TEST] - with mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, mock.patch( - "mozperftest.test.functionaltestrunner.load_class_from_path" - ) as load_class_path_mock, mock.patch( - "mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__" - ) as formatter_mock, mock.patch( - "mozperftest.test.mochitest.install_requirements_file" + with ( + mock.patch("moztest.resolve.TestResolver") as test_resolver_mock, + mock.patch( + "mozperftest.test.functionaltestrunner.load_class_from_path" + ) as load_class_path_mock, + mock.patch( + "mozperftest.test.functionaltestrunner.mozlog.formatters.MachFormatter.__new__" + ) as formatter_mock, + mock.patch("mozperftest.test.mochitest.install_requirements_file"), ): formatter_mock.return_value = lambda x: x @@ -203,12 +212,12 @@ def test_mochitest_profiling(fake_file_path): ) environ_patch.start() - with mock.patch( - "mozperftest.test.functionaltestrunner.FunctionalTestRunner.test" - ) as test_mock, mock.patch( - "mozperftest.test.mochitest.install_requirements_file" - ), mock.patch( - "mozperftest.test.mochitest.Path" + with ( + mock.patch( + "mozperftest.test.functionaltestrunner.FunctionalTestRunner.test" + ) as test_mock, + mock.patch("mozperftest.test.mochitest.install_requirements_file"), + mock.patch("mozperftest.test.mochitest.Path"), ): test_mock.return_value = (0, mock.MagicMock()) try: @@ -234,9 +243,12 @@ def test_mochitest_extra_args(*mocked): sys = env.layers[SYSTEM] mochitest = env.layers[TEST] - with mock.patch( - "mozperftest.test.functionaltestrunner.FunctionalTestRunner.test" - ) as test_mock, mock.patch("mozperftest.test.mochitest.install_requirements_file"): + with ( + mock.patch( + "mozperftest.test.functionaltestrunner.FunctionalTestRunner.test" + ) as test_mock, + mock.patch("mozperftest.test.mochitest.install_requirements_file"), + ): test_mock.return_value = (0, mock.MagicMock()) try: with pytest.raises(NoPerfMetricsError): diff --git a/python/mozperftest/mozperftest/tests/test_notebookupload.py b/python/mozperftest/mozperftest/tests/test_notebookupload.py @@ -54,9 +54,9 @@ def test_notebookupload_with_filter(notebook, no_filter): for data in data_dict["data"]: assert type(data["value"]) in (int, float) - notebook.assert_has_calls( - [mock.call().post_to_iodide(["scatterplot"], start_local_server=True)] - ) + notebook.assert_has_calls([ + mock.call().post_to_iodide(["scatterplot"], start_local_server=True) + ]) @pytest.mark.parametrize("stats", [False, True]) @@ -88,9 +88,9 @@ def test_compare_to_success(notebook, stats): else: assert any("statistics" in element["subtest"] for element in kwargs["data"]) - notebook.assert_has_calls( - [mock.call().post_to_iodide(["compare"], start_local_server=True)] - ) + notebook.assert_has_calls([ + mock.call().post_to_iodide(["compare"], start_local_server=True) + ]) @pytest.mark.parametrize("filepath", ["invalidPath", str(BT_DATA)]) diff --git a/python/mozperftest/mozperftest/tests/test_perfherder.py b/python/mozperftest/mozperftest/tests/test_perfherder.py @@ -229,14 +229,12 @@ def test_perfherder_simple_names(): # Check if only firstPaint/resource metrics were obtained and # that simplifications occurred - assert all( - [ - "firstPaint" in subtest["name"] - or "duration" in subtest["name"] - or "count" in subtest["name"] - for subtest in output["suites"][0]["subtests"] - ] - ) + assert all([ + "firstPaint" in subtest["name"] + or "duration" in subtest["name"] + or "count" in subtest["name"] + for subtest in output["suites"][0]["subtests"] + ]) found_all = {"firstPaint": False, "count": False, "duration": False} for subtest in output["suites"][0]["subtests"]: @@ -253,23 +251,19 @@ def test_perfherder_simple_names(): # Statistics are not simplified by default assert ( - len( - [ - subtest - for subtest in output["suites"][0]["subtests"] - if "statistics" in subtest["name"] - ] - ) + len([ + subtest + for subtest in output["suites"][0]["subtests"] + if "statistics" in subtest["name"] + ]) == 27 ) assert ( - len( - [ - subtest - for subtest in output["suites"][0]["subtests"] - if "statistics" not in subtest["name"] - ] - ) + len([ + subtest + for subtest in output["suites"][0]["subtests"] + if "statistics" not in subtest["name"] + ]) == 3 ) @@ -305,14 +299,12 @@ def test_perfherder_names_simplified_with_no_exclusions(): # In this case, some metrics will be called "median", "mean", etc. # since those are the simplifications of the first statistics entries # that were found. - assert not all( - [ - "firstPaint" in subtest["name"] - or "duration" in subtest["name"] - or "count" in subtest["name"] - for subtest in output["suites"][0]["subtests"] - ] - ) + assert not all([ + "firstPaint" in subtest["name"] + or "duration" in subtest["name"] + or "count" in subtest["name"] + for subtest in output["suites"][0]["subtests"] + ]) found_all = {"firstPaint": False, "count": False, "duration": False} for subtest in output["suites"][0]["subtests"]: @@ -326,23 +318,19 @@ def test_perfherder_names_simplified_with_no_exclusions(): # Only a portion of the metrics should still have statistics in # their name due to a naming conflict that only emits a warning assert ( - len( - [ - subtest - for subtest in output["suites"][0]["subtests"] - if "statistics" in subtest["name"] - ] - ) + len([ + subtest + for subtest in output["suites"][0]["subtests"] + if "statistics" in subtest["name"] + ]) == 18 ) assert ( - len( - [ - subtest - for subtest in output["suites"][0]["subtests"] - if "statistics" not in subtest["name"] - ] - ) + len([ + subtest + for subtest in output["suites"][0]["subtests"] + if "statistics" not in subtest["name"] + ]) == 12 ) @@ -370,9 +358,11 @@ def test_perfherder_with_extra_metadata_options(): output = json.loads(f.read()) assert len(output["suites"]) == 1 - assert sorted(output["suites"][0]["extraOptions"]) == sorted( - ["option", "second-option", "simpleperf"] - ) + assert sorted(output["suites"][0]["extraOptions"]) == sorted([ + "option", + "second-option", + "simpleperf", + ]) def test_perfherder_with_extra_options(): @@ -397,9 +387,10 @@ def test_perfherder_with_extra_options(): output = json.loads(f.read()) assert len(output["suites"]) == 1 - assert sorted(output["suites"][0]["extraOptions"]) == sorted( - ["option", "second-option"] - ) + assert sorted(output["suites"][0]["extraOptions"]) == sorted([ + "option", + "second-option", + ]) def test_perfherder_with_alerting(): @@ -425,20 +416,16 @@ def test_perfherder_with_alerting(): assert len(output["suites"]) == 1 assert sorted(output["suites"][0]["extraOptions"]) == sorted(["option"]) - assert all( - [ - subtest["shouldAlert"] - for subtest in output["suites"][0]["subtests"] - if "resource" in subtest["name"] - ] - ) - assert not all( - [ - subtest["shouldAlert"] - for subtest in output["suites"][0]["subtests"] - if "firstPaint" in subtest["name"] - ] - ) + assert all([ + subtest["shouldAlert"] + for subtest in output["suites"][0]["subtests"] + if "resource" in subtest["name"] + ]) + assert not all([ + subtest["shouldAlert"] + for subtest in output["suites"][0]["subtests"] + if "firstPaint" in subtest["name"] + ]) def test_perfherder_with_subunits(): @@ -463,20 +450,16 @@ def test_perfherder_with_subunits(): output = json.loads(f.read()) assert len(output["suites"]) == 1 - assert all( - [ - subtest["unit"] == "a-unit" - for subtest in output["suites"][0]["subtests"] - if "resource" in subtest["name"] - ] - ) - assert all( - [ - subtest["unit"] == "ms" - for subtest in output["suites"][0]["subtests"] - if "firstPaint" in subtest["name"] - ] - ) + assert all([ + subtest["unit"] == "a-unit" + for subtest in output["suites"][0]["subtests"] + if "resource" in subtest["name"] + ]) + assert all([ + subtest["unit"] == "ms" + for subtest in output["suites"][0]["subtests"] + if "firstPaint" in subtest["name"] + ]) def test_perfherder_with_supraunits(): @@ -503,20 +486,16 @@ def test_perfherder_with_supraunits(): assert len(output["suites"]) == 1 assert output["suites"][0]["unit"] == "new-unit" - assert all( - [ - subtest["unit"] == "a-unit" - for subtest in output["suites"][0]["subtests"] - if "resource" in subtest["name"] - ] - ) - assert all( - [ - subtest["unit"] == "new-unit" - for subtest in output["suites"][0]["subtests"] - if "firstPaint" in subtest["name"] - ] - ) + assert all([ + subtest["unit"] == "a-unit" + for subtest in output["suites"][0]["subtests"] + if "resource" in subtest["name"] + ]) + assert all([ + subtest["unit"] == "new-unit" + for subtest in output["suites"][0]["subtests"] + if "firstPaint" in subtest["name"] + ]) def test_perfherder_transforms(): @@ -560,18 +539,16 @@ def test_perfherder_logcat(): return (float(groups[0]) * 1000) + float(groups[1]) re_w_group = r".*Displayed.*org\.mozilla\.fennec_aurora.*\+([\d]+)s([\d]+)ms.*" - metadata.add_result( - { - "results": str(HERE / "data" / "home_activity.txt"), - "transformer": "LogCatTimeTransformer", - "transformer-options": { - "first-timestamp": re_w_group, - "processor": processor, - "transform-subtest-name": "TimeToDisplayed", - }, - "name": "LogCat", - } - ) + metadata.add_result({ + "results": str(HERE / "data" / "home_activity.txt"), + "transformer": "LogCatTimeTransformer", + "transformer-options": { + "first-timestamp": re_w_group, + "processor": processor, + "transform-subtest-name": "TimeToDisplayed", + }, + "name": "LogCat", + }) with temp_file() as output: env.set_arg("output", output) diff --git a/python/mozperftest/mozperftest/tests/test_perftestnotebook.py b/python/mozperftest/mozperftest/tests/test_perftestnotebook.py @@ -31,11 +31,15 @@ def test_post_to_iodide(ptnb, standarized_data, analysis): def mocked_open(self, *args, **kwargs): return opener(self, *args, **kwargs) - with mock.patch.object(Path, "open", mocked_open), mock.patch( - "mozperftest.metrics.notebook.perftestnotebook.webbrowser.open_new_tab" - ) as browser, mock.patch( - "mozperftest.metrics.notebook.perftestnotebook.HTTPServer" - ) as server: + with ( + mock.patch.object(Path, "open", mocked_open), + mock.patch( + "mozperftest.metrics.notebook.perftestnotebook.webbrowser.open_new_tab" + ) as browser, + mock.patch( + "mozperftest.metrics.notebook.perftestnotebook.HTTPServer" + ) as server, + ): ptnb.post_to_iodide(analysis=analysis) list_of_calls = opener.mock_calls @@ -66,9 +70,10 @@ def test_post_to_iodide(ptnb, standarized_data, analysis): assert list_of_calls.count(mock.call().__enter__()) == 3 browser.assert_called_with(str(upload_file_path)) - server.assert_has_calls( - [mock.call().serve_forever(), mock.call().server_close()] - ) + server.assert_has_calls([ + mock.call().serve_forever(), + mock.call().server_close(), + ]) if __name__ == "__main__": diff --git a/python/mozperftest/mozperftest/tests/test_runner.py b/python/mozperftest/mozperftest/tests/test_runner.py @@ -21,11 +21,13 @@ def test_main(): def test_main_perf_flags(mocked_argparser, set_perf_flags): mocked_parse_args = mock.MagicMock() mocked_argparser.return_value = mocked_parse_args - with mock.patch( - "mozperftest.runner._activate_virtualenvs", return_value="fake_path" - ) as _, mock.patch( - "mozperftest.runner.run_tests", return_value="fake_path" - ) as _, silence(): + with ( + mock.patch( + "mozperftest.runner._activate_virtualenvs", return_value="fake_path" + ) as _, + mock.patch("mozperftest.runner.run_tests", return_value="fake_path") as _, + silence(), + ): main([""]) assert "--gecko-profile" in mocked_argparser.call_args[1]["args"] @@ -42,23 +44,22 @@ def test_tools(): @mock.patch("mozperftest.utils.install_package") @mock.patch("mozperftest.PerftestToolsArgumentParser") def test_side_by_side(arg, patched_mozperftest_tools): - with mock.patch( - "mozperftest.runner._activate_virtualenvs", return_value="fake_path" - ) as _, mock.patch( - "mozperftest.runner._create_artifacts_dir", return_value="fake_path" - ) as _, mock.patch( - "mozperftest.runner._save_params", return_value="fake_path" - ) as _, mock.patch( - "sys.modules", return_value=mock.MagicMock() - ) as _: - main( - [ - "tools", - "side-by-side", - "-t", - "fake-test-name", - ] - ) + with ( + mock.patch( + "mozperftest.runner._activate_virtualenvs", return_value="fake_path" + ) as _, + mock.patch( + "mozperftest.runner._create_artifacts_dir", return_value="fake_path" + ) as _, + mock.patch("mozperftest.runner._save_params", return_value="fake_path") as _, + mock.patch("sys.modules", return_value=mock.MagicMock()) as _, + ): + main([ + "tools", + "side-by-side", + "-t", + "fake-test-name", + ]) if __name__ == "__main__": diff --git a/python/mozperftest/mozperftest/tests/test_shellscript.py b/python/mozperftest/mozperftest/tests/test_shellscript.py @@ -55,9 +55,11 @@ def test_shell_script_metric_parsing(): def test_shell_script( mocked_mozprocess, mocked_metrics, mocked_temp_dir, on_try_setting ): - with mock.patch( - "mozperftest.test.shellscript.ON_TRY", new=on_try_setting - ), temp_dir() as tmp_output_dir, temp_dir() as tmp_testing_dir: + with ( + mock.patch("mozperftest.test.shellscript.ON_TRY", new=on_try_setting), + temp_dir() as tmp_output_dir, + temp_dir() as tmp_testing_dir, + ): mach_cmd, metadata, env = running_env( app="firefox", tests=[str(EXAMPLE_SHELL_TEST)], output=tmp_output_dir ) diff --git a/python/mozperftest/mozperftest/tests/test_simpleperf.py b/python/mozperftest/mozperftest/tests/test_simpleperf.py @@ -397,8 +397,9 @@ def test_simpleperf_invalid_symbolicate_arguments(): profiler._validate_symbolication_paths(None, None) # Verify local symbolication is skipped if args not provided - with mock.patch.object(profiler, "_cleanup") as mock_cleanup, mock.patch( - "mozperftest.system.simpleperf.ON_TRY", False + with ( + mock.patch.object(profiler, "_cleanup") as mock_cleanup, + mock.patch("mozperftest.system.simpleperf.ON_TRY", False), ): profiler.teardown() mock_cleanup.assert_called_once() @@ -412,8 +413,9 @@ def test_simpleperf_invalid_symbolicate_arguments(): profiler.set_arg("symbolicator-path", "/fake/symbolicator/path") # Verify local symbolication is skipped if args are invalid - with mock.patch.object(profiler, "_cleanup") as mock_cleanup, mock.patch( - "mozperftest.system.simpleperf.ON_TRY", False + with ( + mock.patch.object(profiler, "_cleanup") as mock_cleanup, + mock.patch("mozperftest.system.simpleperf.ON_TRY", False), ): profiler.teardown() mock_cleanup.assert_called_once() @@ -453,13 +455,15 @@ def test_local_simpleperf_symbolicate(tmp_path): profiler.test_name = "unit_test" # Test local symbolication - with mock.patch("mozperftest.system.simpleperf.ON_TRY", False), mock.patch( - "tempfile.mkdtemp", return_value=str(mock_work_dir_path) - ), mock.patch("shutil.rmtree") as mock_rmtree, mock.patch( - "subprocess.Popen" - ) as mock_popen, mock.patch( - "mozperftest.system.simpleperf.find_node_executable", - return_value=[str(node_path)], + with ( + mock.patch("mozperftest.system.simpleperf.ON_TRY", False), + mock.patch("tempfile.mkdtemp", return_value=str(mock_work_dir_path)), + mock.patch("shutil.rmtree") as mock_rmtree, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch( + "mozperftest.system.simpleperf.find_node_executable", + return_value=[str(node_path)], + ), ): import_process = make_mock_process(context=True) @@ -573,16 +577,17 @@ def test_local_simpleperf_symbolicate_timeout(tmp_path): profiler.test_name = "unit_test" # Test timeout error in local run - with mock.patch("mozperftest.system.simpleperf.ON_TRY", False), mock.patch( - "tempfile.mkdtemp", return_value=str(mock_work_dir_path) - ), mock.patch("shutil.rmtree") as mock_rmtree, mock.patch( - "subprocess.Popen" - ) as mock_popen, mock.patch( - "mozperftest.system.simpleperf.find_node_executable", - return_value=[str(node_path)], - ), mock.patch.object( - profiler, "_cleanup" - ) as mock_cleanup: + with ( + mock.patch("mozperftest.system.simpleperf.ON_TRY", False), + mock.patch("tempfile.mkdtemp", return_value=str(mock_work_dir_path)), + mock.patch("shutil.rmtree") as mock_rmtree, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch( + "mozperftest.system.simpleperf.find_node_executable", + return_value=[str(node_path)], + ), + mock.patch.object(profiler, "_cleanup") as mock_cleanup, + ): # Mock processes import_process = make_mock_process(context=True) @@ -674,22 +679,20 @@ def test_ci_simpleperf_symbolicate(tmp_path): profiler.test_name = "unit_test" # Test symbolication in CI - with mock.patch.dict( - os.environ, - { - "MOZ_FETCHES_DIR": str(mock_fetch_path), - }, - clear=False, - ), mock.patch("mozperftest.system.simpleperf.ON_TRY", True), mock.patch( - "mozperftest.utils.ON_TRY", True - ), mock.patch( - "tempfile.mkdtemp", return_value=str(mock_work_dir_path) - ), mock.patch( - "shutil.rmtree" - ) as mock_rmtree, mock.patch( - "subprocess.Popen" - ) as mock_popen: - + with ( + mock.patch.dict( + os.environ, + { + "MOZ_FETCHES_DIR": str(mock_fetch_path), + }, + clear=False, + ), + mock.patch("mozperftest.system.simpleperf.ON_TRY", True), + mock.patch("mozperftest.utils.ON_TRY", True), + mock.patch("tempfile.mkdtemp", return_value=str(mock_work_dir_path)), + mock.patch("shutil.rmtree") as mock_rmtree, + mock.patch("subprocess.Popen") as mock_popen, + ): # Mock processes import_process = make_mock_process(context=True) @@ -829,27 +832,25 @@ def test_ci_simpleperf_symbolicate_timeout(tmp_path): profiler.test_name = "unit_test" # Test timeout error in CI - with mock.patch("mozperftest.system.simpleperf.ON_TRY", True), mock.patch( - "mozperftest.utils.ON_TRY", True - ), mock.patch( - "tempfile.mkdtemp", return_value=str(mock_work_dir_path) - ), mock.patch.dict( - os.environ, - { - "MOZ_FETCHES_DIR": str(mock_fetch_path), - }, - clear=False, - ), mock.patch( - "shutil.rmtree" - ) as mock_rmtree, mock.patch( - "subprocess.Popen" - ) as mock_popen, mock.patch( - "mozperftest.system.simpleperf.find_node_executable", - return_value=[str(node_path)], - ), mock.patch.object( - profiler, "_cleanup" - ) as mock_cleanup: - + with ( + mock.patch("mozperftest.system.simpleperf.ON_TRY", True), + mock.patch("mozperftest.utils.ON_TRY", True), + mock.patch("tempfile.mkdtemp", return_value=str(mock_work_dir_path)), + mock.patch.dict( + os.environ, + { + "MOZ_FETCHES_DIR": str(mock_fetch_path), + }, + clear=False, + ), + mock.patch("shutil.rmtree") as mock_rmtree, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch( + "mozperftest.system.simpleperf.find_node_executable", + return_value=[str(node_path)], + ), + mock.patch.object(profiler, "_cleanup") as mock_cleanup, + ): # Mock processes import_process = make_mock_process(context=True) diff --git a/python/mozperftest/mozperftest/tests/test_utils.py b/python/mozperftest/mozperftest/tests/test_utils.py @@ -87,37 +87,34 @@ def test_install_package(): vem.bin_path = "someplace" with mock.patch("subprocess.check_call") as mock_check_call: assert install_package(vem, "foo") - mock_check_call.assert_called_once_with( - [ - vem.python_path, - "-m", - "pip", - "install", - "foo", - ] - ) + mock_check_call.assert_called_once_with([ + vem.python_path, + "-m", + "pip", + "install", + "foo", + ]) def test_install_requirements_file(): vem = mock.Mock() vem.bin_path = "someplace" - with mock.patch("subprocess.check_call") as mock_check_call, mock.patch( - "mozperftest.utils.os" + with ( + mock.patch("subprocess.check_call") as mock_check_call, + mock.patch("mozperftest.utils.os"), ): assert install_requirements_file(vem, "foo") - mock_check_call.assert_called_once_with( - [ - vem.python_path, - "-m", - "pip", - "install", - "-r", - "foo", - "--no-index", - "--find-links", - "https://pypi.pub.build.mozilla.org/pub/", - ] - ) + mock_check_call.assert_called_once_with([ + vem.python_path, + "-m", + "pip", + "install", + "-r", + "foo", + "--no-index", + "--find-links", + "https://pypi.pub.build.mozilla.org/pub/", + ]) @mock.patch("pip._internal.req.constructors.install_req_from_line", new=_req) diff --git a/python/mozperftest/mozperftest/tests/test_xpcshell.py b/python/mozperftest/mozperftest/tests/test_xpcshell.py @@ -28,17 +28,17 @@ class XPCShellTests: self.log.log_raw({"action": "log", "message": "message"}) # these are the metrics sent by the scripts - self.log.log_raw( - { - "action": "log", - "message": '"perfMetrics"', - "extra": {"metrics1": 1, "metrics2": 2}, - } - ) - - self.log.log_raw( - {"action": "log", "message": '"perfMetrics"', "extra": {"metrics3": 3}} - ) + self.log.log_raw({ + "action": "log", + "message": '"perfMetrics"', + "extra": {"metrics1": 1, "metrics2": 2}, + }) + + self.log.log_raw({ + "action": "log", + "message": '"perfMetrics"', + "extra": {"metrics3": 3}, + }) self.log.test_end("test end") self.log.suite_end("suite end") diff --git a/python/mozperftest/mozperftest/utils.py b/python/mozperftest/mozperftest/utils.py @@ -236,9 +236,13 @@ def install_package(virtualenv_manager, package, ignore_failure=False): return True with silence(): try: - subprocess.check_call( - [virtualenv_manager.python_path, "-m", "pip", "install", package] - ) + subprocess.check_call([ + virtualenv_manager.python_path, + "-m", + "pip", + "install", + package, + ]) return True except Exception: if not ignore_failure: @@ -281,19 +285,17 @@ def install_requirements_file( cwd = os.getcwd() try: os.chdir(Path(requirements_file).parent) - subprocess.check_call( - [ - virtualenv_manager.python_path, - "-m", - "pip", - "install", - "-r", - requirements_file, - "--no-index", - "--find-links", - "https://pypi.pub.build.mozilla.org/pub/", - ] - ) + subprocess.check_call([ + virtualenv_manager.python_path, + "-m", + "pip", + "install", + "-r", + requirements_file, + "--no-index", + "--find-links", + "https://pypi.pub.build.mozilla.org/pub/", + ]) return True except Exception: if not ignore_failure: diff --git a/python/mozrelease/mozrelease/attribute_builds.py b/python/mozrelease/mozrelease/attribute_builds.py @@ -181,13 +181,11 @@ def main(): work = [] for i in args.input: fn = os.path.basename(i) - work.append( - { - "input": i, - "output": os.path.join(args.output, fn), - "attribution": args.attribution, - } - ) + work.append({ + "input": i, + "output": os.path.join(args.output, fn), + "attribution": args.attribution, + }) else: log.error("No configuration found. Set ATTRIBUTION_CONFIG or pass arguments.") return 1 diff --git a/python/mozrelease/mozrelease/balrog.py b/python/mozrelease/mozrelease/balrog.py @@ -62,10 +62,8 @@ def generate_update_properties(context, config): conditions = _generate_conditions(context, entry.get("conditions", {})) if conditions is not None: - result.append( - { - "fields": fields, - "for": conditions, - } - ) + result.append({ + "fields": fields, + "for": conditions, + }) return result diff --git a/python/mozrelease/mozrelease/buglist_creator.py b/python/mozrelease/mozrelease/buglist_creator.py @@ -18,7 +18,7 @@ BUGZILLA_BUGLIST_TEMPLATE = "https://bugzilla.mozilla.org/buglist.cgi?bug_id={bu BUG_NUMBER_REGEX = re.compile(r"bug \d+", re.IGNORECASE) CHANGELOG_TO_FROM_STRING = "{product}_{version}_RELEASE" CHANGESET_URL_TEMPLATE = ( - "{repo}/{logtype}" "?rev={to_version}+%25+{from_version}&revcount=1000" + "{repo}/{logtype}?rev={to_version}+%25+{from_version}&revcount=1000" ) FULL_CHANGESET_TEMPLATE = "* [Full Mercurial changelog]({url})\n" LIST_DESCRIPTION_TEMPLATE = "Comparing Mercurial tag {from_version} to {to_version}:\n" @@ -245,10 +245,8 @@ Task group: [{task_group_id}]({root_url}/tasks/groups/{task_group_id}) notify = Notify(notify_options) for address in addresses: - notify.email( - { - "address": address, - "subject": subject, - "content": content, - } - ) + notify.email({ + "address": address, + "subject": subject, + "content": content, + }) diff --git a/python/mozrelease/mozrelease/mach_commands.py b/python/mozrelease/mozrelease/mach_commands.py @@ -59,8 +59,7 @@ def buglist(command_context, version, product, revision, repo): required=True, action="append", dest="addresses", - help="The email address to send the bug list to " - "(may be specified more than once.", + help="The email address to send the bug list to (may be specified more than once.", ) @CommandArgument( "--version", diff --git a/python/mozrelease/mozrelease/paths.py b/python/mozrelease/mozrelease/paths.py @@ -59,34 +59,28 @@ def getReleaseInstallerPath( MozillaVersion(version) > MozillaVersion(last_linux_bz2_version) ): compression = "xz" - return "/".join( - [ - p.strip("/") - for p in [ - platform, - locale, - "%s-%s.tar.%s" % (productName, version, compression), - ] + return "/".join([ + p.strip("/") + for p in [ + platform, + locale, + "%s-%s.tar.%s" % (productName, version, compression), ] - ) + ]) elif "mac" in platform: - return "/".join( - [ - p.strip("/") - for p in [platform, locale, "%s %s.dmg" % (brandName, version)] - ] - ) + return "/".join([ + p.strip("/") + for p in [platform, locale, "%s %s.dmg" % (brandName, version)] + ]) elif platform.startswith("win"): - return "/".join( - [ - p.strip("/") - for p in [ - platform, - locale, - "%s Setup %s.exe" % (brandName, version), - ] + return "/".join([ + p.strip("/") + for p in [ + platform, + locale, + "%s Setup %s.exe" % (brandName, version), ] - ) + ]) else: raise "Unsupported platform" elif platform.startswith("android"): diff --git a/python/mozrelease/mozrelease/scriptworker_canary.py b/python/mozrelease/mozrelease/scriptworker_canary.py @@ -57,9 +57,7 @@ def configure_ssh(ssh_key_secret): ssh_key_file.chmod(0o600) hgrc_content = ( - "[ui]\n" - "username = trybld\n" - "ssh = ssh -i {path} -l {user}\n".format( + "[ui]\nusername = trybld\nssh = ssh -i {path} -l {user}\n".format( path=ssh_key_file, user=ssh_key["user"] ) ) @@ -90,14 +88,12 @@ def push_canary(scriptworkers, addresses, ssh_key_secret): mach = Path(GECKO) / "mach" base_command = [str(mach), "try", "scriptworker", "--closed-tree", "--push-to-vcs"] for address in addresses: - base_command.extend( - [ - "--route", - f"notify.email.{address}.on-failed", - "--route", - f"notify.email.{address}.on-exception", - ] - ) + base_command.extend([ + "--route", + f"notify.email.{address}.on-failed", + "--route", + f"notify.email.{address}.on-exception", + ]) with configure_ssh(ssh_key_secret): env = os.environ.copy() diff --git a/python/mozrelease/mozrelease/update_verify.py b/python/mozrelease/mozrelease/update_verify.py @@ -188,20 +188,18 @@ class UpdateVerifyConfig: locales = sorted(list(locales.split())) if isinstance(patch_types, str): patch_types = list(patch_types.split()) - self.releases.append( - { - "release": release, - "build_id": build_id, - "locales": locales, - "patch_types": patch_types, - "from": from_path, - "ftp_server_from": ftp_server_from, - "ftp_server_to": ftp_server_to, - "mar_channel_IDs": mar_channel_IDs, - "platform": platform, - "updater_package": updater_package, - } - ) + self.releases.append({ + "release": release, + "build_id": build_id, + "locales": locales, + "patch_types": patch_types, + "from": from_path, + "ftp_server_from": ftp_server_from, + "ftp_server_to": ftp_server_to, + "mar_channel_IDs": mar_channel_IDs, + "platform": platform, + "updater_package": updater_package, + }) def addLocaleToRelease(self, build_id, locale, from_path=None): r = self.getRelease(build_id, from_path) diff --git a/python/mozterm/test/test_widgets.py b/python/mozterm/test/test_widgets.py @@ -31,12 +31,10 @@ def terminal(): ) def test_footer(terminal): footer = Footer(terminal=terminal) - footer.write( - [ - ("bright_black", "foo"), - ("green", "bar"), - ] - ) + footer.write([ + ("bright_black", "foo"), + ("green", "bar"), + ]) value = terminal.stream.getvalue() expected = "\x1b7\x1b[90mfoo\x1b(B\x1b[m \x1b[32mbar\x1b(B\x1b[m\x1b8" assert value == expected diff --git a/python/mozversioncontrol/mozversioncontrol/repo/git.py b/python/mozversioncontrol/mozversioncontrol/repo/git.py @@ -415,24 +415,22 @@ class GitRepository(Repository): # adding or modifying the files from `changed_files`. # fast-import will output the sha1 for that temporary commit on stdout # (via `get-mark`). - fast_import = "\n".join( - [ - f"commit refs/machtry/{branch}", - "mark :1", - f"author {author}", - f"committer {committer}", - data(commit_message), - f"from {current_head}", - "\n".join( - f"M 100644 inline {path}\n{data(content)}" - for path, content in (changed_files or {}).items() - ), - f"reset refs/machtry/{branch}", - "from 0000000000000000000000000000000000000000", - "get-mark :1", - "", - ] - ) + fast_import = "\n".join([ + f"commit refs/machtry/{branch}", + "mark :1", + f"author {author}", + f"committer {committer}", + data(commit_message), + f"from {current_head}", + "\n".join( + f"M 100644 inline {path}\n{data(content)}" + for path, content in (changed_files or {}).items() + ), + f"reset refs/machtry/{branch}", + "from 0000000000000000000000000000000000000000", + "get-mark :1", + "", + ]) cmd = (str(self._tool), "fast-import", "--quiet") stdout = subprocess.check_output( diff --git a/python/mozversioncontrol/mozversioncontrol/repo/mercurial.py b/python/mozversioncontrol/mozversioncontrol/repo/mercurial.py @@ -427,12 +427,10 @@ class HgRepository(Repository): print(f"Ensuring {url} is up to date at {dest}") env = os.environ.copy() - env.update( - { - "HGPLAIN": "1", - "HGRCPATH": "!", - } - ) + env.update({ + "HGPLAIN": "1", + "HGRCPATH": "!", + }) try: subprocess.check_call(pull_args, cwd=str(cwd), env=env) diff --git a/python/mozversioncontrol/test/test_commit.py b/python/mozversioncontrol/test/test_commit.py @@ -121,9 +121,9 @@ def test_commit(repo): ] assert matches, f"No diff marker found for '{filename}'" - assert ( - len(matches) == 1 - ), f"More than one diff marker for '{filename}': {matches}" + assert len(matches) == 1, ( + f"More than one diff marker for '{filename}': {matches}" + ) return matches[0] diff --git a/python/mozversioncontrol/test/test_get_mozilla_remote_args.py b/python/mozversioncontrol/test/test_get_mozilla_remote_args.py @@ -51,17 +51,17 @@ def test_get_mozilla_remote_args(is_cinnabar, expected_remotes, repo): remotes = vcs.get_mozilla_remote_args() - assert remotes == [ - "--remotes" - ], "Default `--remotes` passed without finding official remote." + assert remotes == ["--remotes"], ( + "Default `--remotes` passed without finding official remote." + ) repo.execute_next_step() remotes = sorted(vcs.get_mozilla_remote_args()) - assert ( - remotes == expected_remotes - ), "Multiple non-try remote arguments should be found." + assert remotes == expected_remotes, ( + "Multiple non-try remote arguments should be found." + ) if __name__ == "__main__": diff --git a/python/mozversioncontrol/test/test_try_commit.py b/python/mozversioncontrol/test/test_try_commit.py @@ -22,17 +22,17 @@ def test_try_commit(repo): except MissingVCSExtension: pytest.xfail("Requires the Mercurial evolve extension.") - assert ( - vcs.head_ref == initial_head_ref - ), "We should have reverted to previous head after try_commit" + assert vcs.head_ref == initial_head_ref, ( + "We should have reverted to previous head after try_commit" + ) # Create an empty commit. with vcs.try_commit(commit_message) as head: assert vcs.get_changed_files(rev=head) == [] - assert ( - vcs.head_ref == initial_head_ref - ), "We should have reverted to previous head after try_commit" + assert vcs.head_ref == initial_head_ref, ( + "We should have reverted to previous head after try_commit" + ) if __name__ == "__main__": diff --git a/security/manager/ssl/gen_cert_header.py b/security/manager/ssl/gen_cert_header.py @@ -34,7 +34,7 @@ def write_header(output, array_name, certificates): output.write(f" {line}\n") output.write("};\n") output.write( - f'const mozilla::Span<const uint8_t> {array_name}[] = {{ {", ".join(certificate_names)} }};\n' + f"const mozilla::Span<const uint8_t> {array_name}[] = {{ {', '.join(certificate_names)} }};\n" ) diff --git a/security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py b/security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py @@ -12,6 +12,7 @@ duplicates. Requires Python 3. """ + import argparse import io import re diff --git a/security/manager/tools/pypkcs12.py b/security/manager/tools/pypkcs12.py @@ -71,9 +71,11 @@ class PKCS12: self.key = pykey.keyFromSpecification("default") def toDER(self): - with NamedTemporaryFile(mode="wt+") as certTmp, NamedTemporaryFile( - mode="wt+" - ) as keyTmp, NamedTemporaryFile(mode="rb+") as pkcs12Tmp: + with ( + NamedTemporaryFile(mode="wt+") as certTmp, + NamedTemporaryFile(mode="wt+") as keyTmp, + NamedTemporaryFile(mode="rb+") as pkcs12Tmp, + ): certTmp.write(self.cert.toPEM()) certTmp.flush() keyTmp.write(self.key.toPEM()) diff --git a/security/manager/tools/pytlsbinding.py b/security/manager/tools/pytlsbinding.py @@ -28,7 +28,6 @@ Currently only the algorithms RS256 (RSA PKCS#1v1.5 with SHA-256) and S256 (SHA-256) are supported. """ - import base64 import hashlib import json diff --git a/security/manager/tools/regen_root_ca_metadata.py b/security/manager/tools/regen_root_ca_metadata.py @@ -271,12 +271,12 @@ def write_root_hashes(path, certdata, known_root_hashes): tmpl = Template(ROOT_HASHES_ENTRY_TEMPLATE) for root in certdata: root_hash = known_root_hashes[root.sha256base64()] - digest_half_1 = "".join( - [f"0x{c:02x}, " for c in root_hash.digest[: len(root_hash.digest) >> 1]] - ).removesuffix(" ") - digest_half_2 = "".join( - [f"0x{c:02x}, " for c in root_hash.digest[len(root_hash.digest) >> 1 :]] - ).removesuffix(", ") + digest_half_1 = "".join([ + f"0x{c:02x}, " for c in root_hash.digest[: len(root_hash.digest) >> 1] + ]).removesuffix(" ") + digest_half_2 = "".join([ + f"0x{c:02x}, " for c in root_hash.digest[len(root_hash.digest) >> 1 :] + ]).removesuffix(", ") f.write( tmpl.substitute( label=root_hash.label, diff --git a/taskcluster/android_taskgraph/__init__.py b/taskcluster/android_taskgraph/__init__.py @@ -20,15 +20,13 @@ def register(graph_config): Import all modules that are siblings of this one, triggering decorators in the process. """ - _import_modules( - [ - "job", - "parameters", - "target_tasks", - "util.group_by", - "worker_types", - ] - ) + _import_modules([ + "job", + "parameters", + "target_tasks", + "util.group_by", + "worker_types", + ]) def _import_modules(modules): diff --git a/taskcluster/android_taskgraph/job.py b/taskcluster/android_taskgraph/job.py @@ -24,31 +24,27 @@ dummy_secret_schema = { Optional("json"): bool, } -gradlew_schema = Schema( - { - Required("using"): "gradlew", - Optional("pre-gradlew"): [[str]], - Required("gradlew"): [str], - Optional("post-gradlew"): [[str]], - # Base work directory used to set up the task. - Required("workdir"): str, - Optional("use-caches"): Any(bool, [str]), - Optional("secrets"): [secret_schema], - Optional("dummy-secrets"): [dummy_secret_schema], - } -) - -run_commands_schema = Schema( - { - Required("using"): "run-commands", - Optional("pre-commands"): [[str]], - Required("commands"): [[taskref_or_string]], - Required("workdir"): str, - Optional("use-caches"): Any(bool, [str]), - Optional("secrets"): [secret_schema], - Optional("dummy-secrets"): [dummy_secret_schema], - } -) +gradlew_schema = Schema({ + Required("using"): "gradlew", + Optional("pre-gradlew"): [[str]], + Required("gradlew"): [str], + Optional("post-gradlew"): [[str]], + # Base work directory used to set up the task. + Required("workdir"): str, + Optional("use-caches"): Any(bool, [str]), + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], +}) + +run_commands_schema = Schema({ + Required("using"): "run-commands", + Optional("pre-commands"): [[str]], + Required("commands"): [[taskref_or_string]], + Required("workdir"): str, + Optional("use-caches"): Any(bool, [str]), + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], +}) @run_job_using("docker-worker", "run-commands", schema=run_commands_schema) @@ -78,15 +74,13 @@ def configure_gradlew(config, job, taskdesc): fetches_dir = "/builds/worker/fetches" topsrc_dir = "/builds/worker/checkouts/gecko" - worker.setdefault("env", {}).update( - { - "ANDROID_SDK_ROOT": path.join(fetches_dir, "android-sdk-linux"), - "GRADLE_USER_HOME": path.join( - topsrc_dir, "mobile/android/gradle/dotgradle-offline" - ), - "MOZ_BUILD_DATE": config.params["moz_build_date"], - } - ) + worker.setdefault("env", {}).update({ + "ANDROID_SDK_ROOT": path.join(fetches_dir, "android-sdk-linux"), + "GRADLE_USER_HOME": path.join( + topsrc_dir, "mobile/android/gradle/dotgradle-offline" + ), + "MOZ_BUILD_DATE": config.params["moz_build_date"], + }) worker["env"].setdefault( "MOZCONFIG", path.join( @@ -103,14 +97,12 @@ def configure_gradlew(config, job, taskdesc): for secret in run.pop("dummy-secrets", []) ] secrets = [_generate_secret_command(secret) for secret in run.get("secrets", [])] - worker["env"].update( - { - "PRE_GRADLEW": _convert_commands_to_string(run.pop("pre-gradlew", [])), - "GET_SECRETS": _convert_commands_to_string(dummy_secrets + secrets), - "GRADLEW_ARGS": " ".join(run.pop("gradlew")), - "POST_GRADLEW": _convert_commands_to_string(run.pop("post-gradlew", [])), - } - ) + worker["env"].update({ + "PRE_GRADLEW": _convert_commands_to_string(run.pop("pre-gradlew", [])), + "GET_SECRETS": _convert_commands_to_string(dummy_secrets + secrets), + "GRADLEW_ARGS": " ".join(run.pop("gradlew")), + "POST_GRADLEW": _convert_commands_to_string(run.pop("post-gradlew", [])), + }) run["command"] = ( "/builds/worker/checkouts/gecko/taskcluster/scripts/builder/build-android.sh" ) diff --git a/taskcluster/android_taskgraph/transforms/beetmover_android_app.py b/taskcluster/android_taskgraph/transforms/beetmover_android_app.py @@ -102,15 +102,13 @@ def make_task_worker(config, tasks): locale = task["attributes"].get("locale") build_type = task["attributes"]["build-type"] - task["worker"].update( - { - "implementation": "beetmover", - "release-properties": craft_release_properties(config, task), - "artifact-map": generate_beetmover_artifact_map( - config, task, platform=build_type, locale=locale - ), - } - ) + task["worker"].update({ + "implementation": "beetmover", + "release-properties": craft_release_properties(config, task), + "artifact-map": generate_beetmover_artifact_map( + config, task, platform=build_type, locale=locale + ), + }) if locale: task["worker"]["locale"] = locale diff --git a/taskcluster/android_taskgraph/transforms/build_android_app.py b/taskcluster/android_taskgraph/transforms/build_android_app.py @@ -6,7 +6,6 @@ Apply some defaults and minor modifications to the jobs defined in the build-apk and build-bundle kinds. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util import path @@ -82,30 +81,26 @@ def add_shippable_secrets(config, tasks): task.pop("include-shippable-secrets", False) and config.params["level"] == "3" ): - secrets.extend( - [ - { - "key": key, - "name": _get_secret_index(task["name"]), - "path": target_file, - } - for key, target_file in _get_secrets_keys_and_target_files(task) - ] - ) + secrets.extend([ + { + "key": key, + "name": _get_secret_index(task["name"]), + "path": target_file, + } + for key, target_file in _get_secrets_keys_and_target_files(task) + ]) else: - dummy_secrets.extend( - [ - { - "content": fake_value, - "path": target_file, - } - for fake_value, target_file in ( - ("faketoken", ".adjust_token"), - ("faketoken", ".mls_token"), - ("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"), - ) - ] - ) + dummy_secrets.extend([ + { + "content": fake_value, + "path": target_file, + } + for fake_value, target_file in ( + ("faketoken", ".adjust_token"), + ("faketoken", ".mls_token"), + ("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"), + ) + ]) yield task @@ -120,16 +115,14 @@ def _get_secrets_keys_and_target_files(task): if task["name"].startswith("fenix-"): gradle_build_type = task["run"]["gradle-build-type"] - secrets.extend( - [ - ( - "firebase", - f"app/src/{gradle_build_type}/res/values/firebase.xml", - ), - ("wallpaper_url", ".wallpaper_url"), - ("pocket_consumer_key", ".pocket_consumer_key"), - ] - ) + secrets.extend([ + ( + "firebase", + f"app/src/{gradle_build_type}/res/values/firebase.xml", + ), + ("wallpaper_url", ".wallpaper_url"), + ("pocket_consumer_key", ".pocket_consumer_key"), + ]) return secrets @@ -221,13 +214,11 @@ def add_disable_optimization(config, tasks): def add_nightly_version(config, tasks): for task in tasks: if task.pop("include-nightly-version", False): - task["run"]["gradlew"].extend( - [ - # We only set the `official` flag here. The actual version name will be determined - # by Gradle (depending on the Gecko/A-C version being used) - "-Pofficial" - ] - ) + task["run"]["gradlew"].extend([ + # We only set the `official` flag here. The actual version name will be determined + # by Gradle (depending on the Gecko/A-C version being used) + "-Pofficial" + ]) yield task @@ -235,9 +226,10 @@ def add_nightly_version(config, tasks): def add_release_version(config, tasks): for task in tasks: if task.pop("include-release-version", False): - task["run"]["gradlew"].extend( - ["-PversionName={}".format(config.params["version"]), "-Pofficial"] - ) + task["run"]["gradlew"].extend([ + "-PversionName={}".format(config.params["version"]), + "-Pofficial", + ]) yield task @@ -271,19 +263,17 @@ def add_artifacts(config, tasks): apk_name = artifact_template["name"].format( gradle_build=gradle_build, **apk ) - artifacts.append( - { - "type": artifact_template["type"], - "name": apk_name, - "path": artifact_template["path"].format( - gradle_build_type=gradle_build_type, - gradle_build=gradle_build, - apk_path=apk_path, - source_project_name=source_project_name, - **apk, - ), - } - ) + artifacts.append({ + "type": artifact_template["type"], + "name": apk_name, + "path": artifact_template["path"].format( + gradle_build_type=gradle_build_type, + gradle_build=gradle_build, + apk_path=apk_path, + source_project_name=source_project_name, + **apk, + ), + }) apks[apk["abi"]] = { "name": apk_name, } @@ -297,19 +287,17 @@ def add_artifacts(config, tasks): else: aab_filename = f"app-{gradle_build}-{gradle_build_type}.aab" - artifacts.append( - { - "type": artifact_template["type"], - "name": artifact_template["name"], - "path": artifact_template["path"].format( - gradle_build_type=gradle_build_type, - gradle_build=gradle_build, - source_project_name=source_project_name, - variant_name=variant_name, - aab_filename=aab_filename, - ), - } - ) + artifacts.append({ + "type": artifact_template["type"], + "name": artifact_template["name"], + "path": artifact_template["path"].format( + gradle_build_type=gradle_build_type, + gradle_build=gradle_build, + source_project_name=source_project_name, + variant_name=variant_name, + aab_filename=aab_filename, + ), + }) task["attributes"]["aab"] = artifact_template["name"] yield task diff --git a/taskcluster/android_taskgraph/transforms/build_components.py b/taskcluster/android_taskgraph/transforms/build_components.py @@ -148,15 +148,13 @@ def add_artifacts(config, tasks): ]: if key in task: optional_artifact_template = task.pop(key, {}) - build_artifact_definitions.append( - { - "type": optional_artifact_template["type"], - "name": optional_artifact_template["name"], - "path": optional_artifact_template["path"].format( - component_path=get_path(component) - ), - } - ) + build_artifact_definitions.append({ + "type": optional_artifact_template["type"], + "name": optional_artifact_template["name"], + "path": optional_artifact_template["path"].format( + component_path=get_path(component) + ), + }) if artifact_template: all_extensions = get_extensions(component) @@ -185,22 +183,20 @@ def add_artifacts(config, tasks): artifact_full_name = artifact_template["name"].format( artifact_file_name=artifact_file_name, ) - build_artifact_definitions.append( - { - "type": artifact_template["type"], - "name": artifact_full_name, - "path": artifact_template["path"].format( - component_path=get_path(component), - component=component, - version=craft_path_version( - version, - task["attributes"]["build-type"], - nightly_version, - ), - artifact_file_name=artifact_file_name, + build_artifact_definitions.append({ + "type": artifact_template["type"], + "name": artifact_full_name, + "path": artifact_template["path"].format( + component_path=get_path(component), + component=component, + version=craft_path_version( + version, + task["attributes"]["build-type"], + nightly_version, ), - } - ) + artifact_file_name=artifact_file_name, + ), + }) artifacts[extension] = artifact_full_name diff --git a/taskcluster/android_taskgraph/transforms/notify.py b/taskcluster/android_taskgraph/transforms/notify.py @@ -38,12 +38,10 @@ def add_notify_email(config, tasks): } routes = task.setdefault("routes", []) - routes.extend( - [ - f"notify.email.{address}.on-{reason}" - for address in email_config["to-addresses"] - for reason in email_config["on-reasons"] - ] - ) + routes.extend([ + f"notify.email.{address}.on-{reason}" + for address in email_config["to-addresses"] + for reason in email_config["on-reasons"] + ]) yield task diff --git a/taskcluster/android_taskgraph/transforms/push_android_app.py b/taskcluster/android_taskgraph/transforms/push_android_app.py @@ -6,7 +6,6 @@ Apply some defaults and minor modifications to the jobs defined in the build kind. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.schema import resolve_keyed_by @@ -31,6 +30,6 @@ def resolve_keys(config, tasks): **{ "build-type": task["attributes"]["build-type"], "level": config.params["level"], - } + }, ) yield task diff --git a/taskcluster/android_taskgraph/transforms/signing.py b/taskcluster/android_taskgraph/transforms/signing.py @@ -27,7 +27,7 @@ def resolve_keys(config, tasks): **{ "build-type": task["attributes"]["build-type"], "level": config.params["level"], - } + }, ) yield task @@ -65,8 +65,7 @@ _DETACHED_SIGNATURE_EXTENSION = ".asc" def set_detached_signature_artifacts(config, tasks): for task in tasks: task["attributes"]["artifacts"] = { - extension - + _DETACHED_SIGNATURE_EXTENSION: path + extension + _DETACHED_SIGNATURE_EXTENSION: path + _DETACHED_SIGNATURE_EXTENSION for extension, path in task["attributes"]["artifacts"].items() } diff --git a/taskcluster/android_taskgraph/transforms/startup_tests_deps.py b/taskcluster/android_taskgraph/transforms/startup_tests_deps.py @@ -22,7 +22,7 @@ def add_android_startup_test_dependencies(config, jobs): if job_build_type in t.attributes.get("build-type", "") ] if matching_tasks: - job.setdefault("dependencies", {}).update( - {t.label: t.label for t in matching_tasks} - ) + job.setdefault("dependencies", {}).update({ + t.label: t.label for t in matching_tasks + }) yield job diff --git a/taskcluster/android_taskgraph/transforms/ui_tests.py b/taskcluster/android_taskgraph/transforms/ui_tests.py @@ -96,16 +96,14 @@ def define_ui_test_command_line(config, tasks): run = task.setdefault("run", {}) post_gradlew = run.setdefault("post-gradlew", []) - post_gradlew.append( - [ - "python3", - "taskcluster/scripts/tests/test-lab.py", - flank_config, - apk_app, - "--apk_test", - apk_test, - ] - ) + post_gradlew.append([ + "python3", + "taskcluster/scripts/tests/test-lab.py", + flank_config, + apk_app, + "--apk_test", + apk_test, + ]) yield task diff --git a/taskcluster/android_taskgraph/transforms/upstream_artifacts.py b/taskcluster/android_taskgraph/transforms/upstream_artifacts.py @@ -35,23 +35,19 @@ def build_upstream_artifacts(config, tasks): only_archs = task.pop("only-archs", []) for dep in get_dependencies(config, task): paths = list(dep.attributes.get("artifacts", {}).values()) - paths.extend( - [ - apk_metadata["name"] - for arch, apk_metadata in dep.attributes.get("apks", {}).items() - if not only_archs or arch in only_archs - ] - ) + paths.extend([ + apk_metadata["name"] + for arch, apk_metadata in dep.attributes.get("apks", {}).items() + if not only_archs or arch in only_archs + ]) if dep.attributes.get("aab"): paths.extend([dep.attributes.get("aab")]) if paths: - worker_definition["upstream-artifacts"].append( - { - "taskId": {"task-reference": f"<{dep.kind}>"}, - "taskType": _get_task_type(dep.kind), - "paths": sorted(paths), - } - ) + worker_definition["upstream-artifacts"].append({ + "taskId": {"task-reference": f"<{dep.kind}>"}, + "taskType": _get_task_type(dep.kind), + "paths": sorted(paths), + }) task.setdefault("worker", {}).update(worker_definition) yield task diff --git a/taskcluster/android_taskgraph/util/group_by.py b/taskcluster/android_taskgraph/util/group_by.py @@ -25,13 +25,11 @@ def component_grouping(config, tasks): and task.attributes.get("is_final_chunked_task", True) ] for (_, build_type), group_tasks in groups.items(): - group_tasks.extend( - [ - task - for task in tasks_for_all_components - if task.attributes.get("build-type") == build_type - ] - ) + group_tasks.extend([ + task + for task in tasks_for_all_components + if task.attributes.get("build-type") == build_type + ]) return groups.values() diff --git a/taskcluster/android_taskgraph/util/scriptworker.py b/taskcluster/android_taskgraph/util/scriptworker.py @@ -115,14 +115,12 @@ def generate_beetmover_upstream_artifacts( if not paths: continue - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{dep}>"}, - "taskType": map_config["tasktype_map"].get(dep), - "paths": sorted(paths), - "locale": current_locale, - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": f"<{dep}>"}, + "taskType": map_config["tasktype_map"].get(dep), + "paths": sorted(paths), + "locale": current_locale, + }) upstream_artifacts.sort(key=lambda u: u["paths"]) return upstream_artifacts @@ -271,17 +269,17 @@ def generate_beetmover_artifact_map(config, job, **kwargs): else: folder_prefix = f"{version}-candidates/build{build_number}/android/" - kwargs.update( - {"locale": locale, "version": version, "folder_prefix": folder_prefix} - ) + kwargs.update({ + "locale": locale, + "version": version, + "folder_prefix": folder_prefix, + }) kwargs.update(**platforms) paths = jsone.render(paths, kwargs) - artifacts.append( - { - "taskId": {"task-reference": f"<{dep}>"}, - "locale": locale, - "paths": paths, - } - ) + artifacts.append({ + "taskId": {"task-reference": f"<{dep}>"}, + "locale": locale, + "paths": paths, + }) return artifacts diff --git a/taskcluster/android_taskgraph/worker_types.py b/taskcluster/android_taskgraph/worker_types.py @@ -53,12 +53,10 @@ def build_scriptworker_beetmover_payload(config, task, task_def): } scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] - task_def["scopes"].extend( - [ - "{}:beetmover:action:{}".format(scope_prefix, worker["action"]), - "{}:beetmover:bucket:{}".format(scope_prefix, worker["bucket"]), - ] - ) + task_def["scopes"].extend([ + "{}:beetmover:action:{}".format(scope_prefix, worker["action"]), + "{}:beetmover:bucket:{}".format(scope_prefix, worker["bucket"]), + ]) @payload_builder( diff --git a/taskcluster/docker/funsize-update-generator/scripts/funsize.py b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -63,7 +63,7 @@ def strtobool(value: str): if value in false_vals: return 0 - raise ValueError(f'Expected one of: {", ".join(true_vals + false_vals)}') + raise ValueError(f"Expected one of: {', '.join(true_vals + false_vals)}") def verify_signature(mar, cert): diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/basic_tests.py @@ -251,9 +251,10 @@ class SnapTestsBase: return self._is_debug_build def need_allow_system_access(self): - geckodriver_output = subprocess.check_output( - [self._EXE_PATH, "--help"] - ).decode() + geckodriver_output = subprocess.check_output([ + self._EXE_PATH, + "--help", + ]).decode() return "--allow-system-access" in geckodriver_output def update_channel(self): @@ -414,16 +415,16 @@ class SnapTestsBase: (left, upper, right, lower) = bbox assert right >= left, f"Inconsistent boundaries right={right} left={left}" - assert ( - lower >= upper - ), f"Inconsistent boundaries lower={lower} upper={upper}" + assert lower >= upper, ( + f"Inconsistent boundaries lower={lower} upper={upper}" + ) if ((right - left) <= 2) or ((lower - upper) <= 2): self._logger.info("Difference is a <= 2 pixels band, ignoring") return - assert ( - diff_px_on_bbox <= allowance - ), "Mismatching screenshots for {}".format(exp["reference"]) + assert diff_px_on_bbox <= allowance, ( + "Mismatching screenshots for {}".format(exp["reference"]) + ) class SnapTests(SnapTestsBase): @@ -451,9 +452,9 @@ class SnapTests(SnapTestsBase): ) self._wait.until(lambda d: len(distributionid_box.text) > 0) self._logger.info(f"about:support distribution ID: {distributionid_box.text}") - assert ( - distributionid_box.text == exp["distribution_id"] - ), "distribution_id should match" + assert distributionid_box.text == exp["distribution_id"], ( + "distribution_id should match" + ) windowing_protocol = self._driver.execute_script( "return document.querySelector('th[data-l10n-id=\"graphics-window-protocol\"').parentNode.lastChild.textContent;" @@ -471,18 +472,18 @@ class SnapTests(SnapTestsBase): ) self._wait.until(lambda d: len(source_link.text) > 0) self._logger.info(f"about:buildconfig source: {source_link.text}") - assert source_link.text.startswith( - exp["source_repo"] - ), "source repo should exists and match" + assert source_link.text.startswith(exp["source_repo"]), ( + "source repo should exists and match" + ) build_flags_box = self._wait.until( EC.visibility_of_element_located((By.CSS_SELECTOR, "p:last-child")) ) self._wait.until(lambda d: len(build_flags_box.text) > 0) self._logger.info(f"about:support buildflags: {build_flags_box.text}") - assert ( - build_flags_box.text.find(exp["official"]) >= 0 - ), "official build flag should be there" + assert build_flags_box.text.find(exp["official"]) >= 0, ( + "official build flag should be there" + ) return True @@ -499,9 +500,10 @@ class SnapTests(SnapTestsBase): self._logger.info("Wait for consent form") try: self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, "button[aria-label*=Accept]") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + "button[aria-label*=Accept]", + )) ).click() except TimeoutException: self._logger.info("Wait for consent form: timed out, maybe it is not here") @@ -513,9 +515,10 @@ class SnapTests(SnapTestsBase): self._logger.info("Wait for cable proposal") try: self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, "button[aria-label*=Dismiss]") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + "button[aria-label*=Dismiss]", + )) ).click() except TimeoutException: self._logger.info( @@ -547,9 +550,9 @@ class SnapTests(SnapTestsBase): self._logger.info( "video duration: {}".format(video.get_property("duration")) ) - assert ( - video.get_property("duration") > exp["duration"] - ), "youtube video should have duration" + assert video.get_property("duration") > exp["duration"], ( + "youtube video should have duration" + ) self._wait.until( lambda d: video.get_property("currentTime") > exp["playback"] @@ -557,9 +560,9 @@ class SnapTests(SnapTestsBase): self._logger.info( "video played: {}".format(video.get_property("currentTime")) ) - assert ( - video.get_property("currentTime") > exp["playback"] - ), "youtube video should perform playback" + assert video.get_property("currentTime") > exp["playback"], ( + "youtube video should perform playback" + ) except TimeoutException as ex: self._logger.info("video detection timed out") self._logger.info(f"video: {video}") @@ -579,30 +582,34 @@ class SnapTests(SnapTestsBase): ) enable_drm_button = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".notification-button[label='Enable DRM']") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + ".notification-button[label='Enable DRM']", + )) ) self._logger.info("Enabling DRMs") enable_drm_button.click() self._wait.until( - EC.invisibility_of_element_located( - (By.CSS_SELECTOR, ".notification-button[label='Enable DRM']") - ) + EC.invisibility_of_element_located(( + By.CSS_SELECTOR, + ".notification-button[label='Enable DRM']", + )) ) self._logger.info("Installing DRMs") self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".infobar[value='drmContentCDMInstalling']") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + ".infobar[value='drmContentCDMInstalling']", + )) ) self._logger.info("Waiting for DRMs installation to complete") self._longwait.until( - EC.invisibility_of_element_located( - (By.CSS_SELECTOR, ".infobar[value='drmContentCDMInstalling']") - ) + EC.invisibility_of_element_located(( + By.CSS_SELECTOR, + ".infobar[value='drmContentCDMInstalling']", + )) ) self._driver.set_context("content") @@ -620,24 +627,25 @@ class SnapTests(SnapTestsBase): # Wait for duration to be set to something self._logger.info("Wait for video to start") video = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, "video.html5-main-video") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + "video.html5-main-video", + )) ) self._wait.until(lambda d: type(video.get_property("duration")) is float) self._logger.info("video duration: {}".format(video.get_property("duration"))) - assert ( - video.get_property("duration") > exp["duration"] - ), "youtube video should have duration" + assert video.get_property("duration") > exp["duration"], ( + "youtube video should have duration" + ) self._driver.execute_script("arguments[0].click();", video) video.send_keys("k") self._wait.until(lambda d: video.get_property("currentTime") > exp["playback"]) self._logger.info("video played: {}".format(video.get_property("currentTime"))) - assert ( - video.get_property("currentTime") > exp["playback"] - ), "youtube video should perform playback" + assert video.get_property("currentTime") > exp["playback"], ( + "youtube video should perform playback" + ) return True diff --git a/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py b/taskcluster/docker/snap-coreXX-build/snap-tests/qa_tests.py @@ -108,9 +108,10 @@ class QATests(SnapTestsBase): self._logger.info("find video") video = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, video_selector or "video") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + video_selector or "video", + )) ) self._wait.until(lambda d: type(video.get_property("duration")) is float) assert video.get_property("duration") > 0.0, "<video> duration null" @@ -150,9 +151,9 @@ class QATests(SnapTestsBase): self._logger.info("find video: wait currentTime") self._wait.until(lambda d: video.get_property("currentTime") >= 0.01) - assert ( - video.get_property("currentTime") >= 0.01 - ), "<video> currentTime not moved" + assert video.get_property("currentTime") >= 0.01, ( + "<video> currentTime not moved" + ) # this should pause self._logger.info("find video: pause") @@ -178,18 +179,18 @@ class QATests(SnapTestsBase): # we wait for 2s but it's not super accurate on CI (vbox VMs?), # observed values +/- 15% so check for more that should avoid # intermittent failures - assert ( - datum_after_resume >= datum_after_sleep + 0.5 - ), "<video> progressed after pause" + assert datum_after_resume >= datum_after_sleep + 0.5, ( + "<video> progressed after pause" + ) self._logger.info("find video: volume") self._driver.execute_script( "arguments[0].volume = arguments[1]", video, ref_volume * 0.25 ) new_volume = video.get_property("volume") - assert ( - new_volume == ref_volume * 0.25 - ), f"<video> sound volume increased from {ref_volume} to {ref_volume * 0.25} but got {new_volume}" + assert new_volume == ref_volume * 0.25, ( + f"<video> sound volume increased from {ref_volume} to {ref_volume * 0.25} but got {new_volume}" + ) self._logger.info("find video: done") @@ -215,9 +216,10 @@ class QATests(SnapTestsBase): self._logger.info("try fullscreen") fullscreen_button = self._wait.until( - EC.presence_of_element_located( - (By.CSS_SELECTOR, "button[aria-label*='(f)']") - ) + EC.presence_of_element_located(( + By.CSS_SELECTOR, + "button[aria-label*='(f)']", + )) ) self._driver.execute_script("return arguments[0].click();", fullscreen_button) time.sleep(1) @@ -340,9 +342,10 @@ class QATests(SnapTestsBase): def pdf_get_page(self, page, long=False): waiter = self._longwait if long is True else self._wait page = waiter.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, f"div.page[data-page-number='{page}'] canvas") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + f"div.page[data-page-number='{page}'] canvas", + )) ) self._wait.until( @@ -502,18 +505,20 @@ class QATests(SnapTestsBase): action = ActionChains(self._driver) paragraph = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, "span[role=presentation]") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + "span[role=presentation]", + )) ) action.drag_and_drop_by_offset(paragraph, 50, 10).perform() time.sleep(0.75) try: ref_screen_source = "select_text_with_highlight" self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, "button.highlightButton") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + "button.highlightButton", + )) ) except TimeoutException: ref_screen_source = "select_text_without_highlight" @@ -616,16 +621,14 @@ class QATests(SnapTestsBase): EC.visibility_of_element_located((By.ID, "contentAreaContextMenu")) ) copy = self._wait.until( - EC.visibility_of_element_located( + EC.visibility_of_element_located(( + By.ID, ( - By.ID, - ( - "context-copyimage-contents" - if mime_type.startswith("image/") - else "context-copy" - ), - ) - ) + "context-copyimage-contents" + if mime_type.startswith("image/") + else "context-copy" + ), + )) ) copy.click() self.wait_for_element_in_clipboard(mime_type, False) @@ -641,9 +644,9 @@ class QATests(SnapTestsBase): mime_type, ) self._driver.set_context("content") - assert ( - in_clipboard == should_be_present - ), f"type {mime_type} should/should ({should_be_present}) not be in clipboard" + assert in_clipboard == should_be_present, ( + f"type {mime_type} should/should ({should_be_present}) not be in clipboard" + ) def wait_for_element_in_clipboard(self, mime_type, context_change=False): if context_change: @@ -679,9 +682,10 @@ class QATests(SnapTestsBase): self._driver.switch_to.window(mystor) link = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, "#testlist > li:nth-child(11) > a:nth-child(1)") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + "#testlist > li:nth-child(11) > a:nth-child(1)", + )) ) link.click() drop_area = self._wait.until( @@ -698,9 +702,10 @@ class QATests(SnapTestsBase): self._driver.switch_to.window(images) text = self._wait.until( - EC.presence_of_element_located( - (By.CSS_SELECTOR, ".entry-content > p:nth-child(1)") - ) + EC.presence_of_element_located(( + By.CSS_SELECTOR, + ".entry-content > p:nth-child(1)", + )) ) self._driver.execute_script("arguments[0].scrollIntoView();", text) @@ -711,9 +716,10 @@ class QATests(SnapTestsBase): self._driver.switch_to.window(mystor) link = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, "#testlist > li:nth-child(12) > a:nth-child(1)") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + "#testlist > li:nth-child(12) > a:nth-child(1)", + )) ) link.click() drop_area = self._wait.until( @@ -739,17 +745,19 @@ class QATests(SnapTestsBase): time.sleep(1) download_item = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadTarget") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadTarget", + )) ) download_item.click() download_name = download_item.get_property("value") download_allow = self._wait.until( - EC.presence_of_element_located( - (By.ID, "downloadsPanel-blockedSubview-unblockButton") - ) + EC.presence_of_element_located(( + By.ID, + "downloadsPanel-blockedSubview-unblockButton", + )) ) download_allow.click() @@ -768,17 +776,19 @@ class QATests(SnapTestsBase): time.sleep(1) download_item = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadTarget") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadTarget", + )) ) download_name = download_item.get_property("value") self._logger.info(f"Waiting for download: {download_name}") download_progress = self._wait.until( - EC.presence_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadProgress") - ) + EC.presence_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadProgress", + )) ) self._logger.info( "Download progress {}%".format(download_progress.get_property("value")) @@ -788,9 +798,10 @@ class QATests(SnapTestsBase): self._wait.until(lambda d: download_progress.get_property("value") == 100) except TimeoutException as ex: details_normal = self._wait.until( - EC.presence_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadDetailsNormal") - ) + EC.presence_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadDetailsNormal", + )) ) self._logger.info( "Download details normal {}".format( @@ -798,9 +809,10 @@ class QATests(SnapTestsBase): ) ) details_hover = self._wait.until( - EC.presence_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadDetailsHover") - ) + EC.presence_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadDetailsHover", + )) ) self._logger.info( "Download details hover {}".format(details_hover.get_property("value")) @@ -825,9 +837,9 @@ class QATests(SnapTestsBase): ) self._driver.set_context("content") self._logger.info(f"Download folder pref: {download_dir_pref}") - assert ( - download_dir_pref == new - ), "download directory from pref should match new directory" + assert download_dir_pref == new, ( + "download directory from pref should match new directory" + ) def enable_downloads_debug(self): self._driver.set_context("chrome") @@ -850,12 +862,10 @@ class QATests(SnapTestsBase): def get_local_1M(self): return self._wait.until( - EC.presence_of_element_located( - ( - By.ID, - "download-link", - ) - ) + EC.presence_of_element_located(( + By.ID, + "download-link", + )) ) def test_download_folder_change(self, exp): @@ -945,44 +955,48 @@ class QATests(SnapTestsBase): time.sleep(1) download_details = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadDetailsNormal") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadDetailsNormal", + )) + ) + assert download_details.get_property("value").startswith("Completed"), ( + "download should be marked as completed" ) - assert download_details.get_property("value").startswith( - "Completed" - ), "download should be marked as completed" # TemporaryDirectory out of focus so folder removed # Close panel we will re-open it self._driver.execute_script("this.window.DownloadsButton.hide();") self._wait.until( - EC.invisibility_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadDetailsNormal") - ) + EC.invisibility_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadDetailsNormal", + )) ) assert os.path.isdir(tmpdir) is False, "tmpdir should have been removed" - assert ( - os.path.isfile(download_file) is False - ), "downloaded file should have been removed" + assert os.path.isfile(download_file) is False, ( + "downloaded file should have been removed" + ) download_button.click() time.sleep(1) download_item = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadTarget") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadTarget", + )) ) download_name_2 = download_item.get_property("value") assert download_name == download_name_2, "downloaded names should match" download_details = self._wait.until( - EC.visibility_of_element_located( - (By.CSS_SELECTOR, ".download-state .downloadDetailsNormal") - ) + EC.visibility_of_element_located(( + By.CSS_SELECTOR, + ".download-state .downloadDetailsNormal", + )) ) assert download_details.get_property("value").startswith( "File moved or missing" diff --git a/taskcluster/docker/updatebot/run.py b/taskcluster/docker/updatebot/run.py @@ -135,9 +135,7 @@ towrite = """ } } } -""".replace( - "TOKENHERE", phabricator_token -).replace( +""".replace("TOKENHERE", phabricator_token).replace( "PHAB_URL_HERE", phabricator_url + "api/" ) arcrc.write(towrite) diff --git a/taskcluster/gecko_taskgraph/__init__.py b/taskcluster/gecko_taskgraph/__init__.py @@ -32,16 +32,14 @@ tc_util.PRODUCTION_TASKCLUSTER_ROOT_URL = "https://firefox-ci-tc.services.mozill # Schemas for YAML files should use dashed identifiers by default. If there are # components of the schema for which there is a good reason to use another format, # exceptions can be added here. -schema.EXCEPTED_SCHEMA_IDENTIFIERS.extend( - [ - "test_name", - "json_location", - "video_location", - "profile_name", - "target_path", - "try_task_config", - ] -) +schema.EXCEPTED_SCHEMA_IDENTIFIERS.extend([ + "test_name", + "json_location", + "video_location", + "profile_name", + "target_path", + "try_task_config", +]) # TODO: These are temporarily redefined in gecko_taskgraph. Remove them from # upstream until they can be consolidated. diff --git a/taskcluster/gecko_taskgraph/actions/backfill.py b/taskcluster/gecko_taskgraph/actions/backfill.py @@ -278,7 +278,7 @@ def new_label(label, tasks): "maximum": 10, "title": "Times", "description": ( - "The number of times to execute each job " "you are backfilling." + "The number of times to execute each job you are backfilling." ), }, "retrigger": { diff --git a/taskcluster/gecko_taskgraph/actions/registry.py b/taskcluster/gecko_taskgraph/actions/registry.py @@ -149,18 +149,18 @@ def register_callback_action( def register_callback(cb): assert isinstance(name, str), "name must be a string" assert isinstance(order, int), "order must be an integer" - assert callable(schema) or is_json( - schema - ), "schema must be a JSON compatible object" + assert callable(schema) or is_json(schema), ( + "schema must be a JSON compatible object" + ) assert isinstance(cb, FunctionType), "callback must be a function" # Allow for json-e > 25 chars in the symbol. if "$" not in symbol: assert 1 <= len(symbol) <= 25, "symbol must be between 1 and 25 characters" assert isinstance(symbol, str), "symbol must be a string" - assert not mem[ - "registered" - ], "register_callback_action must be used as decorator" + assert not mem["registered"], ( + "register_callback_action must be used as decorator" + ) assert cb_name not in callbacks, f"callback name {cb_name} is not unique" def action_builder(parameters, graph_config, decision_task_id): @@ -223,32 +223,28 @@ def register_callback_action( if "/" in permission: raise Exception("`/` is not allowed in action names; use `-`") - rv.update( - { - "kind": "hook", - "hookGroupId": f"project-{trustDomain}", - "hookId": f"in-tree-action-{level}-{permission}/{tcyml_hash}", - "hookPayload": { - # provide the decision-task parameters as context for triggerHook - "decision": { - "action": action, - "repository": repository, - "push": push, - }, - # and pass everything else through from our own context - "user": { - "input": {"$eval": "input"}, - "taskId": {"$eval": "taskId"}, # target taskId (or null) - "taskGroupId": { - "$eval": "taskGroupId" - }, # target task group - }, + rv.update({ + "kind": "hook", + "hookGroupId": f"project-{trustDomain}", + "hookId": f"in-tree-action-{level}-{permission}/{tcyml_hash}", + "hookPayload": { + # provide the decision-task parameters as context for triggerHook + "decision": { + "action": action, + "repository": repository, + "push": push, }, - "extra": { - "actionPerm": permission, + # and pass everything else through from our own context + "user": { + "input": {"$eval": "input"}, + "taskId": {"$eval": "taskId"}, # target taskId (or null) + "taskGroupId": {"$eval": "taskGroupId"}, # target task group }, - } - ) + }, + "extra": { + "actionPerm": permission, + }, + }) return rv diff --git a/taskcluster/gecko_taskgraph/actions/release_promotion.py b/taskcluster/gecko_taskgraph/actions/release_promotion.py @@ -125,8 +125,7 @@ def get_flavors(graph_config, param): "rebuild_kinds": { "type": "array", "description": ( - "Optional: an array of kinds to ignore from the previous " - "graph(s)." + "Optional: an array of kinds to ignore from the previous graph(s)." ), "default": graph_config["release-promotion"].get("rebuild-kinds", []), "items": { @@ -158,8 +157,9 @@ def get_flavors(graph_config, param): "next_version": { "type": "string", "description": ( - "Next version. Required in the following flavors: " - "{}".format(get_flavors(graph_config, "version-bump")) + "Next version. Required in the following flavors: {}".format( + get_flavors(graph_config, "version-bump") + ) ), "default": "", }, @@ -177,8 +177,9 @@ def get_flavors(graph_config, param): "partial_updates": { "type": "object", "description": ( - "Partial updates. Required in the following flavors: " - "{}".format(get_flavors(graph_config, "partial-updates")) + "Partial updates. Required in the following flavors: {}".format( + get_flavors(graph_config, "partial-updates") + ) ), "default": {}, "additionalProperties": { @@ -396,13 +397,11 @@ def release_promotion_action(parameters, graph_config, input, task_group_id, tas parameters["release_enable_emefree"] = release_enable_emefree partner_config = input.get("release_partner_config") - if not partner_config and any( - [ - release_enable_partner_repack, - release_enable_partner_attribution, - release_enable_emefree, - ] - ): + if not partner_config and any([ + release_enable_partner_repack, + release_enable_partner_attribution, + release_enable_emefree, + ]): github_token = get_token(parameters) partner_config = get_partner_config(partner_url_config, github_token) if partner_config: diff --git a/taskcluster/gecko_taskgraph/actions/retrigger.py b/taskcluster/gecko_taskgraph/actions/retrigger.py @@ -293,13 +293,11 @@ def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id): for rerun_taskid in label_to_taskids[label]: _rerun_task(rerun_taskid, label) - retrigger_tasks.extend( - [ - label - for label in request.get("tasks") - if _should_retrigger(full_task_graph, label) - ] - ) + retrigger_tasks.extend([ + label + for label in request.get("tasks") + if _should_retrigger(full_task_graph, label) + ]) create_tasks( graph_config, diff --git a/taskcluster/gecko_taskgraph/actions/retrigger_custom.py b/taskcluster/gecko_taskgraph/actions/retrigger_custom.py @@ -41,36 +41,33 @@ basic_properties = { # Additional properties available for custom retrigger of some additional test suites extended_properties = basic_properties.copy() -extended_properties.update( - { - "runUntilFail": { - "type": "boolean", - "default": False, - "title": "Run until failure", - "description": ( - "Runs the specified set of tests repeatedly " - "until failure (up to REPEAT times)" - ), - }, - "repeat": { - "type": "integer", - "default": 0, - "minimum": 0, - "title": "Repeat test(s) N times", - "description": ( - "Run test(s) repeatedly (usually used in " - "conjunction with runUntilFail)" - ), - }, - "preferences": { - "type": "object", - "default": {"remote.log.level": "Info"}, - "title": "Extra gecko (about:config) preferences", - "description": "Extra gecko (about:config) preferences to use for this run", - "additionalProperties": {"type": "string"}, - }, - } -) +extended_properties.update({ + "runUntilFail": { + "type": "boolean", + "default": False, + "title": "Run until failure", + "description": ( + "Runs the specified set of tests repeatedly " + "until failure (up to REPEAT times)" + ), + }, + "repeat": { + "type": "integer", + "default": 0, + "minimum": 0, + "title": "Repeat test(s) N times", + "description": ( + "Run test(s) repeatedly (usually used in conjunction with runUntilFail)" + ), + }, + "preferences": { + "type": "object", + "default": {"remote.log.level": "Info"}, + "title": "Extra gecko (about:config) preferences", + "description": "Extra gecko (about:config) preferences to use for this run", + "additionalProperties": {"type": "string"}, + }, +}) @register_callback_action( diff --git a/taskcluster/gecko_taskgraph/actions/side_by_side.py b/taskcluster/gecko_taskgraph/actions/side_by_side.py @@ -46,9 +46,9 @@ def side_by_side_modifier(task, input): # Make side-by-side job searchable by the platform, test name, and revisions # it was triggered for - task.task["metadata"][ - "name" - ] = f"{input['platform']} {input['test_name']} {input['base_revision'][:12]} {input['new_revision'][:12]}" + task.task["metadata"]["name"] = ( + f"{input['platform']} {input['test_name']} {input['base_revision'][:12]} {input['new_revision'][:12]}" + ) # Use a job symbol to include the symbol of the job the side-by-side # is running for task.task["extra"]["treeherder"]["symbol"] += f"-{input['symbol']}" diff --git a/taskcluster/gecko_taskgraph/config.py b/taskcluster/gecko_taskgraph/config.py @@ -6,127 +6,123 @@ from taskgraph.util.schema import Schema, optionally_keyed_by from voluptuous import All, Any, Extra, Optional, Required from voluptuous.validators import Length -graph_config_schema = Schema( - { - # The trust-domain for this graph. - # (See https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/taskgraph.html#taskgraph-trust-domain) # noqa - Required("trust-domain"): str, - # This specifes the prefix for repo parameters that refer to the project being built. - # This selects between `head_rev` and `comm_head_rev` and related paramters. - # (See http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#push-information # noqa - # and http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#comm-push-information) # noqa - Required("project-repo-param-prefix"): str, - # This specifies the top level directory of the application being built. - # ie. "browser/" for Firefox, "comm/mail/" for Thunderbird. - Required("product-dir"): str, - Required("treeherder"): { - # Mapping of treeherder group symbols to descriptive names - Required("group-names"): {str: Length(max=100)} +graph_config_schema = Schema({ + # The trust-domain for this graph. + # (See https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/taskgraph.html#taskgraph-trust-domain) # noqa + Required("trust-domain"): str, + # This specifes the prefix for repo parameters that refer to the project being built. + # This selects between `head_rev` and `comm_head_rev` and related paramters. + # (See http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#push-information # noqa + # and http://firefox-source-docs.mozilla.org/taskcluster/taskcluster/parameters.html#comm-push-information) # noqa + Required("project-repo-param-prefix"): str, + # This specifies the top level directory of the application being built. + # ie. "browser/" for Firefox, "comm/mail/" for Thunderbird. + Required("product-dir"): str, + Required("treeherder"): { + # Mapping of treeherder group symbols to descriptive names + Required("group-names"): {str: Length(max=100)} + }, + Required("index"): {Required("products"): [str]}, + Required("try"): { + # We have a few platforms for which we want to do some "extra" builds, or at + # least build-ish things. Sort of. Anyway, these other things are implemented + # as different "platforms". These do *not* automatically ride along with "-p + # all" + Required("ridealong-builds"): {str: [str]}, + }, + Required("release-promotion"): { + Required("products"): [str], + Required("flavors"): { + str: { + Required("product"): str, + Required("target-tasks-method"): str, + Optional("is-rc"): bool, + Optional("rebuild-kinds"): [str], + Optional("version-bump"): bool, + Optional("partial-updates"): bool, + } }, - Required("index"): {Required("products"): [str]}, - Required("try"): { - # We have a few platforms for which we want to do some "extra" builds, or at - # least build-ish things. Sort of. Anyway, these other things are implemented - # as different "platforms". These do *not* automatically ride along with "-p - # all" - Required("ridealong-builds"): {str: [str]}, + Optional("rebuild-kinds"): [str], + }, + Required("scriptworker"): { + # Prefix to add to scopes controlling scriptworkers + Required("scope-prefix"): str, + }, + Required("task-priority"): optionally_keyed_by( + "project", + Any( + "highest", + "very-high", + "high", + "medium", + "low", + "very-low", + "lowest", + ), + ), + Required("partner-urls"): { + Required("release-partner-repack"): optionally_keyed_by( + "release-product", "release-level", "release-type", Any(str, None) + ), + Optional("release-partner-attribution"): optionally_keyed_by( + "release-product", "release-level", "release-type", Any(str, None) + ), + Required("release-eme-free-repack"): optionally_keyed_by( + "release-product", "release-level", "release-type", Any(str, None) + ), + }, + Required("workers"): { + Required("aliases"): { + str: { + Required("provisioner"): optionally_keyed_by("level", str), + Required("implementation"): str, + Required("os"): str, + Required("worker-type"): optionally_keyed_by( + "level", "release-level", "project", str + ), + } }, - Required("release-promotion"): { - Required("products"): [str], - Required("flavors"): { - str: { - Required("product"): str, - Required("target-tasks-method"): str, - Optional("is-rc"): bool, - Optional("rebuild-kinds"): [str], - Optional("version-bump"): bool, - Optional("partial-updates"): bool, + }, + Required("mac-signing"): { + Required("mac-requirements"): optionally_keyed_by("platform", str), + Required("hardened-sign-config"): optionally_keyed_by( + "hardened-signing-type", + [ + { + Optional("deep"): bool, + Optional("runtime"): bool, + Optional("force"): bool, + Optional("requirements"): optionally_keyed_by( + "release-product", "release-level", str + ), + Optional("entitlements"): optionally_keyed_by( + "build-platform", "project", str + ), + Required("globs"): [str], } - }, - Optional("rebuild-kinds"): [str], - }, - Required("scriptworker"): { - # Prefix to add to scopes controlling scriptworkers - Required("scope-prefix"): str, - }, - Required("task-priority"): optionally_keyed_by( - "project", - Any( - "highest", - "very-high", - "high", - "medium", - "low", - "very-low", - "lowest", - ), + ], ), - Required("partner-urls"): { - Required("release-partner-repack"): optionally_keyed_by( - "release-product", "release-level", "release-type", Any(str, None) - ), - Optional("release-partner-attribution"): optionally_keyed_by( - "release-product", "release-level", "release-type", Any(str, None) - ), - Required("release-eme-free-repack"): optionally_keyed_by( - "release-product", "release-level", "release-type", Any(str, None) - ), + }, + Required("taskgraph"): { + Optional( + "register", + description="Python function to call to register extensions.", + ): str, + Optional("decision-parameters"): str, + Optional("run"): { + Optional("use-caches"): Any(bool, [str]), }, - Required("workers"): { - Required("aliases"): { + Required("repositories"): All( + { str: { - Required("provisioner"): optionally_keyed_by("level", str), - Required("implementation"): str, - Required("os"): str, - Required("worker-type"): optionally_keyed_by( - "level", "release-level", "project", str - ), + Required("name"): str, + Optional("project-regex"): str, + Optional("ssh-secret-name"): str, + Extra: str, } }, - }, - Required("mac-signing"): { - Required("mac-requirements"): optionally_keyed_by("platform", str), - Required("hardened-sign-config"): optionally_keyed_by( - "hardened-signing-type", - [ - { - Optional("deep"): bool, - Optional("runtime"): bool, - Optional("force"): bool, - Optional("requirements"): optionally_keyed_by( - "release-product", "release-level", str - ), - Optional("entitlements"): optionally_keyed_by( - "build-platform", "project", str - ), - Required("globs"): [str], - } - ], - ), - }, - Required("taskgraph"): { - Optional( - "register", - description="Python function to call to register extensions.", - ): str, - Optional("decision-parameters"): str, - Optional("run"): { - Optional("use-caches"): Any(bool, [str]), - }, - Required("repositories"): All( - { - str: { - Required("name"): str, - Optional("project-regex"): str, - Optional("ssh-secret-name"): str, - Extra: str, - } - }, - Length(min=1), - ), - }, - Required("expiration-policy"): optionally_keyed_by( - "project", "level", {str: str} + Length(min=1), ), - } -) + }, + Required("expiration-policy"): optionally_keyed_by("project", "level", {str: str}), +}) diff --git a/taskcluster/gecko_taskgraph/files_changed.py b/taskcluster/gecko_taskgraph/files_changed.py @@ -58,7 +58,7 @@ def check(params, file_patterns): revision = params.get("comm_head_rev") if not revision: logger.warning( - "Missing `comm_head_rev` parameters; " "assuming all files have changed" + "Missing `comm_head_rev` parameters; assuming all files have changed" ) return True diff --git a/taskcluster/gecko_taskgraph/main.py b/taskcluster/gecko_taskgraph/main.py @@ -323,13 +323,11 @@ def show_taskgraph(options): for param in parameters[:]: if isinstance(param, str) and os.path.isdir(param): parameters.remove(param) - parameters.extend( - [ - p.as_posix() - for p in Path(param).iterdir() - if p.suffix in (".yml", ".json") - ] - ) + parameters.extend([ + p.as_posix() + for p in Path(param).iterdir() + if p.suffix in (".yml", ".json") + ]) logdir = None if len(parameters) > 1: diff --git a/taskcluster/gecko_taskgraph/morph.py b/taskcluster/gecko_taskgraph/morph.py @@ -17,7 +17,6 @@ the graph. # `{'relative-datestamp': '..'}` is handled at the last possible moment during # task creation. - import copy import logging import os diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py @@ -888,17 +888,14 @@ def make_desktop_nightly_filter(platforms): """Returns a filter that gets all nightly tasks on the given platform.""" def filter(task, parameters): - return all( - [ - filter_on_platforms(task, platforms), - filter_for_project(task, parameters), - task.attributes.get("shippable", False), - # Tests and nightly only builds don't have `shipping_product` set - task.attributes.get("shipping_product") - in {None, "firefox", "thunderbird"}, - task.kind not in {"l10n"}, # no on-change l10n - ] - ) + return all([ + filter_on_platforms(task, platforms), + filter_for_project(task, parameters), + task.attributes.get("shippable", False), + # Tests and nightly only builds don't have `shipping_product` set + task.attributes.get("shipping_product") in {None, "firefox", "thunderbird"}, + task.kind not in {"l10n"}, # no on-change l10n + ]) return filter @@ -939,9 +936,10 @@ def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux. The nightly build process involves a pipeline of builds, signing, and, eventually, uploading the tasks to balrog.""" - filter = make_desktop_nightly_filter( - {"linux64-shippable", "linux64-aarch64-shippable"} - ) + filter = make_desktop_nightly_filter({ + "linux64-shippable", + "linux64-aarch64-shippable", + }) return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] @@ -986,9 +984,10 @@ def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of asan. The nightly build process involves a pipeline of builds, signing, and, eventually, uploading the tasks to balrog.""" - filter = make_desktop_nightly_filter( - {"linux64-asan-reporter-shippable", "win64-asan-reporter-shippable"} - ) + filter = make_desktop_nightly_filter({ + "linux64-asan-reporter-shippable", + "win64-asan-reporter-shippable", + }) return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] diff --git a/taskcluster/gecko_taskgraph/test/conftest.py b/taskcluster/gecko_taskgraph/test/conftest.py @@ -190,20 +190,18 @@ def maketgg(monkeypatch): target_tasks_mod._target_task_methods["test_method"] = target_tasks_method monkeypatch.setattr(optimize_mod, "registry", fake_registry) - parameters = FakeParameters( - { - "_kinds": kinds, - "backstop": False, - "enable_always_target": False, - "level": 1, - "target_tasks_method": "test_method", - "test_manifest_loader": "default", - "try_mode": None, - "try_task_config": {}, - "tasks_for": "hg-push", - "project": "mozilla-central", - } - ) + parameters = FakeParameters({ + "_kinds": kinds, + "backstop": False, + "enable_always_target": False, + "level": 1, + "target_tasks_method": "test_method", + "test_manifest_loader": "default", + "try_mode": None, + "try_task_config": {}, + "tasks_for": "hg-push", + "project": "mozilla-central", + }) parameters.update(params) monkeypatch.setattr(generator_mod, "load_graph_config", fake_load_graph_config) diff --git a/taskcluster/gecko_taskgraph/test/test_actions.py b/taskcluster/gecko_taskgraph/test/test_actions.py @@ -435,12 +435,10 @@ def test_confirm_failures(mocker, responses, run_action, get_artifact): }, ) - errorsummary_content = b"\n".join( - [ - b'{"test": "dom/tests/test_example.html", "status": "FAIL", "expected": "PASS", "group": "dom/tests"}', - b'{"test": "dom/tests/test_another.html", "status": "FAIL", "expected": "PASS", "group": "dom/tests"}', - ] - ) + errorsummary_content = b"\n".join([ + b'{"test": "dom/tests/test_example.html", "status": "FAIL", "expected": "PASS", "group": "dom/tests"}', + b'{"test": "dom/tests/test_another.html", "status": "FAIL", "expected": "PASS", "group": "dom/tests"}', + ]) responses.get( f"{ROOT_URL}/api/queue/v1/task/{task_id}/artifacts/public%2Flogs%2Ferrorsummary.log", status=200, @@ -736,20 +734,18 @@ def test_release_promotion( responses.get( f"{ROOT_URL}/api/queue/v1/task/decision-task-id/artifacts/public%2Fparameters.yml", status=200, - body=yaml.dump( - { - "base_repository": "http://hg.example.com", - "head_repository": "http://hg.example.com", - "head_rev": "abcdef", - "project": "try", - "level": "1", - "pushlog_id": "100", - "required_signoffs": [], - "signoff_urls": {}, - "release_product": "firefox", - "release_type": "nightly", - } - ), + body=yaml.dump({ + "base_repository": "http://hg.example.com", + "head_repository": "http://hg.example.com", + "head_rev": "abcdef", + "project": "try", + "level": "1", + "pushlog_id": "100", + "required_signoffs": [], + "signoff_urls": {}, + "release_product": "firefox", + "release_type": "nightly", + }), content_type="application/x-yaml", ) responses.get( diff --git a/taskcluster/gecko_taskgraph/test/test_main.py b/taskcluster/gecko_taskgraph/test/test_main.py @@ -58,9 +58,11 @@ def test_output_file(run_main, tmpdir): run_main(["full", f"--output-file={output_file.strpath}"]) assert output_file.check() - assert output_file.read_text("utf-8").strip() == "\n".join( - ["_fake-t-0", "_fake-t-1", "_fake-t-2"] - ) + assert output_file.read_text("utf-8").strip() == "\n".join([ + "_fake-t-0", + "_fake-t-1", + "_fake-t-2", + ]) if __name__ == "__main__": diff --git a/taskcluster/gecko_taskgraph/test/test_morph.py b/taskcluster/gecko_taskgraph/test/test_morph.py @@ -106,13 +106,11 @@ def test_try_task_duplicates(make_taskgraph, graph_config, params, expected): taskb = Task(kind="test", label="b", attributes={}, task={}) task1 = Task(kind="test", label="a-1", attributes={}, task={}) task2 = Task(kind="test", label="a-2", attributes={}, task={}) - taskgraph, label_to_taskid = make_taskgraph( - { - taskb.label: taskb, - task1.label: task1, - task2.label: task2, - } - ) + taskgraph, label_to_taskid = make_taskgraph({ + taskb.label: taskb, + task1.label: task1, + task2.label: task2, + }) taskgraph, label_to_taskid = morph._add_try_task_duplicates( taskgraph, label_to_taskid, params, graph_config @@ -170,12 +168,10 @@ def test_make_index_tasks(make_taskgraph, graph_config): docker_task = Task( kind="docker-image", label="docker-image-index-task", attributes={}, task={} ) - taskgraph, label_to_taskid = make_taskgraph( - { - task.label: task, - docker_task.label: docker_task, - } - ) + taskgraph, label_to_taskid = make_taskgraph({ + task.label: task, + docker_task.label: docker_task, + }) index_paths = [ r.split(".", 1)[1] for r in task_def["routes"] if r.startswith("index.") diff --git a/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py b/taskcluster/gecko_taskgraph/test/test_optimize_strategies.py @@ -358,9 +358,12 @@ def test_bugbug_multiple_pushes(responses, params): labels = [ t.label for t in default_tasks if not opt.should_remove_task(t, params, {}) ] - assert sorted(labels) == sorted( - ["task-0-label", "task-1-label", "task-2-label", "task-4-label"] - ) + assert sorted(labels) == sorted([ + "task-0-label", + "task-1-label", + "task-2-label", + "task-4-label", + ]) def test_bugbug_timeout(monkeypatch, responses, params): diff --git a/taskcluster/gecko_taskgraph/test/test_target_tasks.py b/taskcluster/gecko_taskgraph/test/test_target_tasks.py @@ -127,19 +127,17 @@ class TestTargetTasks(unittest.TestCase): "ddd-var-2": Task(kind="test", label="ddd-var-2", attributes={}, task={}), } graph = Graph( - nodes=set( - [ - "a", - "b", - "c", - "ddd-1", - "ddd-2", - "ddd-1-cf", - "ddd-2-cf", - "ddd-var-1", - "ddd-var-2", - ] - ), + nodes=set([ + "a", + "b", + "c", + "ddd-1", + "ddd-2", + "ddd-1-cf", + "ddd-2-cf", + "ddd-var-1", + "ddd-var-2", + ]), edges=set(), ) return TaskGraph(tasks, graph) diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py @@ -6,7 +6,6 @@ Tests for the 'job' transform subsystem. """ - import os from copy import deepcopy @@ -36,15 +35,13 @@ TASK_DEFAULTS = { @pytest.fixture(scope="module") def config(): graph_config = load_graph_config(os.path.join(GECKO, "taskcluster")) - params = FakeParameters( - { - "base_repository": "http://hg.example.com", - "head_repository": "http://hg.example.com", - "head_rev": "abcdef", - "level": 1, - "project": "example", - } - ) + params = FakeParameters({ + "base_repository": "http://hg.example.com", + "head_repository": "http://hg.example.com", + "head_rev": "abcdef", + "level": 1, + "project": "example", + }) return TransformConfig( "job_test", here, {}, params, {}, graph_config, write_artifacts=False ) diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_test.py b/taskcluster/gecko_taskgraph/test/test_transforms_test.py @@ -78,28 +78,24 @@ def test_split_variants(monkeypatch, run_full_config_transform, make_test_task): def make_expected(variant): """Helper to generate expected tasks.""" - return make_test_task( - **{ - "attributes": {"unittest_variant": variant}, - "description": f"{variant} variant", - "mozharness": { - "extra-options": [f"--setpref={variant}=1"], - }, - "treeherder-symbol": f"g-{variant}(t)", - "variant-suffix": f"-{variant}", - } - ) + return make_test_task(**{ + "attributes": {"unittest_variant": variant}, + "description": f"{variant} variant", + "mozharness": { + "extra-options": [f"--setpref={variant}=1"], + }, + "treeherder-symbol": f"g-{variant}(t)", + "variant-suffix": f"-{variant}", + }) run_split_variants = partial( run_full_config_transform, test_transforms.variant.split_variants ) # test no variants - input_task = make_test_task( - **{ - "run-without-variant": True, - } - ) + input_task = make_test_task(**{ + "run-without-variant": True, + }) tasks = list(run_split_variants(input_task)) assert len(tasks) == 1 @@ -108,12 +104,10 @@ def test_split_variants(monkeypatch, run_full_config_transform, make_test_task): assert tasks[0] == expected # test variants are split into expected tasks - input_task = make_test_task( - **{ - "run-without-variant": True, - "variants": ["foo", "bar"], - } - ) + input_task = make_test_task(**{ + "run-without-variant": True, + "variants": ["foo", "bar"], + }) tasks = list(run_split_variants(input_task)) assert len(tasks) == 3 @@ -127,12 +121,10 @@ def test_split_variants(monkeypatch, run_full_config_transform, make_test_task): assert tasks[2] == expected # test composite variants - input_task = make_test_task( - **{ - "run-without-variant": True, - "variants": ["foo+bar"], - } - ) + input_task = make_test_task(**{ + "run-without-variant": True, + "variants": ["foo+bar"], + }) tasks = list(run_split_variants(input_task)) assert len(tasks) == 2 assert tasks[1]["attributes"]["unittest_variant"] == "foo+bar" @@ -143,26 +135,22 @@ def test_split_variants(monkeypatch, run_full_config_transform, make_test_task): assert tasks[1]["treeherder-symbol"] == "g-foo-bar(t)" # test 'when' filter - input_task = make_test_task( - **{ - "run-without-variant": True, - # this should cause task to be filtered out of 'bar' and 'foo+bar' variants - "test-platform": "windows", - "variants": ["foo", "bar", "foo+bar"], - } - ) + input_task = make_test_task(**{ + "run-without-variant": True, + # this should cause task to be filtered out of 'bar' and 'foo+bar' variants + "test-platform": "windows", + "variants": ["foo", "bar", "foo+bar"], + }) tasks = list(run_split_variants(input_task)) assert len(tasks) == 2 assert tasks[0]["attributes"]["unittest_variant"] is None assert tasks[1]["attributes"]["unittest_variant"] == "foo" # test 'run-without-variants=False' - input_task = make_test_task( - **{ - "run-without-variant": False, - "variants": ["foo"], - } - ) + input_task = make_test_task(**{ + "run-without-variant": False, + "variants": ["foo"], + }) tasks = list(run_split_variants(input_task)) assert len(tasks) == 1 assert tasks[0]["attributes"]["unittest_variant"] == "foo" diff --git a/taskcluster/gecko_taskgraph/test/test_util_backstop.py b/taskcluster/gecko_taskgraph/test/test_util_backstop.py @@ -170,9 +170,10 @@ def test_is_backstop( urls = { "index": get_index_url( - BACKSTOP_INDEX.format( - **{"trust-domain": "gecko", "project": params["project"]} - ) + BACKSTOP_INDEX.format(**{ + "trust-domain": "gecko", + "project": params["project"], + }) ), "artifact": get_artifact_url(LAST_BACKSTOP_PUSHID, "public%2Fparameters.yml"), "status": get_task_url(LAST_BACKSTOP_PUSHID) + "/status", diff --git a/taskcluster/gecko_taskgraph/test/test_util_chunking.py b/taskcluster/gecko_taskgraph/test/test_util_chunking.py @@ -365,12 +365,10 @@ def test_get_manifests(suite, platform, mock_mozinfo): if suite == "xpcshell": assert all([re.search(r"xpcshell(.*)?(.ini|.toml)", m) for m in items]) if "mochitest" in suite: - assert all( - [ - re.search(r"(perftest|mochitest|chrome|browser).*(.ini|.toml)", m) - for m in items - ] - ) + assert all([ + re.search(r"(perftest|mochitest|chrome|browser).*(.ini|.toml)", m) + for m in items + ]) if "web-platform" in suite: assert all([m.startswith("/") and m.count("/") <= 4 for m in items]) diff --git a/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py b/taskcluster/gecko_taskgraph/test/test_util_runnable_jobs.py @@ -50,9 +50,9 @@ class TestRunnableJobs(unittest.TestCase): return taskgraph, label_to_taskid def test_taskgraph_to_runnable_jobs(self): - tg, label_to_taskid = self.make_taskgraph( - {t["label"]: Task(**t) for t in self.tasks[:]} - ) + tg, label_to_taskid = self.make_taskgraph({ + t["label"]: Task(**t) for t in self.tasks[:] + }) res = full_task_graph_to_runnable_jobs(tg.to_json()) diff --git a/taskcluster/gecko_taskgraph/test/test_util_verify.py b/taskcluster/gecko_taskgraph/test/test_util_verify.py @@ -9,7 +9,6 @@ gecko_taskgraph.util.verify.verify_docs with different parameters to do the actual checking. """ - import os.path import pytest diff --git a/taskcluster/gecko_taskgraph/transforms/artifact.py b/taskcluster/gecko_taskgraph/transforms/artifact.py @@ -4,6 +4,7 @@ """ Apply different expiration dates to different artifacts based on a manifest file (artifacts.yml) """ + import logging import os import sys diff --git a/taskcluster/gecko_taskgraph/transforms/balrog_submit.py b/taskcluster/gecko_taskgraph/transforms/balrog_submit.py @@ -14,27 +14,25 @@ from voluptuous import Optional, Required from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job -balrog_description_schema = Schema( - { - # unique label to describe this balrog task, defaults to balrog-{dep.label} - Required("label"): str, - Optional( - "update-no-wnp", - description="Whether the parallel `-No-WNP` blob should be updated as well.", - ): optionally_keyed_by("release-type", bool), - # treeherder is allowed here to override any defaults we use for beetmover. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the - # below transforms for defaults of various values. - Optional("treeherder"): task_description_schema["treeherder"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("task-from"): task_description_schema["task-from"], - # Shipping product / phase - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +balrog_description_schema = Schema({ + # unique label to describe this balrog task, defaults to balrog-{dep.label} + Required("label"): str, + Optional( + "update-no-wnp", + description="Whether the parallel `-No-WNP` blob should be updated as well.", + ): optionally_keyed_by("release-type", bool), + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("task-from"): task_description_schema["task-from"], + # Shipping product / phase + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover.py b/taskcluster/gecko_taskgraph/transforms/beetmover.py @@ -22,24 +22,22 @@ from gecko_taskgraph.util.scriptworker import ( transforms = TransformSequence() -beetmover_description_schema = Schema( - { - # unique label to describe this beetmover task - Required("label"): str, - Required("dependencies"): task_description_schema["dependencies"], - # treeherder is allowed here to override any defaults we use for beetmover. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the - # below transforms for defaults of various values. - Optional("treeherder"): task_description_schema["treeherder"], - # locale is passed only for l10n beetmoving - Optional("locale"): str, - Required("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("attributes"): task_description_schema["attributes"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_description_schema = Schema({ + # unique label to describe this beetmover task + Required("label"): str, + Required("dependencies"): task_description_schema["dependencies"], + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + # locale is passed only for l10n beetmoving + Optional("locale"): str, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) @transforms.add @@ -144,9 +142,9 @@ def craft_release_properties(config, job): @transforms.add def make_task_worker(config, jobs): for job in jobs: - valid_beetmover_job = len(job["dependencies"]) == 2 and any( - ["signing" in j for j in job["dependencies"]] - ) + valid_beetmover_job = len(job["dependencies"]) == 2 and any([ + "signing" in j for j in job["dependencies"] + ]) if not valid_beetmover_job: raise NotImplementedError("Beetmover must have two dependencies.") diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_checksums.py @@ -21,19 +21,17 @@ from gecko_taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, ) -beetmover_checksums_description_schema = Schema( - { - Required("attributes"): {str: object}, - Required("dependencies"): task_description_schema["dependencies"], - Optional("label"): str, - Optional("treeherder"): task_description_schema["treeherder"], - Optional("locale"): str, - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_checksums_description_schema = Schema({ + Required("attributes"): {str: object}, + Required("dependencies"): task_description_schema["dependencies"], + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("locale"): str, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_emefree_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_emefree_checksums.py @@ -14,18 +14,16 @@ from gecko_taskgraph.transforms.beetmover import craft_release_properties from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job -beetmover_checksums_description_schema = Schema( - { - Optional("label"): str, - Optional("extra"): object, - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("task-from"): task_description_schema["task-from"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_checksums_description_schema = Schema({ + Optional("label"): str, + Optional("extra"): object, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("task-from"): task_description_schema["task-from"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py b/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py @@ -5,7 +5,6 @@ Transform the beetmover task into an actual task description. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.copy import deepcopy from taskgraph.util.dependencies import get_primary_dependency @@ -26,23 +25,21 @@ from gecko_taskgraph.util.declarative_artifacts import ( get_geckoview_upstream_artifacts, ) -beetmover_description_schema = Schema( - { - Required("label"): str, - Required("dependencies"): task_description_schema["dependencies"], - Optional("treeherder"): task_description_schema["treeherder"], - Required("run-on-projects"): task_description_schema["run-on-projects"], - Required("run-on-hg-branches"): task_description_schema["run-on-hg-branches"], - Optional("bucket-scope"): optionally_keyed_by("release-level", str), - Optional("shipping-phase"): optionally_keyed_by( - "project", task_description_schema["shipping-phase"] - ), - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("attributes"): task_description_schema["attributes"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_description_schema = Schema({ + Required("label"): str, + Required("dependencies"): task_description_schema["dependencies"], + Optional("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("run-on-hg-branches"): task_description_schema["run-on-hg-branches"], + Optional("bucket-scope"): optionally_keyed_by("release-level", str), + Optional("shipping-phase"): optionally_keyed_by( + "project", task_description_schema["shipping-phase"] + ), + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() @@ -123,8 +120,7 @@ def make_task_description(config, jobs): treeherder.setdefault("symbol", f"BM-{package}{symbol_suffix}") label = job["label"] description = ( - "Beetmover submission for geckoview" - "{build_platform}/{build_type}'".format( + "Beetmover submission for geckoview{build_platform}/{build_type}'".format( build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_integration.py b/taskcluster/gecko_taskgraph/transforms/beetmover_integration.py @@ -5,7 +5,6 @@ Transform the beetmover task into an actual task description. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.dependencies import get_primary_dependency from taskgraph.util.keyed_by import evaluate_keyed_by @@ -20,21 +19,19 @@ from gecko_taskgraph.util.scriptworker import ( generate_beetmover_upstream_artifacts, ) -beetmover_description_schema = Schema( - { - Required("label"): str, - Required("description"): str, - Required("dependencies"): task_description_schema["dependencies"], - Required("if-dependencies"): task_description_schema["if-dependencies"], - Optional("treeherder"): task_description_schema["treeherder"], - Required("run-on-projects"): task_description_schema["run-on-projects"], - Optional("attributes"): task_description_schema["attributes"], - Optional("task-from"): task_description_schema["task-from"], - Required("worker-type"): task_description_schema["worker-type"], - Required("scopes"): optionally_keyed_by("project", [str]), - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_description_schema = Schema({ + Required("label"): str, + Required("description"): str, + Required("dependencies"): task_description_schema["dependencies"], + Required("if-dependencies"): task_description_schema["if-dependencies"], + Optional("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Optional("attributes"): task_description_schema["attributes"], + Optional("task-from"): task_description_schema["task-from"], + Required("worker-type"): task_description_schema["worker-type"], + Required("scopes"): optionally_keyed_by("project", [str]), + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_langpack_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_langpack_checksums.py @@ -21,19 +21,17 @@ from gecko_taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, ) -beetmover_checksums_description_schema = Schema( - { - Required("attributes"): {str: object}, - Optional("label"): str, - Optional("treeherder"): task_description_schema["treeherder"], - Optional("locale"): str, - Optional("dependencies"): task_description_schema["dependencies"], - Optional("task-from"): task_description_schema["task-from"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_checksums_description_schema = Schema({ + Required("attributes"): {str: object}, + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("locale"): str, + Optional("dependencies"): task_description_schema["dependencies"], + Optional("task-from"): task_description_schema["task-from"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py b/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py @@ -15,24 +15,22 @@ from gecko_taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, ) -beetmover_push_to_release_description_schema = Schema( - { - Required("name"): str, - Required("product"): str, - Required("treeherder-platform"): str, - Optional("attributes"): {str: object}, - Optional("task-from"): task_description_schema["task-from"], - Optional("run"): {str: object}, - Optional("run-on-projects"): task_description_schema["run-on-projects"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - Optional("dependencies"): {str: taskref_or_string}, - Optional("index"): {str: str}, - Optional("routes"): [str], - Required("shipping-phase"): task_description_schema["shipping-phase"], - Required("shipping-product"): task_description_schema["shipping-product"], - Optional("extra"): task_description_schema["extra"], - } -) +beetmover_push_to_release_description_schema = Schema({ + Required("name"): str, + Required("product"): str, + Required("treeherder-platform"): str, + Optional("attributes"): {str: object}, + Optional("task-from"): task_description_schema["task-from"], + Optional("run"): {str: object}, + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], + Optional("dependencies"): {str: taskref_or_string}, + Optional("index"): {str: str}, + Optional("routes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Optional("extra"): task_description_schema["extra"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage.py @@ -36,23 +36,21 @@ from gecko_taskgraph.util.scriptworker import ( logger = logging.getLogger(__name__) -beetmover_description_schema = Schema( - { - # unique label to describe this beetmover task, defaults to {dep.label}-beetmover - Required("label"): str, - Required("dependencies"): task_description_schema["dependencies"], - # treeherder is allowed here to override any defaults we use for beetmover. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the - # below transforms for defaults of various values. - Optional("treeherder"): task_description_schema["treeherder"], - Optional("attributes"): task_description_schema["attributes"], - # locale is passed only for l10n beetmoving - Optional("locale"): str, - Required("shipping-phase"): task_description_schema["shipping-phase"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_description_schema = Schema({ + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Required("label"): str, + Required("dependencies"): task_description_schema["dependencies"], + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("attributes"): task_description_schema["attributes"], + # locale is passed only for l10n beetmoving + Optional("locale"): str, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() @@ -282,9 +280,9 @@ def strip_unwanted_langpacks_from_worker(config, jobs): job["worker"]["artifact-map"].remove(map) for artifact in job["worker"].get("upstream-artifacts", []): - if not any( - [path.endswith("target.langpack.xpi") for path in artifact["paths"]] - ): + if not any([ + path.endswith("target.langpack.xpi") for path in artifact["paths"] + ]): continue if artifact["locale"] == "ja-JP-mac": # This locale should only exist on mac diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_l10n.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_l10n.py @@ -5,7 +5,6 @@ Transform the signing task into an actual task description. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.dependencies import get_primary_dependency from taskgraph.util.treeherder import join_symbol diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py @@ -27,21 +27,19 @@ from gecko_taskgraph.util.scriptworker import ( logger = logging.getLogger(__name__) -beetmover_description_schema = Schema( - { - # unique label to describe this beetmover task, defaults to {dep.label}-beetmover - Optional("label"): str, - Required("partner-path"): str, - Optional("extra"): object, - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Required("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("priority"): task_description_schema["priority"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_description_schema = Schema({ + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + Required("partner-path"): str, + Optional("extra"): object, + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("priority"): task_description_schema["priority"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() @@ -146,77 +144,59 @@ def generate_upstream_artifacts( artifact_prefix = get_artifact_prefix(job) if "linux" in platform: - upstream_artifacts.append( - { - "taskId": {"task-reference": build_task_ref}, - "taskType": "build", - "paths": [f"{artifact_prefix}/{repack_id}/target.tar.xz"], - "locale": partner_path, - } - ) - upstream_artifacts.append( - { - "taskId": {"task-reference": repackage_signing_task_ref}, - "taskType": "repackage", - "paths": [f"{artifact_prefix}/{repack_id}/target.tar.xz.asc"], - "locale": partner_path, - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": build_task_ref}, + "taskType": "build", + "paths": [f"{artifact_prefix}/{repack_id}/target.tar.xz"], + "locale": partner_path, + }) + upstream_artifacts.append({ + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.tar.xz.asc"], + "locale": partner_path, + }) elif "macosx" in platform: - upstream_artifacts.append( - { - "taskId": {"task-reference": repackage_task_ref}, - "taskType": "repackage", - "paths": [f"{artifact_prefix}/{repack_id}/target.dmg"], - "locale": partner_path, - } - ) - upstream_artifacts.append( - { - "taskId": {"task-reference": repackage_signing_task_ref}, - "taskType": "repackage", - "paths": [f"{artifact_prefix}/{repack_id}/target.dmg.asc"], - "locale": partner_path, - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": repackage_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.dmg"], + "locale": partner_path, + }) + upstream_artifacts.append({ + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.dmg.asc"], + "locale": partner_path, + }) elif "win" in platform: - upstream_artifacts.append( - { + upstream_artifacts.append({ + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe"], + "locale": partner_path, + }) + upstream_artifacts.append({ + "taskId": {"task-reference": repackage_signing_task_ref}, + "taskType": "repackage", + "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe.asc"], + "locale": partner_path, + }) + if platform.startswith("win32") and repack_stub_installer: + upstream_artifacts.append({ "taskId": {"task-reference": repackage_signing_task_ref}, "taskType": "repackage", - "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe"], + "paths": [f"{artifact_prefix}/{repack_id}/target.stub-installer.exe"], "locale": partner_path, - } - ) - upstream_artifacts.append( - { + }) + upstream_artifacts.append({ "taskId": {"task-reference": repackage_signing_task_ref}, "taskType": "repackage", - "paths": [f"{artifact_prefix}/{repack_id}/target.installer.exe.asc"], + "paths": [ + f"{artifact_prefix}/{repack_id}/target.stub-installer.exe.asc" + ], "locale": partner_path, - } - ) - if platform.startswith("win32") and repack_stub_installer: - upstream_artifacts.append( - { - "taskId": {"task-reference": repackage_signing_task_ref}, - "taskType": "repackage", - "paths": [ - f"{artifact_prefix}/{repack_id}/target.stub-installer.exe" - ], - "locale": partner_path, - } - ) - upstream_artifacts.append( - { - "taskId": {"task-reference": repackage_signing_task_ref}, - "taskType": "repackage", - "paths": [ - f"{artifact_prefix}/{repack_id}/target.stub-installer.exe.asc" - ], - "locale": partner_path, - } - ) + }) if not upstream_artifacts: raise Exception("Couldn't find any upstream artifacts.") diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_rpm.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_rpm.py @@ -22,16 +22,14 @@ from gecko_taskgraph.util.scriptworker import ( transforms = TransformSequence() -beetmover_description_schema = Schema( - { - Required("attributes"): task_description_schema["attributes"], - Required("dependencies"): task_description_schema["dependencies"], - Required("label"): str, - Required("name"): str, - Required("shipping-phase"): task_description_schema["shipping-phase"], - Required("task-from"): task_description_schema["task-from"], - } -) +beetmover_description_schema = Schema({ + Required("attributes"): task_description_schema["attributes"], + Required("dependencies"): task_description_schema["dependencies"], + Required("label"): str, + Required("name"): str, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("task-from"): task_description_schema["task-from"], +}) transforms.add_validate(beetmover_description_schema) diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_source_checksums.py b/taskcluster/gecko_taskgraph/transforms/beetmover_source_checksums.py @@ -20,19 +20,17 @@ from gecko_taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, ) -beetmover_checksums_description_schema = Schema( - { - Optional("label"): str, - Optional("treeherder"): task_description_schema["treeherder"], - Optional("locale"): str, - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_checksums_description_schema = Schema({ + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("locale"): str, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/bootstrap.py b/taskcluster/gecko_taskgraph/transforms/bootstrap.py @@ -11,21 +11,19 @@ from gecko_taskgraph.transforms.task import task_description_schema transforms = TransformSequence() -bootstrap_schema = Schema( - { - # Name of the bootstrap task. - Required("name"): str, - # Name of the docker image. Ideally, we'd also have tasks for mac and windows, - # but we unfortunately don't have workers barebones enough for such testing - # to be satisfactory. - Required("image"): Any(str, {"in-tree": str}), - # Initialization commands. - Required("pre-commands"): [str], - # relative path (from config.path) to the file task was defined in - Optional("task-from"): str, - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +bootstrap_schema = Schema({ + # Name of the bootstrap task. + Required("name"): str, + # Name of the docker image. Ideally, we'd also have tasks for mac and windows, + # but we unfortunately don't have workers barebones enough for such testing + # to be satisfactory. + Required("image"): Any(str, {"in-tree": str}), + # Initialization commands. + Required("pre-commands"): [str], + # relative path (from config.path) to the file task was defined in + Optional("task-from"): str, + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms.add_validate(bootstrap_schema) diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py b/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py @@ -5,7 +5,6 @@ Add from parameters.yml into bouncer submission tasks. """ - import logging from taskgraph.transforms.base import TransformSequence @@ -64,9 +63,7 @@ def make_task_worker(config, jobs): else: logger.warning( 'No bouncer entries defined in bouncer submission task for "{}". \ -Job deleted.'.format( - job["name"] - ) +Job deleted.'.format(job["name"]) ) @@ -90,19 +87,17 @@ def craft_bouncer_entries(config, job): if partner_bouncer_products_per_alias: partners = get_partners_to_be_published(config) for partner, sub_config_name, _ in partners: - entries.update( - { - bouncer_alias.replace( - "PARTNER", f"{partner}-{sub_config_name}" - ): craft_partner_bouncer_product_name( - product, - bouncer_product, - current_version, - partner, - sub_config_name, - ) - for bouncer_alias, bouncer_product in partner_bouncer_products_per_alias.items() # NOQA: E501 - } - ) + entries.update({ + bouncer_alias.replace( + "PARTNER", f"{partner}-{sub_config_name}" + ): craft_partner_bouncer_product_name( + product, + bouncer_product, + current_version, + partner, + sub_config_name, + ) + for bouncer_alias, bouncer_product in partner_bouncer_products_per_alias.items() # NOQA: E501 + }) return entries diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_check.py b/taskcluster/gecko_taskgraph/transforms/bouncer_check.py @@ -24,12 +24,10 @@ def add_command(config, jobs): "python", "testing/mozharness/scripts/release/bouncer_check.py", ] - job["run"].update( - { - "using": "mach", - "mach": command, - } - ) + job["run"].update({ + "using": "mach", + "mach": command, + }) yield job @@ -80,12 +78,10 @@ def handle_keyed_by(config, jobs): job["run"]["mach"].extend(["--config", cfg]) if config.kind == "cron-bouncer-check": - job["run"]["mach"].extend( - [ - "--product-field={}".format(job["run"]["product-field"]), - "--products-url={}".format(job["run"]["products-url"]), - ] - ) + job["run"]["mach"].extend([ + "--product-field={}".format(job["run"]["product-field"]), + "--products-url={}".format(job["run"]["products-url"]), + ]) del job["run"]["product-field"] del job["run"]["products-url"] elif config.kind == "release-bouncer-check": diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py @@ -5,7 +5,6 @@ Add from parameters.yml into bouncer submission tasks. """ - import copy import logging @@ -30,8 +29,10 @@ FTP_PLATFORMS_PER_BOUNCER_PLATFORM = { } # :lang is interpolated by bouncer at runtime -CANDIDATES_PATH_TEMPLATE = "/{ftp_product}/candidates/{version}-candidates/build{build_number}/\ +CANDIDATES_PATH_TEMPLATE = ( + "/{ftp_product}/candidates/{version}-candidates/build{build_number}/\ {update_folder}{ftp_platform}/:lang/{file}" +) RELEASES_PATH_TEMPLATE = "/{ftp_product}/releases/{version}/\ {update_folder}{ftp_platform}/:lang/{file}" @@ -167,9 +168,7 @@ def make_task_worker(config, jobs): else: logger.warning( 'No bouncer entries defined in bouncer submission task for "{}". \ -Job deleted.'.format( - job["name"] - ) +Job deleted.'.format(job["name"]) ) @@ -189,9 +188,7 @@ def craft_bouncer_entries(config, job): else: logger.warning( 'No partials defined! Bouncer submission task won\'t send any \ -partial-related entry for "{}"'.format( - job["name"] - ) +partial-related entry for "{}"'.format(job["name"]) ) bouncer_products = [ bouncer_product diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py @@ -5,7 +5,6 @@ Add from parameters.yml into bouncer submission tasks. """ - import logging from taskgraph.transforms.base import TransformSequence @@ -38,8 +37,10 @@ PARTNER_PLATFORMS_TO_BOUNCER = { } # :lang is interpolated by bouncer at runtime -RELEASES_PARTNERS_PATH_TEMPLATE = "/{ftp_product}/releases/partners/{partner}/{sub_config}/\ +RELEASES_PARTNERS_PATH_TEMPLATE = ( + "/{ftp_product}/releases/partners/{partner}/{sub_config}/\ {version}/{ftp_platform}/:lang/{file}" +) CONFIG_PER_BOUNCER_PRODUCT = { "installer": { @@ -109,27 +110,25 @@ def craft_bouncer_entries(config, job): entries = {} for partner, sub_config_name, platforms in partners: platforms = [PARTNER_PLATFORMS_TO_BOUNCER[p] for p in platforms] - entries.update( - { - craft_partner_bouncer_product_name( - product, bouncer_product, current_version, partner, sub_config_name - ): { - "options": { - "add_locales": False, # partners may use different sets of locales - "ssl_only": craft_ssl_only(bouncer_product), - }, - "paths_per_bouncer_platform": craft_paths_per_bouncer_platform( - product, - bouncer_product, - platforms, - current_version, - partner, - sub_config_name, - ), - } - for bouncer_product in bouncer_products + entries.update({ + craft_partner_bouncer_product_name( + product, bouncer_product, current_version, partner, sub_config_name + ): { + "options": { + "add_locales": False, # partners may use different sets of locales + "ssl_only": craft_ssl_only(bouncer_product), + }, + "paths_per_bouncer_platform": craft_paths_per_bouncer_platform( + product, + bouncer_product, + platforms, + current_version, + partner, + sub_config_name, + ), } - ) + for bouncer_product in bouncer_products + }) return entries diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py @@ -5,6 +5,7 @@ Apply some defaults and minor modifications to the jobs defined in the build kind. """ + import logging from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS @@ -118,9 +119,9 @@ def mozconfig(config, jobs): ) mozconfig_variant = job["run"].pop("mozconfig-variant", None) if mozconfig_variant: - job["run"].setdefault("extra-config", {})[ - "mozconfig_variant" - ] = mozconfig_variant + job["run"].setdefault("extra-config", {})["mozconfig_variant"] = ( + mozconfig_variant + ) yield job @@ -247,9 +248,9 @@ def add_signing_artifacts(config, jobs): # Not macosx build or no artifacts defined, so skip yield job continue - assert ( - "artifacts" in job["worker"] - ), "macosx build jobs must have worker.artifacts defined." + assert "artifacts" in job["worker"], ( + "macosx build jobs must have worker.artifacts defined." + ) is_shippable = ( ("shippable" in job["attributes"] and job["attributes"]["shippable"]) # Instrumented builds don't have attributes.shippable set @@ -283,13 +284,11 @@ def add_signing_artifacts(config, jobs): ) # Add utility.xml if not prod/shippable if not is_prod_project or not is_shippable: - job["worker"]["artifacts"].append( - { - "name": "public/build/security/utility.xml", - "path": "checkouts/gecko/security/mac/hardenedruntime/developer/utility.xml", - "type": "file", - } - ) + job["worker"]["artifacts"].append({ + "name": "public/build/security/utility.xml", + "path": "checkouts/gecko/security/mac/hardenedruntime/developer/utility.xml", + "type": "file", + }) impl, _ = worker_type_implementation( config.graph_config, config.params, job["worker-type"] ) diff --git a/taskcluster/gecko_taskgraph/transforms/build_attrs.py b/taskcluster/gecko_taskgraph/transforms/build_attrs.py @@ -24,11 +24,9 @@ def set_build_attributes(config, jobs): build_type = "opt" attributes = job.setdefault("attributes", {}) - attributes.update( - { - "build_platform": build_platform, - "build_type": build_type, - } - ) + attributes.update({ + "build_platform": build_platform, + "build_type": build_type, + }) yield job diff --git a/taskcluster/gecko_taskgraph/transforms/build_signing.py b/taskcluster/gecko_taskgraph/transforms/build_signing.py @@ -5,7 +5,6 @@ Transform the signing task into an actual task description. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.dependencies import get_primary_dependency diff --git a/taskcluster/gecko_taskgraph/transforms/cached_tasks.py b/taskcluster/gecko_taskgraph/transforms/cached_tasks.py @@ -36,13 +36,11 @@ def order_tasks(config, tasks): def format_task_digest(cached_task): - return "/".join( - [ - cached_task["type"], - cached_task["name"], - cached_task["digest"], - ] - ) + return "/".join([ + cached_task["type"], + cached_task["name"], + cached_task["digest"], + ]) @transforms.add diff --git a/taskcluster/gecko_taskgraph/transforms/chunk_partners.py b/taskcluster/gecko_taskgraph/transforms/chunk_partners.py @@ -5,7 +5,6 @@ Chunk the partner repack tasks by subpartner and locale """ - import copy from mozbuild.chunkify import chunkify diff --git a/taskcluster/gecko_taskgraph/transforms/code_review.py b/taskcluster/gecko_taskgraph/transforms/code_review.py @@ -5,7 +5,6 @@ Add soft dependencies and configuration to code-review tasks. """ - from taskgraph.transforms.base import TransformSequence transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/condprof.py b/taskcluster/gecko_taskgraph/transforms/condprof.py @@ -14,26 +14,24 @@ from voluptuous import Optional from gecko_taskgraph.transforms.job import job_description_schema from gecko_taskgraph.transforms.task import task_description_schema -diff_description_schema = Schema( - { - # default is settled, but add 'full' to get both - Optional("scenarios"): [str], - Optional("description"): task_description_schema["description"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("fetches"): job_description_schema["fetches"], - Optional("index"): task_description_schema["index"], - Optional("task-from"): str, - Optional("name"): str, - Optional("run"): job_description_schema["run"], - Optional("run-on-projects"): task_description_schema["run-on-projects"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - Optional("scopes"): task_description_schema["scopes"], - Optional("treeherder"): task_description_schema["treeherder"], - Optional("use-python"): job_description_schema["use-python"], - Optional("worker"): job_description_schema["worker"], - Optional("worker-type"): task_description_schema["worker-type"], - } -) +diff_description_schema = Schema({ + # default is settled, but add 'full' to get both + Optional("scenarios"): [str], + Optional("description"): task_description_schema["description"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("fetches"): job_description_schema["fetches"], + Optional("index"): task_description_schema["index"], + Optional("task-from"): str, + Optional("name"): str, + Optional("run"): job_description_schema["run"], + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], + Optional("scopes"): task_description_schema["scopes"], + Optional("treeherder"): task_description_schema["treeherder"], + Optional("use-python"): job_description_schema["use-python"], + Optional("worker"): job_description_schema["worker"], + Optional("worker-type"): task_description_schema["worker-type"], +}) transforms = TransformSequence() transforms.add_validate(diff_description_schema) diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py @@ -18,42 +18,40 @@ index_or_string = Any( {Required("index-search"): str}, ) -diff_description_schema = Schema( - { - # Name of the diff task. - Required("name"): str, - # Treeherder tier. - Required("tier"): int, - # Treeherder symbol. - Required("symbol"): str, - # relative path (from config.path) to the file the task was defined in. - Optional("task-from"): str, - # Original and new builds to compare. - Required("original"): index_or_string, - Required("new"): index_or_string, - # Arguments to pass to diffoscope, used for task-defaults in - # taskcluster/kinds/diffoscope/kind.yml - Optional("args"): str, - # Extra arguments to pass to diffoscope, that can be set per job. - Optional("extra-args"): str, - # Fail the task when differences are detected. - Optional("fail-on-diff"): bool, - # What artifact to check the differences of. Defaults to target.tar.xz - # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for - # Android. - Optional("artifact"): str, - # Whether to unpack first. Diffoscope can normally work without unpacking, - # but when one needs to --exclude some contents, that doesn't work out well - # if said content is packed (e.g. in omni.ja). - Optional("unpack"): bool, - # Commands to run before performing the diff. - Optional("pre-diff-commands"): [str], - # Only run the task on a set of projects/branches. - Optional("run-on-projects"): task_description_schema["run-on-projects"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - Optional("optimization"): task_description_schema["optimization"], - } -) +diff_description_schema = Schema({ + # Name of the diff task. + Required("name"): str, + # Treeherder tier. + Required("tier"): int, + # Treeherder symbol. + Required("symbol"): str, + # relative path (from config.path) to the file the task was defined in. + Optional("task-from"): str, + # Original and new builds to compare. + Required("original"): index_or_string, + Required("new"): index_or_string, + # Arguments to pass to diffoscope, used for task-defaults in + # taskcluster/kinds/diffoscope/kind.yml + Optional("args"): str, + # Extra arguments to pass to diffoscope, that can be set per job. + Optional("extra-args"): str, + # Fail the task when differences are detected. + Optional("fail-on-diff"): bool, + # What artifact to check the differences of. Defaults to target.tar.xz + # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for + # Android. + Optional("artifact"): str, + # Whether to unpack first. Diffoscope can normally work without unpacking, + # but when one needs to --exclude some contents, that doesn't work out well + # if said content is packed (e.g. in omni.ja). + Optional("unpack"): bool, + # Commands to run before performing the diff. + Optional("pre-diff-commands"): [str], + # Only run the task on a set of projects/branches. + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], + Optional("optimization"): task_description_schema["optimization"], +}) transforms = TransformSequence() transforms.add_validate(diff_description_schema) @@ -162,11 +160,9 @@ def fill_template(config, tasks): taskdesc["run-on-projects"] = task["run-on-projects"] if artifact.endswith(".dmg"): - taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend( - [ - "linux64-cctools-port", - "linux64-libdmg", - ] - ) + taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend([ + "linux64-cctools-port", + "linux64-libdmg", + ]) yield taskdesc diff --git a/taskcluster/gecko_taskgraph/transforms/docker_image.py b/taskcluster/gecko_taskgraph/transforms/docker_image.py @@ -36,36 +36,34 @@ IMAGE_BUILDER_IMAGE = ( transforms = TransformSequence() -docker_image_schema = Schema( - { - # Name of the docker image. - Required("name"): str, - # Name of the parent docker image. - Optional("parent"): str, - # Treeherder symbol. - Required("symbol"): str, - # relative path (from config.path) to the file the docker image was defined - # in. - Optional("task-from"): str, - # Arguments to use for the Dockerfile. - Optional("args"): {str: str}, - # Name of the docker image definition under taskcluster/docker, when - # different from the docker image name. - Optional("definition"): str, - # List of package tasks this docker image depends on. - Optional("packages"): [str], - Optional("arch"): str, - Optional( - "index", - description="information for indexing this build so its artifacts can be discovered", - ): task_description_schema["index"], - Optional( - "cache", - description="Whether this image should be cached based on inputs.", - ): bool, - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +docker_image_schema = Schema({ + # Name of the docker image. + Required("name"): str, + # Name of the parent docker image. + Optional("parent"): str, + # Treeherder symbol. + Required("symbol"): str, + # relative path (from config.path) to the file the docker image was defined + # in. + Optional("task-from"): str, + # Arguments to use for the Dockerfile. + Optional("args"): {str: str}, + # Name of the docker image definition under taskcluster/docker, when + # different from the docker image name. + Optional("definition"): str, + # List of package tasks this docker image depends on. + Optional("packages"): [str], + Optional("arch"): str, + Optional( + "index", + description="information for indexing this build so its artifacts can be discovered", + ): task_description_schema["index"], + Optional( + "cache", + description="Whether this image should be cached based on inputs.", + ): bool, + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms.add_validate(docker_image_schema) diff --git a/taskcluster/gecko_taskgraph/transforms/fetch.py b/taskcluster/gecko_taskgraph/transforms/fetch.py @@ -25,34 +25,32 @@ from ..util.cached_tasks import add_optimization CACHE_TYPE = "content.v1" -FETCH_SCHEMA = Schema( - { - # Name of the task. - Required("name"): str, - # Relative path (from config.path) to the file the task was defined - # in. - Optional("task-from"): str, - # Description of the task. - Required("description"): str, - Optional( - "fetch-alias", - description="An alias that can be used instead of the real fetch job name in " - "fetch stanzas for jobs.", - ): str, - Optional( - "artifact-prefix", - description="The prefix of the taskcluster artifact being uploaded. " - "Defaults to `public/`; if it starts with something other than " - "`public/` the artifact will require scopes to access.", - ): str, - Optional("attributes"): {str: object}, - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - Required("fetch"): { - Required("type"): str, - Extra: object, - }, - } -) +FETCH_SCHEMA = Schema({ + # Name of the task. + Required("name"): str, + # Relative path (from config.path) to the file the task was defined + # in. + Optional("task-from"): str, + # Description of the task. + Required("description"): str, + Optional( + "fetch-alias", + description="An alias that can be used instead of the real fetch job name in " + "fetch stanzas for jobs.", + ): str, + Optional( + "artifact-prefix", + description="The prefix of the taskcluster artifact being uploaded. " + "Defaults to `public/`; if it starts with something other than " + "`public/` the artifact will require scopes to access.", + ): str, + Optional("attributes"): {str: object}, + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], + Required("fetch"): { + Required("type"): str, + Extra: object, + }, +}) # define a collection of payload builders, depending on the worker implementation @@ -265,24 +263,20 @@ def create_fetch_url_task(config, name, fetch): gpg_key = fh.read() env["FETCH_GPG_KEY"] = gpg_key - command.extend( - [ - "--gpg-sig-url", - sig_url, - "--gpg-key-env", - "FETCH_GPG_KEY", - ] - ) + command.extend([ + "--gpg-sig-url", + sig_url, + "--gpg-key-env", + "FETCH_GPG_KEY", + ]) for header in fetch.get("headers", []): command.extend(["--header", header]) - command.extend( - [ - fetch["url"], - "/builds/worker/artifacts/%s" % artifact_name, - ] - ) + command.extend([ + fetch["url"], + "/builds/worker/artifacts/%s" % artifact_name, + ]) return { "command": command, @@ -416,7 +410,7 @@ def create_chromium_fetch_task(config, name, fetch): cmd = [ "bash", "-c", - "cd {} && " "/usr/bin/python3 {} {}".format(workdir, fetch["script"], args), + "cd {} && /usr/bin/python3 {} {}".format(workdir, fetch["script"], args), ] return { @@ -470,7 +464,7 @@ def create_cft_canary_fetch_task(config, name, fetch): cmd = [ "bash", "-c", - "cd {} && " "/usr/bin/python3 {} {}".format(workdir, fetch["script"], args), + "cd {} && /usr/bin/python3 {} {}".format(workdir, fetch["script"], args), ] return { diff --git a/taskcluster/gecko_taskgraph/transforms/final_verify.py b/taskcluster/gecko_taskgraph/transforms/final_verify.py @@ -5,7 +5,6 @@ Transform the beetmover task into an actual task description. """ - from taskgraph.transforms.base import TransformSequence transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_mac_notarization.py @@ -13,19 +13,17 @@ from voluptuous import Optional from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job -geckodriver_notarization_description_schema = Schema( - { - Optional("label"): str, - Optional("treeherder"): task_description_schema["treeherder"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("worker"): task_description_schema["worker"], - Optional("worker-type"): task_description_schema["worker-type"], - Optional("task-from"): task_description_schema["task-from"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +geckodriver_notarization_description_schema = Schema({ + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("worker"): task_description_schema["worker"], + Optional("worker-type"): task_description_schema["worker-type"], + Optional("task-from"): task_description_schema["task-from"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py b/taskcluster/gecko_taskgraph/transforms/geckodriver_signing.py @@ -14,17 +14,15 @@ from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.scriptworker import get_signing_type_per_platform -repackage_signing_description_schema = Schema( - { - Optional("label"): str, - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("treeherder"): task_description_schema["treeherder"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +repackage_signing_description_schema = Schema({ + Optional("label"): str, + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() @@ -62,9 +60,9 @@ def make_signing_description(config, jobs): dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies - dependencies.update( - {k: v for k, v in signing_dependencies.items() if k != "docker-image"} - ) + dependencies.update({ + k: v for k, v in signing_dependencies.items() if k != "docker-image" + }) description = "Signing Geckodriver for build '{}'".format( attributes.get("build_platform"), diff --git a/taskcluster/gecko_taskgraph/transforms/hardened_signing.py b/taskcluster/gecko_taskgraph/transforms/hardened_signing.py @@ -122,12 +122,10 @@ def add_upstream_signing_resources(config, jobs): task_type = "build" if "notarization" in dep_job.kind: task_type = "scriptworker" - job["worker"].setdefault("upstream-artifacts", []).append( - { - "paths": sorted(upstream_files), - "taskId": {"task-reference": task_ref}, - "taskType": task_type, - "formats": [], # Not for signing - } - ) + job["worker"].setdefault("upstream-artifacts", []).append({ + "paths": sorted(upstream_files), + "taskId": {"task-reference": task_ref}, + "taskType": task_type, + "formats": [], # Not for signing + }) yield job diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py @@ -30,87 +30,83 @@ from gecko_taskgraph.util.workertypes import worker_type_implementation logger = logging.getLogger(__name__) # Schema for a build description -job_description_schema = Schema( - { - # The name of the job and the job's label. At least one must be specified, - # and the label will be generated from the name if necessary, by prepending - # the kind. - Optional("name"): str, - Optional("label"): str, - # the following fields are passed directly through to the task description, - # possibly modified by the run implementation. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details. - Required("description"): task_description_schema["description"], - Optional("attributes"): task_description_schema["attributes"], - Optional("task-from"): task_description_schema["task-from"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("if-dependencies"): task_description_schema["if-dependencies"], - Optional("soft-dependencies"): task_description_schema["soft-dependencies"], - Optional("if-dependencies"): task_description_schema["if-dependencies"], - Optional("requires"): task_description_schema["requires"], - Optional("expires-after"): task_description_schema["expires-after"], - Optional("expiration-policy"): task_description_schema["expiration-policy"], - Optional("routes"): task_description_schema["routes"], - Optional("scopes"): task_description_schema["scopes"], - Optional("tags"): task_description_schema["tags"], - Optional("extra"): task_description_schema["extra"], - Optional("treeherder"): task_description_schema["treeherder"], - Optional("index"): task_description_schema["index"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - Optional("run-on-projects"): task_description_schema["run-on-projects"], - Optional("run-on-git-branches"): task_description_schema["run-on-git-branches"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("always-target"): task_description_schema["always-target"], - Exclusive("optimization", "optimization"): task_description_schema[ - "optimization" - ], - Optional("use-sccache"): task_description_schema["use-sccache"], - Optional("use-python"): Any("system", "default", Coerce(Version)), - # Fetch uv binary and add it to PATH - Optional("use-uv"): bool, - Optional("priority"): task_description_schema["priority"], - # The "when" section contains descriptions of the circumstances under which - # this task should be included in the task graph. This will be converted - # into an optimization, so it cannot be specified in a job description that - # also gives 'optimization'. - Exclusive("when", "optimization"): Any( - None, +job_description_schema = Schema({ + # The name of the job and the job's label. At least one must be specified, + # and the label will be generated from the name if necessary, by prepending + # the kind. + Optional("name"): str, + Optional("label"): str, + # the following fields are passed directly through to the task description, + # possibly modified by the run implementation. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details. + Required("description"): task_description_schema["description"], + Optional("attributes"): task_description_schema["attributes"], + Optional("task-from"): task_description_schema["task-from"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("if-dependencies"): task_description_schema["if-dependencies"], + Optional("soft-dependencies"): task_description_schema["soft-dependencies"], + Optional("if-dependencies"): task_description_schema["if-dependencies"], + Optional("requires"): task_description_schema["requires"], + Optional("expires-after"): task_description_schema["expires-after"], + Optional("expiration-policy"): task_description_schema["expiration-policy"], + Optional("routes"): task_description_schema["routes"], + Optional("scopes"): task_description_schema["scopes"], + Optional("tags"): task_description_schema["tags"], + Optional("extra"): task_description_schema["extra"], + Optional("treeherder"): task_description_schema["treeherder"], + Optional("index"): task_description_schema["index"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], + Optional("run-on-projects"): task_description_schema["run-on-projects"], + Optional("run-on-git-branches"): task_description_schema["run-on-git-branches"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("always-target"): task_description_schema["always-target"], + Exclusive("optimization", "optimization"): task_description_schema["optimization"], + Optional("use-sccache"): task_description_schema["use-sccache"], + Optional("use-python"): Any("system", "default", Coerce(Version)), + # Fetch uv binary and add it to PATH + Optional("use-uv"): bool, + Optional("priority"): task_description_schema["priority"], + # The "when" section contains descriptions of the circumstances under which + # this task should be included in the task graph. This will be converted + # into an optimization, so it cannot be specified in a job description that + # also gives 'optimization'. + Exclusive("when", "optimization"): Any( + None, + { + # This task only needs to be run if a file matching one of the given + # patterns has changed in the push. The patterns use the mozpack + # match function (python/mozbuild/mozpack/path.py). + Optional("files-changed"): [str], + }, + ), + # A list of artifacts to install from 'fetch' tasks. + Optional("fetches"): { + str: [ + str, { - # This task only needs to be run if a file matching one of the given - # patterns has changed in the push. The patterns use the mozpack - # match function (python/mozbuild/mozpack/path.py). - Optional("files-changed"): [str], + Required("artifact"): str, + Optional("dest"): str, + Optional("extract"): bool, + Optional("verify-hash"): bool, }, - ), - # A list of artifacts to install from 'fetch' tasks. - Optional("fetches"): { - str: [ - str, - { - Required("artifact"): str, - Optional("dest"): str, - Optional("extract"): bool, - Optional("verify-hash"): bool, - }, - ], - }, - # A description of how to run this job. - "run": { - # The key to a job implementation in a peer module to this one - "using": str, - # Base work directory used to set up the task. - Optional("workdir"): str, - # Any remaining content is verified against that job implementation's - # own schema. - Extra: object, - }, - Required("worker-type"): task_description_schema["worker-type"], - # This object will be passed through to the task description, with additions - # provided by the job's run-using function - Optional("worker"): dict, - } -) + ], + }, + # A description of how to run this job. + "run": { + # The key to a job implementation in a peer module to this one + "using": str, + # Base work directory used to set up the task. + Optional("workdir"): str, + # Any remaining content is verified against that job implementation's + # own schema. + Extra: object, + }, + Required("worker-type"): task_description_schema["worker-type"], + # This object will be passed through to the task description, with additions + # provided by the job's run-using function + Optional("worker"): dict, +}) transforms = TransformSequence() transforms.add_validate(job_description_schema) @@ -265,13 +261,11 @@ def add_perfherder_fetch_content_artifact(config, jobs): if worker.get("implementation") == "docker-worker" else "./perf/perfherder-data-fetch-content.json" ) - artifacts.append( - { - "type": "file", - "name": "public/fetch/perfherder-data-fetch-content.json", - "path": perfherder_fetch_content_json_path, - } - ) + artifacts.append({ + "type": "file", + "name": "public/fetch/perfherder-data-fetch-content.json", + "path": perfherder_fetch_content_json_path, + }) env["PERFHERDER_FETCH_CONTENT_JSON_PATH"] = perfherder_fetch_content_json_path yield job @@ -345,13 +339,11 @@ def use_fetches(config, jobs): path = artifact_names[label] dependencies[label] = label - job_fetches.append( - { - "artifact": path, - "task": f"<{label}>", - "extract": should_extract.get(label, True), - } - ) + job_fetches.append({ + "artifact": path, + "task": f"<{label}>", + "extract": should_extract.get(label, True), + }) if kind == "toolchain" and fetch_name.endswith("-sccache"): has_sccache = True diff --git a/taskcluster/gecko_taskgraph/transforms/job/common.py b/taskcluster/gecko_taskgraph/transforms/job/common.py @@ -16,13 +16,11 @@ SECRET_SCOPE = "secrets:get:project/releng/{trust_domain}/{kind}/level-{level}/{ def add_artifacts(config, job, taskdesc, path): - taskdesc["worker"].setdefault("artifacts", []).append( - { - "name": get_artifact_prefix(taskdesc), - "path": path, - "type": "directory", - } - ) + taskdesc["worker"].setdefault("artifacts", []).append({ + "name": get_artifact_prefix(taskdesc), + "path": path, + "type": "directory", + }) def docker_worker_add_artifacts(config, job, taskdesc): @@ -103,42 +101,36 @@ def support_vcs_checkout(config, job, taskdesc, repo_configs): CACHES["checkout"]["cache_name"] = get_cache_name env = taskdesc["worker"].setdefault("env", {}) - env.update( - { - "HG_STORE_PATH": hgstore, - "REPOSITORIES": json.dumps( - {repo.prefix: repo.name for repo in repo_configs.values()} - ), - } - ) + env.update({ + "HG_STORE_PATH": hgstore, + "REPOSITORIES": json.dumps({ + repo.prefix: repo.name for repo in repo_configs.values() + }), + }) for repo_config in repo_configs.values(): - env.update( - { - f"{repo_config.prefix.upper()}_{key}": value - for key, value in { - "BASE_REPOSITORY": repo_config.base_repository, - "HEAD_REPOSITORY": repo_config.head_repository, - "HEAD_REV": repo_config.head_rev, - "HEAD_REF": repo_config.head_ref, - "REPOSITORY_TYPE": repo_config.type, - "SSH_SECRET_NAME": repo_config.ssh_secret_name, - }.items() - if value is not None - } - ) + env.update({ + f"{repo_config.prefix.upper()}_{key}": value + for key, value in { + "BASE_REPOSITORY": repo_config.base_repository, + "HEAD_REPOSITORY": repo_config.head_repository, + "HEAD_REV": repo_config.head_rev, + "HEAD_REF": repo_config.head_ref, + "REPOSITORY_TYPE": repo_config.type, + "SSH_SECRET_NAME": repo_config.ssh_secret_name, + }.items() + if value is not None + }) if repo_config.ssh_secret_name: taskdesc["scopes"].append(f"secrets:get:{repo_config.ssh_secret_name}") gecko_path = env.setdefault("GECKO_PATH", geckodir) if "comm_base_repository" in config.params: - taskdesc["worker"]["env"].update( - { - "COMM_BASE_REPOSITORY": config.params["comm_base_repository"], - "COMM_HEAD_REPOSITORY": config.params["comm_head_repository"], - "COMM_HEAD_REV": config.params["comm_head_rev"], - } - ) + taskdesc["worker"]["env"].update({ + "COMM_BASE_REPOSITORY": config.params["comm_base_repository"], + "COMM_HEAD_REPOSITORY": config.params["comm_head_repository"], + "COMM_HEAD_REV": config.params["comm_head_rev"], + }) elif job["run"].get("comm-checkout", False): raise Exception( "Can't checkout from comm-* repository if not given a repository." @@ -198,27 +190,21 @@ def add_tooltool(config, job, taskdesc, internal=False): "{workdir}/tooltool-cache".format(**job["run"]), ) - taskdesc["worker"].setdefault("env", {}).update( - { - "TOOLTOOL_CACHE": "{workdir}/tooltool-cache".format(**job["run"]), - } - ) + taskdesc["worker"].setdefault("env", {}).update({ + "TOOLTOOL_CACHE": "{workdir}/tooltool-cache".format(**job["run"]), + }) elif not internal: return taskdesc["worker"]["taskcluster-proxy"] = True - taskdesc["scopes"].extend( - [ - "project:releng:services/tooltool/api/download/public", - ] - ) + taskdesc["scopes"].extend([ + "project:releng:services/tooltool/api/download/public", + ]) if internal: - taskdesc["scopes"].extend( - [ - "project:releng:services/tooltool/api/download/internal", - ] - ) + taskdesc["scopes"].extend([ + "project:releng:services/tooltool/api/download/internal", + ]) def get_expiration(config, policy="default"): diff --git a/taskcluster/gecko_taskgraph/transforms/job/distro_package.py b/taskcluster/gecko_taskgraph/transforms/job/distro_package.py @@ -5,7 +5,6 @@ Support for running spidermonkey jobs via dedicated scripts """ - import os import re @@ -26,48 +25,42 @@ source_definition = { Required("sha256"): str, } -common_schema = Schema( - { - # URL/SHA256 of a source file to build, which can either be a source - # control (.dsc), or a tarball. - Required(Any("dsc", "tarball")): source_definition, - # Package name. Normally derived from the source control or tarball file - # name. Use in case the name doesn't match DSC_PACKAGE_RE or - # SOURCE_PACKAGE_RE. - Optional("name"): str, - # Patch to apply to the extracted source. - Optional("patch"): str, - # Command to run before dpkg-buildpackage. - Optional("pre-build-command"): str, - # Architecture to build the package for. - Optional("arch"): str, - # List of package tasks to get build dependencies from. - Optional("packages"): [str], - # What resolver to use to install build dependencies. The default - # (apt-get) is good in most cases, but in subtle cases involving - # a *-backports archive, its solver might not be able to find a - # solution that satisfies the build dependencies. - Optional("resolver"): Any("apt-get", "aptitude"), - # Base work directory used to set up the task. - Required("workdir"): str, - } -) - -debian_schema = common_schema.extend( - { - Required("using"): "debian-package", - # Debian distribution - Required("dist"): str, - } -) - -ubuntu_schema = common_schema.extend( - { - Required("using"): "ubuntu-package", - # Ubuntu distribution - Required("dist"): str, - } -) +common_schema = Schema({ + # URL/SHA256 of a source file to build, which can either be a source + # control (.dsc), or a tarball. + Required(Any("dsc", "tarball")): source_definition, + # Package name. Normally derived from the source control or tarball file + # name. Use in case the name doesn't match DSC_PACKAGE_RE or + # SOURCE_PACKAGE_RE. + Optional("name"): str, + # Patch to apply to the extracted source. + Optional("patch"): str, + # Command to run before dpkg-buildpackage. + Optional("pre-build-command"): str, + # Architecture to build the package for. + Optional("arch"): str, + # List of package tasks to get build dependencies from. + Optional("packages"): [str], + # What resolver to use to install build dependencies. The default + # (apt-get) is good in most cases, but in subtle cases involving + # a *-backports archive, its solver might not be able to find a + # solution that satisfies the build dependencies. + Optional("resolver"): Any("apt-get", "aptitude"), + # Base work directory used to set up the task. + Required("workdir"): str, +}) + +debian_schema = common_schema.extend({ + Required("using"): "debian-package", + # Debian distribution + Required("dist"): str, +}) + +ubuntu_schema = common_schema.extend({ + Required("using"): "ubuntu-package", + # Ubuntu distribution + Required("dist"): str, +}) def common_package(config, job, taskdesc, distro, version): @@ -99,8 +92,7 @@ def common_package(config, job, taskdesc, distro, version): elif "tarball" in run: src = run["tarball"] unpack = ( - "mkdir {package} && " - "tar -C {package} -axf {src_file} --strip-components=1" + "mkdir {package} && tar -C {package} -axf {src_file} --strip-components=1" ) package_re = SOURCE_PACKAGE_RE else: @@ -158,7 +150,8 @@ def common_package(config, job, taskdesc, distro, version): "/usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $PACKAGES && " "apt-get update && " # Upgrade packages that might have new versions in package tasks. - "apt-get dist-upgrade && " "cd /tmp && " + "apt-get dist-upgrade && " + "cd /tmp && " # Get, validate and extract the package source. "(dget -d -u {src_url} || exit 100) && " 'echo "{src_sha256} {src_file}" | sha256sum -c && ' diff --git a/taskcluster/gecko_taskgraph/transforms/job/hazard.py b/taskcluster/gecko_taskgraph/transforms/job/hazard.py @@ -5,7 +5,6 @@ Support for running hazard jobs via dedicated scripts """ - from taskgraph.util.schema import Schema from voluptuous import Any, Optional, Required @@ -16,23 +15,21 @@ from gecko_taskgraph.transforms.job.common import ( setup_secrets, ) -haz_run_schema = Schema( - { - Required("using"): "hazard", - # The command to run within the task image (passed through to the worker) - Required("command"): str, - # The mozconfig to use; default in the script is used if omitted - Optional("mozconfig"): str, - # The set of secret names to which the task has access; these are prefixed - # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting - # this will enable any worker features required and set the task's scopes - # appropriately. `true` here means ['*'], all secrets. Not supported on - # Windows - Optional("secrets"): Any(bool, [str]), - # Base work directory used to set up the task. - Optional("workdir"): str, - } -) +haz_run_schema = Schema({ + Required("using"): "hazard", + # The command to run within the task image (passed through to the worker) + Required("command"): str, + # The mozconfig to use; default in the script is used if omitted + Optional("mozconfig"): str, + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Optional("secrets"): Any(bool, [str]), + # Base work directory used to set up the task. + Optional("workdir"): str, +}) @run_job_using("docker-worker", "hazard", schema=haz_run_schema) @@ -50,12 +47,10 @@ def docker_worker_hazard(config, job, taskdesc): setup_secrets(config, job, taskdesc) env = worker["env"] - env.update( - { - "MOZ_BUILD_DATE": config.params["moz_build_date"], - "MOZ_SCM_LEVEL": config.params["level"], - } - ) + env.update({ + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + }) # script parameters if run.get("mozconfig"): diff --git a/taskcluster/gecko_taskgraph/transforms/job/mach.py b/taskcluster/gecko_taskgraph/transforms/job/mach.py @@ -10,26 +10,24 @@ from voluptuous import Any, Optional, Required from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using -mach_schema = Schema( - { - Required("using"): "mach", - # The mach command (omitting `./mach`) to run - Required("mach"): taskref_or_string, - # The sparse checkout profile to use. Value is the filename relative to the - # directory where sparse profiles are defined (build/sparse-profiles/). - Optional("sparse-profile"): Any(str, None), - # if true, perform a checkout of a comm-central based branch inside the - # gecko checkout - Required("comm-checkout"): bool, - # Prepend the specified ENV variables to the command. This can be useful - # if the value of the ENV needs to be interpolated with another ENV. - Optional("prepend-env"): {str: str}, - # Base work directory used to set up the task. - Optional("workdir"): str, - # Use the specified caches. - Optional("use-caches"): Any(bool, [str]), - } -) +mach_schema = Schema({ + Required("using"): "mach", + # The mach command (omitting `./mach`) to run + Required("mach"): taskref_or_string, + # The sparse checkout profile to use. Value is the filename relative to the + # directory where sparse profiles are defined (build/sparse-profiles/). + Optional("sparse-profile"): Any(str, None), + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Required("comm-checkout"): bool, + # Prepend the specified ENV variables to the command. This can be useful + # if the value of the ENV needs to be interpolated with another ENV. + Optional("prepend-env"): {str: str}, + # Base work directory used to set up the task. + Optional("workdir"): str, + # Use the specified caches. + Optional("use-caches"): Any(bool, [str]), +}) defaults = { diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py @@ -25,72 +25,70 @@ from gecko_taskgraph.transforms.job.common import ( ) from gecko_taskgraph.util.attributes import is_try -mozharness_run_schema = Schema( - { - Required("using"): "mozharness", - # the mozharness script used to run this task, relative to the testing/ - # directory and using forward slashes even on Windows - Required("script"): str, - # Additional paths to look for mozharness configs in. These should be - # relative to the base of the source checkout - Optional("config-paths"): [str], - # the config files required for the task, relative to - # testing/mozharness/configs or one of the paths specified in - # `config-paths` and using forward slashes even on Windows - Required("config"): [str], - # any additional actions to pass to the mozharness command - Optional("actions"): [ - Match("^[a-z0-9-]+$", "actions must be `-` seperated alphanumeric strings") - ], - # any additional options (without leading --) to be passed to mozharness - Optional("options"): [ - Match( - "^[a-z0-9-]+(=[^ ]+)?$", - "options must be `-` seperated alphanumeric strings (with optional argument)", - ) - ], - # --custom-build-variant-cfg value - Optional("custom-build-variant-cfg"): str, - # Extra configuration options to pass to mozharness. - Optional("extra-config"): dict, - # If not false, tooltool downloads will be enabled via relengAPIProxy - # for either just public files, or all files. Not supported on Windows - Required("tooltool-downloads"): Any( - False, - "public", - "internal", - ), - # The set of secret names to which the task has access; these are prefixed - # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting - # this will enable any worker features required and set the task's scopes - # appropriately. `true` here means ['*'], all secrets. Not supported on - # Windows - Required("secrets"): Any(bool, [str]), - # If true, taskcluster proxy will be enabled; note that it may also be enabled - # automatically e.g., for secrets support. Not supported on Windows. - Required("taskcluster-proxy"): bool, - # If false, indicate that builds should skip producing artifacts. Not - # supported on Windows. - Required("keep-artifacts"): bool, - # If specified, use the in-tree job script specified. - Optional("job-script"): str, - Required("requires-signed-builds"): bool, - # Whether or not to use caches. - Optional("use-caches"): Any(bool, [str]), - # If false, don't set MOZ_SIMPLE_PACKAGE_NAME - # Only disableable on windows - Required("use-simple-package"): bool, - # If false don't pass --branch mozharness script - # Only disableable on windows - Required("use-magic-mh-args"): bool, - # if true, perform a checkout of a comm-central based branch inside the - # gecko checkout - Required("comm-checkout"): bool, - # Base work directory used to set up the task. - Optional("workdir"): str, - Optional("run-as-root"): bool, - } -) +mozharness_run_schema = Schema({ + Required("using"): "mozharness", + # the mozharness script used to run this task, relative to the testing/ + # directory and using forward slashes even on Windows + Required("script"): str, + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # the config files required for the task, relative to + # testing/mozharness/configs or one of the paths specified in + # `config-paths` and using forward slashes even on Windows + Required("config"): [str], + # any additional actions to pass to the mozharness command + Optional("actions"): [ + Match("^[a-z0-9-]+$", "actions must be `-` seperated alphanumeric strings") + ], + # any additional options (without leading --) to be passed to mozharness + Optional("options"): [ + Match( + "^[a-z0-9-]+(=[^ ]+)?$", + "options must be `-` seperated alphanumeric strings (with optional argument)", + ) + ], + # --custom-build-variant-cfg value + Optional("custom-build-variant-cfg"): str, + # Extra configuration options to pass to mozharness. + Optional("extra-config"): dict, + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Not supported on Windows + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Required("secrets"): Any(bool, [str]), + # If true, taskcluster proxy will be enabled; note that it may also be enabled + # automatically e.g., for secrets support. Not supported on Windows. + Required("taskcluster-proxy"): bool, + # If false, indicate that builds should skip producing artifacts. Not + # supported on Windows. + Required("keep-artifacts"): bool, + # If specified, use the in-tree job script specified. + Optional("job-script"): str, + Required("requires-signed-builds"): bool, + # Whether or not to use caches. + Optional("use-caches"): Any(bool, [str]), + # If false, don't set MOZ_SIMPLE_PACKAGE_NAME + # Only disableable on windows + Required("use-simple-package"): bool, + # If false don't pass --branch mozharness script + # Only disableable on windows + Required("use-magic-mh-args"): bool, + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Required("comm-checkout"): bool, + # Base work directory used to set up the task. + Optional("workdir"): str, + Optional("run-as-root"): bool, +}) mozharness_defaults = { @@ -133,27 +131,23 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc): # android-build). worker.setdefault("docker-image", {"in-tree": "debian12-amd64-build"}) - worker.setdefault("artifacts", []).append( - { - "name": "public/logs", - "path": "{workdir}/logs/".format(**run), - "type": "directory", - "expires-after": get_expiration(config, "medium"), - } - ) + worker.setdefault("artifacts", []).append({ + "name": "public/logs", + "path": "{workdir}/logs/".format(**run), + "type": "directory", + "expires-after": get_expiration(config, "medium"), + }) worker["taskcluster-proxy"] = run.pop("taskcluster-proxy", None) docker_worker_add_artifacts(config, job, taskdesc) env = worker.setdefault("env", {}) - env.update( - { - "WORKSPACE": "{workdir}/workspace".format(**run), - "MOZHARNESS_CONFIG": " ".join(run.pop("config")), - "MOZHARNESS_SCRIPT": run.pop("script"), - "MH_BRANCH": config.params["project"], - "PYTHONUNBUFFERED": "1", - } - ) + env.update({ + "WORKSPACE": "{workdir}/workspace".format(**run), + "MOZHARNESS_CONFIG": " ".join(run.pop("config")), + "MOZHARNESS_SCRIPT": run.pop("script"), + "MH_BRANCH": config.params["project"], + "PYTHONUNBUFFERED": "1", + }) worker.setdefault("required-volumes", []).append(env["WORKSPACE"]) @@ -233,24 +227,20 @@ def mozharness_on_generic_worker(config, job, taskdesc): setup_secrets(config, job, taskdesc) - taskdesc["worker"].setdefault("artifacts", []).append( - { - "name": "public/logs", - "path": "logs", - "type": "directory", - "expires-after": get_expiration(config, "medium"), - } - ) + taskdesc["worker"].setdefault("artifacts", []).append({ + "name": "public/logs", + "path": "logs", + "type": "directory", + "expires-after": get_expiration(config, "medium"), + }) if not worker.get("skip-artifacts", False): generic_worker_add_artifacts(config, job, taskdesc) env = worker.setdefault("env", {}) - env.update( - { - "MH_BRANCH": config.params["project"], - } - ) + env.update({ + "MH_BRANCH": config.params["project"], + }) if run.pop("use-simple-package"): env.update({"MOZ_SIMPLE_PACKAGE_NAME": "target"}) @@ -336,21 +326,18 @@ def mozharness_on_generic_worker(config, job, taskdesc): return if taskdesc.get("use-sccache"): - worker["command"] = ( - [ - # Make the comment part of the first command, as it will help users to - # understand what is going on, and why these steps are implemented. - dedent( - """\ + worker["command"] = [ + # Make the comment part of the first command, as it will help users to + # understand what is going on, and why these steps are implemented. + dedent( + """\ :: sccache currently uses the full compiler commandline as input to the :: cache hash key, so create a symlink to the task dir and build from :: the symlink dir to get consistent paths. if exist z:\\build rmdir z:\\build""" - ), - r"mklink /d z:\build %cd%", - # Grant delete permission on the link to everyone. - r"icacls z:\build /grant *S-1-1-0:D /L", - r"cd /d z:\build", - ] - + worker["command"] - ) + ), + r"mklink /d z:\build %cd%", + # Grant delete permission on the link to everyone. + r"icacls z:\build /grant *S-1-1-0:D /L", + r"cd /d z:\build", + ] + worker["command"] diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py @@ -44,23 +44,21 @@ def get_variant(test_platform): return "" -mozharness_test_run_schema = Schema( - { - Required("using"): "mozharness-test", - Required("test"): { - Required("test-platform"): str, - Required("mozharness"): test_description_schema["mozharness"], - Required("docker-image"): test_description_schema["docker-image"], - Required("loopback-video"): test_description_schema["loopback-video"], - Required("loopback-audio"): test_description_schema["loopback-audio"], - Required("max-run-time"): test_description_schema["max-run-time"], - Optional("retry-exit-status"): test_description_schema["retry-exit-status"], - Extra: object, - }, - # Base work directory used to set up the task. - Optional("workdir"): str, - } -) +mozharness_test_run_schema = Schema({ + Required("using"): "mozharness-test", + Required("test"): { + Required("test-platform"): str, + Required("mozharness"): test_description_schema["mozharness"], + Required("docker-image"): test_description_schema["docker-image"], + Required("loopback-video"): test_description_schema["loopback-video"], + Required("loopback-audio"): test_description_schema["loopback-audio"], + Required("max-run-time"): test_description_schema["max-run-time"], + Optional("retry-exit-status"): test_description_schema["retry-exit-status"], + Extra: object, + }, + # Base work directory used to set up the task. + Optional("workdir"): str, +}) def test_packages_url(taskdesc): @@ -120,38 +118,32 @@ def mozharness_test_on_docker(config, job, taskdesc): installer = installer_url(taskdesc) worker.setdefault("artifacts", []) - worker["artifacts"].extend( - [ - { - "name": prefix, - "path": os.path.join("{workdir}/workspace".format(**run), path), - "type": "directory", - "expires-after": get_expiration(config, "default"), - } - for (prefix, path) in artifacts - ] - ) - worker["artifacts"].append( + worker["artifacts"].extend([ { - "name": "public/xsession-errors.log", - "path": "{workdir}/.xsession-errors".format(**run), - "type": "file", + "name": prefix, + "path": os.path.join("{workdir}/workspace".format(**run), path), + "type": "directory", "expires-after": get_expiration(config, "default"), } - ) + for (prefix, path) in artifacts + ]) + worker["artifacts"].append({ + "name": "public/xsession-errors.log", + "path": "{workdir}/.xsession-errors".format(**run), + "type": "file", + "expires-after": get_expiration(config, "default"), + }) docker_worker_add_artifacts(config, job, taskdesc) env = worker.setdefault("env", {}) - env.update( - { - "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), - "MOZHARNESS_SCRIPT": mozharness["script"], - "MOZILLA_BUILD_URL": {"artifact-reference": installer}, - "NEED_WINDOW_MANAGER": "true", - "ENABLE_E10S": str(bool(test.get("e10s"))).lower(), - "WORKING_DIR": "/builds/worker", - } - ) + env.update({ + "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), + "MOZHARNESS_SCRIPT": mozharness["script"], + "MOZILLA_BUILD_URL": {"artifact-reference": installer}, + "NEED_WINDOW_MANAGER": "true", + "ENABLE_E10S": str(bool(test.get("e10s"))).lower(), + "WORKING_DIR": "/builds/worker", + }) env["PYTHON"] = "python3" @@ -298,14 +290,12 @@ def mozharness_test_on_generic_worker(config, job, taskdesc): # jittest doesn't have blob_upload_dir if test["test-name"] != "jittest": - artifacts.append( - { - "name": "public/test_info", - "path": "build/blobber_upload_dir", - "type": "directory", - "expires-after": get_expiration(config, "default"), - } - ) + artifacts.append({ + "name": "public/test_info", + "path": "build/blobber_upload_dir", + "type": "directory", + "expires-after": get_expiration(config, "default"), + }) if is_bitbar or is_lambda: artifacts = [ @@ -365,33 +355,29 @@ def mozharness_test_on_generic_worker(config, job, taskdesc): # this list will get cleaned up / reduced / removed in bug 1354088 if is_macosx: - env.update( - { - "LC_ALL": "en_US.UTF-8", - "LANG": "en_US.UTF-8", - "MOZ_NODE_PATH": "/usr/local/bin/node", - "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", - "SHELL": "/bin/bash", - } - ) + env.update({ + "LC_ALL": "en_US.UTF-8", + "LANG": "en_US.UTF-8", + "MOZ_NODE_PATH": "/usr/local/bin/node", + "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", + "SHELL": "/bin/bash", + }) elif is_bitbar or is_lambda: - env.update( - { - "LANG": "en_US.UTF-8", - "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), - "MOZHARNESS_SCRIPT": mozharness["script"], - "MOZHARNESS_URL": { - "artifact-reference": "<build/public/build/mozharness.zip>" - }, - "MOZILLA_BUILD_URL": {"artifact-reference": installer}, - "NEED_XVFB": "false", - "XPCOM_DEBUG_BREAK": "warn", - "NO_FAIL_ON_TEST_ERRORS": "1", - "MOZ_HIDE_RESULTS_TABLE": "1", - "MOZ_NODE_PATH": "/usr/local/bin/node", - "TASKCLUSTER_WORKER_TYPE": job["worker-type"], - } - ) + env.update({ + "LANG": "en_US.UTF-8", + "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), + "MOZHARNESS_SCRIPT": mozharness["script"], + "MOZHARNESS_URL": { + "artifact-reference": "<build/public/build/mozharness.zip>" + }, + "MOZILLA_BUILD_URL": {"artifact-reference": installer}, + "NEED_XVFB": "false", + "XPCOM_DEBUG_BREAK": "warn", + "NO_FAIL_ON_TEST_ERRORS": "1", + "MOZ_HIDE_RESULTS_TABLE": "1", + "MOZ_NODE_PATH": "/usr/local/bin/node", + "TASKCLUSTER_WORKER_TYPE": job["worker-type"], + }) extra_config = { "installer_url": installer, diff --git a/taskcluster/gecko_taskgraph/transforms/job/python_test.py b/taskcluster/gecko_taskgraph/transforms/job/python_test.py @@ -5,26 +5,23 @@ Support for running mach python-test tasks (via run-task) """ - from taskgraph.util.schema import Schema from voluptuous import Any, Optional, Required from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using -python_test_schema = Schema( - { - Required("using"): "python-test", - # The subsuite to run - Required("subsuite"): str, - # Base work directory used to set up the task. - Optional("workdir"): str, - # Use the specified caches. - Optional("use-caches"): Any(bool, [str]), - # Prepend the specified ENV variables to the command. This can be useful - # if the value of the ENV needs to be interpolated with another ENV. - Optional("prepend-env"): {str: str}, - } -) +python_test_schema = Schema({ + Required("using"): "python-test", + # The subsuite to run + Required("subsuite"): str, + # Base work directory used to set up the task. + Optional("workdir"): str, + # Use the specified caches. + Optional("use-caches"): Any(bool, [str]), + # Prepend the specified ENV variables to the command. This can be useful + # if the value of the ENV needs to be interpolated with another ENV. + Optional("prepend-env"): {str: str}, +}) defaults = { diff --git a/taskcluster/gecko_taskgraph/transforms/job/run_task.py b/taskcluster/gecko_taskgraph/transforms/job/run_task.py @@ -19,46 +19,44 @@ from gecko_taskgraph.transforms.job import run_job_using from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout from gecko_taskgraph.transforms.task import taskref_or_string -run_task_schema = Schema( - { - Required("using"): "run-task", - # Use the specified caches. - Optional("use-caches"): Any(bool, [str]), - # if true (the default), perform a checkout of gecko on the worker - Required("checkout"): bool, - Optional( - "cwd", - description="Path to run command in. If a checkout is present, the path " - "to the checkout will be interpolated with the key `checkout`", - ): str, - # The sparse checkout profile to use. Value is the filename relative to - # "sparse-profile-prefix" which defaults to "build/sparse-profiles/". - Required("sparse-profile"): Any(str, None), - # The relative path to the sparse profile. - Optional("sparse-profile-prefix"): str, - # Whether to use a shallow clone or not, default True (git only). - Optional("shallow-clone"): bool, - # if true, perform a checkout of a comm-central based branch inside the - # gecko checkout - Required("comm-checkout"): bool, - # The command arguments to pass to the `run-task` script, after the - # checkout arguments. If a list, it will be passed directly; otherwise - # it will be included in a single argument to `bash -cx`. - Required("command"): Any([taskref_or_string], taskref_or_string), - # Base work directory used to set up the task. - Optional("workdir"): str, - # If not false, tooltool downloads will be enabled via relengAPIProxy - # for either just public files, or all files. Only supported on - # docker-worker. - Required("tooltool-downloads"): Any( - False, - "public", - "internal", - ), - # Whether to run as root. (defaults to False) - Optional("run-as-root"): bool, - } -) +run_task_schema = Schema({ + Required("using"): "run-task", + # Use the specified caches. + Optional("use-caches"): Any(bool, [str]), + # if true (the default), perform a checkout of gecko on the worker + Required("checkout"): bool, + Optional( + "cwd", + description="Path to run command in. If a checkout is present, the path " + "to the checkout will be interpolated with the key `checkout`", + ): str, + # The sparse checkout profile to use. Value is the filename relative to + # "sparse-profile-prefix" which defaults to "build/sparse-profiles/". + Required("sparse-profile"): Any(str, None), + # The relative path to the sparse profile. + Optional("sparse-profile-prefix"): str, + # Whether to use a shallow clone or not, default True (git only). + Optional("shallow-clone"): bool, + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Required("comm-checkout"): bool, + # The command arguments to pass to the `run-task` script, after the + # checkout arguments. If a list, it will be passed directly; otherwise + # it will be included in a single argument to `bash -cx`. + Required("command"): Any([taskref_or_string], taskref_or_string), + # Base work directory used to set up the task. + Optional("workdir"): str, + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Only supported on + # docker-worker. + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + # Whether to run as root. (defaults to False) + Optional("run-as-root"): bool, +}) def common_setup(config, job, taskdesc, command): @@ -184,37 +182,31 @@ def generic_worker_run_task(config, job, taskdesc): run_task_bin = ( "run-task-git" if config.params["repository_type"] == "git" else "run-task-hg" ) - worker["mounts"].append( - { - "content": { - "url": script_url(config, run_task_bin), - }, - "file": "./run-task", - } - ) + worker["mounts"].append({ + "content": { + "url": script_url(config, run_task_bin), + }, + "file": "./run-task", + }) if ( job.get("fetches") or job.get("use-uv") or job.get("use-python", "system") != "system" ): - worker["mounts"].append( - { - "content": { - "url": script_url(config, "fetch-content"), - }, - "file": "./fetch-content", - } - ) + worker["mounts"].append({ + "content": { + "url": script_url(config, "fetch-content"), + }, + "file": "./fetch-content", + }) if run.get("checkout"): - worker["mounts"].append( - { - "content": { - "url": script_url(config, "robustcheckout.py"), - }, - "file": "./robustcheckout.py", - } - ) + worker["mounts"].append({ + "content": { + "url": script_url(config, "robustcheckout.py"), + }, + "file": "./robustcheckout.py", + }) run_command = run["command"] diff --git a/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py b/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py @@ -5,7 +5,6 @@ Support for running spidermonkey jobs via dedicated scripts """ - from taskgraph.util.schema import Schema from voluptuous import Any, Optional, Required @@ -15,24 +14,22 @@ from gecko_taskgraph.transforms.job.common import ( generic_worker_add_artifacts, ) -sm_run_schema = Schema( - { - Required("using"): Any( - "spidermonkey", - "spidermonkey-package", - ), - # SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM - Required("spidermonkey-variant"): str, - Optional("spidermonkey-platform"): str, - # Base work directory used to set up the task. - Optional("workdir"): str, - Required("tooltool-downloads"): Any( - False, - "public", - "internal", - ), - } -) +sm_run_schema = Schema({ + Required("using"): Any( + "spidermonkey", + "spidermonkey-package", + ), + # SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM + Required("spidermonkey-variant"): str, + Optional("spidermonkey-platform"): str, + # Base work directory used to set up the task. + Optional("workdir"): str, + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), +}) @run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema) @@ -46,14 +43,12 @@ def docker_worker_spidermonkey(config, job, taskdesc): docker_worker_add_artifacts(config, job, taskdesc) env = worker.setdefault("env", {}) - env.update( - { - "MOZHARNESS_DISABLE": "true", - "SPIDERMONKEY_VARIANT": run.pop("spidermonkey-variant"), - "MOZ_BUILD_DATE": config.params["moz_build_date"], - "MOZ_SCM_LEVEL": config.params["level"], - } - ) + env.update({ + "MOZHARNESS_DISABLE": "true", + "SPIDERMONKEY_VARIANT": run.pop("spidermonkey-variant"), + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + }) if "spidermonkey-platform" in run: env["SPIDERMONKEY_PLATFORM"] = run.pop("spidermonkey-platform") @@ -79,18 +74,16 @@ def generic_worker_spidermonkey(config, job, taskdesc): generic_worker_add_artifacts(config, job, taskdesc) env = worker.setdefault("env", {}) - env.update( - { - "MOZHARNESS_DISABLE": "true", - "SPIDERMONKEY_VARIANT": run.pop("spidermonkey-variant"), - "MOZ_BUILD_DATE": config.params["moz_build_date"], - "MOZ_SCM_LEVEL": config.params["level"], - "SCCACHE_DISABLE": "1", - "WORK": ".", # Override the defaults in build scripts - "GECKO_PATH": "./src", # with values suiteable for windows generic worker - "UPLOAD_DIR": "./public/build", - } - ) + env.update({ + "MOZHARNESS_DISABLE": "true", + "SPIDERMONKEY_VARIANT": run.pop("spidermonkey-variant"), + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "SCCACHE_DISABLE": "1", + "WORK": ".", # Override the defaults in build scripts + "GECKO_PATH": "./src", # with values suiteable for windows generic worker + "UPLOAD_DIR": "./public/build", + }) if "spidermonkey-platform" in run: env["SPIDERMONKEY_PLATFORM"] = run.pop("spidermonkey-platform") diff --git a/taskcluster/gecko_taskgraph/transforms/job/toolchain.py b/taskcluster/gecko_taskgraph/transforms/job/toolchain.py @@ -23,54 +23,52 @@ from gecko_taskgraph.util.hash import hash_paths CACHE_TYPE = "toolchains.v3" -toolchain_run_schema = Schema( - { - Required("using"): "toolchain-script", - # The script (in taskcluster/scripts/misc) to run. - # Python scripts are invoked with `mach python` so vendored libraries - # are available. - Required("script"): str, - # Arguments to pass to the script. - Optional("arguments"): [str], - # If not false, tooltool downloads will be enabled via relengAPIProxy - # for either just public files, or all files. Not supported on Windows - Required("tooltool-downloads"): Any( - False, - "public", - "internal", - ), - # Sparse profile to give to checkout using `run-task`. If given, - # Defaults to "toolchain-build". The value is relative to - # "sparse-profile-prefix", optionally defined below is the path, - # defaulting to "build/sparse-profiles". - # i.e. `build/sparse-profiles/toolchain-build`. - # If `None`, instructs `run-task` to not use a sparse profile at all. - Required("sparse-profile"): Any(str, None), - # The relative path to the sparse profile. - Optional("sparse-profile-prefix"): str, - # Paths/patterns pointing to files that influence the outcome of a - # toolchain build. - Optional("resources"): [str], - # Path to the artifact produced by the toolchain job - Required("toolchain-artifact"): str, - Optional( - "toolchain-alias", - description="An alias that can be used instead of the real toolchain job name in " - "fetch stanzas for jobs.", - ): optionally_keyed_by("project", Any(None, str, [str])), - Optional( - "toolchain-env", - description="Additional env variables to add to the worker when using this toolchain", - ): {str: object}, - Optional( - "toolchain-extract", - description="Whether the toolchain should be extracted after it is fetched " - + "(default: True)", - ): bool, - # Base work directory used to set up the task. - Optional("workdir"): str, - } -) +toolchain_run_schema = Schema({ + Required("using"): "toolchain-script", + # The script (in taskcluster/scripts/misc) to run. + # Python scripts are invoked with `mach python` so vendored libraries + # are available. + Required("script"): str, + # Arguments to pass to the script. + Optional("arguments"): [str], + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Not supported on Windows + Required("tooltool-downloads"): Any( + False, + "public", + "internal", + ), + # Sparse profile to give to checkout using `run-task`. If given, + # Defaults to "toolchain-build". The value is relative to + # "sparse-profile-prefix", optionally defined below is the path, + # defaulting to "build/sparse-profiles". + # i.e. `build/sparse-profiles/toolchain-build`. + # If `None`, instructs `run-task` to not use a sparse profile at all. + Required("sparse-profile"): Any(str, None), + # The relative path to the sparse profile. + Optional("sparse-profile-prefix"): str, + # Paths/patterns pointing to files that influence the outcome of a + # toolchain build. + Optional("resources"): [str], + # Path to the artifact produced by the toolchain job + Required("toolchain-artifact"): str, + Optional( + "toolchain-alias", + description="An alias that can be used instead of the real toolchain job name in " + "fetch stanzas for jobs.", + ): optionally_keyed_by("project", Any(None, str, [str])), + Optional( + "toolchain-env", + description="Additional env variables to add to the worker when using this toolchain", + ): {str: object}, + Optional( + "toolchain-extract", + description="Whether the toolchain should be extracted after it is fetched " + + "(default: True)", + ): bool, + # Base work directory used to set up the task. + Optional("workdir"): str, +}) def get_digest_data(config, run, taskdesc): @@ -151,13 +149,11 @@ def common_toolchain(config, job, taskdesc, is_docker): digest_data = get_digest_data(config, run, taskdesc) env = worker.setdefault("env", {}) - env.update( - { - "MOZ_BUILD_DATE": config.params["moz_build_date"], - "MOZ_SCM_LEVEL": config.params["level"], - "TOOLCHAIN_ARTIFACT": run.pop("toolchain-artifact"), - } - ) + env.update({ + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "TOOLCHAIN_ARTIFACT": run.pop("toolchain-artifact"), + }) if is_docker: # Toolchain checkouts don't live under {workdir}/checkouts diff --git a/taskcluster/gecko_taskgraph/transforms/l10n.py b/taskcluster/gecko_taskgraph/transforms/l10n.py @@ -33,94 +33,92 @@ def _by_platform(arg): return optionally_keyed_by("build-platform", arg) -l10n_description_schema = Schema( - { - # Name for this job, inferred from the dependent job before validation - Required("name"): str, - # build-platform, inferred from dependent job before validation - Required("build-platform"): str, - # max run time of the task - Required("run-time"): _by_platform(int), - # Locales not to repack for - Required("ignore-locales"): _by_platform([str]), - # All l10n jobs use mozharness - Required("mozharness"): { - # Script to invoke for mozharness - Required("script"): _by_platform(str), - # Config files passed to the mozharness script - Required("config"): _by_platform([str]), - # Additional paths to look for mozharness configs in. These should be - # relative to the base of the source checkout - Optional("config-paths"): [str], - # Options to pass to the mozharness script - Optional("options"): _by_platform([str]), - # Action commands to provide to mozharness script - Required("actions"): _by_platform([str]), - # if true, perform a checkout of a comm-central based branch inside the - # gecko checkout - Optional("comm-checkout"): bool, - }, - # Items for the taskcluster index - Optional("index"): { - # Product to identify as in the taskcluster index - Required("product"): _by_platform(str), - # Job name to identify as in the taskcluster index - Required("job-name"): _by_platform(str), - # Type of index - Optional("type"): _by_platform(str), - }, - # Description of the localized task - Required("description"): _by_platform(str), - Optional("run-on-projects"): job_description_schema["run-on-projects"], - Optional("run-on-repo-type"): job_description_schema["run-on-repo-type"], - # worker-type to utilize - Required("worker-type"): _by_platform(str), - # File which contains the used locales - Required("locales-file"): _by_platform(str), - # Tooltool visibility required for task. - Required("tooltool"): _by_platform(Any("internal", "public")), - # Docker image required for task. We accept only in-tree images - # -- generally desktop-build or android-build -- for now. - Optional("docker-image"): _by_platform( - # an in-tree generated docker image (from `taskcluster/docker/<name>`) - {"in-tree": str}, - ), - Optional("fetches"): { - str: _by_platform([str]), - }, - # The set of secret names to which the task has access; these are prefixed - # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting - # this will enable any worker features required and set the task's scopes - # appropriately. `true` here means ['*'], all secrets. Not supported on - # Windows - Optional("secrets"): _by_platform(Any(bool, [str])), - # Information for treeherder - Required("treeherder"): { - # Platform to display the task on in treeherder - Required("platform"): _by_platform(str), - # Symbol to use - Required("symbol"): str, - # Tier this task is - Required("tier"): _by_platform(int), - }, - # Extra environment values to pass to the worker - Optional("env"): _by_platform({str: taskref_or_string}), - # Max number locales per chunk - Optional("locales-per-chunk"): _by_platform(int), - # Task deps to chain this task with, added in transforms from primary dependency - # if this is a shippable-style build - Optional("dependencies"): {str: str}, - # Run the task when the listed files change (if present). - Optional("when"): {"files-changed": [str]}, - # passed through directly to the job description - Optional("attributes"): job_description_schema["attributes"], - Optional("extra"): job_description_schema["extra"], - # Shipping product and phase - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("task-from"): task_description_schema["task-from"], - } -) +l10n_description_schema = Schema({ + # Name for this job, inferred from the dependent job before validation + Required("name"): str, + # build-platform, inferred from dependent job before validation + Required("build-platform"): str, + # max run time of the task + Required("run-time"): _by_platform(int), + # Locales not to repack for + Required("ignore-locales"): _by_platform([str]), + # All l10n jobs use mozharness + Required("mozharness"): { + # Script to invoke for mozharness + Required("script"): _by_platform(str), + # Config files passed to the mozharness script + Required("config"): _by_platform([str]), + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # Options to pass to the mozharness script + Optional("options"): _by_platform([str]), + # Action commands to provide to mozharness script + Required("actions"): _by_platform([str]), + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Optional("comm-checkout"): bool, + }, + # Items for the taskcluster index + Optional("index"): { + # Product to identify as in the taskcluster index + Required("product"): _by_platform(str), + # Job name to identify as in the taskcluster index + Required("job-name"): _by_platform(str), + # Type of index + Optional("type"): _by_platform(str), + }, + # Description of the localized task + Required("description"): _by_platform(str), + Optional("run-on-projects"): job_description_schema["run-on-projects"], + Optional("run-on-repo-type"): job_description_schema["run-on-repo-type"], + # worker-type to utilize + Required("worker-type"): _by_platform(str), + # File which contains the used locales + Required("locales-file"): _by_platform(str), + # Tooltool visibility required for task. + Required("tooltool"): _by_platform(Any("internal", "public")), + # Docker image required for task. We accept only in-tree images + # -- generally desktop-build or android-build -- for now. + Optional("docker-image"): _by_platform( + # an in-tree generated docker image (from `taskcluster/docker/<name>`) + {"in-tree": str}, + ), + Optional("fetches"): { + str: _by_platform([str]), + }, + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Optional("secrets"): _by_platform(Any(bool, [str])), + # Information for treeherder + Required("treeherder"): { + # Platform to display the task on in treeherder + Required("platform"): _by_platform(str), + # Symbol to use + Required("symbol"): str, + # Tier this task is + Required("tier"): _by_platform(int), + }, + # Extra environment values to pass to the worker + Optional("env"): _by_platform({str: taskref_or_string}), + # Max number locales per chunk + Optional("locales-per-chunk"): _by_platform(int), + # Task deps to chain this task with, added in transforms from primary dependency + # if this is a shippable-style build + Optional("dependencies"): {str: str}, + # Run the task when the listed files change (if present). + Optional("when"): {"files-changed": [str]}, + # passed through directly to the job description + Optional("attributes"): job_description_schema["attributes"], + Optional("extra"): job_description_schema["extra"], + # Shipping product and phase + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("task-from"): task_description_schema["task-from"], +}) transforms = TransformSequence() @@ -296,12 +294,10 @@ def chunk_locales(config, jobs): chunked_locales = chunkify( locales_with_changesets_as_list, this_chunk, chunks ) - chunked["mozharness"]["options"].extend( - [ - f"locale={locale}:{changeset}" - for locale, changeset in chunked_locales - ] - ) + chunked["mozharness"]["options"].extend([ + f"locale={locale}:{changeset}" + for locale, changeset in chunked_locales + ]) chunked["attributes"]["l10n_chunk"] = str(this_chunk) # strip revision chunked["attributes"]["chunk_locales"] = [ @@ -315,12 +311,10 @@ def chunk_locales(config, jobs): yield chunked else: job["mozharness"]["options"] = job["mozharness"].get("options", []) - job["mozharness"]["options"].extend( - [ - f"locale={locale}:{changeset}" - for locale, changeset in sorted(locales_with_changesets.items()) - ] - ) + job["mozharness"]["options"].extend([ + f"locale={locale}:{changeset}" + for locale, changeset in sorted(locales_with_changesets.items()) + ]) yield job @@ -353,13 +347,11 @@ def set_extra_config(config, jobs): @transforms.add def make_job_description(config, jobs): for job in jobs: - job["mozharness"].update( - { - "using": "mozharness", - "job-script": "taskcluster/scripts/builder/build-l10n.sh", - "secrets": job.get("secrets", False), - } - ) + job["mozharness"].update({ + "using": "mozharness", + "job-script": "taskcluster/scripts/builder/build-l10n.sh", + "secrets": job.get("secrets", False), + }) job_description = { "name": job["name"], "worker-type": job["worker-type"], @@ -437,11 +429,9 @@ def add_macos_signing_artifacts(config, jobs): assert build_dep, f"l10n job {job['name']} has no build dependency" for path, artifact in build_dep.task["payload"]["artifacts"].items(): if path.startswith("public/build/security/"): - job["worker"].setdefault("artifacts", []).append( - { - "name": path, - "path": artifact["path"], - "type": "file", - } - ) + job["worker"].setdefault("artifacts", []).append({ + "name": path, + "path": artifact["path"], + "type": "file", + }) yield job diff --git a/taskcluster/gecko_taskgraph/transforms/mar_signing.py b/taskcluster/gecko_taskgraph/transforms/mar_signing.py @@ -62,14 +62,12 @@ def generate_complete_artifacts(job, kind): for artifact in job.attributes["release_artifacts"]: basename = os.path.basename(artifact) if basename in SIGNING_FORMATS[kind]: - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{job.kind}>"}, - "taskType": "build", - "paths": [artifact], - "formats": SIGNING_FORMATS[kind][basename], - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": f"<{job.kind}>"}, + "taskType": "build", + "paths": [artifact], + "formats": SIGNING_FORMATS[kind][basename], + }) return upstream_artifacts @@ -112,7 +110,8 @@ def make_task_description(config, jobs): upstream_artifacts = generate_complete_artifacts(dep_job, config.kind) is_shippable = job.get( - "shippable", dep_job.attributes.get("shippable") # First check current job + "shippable", + dep_job.attributes.get("shippable"), # First check current job ) # Then dep job for 'shippable' signing_type = get_signing_type_per_platform( build_platform, is_shippable, config diff --git a/taskcluster/gecko_taskgraph/transforms/maybe_release.py b/taskcluster/gecko_taskgraph/transforms/maybe_release.py @@ -18,6 +18,6 @@ def make_task_description(config, jobs): job, key, item_name=job["name"], - **{"release-level": release_level(config.params)} + **{"release-level": release_level(config.params)}, ) yield job diff --git a/taskcluster/gecko_taskgraph/transforms/merge_automation.py b/taskcluster/gecko_taskgraph/transforms/merge_automation.py @@ -38,7 +38,7 @@ def handle_keyed_by(config, tasks): "release-type": config.params["release_type"], "behavior": merge_config["behavior"], "level": config.params["level"], - } + }, ) yield task diff --git a/taskcluster/gecko_taskgraph/transforms/openh264_signing.py b/taskcluster/gecko_taskgraph/transforms/openh264_signing.py @@ -17,18 +17,16 @@ from gecko_taskgraph.util.scriptworker import get_signing_type_per_platform transforms = TransformSequence() -signing_description_schema = Schema( - { - Optional("label"): str, - Optional("extra"): object, - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +signing_description_schema = Schema({ + Optional("label"): str, + Optional("extra"): object, + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) @transforms.add @@ -53,8 +51,7 @@ def make_signing_description(config, jobs): is_nightly = True # cert_scope_per_platform uses this to choose the right cert description = ( - "Signing of OpenH264 Binaries for '" - "{build_platform}/{build_type}'".format( + "Signing of OpenH264 Binaries for '{build_platform}/{build_type}'".format( build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) diff --git a/taskcluster/gecko_taskgraph/transforms/partials.py b/taskcluster/gecko_taskgraph/transforms/partials.py @@ -30,20 +30,16 @@ def _generate_task_output_files(job, filenames, locale=None): data = list() for filename in filenames: - data.append( - { - "type": "file", - "path": f"/home/worker/artifacts/{filename}", - "name": f"{artifact_prefix}/{locale_output_path}{filename}", - } - ) - data.append( - { + data.append({ "type": "file", - "path": "/home/worker/artifacts/manifest.json", - "name": f"{artifact_prefix}/{locale_output_path}manifest.json", - } - ) + "path": f"/home/worker/artifacts/{filename}", + "name": f"{artifact_prefix}/{locale_output_path}{filename}", + }) + data.append({ + "type": "file", + "path": "/home/worker/artifacts/manifest.json", + "name": f"{artifact_prefix}/{locale_output_path}manifest.json", + }) return data @@ -167,9 +163,9 @@ def make_task_description(config, jobs): } # We only want caching on linux/windows due to bug 1436977 - if int(level) == 3 and any( - [build_platform.startswith(prefix) for prefix in ["linux", "win"]] - ): + if int(level) == 3 and any([ + build_platform.startswith(prefix) for prefix in ["linux", "win"] + ]): task["scopes"].append( "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" ) diff --git a/taskcluster/gecko_taskgraph/transforms/partials_zucchini.py b/taskcluster/gecko_taskgraph/transforms/partials_zucchini.py @@ -4,6 +4,7 @@ """ Transform the partials task into an actual task description. """ + from taskgraph.transforms.base import TransformSequence from taskgraph.util.dependencies import get_primary_dependency from taskgraph.util.treeherder import inherit_treeherder_from_dep @@ -78,15 +79,13 @@ def make_task_description(config, tasks): task["worker"]["env"]["TO_MAR_TASK_ID"] = { "task-reference": f"<{dep_task.kind}>" } - extra_params.extend( - [ - f"--arch={architecture(build_platform)}", - f"--locale={build_locale}", - # This isn't a great approach to resolving the source URL of to_mar, but it's the same as - # It's only being used to fill manifest.json information, the actual file is downloaded via run-task - f"--to_mar_url=https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/$TO_MAR_TASK_ID/artifacts/{to_mar_path}", - ] - ) + extra_params.extend([ + f"--arch={architecture(build_platform)}", + f"--locale={build_locale}", + # This isn't a great approach to resolving the source URL of to_mar, but it's the same as + # It's only being used to fill manifest.json information, the actual file is downloaded via run-task + f"--to_mar_url=https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/$TO_MAR_TASK_ID/artifacts/{to_mar_path}", + ]) # Add a space from the last command + list of extra_params task["run"]["command"] += " " + " ".join(extra_params) diff --git a/taskcluster/gecko_taskgraph/transforms/partner_attribution.py b/taskcluster/gecko_taskgraph/transforms/partner_attribution.py @@ -68,13 +68,11 @@ def add_command_arguments(config, tasks): fetches[upstream_label].add(attributed_build_config["fetch_config"]) - attributions.append( - { - "input": attributed_build_config["input_path"], - "output": attributed_build_config["output_path"], - "attribution": attribution_code, - } - ) + attributions.append({ + "input": attributed_build_config["input_path"], + "output": attributed_build_config["output_path"], + "attribution": attribution_code, + }) release_artifacts.append( attributed_build_config["release_artifact"] ) diff --git a/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py b/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py @@ -24,26 +24,24 @@ from gecko_taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, ) -beetmover_description_schema = Schema( - { - # from the loader: - Optional("task-from"): str, - Optional("name"): str, - # from the from_deps transforms: - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - # depname is used in taskref's to identify the taskID of the unsigned things - Required("depname", default="build"): str, - # unique label to describe this beetmover task, defaults to {dep.label}-beetmover - Optional("label"): str, - Required("partner-path"): str, - Optional("extra"): object, - Required("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("priority"): task_description_schema["priority"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_description_schema = Schema({ + # from the loader: + Optional("task-from"): str, + Optional("name"): str, + # from the from_deps transforms: + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + # depname is used in taskref's to identify the taskID of the unsigned things + Required("depname", default="build"): str, + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + Required("partner-path"): str, + Optional("extra"): object, + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("priority"): task_description_schema["priority"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() transforms.add_validate(beetmover_description_schema) @@ -124,19 +122,17 @@ def make_task_description(config, jobs): def generate_upstream_artifacts(attribution_task_kind, artifacts, partner_path): upstream_artifacts = [] for artifact, partner, subpartner, platform, locale in artifacts: - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{attribution_task_kind}>"}, - "taskType": "repackage", - "paths": [artifact], - "locale": partner_path.format( - partner=partner, - subpartner=subpartner, - platform=platform, - locale=locale, - ), - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": f"<{attribution_task_kind}>"}, + "taskType": "repackage", + "paths": [artifact], + "locale": partner_path.format( + partner=partner, + subpartner=subpartner, + platform=platform, + locale=locale, + ), + }) if not upstream_artifacts: raise Exception("Couldn't find any upstream artifacts.") diff --git a/taskcluster/gecko_taskgraph/transforms/partner_repack.py b/taskcluster/gecko_taskgraph/transforms/partner_repack.py @@ -129,13 +129,11 @@ def add_command_arguments(config, tasks): # blocks that. It's space-separated string of ids in the end. task["worker"]["env"]["UPSTREAM_TASKIDS"] = { # We only want signing related tasks here, not build (used by mac builds for signing artifact resolution) - "task-reference": " ".join( - [ - f"<{dep}>" - for dep in task["dependencies"] - if ("signing" in dep or "notarization" in dep) - ] - ) + "task-reference": " ".join([ + f"<{dep}>" + for dep in task["dependencies"] + if ("signing" in dep or "notarization" in dep) + ]) } # Forward the release type for bouncer product construction @@ -158,11 +156,9 @@ def add_macos_signing_artifacts(config, tasks): assert build_dep, f"repackage job {task['name']} has no build dependency" for path, artifact in build_dep.task["payload"]["artifacts"].items(): if path.startswith("public/build/security/"): - task["worker"].setdefault("artifacts", []).append( - { - "name": path, - "path": artifact["path"], - "type": "file", - } - ) + task["worker"].setdefault("artifacts", []).append({ + "name": path, + "path": artifact["path"], + "type": "file", + }) yield task diff --git a/taskcluster/gecko_taskgraph/transforms/partner_signing.py b/taskcluster/gecko_taskgraph/transforms/partner_signing.py @@ -24,9 +24,9 @@ def set_mac_label(config, jobs): if "mac-notarization" in config.kind: default_label = dep_job.label.replace("mac-signing", "mac-notarization") job.setdefault("label", default_label) - assert ( - job["label"] != dep_job.label - ), f"Unable to determine label for {config.kind}" + assert job["label"] != dep_job.label, ( + f"Unable to determine label for {config.kind}" + ) yield job diff --git a/taskcluster/gecko_taskgraph/transforms/perftest.py b/taskcluster/gecko_taskgraph/transforms/perftest.py @@ -17,39 +17,37 @@ from voluptuous import Any, Extra, Optional transforms = TransformSequence() -perftest_description_schema = Schema( - { - # The test names and the symbols to use for them: [test-symbol, test-path] - Optional("perftest"): [[str]], - # Metrics to gather for the test. These will be merged - # with options specified through perftest-perfherder-global - Optional("perftest-metrics"): optionally_keyed_by( - "perftest", - Any( - [str], - {str: Any(None, {str: Any(None, str, [str])})}, - ), +perftest_description_schema = Schema({ + # The test names and the symbols to use for them: [test-symbol, test-path] + Optional("perftest"): [[str]], + # Metrics to gather for the test. These will be merged + # with options specified through perftest-perfherder-global + Optional("perftest-metrics"): optionally_keyed_by( + "perftest", + Any( + [str], + {str: Any(None, {str: Any(None, str, [str])})}, ), - # Perfherder data options that will be applied to - # all metrics gathered. - Optional("perftest-perfherder-global"): optionally_keyed_by( - "perftest", {str: Any(None, str, [str])} - ), - # Extra options to add to the test's command - Optional("perftest-extra-options"): optionally_keyed_by("perftest", [str]), - # Variants of the test to make based on extra browsertime - # arguments. Expecting: - # [variant-suffix, options-to-use] - # If variant-suffix is `null` then the options will be added - # to the existing task. Otherwise, a new variant is created - # with the given suffix and with its options replaced. - Optional("perftest-btime-variants"): optionally_keyed_by( - "perftest", [[Any(None, str)]] - ), - # These options will be parsed in the next schemas - Extra: object, - } -) + ), + # Perfherder data options that will be applied to + # all metrics gathered. + Optional("perftest-perfherder-global"): optionally_keyed_by( + "perftest", {str: Any(None, str, [str])} + ), + # Extra options to add to the test's command + Optional("perftest-extra-options"): optionally_keyed_by("perftest", [str]), + # Variants of the test to make based on extra browsertime + # arguments. Expecting: + # [variant-suffix, options-to-use] + # If variant-suffix is `null` then the options will be added + # to the existing task. Otherwise, a new variant is created + # with the given suffix and with its options replaced. + Optional("perftest-btime-variants"): optionally_keyed_by( + "perftest", [[Any(None, str)]] + ), + # These options will be parsed in the next schemas + Extra: object, +}) transforms.add_validate(perftest_description_schema) @@ -216,24 +214,16 @@ def setup_perftest_metrics(config, jobs): job["run"]["command"] = job["run"]["command"].replace( "{perftest_metrics}", - " ".join( - [ - ",".join( - [ - ":".join( - [ - option, - str(value) - .replace(" ", "") - .replace("'", quote_escape), - ] - ) - for option, value in metric_info.items() - ] - ) - for metric_info in perftest_metrics - ] - ), + " ".join([ + ",".join([ + ":".join([ + option, + str(value).replace(" ", "").replace("'", quote_escape), + ]) + for option, value in metric_info.items() + ]) + for metric_info in perftest_metrics + ]), ) yield job @@ -246,9 +236,9 @@ def setup_perftest_browsertime_variants(config, jobs): yield job continue - job["run"]["command"] += " --browsertime-extra-options %s" % ",".join( - [opt.strip() for opt in job.pop("perftest-btime-variants")] - ) + job["run"]["command"] += " --browsertime-extra-options %s" % ",".join([ + opt.strip() for opt in job.pop("perftest-btime-variants") + ]) yield job @@ -278,22 +268,20 @@ def create_duplicate_simpleperf_jobs(config, jobs): "android-aarch64-shippable": "build-android-aarch64-shippable/opt" } new_job["name"] += "-profiling" - new_job["run"][ - "command" - ] += " --simpleperf --simpleperf-path $MOZ_FETCHES_DIR/android-simpleperf --geckoprofiler" + new_job["run"]["command"] += ( + " --simpleperf --simpleperf-path $MOZ_FETCHES_DIR/android-simpleperf --geckoprofiler" + ) new_job["description"] = str(new_job["description"]).replace( "Run", "Profile" ) new_job["treeherder"]["symbol"] = str( new_job["treeherder"]["symbol"] ).replace(")", "-profile)") - new_job["fetches"]["toolchain"].extend( - [ - "linux64-android-simpleperf-linux-repack", - "linux64-samply", - "symbolicator-cli", - ] - ) + new_job["fetches"]["toolchain"].extend([ + "linux64-android-simpleperf-linux-repack", + "linux64-samply", + "symbolicator-cli", + ]) new_job["fetches"]["android-aarch64-shippable"] = [ { "artifact": "target.crashreporter-symbols.zip", diff --git a/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py b/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py @@ -33,27 +33,25 @@ logger = logging.getLogger(__name__) transforms = TransformSequence() -beetmover_description_schema = Schema( - { - # attributes is used for enabling artifact-map by declarative artifacts - Required("attributes"): {str: object}, - # unique label to describe this beetmover task, defaults to {dep.label}-beetmover - Optional("label"): str, - # treeherder is allowed here to override any defaults we use for beetmover. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the - # below transforms for defaults of various values. - Optional("treeherder"): task_description_schema["treeherder"], - Required("description"): str, - Required("worker-type"): optionally_keyed_by("release-level", str), - Required("run-on-projects"): [], - # locale is passed only for l10n beetmoving - Optional("locale"): str, - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("task-from"): task_description_schema["task-from"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +beetmover_description_schema = Schema({ + # attributes is used for enabling artifact-map by declarative artifacts + Required("attributes"): {str: object}, + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Required("description"): str, + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("run-on-projects"): [], + # locale is passed only for l10n beetmoving + Optional("locale"): str, + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("task-from"): task_description_schema["task-from"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) @transforms.add diff --git a/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py b/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py @@ -13,25 +13,23 @@ from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import release_level from gecko_taskgraph.util.scriptworker import add_scope_prefix -push_flatpak_description_schema = Schema( - { - Required("name"): str, - Required("task-from"): task_description_schema["task-from"], - Required("dependencies"): task_description_schema["dependencies"], - Required("description"): task_description_schema["description"], - Required("treeherder"): task_description_schema["treeherder"], - Required("run-on-projects"): task_description_schema["run-on-projects"], - Required("worker-type"): optionally_keyed_by("release-level", str), - Required("worker"): object, - Optional("scopes"): [str], - Required("shipping-phase"): task_description_schema["shipping-phase"], - Required("shipping-product"): task_description_schema["shipping-product"], - Required("flathub-scope"): str, - Optional("extra"): task_description_schema["extra"], - Optional("attributes"): task_description_schema["attributes"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +push_flatpak_description_schema = Schema({ + Required("name"): str, + Required("task-from"): task_description_schema["task-from"], + Required("dependencies"): task_description_schema["dependencies"], + Required("description"): task_description_schema["description"], + Required("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): object, + Optional("scopes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Required("flathub-scope"): str, + Optional("extra"): task_description_schema["extra"], + Optional("attributes"): task_description_schema["attributes"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() transforms.add_validate(push_flatpak_description_schema) diff --git a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py @@ -33,7 +33,7 @@ def handle_keyed_by(config, jobs): item=job, field=field, item_name=job["name"], - **{"release-level": release_level(config.params)} + **{"release-level": release_level(config.params)}, ) yield job diff --git a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_beetmover.py b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_beetmover.py @@ -23,22 +23,20 @@ from gecko_taskgraph.util.scriptworker import ( transforms = TransformSequence() -release_generate_checksums_beetmover_schema = Schema( - { - # unique label to describe this beetmover task, defaults to {dep.label}-beetmover - Optional("label"): str, - # treeherder is allowed here to override any defaults we use for beetmover. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the - # below transforms for defaults of various values. - Optional("treeherder"): task_description_schema["treeherder"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("attributes"): task_description_schema["attributes"], - Optional("task-from"): task_description_schema["task-from"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +release_generate_checksums_beetmover_schema = Schema({ + # unique label to describe this beetmover task, defaults to {dep.label}-beetmover + Optional("label"): str, + # treeherder is allowed here to override any defaults we use for beetmover. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("attributes"): task_description_schema["attributes"], + Optional("task-from"): task_description_schema["task-from"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() @@ -112,9 +110,9 @@ def make_task_description(config, jobs): @transforms.add def make_task_worker(config, jobs): for job in jobs: - valid_beetmover_job = len(job["dependencies"]) == 2 and any( - ["signing" in j for j in job["dependencies"]] - ) + valid_beetmover_job = len(job["dependencies"]) == 2 and any([ + "signing" in j for j in job["dependencies"] + ]) if not valid_beetmover_job: raise NotImplementedError("Beetmover must have two dependencies.") diff --git a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums_signing.py @@ -15,18 +15,16 @@ from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.scriptworker import get_signing_type -release_generate_checksums_signing_schema = Schema( - { - Optional("label"): str, - Optional("dependencies"): task_description_schema["dependencies"], - Optional("attributes"): task_description_schema["attributes"], - Optional("treeherder"): task_description_schema["treeherder"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +release_generate_checksums_signing_schema = Schema({ + Optional("label"): str, + Optional("dependencies"): task_description_schema["dependencies"], + Optional("attributes"): task_description_schema["attributes"], + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py b/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py @@ -19,13 +19,13 @@ def make_task_description(config, jobs): job, "worker-type", item_name=job["name"], - **{"release-level": release_level(config.params)} + **{"release-level": release_level(config.params)}, ) resolve_keyed_by( job, "scopes", item_name=job["name"], - **{"release-level": release_level(config.params)} + **{"release-level": release_level(config.params)}, ) job["worker"]["release-name"] = ( diff --git a/taskcluster/gecko_taskgraph/transforms/release_msix_push.py b/taskcluster/gecko_taskgraph/transforms/release_msix_push.py @@ -13,24 +13,22 @@ from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import release_level from gecko_taskgraph.util.scriptworker import add_scope_prefix -push_msix_description_schema = Schema( - { - Required("name"): str, - Required("task-from"): task_description_schema["task-from"], - Required("dependencies"): task_description_schema["dependencies"], - Required("description"): task_description_schema["description"], - Required("treeherder"): task_description_schema["treeherder"], - Required("run-on-projects"): task_description_schema["run-on-projects"], - Required("worker-type"): optionally_keyed_by("release-level", str), - Required("worker"): object, - Optional("scopes"): [str], - Required("shipping-phase"): task_description_schema["shipping-phase"], - Required("shipping-product"): task_description_schema["shipping-product"], - Optional("extra"): task_description_schema["extra"], - Optional("attributes"): task_description_schema["attributes"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +push_msix_description_schema = Schema({ + Required("name"): str, + Required("task-from"): task_description_schema["task-from"], + Required("dependencies"): task_description_schema["dependencies"], + Required("description"): task_description_schema["description"], + Required("treeherder"): task_description_schema["treeherder"], + Required("run-on-projects"): task_description_schema["run-on-projects"], + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): object, + Optional("scopes"): [str], + Required("shipping-phase"): task_description_schema["shipping-phase"], + Required("shipping-product"): task_description_schema["shipping-product"], + Optional("extra"): task_description_schema["extra"], + Optional("attributes"): task_description_schema["attributes"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() transforms.add_validate(push_msix_description_schema) diff --git a/taskcluster/gecko_taskgraph/transforms/release_notifications.py b/taskcluster/gecko_taskgraph/transforms/release_notifications.py @@ -4,6 +4,7 @@ """ Add notifications via taskcluster-notify for release tasks """ + from string import Formatter from taskgraph.transforms.base import TransformSequence @@ -67,20 +68,18 @@ def add_notifications(config, jobs): # candidates dir' when cancelling graphs, dummy job failure, etc status_types = notifications.get("status-types", ["on-completed"]) for s in status_types: - job.setdefault("routes", []).extend( - [f"notify.email.{email}.{s}" for email in emails] - ) + job.setdefault("routes", []).extend([ + f"notify.email.{email}.{s}" for email in emails + ]) # Customize the email subject to include release name and build number - job.setdefault("extra", {}).update( - { - "notify": { - "email": { - "subject": subject, - } + job.setdefault("extra", {}).update({ + "notify": { + "email": { + "subject": subject, } } - ) + }) if message: job["extra"]["notify"]["email"]["content"] = message diff --git a/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py b/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py @@ -19,26 +19,24 @@ from gecko_taskgraph.util.attributes import ( transforms = TransformSequence() -langpack_sign_push_description_schema = Schema( - { - Required("label"): str, - Required("description"): str, - Required("worker-type"): optionally_keyed_by("release-level", str), - Required("worker"): { - Required("channel"): optionally_keyed_by( - "project", "platform", Any("listed", "unlisted") - ), - Required("upstream-artifacts"): None, # Processed here below - }, - Required("run-on-projects"): [], - Required("scopes"): optionally_keyed_by("release-level", [str]), - Required("shipping-phase"): task_description_schema["shipping-phase"], - Optional("task-from"): task_description_schema["task-from"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +langpack_sign_push_description_schema = Schema({ + Required("label"): str, + Required("description"): str, + Required("worker-type"): optionally_keyed_by("release-level", str), + Required("worker"): { + Required("channel"): optionally_keyed_by( + "project", "platform", Any("listed", "unlisted") + ), + Required("upstream-artifacts"): None, # Processed here below + }, + Required("run-on-projects"): [], + Required("scopes"): optionally_keyed_by("release-level", [str]), + Required("shipping-phase"): task_description_schema["shipping-phase"], + Optional("task-from"): task_description_schema["task-from"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) @transforms.add diff --git a/taskcluster/gecko_taskgraph/transforms/release_started.py b/taskcluster/gecko_taskgraph/transforms/release_started.py @@ -4,6 +4,7 @@ """ Add notifications via taskcluster-notify for release tasks """ + from shlex import quote as shell_quote from taskgraph.transforms.base import TransformSequence diff --git a/taskcluster/gecko_taskgraph/transforms/release_update_product_channel_version.py b/taskcluster/gecko_taskgraph/transforms/release_update_product_channel_version.py @@ -22,7 +22,7 @@ def make_task_worker(config, jobs): job, "scopes", item_name=job["name"], - **{"release-level": release_level(config.params)} + **{"release-level": release_level(config.params)}, ) job["worker"]["product"] = job["shipping-product"] job["worker"]["version"] = config.params["version"] diff --git a/taskcluster/gecko_taskgraph/transforms/repackage.py b/taskcluster/gecko_taskgraph/transforms/repackage.py @@ -17,102 +17,100 @@ from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.platforms import architecture, archive_format from gecko_taskgraph.util.workertypes import worker_type_implementation -packaging_description_schema = Schema( - { - # unique label to describe this repackaging task - Optional("label"): str, - Optional("worker-type"): str, - Optional("worker"): object, - Optional("attributes"): job_description_schema["attributes"], - Optional("dependencies"): job_description_schema["dependencies"], - # treeherder is allowed here to override any defaults we use for repackaging. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the - # below transforms for defaults of various values. - Optional("treeherder"): job_description_schema["treeherder"], - # If a l10n task, the corresponding locale - Optional("locale"): str, - # Routes specific to this task, if defined - Optional("routes"): [str], - # passed through directly to the job description - Optional("extra"): job_description_schema["extra"], - # passed through to job description - Optional("fetches"): job_description_schema["fetches"], - Optional("run-on-projects"): job_description_schema["run-on-projects"], - Optional("run-on-repo-type"): job_description_schema["run-on-repo-type"], - # Shipping product and phase - Optional("shipping-product"): job_description_schema["shipping-product"], - Optional("shipping-phase"): job_description_schema["shipping-phase"], - Required("package-formats"): optionally_keyed_by( - "build-platform", "release-type", "build-type", [str] +packaging_description_schema = Schema({ + # unique label to describe this repackaging task + Optional("label"): str, + Optional("worker-type"): str, + Optional("worker"): object, + Optional("attributes"): job_description_schema["attributes"], + Optional("dependencies"): job_description_schema["dependencies"], + # treeherder is allowed here to override any defaults we use for repackaging. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): job_description_schema["treeherder"], + # If a l10n task, the corresponding locale + Optional("locale"): str, + # Routes specific to this task, if defined + Optional("routes"): [str], + # passed through directly to the job description + Optional("extra"): job_description_schema["extra"], + # passed through to job description + Optional("fetches"): job_description_schema["fetches"], + Optional("run-on-projects"): job_description_schema["run-on-projects"], + Optional("run-on-repo-type"): job_description_schema["run-on-repo-type"], + # Shipping product and phase + Optional("shipping-product"): job_description_schema["shipping-product"], + Optional("shipping-phase"): job_description_schema["shipping-phase"], + Required("package-formats"): optionally_keyed_by( + "build-platform", "release-type", "build-type", [str] + ), + Optional("msix"): { + Optional("channel"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, ), - Optional("msix"): { - Optional("channel"): optionally_keyed_by( - "package-format", - "level", - "build-platform", - "release-type", - "shipping-product", - str, - ), - Optional("identity-name"): optionally_keyed_by( - "package-format", - "level", - "build-platform", - "release-type", - "shipping-product", - str, - ), - Optional("publisher"): optionally_keyed_by( - "package-format", - "level", - "build-platform", - "release-type", - "shipping-product", - str, - ), - Optional("publisher-display-name"): optionally_keyed_by( - "package-format", - "level", - "build-platform", - "release-type", - "shipping-product", - str, - ), - Optional("vendor"): str, - }, - Optional("flatpak"): { - Required("name"): optionally_keyed_by( - "level", - "build-platform", - "release-type", - "shipping-product", - str, - ), - Required("branch"): optionally_keyed_by( - "level", - "build-platform", - "release-type", - "shipping-product", - str, - ), - }, - # All l10n jobs use mozharness - Required("mozharness"): { - Extra: object, - # Config files passed to the mozharness script - Required("config"): optionally_keyed_by("build-platform", [str]), - # Additional paths to look for mozharness configs in. These should be - # relative to the base of the source checkout - Optional("config-paths"): [str], - # if true, perform a checkout of a comm-central based branch inside the - # gecko checkout - Optional("comm-checkout"): bool, - Optional("run-as-root"): bool, - Optional("use-caches"): Any(bool, [str]), - }, - Optional("task-from"): job_description_schema["task-from"], - } -) + Optional("identity-name"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Optional("publisher"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Optional("publisher-display-name"): optionally_keyed_by( + "package-format", + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Optional("vendor"): str, + }, + Optional("flatpak"): { + Required("name"): optionally_keyed_by( + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + Required("branch"): optionally_keyed_by( + "level", + "build-platform", + "release-type", + "shipping-product", + str, + ), + }, + # All l10n jobs use mozharness + Required("mozharness"): { + Extra: object, + # Config files passed to the mozharness script + Required("config"): optionally_keyed_by("build-platform", [str]), + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Optional("comm-checkout"): bool, + Optional("run-as-root"): bool, + Optional("use-caches"): Any(bool, [str]), + }, + Optional("task-from"): job_description_schema["task-from"], +}) # The configuration passed to the mozharness repackage script. This defines the # arguments passed to `mach repackage` @@ -532,19 +530,17 @@ def make_job_description(config, jobs): dependencies.update({t.label: t.label}) - fetches.update( - { - t.label: [ - { - "artifact": f"{loc}/target.langpack.xpi", - "extract": False, - # Otherwise we can't disambiguate locales! - "dest": f"distribution/extensions/{loc}", - } - for loc in t.attributes["chunk_locales"] - ] - } - ) + fetches.update({ + t.label: [ + { + "artifact": f"{loc}/target.langpack.xpi", + "extract": False, + # Otherwise we can't disambiguate locales! + "dest": f"distribution/extensions/{loc}", + } + for loc in t.attributes["chunk_locales"] + ] + }) elif config.kind in ("repackage-deb", "repackage-rpm"): attributes["repackage_type"] = config.kind @@ -593,19 +589,17 @@ def make_job_description(config, jobs): dependencies.update({t.label: t.label}) - fetches.update( - { - t.label: [ - { - "artifact": f"{loc}/target.langpack.xpi", - "extract": False, - # Otherwise we can't disambiguate locales! - "dest": f"extensions/{loc}", - } - for loc in locales - ] - } - ) + fetches.update({ + t.label: [ + { + "artifact": f"{loc}/target.langpack.xpi", + "extract": False, + # Otherwise we can't disambiguate locales! + "dest": f"extensions/{loc}", + } + for loc in locales + ] + }) _fetch_subst_locale = "en-US" if locale: @@ -678,27 +672,23 @@ def make_job_description(config, jobs): repackage_config.append(command) run = job.get("mozharness", {}) - run.update( - { - "using": "mozharness", - "script": "mozharness/scripts/repackage.py", - "job-script": "taskcluster/scripts/builder/repackage.sh", - "actions": ["setup", "repackage"], - "extra-config": { - "repackage_config": repackage_config, - }, - "run-as-root": run.get("run-as-root", False), - } - ) + run.update({ + "using": "mozharness", + "script": "mozharness/scripts/repackage.py", + "job-script": "taskcluster/scripts/builder/repackage.sh", + "actions": ["setup", "repackage"], + "extra-config": { + "repackage_config": repackage_config, + }, + "run-as-root": run.get("run-as-root", False), + }) worker = job.get("worker", {}) - worker.update( - { - "chain-of-trust": True, - # Don't add generic artifact directory. - "skip-artifacts": True, - } - ) + worker.update({ + "chain-of-trust": True, + # Don't add generic artifact directory. + "skip-artifacts": True, + }) worker.setdefault("max-run-time", 3600) if locale: @@ -744,15 +734,13 @@ def make_job_description(config, jobs): } if build_platform.startswith("macosx"): - task.setdefault("fetches", {}).setdefault("toolchain", []).extend( - [ - "linux64-libdmg", - "linux64-hfsplus", - "linux64-node", - "linux64-xar", - "linux64-mkbom", - ] - ) + task.setdefault("fetches", {}).setdefault("toolchain", []).extend([ + "linux64-libdmg", + "linux64-hfsplus", + "linux64-node", + "linux64-xar", + "linux64-mkbom", + ]) if "shipping-phase" in job: task["shipping-phase"] = job["shipping-phase"] @@ -778,11 +766,9 @@ def _generate_download_config( fetch.update(existing_fetch) if repackage_signing_task and build_platform.startswith("win"): - fetch.update( - { - repackage_signing_task: [f"{locale_path}target.installer.exe"], - } - ) + fetch.update({ + repackage_signing_task: [f"{locale_path}target.installer.exe"], + }) elif build_platform.startswith("linux") or build_platform.startswith("macosx"): signing_fetch = [ { @@ -791,25 +777,21 @@ def _generate_download_config( }, ] if config.kind == "repackage-deb-l10n": - signing_fetch.append( - { - "artifact": f"{locale_path}target.langpack.xpi", - "extract": False, - } - ) + signing_fetch.append({ + "artifact": f"{locale_path}target.langpack.xpi", + "extract": False, + }) fetch.update({signing_task: signing_fetch}) elif build_platform.startswith("win"): - fetch.update( - { - signing_task: [ - { - "artifact": f"{locale_path}target.zip", - "extract": False, - }, - f"{locale_path}setup.exe", - ], - } - ) + fetch.update({ + signing_task: [ + { + "artifact": f"{locale_path}target.zip", + "extract": False, + }, + f"{locale_path}setup.exe", + ], + }) use_stub = task.attributes.get("stub-installer") if use_stub: @@ -838,15 +820,13 @@ def _generate_task_output_files( output_files = [] for config in repackage_config: - output_files.append( - { - "type": "file", - "path": "{}outputs/{}{}".format( - local_prefix, locale_output_path, config["output"] - ), - "name": "{}/{}{}".format( - artifact_prefix, locale_output_path, config["output"] - ), - } - ) + output_files.append({ + "type": "file", + "path": "{}outputs/{}{}".format( + local_prefix, locale_output_path, config["output"] + ), + "name": "{}/{}{}".format( + artifact_prefix, locale_output_path, config["output"] + ), + }) return output_files diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_l10n.py b/taskcluster/gecko_taskgraph/transforms/repackage_l10n.py @@ -5,7 +5,6 @@ Transform the repackage task into an actual task description. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.copy import deepcopy from taskgraph.util.dependencies import get_primary_dependency diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py @@ -5,7 +5,6 @@ Transform the repackage task into an actual task description. """ - import copy from taskgraph.transforms.base import TransformSequence @@ -29,39 +28,37 @@ PACKAGE_FORMATS = copy.deepcopy(PACKAGE_FORMATS_VANILLA) PACKAGE_FORMATS["installer-stub"]["inputs"]["package"] = "target-stub{archive_format}" PACKAGE_FORMATS["installer-stub"]["args"].extend(["--package-name", "{package-name}"]) -packaging_description_schema = Schema( - { - # unique label to describe this repackaging task - Optional("label"): str, - # Routes specific to this task, if defined - Optional("routes"): [str], - # passed through directly to the job description - Optional("extra"): task_description_schema["extra"], - # Shipping product and phase - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Required("package-formats"): optionally_keyed_by( - "build-platform", "build-type", [str] - ), - # All l10n jobs use mozharness - Required("mozharness"): { - # Config files passed to the mozharness script - Required("config"): optionally_keyed_by("build-platform", [str]), - # Additional paths to look for mozharness configs in. These should be - # relative to the base of the source checkout - Optional("config-paths"): [str], - # if true, perform a checkout of a comm-central based branch inside the - # gecko checkout - Optional("comm-checkout"): bool, - }, - # Override the default priority for the project - Optional("priority"): task_description_schema["priority"], - Optional("task-from"): task_description_schema["task-from"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +packaging_description_schema = Schema({ + # unique label to describe this repackaging task + Optional("label"): str, + # Routes specific to this task, if defined + Optional("routes"): [str], + # passed through directly to the job description + Optional("extra"): task_description_schema["extra"], + # Shipping product and phase + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Required("package-formats"): optionally_keyed_by( + "build-platform", "build-type", [str] + ), + # All l10n jobs use mozharness + Required("mozharness"): { + # Config files passed to the mozharness script + Required("config"): optionally_keyed_by("build-platform", [str]), + # Additional paths to look for mozharness configs in. These should be + # relative to the base of the source checkout + Optional("config-paths"): [str], + # if true, perform a checkout of a comm-central based branch inside the + # gecko checkout + Optional("comm-checkout"): bool, + }, + # Override the default priority for the project + Optional("priority"): task_description_schema["priority"], + Optional("task-from"): task_description_schema["task-from"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() @@ -165,17 +162,15 @@ def make_job_description(config, jobs): repackage_config.append(command) run = job.get("mozharness", {}) - run.update( - { - "using": "mozharness", - "script": "mozharness/scripts/repackage.py", - "job-script": "taskcluster/scripts/builder/repackage.sh", - "actions": ["setup", "repackage"], - "extra-config": { - "repackage_config": repackage_config, - }, - } - ) + run.update({ + "using": "mozharness", + "script": "mozharness/scripts/repackage.py", + "job-script": "taskcluster/scripts/builder/repackage.sh", + "actions": ["setup", "repackage"], + "extra-config": { + "repackage_config": repackage_config, + }, + }) worker = { "chain-of-trust": True, @@ -232,13 +227,11 @@ def make_job_description(config, jobs): if job.get("priority"): task["priority"] = job["priority"] if build_platform.startswith("macosx"): - task.setdefault("fetches", {}).setdefault("toolchain", []).extend( - [ - "linux64-libdmg", - "linux64-hfsplus", - "linux64-node", - ] - ) + task.setdefault("fetches", {}).setdefault("toolchain", []).extend([ + "linux64-libdmg", + "linux64-hfsplus", + "linux64-node", + ]) yield task @@ -270,15 +263,13 @@ def _generate_download_config( f"{locale_path}setup.exe", ] if build_platform.startswith("win32") and repack_stub_installer: - download_config.extend( - [ - { - "artifact": f"{locale_path}target-stub.zip", - "extract": False, - }, - f"{locale_path}setup-stub.exe", - ] - ) + download_config.extend([ + { + "artifact": f"{locale_path}target-stub.zip", + "extract": False, + }, + f"{locale_path}setup-stub.exe", + ]) return {signing_task: download_config} raise NotImplementedError(f'Unsupported build_platform: "{build_platform}"') @@ -303,15 +294,13 @@ def _generate_task_output_files(task, worker_implementation, repackage_config, p output_files = [] for config in repackage_config: - output_files.append( - { - "type": "file", - "path": "{}outputs/{}{}".format( - local_prefix, partner_output_path, config["output"] - ), - "name": "{}/{}{}".format( - artifact_prefix, partner_output_path, config["output"] - ), - } - ) + output_files.append({ + "type": "file", + "path": "{}outputs/{}{}".format( + local_prefix, partner_output_path, config["output"] + ), + "name": "{}/{}{}".format( + artifact_prefix, partner_output_path, config["output"] + ), + }) return output_files diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_set_upstream_mac_kind.py b/taskcluster/gecko_taskgraph/transforms/repackage_set_upstream_mac_kind.py @@ -36,7 +36,7 @@ def repackage_set_upstream_mac_kind(config, tasks): **{ "build-platform": primary_dep.attributes["build_platform"], "release-level": release_level(config.params), - } + }, ) upstream_mac_kind = task.pop("upstream-mac-kind") diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing.py @@ -16,18 +16,16 @@ from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.scriptworker import get_signing_type_per_platform -repackage_signing_description_schema = Schema( - { - Optional("label"): str, - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("task-from"): task_description_schema["task-from"], - Optional("treeherder"): task_description_schema["treeherder"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +repackage_signing_description_schema = Schema({ + Optional("label"): str, + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("task-from"): task_description_schema["task-from"], + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) SIGNING_FORMATS = { "target.installer.exe": ["gcp_prod_autograph_authenticode_202412_stub"], @@ -96,14 +94,12 @@ def make_repackage_signing_description(config, jobs): # This is so we get the build task etc in our dependencies to have better beetmover # support. But for multi-locale MSIX packages, we don't want the signing task to directly # depend on the langpack tasks. - dependencies.update( - { - k: v - for k, v in signing_dependencies.items() - if k != "docker-image" - and not k.startswith("shippable-l10n-signing-linux64") - } - ) + dependencies.update({ + k: v + for k, v in signing_dependencies.items() + if k != "docker-image" + and not k.startswith("shippable-l10n-signing-linux64") + }) description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" @@ -124,14 +120,12 @@ def make_repackage_signing_description(config, jobs): for artifact in sorted(dep_job.attributes.get("release_artifacts")): basename = os.path.basename(artifact) if basename in SIGNING_FORMATS: - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{dep_kind}>"}, - "taskType": "repackage", - "paths": [artifact], - "formats": SIGNING_FORMATS[os.path.basename(artifact)], - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": f"<{dep_kind}>"}, + "taskType": "repackage", + "paths": [artifact], + "formats": SIGNING_FORMATS[os.path.basename(artifact)], + }) task = { "label": label, diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_signing_partner.py @@ -18,19 +18,17 @@ from gecko_taskgraph.util.scriptworker import get_signing_type_per_platform transforms = TransformSequence() -repackage_signing_description_schema = Schema( - { - Optional("label"): str, - Optional("extra"): object, - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("priority"): task_description_schema["priority"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +repackage_signing_description_schema = Schema({ + Optional("label"): str, + Optional("extra"): object, + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("priority"): task_description_schema["priority"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) @transforms.add @@ -106,22 +104,20 @@ def make_repackage_signing_description(config, jobs): "repack_stub_installer" ) if build_platform.startswith("win32") and repack_stub_installer: - upstream_artifacts.append( - { - "taskId": {"task-reference": "<repackage>"}, - "taskType": "repackage", - "paths": [ - get_artifact_path( - dep_job, - f"{repack_id}/target.stub-installer.exe", - ), - ], - "formats": [ - "gcp_prod_autograph_authenticode_202412", - "gcp_prod_autograph_gpg", - ], - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": "<repackage>"}, + "taskType": "repackage", + "paths": [ + get_artifact_path( + dep_job, + f"{repack_id}/target.stub-installer.exe", + ), + ], + "formats": [ + "gcp_prod_autograph_authenticode_202412", + "gcp_prod_autograph_gpg", + ], + }) elif "mac" in build_platform: upstream_artifacts = [ { diff --git a/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py b/taskcluster/gecko_taskgraph/transforms/reprocess_symbols.py @@ -6,7 +6,6 @@ Transform the reprocess-symbols task description template, taskcluster/kinds/reprocess-symbols/job-template.yml into an actual task description. """ - import logging from taskgraph.transforms.base import TransformSequence diff --git a/taskcluster/gecko_taskgraph/transforms/scriptworker.py b/taskcluster/gecko_taskgraph/transforms/scriptworker.py @@ -6,7 +6,6 @@ Transforms for adding appropriate scopes to scriptworker tasks. """ - from gecko_taskgraph.util.scriptworker import get_balrog_server_scope diff --git a/taskcluster/gecko_taskgraph/transforms/scriptworker_canary.py b/taskcluster/gecko_taskgraph/transforms/scriptworker_canary.py @@ -5,7 +5,6 @@ Build a command to run `mach release push-scriptworker-canaries`. """ - from shlex import quote as shell_quote from mozrelease.scriptworker_canary import TASK_TYPES @@ -40,10 +39,11 @@ def build_command(config, jobs): command.extend(["--ssh-key-secret", ssh_key_secret]) job.setdefault("scopes", []).append(f"secrets:get:{ssh_key_secret}") - job.setdefault("run", {}).update( - {"using": "mach", "mach": " ".join(map(shell_quote, command))} - ) - job.setdefault("routes", []).extend( - [f"notify.email.{address}.on-failed" for address in addresses] - ) + job.setdefault("run", {}).update({ + "using": "mach", + "mach": " ".join(map(shell_quote, command)), + }) + job.setdefault("routes", []).extend([ + f"notify.email.{address}.on-failed" for address in addresses + ]) yield job diff --git a/taskcluster/gecko_taskgraph/transforms/shippable_l10n_signing.py b/taskcluster/gecko_taskgraph/transforms/shippable_l10n_signing.py @@ -59,24 +59,20 @@ def define_upstream_artifacts(config, jobs): if dep_job.kind.endswith(("-mac-notarization", "-mac-signing")): # Upstream is mac signing or notarization upstream_task_type = "scriptworker" - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{dep_job.kind}>"}, - "taskType": upstream_task_type, - # Set paths based on artifacts in the specs (above) one per - # locale present in the chunk this is signing stuff for. - # Pass paths through set and sorted() so we get a list back - # and we remove any duplicates (e.g. hardcoded ja-JP-mac langpack) - "paths": sorted( - { - path_template.format(locale=locale) - for locale in dep_job.attributes.get("chunk_locales", []) - for path_template in spec["artifacts"] - } - ), - "formats": spec["formats"], - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": f"<{dep_job.kind}>"}, + "taskType": upstream_task_type, + # Set paths based on artifacts in the specs (above) one per + # locale present in the chunk this is signing stuff for. + # Pass paths through set and sorted() so we get a list back + # and we remove any duplicates (e.g. hardcoded ja-JP-mac langpack) + "paths": sorted({ + path_template.format(locale=locale) + for locale in dep_job.attributes.get("chunk_locales", []) + for path_template in spec["artifacts"] + }), + "formats": spec["formats"], + }) job["upstream-artifacts"] = upstream_artifacts diff --git a/taskcluster/gecko_taskgraph/transforms/signing.py b/taskcluster/gecko_taskgraph/transforms/signing.py @@ -20,44 +20,42 @@ from gecko_taskgraph.util.scriptworker import ( transforms = TransformSequence() -signing_description_schema = Schema( - { - # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py - # because this is passed directly into the signingscript worker - Required("upstream-artifacts"): [ - { - # taskId of the task with the artifact - Required("taskId"): taskref_or_string, - # type of signing task (for CoT) - Required("taskType"): str, - # Paths to the artifacts to sign - Required("paths"): [str], - # Signing formats to use on each of the paths - Required("formats"): [str], - } - ], - # attributes for this task - Optional("attributes"): {str: object}, - # unique label to describe this signing task, defaults to {dep.label}-signing - Optional("label"): str, - # treeherder is allowed here to override any defaults we use for signing. See - # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the - # below transforms for defaults of various values. - Optional("treeherder"): task_description_schema["treeherder"], - # Routes specific to this task, if defined - Optional("routes"): [str], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Required("dependencies"): task_description_schema["dependencies"], - Optional("extra"): {str: object}, - # Max number of partner repacks per chunk - Optional("repacks-per-chunk"): int, - # Override the default priority for the project - Optional("priority"): task_description_schema["priority"], - Optional("task-from"): task_description_schema["task-from"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +signing_description_schema = Schema({ + # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py + # because this is passed directly into the signingscript worker + Required("upstream-artifacts"): [ + { + # taskId of the task with the artifact + Required("taskId"): taskref_or_string, + # type of signing task (for CoT) + Required("taskType"): str, + # Paths to the artifacts to sign + Required("paths"): [str], + # Signing formats to use on each of the paths + Required("formats"): [str], + } + ], + # attributes for this task + Optional("attributes"): {str: object}, + # unique label to describe this signing task, defaults to {dep.label}-signing + Optional("label"): str, + # treeherder is allowed here to override any defaults we use for signing. See + # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional("treeherder"): task_description_schema["treeherder"], + # Routes specific to this task, if defined + Optional("routes"): [str], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Required("dependencies"): task_description_schema["dependencies"], + Optional("extra"): {str: object}, + # Max number of partner repacks per chunk + Optional("repacks-per-chunk"): int, + # Override the default priority for the project + Optional("priority"): task_description_schema["priority"], + Optional("task-from"): task_description_schema["task-from"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) def get_locales_description(attributes, default): diff --git a/taskcluster/gecko_taskgraph/transforms/snap_test.py b/taskcluster/gecko_taskgraph/transforms/snap_test.py @@ -1,9 +1,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -""" - +""" """ import logging diff --git a/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py b/taskcluster/gecko_taskgraph/transforms/source_checksums_signing.py @@ -14,18 +14,16 @@ from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.scriptworker import get_signing_type -checksums_signing_description_schema = Schema( - { - Optional("label"): str, - Optional("treeherder"): task_description_schema["treeherder"], - Optional("shipping-product"): task_description_schema["shipping-product"], - Optional("shipping-phase"): task_description_schema["shipping-phase"], - Optional("task-from"): task_description_schema["task-from"], - Optional("attributes"): task_description_schema["attributes"], - Optional("dependencies"): task_description_schema["dependencies"], - Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], - } -) +checksums_signing_description_schema = Schema({ + Optional("label"): str, + Optional("treeherder"): task_description_schema["treeherder"], + Optional("shipping-product"): task_description_schema["shipping-product"], + Optional("shipping-phase"): task_description_schema["shipping-phase"], + Optional("task-from"): task_description_schema["task-from"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], + Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py @@ -7,7 +7,6 @@ with either `platform` or a list of `platforms`, and set the appropriate treeherder configuration and attributes for that platform. """ - import copy import os @@ -19,40 +18,36 @@ from voluptuous import Any, Extra, Optional, Required from gecko_taskgraph.transforms.job import job_description_schema -source_test_description_schema = Schema( - { - # most fields are passed directly through as job fields, and are not - # repeated here - Extra: object, - # The platform on which this task runs. This will be used to set up attributes - # (for try selection) and treeherder metadata (for display). If given as a list, - # the job will be "split" into multiple tasks, one with each platform. - Required("platform"): Any(str, [str]), - # Build labels required for the task. If this key is provided it must - # contain a build label for the task platform. - # The task will then depend on a build task, and the installer url will be - # saved to the GECKO_INSTALLER_URL environment variable. - Optional("require-build"): optionally_keyed_by("project", {str: str}), - # These fields can be keyed by "platform", and are otherwise identical to - # job descriptions. - Required("worker-type"): optionally_keyed_by( - "platform", job_description_schema["worker-type"] - ), - Required("worker"): optionally_keyed_by( - "platform", job_description_schema["worker"] - ), - Optional("dependencies"): { - k: optionally_keyed_by("platform", v) - for k, v in job_description_schema["dependencies"].items() - }, - # A list of artifacts to install from 'fetch' tasks. - Optional("fetches"): { - str: optionally_keyed_by( - "platform", job_description_schema["fetches"][str] - ), - }, - } -) +source_test_description_schema = Schema({ + # most fields are passed directly through as job fields, and are not + # repeated here + Extra: object, + # The platform on which this task runs. This will be used to set up attributes + # (for try selection) and treeherder metadata (for display). If given as a list, + # the job will be "split" into multiple tasks, one with each platform. + Required("platform"): Any(str, [str]), + # Build labels required for the task. If this key is provided it must + # contain a build label for the task platform. + # The task will then depend on a build task, and the installer url will be + # saved to the GECKO_INSTALLER_URL environment variable. + Optional("require-build"): optionally_keyed_by("project", {str: str}), + # These fields can be keyed by "platform", and are otherwise identical to + # job descriptions. + Required("worker-type"): optionally_keyed_by( + "platform", job_description_schema["worker-type"] + ), + Required("worker"): optionally_keyed_by( + "platform", job_description_schema["worker"] + ), + Optional("dependencies"): { + k: optionally_keyed_by("platform", v) + for k, v in job_description_schema["dependencies"].items() + }, + # A list of artifacts to install from 'fetch' tasks. + Optional("fetches"): { + str: optionally_keyed_by("platform", job_description_schema["fetches"][str]), + }, +}) transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/split_by_locale.py b/taskcluster/gecko_taskgraph/transforms/split_by_locale.py @@ -17,22 +17,20 @@ from gecko_taskgraph.transforms.l10n import parse_locales_file transforms = TransformSequence() -split_by_locale_schema = Schema( - { - # The file to pull locale information from. This should be a json file - # such as browser/locales/l10n-changesets.json. - Required("locales-file"): str, - # The platform name in the form used by the locales files. Defaults to - # attributes.build_platform if not provided. - Optional("locale-file-platform"): str, - # A list of properties elsewhere in the job that need to have the locale - # name substituted into them. The referenced properties may be strings - # or lists. In the case of the latter, all list values will have - # substitutions performed. - Optional("properties-with-locale"): [str], - Extra: object, - } -) +split_by_locale_schema = Schema({ + # The file to pull locale information from. This should be a json file + # such as browser/locales/l10n-changesets.json. + Required("locales-file"): str, + # The platform name in the form used by the locales files. Defaults to + # attributes.build_platform if not provided. + Optional("locale-file-platform"): str, + # A list of properties elsewhere in the job that need to have the locale + # name substituted into them. The referenced properties may be strings + # or lists. In the case of the latter, all list values will have + # substitutions performed. + Optional("properties-with-locale"): [str], + Extra: object, +}) transforms.add_validate(split_by_locale_schema) diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py @@ -74,152 +74,150 @@ def _compute_geckoview_version(app_version, moz_build_date): # A task description is a general description of a TaskCluster task -task_description_schema = Schema( - { - # the label for this task - Required("label"): str, - # description of the task (for metadata) - Required("description"): str, - # attributes for this task - Optional("attributes"): {str: object}, - # relative path (from config.path) to the file task was defined in - Optional("task-from"): str, - # dependencies of this task, keyed by name; these are passed through - # verbatim and subject to the interpretation of the Task's get_dependencies - # method. - Optional("dependencies"): { - All( - str, - NotIn( - ["self", "decision"], - "Can't use 'self` or 'decision' as depdency names.", - ), - ): object, - }, - # Soft dependencies of this task, as a list of tasks labels - Optional("soft-dependencies"): [str], - # Dependencies that must be scheduled in order for this task to run. - Optional("if-dependencies"): [str], - Optional("requires"): Any("all-completed", "all-resolved"), - # expiration and deadline times, relative to task creation, with units - # (e.g., "14 days"). Defaults are set based on the project. - Optional("expires-after"): str, - Optional("deadline-after"): str, - Optional("expiration-policy"): str, - # custom routes for this task; the default treeherder routes will be added - # automatically - Optional("routes"): [str], - # custom scopes for this task; any scopes required for the worker will be - # added automatically. The following parameters will be substituted in each - # scope: - # {level} -- the scm level of this push - # {project} -- the project of this push - Optional("scopes"): [str], - # Tags - Optional("tags"): {str: str}, - # custom "task.extra" content - Optional("extra"): {str: object}, - # treeherder-related information; see - # https://firefox-ci-tc.services.mozilla.com/schemas/taskcluster-treeherder/v1/task-treeherder-config.json - # If not specified, no treeherder extra information or routes will be - # added to the task - Optional("treeherder"): { - # either a bare symbol, or "grp(sym)". - "symbol": str, - # the job kind - "kind": Any("build", "test", "other"), - # tier for this task - "tier": int, - # task platform, in the form platform/collection, used to set - # treeherder.machine.platform and treeherder.collection or - # treeherder.labels - "platform": Match("^[A-Za-z0-9_-]{1,50}/[A-Za-z0-9_-]{1,50}$"), - }, - # information for indexing this build so its artifacts can be discovered; - # if omitted, the build will not be indexed. - Optional("index"): { - # the name of the product this build produces - "product": str, - # the names to use for this job in the TaskCluster index - "job-name": str, - # Type of gecko v2 index to use - "type": Any( - "generic", - "l10n", - "shippable", - "shippable-l10n", - "android-shippable", - "android-shippable-with-multi-l10n", - "shippable-with-multi-l10n", - ), - # The rank that the task will receive in the TaskCluster - # index. A newly completed task supercedes the currently - # indexed task iff it has a higher rank. If unspecified, - # 'by-tier' behavior will be used. - "rank": Any( - # Rank is equal the timestamp of the build_date for tier-1 - # tasks, and one for non-tier-1. This sorts tier-{2,3} - # builds below tier-1 in the index, but above eager-index. - "by-tier", - # Rank is given as an integer constant (e.g. zero to make - # sure a task is last in the index). - int, - # Rank is equal to the timestamp of the build_date. This - # option can be used to override the 'by-tier' behavior - # for non-tier-1 tasks. - "build_date", +task_description_schema = Schema({ + # the label for this task + Required("label"): str, + # description of the task (for metadata) + Required("description"): str, + # attributes for this task + Optional("attributes"): {str: object}, + # relative path (from config.path) to the file task was defined in + Optional("task-from"): str, + # dependencies of this task, keyed by name; these are passed through + # verbatim and subject to the interpretation of the Task's get_dependencies + # method. + Optional("dependencies"): { + All( + str, + NotIn( + ["self", "decision"], + "Can't use 'self` or 'decision' as depdency names.", ), - }, - # The `run_on_repo_type` attribute, defaulting to "hg". This dictates - # the types of repositories on which this task should be included in - # the target task set. See the attributes documentation for details. - Optional("run-on-repo-type"): [Any("git", "hg")], - # The `run_on_projects` attribute, defaulting to "all". This dictates the - # projects on which this task should be included in the target task set. - # See the attributes documentation for details. - Optional("run-on-projects"): optionally_keyed_by("build-platform", [str]), - # Like `run_on_projects`, `run-on-hg-branches` defaults to "all". - Optional("run-on-hg-branches"): optionally_keyed_by("project", [str]), - # Specifies git branches for which this task should run. - Optional("run-on-git-branches"): [str], - # The `shipping_phase` attribute, defaulting to None. This specifies the - # release promotion phase that this task belongs to. - Required("shipping-phase"): Any( - None, - "build", - "promote", - "push", - "ship", + ): object, + }, + # Soft dependencies of this task, as a list of tasks labels + Optional("soft-dependencies"): [str], + # Dependencies that must be scheduled in order for this task to run. + Optional("if-dependencies"): [str], + Optional("requires"): Any("all-completed", "all-resolved"), + # expiration and deadline times, relative to task creation, with units + # (e.g., "14 days"). Defaults are set based on the project. + Optional("expires-after"): str, + Optional("deadline-after"): str, + Optional("expiration-policy"): str, + # custom routes for this task; the default treeherder routes will be added + # automatically + Optional("routes"): [str], + # custom scopes for this task; any scopes required for the worker will be + # added automatically. The following parameters will be substituted in each + # scope: + # {level} -- the scm level of this push + # {project} -- the project of this push + Optional("scopes"): [str], + # Tags + Optional("tags"): {str: str}, + # custom "task.extra" content + Optional("extra"): {str: object}, + # treeherder-related information; see + # https://firefox-ci-tc.services.mozilla.com/schemas/taskcluster-treeherder/v1/task-treeherder-config.json + # If not specified, no treeherder extra information or routes will be + # added to the task + Optional("treeherder"): { + # either a bare symbol, or "grp(sym)". + "symbol": str, + # the job kind + "kind": Any("build", "test", "other"), + # tier for this task + "tier": int, + # task platform, in the form platform/collection, used to set + # treeherder.machine.platform and treeherder.collection or + # treeherder.labels + "platform": Match("^[A-Za-z0-9_-]{1,50}/[A-Za-z0-9_-]{1,50}$"), + }, + # information for indexing this build so its artifacts can be discovered; + # if omitted, the build will not be indexed. + Optional("index"): { + # the name of the product this build produces + "product": str, + # the names to use for this job in the TaskCluster index + "job-name": str, + # Type of gecko v2 index to use + "type": Any( + "generic", + "l10n", + "shippable", + "shippable-l10n", + "android-shippable", + "android-shippable-with-multi-l10n", + "shippable-with-multi-l10n", ), - # The `shipping_product` attribute, defaulting to None. This specifies the - # release promotion product that this task belongs to. - Required("shipping-product"): Any(None, str), - # The `always-target` attribute will cause the task to be included in the - # target_task_graph regardless of filtering. Tasks included in this manner - # will be candidates for optimization even when `optimize_target_tasks` is - # False, unless the task was also explicitly chosen by the target_tasks - # method. - Required("always-target"): bool, - # Optimization to perform on this task during the optimization phase. - # Optimizations are defined in taskcluster/gecko_taskgraph/optimize.py. - Required("optimization"): OptimizationSchema, - # the provisioner-id/worker-type for the task. The following parameters will - # be substituted in this string: - # {level} -- the scm level of this push - "worker-type": str, - # Whether the job should use sccache compiler caching. - Required("use-sccache"): bool, - # information specific to the worker implementation that will run this task - Optional("worker"): { - Required("implementation"): str, - Extra: object, - }, - # Override the default priority for the project - Optional("priority"): str, - # Override the default 5 retries - Optional("retries"): int, - } -) + # The rank that the task will receive in the TaskCluster + # index. A newly completed task supercedes the currently + # indexed task iff it has a higher rank. If unspecified, + # 'by-tier' behavior will be used. + "rank": Any( + # Rank is equal the timestamp of the build_date for tier-1 + # tasks, and one for non-tier-1. This sorts tier-{2,3} + # builds below tier-1 in the index, but above eager-index. + "by-tier", + # Rank is given as an integer constant (e.g. zero to make + # sure a task is last in the index). + int, + # Rank is equal to the timestamp of the build_date. This + # option can be used to override the 'by-tier' behavior + # for non-tier-1 tasks. + "build_date", + ), + }, + # The `run_on_repo_type` attribute, defaulting to "hg". This dictates + # the types of repositories on which this task should be included in + # the target task set. See the attributes documentation for details. + Optional("run-on-repo-type"): [Any("git", "hg")], + # The `run_on_projects` attribute, defaulting to "all". This dictates the + # projects on which this task should be included in the target task set. + # See the attributes documentation for details. + Optional("run-on-projects"): optionally_keyed_by("build-platform", [str]), + # Like `run_on_projects`, `run-on-hg-branches` defaults to "all". + Optional("run-on-hg-branches"): optionally_keyed_by("project", [str]), + # Specifies git branches for which this task should run. + Optional("run-on-git-branches"): [str], + # The `shipping_phase` attribute, defaulting to None. This specifies the + # release promotion phase that this task belongs to. + Required("shipping-phase"): Any( + None, + "build", + "promote", + "push", + "ship", + ), + # The `shipping_product` attribute, defaulting to None. This specifies the + # release promotion product that this task belongs to. + Required("shipping-product"): Any(None, str), + # The `always-target` attribute will cause the task to be included in the + # target_task_graph regardless of filtering. Tasks included in this manner + # will be candidates for optimization even when `optimize_target_tasks` is + # False, unless the task was also explicitly chosen by the target_tasks + # method. + Required("always-target"): bool, + # Optimization to perform on this task during the optimization phase. + # Optimizations are defined in taskcluster/gecko_taskgraph/optimize.py. + Required("optimization"): OptimizationSchema, + # the provisioner-id/worker-type for the task. The following parameters will + # be substituted in this string: + # {level} -- the scm level of this push + "worker-type": str, + # Whether the job should use sccache compiler caching. + Required("use-sccache"): bool, + # information specific to the worker implementation that will run this task + Optional("worker"): { + Required("implementation"): str, + Extra: object, + }, + # Override the default priority for the project + Optional("priority"): str, + # Override the default 5 retries + Optional("retries"): int, +}) TC_TREEHERDER_SCHEMA_URL = ( "https://github.com/taskcluster/taskcluster-treeherder/" @@ -228,7 +226,7 @@ TC_TREEHERDER_SCHEMA_URL = ( UNKNOWN_GROUP_NAME = ( - "Treeherder group {} (from {}) has no name; " "add it to taskcluster/config.yml" + "Treeherder group {} (from {}) has no name; add it to taskcluster/config.yml" ) V2_ROUTE_TEMPLATES = [ @@ -657,7 +655,8 @@ def build_docker_worker_payload(config, task, task_def): # on Windows, each command is a string, on OS X and Linux, each command is # a string array Required("command"): Any( - [taskref_or_string], [[taskref_or_string]] # Windows # Linux / OS X + [taskref_or_string], + [[taskref_or_string]], # Windows # Linux / OS X ), # artifacts to extract from the task image after completion; note that artifacts # for the generic worker cannot have names @@ -819,12 +818,10 @@ def build_generic_worker_payload(config, task, task_def): if worker.get("os-groups"): task_def["payload"]["osGroups"] = worker["os-groups"] - task_def["scopes"].extend( - [ - "generic-worker:os-group:{}/{}".format(task["worker-type"], group) - for group in worker["os-groups"] - ] - ) + task_def["scopes"].extend([ + "generic-worker:os-group:{}/{}".format(task["worker-type"], group) + for group in worker["os-groups"] + ]) if worker.get("chain-of-trust"): features["chainOfTrust"] = True @@ -1123,12 +1120,10 @@ def build_balrog_payload(config, task, task_def): worker["balrog-action"] == "submit-locale" or worker["balrog-action"] == "v2-submit-locale" ): - task_def["payload"].update( - { - "upstreamArtifacts": worker["upstream-artifacts"], - "suffixes": worker["suffixes"], - } - ) + task_def["payload"].update({ + "upstreamArtifacts": worker["upstream-artifacts"], + "suffixes": worker["suffixes"], + }) else: for prop in ( "archive-domain", @@ -1151,13 +1146,11 @@ def build_balrog_payload(config, task, task_def): "beta-number": beta_number, }, ) - task_def["payload"].update( - { - "build_number": release_config["build_number"], - "product": worker["product"], - "version": release_config["version"], - } - ) + task_def["payload"].update({ + "build_number": release_config["build_number"], + "product": worker["product"], + "version": release_config["version"], + }) for prop in ( "blob-suffix", "complete-mar-filename-pattern", @@ -1170,29 +1163,25 @@ def build_balrog_payload(config, task, task_def): worker["balrog-action"] == "submit-toplevel" or worker["balrog-action"] == "v2-submit-toplevel" ): - task_def["payload"].update( - { - "app_version": release_config["appVersion"], - "archive_domain": worker["archive-domain"], - "channel_names": worker["channel-names"], - "download_domain": worker["download-domain"], - "partial_versions": release_config.get("partial_versions", ""), - "platforms": worker["platforms"], - "rules_to_update": worker["rules-to-update"], - "require_mirrors": worker["require-mirrors"], - "update_line": worker["update-line"], - } - ) + task_def["payload"].update({ + "app_version": release_config["appVersion"], + "archive_domain": worker["archive-domain"], + "channel_names": worker["channel-names"], + "download_domain": worker["download-domain"], + "partial_versions": release_config.get("partial_versions", ""), + "platforms": worker["platforms"], + "rules_to_update": worker["rules-to-update"], + "require_mirrors": worker["require-mirrors"], + "update_line": worker["update-line"], + }) else: # schedule / ship - task_def["payload"].update( - { - "publish_rules": worker["publish-rules"], - "release_eta": worker.get( - "release-eta", config.params.get("release_eta") - ) - or "", - } - ) + task_def["payload"].update({ + "publish_rules": worker["publish-rules"], + "release_eta": worker.get( + "release-eta", config.params.get("release_eta") + ) + or "", + }) if worker.get("force-fallback-mapping-update"): task_def["payload"]["force_fallback_mapping_update"] = worker[ "force-fallback-mapping-update" @@ -1429,11 +1418,9 @@ def build_treescript_payload(config, task, task_def): version = release_config["version"].replace(".", "_") buildnum = release_config["build_number"] if "buildN" in worker["tags"]: - tag_names.extend( - [ - f"{product}_{version}_BUILD{buildnum}", - ] - ) + tag_names.extend([ + f"{product}_{version}_BUILD{buildnum}", + ]) if "release" in worker["tags"]: tag_names.extend([f"{product}_{version}_RELEASE"]) tag_info = { @@ -1663,11 +1650,9 @@ def build_landoscript_payload(config, task, task_def): version = release_config["version"].replace(".", "_") buildnum = release_config["build_number"] if "buildN" in worker["tags"]: - tag_names.extend( - [ - f"{product}_{version}_BUILD{buildnum}", - ] - ) + tag_names.extend([ + f"{product}_{version}_BUILD{buildnum}", + ]) if "release" in worker["tags"]: tag_names.extend([f"{product}_{version}_RELEASE"]) tag_info = { @@ -2303,16 +2288,14 @@ def build_task(config, tasks): tags = task.get("tags", {}) attributes = task.get("attributes", {}) - tags.update( - { - "createdForUser": config.params["owner"], - "kind": config.kind, - "label": task["label"], - "retrigger": "true" if attributes.get("retrigger", False) else "false", - "project": config.params["project"], - "trust-domain": config.graph_config["trust-domain"], - } - ) + tags.update({ + "createdForUser": config.params["owner"], + "kind": config.kind, + "label": task["label"], + "retrigger": "true" if attributes.get("retrigger", False) else "false", + "project": config.params["project"], + "trust-domain": config.graph_config["trust-domain"], + }) task_def = { "provisionerId": provisioner_id, @@ -2405,15 +2388,13 @@ def build_task(config, tasks): payload = task_def.get("payload") if payload: env = payload.setdefault("env", {}) - env.update( - { - "MOZ_AUTOMATION": "1", - "MOZ_BUILD_DATE": config.params["moz_build_date"], - "MOZ_SCM_LEVEL": config.params["level"], - "MOZ_SOURCE_CHANGESET": get_branch_rev(config), - "MOZ_SOURCE_REPO": get_branch_repo(config), - } - ) + env.update({ + "MOZ_AUTOMATION": "1", + "MOZ_BUILD_DATE": config.params["moz_build_date"], + "MOZ_SCM_LEVEL": config.params["level"], + "MOZ_SOURCE_CHANGESET": get_branch_rev(config), + "MOZ_SOURCE_REPO": get_branch_repo(config), + }) dependencies = task.get("dependencies", {}) if_dependencies = task.get("if-dependencies", []) diff --git a/taskcluster/gecko_taskgraph/transforms/test/__init__.py b/taskcluster/gecko_taskgraph/transforms/test/__init__.py @@ -44,279 +44,275 @@ transforms = TransformSequence() # See the warnings in taskcluster/docs/how-tos.rst # # *****WARNING***** -test_description_schema = Schema( - { - # description of the suite, for the task metadata - Required("description"): str, - # test suite category and name - Optional("suite"): Any( - optionally_keyed_by("variant", str), - { - Optional("category"): str, - Optional("name"): optionally_keyed_by("variant", str), - }, - ), - # base work directory used to set up the task. - Optional("workdir"): optionally_keyed_by("test-platform", Any(str, "default")), - # the name by which this test suite is addressed in try syntax; defaults to - # the test-name. This will translate to the `unittest_try_name` or - # `talos_try_name` attribute. - Optional("try-name"): str, - # additional tags to mark up this type of test - Optional("tags"): {str: object}, - # the symbol, or group(symbol), under which this task should appear in - # treeherder. - Required("treeherder-symbol"): str, - # the value to place in task.extra.treeherder.machine.platform; ideally - # this is the same as build-platform, and that is the default, but in - # practice it's not always a match. - Optional("treeherder-machine-platform"): str, - # attributes to appear in the resulting task (later transforms will add the - # common attributes) - Optional("attributes"): {str: object}, - # relative path (from config.path) to the file task was defined in - Optional("task-from"): str, - # The `run_on_projects` attribute, defaulting to "all". This dictates the - # projects on which this task should be included in the target task set. - # See the attributes documentation for details. - # - # Note that the special case 'built-projects', the default, uses the parent - # build task's run-on-projects, meaning that tests run only on platforms - # that are built. - Optional("run-on-projects"): optionally_keyed_by( - "app", - "subtest", - "test-platform", - "test-name", - "variant", - Any([str], "built-projects"), - ), - # Whether tasks should run on only a specific type of repository. - Optional("run-on-repo-type"): job_description_schema["run-on-repo-type"], - # Whether tasks should run on specified Git branches. - Optional("run-on-git-branches"): job_description_schema["run-on-git-branches"], - # When set only run on projects where the build would already be running. - # This ensures tasks where this is True won't be the cause of the build - # running on a project it otherwise wouldn't have. - Optional("built-projects-only"): bool, - # the sheriffing tier for this task (default: set based on test platform) - Optional("tier"): optionally_keyed_by( - "test-platform", "variant", "app", "subtest", Any(int, "default") - ), - # number of chunks to create for this task. This can be keyed by test - # platform by passing a dictionary in the `by-test-platform` key. If the - # test platform is not found, the key 'default' will be tried. - Required("chunks"): optionally_keyed_by( - "test-platform", "variant", Any(int, "dynamic") - ), - # Timeout multiplier to apply to default test timeout values. Can be keyed - # by test platform. - Optional("timeoutfactor"): optionally_keyed_by( - "test-platform", Any(int, float) - ), - # Custom 'test_manifest_loader' to use, overriding the one configured in the - # parameters. When 'null', no test chunking will be performed. Can also - # be used to disable "manifest scheduling". - Optional("test-manifest-loader"): optionally_keyed_by( - "test-platform", Any(None, *list(manifest_loaders)) - ), - # the time (with unit) after which this task is deleted; default depends on - # the branch (see below) - Optional("expires-after"): str, - # The different configurations that should be run against this task, defined - # in the TEST_VARIANTS object in the variant.py transforms. - Optional("variants"): [str], - # Whether to run this task without any variants applied. - Required("run-without-variant"): optionally_keyed_by("test-platform", bool), - # The EC2 instance size to run these tests on. - Required("instance-size"): optionally_keyed_by( - "test-platform", - "variant", - Any( - "default", - "large-legacy", - "large", - "large-noscratch", - "xlarge", - "xlarge-noscratch", - "highcpu", - ), - ), - # type of virtualization or hardware required by test. - Required("virtualization"): optionally_keyed_by( - "test-platform", Any("virtual", "virtual-with-gpu", "hardware") - ), - # Whether the task requires loopback audio or video (whatever that may mean - # on the platform) - Required("loopback-audio"): bool, - Required("loopback-video"): bool, - # Whether the test can run using a software GL implementation on Linux - # using the GL compositor. May not be used with "legacy" sized instances - # due to poor LLVMPipe performance (bug 1296086). Defaults to true for - # unit tests on linux platforms and false otherwise - Optional("allow-software-gl-layers"): bool, - # For tasks that will run in docker-worker, this is the - # name of the docker image or in-tree docker image to run the task in. If - # in-tree, then a dependency will be created automatically. This is - # generally `desktop-test`, or an image that acts an awful lot like it. - Required("docker-image"): optionally_keyed_by( - "test-platform", - Any( - # a raw Docker image path (repo/image:tag) - str, - # an in-tree generated docker image (from `taskcluster/docker/<name>`) - {"in-tree": str}, - # an indexed docker image - {"indexed": str}, - ), - ), - # seconds of runtime after which the task will be killed. Like 'chunks', - # this can be keyed by test platform, but also variant. - Required("max-run-time"): optionally_keyed_by( - "test-platform", "subtest", "variant", "app", int - ), - # the exit status code that indicates the task should be retried - Optional("retry-exit-status"): [int], - # Whether to perform a gecko checkout. - Required("checkout"): bool, - # Wheter to perform a machine reboot after test is done - Optional("reboot"): Any(False, "always", "on-exception", "on-failure"), - # What to run - Required("mozharness"): { - # the mozharness script used to run this task - Required("script"): optionally_keyed_by("test-platform", str), - # the config files required for the task - Required("config"): optionally_keyed_by("test-platform", [str]), - # mochitest flavor for mochitest runs - Optional("mochitest-flavor"): str, - # any additional actions to pass to the mozharness command - Optional("actions"): [str], - # additional command-line options for mozharness, beyond those - # automatically added - Required("extra-options"): optionally_keyed_by( - "test-platform", "variant", "subtest", "app", [str] - ), - # the artifact name (including path) to test on the build task; this is - # generally set in a per-kind transformation - Optional("build-artifact-name"): str, - Optional("installer-url"): str, - # If not false, tooltool downloads will be enabled via relengAPIProxy - # for either just public files, or all files. Not supported on Windows - Required("tooltool-downloads"): Any( - False, - "public", - "internal", - ), - # Add --blob-upload-branch=<project> mozharness parameter - Optional("include-blob-upload-branch"): bool, - # The setting for --download-symbols (if omitted, the option will not - # be passed to mozharness) - Optional("download-symbols"): Any(True, "ondemand"), - # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the - # environment. This is more than just a helpful path setting -- it - # causes xpcshell tests to start additional servers, and runs - # additional tests. - Required("set-moz-node-path"): bool, - # If true, include chunking information in the command even if the number - # of chunks is 1 - Required("chunked"): optionally_keyed_by("test-platform", bool), - Required("requires-signed-builds"): optionally_keyed_by( - "test-platform", "variant", bool - ), +test_description_schema = Schema({ + # description of the suite, for the task metadata + Required("description"): str, + # test suite category and name + Optional("suite"): Any( + optionally_keyed_by("variant", str), + { + Optional("category"): str, + Optional("name"): optionally_keyed_by("variant", str), }, - # The set of test manifests to run. - Optional("test-manifests"): Any( - [str], - {"active": [str], "skipped": [str]}, + ), + # base work directory used to set up the task. + Optional("workdir"): optionally_keyed_by("test-platform", Any(str, "default")), + # the name by which this test suite is addressed in try syntax; defaults to + # the test-name. This will translate to the `unittest_try_name` or + # `talos_try_name` attribute. + Optional("try-name"): str, + # additional tags to mark up this type of test + Optional("tags"): {str: object}, + # the symbol, or group(symbol), under which this task should appear in + # treeherder. + Required("treeherder-symbol"): str, + # the value to place in task.extra.treeherder.machine.platform; ideally + # this is the same as build-platform, and that is the default, but in + # practice it's not always a match. + Optional("treeherder-machine-platform"): str, + # attributes to appear in the resulting task (later transforms will add the + # common attributes) + Optional("attributes"): {str: object}, + # relative path (from config.path) to the file task was defined in + Optional("task-from"): str, + # The `run_on_projects` attribute, defaulting to "all". This dictates the + # projects on which this task should be included in the target task set. + # See the attributes documentation for details. + # + # Note that the special case 'built-projects', the default, uses the parent + # build task's run-on-projects, meaning that tests run only on platforms + # that are built. + Optional("run-on-projects"): optionally_keyed_by( + "app", + "subtest", + "test-platform", + "test-name", + "variant", + Any([str], "built-projects"), + ), + # Whether tasks should run on only a specific type of repository. + Optional("run-on-repo-type"): job_description_schema["run-on-repo-type"], + # Whether tasks should run on specified Git branches. + Optional("run-on-git-branches"): job_description_schema["run-on-git-branches"], + # When set only run on projects where the build would already be running. + # This ensures tasks where this is True won't be the cause of the build + # running on a project it otherwise wouldn't have. + Optional("built-projects-only"): bool, + # the sheriffing tier for this task (default: set based on test platform) + Optional("tier"): optionally_keyed_by( + "test-platform", "variant", "app", "subtest", Any(int, "default") + ), + # number of chunks to create for this task. This can be keyed by test + # platform by passing a dictionary in the `by-test-platform` key. If the + # test platform is not found, the key 'default' will be tried. + Required("chunks"): optionally_keyed_by( + "test-platform", "variant", Any(int, "dynamic") + ), + # Timeout multiplier to apply to default test timeout values. Can be keyed + # by test platform. + Optional("timeoutfactor"): optionally_keyed_by("test-platform", Any(int, float)), + # Custom 'test_manifest_loader' to use, overriding the one configured in the + # parameters. When 'null', no test chunking will be performed. Can also + # be used to disable "manifest scheduling". + Optional("test-manifest-loader"): optionally_keyed_by( + "test-platform", Any(None, *list(manifest_loaders)) + ), + # the time (with unit) after which this task is deleted; default depends on + # the branch (see below) + Optional("expires-after"): str, + # The different configurations that should be run against this task, defined + # in the TEST_VARIANTS object in the variant.py transforms. + Optional("variants"): [str], + # Whether to run this task without any variants applied. + Required("run-without-variant"): optionally_keyed_by("test-platform", bool), + # The EC2 instance size to run these tests on. + Required("instance-size"): optionally_keyed_by( + "test-platform", + "variant", + Any( + "default", + "large-legacy", + "large", + "large-noscratch", + "xlarge", + "xlarge-noscratch", + "highcpu", ), - # flag to determine if this is a confirm failure task - Optional("confirm-failure"): bool, - # The current chunk (if chunking is enabled). - Optional("this-chunk"): int, - # os user groups for test task workers; required scopes, will be - # added automatically - Optional("os-groups"): optionally_keyed_by("test-platform", [str]), - Optional("run-as-administrator"): optionally_keyed_by("test-platform", bool), - # -- values supplied by the task-generation infrastructure - # the platform of the build this task is testing - Required("build-platform"): str, - # the label of the build task generating the materials to test - Required("build-label"): str, - # the label of the signing task generating the materials to test. - # Signed builds are used in xpcshell tests on Windows, for instance. - Optional("build-signing-label"): optionally_keyed_by("variant", str), - # the build's attributes - Required("build-attributes"): {str: object}, - # the platform on which the tests will run - Required("test-platform"): str, - # limit the test-platforms (as defined in test-platforms.yml) - # that the test will run on - Optional("limit-platforms"): optionally_keyed_by("app", "subtest", [str]), - # the name of the test (the key in tests.yml) - Required("test-name"): str, - # the product name, defaults to firefox - Optional("product"): str, - # conditional files to determine when these tests should be run - Exclusive("when", "optimization"): { - Optional("files-changed"): [str], - }, - # Optimization to perform on this task during the optimization phase. - # Optimizations are defined in taskcluster/gecko_taskgraph/optimize.py. - Exclusive("optimization", "optimization"): OptimizationSchema, - # The SCHEDULES component for this task; this defaults to the suite - # (not including the flavor) but can be overridden here. - Exclusive("schedules-component", "optimization"): Any( + ), + # type of virtualization or hardware required by test. + Required("virtualization"): optionally_keyed_by( + "test-platform", Any("virtual", "virtual-with-gpu", "hardware") + ), + # Whether the task requires loopback audio or video (whatever that may mean + # on the platform) + Required("loopback-audio"): bool, + Required("loopback-video"): bool, + # Whether the test can run using a software GL implementation on Linux + # using the GL compositor. May not be used with "legacy" sized instances + # due to poor LLVMPipe performance (bug 1296086). Defaults to true for + # unit tests on linux platforms and false otherwise + Optional("allow-software-gl-layers"): bool, + # For tasks that will run in docker-worker, this is the + # name of the docker image or in-tree docker image to run the task in. If + # in-tree, then a dependency will be created automatically. This is + # generally `desktop-test`, or an image that acts an awful lot like it. + Required("docker-image"): optionally_keyed_by( + "test-platform", + Any( + # a raw Docker image path (repo/image:tag) str, - [str], + # an in-tree generated docker image (from `taskcluster/docker/<name>`) + {"in-tree": str}, + # an indexed docker image + {"indexed": str}, ), - Optional("worker-type"): optionally_keyed_by( - "test-platform", - "variant", - Any(str, None), + ), + # seconds of runtime after which the task will be killed. Like 'chunks', + # this can be keyed by test platform, but also variant. + Required("max-run-time"): optionally_keyed_by( + "test-platform", "subtest", "variant", "app", int + ), + # the exit status code that indicates the task should be retried + Optional("retry-exit-status"): [int], + # Whether to perform a gecko checkout. + Required("checkout"): bool, + # Wheter to perform a machine reboot after test is done + Optional("reboot"): Any(False, "always", "on-exception", "on-failure"), + # What to run + Required("mozharness"): { + # the mozharness script used to run this task + Required("script"): optionally_keyed_by("test-platform", str), + # the config files required for the task + Required("config"): optionally_keyed_by("test-platform", [str]), + # mochitest flavor for mochitest runs + Optional("mochitest-flavor"): str, + # any additional actions to pass to the mozharness command + Optional("actions"): [str], + # additional command-line options for mozharness, beyond those + # automatically added + Required("extra-options"): optionally_keyed_by( + "test-platform", "variant", "subtest", "app", [str] ), - Optional( - "require-signed-extensions", - description="Whether the build being tested requires extensions be signed.", - ): optionally_keyed_by("release-type", "test-platform", bool), - # The target name, specifying the build artifact to be tested. - # If None or not specified, a transform sets the target based on OS: - # target.dmg (Mac), target.apk (Android), target.tar.xz (Linux), - # or target.zip (Windows). - Optional("target"): optionally_keyed_by( - "app", - "test-platform", - "variant", - Any( - str, - None, - {Required("index"): str, Required("name"): str}, - {Required("upstream-task"): str, Required("name"): str}, - ), + # the artifact name (including path) to test on the build task; this is + # generally set in a per-kind transformation + Optional("build-artifact-name"): str, + Optional("installer-url"): str, + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Not supported on Windows + Required("tooltool-downloads"): Any( + False, + "public", + "internal", ), - # A list of artifacts to install from 'fetch' tasks. Validation deferred - # to 'job' transforms. - Optional("fetches"): object, - # A list of extra dependencies - Optional("dependencies"): object, - # Raptor / browsertime specific keys, defer validation to 'raptor.py' - # transform. - Optional("raptor"): object, - # Raptor / browsertime specific keys that need to be here since 'raptor' schema - # is evluated *before* test_description_schema - Optional("app"): str, - Optional("subtest"): str, - # Define if a given task supports artifact builds or not, see bug 1695325. - Optional("supports-artifact-builds"): bool, - # Version of python used to run the task - Optional("use-python"): job_description_schema["use-python"], - # Fetch uv binary and add it to PATH - Optional("use-uv"): bool, - # Cache mounts / volumes to set up - Optional("use-caches"): optionally_keyed_by( - "test-platform", run_task_schema["use-caches"] + # Add --blob-upload-branch=<project> mozharness parameter + Optional("include-blob-upload-branch"): bool, + # The setting for --download-symbols (if omitted, the option will not + # be passed to mozharness) + Optional("download-symbols"): Any(True, "ondemand"), + # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the + # environment. This is more than just a helpful path setting -- it + # causes xpcshell tests to start additional servers, and runs + # additional tests. + Required("set-moz-node-path"): bool, + # If true, include chunking information in the command even if the number + # of chunks is 1 + Required("chunked"): optionally_keyed_by("test-platform", bool), + Required("requires-signed-builds"): optionally_keyed_by( + "test-platform", "variant", bool + ), + }, + # The set of test manifests to run. + Optional("test-manifests"): Any( + [str], + {"active": [str], "skipped": [str]}, + ), + # flag to determine if this is a confirm failure task + Optional("confirm-failure"): bool, + # The current chunk (if chunking is enabled). + Optional("this-chunk"): int, + # os user groups for test task workers; required scopes, will be + # added automatically + Optional("os-groups"): optionally_keyed_by("test-platform", [str]), + Optional("run-as-administrator"): optionally_keyed_by("test-platform", bool), + # -- values supplied by the task-generation infrastructure + # the platform of the build this task is testing + Required("build-platform"): str, + # the label of the build task generating the materials to test + Required("build-label"): str, + # the label of the signing task generating the materials to test. + # Signed builds are used in xpcshell tests on Windows, for instance. + Optional("build-signing-label"): optionally_keyed_by("variant", str), + # the build's attributes + Required("build-attributes"): {str: object}, + # the platform on which the tests will run + Required("test-platform"): str, + # limit the test-platforms (as defined in test-platforms.yml) + # that the test will run on + Optional("limit-platforms"): optionally_keyed_by("app", "subtest", [str]), + # the name of the test (the key in tests.yml) + Required("test-name"): str, + # the product name, defaults to firefox + Optional("product"): str, + # conditional files to determine when these tests should be run + Exclusive("when", "optimization"): { + Optional("files-changed"): [str], + }, + # Optimization to perform on this task during the optimization phase. + # Optimizations are defined in taskcluster/gecko_taskgraph/optimize.py. + Exclusive("optimization", "optimization"): OptimizationSchema, + # The SCHEDULES component for this task; this defaults to the suite + # (not including the flavor) but can be overridden here. + Exclusive("schedules-component", "optimization"): Any( + str, + [str], + ), + Optional("worker-type"): optionally_keyed_by( + "test-platform", + "variant", + Any(str, None), + ), + Optional( + "require-signed-extensions", + description="Whether the build being tested requires extensions be signed.", + ): optionally_keyed_by("release-type", "test-platform", bool), + # The target name, specifying the build artifact to be tested. + # If None or not specified, a transform sets the target based on OS: + # target.dmg (Mac), target.apk (Android), target.tar.xz (Linux), + # or target.zip (Windows). + Optional("target"): optionally_keyed_by( + "app", + "test-platform", + "variant", + Any( + str, + None, + {Required("index"): str, Required("name"): str}, + {Required("upstream-task"): str, Required("name"): str}, ), - } -) + ), + # A list of artifacts to install from 'fetch' tasks. Validation deferred + # to 'job' transforms. + Optional("fetches"): object, + # A list of extra dependencies + Optional("dependencies"): object, + # Raptor / browsertime specific keys, defer validation to 'raptor.py' + # transform. + Optional("raptor"): object, + # Raptor / browsertime specific keys that need to be here since 'raptor' schema + # is evluated *before* test_description_schema + Optional("app"): str, + Optional("subtest"): str, + # Define if a given task supports artifact builds or not, see bug 1695325. + Optional("supports-artifact-builds"): bool, + # Version of python used to run the task + Optional("use-python"): job_description_schema["use-python"], + # Fetch uv binary and add it to PATH + Optional("use-uv"): bool, + # Cache mounts / volumes to set up + Optional("use-caches"): optionally_keyed_by( + "test-platform", run_task_schema["use-caches"] + ), +}) @transforms.add @@ -513,16 +509,14 @@ def make_job_description(config, tasks): attr_try_name = "unittest_try_name" attr_build_platform, attr_build_type = task["build-platform"].split("/", 1) - attributes.update( - { - "build_platform": attr_build_platform, - "build_type": attr_build_type, - "test_platform": task["test-platform"], - "test_chunk": str(task["this-chunk"]), - "supports-artifact-builds": task["supports-artifact-builds"], - attr_try_name: try_name, - } - ) + attributes.update({ + "build_platform": attr_build_platform, + "build_type": attr_build_type, + "test_platform": task["test-platform"], + "test_chunk": str(task["this-chunk"]), + "supports-artifact-builds": task["supports-artifact-builds"], + attr_try_name: try_name, + }) if "test-manifests" in task: attributes["test_manifests"] = task["test-manifests"] diff --git a/taskcluster/gecko_taskgraph/transforms/test/chunk.py b/taskcluster/gecko_taskgraph/transforms/test/chunk.py @@ -283,9 +283,9 @@ def split_chunks(config, tasks): and manifests["active"] and "skipped" in manifests ): - chunked_manifests[0].extend( - [m for m in manifests["skipped"] if not m.endswith(".list")] - ) + chunked_manifests[0].extend([ + m for m in manifests["skipped"] if not m.endswith(".list") + ]) last_chunk = task["chunks"] for i in range(task["chunks"]): this_chunk = i + 1 diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py @@ -313,9 +313,9 @@ def set_target(config, tasks): ) task["mozharness"]["installer-url"] = installer_url else: - task["mozharness"][ - "installer-url" - ] = f"<{target['upstream-task']}/{target['name']}>" + task["mozharness"]["installer-url"] = ( + f"<{target['upstream-task']}/{target['name']}>" + ) else: task["mozharness"]["build-artifact-name"] = get_artifact_path(task, target) @@ -458,12 +458,10 @@ def setup_browsertime(config, tasks): evaluate_keyed_by(fs, "fetches.fetch", task) ) - extra_options.extend( - ( - "--browsertime-browsertimejs", - "$MOZ_FETCHES_DIR/browsertime/node_modules/browsertime/bin/browsertime.js", - ) - ) # noqa: E501 + extra_options.extend(( + "--browsertime-browsertimejs", + "$MOZ_FETCHES_DIR/browsertime/node_modules/browsertime/bin/browsertime.js", + )) # noqa: E501 eos = { "by-test-platform": { @@ -1080,11 +1078,9 @@ def add_gecko_profile_symbolication_deps(config, tasks): startup_profile = env.get("MOZ_PROFILER_STARTUP") == "1" for task in tasks: - if (gecko_profile and task["suite"] in ["talos", "raptor"]) or ( startup_profile and "mochitest" in task["suite"] ): - fetches = task.setdefault("fetches", {}) fetch_toolchains = fetches.setdefault("toolchain", []) fetch_toolchains.append("symbolicator-cli") diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py @@ -15,55 +15,49 @@ from gecko_taskgraph.util.perftest import is_external_browser transforms = TransformSequence() task_transforms = TransformSequence() -raptor_description_schema = Schema( - { - # Raptor specific configs. - Optional("raptor"): { - Optional("activity"): optionally_keyed_by("app", str), - Optional("apps"): optionally_keyed_by("test-platform", "subtest", [str]), - Optional("binary-path"): optionally_keyed_by("app", str), - Optional("run-visual-metrics"): optionally_keyed_by( - "app", "test-platform", bool - ), - Optional("subtests"): optionally_keyed_by("app", "test-platform", list), - Optional("test"): str, - Optional("test-url-param"): optionally_keyed_by( - "subtest", "test-platform", str - ), - Optional("lull-schedule"): optionally_keyed_by( - "subtest", "test-platform", str - ), - Optional("network-conditions"): optionally_keyed_by("subtest", list), - }, - # Configs defined in the 'test_description_schema'. - Optional("max-run-time"): optionally_keyed_by( - "app", "subtest", "test-platform", test_description_schema["max-run-time"] +raptor_description_schema = Schema({ + # Raptor specific configs. + Optional("raptor"): { + Optional("activity"): optionally_keyed_by("app", str), + Optional("apps"): optionally_keyed_by("test-platform", "subtest", [str]), + Optional("binary-path"): optionally_keyed_by("app", str), + Optional("run-visual-metrics"): optionally_keyed_by( + "app", "test-platform", bool ), - Optional("run-on-projects"): optionally_keyed_by( - "app", - "test-name", - "raptor.test", - "subtest", - "variant", - test_description_schema["run-on-projects"], + Optional("subtests"): optionally_keyed_by("app", "test-platform", list), + Optional("test"): str, + Optional("test-url-param"): optionally_keyed_by( + "subtest", "test-platform", str ), - Optional("variants"): test_description_schema["variants"], - Optional("target"): optionally_keyed_by( - "app", test_description_schema["target"] - ), - Optional("tier"): optionally_keyed_by( - "app", "raptor.test", "subtest", "variant", test_description_schema["tier"] - ), - Required("test-name"): test_description_schema["test-name"], - Required("test-platform"): test_description_schema["test-platform"], - Required("require-signed-extensions"): test_description_schema[ - "require-signed-extensions" - ], - Required("treeherder-symbol"): test_description_schema["treeherder-symbol"], - # Any unrecognized keys will be validated against the test_description_schema. - Extra: object, - } -) + Optional("lull-schedule"): optionally_keyed_by("subtest", "test-platform", str), + Optional("network-conditions"): optionally_keyed_by("subtest", list), + }, + # Configs defined in the 'test_description_schema'. + Optional("max-run-time"): optionally_keyed_by( + "app", "subtest", "test-platform", test_description_schema["max-run-time"] + ), + Optional("run-on-projects"): optionally_keyed_by( + "app", + "test-name", + "raptor.test", + "subtest", + "variant", + test_description_schema["run-on-projects"], + ), + Optional("variants"): test_description_schema["variants"], + Optional("target"): optionally_keyed_by("app", test_description_schema["target"]), + Optional("tier"): optionally_keyed_by( + "app", "raptor.test", "subtest", "variant", test_description_schema["tier"] + ), + Required("test-name"): test_description_schema["test-name"], + Required("test-platform"): test_description_schema["test-platform"], + Required("require-signed-extensions"): test_description_schema[ + "require-signed-extensions" + ], + Required("treeherder-symbol"): test_description_schema["treeherder-symbol"], + # Any unrecognized keys will be validated against the test_description_schema. + Extra: object, +}) transforms.add_validate(raptor_description_schema) @@ -211,12 +205,10 @@ def handle_network_conditions(config, tests): mozharness = new_test.setdefault("mozharness", {}) extra_options = mozharness.setdefault("extra-options", []) - extra_options.extend( - [ - f"--browsertime-arg=network_type={network_type}", - f"--browsertime-arg=pkt_loss_rate={packet_loss_rate}", - ] - ) + extra_options.extend([ + f"--browsertime-arg=network_type={network_type}", + f"--browsertime-arg=pkt_loss_rate={packet_loss_rate}", + ]) new_test["test-name"] += f"-{subtest}-{network_type}-{packet_loss_rate}" new_test["try-name"] += f"-{subtest}-{network_type}-{packet_loss_rate}" @@ -468,21 +460,17 @@ def setup_internal_artifacts(config, tasks): task["worker"]["os"] == "linux-bitbar" or task["worker"]["os"] == "linux-lambda" ): - task["worker"].setdefault("artifacts", []).append( - { - "name": "perftest", - "path": "workspace/build/perftest", - "type": "directory", - } - ) + task["worker"].setdefault("artifacts", []).append({ + "name": "perftest", + "path": "workspace/build/perftest", + "type": "directory", + }) else: - task["worker"].setdefault("artifacts", []).append( - { - "name": "perftest", - "path": "build/perftest", - "type": "directory", - } - ) + task["worker"].setdefault("artifacts", []).append({ + "name": "perftest", + "path": "build/perftest", + "type": "directory", + }) yield task @@ -514,22 +502,20 @@ def select_tasks_to_lambda(config, tasks): task["tags"]["os"] = "linux-lambda" task["worker"]["os"] = "linux-lambda" task["worker-type"] = "t-lambda-perf-a55" - task["worker"]["env"][ - "TASKCLUSTER_WORKER_TYPE" - ] = "t-lambda-perf-a55" + task["worker"]["env"]["TASKCLUSTER_WORKER_TYPE"] = ( + "t-lambda-perf-a55" + ) cmds = [] for cmd in task["worker"]["command"]: # Bug 1981862 - issues with condprof setup @ lambdatest - cmds.append( - [ - c.replace( - "/builds/taskcluster/script.py", - "/home/ltuser/taskcluster/script.py", - ) - for c in cmd - if not c.startswith("--conditioned-profile") - ] - ) + cmds.append([ + c.replace( + "/builds/taskcluster/script.py", + "/home/ltuser/taskcluster/script.py", + ) + for c in cmd + if not c.startswith("--conditioned-profile") + ]) task["worker"]["command"] = cmds task["worker"]["env"]["DISABLE_USB_POWER_METER_RESET"] = "1" yield task diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py @@ -18,20 +18,18 @@ transforms = TransformSequence() """List of available test variants defined.""" -variant_description_schema = Schema( - { - str: { - Required("description"): str, - Required("suffix"): str, - Optional("mozinfo"): str, - Required("component"): str, - Required("expiration"): str, - Optional("when"): {Any("$eval", "$if"): str}, - Optional("replace"): {str: object}, - Optional("merge"): {str: object}, - } +variant_description_schema = Schema({ + str: { + Required("description"): str, + Required("suffix"): str, + Optional("mozinfo"): str, + Required("component"): str, + Required("expiration"): str, + Optional("when"): {Any("$eval", "$if"): str}, + Optional("replace"): {str: object}, + Optional("merge"): {str: object}, } -) +}) """variant description schema""" diff --git a/taskcluster/gecko_taskgraph/transforms/test_apk.py b/taskcluster/gecko_taskgraph/transforms/test_apk.py @@ -6,7 +6,6 @@ Apply some defaults and minor modifications to the jobs defined in the test kinds. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.schema import resolve_keyed_by @@ -30,6 +29,6 @@ def resolve_keys(config, tasks): "shipping-product": task.get("attributes", {}).get( "shipping-product" ), - } + }, ) yield task diff --git a/taskcluster/gecko_taskgraph/transforms/update_test.py b/taskcluster/gecko_taskgraph/transforms/update_test.py @@ -174,9 +174,9 @@ def parametrize_by_locale(config, tasks): f"{this_task['description']}, locale coverage: {locale}" ) this_task["name"] = this_task["name"].replace("locale", locale) - this_task["index"][ - "job-name" - ] = f"{this_task['index']['job-name']}-{locale}" + this_task["index"]["job-name"] = ( + f"{this_task['index']['job-name']}-{locale}" + ) this_task["treeherder"]["symbol"] = infix_treeherder_symbol( this_task["treeherder"]["symbol"], locale.replace("-", "") ) diff --git a/taskcluster/gecko_taskgraph/transforms/update_verify.py b/taskcluster/gecko_taskgraph/transforms/update_verify.py @@ -5,7 +5,6 @@ Transform the beetmover task into an actual task description. """ - from taskgraph.transforms.base import TransformSequence from taskgraph.util.copy import deepcopy from taskgraph.util.treeherder import add_suffix, inherit_treeherder_from_dep diff --git a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py @@ -143,12 +143,10 @@ def add_command(config, tasks): command.append(f"--{arg}") command.append(task["extra"][arg]) - task["run"].update( - { - "using": "mach", - "mach": " ".join(command), - } - ) + task["run"].update({ + "using": "mach", + "mach": " ".join(command), + }) if task.get("index"): task["index"].setdefault( diff --git a/taskcluster/gecko_taskgraph/transforms/upload_symbols.py b/taskcluster/gecko_taskgraph/transforms/upload_symbols.py @@ -6,7 +6,6 @@ Transform the upload-symbols task description template, taskcluster/kinds/upload-symbols/job-template.yml into an actual task description. """ - import logging from taskgraph.transforms.base import TransformSequence diff --git a/taskcluster/gecko_taskgraph/util/attributes.py b/taskcluster/gecko_taskgraph/util/attributes.py @@ -57,12 +57,10 @@ RELEASE_PROMOTION_PROJECTS = { "try-comm-central", } | RELEASE_PROJECTS -TEMPORARY_PROJECTS = set( - { - # When using a "Disposable Project Branch" you can specify your branch here. e.g.: - "oak", - } -) +TEMPORARY_PROJECTS = set({ + # When using a "Disposable Project Branch" you can specify your branch here. e.g.: + "oak", +}) TRY_PROJECTS = { "staging-firefox", # https://github.com/mozilla-releng/staging-firefox diff --git a/taskcluster/gecko_taskgraph/util/cached_tasks.py b/taskcluster/gecko_taskgraph/util/cached_tasks.py @@ -67,9 +67,9 @@ def add_optimization( subs["build_date_long"] = time.strftime( "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) ) - taskdesc["routes"].extend( - [f"index.{route.format(**subs)}" for route in EXTRA_CACHE_INDEXES] - ) + taskdesc["routes"].extend([ + f"index.{route.format(**subs)}" for route in EXTRA_CACHE_INDEXES + ]) taskdesc["attributes"]["cached_task"] = { "type": cache_type, diff --git a/taskcluster/gecko_taskgraph/util/chunking.py b/taskcluster/gecko_taskgraph/util/chunking.py @@ -260,7 +260,6 @@ class DefaultLoader(BaseManifestLoader): if any(x in manifest for x in subsuite_paths): manifests.add(manifest) else: - all_subsuite_paths = [ path for paths in WPT_SUBSUITES.values() for path in paths ] diff --git a/taskcluster/gecko_taskgraph/util/docker.py b/taskcluster/gecko_taskgraph/util/docker.py @@ -21,9 +21,14 @@ IMAGE_DIR = os.path.join(GECKO, "taskcluster", "docker") def docker_url(path, **kwargs): docker_socket = os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") - return urlunparse( - ("http+unix", quote(docker_socket, safe=""), path, "", urlencode(kwargs), "") - ) + return urlunparse(( + "http+unix", + quote(docker_socket, safe=""), + path, + "", + urlencode(kwargs), + "", + )) def post_to_docker(tar, api_path, **kwargs): @@ -141,12 +146,9 @@ class ImagePathsMap(Mapping): return len(self.__dict__) def __update_image_paths(self, jobs, image_dir): - self.__dict__.update( - { - k: os.path.join(image_dir, v.get("definition", k)) - for k, v in jobs.items() - } - ) + self.__dict__.update({ + k: os.path.join(image_dir, v.get("definition", k)) for k, v in jobs.items() + }) def register(self, jobs_config_path, image_dir): """Register additional image_paths. In this case, there is no 'jobs' @@ -181,8 +183,7 @@ def parse_volumes(image): v = line.split(None, 1)[1] if v.startswith(b"["): raise ValueError( - "cannot parse array syntax for VOLUME; " - "convert to multiple entries" + "cannot parse array syntax for VOLUME; convert to multiple entries" ) volumes |= {v.decode("utf-8") for v in v.split()} diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py @@ -542,9 +542,10 @@ def apply_partner_priority(config, jobs): # integration branches because we don't want to wait a lot for the graph to be done, but # for multiple releases the partner tasks always wait for non-partner. if ( - config.kind.startswith( - ("release-partner-repack", "release-partner-attribution") - ) + config.kind.startswith(( + "release-partner-repack", + "release-partner-attribution", + )) and release_level(config.params) == "production" ): priority = "medium" @@ -582,16 +583,14 @@ def build_macos_attribution_dmg_command(dmg_app_path, attributions): command.append(create_dir_command) command.append( - " ".join( - [ - dmg_app_path, - "attribute", - a["input"], - a["output"], - MACOS_ATTRIBUTION_SENTINEL, - _build_macos_attribution_string(attribution_code=a["attribution"]), - ] - ) + " ".join([ + dmg_app_path, + "attribute", + a["input"], + a["output"], + MACOS_ATTRIBUTION_SENTINEL, + _build_macos_attribution_string(attribution_code=a["attribution"]), + ]) ) return " && ".join(command) diff --git a/taskcluster/gecko_taskgraph/util/scriptworker.py b/taskcluster/gecko_taskgraph/util/scriptworker.py @@ -15,6 +15,7 @@ happen on mozilla-beta and mozilla-release. Additional configuration is found in the :ref:`graph config <taskgraph-graph-config>`. """ + import functools import itertools import json @@ -457,12 +458,10 @@ def get_release_config(config): "release-secondary-balrog-submit-toplevel", ): partial_updates = json.loads(partial_updates) - release_config["partial_versions"] = ", ".join( - [ - "{}build{}".format(v, info["buildNumber"]) - for v, info in partial_updates.items() - ] - ) + release_config["partial_versions"] = ", ".join([ + "{}build{}".format(v, info["buildNumber"]) + for v, info in partial_updates.items() + ]) if release_config["partial_versions"] == "{}": del release_config["partial_versions"] @@ -593,14 +592,12 @@ def generate_beetmover_upstream_artifacts( if not paths: continue - upstream_artifacts.append( - { - "taskId": {"task-reference": f"<{dep}>"}, - "taskType": map_config["tasktype_map"].get(dep), - "paths": sorted(paths), - "locale": current_locale, - } - ) + upstream_artifacts.append({ + "taskId": {"task-reference": f"<{dep}>"}, + "taskType": map_config["tasktype_map"].get(dep), + "paths": sorted(paths), + "locale": current_locale, + }) upstream_artifacts.sort(key=lambda u: u["paths"]) return upstream_artifacts @@ -788,28 +785,24 @@ def generate_beetmover_artifact_map(config, job, **kwargs): upload_date = datetime.fromtimestamp(config.params["build_date"]) - kwargs.update( - { - "locale": locale, - "version": config.params["version"], - "branch": config.params["project"], - "build_number": config.params["build_number"], - "year": upload_date.year, - "month": upload_date.strftime("%m"), # zero-pad the month - "day": upload_date.strftime("%d"), - "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), - "head_rev": config.params["head_rev"], - } - ) + kwargs.update({ + "locale": locale, + "version": config.params["version"], + "branch": config.params["project"], + "build_number": config.params["build_number"], + "year": upload_date.year, + "month": upload_date.strftime("%m"), # zero-pad the month + "day": upload_date.strftime("%d"), + "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), + "head_rev": config.params["head_rev"], + }) kwargs.update(**platforms) paths = jsone.render(paths, kwargs) - artifacts.append( - { - "taskId": {"task-reference": f"<{dep}>"}, - "locale": locale, - "paths": paths, - } - ) + artifacts.append({ + "taskId": {"task-reference": f"<{dep}>"}, + "locale": locale, + "paths": paths, + }) return artifacts @@ -942,34 +935,30 @@ def generate_beetmover_partials_artifact_map(config, job, partials_info, **kwarg } # render buildid - kwargs.update( - { - "partial": pname, - "from_buildid": info["buildid"], - "previous_version": info.get("previousVersion"), - "buildid": str(config.params["moz_build_date"]), - "locale": locale, - "version": config.params["version"], - "branch": config.params["project"], - "build_number": config.params["build_number"], - "year": upload_date.year, - "month": upload_date.strftime("%m"), # zero-pad the month - "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), - } - ) + kwargs.update({ + "partial": pname, + "from_buildid": info["buildid"], + "previous_version": info.get("previousVersion"), + "buildid": str(config.params["moz_build_date"]), + "locale": locale, + "version": config.params["version"], + "branch": config.params["project"], + "build_number": config.params["build_number"], + "year": upload_date.year, + "month": upload_date.strftime("%m"), # zero-pad the month + "upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"), + }) kwargs.update(**platforms) paths.update(jsone.render(partials_paths, kwargs)) if not paths: continue - artifacts.append( - { - "taskId": {"task-reference": f"<{dep}>"}, - "locale": locale, - "paths": paths, - } - ) + artifacts.append({ + "taskId": {"task-reference": f"<{dep}>"}, + "locale": locale, + "paths": paths, + }) artifacts.sort(key=lambda a: sorted(a["paths"].items())) return artifacts diff --git a/taskcluster/gecko_taskgraph/util/verify.py b/taskcluster/gecko_taskgraph/util/verify.py @@ -397,14 +397,12 @@ def verify_test_packaging(task, taskgraph, scratch_pad, graph_config, parameters if task is None: # In certain cases there are valid reasons for tests to be missing, # don't error out when that happens. - missing_tests_allowed = any( - ( - # user specified `--target-kind` - bool(parameters.get("target-kinds")), - # manifest scheduling is enabled - parameters["test_manifest_loader"] != "default", - ) - ) + missing_tests_allowed = any(( + # user specified `--target-kind` + bool(parameters.get("target-kinds")), + # manifest scheduling is enabled + parameters["test_manifest_loader"] != "default", + )) test_env = parameters["try_task_config"].get("env", {}) if test_env.get("MOZHARNESS_TEST_PATHS", "") or test_env.get( diff --git a/taskcluster/gecko_taskgraph/util/workertypes.py b/taskcluster/gecko_taskgraph/util/workertypes.py @@ -53,13 +53,11 @@ def _get(graph_config, alias, level, release_level, project): worker_config["provisioner"], f"worker-type alias {alias} field provisioner", {"level": level}, - ).format( - **{ - "trust-domain": graph_config["trust-domain"], - "level": level, - "alias": alias, - } - ) + ).format(**{ + "trust-domain": graph_config["trust-domain"], + "level": level, + "alias": alias, + }) attrs = {"level": level, "release-level": release_level} if project: attrs["project"] = project @@ -67,13 +65,11 @@ def _get(graph_config, alias, level, release_level, project): worker_config["worker-type"], f"worker-type alias {alias} field worker-type", attrs, - ).format( - **{ - "trust-domain": graph_config["trust-domain"], - "level": level, - "alias": alias, - } - ) + ).format(**{ + "trust-domain": graph_config["trust-domain"], + "level": level, + "alias": alias, + }) return worker_config diff --git a/taskcluster/scripts/are_dependencies_completed.py b/taskcluster/scripts/are_dependencies_completed.py @@ -10,11 +10,9 @@ import sys import taskcluster -queue = taskcluster.Queue( - { - "rootUrl": os.environ.get("TASKCLUSTER_PROXY_URL", "https://taskcluster.net"), - } -) +queue = taskcluster.Queue({ + "rootUrl": os.environ.get("TASKCLUSTER_PROXY_URL", "https://taskcluster.net"), +}) def check_all_dependencies_are_completed(current_task_id): diff --git a/taskcluster/scripts/get-secret.py b/taskcluster/scripts/get-secret.py @@ -39,21 +39,17 @@ def write_secret_to_file( def fetch_secret_from_taskcluster(name): try: - secrets = taskcluster.Secrets( - { - # BaseUrl is still needed for tasks that haven't migrated to taskgraph yet. - "baseUrl": "http://taskcluster/secrets/v1", - } - ) + secrets = taskcluster.Secrets({ + # BaseUrl is still needed for tasks that haven't migrated to taskgraph yet. + "baseUrl": "http://taskcluster/secrets/v1", + }) except taskcluster.exceptions.TaskclusterFailure: # taskcluster library >=5 errors out when `baseUrl` is used - secrets = taskcluster.Secrets( - { - "rootUrl": os.environ.get( - "TASKCLUSTER_PROXY_URL", "https://taskcluster.net" - ), - } - ) + secrets = taskcluster.Secrets({ + "rootUrl": os.environ.get( + "TASKCLUSTER_PROXY_URL", "https://taskcluster.net" + ), + }) return secrets.get(name) diff --git a/taskcluster/scripts/lint/update_buildconfig_from_gradle.py b/taskcluster/scripts/lint/update_buildconfig_from_gradle.py @@ -35,9 +35,9 @@ def _get_upstream_deps_per_gradle_project(gradle_root, existing_build_config): # command line too long. If that happens, we'll need to split this list up and # run gradle more than once. cmd = list(_DEFAULT_GRADLE_COMMAND) - cmd.extend( - [f"{gradle_project}:dependencies" for gradle_project in sorted(gradle_projects)] - ) + cmd.extend([ + f"{gradle_project}:dependencies" for gradle_project in sorted(gradle_projects) + ]) # Parsing output like this is not ideal but bhearsum couldn't find a way # to get the dependencies printed in a better format. If we could convince diff --git a/taskcluster/scripts/misc/fetch-cft-chromedriver.py b/taskcluster/scripts/misc/fetch-cft-chromedriver.py @@ -13,7 +13,6 @@ channel is specified as it is required for the Chromium-as-Release performance tests. """ - import argparse import errno import os diff --git a/taskcluster/scripts/misc/repack_rust.py b/taskcluster/scripts/misc/repack_rust.py @@ -191,9 +191,14 @@ def fetch(url, validate=True): if validate: log("Verifying %s..." % base) verify_sha(base, sha) - subprocess.check_call( - ["gpg", "--keyid-format", "0xlong", "--verify", base + ".asc", base] - ) + subprocess.check_call([ + "gpg", + "--keyid-format", + "0xlong", + "--verify", + base + ".asc", + base, + ]) return sha @@ -237,8 +242,7 @@ def fetch_package(manifest, pkg, host): sha = fetch(info["url"], info["hash"] is not None) if info["hash"] and sha != info["hash"]: log( - "Checksum mismatch: package resource is different from manifest" - "\n %s" % sha + "Checksum mismatch: package resource is different from manifest\n %s" % sha ) raise AssertionError return info @@ -507,24 +511,20 @@ def build_src(install_dir, host, targets, patches): shutil.copy(f"{clang_bin}/llvm-ml.exe", f"{tmpdir}/ml64.exe") env = os.environ.copy() - env.update( - { - "PATH": os.pathsep.join((tmpdir, clang_bin, os.environ["PATH"])), - "LD_LIBRARY_PATH": clang_lib, - "DESTDIR": install_dir, - } - ) + env.update({ + "PATH": os.pathsep.join((tmpdir, clang_bin, os.environ["PATH"])), + "LD_LIBRARY_PATH": clang_lib, + "DESTDIR": install_dir, + }) if "msvc" in host: cmake_bin = os.path.join(fetches, "cmake", "bin") ninja_bin = os.path.join(fetches, "ninja", "bin") - env.update( - { - "PATH": os.pathsep.join((env["PATH"], cmake_bin, ninja_bin)), - "CC_x86_64_pc_windows_msvc": "clang-cl.bat", - "CXX_x86_64_pc_windows_msvc": "clang-cl.bat", - "AR_x86_64_pc_windows_msvc": "llvm-lib", - } - ) + env.update({ + "PATH": os.pathsep.join((env["PATH"], cmake_bin, ninja_bin)), + "CC_x86_64_pc_windows_msvc": "clang-cl.bat", + "CXX_x86_64_pc_windows_msvc": "clang-cl.bat", + "AR_x86_64_pc_windows_msvc": "llvm-lib", + }) # x.py install does everything we need for us. # If you're running into issues, consider using `-vv` to debug it. diff --git a/taskcluster/scripts/tests/generate-flaky-report-from-ftl.py b/taskcluster/scripts/tests/generate-flaky-report-from-ftl.py @@ -98,20 +98,18 @@ def extract_flaky_tests(xml_files: list[Path]) -> list[dict[str, Any]]: if cur_case.result: result_info = [] for result in cur_case.result: - result_info.append( - { - "type": ( - result.type - if result.type - else "Failure Stack Trace" - ), - "message": ( - result.message - if result.message - else result.text - ), - } - ) + result_info.append({ + "type": ( + result.type + if result.type + else "Failure Stack Trace" + ), + "message": ( + result.message + if result.message + else result.text + ), + }) flaky_info["results"] = result_info flaky_tests.append(flaky_info) @@ -233,24 +231,24 @@ def generate_html_report(flaky_tests: list[dict[str, Any]], output_path: Path) - for test in flaky_tests: html_content += f""" <div class="test-case"> - <div class="test-name">{test['name']}</div> - <div class="test-class">Class: {test['classname']}</div> + <div class="test-name">{test["name"]}</div> + <div class="test-class">Class: {test["classname"]}</div> <div class="test-meta"> - Execution Time: {test.get('time', 'N/A')}s | Source: {test['file']} + Execution Time: {test.get("time", "N/A")}s | Source: {test["file"]} </div> """ if "weblink" in test: html_content += f""" - <a href="{test['weblink']}" class="firebase-link" target="_blank">View in Firebase Test Lab</a> + <a href="{test["weblink"]}" class="firebase-link" target="_blank">View in Firebase Test Lab</a> """ if "results" in test: for result in test["results"]: html_content += f""" <div class="result-info"> - <div class="result-type">{result.get('type', 'Unknown')}</div> - <div>{result.get('message', 'No message available')}</div> + <div class="result-type">{result.get("type", "Unknown")}</div> + <div>{result.get("message", "No message available")}</div> </div> """ diff --git a/taskcluster/scripts/tests/parse-junit-results.py b/taskcluster/scripts/tests/parse-junit-results.py @@ -107,13 +107,11 @@ def parse_print_failure_results(results): ) # Check if the current failure details are different from the last seen ones if details != last_seen_failures.get(test_id, ""): - table.rows.append( - [ - test_id, - "Flaky", - details, - ] - ) + table.rows.append([ + test_id, + "Flaky", + details, + ]) last_seen_failures[test_id] = details else: test_id = "%s#%s" % (case.classname, case.name) @@ -122,13 +120,11 @@ def parse_print_failure_results(results): ) # Check if the current failure details are different from the last seen ones if details != last_seen_failures.get(test_id, ""): - table.rows.append( - [ - test_id, - "Failure", - details, - ] - ) + table.rows.append([ + test_id, + "Failure", + details, + ]) print(f"TEST-UNEXPECTED-FAIL | {test_id} | {details}") failure_count += 1 # Update the last seen failure details for this test case diff --git a/taskcluster/scripts/tests/test-lab.py b/taskcluster/scripts/tests/test-lab.py @@ -90,9 +90,13 @@ def setup_environment(): sys.exit(1) run_command(["gcloud", "config", "set", "project", project_id]) - run_command( - ["gcloud", "auth", "activate-service-account", "--key-file", credentials_file] - ) + run_command([ + "gcloud", + "auth", + "activate-service-account", + "--key-file", + credentials_file, + ]) def execute_tests( @@ -131,12 +135,10 @@ def execute_tests( geckoRev = os.environ.get("GECKO_HEAD_REV") if matrixLabel is not None and geckoRev is not None: - flank_command.extend( - [ - "--client-details", - f"matrixLabel={urlparse(matrixLabel).path.rpartition('/')[-1]},geckoRev={geckoRev}", - ] - ) + flank_command.extend([ + "--client-details", + f"matrixLabel={urlparse(matrixLabel).path.rpartition('/')[-1]},geckoRev={geckoRev}", + ]) # Add androidTest APK if provided (optional) as robo test or instrumentation test if apk_test: diff --git a/taskcluster/test/test_mach_try_auto.py b/taskcluster/test/test_mach_try_auto.py @@ -9,22 +9,20 @@ from tryselect.selectors.auto import TRY_AUTO_PARAMETERS pytestmark = pytest.mark.slow PARAMS = TRY_AUTO_PARAMETERS.copy() -PARAMS.update( - { - "files_changed": [ - "dom/html/HTMLDetailsElement.cpp", - "gfx/thebes/gfxUserFontSet.cpp", - ], - "head_repository": "https://hg.mozilla.org/try", - "project": "try", - "target_kind": "mochitest", - # These ensure this isn't considered a backstop. The pushdate must - # be slightly higher than the one in data/pushes.json, and - # pushlog_id must not be a multiple of 10. - "pushdate": 1593029536, - "pushlog_id": "2", - } -) +PARAMS.update({ + "files_changed": [ + "dom/html/HTMLDetailsElement.cpp", + "gfx/thebes/gfxUserFontSet.cpp", + ], + "head_repository": "https://hg.mozilla.org/try", + "project": "try", + "target_kind": "mochitest", + # These ensure this isn't considered a backstop. The pushdate must + # be slightly higher than the one in data/pushes.json, and + # pushlog_id must not be a multiple of 10. + "pushdate": 1593029536, + "pushlog_id": "2", +}) def test_generate_graph(optimized_task_graph): diff --git a/taskcluster/test/test_new_config.py b/taskcluster/test/test_new_config.py @@ -8,36 +8,32 @@ from tryselect.selectors.auto import TRY_AUTO_PARAMETERS pytestmark = pytest.mark.slow PARAMS = TRY_AUTO_PARAMETERS.copy() -PARAMS.update( - { - "head_repository": "https://hg.mozilla.org/try", - "project": "try", - "target_kind": "mochitest", - # These ensure this isn't considered a backstop. The pushdate must - # be slightly higher than the one in data/pushes.json, and - # pushlog_id must not be a multiple of 10. - "pushdate": 1593029536, - "pushlog_id": "2", - } -) +PARAMS.update({ + "head_repository": "https://hg.mozilla.org/try", + "project": "try", + "target_kind": "mochitest", + # These ensure this isn't considered a backstop. The pushdate must + # be slightly higher than the one in data/pushes.json, and + # pushlog_id must not be a multiple of 10. + "pushdate": 1593029536, + "pushlog_id": "2", +}) PARAMS_NEW_CONFIG = TRY_AUTO_PARAMETERS.copy() -PARAMS_NEW_CONFIG.update( - { - "head_repository": "https://hg.mozilla.org/try", - "project": "try", - "target_kind": "mochitest", - # These ensure this isn't considered a backstop. The pushdate must - # be slightly higher than the one in data/pushes.json, and - # pushlog_id must not be a multiple of 10. - "pushdate": 1593029536, - "pushlog_id": "2", - "try_task_config": {"new-test-config": True}, - "try_mode": "try_task_config", - "target_tasks_method": "try_tasks", - "test_manifest_loader": "default", - } -) +PARAMS_NEW_CONFIG.update({ + "head_repository": "https://hg.mozilla.org/try", + "project": "try", + "target_kind": "mochitest", + # These ensure this isn't considered a backstop. The pushdate must + # be slightly higher than the one in data/pushes.json, and + # pushlog_id must not be a multiple of 10. + "pushdate": 1593029536, + "pushlog_id": "2", + "try_task_config": {"new-test-config": True}, + "try_mode": "try_task_config", + "target_tasks_method": "try_tasks", + "test_manifest_loader": "default", +}) @pytest.mark.parametrize( diff --git a/testing/awsy/awsy/parse_about_memory.py b/testing/awsy/awsy/parse_about_memory.py @@ -145,12 +145,12 @@ if __name__ == "__main__": action="store", nargs="*", default=None, - help="Process name filter. " "If not provided all processes will be included.", + help="Process name filter. If not provided all processes will be included.", ) parser.add_argument( "--mebi", action="store_true", - help="Output values as mebibytes (instead of bytes)" " to match about:memory.", + help="Output values as mebibytes (instead of bytes) to match about:memory.", ) args = parser.parse_args() @@ -159,15 +159,15 @@ if __name__ == "__main__": sorted_totals = sorted(totals.items(), key=lambda item: (-item[1], item[0])) for k, v in sorted_totals: if v: - print(f"{k}\t"), + (print(f"{k}\t"),) print("") bytes_per_mebibyte = 1024.0 * 1024.0 for k, v in sorted_totals: if v: if args.mebi: - print(f"{v / bytes_per_mebibyte:.2f} MiB"), + (print(f"{v / bytes_per_mebibyte:.2f} MiB"),) else: - print(f"{v} bytes"), - print("\t"), + (print(f"{v} bytes"),) + (print("\t"),) print("") diff --git a/testing/awsy/awsy/process_perf_data.py b/testing/awsy/awsy/process_perf_data.py @@ -148,9 +148,9 @@ def create_suite( totals_uss = parse_about_memory.calculate_memory_report_values( memory_report_path, "resident-unique" ) - value = list(totals_rss.values())[0] + sum( - [v for k, v in totals_uss.items() if "Main" not in k] - ) + value = list(totals_rss.values())[0] + sum([ + v for k, v in totals_uss.items() if "Main" not in k + ]) subtest = { "name": checkpoint["name"], diff --git a/testing/awsy/mach_commands.py b/testing/awsy/mach_commands.py @@ -249,8 +249,7 @@ def run_awsy(command_context, tests, binary=None, **kwargs): action="store", type=int, dest="iterations", - help="Number of times to run through the test suite. " - "Defaults to %s." % ITERATIONS, + help="Number of times to run through the test suite. Defaults to %s." % ITERATIONS, ) @CommandArgument( "--per-tab-pause", diff --git a/testing/condprofile/condprof/archiver.py b/testing/condprofile/condprof/archiver.py @@ -1,8 +1,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -"""Helper to create tarballs. -""" +"""Helper to create tarballs.""" + import copy import glob import os diff --git a/testing/condprofile/condprof/changelog.py b/testing/condprofile/condprof/changelog.py @@ -7,6 +7,7 @@ """ Maintains a unique file that lists all artifacts operations. """ + import json import os import sys diff --git a/testing/condprofile/condprof/check_install.py b/testing/condprofile/condprof/check_install.py @@ -1,11 +1,12 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" Installs dependencies at runtime to simplify deployment. +"""Installs dependencies at runtime to simplify deployment. This module tries to make sure we have all dependencies installed on all our environments. """ + import os import subprocess import sys @@ -43,19 +44,17 @@ def install_reqs(): if req.strip() != "" and not req.startswith("#") ] for req in reqs: - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - "--no-cache-dir", - "--isolated", - "--find-links", - "https://pypi.pub.build.mozilla.org/pub/", - req, - ] - ) + subprocess.check_call([ + sys.executable, + "-m", + "pip", + "install", + "--no-cache-dir", + "--isolated", + "--find-links", + "https://pypi.pub.build.mozilla.org/pub/", + req, + ]) return True diff --git a/testing/condprofile/condprof/main.py b/testing/condprofile/condprof/main.py @@ -1,8 +1,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" Script that launches profiles creation. -""" +"""Script that launches profiles creation.""" + import argparse import os import sys diff --git a/testing/condprofile/condprof/metadata.py b/testing/condprofile/condprof/metadata.py @@ -1,8 +1,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" Manages a metadata file. -""" +"""Manages a metadata file.""" + import datetime import json import os diff --git a/testing/condprofile/condprof/tests/test_runner.py b/testing/condprofile/condprof/tests/test_runner.py @@ -25,7 +25,7 @@ with open(os.path.join(os.path.dirname(__file__), "ftp_mozilla.html")) as f: FTP_PAGE = f.read() FTP_ARCHIVE = re.compile( - "https://ftp.mozilla.org/pub/firefox/nightly/" "latest-mozilla-central/firefox.*" + "https://ftp.mozilla.org/pub/firefox/nightly/latest-mozilla-central/firefox.*" ) ADDON = re.compile("https://addons.mozilla.org/.*/.*xpi") diff --git a/testing/firefox-ui/tests/functional/safebrowsing/test_initial_download.py b/testing/firefox-ui/tests/functional/safebrowsing/test_initial_download.py @@ -59,13 +59,11 @@ class TestSafeBrowsingInitialDownload(MarionetteTestCase): ) for ext in my_file_extensions: - files.extend( - [ - f"{f}.{ext}" - for f in base_names - if f and f.endswith("-proto") == is_v4 - ] - ) + files.extend([ + f"{f}.{ext}" + for f in base_names + if f and f.endswith("-proto") == is_v4 + ]) return set(sorted(files)) @@ -77,11 +75,9 @@ class TestSafeBrowsingInitialDownload(MarionetteTestCase): f.startswith("goog-") or f.startswith("googpub-") for f in self.safebrowsing_shavar_files ): - self.prefs_provider_update_time.update( - { - "browser.safebrowsing.provider.google.nextupdatetime": 1, - } - ) + self.prefs_provider_update_time.update({ + "browser.safebrowsing.provider.google.nextupdatetime": 1, + }) # if V5 is enabled, we use the V5 update time to determine if the files # have been downloaded. Otherwise, we use the V4 update time. @@ -95,17 +91,13 @@ class TestSafeBrowsingInitialDownload(MarionetteTestCase): for f in self.safebrowsing_protobuf_files ): if is_safebrowsing_v5_enabled: - self.prefs_provider_update_time.update( - { - "browser.safebrowsing.provider.google5.nextupdatetime": 1, - } - ) + self.prefs_provider_update_time.update({ + "browser.safebrowsing.provider.google5.nextupdatetime": 1, + }) else: - self.prefs_provider_update_time.update( - { - "browser.safebrowsing.provider.google4.nextupdatetime": 1, - } - ) + self.prefs_provider_update_time.update({ + "browser.safebrowsing.provider.google4.nextupdatetime": 1, + }) # Force the preferences for the new profile enforce_prefs = self.prefs_safebrowsing diff --git a/testing/firefox-ui/tests/functional/security/test_https_first.py b/testing/firefox-ui/tests/functional/security/test_https_first.py @@ -16,7 +16,6 @@ from marionette_harness import MarionetteTestCase, WindowManagerMixin class TestHTTPSFirst(WindowManagerMixin, MarionetteTestCase): - def setUp(self): super().setUp() diff --git a/testing/gtest/reports.py b/testing/gtest/reports.py @@ -113,9 +113,13 @@ class AggregatedGTestReport(dict): def reset(self): """Clear all results.""" self.clear() - self.update( - {"tests": 0, "failures": 0, "disabled": 0, "errors": 0, "testsuites": []} - ) + self.update({ + "tests": 0, + "failures": 0, + "disabled": 0, + "errors": 0, + "testsuites": [], + }) def gtest_output(self, job_id): """ diff --git a/testing/gtest/rungtests.py b/testing/gtest/rungtests.py @@ -265,13 +265,13 @@ class gtestOptions(argparse.ArgumentParser): "--cwd", dest="cwd", default=os.getcwd(), - help="absolute path to directory from which " "to run the binary", + help="absolute path to directory from which to run the binary", ) self.add_argument( "--xre-path", dest="xre_path", default=None, - help="absolute path to directory containing XRE " "(probably xulrunner)", + help="absolute path to directory containing XRE (probably xulrunner)", ) self.add_argument( "--symbols-path", diff --git a/testing/mach_commands.py b/testing/mach_commands.py @@ -404,12 +404,10 @@ def test(command_context, what, extra_args, **log_args): repo = get_specific_repository_object(".", "git") base_commit = repo.base_ref_as_commit() - patch = "\n".join( - [ - repo.get_patches_after_ref(base_commit), - repo.get_patch_for_uncommitted_changes(), - ] - ) + patch = "\n".join([ + repo.get_patches_after_ref(base_commit), + repo.get_patch_for_uncommitted_changes(), + ]) if not patch.strip(): print("No local changes detected; no tests to run.") return 1 @@ -444,9 +442,10 @@ def test(command_context, what, extra_args, **log_args): if not mozdebug.get_debugger_info(log_args.get("debugger")): sys.exit(1) - extra_args_debugger_notation = "=".join( - ["--debugger", log_args.get("debugger")] - ) + extra_args_debugger_notation = "=".join([ + "--debugger", + log_args.get("debugger"), + ]) if extra_args: extra_args.append(extra_args_debugger_notation) else: @@ -1340,12 +1339,10 @@ def run_migration_tests(command_context, test_paths=None, **kwargs): ) if context["issues"]: continue - with_context.append( - { - "to_test": to_test, - "references": context["references"], - } - ) + with_context.append({ + "to_test": to_test, + "references": context["references"], + }) except Exception as e: command_context.log( logging.ERROR, diff --git a/testing/manifest/clean_skipfails.py b/testing/manifest/clean_skipfails.py @@ -17,7 +17,6 @@ ERROR = "error" class CleanSkipfails: - def __init__( self, command_context=None, @@ -108,10 +107,8 @@ class CleanSkipfails: manifest_paths = set(t["manifest"] for t in tests) else: myPath = Path(".").parent - manifest_paths = set( - [ - str(x).replace("\\", "/") - for x in myPath.glob(self.manifest_search_path) - ] - ) + manifest_paths = set([ + str(x).replace("\\", "/") + for x in myPath.glob(self.manifest_search_path) + ]) return manifest_paths diff --git a/testing/manifest/failedplatform.py b/testing/manifest/failedplatform.py @@ -60,12 +60,10 @@ class FailedPlatform: # This avoids creating a too broad skip-if condition if len(possible_build_types) == 0: return False - return all( - [ - bt in build_types and self.is_full_test_variants_fail(bt) - for bt in possible_build_types - ] - ) + return all([ + bt in build_types and self.is_full_test_variants_fail(bt) + for bt in possible_build_types + ]) def is_full_high_freq_fail(self) -> bool: """ @@ -77,12 +75,10 @@ class FailedPlatform: # This avoids creating a too broad skip-if condition if len(possible_build_types) == 0: return False - return all( - [ - bt in build_types and sum(list(self.failures[bt].values())) >= 7 - for bt in possible_build_types - ] - ) + return all([ + bt in build_types and sum(list(self.failures[bt].values())) >= 7 + for bt in possible_build_types + ]) def is_full_test_variants_fail(self, build_type: str) -> bool: """ diff --git a/testing/manifest/parse_reftest.py b/testing/manifest/parse_reftest.py @@ -563,13 +563,13 @@ class ListManifestParser: self.error(f"line {lineno} does not match: {line}") return result if os.path.basename(words[n - 1]) != url_ref: - self.error(f"words[n-1] not url_ref: {words[n-1]} != {url_ref}") + self.error(f"words[n-1] not url_ref: {words[n - 1]} != {url_ref}") return result if os.path.basename(words[n - 2]) != url: - self.error(f"words[n-2] not url: {words[n-2]} != {url}") + self.error(f"words[n-2] not url: {words[n - 2]} != {url}") return result if words[n - 3] != test_type: - self.error(f"words[n-3] not '{test_type}': {words[n-3]}") + self.error(f"words[n-3] not '{test_type}': {words[n - 3]}") return result d_min = 0 d_max = 0 @@ -609,13 +609,13 @@ class ListManifestParser: d_max2 = int((1.0 + MARGIN) * d_max) if d_max2 > d_max: self.info( - f"Increased max difference from {d_max} by {int(MARGIN*100)}% to {d_max2}" + f"Increased max difference from {d_max} by {int(MARGIN * 100)}% to {d_max2}" ) d_max = d_max2 p_max2 = int((1.0 + MARGIN) * p_max) if p_max2 > p_max: self.info( - f"Increased differing pixels from {p_max} by {int(MARGIN*100)}% to {p_max2}" + f"Increased differing pixels from {p_max} by {int(MARGIN * 100)}% to {p_max2}" ) p_max = p_max2 if comment: diff --git a/testing/manifest/skipfails.py b/testing/manifest/skipfails.py @@ -261,14 +261,12 @@ class SkipfailsMode(Mode): replace_tbd_mode: bool, ) -> int: if ( - sum( - [ - carryover_mode, - known_intermittents_mode, - new_failures_mode, - replace_tbd_mode, - ] - ) + sum([ + carryover_mode, + known_intermittents_mode, + new_failures_mode, + replace_tbd_mode, + ]) > 1 ): raise Exception( @@ -797,7 +795,7 @@ class Skipfails: tcs = i if tcs < 0: raise PermissionError("Error querying mozci with default User-Agent") - msg = f'Error querying mozci with User-Agent: {mozci.data.handler.sources[tcs].session.headers["User-Agent"]}' + msg = f"Error querying mozci with User-Agent: {mozci.data.handler.sources[tcs].session.headers['User-Agent']}" if self.user_agent is None: raise PermissionError(msg) else: @@ -1900,17 +1898,13 @@ class Skipfails: import taskcluster url: OptStr = None - index = taskcluster.Index( - { - "rootUrl": "https://firefox-ci-tc.services.mozilla.com", - } - ) + index = taskcluster.Index({ + "rootUrl": "https://firefox-ci-tc.services.mozilla.com", + }) route = "gecko.v2.mozilla-central.latest.source.test-info-all" - queue = taskcluster.Queue( - { - "rootUrl": "https://firefox-ci-tc.services.mozilla.com", - } - ) + queue = taskcluster.Queue({ + "rootUrl": "https://firefox-ci-tc.services.mozilla.com", + }) # Typing from findTask is wrong, so we need to convert to Any result: OptTaskResult = index.findTask(route) @@ -2694,9 +2688,9 @@ class Skipfails: difference = 0 pixels = 0 if difference > 0: - self.error_summary[group][allmods][RUNS][task_id][ - DIFFERENCE - ] = difference + self.error_summary[group][allmods][RUNS][task_id][DIFFERENCE] = ( + difference + ) if pixels > 0: self.error_summary[group][allmods][RUNS][task_id][PIXELS] = pixels if status != FAIL: @@ -2756,9 +2750,7 @@ class Skipfails: if summary.endswith("single tracking bug"): bugid: int = bug.get("id", None) line_number = top["line_number"] + 1 - log_url: str = ( - f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}" - ) + log_url: str = f"https://treeherder.mozilla.org/logviewer?repo={repo}&job_id={job_id}&lineNumber={line_number}" comment += f"\nError log line {line_number}: {log_url}" return (bugid, comment, line_number) diff --git a/testing/manifest/test/test_failedplatform.py b/testing/manifest/test/test_failedplatform.py @@ -15,9 +15,10 @@ def test_get_possible_build_types(): fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) assert fp.get_possible_build_types() == ["build_type1"] - fp = FailedPlatform( - {"build_type1": {"test_variant1": {}}, "build_type2": {"test_variant1": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}}, + "build_type2": {"test_variant1": {}}, + }) assert fp.get_possible_build_types() == ["build_type1", "build_type2"] @@ -33,9 +34,10 @@ def test_get_possible_test_variants(): fp = FailedPlatform({"build_type1": {"test_variant1": {}}}) assert fp.get_possible_test_variants("build_type1") == ["test_variant1"] - fp = FailedPlatform( - {"build_type1": {"test_variant1": {}}, "build_type2": {"test_variant2": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}}, + "build_type2": {"test_variant2": {}}, + }) assert fp.get_possible_test_variants("build_type2") == ["test_variant2"] fp = FailedPlatform({"build_type1": {"test_variant1": {}, "test_variant2": {}}}) @@ -110,21 +112,17 @@ def test_is_full_fail(): fp.failures["build_type1"] = {"test_variant1": 1, "test_variant2": 1} assert fp.is_full_fail() - fp = FailedPlatform( - { - "build_type1": {"test_variant1": {}, "test_variant2": {}}, - "build_type2": {"test_variant1": {}, "test_variant2": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}, "test_variant2": {}}, + "build_type2": {"test_variant1": {}, "test_variant2": {}}, + }) fp.failures["build_type1"] = {"test_variant1": 1, "test_variant2": 1} assert not fp.is_full_fail() - fp = FailedPlatform( - { - "build_type1": {"test_variant1": {}, "test_variant2": {}}, - "build_type2": {"test_variant1": {}, "test_variant2": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}, "test_variant2": {}}, + "build_type2": {"test_variant1": {}, "test_variant2": {}}, + }) fp.failures["build_type1"] = {"test_variant1": 1, "test_variant2": 1} fp.failures["build_type2"] = {"test_variant1": 1, "test_variant2": 1} assert fp.is_full_fail() @@ -150,30 +148,26 @@ def test_get_no_variant_conditions(): ) # Handle composite variants - fp = FailedPlatform( - { - "build_type1": { - "test_variant1": {}, - "socketprocess_networking+!fission": {}, - "no_variant": {}, - } + fp = FailedPlatform({ + "build_type1": { + "test_variant1": {}, + "socketprocess_networking+!fission": {}, + "no_variant": {}, } - ) + }) assert ( fp.get_no_variant_conditions(" && ", "build_type1") == " && !test_variant1 && !socketprocess_networking && fission" ) # Handle mutually exclusive variants - fp = FailedPlatform( - { - "build_type1": { - "fission": {}, - "socketprocess_networking+!fission": {}, - "no_variant": {}, - } + fp = FailedPlatform({ + "build_type1": { + "fission": {}, + "socketprocess_networking+!fission": {}, + "no_variant": {}, } - ) + }) assert ( fp.get_no_variant_conditions(" && ", "build_type1") == " && !socketprocess_networking" @@ -203,27 +197,27 @@ def test_get_test_variant_condition(): ) # Returns the negation of the composite if any - fp = FailedPlatform( - {"build_type1": {"test_variant1": {}, "test_variant1+test_variant2": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}, "test_variant1+test_variant2": {}} + }) assert ( fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") == " && test_variant1 && !test_variant2" ) # Ignores composite variants it is not part of - fp = FailedPlatform( - {"build_type1": {"test_variant1": {}, "test_variant2+test_variant3": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}, "test_variant2+test_variant3": {}} + }) assert ( fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") == " && test_variant1" ) # Convert a composite test variant - fp = FailedPlatform( - {"build_type1": {"test_variant1": {}, "test_variant1+test_variant2": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}, "test_variant1+test_variant2": {}} + }) assert ( fp.get_test_variant_condition( " && ", "build_type1", "test_variant1+test_variant2" @@ -232,14 +226,12 @@ def test_get_test_variant_condition(): ) # Handles composite variants included in other composites - fp = FailedPlatform( - { - "build_type1": { - "test_variant1+test_variant2": {}, - "test_variant1+test_variant2+test_variant3": {}, - } + fp = FailedPlatform({ + "build_type1": { + "test_variant1+test_variant2": {}, + "test_variant1+test_variant2+test_variant3": {}, } - ) + }) assert ( fp.get_test_variant_condition( " && ", "build_type1", "test_variant1+test_variant2" @@ -248,14 +240,12 @@ def test_get_test_variant_condition(): ) # Detects if the composite or in scrambled order - fp = FailedPlatform( - { - "build_type1": { - "test_variant2+test_variant1": {}, - "test_variant1+test_variant3+test_variant2": {}, - } + fp = FailedPlatform({ + "build_type1": { + "test_variant2+test_variant1": {}, + "test_variant1+test_variant3+test_variant2": {}, } - ) + }) assert ( fp.get_test_variant_condition( " && ", "build_type1", "test_variant2+test_variant1" @@ -264,15 +254,13 @@ def test_get_test_variant_condition(): ) # Handles when variant is included in several composites - fp = FailedPlatform( - { - "build_type1": { - "test_variant1+test_variant2": {}, - "test_variant1+test_variant2+test_variant3": {}, - "test_variant1+test_variant4+test_variant2": {}, - } + fp = FailedPlatform({ + "build_type1": { + "test_variant1+test_variant2": {}, + "test_variant1+test_variant2+test_variant3": {}, + "test_variant1+test_variant4+test_variant2": {}, } - ) + }) assert ( fp.get_test_variant_condition( " && ", "build_type1", "test_variant2+test_variant1" @@ -281,14 +269,12 @@ def test_get_test_variant_condition(): ) # Simply returns the current composite variant if it is not fully contained in another - fp = FailedPlatform( - { - "build_type1": { - "test_variant1+test_variant2": {}, - "test_variant1+test_variant3": {}, - } + fp = FailedPlatform({ + "build_type1": { + "test_variant1+test_variant2": {}, + "test_variant1+test_variant3": {}, } - ) + }) assert ( fp.get_test_variant_condition( " && ", "build_type1", "test_variant1+test_variant2" @@ -297,12 +283,10 @@ def test_get_test_variant_condition(): ) # Ignore matching composite variants in other build types - fp = FailedPlatform( - { - "build_type1": {"test_variant1": {}}, - "build_type2": {"test_variant1+test_variant2": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}}, + "build_type2": {"test_variant1+test_variant2": {}}, + }) assert ( fp.get_test_variant_condition(" && ", "build_type1", "test_variant1") == " && test_variant1" @@ -324,9 +308,10 @@ def test_is_full_high_freq_fail(): assert fp.is_full_high_freq_fail() # Several build types - fp = FailedPlatform( - {"build_type1": {"no_variant": {}}, "build_type2": {"no_variant": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"no_variant": {}}, + "build_type2": {"no_variant": {}}, + }) assert not fp.is_full_high_freq_fail() for i in range(0, 7): @@ -341,12 +326,10 @@ def test_is_full_high_freq_fail(): assert fp.is_full_high_freq_fail() # Several build types and test variants - fp = FailedPlatform( - { - "build_type1": {"no_variant": {}, "test_variant1": {}}, - "build_type2": {"no_variant": {}, "test_variant1": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"no_variant": {}, "test_variant1": {}}, + "build_type2": {"no_variant": {}, "test_variant1": {}}, + }) assert not fp.is_full_high_freq_fail() for i in range(0, 7): @@ -382,69 +365,64 @@ def test_get_skip_string(): # Fail only on some build types without test variant # => only failed build types returned - fp = FailedPlatform( - {"build_type1": {"no_variant": {}}, "build_type2": {"no_variant": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"no_variant": {}}, + "build_type2": {"no_variant": {}}, + }) assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" # Fail only on one build type with test variant # => only failed build types returned with negated test variants - fp = FailedPlatform( - { - "build_type1": {"no_variant": {}, "test_variant1": {}}, - "build_type2": {"no_variant": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"no_variant": {}, "test_variant1": {}}, + "build_type2": {"no_variant": {}}, + }) assert ( fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1 && !test_variant1" ) # Full test variant fail on single line => only build type returned - fp = FailedPlatform( - {"build_type1": {"test_variant1": {}}, "build_type2": {"no_variant": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}}, + "build_type2": {"no_variant": {}}, + }) assert ( fp.get_skip_string(" && ", "build_type1", "test_variant1") == " && build_type1" ) - fp = FailedPlatform( - { - "build_type1": {"test_variant1+test_variant2": {}}, - "build_type2": {"no_variant": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1+test_variant2": {}}, + "build_type2": {"no_variant": {}}, + }) assert ( fp.get_skip_string(" && ", "build_type1", "test_variant1+test_variant2") == " && build_type1" ) # Fail only on some test variants => build type and test variant returned - fp = FailedPlatform( - { - "build_type1": {"test_variant1": {}, "test_variant2": {}}, - "build_type2": {"no_variant": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}, "test_variant2": {}}, + "build_type2": {"no_variant": {}}, + }) assert ( fp.get_skip_string(" && ", "build_type1", "test_variant1") == " && build_type1 && test_variant1" ) # Full fail on second call - fp = FailedPlatform( - {"build_type1": {"no_variant": {}}, "build_type2": {"no_variant": {}}} - ) + fp = FailedPlatform({ + "build_type1": {"no_variant": {}}, + "build_type2": {"no_variant": {}}, + }) assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" assert fp.get_skip_string(" && ", "build_type2", "no_variant") == "" # Full fail on second call with test variants - fp = FailedPlatform( - { - "build_type1": {"test_variant1+test_variant2": {}}, - "build_type2": {"test_variant1+test_variant2": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1+test_variant2": {}}, + "build_type2": {"test_variant1+test_variant2": {}}, + }) assert ( fp.get_skip_string(" && ", "build_type1", "test_variant1+test_variant2") == " && build_type1" @@ -461,12 +439,10 @@ def test_get_skip_string(): assert fp.get_skip_string(" && ", "build_type1", "test_variant2") == "" # Fail on variant and no_variant - fp = FailedPlatform( - { - "build_type1": {"test_variant1": {}, "no_variant": {}}, - "build_type2": {"no_variant": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": {"test_variant1": {}, "no_variant": {}}, + "build_type2": {"no_variant": {}}, + }) assert ( fp.get_skip_string(" && ", "build_type1", "test_variant1") == " && build_type1 && test_variant1" @@ -474,16 +450,14 @@ def test_get_skip_string(): assert fp.get_skip_string(" && ", "build_type1", "no_variant") == " && build_type1" # Complex cases - fp = FailedPlatform( - { - "build_type1": { - "test_variant1": {}, - "test_variant2": {}, - "test_variant1+test_variant2": {}, - }, - "build_type2": {"no_variant": {}, "test_variant1": {}, "test_variant2": {}}, - } - ) + fp = FailedPlatform({ + "build_type1": { + "test_variant1": {}, + "test_variant2": {}, + "test_variant1+test_variant2": {}, + }, + "build_type2": {"no_variant": {}, "test_variant1": {}, "test_variant2": {}}, + }) assert ( fp.get_skip_string(" && ", "build_type1", "test_variant1") == " && build_type1 && test_variant1 && !test_variant2" diff --git a/testing/marionette/client/marionette_driver/geckoinstance.py b/testing/marionette/client/marionette_driver/geckoinstance.py @@ -343,15 +343,13 @@ class GeckoInstance: args["preferences"]["remote.log.level"] = level if "-jsdebugger" in self.app_args: - args["preferences"].update( - { - "devtools.browsertoolbox.panel": "jsdebugger", - "devtools.chrome.enabled": True, - "devtools.debugger.prompt-connection": False, - "devtools.debugger.remote-enabled": True, - "devtools.testing": True, - } - ) + args["preferences"].update({ + "devtools.browsertoolbox.panel": "jsdebugger", + "devtools.chrome.enabled": True, + "devtools.debugger.prompt-connection": False, + "devtools.debugger.remote-enabled": True, + "devtools.testing": True, + }) if self.addons: args["addons"] = self.addons @@ -420,13 +418,11 @@ class GeckoInstance: # environment variables needed for crashreporting # https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting - env.update( - { - "MOZ_CRASHREPORTER": "1", - "MOZ_CRASHREPORTER_NO_REPORT": "1", - "MOZ_CRASHREPORTER_SHUTDOWN": "1", - } - ) + env.update({ + "MOZ_CRASHREPORTER": "1", + "MOZ_CRASHREPORTER_NO_REPORT": "1", + "MOZ_CRASHREPORTER_SHUTDOWN": "1", + }) extra_args = ["-marionette", "-remote-allow-system-access"] args = { diff --git a/testing/marionette/client/marionette_driver/marionette.py b/testing/marionette/client/marionette_driver/marionette.py @@ -1104,7 +1104,7 @@ class Marionette: """ if not self.instance: raise errors.MarionetteException( - "quit() can only be called " "on Gecko instances launched by Marionette" + "quit() can only be called on Gecko instances launched by Marionette" ) quit_details = {"cause": "shutdown", "forced": False} @@ -1164,8 +1164,9 @@ class Marionette: if quit_details.get("cause") not in (None, "shutdown"): raise errors.MarionetteException( - "Unexpected shutdown reason '{}' for " - "quitting the process.".format(quit_details["cause"]) + "Unexpected shutdown reason '{}' for quitting the process.".format( + quit_details["cause"] + ) ) return quit_details @@ -1204,8 +1205,7 @@ class Marionette: """ if not self.instance: raise errors.MarionetteException( - "restart() can only be called " - "on Gecko instances launched by Marionette" + "restart() can only be called on Gecko instances launched by Marionette" ) context = self._send_message("Marionette:GetContext", key="value") @@ -1299,8 +1299,9 @@ class Marionette: if restart_details.get("cause") not in (None, "restart"): raise errors.MarionetteException( - "Unexpected shutdown reason '{}' for " - "restarting the process".format(restart_details["cause"]) + "Unexpected shutdown reason '{}' for restarting the process".format( + restart_details["cause"] + ) ) return restart_details diff --git a/testing/marionette/harness/marionette_harness/marionette_test/testcases.py b/testing/marionette/harness/marionette_harness/marionette_test/testcases.py @@ -132,8 +132,7 @@ class CommonTestCase(unittest.TestCase, metaclass=MetaParameterized): addExpectedFailure(self, exc_info) else: warnings.warn( - "TestResult has no addExpectedFailure method, " - "reporting as passes", + "TestResult has no addExpectedFailure method, reporting as passes", RuntimeWarning, ) result.addSuccess(self) diff --git a/testing/marionette/harness/marionette_harness/runner/base.py b/testing/marionette/harness/marionette_harness/runner/base.py @@ -596,8 +596,7 @@ class RemoteMarionetteArguments: ["--device"], { "help": ( - "Serial ID to connect to as seen in `adb devices`," - "e.g emulator-5444" + "Serial ID to connect to as seen in `adb devices`,e.g emulator-5444" ), "dest": "device_serial", }, @@ -735,9 +734,7 @@ class BaseMarionetteTestRunner: self.reset_test_stats() - self.logger.info( - "Using workspace for temporary data: " f'"{self.workspace_path}"' - ) + self.logger.info(f'Using workspace for temporary data: "{self.workspace_path}"') if not gecko_log: self.gecko_log = os.path.join(self.workspace_path or "", "gecko.log") @@ -861,53 +858,43 @@ class BaseMarionetteTestRunner: debugger_info = mozdebug.get_debugger_info( self.debugger, self.debugger_args ) - kwargs.update( - { - "host": "127.0.0.1", - "port": 2828, - "app": self.app, - "app_args": self.app_args, - "debugger_info": debugger_info, - "profile": self.profile, - "addons": self.addons, - "gecko_log": self.gecko_log, - # ensure Marionette class takes care of starting gecko instance - "bin": True, - } - ) + kwargs.update({ + "host": "127.0.0.1", + "port": 2828, + "app": self.app, + "app_args": self.app_args, + "debugger_info": debugger_info, + "profile": self.profile, + "addons": self.addons, + "gecko_log": self.gecko_log, + # ensure Marionette class takes care of starting gecko instance + "bin": True, + }) if self.bin: - kwargs.update( - { - "bin": self.bin, - } - ) + kwargs.update({ + "bin": self.bin, + }) if self.emulator: - kwargs.update( - { - "avd_home": self.extra_kwargs.get("avd_home"), - "adb_path": self.extra_kwargs.get("adb_path"), - "emulator_binary": self.extra_kwargs.get("emulator_bin"), - "avd": self.extra_kwargs.get("avd"), - "package_name": self.extra_kwargs.get("package_name"), - } - ) + kwargs.update({ + "avd_home": self.extra_kwargs.get("avd_home"), + "adb_path": self.extra_kwargs.get("adb_path"), + "emulator_binary": self.extra_kwargs.get("emulator_bin"), + "avd": self.extra_kwargs.get("avd"), + "package_name": self.extra_kwargs.get("package_name"), + }) if self.address: host, port = self.address.split(":") - kwargs.update( - { - "host": host, - "port": int(port), - } - ) + kwargs.update({ + "host": host, + "port": int(port), + }) if self.emulator: - kwargs.update( - { - "connect_to_running_emulator": True, - } - ) + kwargs.update({ + "connect_to_running_emulator": True, + }) if not self.bin and not self.emulator: try: # Establish a socket connection so we can vertify the data come back @@ -1128,13 +1115,11 @@ class BaseMarionetteTestRunner: manifest.read(filepath) json_path = update_mozinfo(filepath) - mozinfo.update( - { - "appname": self.appName, - "manage_instance": self.marionette.instance is not None, - "headless": self.headless, - } - ) + mozinfo.update({ + "appname": self.appName, + "manage_instance": self.marionette.instance is not None, + "headless": self.headless, + }) self.logger.info(f"mozinfo updated from: {json_path}") self.logger.info(f"mozinfo is: {mozinfo.info}") @@ -1203,20 +1188,20 @@ class BaseMarionetteTestRunner: self.todo += len(results.skipped) self.passed += results.passed for failure in results.failures + results.errors: - self.failures.append( - (results.getInfo(failure), failure.output, "TEST-UNEXPECTED-FAIL") - ) + self.failures.append(( + results.getInfo(failure), + failure.output, + "TEST-UNEXPECTED-FAIL", + )) if hasattr(results, "unexpectedSuccesses"): self.failed += len(results.unexpectedSuccesses) self.unexpected_successes += len(results.unexpectedSuccesses) for failure in results.unexpectedSuccesses: - self.failures.append( - ( - results.getInfo(failure), - failure.output, - "TEST-UNEXPECTED-PASS", - ) - ) + self.failures.append(( + results.getInfo(failure), + failure.output, + "TEST-UNEXPECTED-PASS", + )) if hasattr(results, "expectedFailures"): self.todo += len(results.expectedFailures) diff --git a/testing/mochitest/leaks.py b/testing/mochitest/leaks.py @@ -118,13 +118,11 @@ class ShutdownLeaks: errors = [] for test in self._parseLeakingTests(): for url, count in self._zipLeakedWindows(test["leakedWindows"]): - errors.append( - { - "test": test["fileName"], - "msg": "leaked %d window(s) until shutdown [url = %s]" - % (count, url), - } - ) + errors.append({ + "test": test["fileName"], + "msg": "leaked %d window(s) until shutdown [url = %s]" + % (count, url), + }) failures += 1 if test["leakedWindowsString"]: @@ -134,24 +132,19 @@ class ShutdownLeaks: ) if test["leakedDocShells"]: - errors.append( - { - "test": test["fileName"], - "msg": "leaked %d docShell(s) until shutdown" - % (len(test["leakedDocShells"])), - } - ) + errors.append({ + "test": test["fileName"], + "msg": "leaked %d docShell(s) until shutdown" + % (len(test["leakedDocShells"])), + }) failures += 1 self.logger.info( "TEST-INFO | %s | docShell(s) leaked: %s" % ( test["fileName"], - ", ".join( - [ - "[pid = %s] [id = %s]" % x - for x in test["leakedDocShells"] - ] - ), + ", ".join([ + "[pid = %s] [id = %s]" % x for x in test["leakedDocShells"] + ]), ) ) @@ -244,9 +237,9 @@ class ShutdownLeaks: leakedWindows = [id for id in test["windows"] if id in self.leakedWindows] test["leakedWindows"] = [self.leakedWindows[id] for id in leakedWindows] test["hiddenWindowsCount"] = self.hiddenWindowsCount - test["leakedWindowsString"] = ", ".join( - ["[pid = %s] [serial = %s]" % x for x in leakedWindows] - ) + test["leakedWindowsString"] = ", ".join([ + "[pid = %s] [serial = %s]" % x for x in leakedWindows + ]) test["leakedDocShells"] = [ id for id in test["docShells"] if id in self.leakedDocShells ] diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py @@ -84,7 +84,7 @@ def strtobool(value: str): if value in false_vals: return 0 - raise ValueError(f'Expected one of: {", ".join(true_vals + false_vals)}') + raise ValueError(f"Expected one of: {', '.join(true_vals + false_vals)}") def get_default_valgrind_suppression_files(): @@ -1217,7 +1217,7 @@ class MochitestArguments(ArgumentContainer): if not pactl: parser.error( - "Missing binary pactl required for " "--use-test-media-devices" + "Missing binary pactl required for --use-test-media-devices" ) # The a11y and chrome flavors can't run with e10s. diff --git a/testing/mochitest/runjunit.py b/testing/mochitest/runjunit.py @@ -636,14 +636,16 @@ class JunitArgumentParser(argparse.ArgumentParser): help="Defines an extra user preference.", ) # Additional options for server. - self.add_argument( - "--certificate-path", - action="store", - type=str, - dest="certPath", - default=None, - help="Path to directory containing certificate store.", - ), + ( + self.add_argument( + "--certificate-path", + action="store", + type=str, + dest="certPath", + default=None, + help="Path to directory containing certificate store.", + ), + ) self.add_argument( "--http-port", action="store", diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py @@ -150,21 +150,19 @@ class MessageLogger: # This is a delimiter used by the JS side to avoid logs interleaving DELIMITER = "\ue175\uee31\u2c32\uacbf" BUFFERED_ACTIONS = set(["test_status", "log"]) - VALID_ACTIONS = set( - [ - "suite_start", - "suite_end", - "group_start", - "group_end", - "test_start", - "test_end", - "test_status", - "log", - "assertion_count", - "buffering_on", - "buffering_off", - ] - ) + VALID_ACTIONS = set([ + "suite_start", + "suite_end", + "group_start", + "group_end", + "test_start", + "test_end", + "test_status", + "log", + "assertion_count", + "buffering_on", + "buffering_off", + ]) # Regexes that will be replaced with an empty string if found in a test # name. We do this to normalize test names which may contain URLs and test # package prefixes. @@ -536,9 +534,10 @@ class MochitestServer: env["LD_LIBRARY_PATH"] = ":".join([self._xrePath, env["LD_LIBRARY_PATH"]]) if self._trainHop: - env["LD_LIBRARY_PATH"] = ":".join( - [os.path.join(os.path.dirname(here), "bin"), env["LD_LIBRARY_PATH"]] - ) + env["LD_LIBRARY_PATH"] = ":".join([ + os.path.join(os.path.dirname(here), "bin"), + env["LD_LIBRARY_PATH"], + ]) # When running with an ASan build, our xpcshell server will also be ASan-enabled, # thus consuming too much resources when running together with the browser on @@ -912,18 +911,16 @@ def findTestMediaDevices(log): gst = gst010 else: gst = gst10 - process = subprocess.Popen( - [ - gst, - "--no-fault", - "videotestsrc", - "pattern=green", - "num-buffers=1", - "!", - "v4l2sink", - "device=%s" % device, - ] - ) + process = subprocess.Popen([ + gst, + "--no-fault", + "videotestsrc", + "pattern=green", + "num-buffers=1", + "!", + "v4l2sink", + "device=%s" % device, + ]) info["video"] = {"name": name, "process": process} info["speaker"] = {"name": "44100Hz Null Output"} info["audio"] = {"name": "Monitor of {}".format(info["speaker"]["name"])} @@ -3124,13 +3121,11 @@ toolbar#nav-bar { input_devices = [] for i in range(1, INPUT_DEVICES_COUNT + 1): freq = i * DEVICES_BASE_FREQUENCY - input_devices.append( - { - "name": f"sine-{freq}", - "description": f"{freq}Hz Sine Source", - "frequency": freq, - } - ) + input_devices.append({ + "name": f"sine-{freq}", + "description": f"{freq}Hz Sine Source", + "frequency": freq, + }) # Determine if this is running PulseAudio or PipeWire # `pactl info` works on both systems, but when running on PipeWire it says @@ -3272,8 +3267,9 @@ toolbar#nav-bar { except subprocess.CalledProcessError: self.log.error( - "Could not create device with module-sine-source" - " (freq={})".format(device["frequency"]) + "Could not create device with module-sine-source (freq={})".format( + device["frequency"] + ) ) self.virtualDeviceIdList = idList @@ -3507,53 +3503,49 @@ toolbar#nav-bar { self.extraPrefs["fission.autostart"] = not options.disable_fission # for test manifest parsing. - mozinfo.update( - { - "a11y_checks": options.a11y_checks, - "e10s": options.e10s, - "fission": not options.disable_fission, - "headless": options.headless, - "http3": options.useHttp3Server, - "http2": options.useHttp2Server, - "inc_origin_init": os.environ.get("MOZ_ENABLE_INC_ORIGIN_INIT") == "1", - # Until the test harness can understand default pref values, - # (https://bugzilla.mozilla.org/show_bug.cgi?id=1577912) this value - # should by synchronized with the default pref value indicated in - # StaticPrefList.yaml. - # - # Currently for automation, the pref defaults to true (but can be - # overridden with --setpref). - "sessionHistoryInParent": not options.disable_fission - or not self.extraPrefs.get("fission.disableSessionHistoryInParent"), - "socketprocess_e10s": self.extraPrefs.get( - "network.process.enabled", False - ), - "socketprocess_networking": self.extraPrefs.get( - "network.http.network_access_on_socket_process.enabled", False - ), - "swgl": self.extraPrefs.get("gfx.webrender.software", False), - "verify": options.verify, - "verify_fission": options.verify_fission, - "vertical_tab": self.extraPrefs.get("sidebar.verticalTabs", False), - "webgl_ipc": self.extraPrefs.get("webgl.out-of-process", False), - "wmfme": ( - self.extraPrefs.get("media.wmf.media-engine.enabled", 0) - and self.extraPrefs.get( - "media.wmf.media-engine.channel-decoder.enabled", False - ) - ), - "mda_gpu": self.extraPrefs.get( - "media.hardware-video-decoding.force-enabled", False - ), - "xorigin": options.xOriginTests, - "condprof": options.conditionedProfile, - "msix": "WindowsApps" in options.app, - "android": mozinfo.info.get("android", False), - "is_emulator": mozinfo.info.get("is_emulator", False), - "coverage": mozinfo.info.get("coverage", False), - "nogpu": mozinfo.info.get("nogpu", False), - } - ) + mozinfo.update({ + "a11y_checks": options.a11y_checks, + "e10s": options.e10s, + "fission": not options.disable_fission, + "headless": options.headless, + "http3": options.useHttp3Server, + "http2": options.useHttp2Server, + "inc_origin_init": os.environ.get("MOZ_ENABLE_INC_ORIGIN_INIT") == "1", + # Until the test harness can understand default pref values, + # (https://bugzilla.mozilla.org/show_bug.cgi?id=1577912) this value + # should by synchronized with the default pref value indicated in + # StaticPrefList.yaml. + # + # Currently for automation, the pref defaults to true (but can be + # overridden with --setpref). + "sessionHistoryInParent": not options.disable_fission + or not self.extraPrefs.get("fission.disableSessionHistoryInParent"), + "socketprocess_e10s": self.extraPrefs.get("network.process.enabled", False), + "socketprocess_networking": self.extraPrefs.get( + "network.http.network_access_on_socket_process.enabled", False + ), + "swgl": self.extraPrefs.get("gfx.webrender.software", False), + "verify": options.verify, + "verify_fission": options.verify_fission, + "vertical_tab": self.extraPrefs.get("sidebar.verticalTabs", False), + "webgl_ipc": self.extraPrefs.get("webgl.out-of-process", False), + "wmfme": ( + self.extraPrefs.get("media.wmf.media-engine.enabled", 0) + and self.extraPrefs.get( + "media.wmf.media-engine.channel-decoder.enabled", False + ) + ), + "mda_gpu": self.extraPrefs.get( + "media.hardware-video-decoding.force-enabled", False + ), + "xorigin": options.xOriginTests, + "condprof": options.conditionedProfile, + "msix": "WindowsApps" in options.app, + "android": mozinfo.info.get("android", False), + "is_emulator": mozinfo.info.get("is_emulator", False), + "coverage": mozinfo.info.get("coverage", False), + "nogpu": mozinfo.info.get("nogpu", False), + }) if not self.mozinfo_variables_shown: self.mozinfo_variables_shown = True @@ -3773,8 +3765,7 @@ toolbar#nav-bar { if (valgrindArgs or valgrindSuppFiles) and not valgrindPath: self.log.error( - "Specified --valgrind-args or --valgrind-supp-files," - " but not --valgrind" + "Specified --valgrind-args or --valgrind-supp-files, but not --valgrind" ) return 1 diff --git a/testing/mochitest/tests/python/conftest.py b/testing/mochitest/tests/python/conftest.py @@ -71,16 +71,14 @@ def runtests(setup_test_harness, binary, parser, request): buf = io.StringIO() options = vars(parser.parse_args([])) - options.update( - { - "app": binary, - "flavor": flavor, - "runFailures": runFailures, - "restartAfterFailure": restartAfterFailure, - "keep_open": False, - "log_raw": [buf], - } - ) + options.update({ + "app": binary, + "flavor": flavor, + "runFailures": runFailures, + "restartAfterFailure": restartAfterFailure, + "keep_open": False, + "log_raw": [buf], + }) if runFailures == "selftest": options["crashAsPass"] = True @@ -89,12 +87,10 @@ def runtests(setup_test_harness, binary, parser, request): if not os.path.isdir(runtests.build_obj.bindir): package_root = os.path.dirname(mochitest_root) - options.update( - { - "certPath": os.path.join(package_root, "certs"), - "utilityPath": os.path.join(package_root, "bin"), - } - ) + options.update({ + "certPath": os.path.join(package_root, "certs"), + "utilityPath": os.path.join(package_root, "bin"), + }) options["extraProfileFiles"].append( os.path.join(package_root, "bin", "plugins") ) diff --git a/testing/mochitest/tests/python/test_create_directories.py b/testing/mochitest/tests/python/test_create_directories.py @@ -134,9 +134,10 @@ def test_one_entry(prepareRunTests, create_manifest): mock_os_makedirs.assert_called_once_with(".snap_firefox_current_real") opts = mock.Mock(pidFile="") # so cleanup() does not check it - with mock.patch("os.path.exists") as mock_os_path_exists, mock.patch( - "shutil.rmtree" - ) as mock_shutil_rmtree: + with ( + mock.patch("os.path.exists") as mock_os_path_exists, + mock.patch("shutil.rmtree") as mock_shutil_rmtree, + ): md.cleanup(opts, False) mock_os_path_exists.assert_called_once_with(".snap_firefox_current_real") mock_shutil_rmtree.assert_called_once_with(".snap_firefox_current_real") @@ -144,9 +145,10 @@ def test_one_entry(prepareRunTests, create_manifest): def test_one_entry_already_exists(prepareRunTests, create_manifest): options = create_manifest_one(create_manifest) - with mock.patch( - "os.path.exists", return_value=True - ) as mock_os_path_exists, mock.patch("os.makedirs") as mock_os_makedirs: + with ( + mock.patch("os.path.exists", return_value=True) as mock_os_path_exists, + mock.patch("os.makedirs") as mock_os_makedirs, + ): with pytest.raises(FileExistsError): _ = prepareRunTests(**options) mock_os_path_exists.assert_called_once_with(".snap_firefox_current_real") @@ -158,60 +160,55 @@ def test_mult_entry(prepareRunTests, create_manifest): with mock.patch("os.makedirs") as mock_os_makedirs: md = prepareRunTests(**options) assert mock_os_makedirs.call_count == 2 - mock_os_makedirs.assert_has_calls( - [ - mock.call(".snap_firefox_current_real"), - mock.call(".snap_firefox_current_real2"), - ] - ) + mock_os_makedirs.assert_has_calls([ + mock.call(".snap_firefox_current_real"), + mock.call(".snap_firefox_current_real2"), + ]) opts = mock.Mock(pidFile="") # so cleanup() does not check it - with mock.patch("os.path.exists") as mock_os_path_exists, mock.patch( - "shutil.rmtree" - ) as mock_shutil_rmtree: + with ( + mock.patch("os.path.exists") as mock_os_path_exists, + mock.patch("shutil.rmtree") as mock_shutil_rmtree, + ): md.cleanup(opts, False) assert mock_os_path_exists.call_count == 2 - mock_os_path_exists.assert_has_calls( - [ - mock.call(".snap_firefox_current_real"), - mock.call().__bool__(), - mock.call(".snap_firefox_current_real2"), - mock.call().__bool__(), - ] - ) + mock_os_path_exists.assert_has_calls([ + mock.call(".snap_firefox_current_real"), + mock.call().__bool__(), + mock.call(".snap_firefox_current_real2"), + mock.call().__bool__(), + ]) assert mock_os_makedirs.call_count == 2 - mock_shutil_rmtree.assert_has_calls( - [ - mock.call(".snap_firefox_current_real"), - mock.call(".snap_firefox_current_real2"), - ] - ) + mock_shutil_rmtree.assert_has_calls([ + mock.call(".snap_firefox_current_real"), + mock.call(".snap_firefox_current_real2"), + ]) def test_mult_entry_one_already_exists(prepareRunTests, create_manifest): options = create_manifest_mult(create_manifest) - with mock.patch( - "os.path.exists", side_effect=[True, False] - ) as mock_os_path_exists, mock.patch("os.makedirs") as mock_os_makedirs: + with ( + mock.patch("os.path.exists", side_effect=[True, False]) as mock_os_path_exists, + mock.patch("os.makedirs") as mock_os_makedirs, + ): with pytest.raises(FileExistsError): _ = prepareRunTests(**options) mock_os_path_exists.assert_called_once_with(".snap_firefox_current_real") mock_os_makedirs.assert_not_called() - with mock.patch( - "os.path.exists", side_effect=[False, True] - ) as mock_os_path_exists, mock.patch("os.makedirs") as mock_os_makedirs: + with ( + mock.patch("os.path.exists", side_effect=[False, True]) as mock_os_path_exists, + mock.patch("os.makedirs") as mock_os_makedirs, + ): with pytest.raises(FileExistsError): _ = prepareRunTests(**options) assert mock_os_path_exists.call_count == 2 - mock_os_path_exists.assert_has_calls( - [ - mock.call(".snap_firefox_current_real"), - mock.call(".snap_firefox_current_real2"), - ] - ) + mock_os_path_exists.assert_has_calls([ + mock.call(".snap_firefox_current_real"), + mock.call(".snap_firefox_current_real2"), + ]) mock_os_makedirs.assert_not_called() diff --git a/testing/mochitest/tests/python/test_message_logger.py b/testing/mochitest/tests/python/test_message_logger.py @@ -119,17 +119,15 @@ def test_buffering_on(get_message_logger, assert_actions): # test end, it failed! All previsouly buffered messages are now logged. ml.fake_message("test_end", status="FAIL") - assert_actions( - [ - "log", # "Buffered messages logged" - "test_status", - "log", - "test_status", - "log", - "log", # "Buffered messages finished" - "test_end", - ] - ) + assert_actions([ + "log", # "Buffered messages logged" + "test_status", + "log", + "test_status", + "log", + "log", # "Buffered messages finished" + "test_end", + ]) # enabling buffering outside of a test has no affect ml.fake_message("buffering_on") diff --git a/testing/mochitest/tests/python/test_mochitest_integration.py b/testing/mochitest/tests/python/test_mochitest_integration.py @@ -321,12 +321,10 @@ def test_output_testfile_in_dupe_manifests(flavor, runtests, test_name, test_man # Explicitly provide a manifestFile property that includes the # two manifest files that share the same test file. extra_opts = { - "manifestFile": test_manifest( - [ - "mochitest-dupemanifest-1.ini", - "mochitest-dupemanifest-2.ini", - ] - ), + "manifestFile": test_manifest([ + "mochitest-dupemanifest-1.ini", + "mochitest-dupemanifest-2.ini", + ]), "runByManifest": True, } diff --git a/testing/mozbase/manifestparser/manifestparser/cli.py b/testing/mozbase/manifestparser/manifestparser/cli.py @@ -6,6 +6,7 @@ """ Mozilla universal manifest parser """ + import os import sys from optparse import OptionParser diff --git a/testing/mozbase/manifestparser/manifestparser/expression.py b/testing/mozbase/manifestparser/manifestparser/expression.py @@ -187,27 +187,25 @@ class ParseError(Exception): @functools.lru_cache(maxsize=None) # noqa: UP033 def scan(text): if not ExpressionParser.scanner: - ExpressionParser.scanner = re.Scanner( - [ - # Note: keep these in sync with the class docstring above. - (r"true|false", bool_token), - (r"[a-zA-Z_]\w*", ident_token), - (r"[0-9]+", int_token), - (r'("[^"]*")|(\'[^\']*\')', string_token), - (r"==", eq_op_token()), - (r"!=", neq_op_token()), - (r"<=", le_op_token()), - (r">=", ge_op_token()), - (r"<", lt_op_token()), - (r">", gt_op_token()), - (r"\|\|", or_op_token()), - (r"!", not_op_token()), - (r"&&", and_op_token()), - (r"\(", lparen_token()), - (r"\)", rparen_token()), - (r"\s+", None), # skip whitespace - ] - ) + ExpressionParser.scanner = re.Scanner([ + # Note: keep these in sync with the class docstring above. + (r"true|false", bool_token), + (r"[a-zA-Z_]\w*", ident_token), + (r"[0-9]+", int_token), + (r'("[^"]*")|(\'[^\']*\')', string_token), + (r"==", eq_op_token()), + (r"!=", neq_op_token()), + (r"<=", le_op_token()), + (r">=", ge_op_token()), + (r"<", lt_op_token()), + (r">", gt_op_token()), + (r"\|\|", or_op_token()), + (r"!", not_op_token()), + (r"&&", and_op_token()), + (r"\(", lparen_token()), + (r"\)", rparen_token()), + (r"\s+", None), # skip whitespace + ]) tokens, _ = ExpressionParser.scanner.scan(text) return tokens diff --git a/testing/mozbase/manifestparser/manifestparser/ini.py b/testing/mozbase/manifestparser/manifestparser/ini.py @@ -106,9 +106,9 @@ def read_ini( if strict: # make sure this section doesn't already exist - assert ( - section not in section_names - ), "Section '%s' already found in '%s'" % (section, section_names) + assert section not in section_names, ( + "Section '%s' already found in '%s'" % (section, section_names) + ) section_names.add(section) current_section = {} @@ -121,7 +121,7 @@ def read_ini( raise IniParseError( fp, linenum, - "Expected a comment or section, " f"instead found '{stripped}'", + f"Expected a comment or section, instead found '{stripped}'", ) # continuation line ? @@ -149,9 +149,9 @@ def read_ini( # make sure this key isn't already in the section if key: - assert ( - key not in current_section - ), f"Found duplicate key {key} in section {section}" + assert key not in current_section, ( + f"Found duplicate key {key} in section {section}" + ) if strict: # make sure this key isn't empty diff --git a/testing/mozbase/manifestparser/manifestparser/manifestparser.py b/testing/mozbase/manifestparser/manifestparser/manifestparser.py @@ -505,9 +505,9 @@ class ManifestParser: if os.path.splitext(filename)[-1] in extensions ] - files.update( - [os.path.join(dirpath, filename) for filename in filenames] - ) + files.update([ + os.path.join(dirpath, filename) for filename in filenames + ]) paths = set(self.paths()) missing_from_filesystem = paths.difference(files) @@ -854,9 +854,9 @@ class ManifestParser: filenames = [relpath(filename, relative_to) for filename in filenames] # write to manifest - write_content = "\n".join( - [f"[{denormalize_path(filename)}]" for filename in filenames] - ) + write_content = "\n".join([ + f"[{denormalize_path(filename)}]" for filename in filenames + ]) print(write_content, file=write) cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) diff --git a/testing/mozbase/manifestparser/tests/test_manifestparser.py b/testing/mozbase/manifestparser/tests/test_manifestparser.py @@ -39,7 +39,7 @@ class TestManifestParser(unittest.TestCase): ) # Ensure that capitalization and order aren't an issue: - lines = [f"[\"{test['name']}\"]" for test in tests] + lines = [f'["{test["name"]}"]' for test in tests] self.assertEqual(lines, open(mozmill_example).read().strip().splitlines()) # Show how you select subsets of tests: @@ -50,14 +50,11 @@ class TestManifestParser(unittest.TestCase): self.assertEqual( len(restart_tests), len(parser.get(manifest=mozmill_restart_example)) ) - self.assertFalse( - [ - test - for test in restart_tests - if test["manifest"] - != os.path.join(here, "mozmill-restart-example.toml") - ] - ) + self.assertFalse([ + test + for test in restart_tests + if test["manifest"] != os.path.join(here, "mozmill-restart-example.toml") + ]) self.assertEqual( parser.get("name", tags=["foo"]), [ @@ -92,13 +89,11 @@ class TestManifestParser(unittest.TestCase): # The including manifest is always reported as a part of the generated test object. self.assertTrue( - all( - [ - t["ancestor_manifest"] == "include-example.toml" - for t in parser.tests - if t["name"] != "fleem" - ] - ) + all([ + t["ancestor_manifest"] == "include-example.toml" + for t in parser.tests + if t["name"] != "fleem" + ]) ) # The manifests should be there too: @@ -172,13 +167,11 @@ yellow = submarine""" # noqa # The including manifest is always reported as a part of the generated test object. self.assertTrue( - all( - [ - t["ancestor_manifest"] == "include-example.toml" - for t in parser.tests - if t["name"] != "fleem" - ] - ) + all([ + t["ancestor_manifest"] == "include-example.toml" + for t in parser.tests + if t["name"] != "fleem" + ]) ) # The manifests should be there too: diff --git a/testing/mozbase/mozcrash/mozcrash/__init__.py b/testing/mozbase/mozcrash/mozcrash/__init__.py @@ -6,4 +6,5 @@ mozcrash is a library for getting a stack trace out of processes that have crashed and left behind a minidump file using the Google Breakpad library. """ + from .mozcrash import * diff --git a/testing/mozbase/mozcrash/tests/test_symbols_path.py b/testing/mozbase/mozcrash/tests/test_symbols_path.py @@ -46,9 +46,13 @@ def test_symbols_path_url(check_for_crashes, minidump_files): urlhandlers=[{"method": "GET", "path": "/symbols", "function": get_symbols}], ) httpd.start() - symbol_url = urlunsplit( - ("http", "%s:%d" % httpd.httpd.server_address, "/symbols", "", "") - ) + symbol_url = urlunsplit(( + "http", + "%s:%d" % httpd.httpd.server_address, + "/symbols", + "", + "", + )) assert 1 == check_for_crashes(symbols_path=symbol_url) assert data["retrieved"] @@ -83,9 +87,13 @@ def test_symbols_retry(check_for_crashes, minidump_files): urlhandlers=[{"method": "GET", "path": "/symbols", "function": get_symbols}], ) httpd.start() - symbol_url = urlunsplit( - ("http", "%s:%d" % httpd.httpd.server_address, "/symbols", "", "") - ) + symbol_url = urlunsplit(( + "http", + "%s:%d" % httpd.httpd.server_address, + "/symbols", + "", + "", + )) assert 1 == check_for_crashes(symbols_path=symbol_url) assert data["retrieved"] diff --git a/testing/mozbase/mozdebug/mozdebug/__init__.py b/testing/mozbase/mozdebug/mozdebug/__init__.py @@ -27,4 +27,5 @@ debugger-specific arguments: run_process(processArgs, ...) """ + from .mozdebug import * diff --git a/testing/mozbase/mozdevice/mozdevice/adb.py b/testing/mozbase/mozdevice/mozdevice/adb.py @@ -583,7 +583,7 @@ class ADBHost(ADBCommand): ) except ValueError: self._logger.warning( - "devices: Unable to parse " "remainder for device %s" % line + "devices: Unable to parse remainder for device %s" % line ) devices.append(device) for device in devices: @@ -639,9 +639,9 @@ def ADBDeviceFactory( # already created an ADBDevice which means we must only have # one device connected and we can re-use the existing ADBDevice. devices = list(ADBDEVICES.keys()) - assert ( - len(devices) == 1 - ), "Only one device may be connected if the device serial number is not specified." + assert len(devices) == 1, ( + "Only one device may be connected if the device serial number is not specified." + ) adbdevice = ADBDEVICES[devices[0]] elif ( device is not None @@ -750,67 +750,65 @@ class ADBDevice(ADBCommand): # BUILTINS is used to determine which commands can not be executed # via su or run-as. This set of possible builtin commands was # obtained from `man builtin` on Linux. - BUILTINS = set( - [ - "alias", - "bg", - "bind", - "break", - "builtin", - "caller", - "cd", - "command", - "compgen", - "complete", - "compopt", - "continue", - "declare", - "dirs", - "disown", - "echo", - "enable", - "eval", - "exec", - "exit", - "export", - "false", - "fc", - "fg", - "getopts", - "hash", - "help", - "history", - "jobs", - "kill", - "let", - "local", - "logout", - "mapfile", - "popd", - "printf", - "pushd", - "pwd", - "read", - "readonly", - "return", - "set", - "shift", - "shopt", - "source", - "suspend", - "test", - "times", - "trap", - "true", - "type", - "typeset", - "ulimit", - "umask", - "unalias", - "unset", - "wait", - ] - ) + BUILTINS = set([ + "alias", + "bg", + "bind", + "break", + "builtin", + "caller", + "cd", + "command", + "compgen", + "complete", + "compopt", + "continue", + "declare", + "dirs", + "disown", + "echo", + "enable", + "eval", + "exec", + "exit", + "export", + "false", + "fc", + "fg", + "getopts", + "hash", + "help", + "history", + "jobs", + "kill", + "let", + "local", + "logout", + "mapfile", + "popd", + "printf", + "pushd", + "pwd", + "read", + "readonly", + "return", + "set", + "shift", + "shopt", + "source", + "suspend", + "test", + "times", + "trap", + "true", + "type", + "typeset", + "ulimit", + "umask", + "unalias", + "unset", + "wait", + ]) def __init__( self, @@ -1600,13 +1598,11 @@ class ADBDevice(ADBCommand): if not self.is_rooted: # Note that /sdcard may be accessible while # /mnt/sdcard is not. - paths.extend( - [ - "/sdcard/test_root", - "/storage/sdcard/test_root", - "/mnt/sdcard/test_root", - ] - ) + paths.extend([ + "/sdcard/test_root", + "/storage/sdcard/test_root", + "/mnt/sdcard/test_root", + ]) return self._try_test_root_candidates(paths) @@ -3385,9 +3381,9 @@ class ADBDevice(ADBCommand): if "No such process" not in str(e): raise pid_set = set(pid_list) - current_pid_set = set( - [str(proc[0]) for proc in self.get_process_list(timeout=timeout)] - ) + current_pid_set = set([ + str(proc[0]) for proc in self.get_process_list(timeout=timeout) + ]) pid_list = list(pid_set.intersection(current_pid_set)) if not pid_list: break @@ -3683,9 +3679,9 @@ class ADBDevice(ADBCommand): if uptime: m = re.match(r"up time: ((\d+) days, )*(\d{2}):(\d{2}):(\d{2})", uptime) if m: - uptime = "%d days %d hours %d minutes %d seconds" % tuple( - [int(g or 0) for g in m.groups()[1:]] - ) + uptime = "%d days %d hours %d minutes %d seconds" % tuple([ + int(g or 0) for g in m.groups()[1:] + ]) info["uptime"] = uptime return info @@ -4219,9 +4215,7 @@ class ADBDevice(ADBCommand): # starting a new instance may not be what we want depending on what # we want to do if fail_if_running and self.process_exist(app_name, timeout=timeout): - raise ADBError( - "Only one instance of an application may be running " "at once" - ) + raise ADBError("Only one instance of an application may be running at once") if grant_runtime_permissions: self.grant_runtime_permissions(app_name) @@ -4229,12 +4223,10 @@ class ADBDevice(ADBCommand): acmd = ["am"] + ["startservice" if is_service else "start"] if wait: acmd.extend(["-W"]) - acmd.extend( - [ - "-n", - f"{app_name}/{activity_name}", - ] - ) + acmd.extend([ + "-n", + f"{app_name}/{activity_name}", + ]) if intent: acmd.extend(["-a", intent]) diff --git a/testing/mozbase/mozdevice/mozdevice/ios.py b/testing/mozbase/mozdevice/mozdevice/ios.py @@ -53,18 +53,16 @@ class IosDeviceReal(IosDevice): super().__init__(uuid, name, False) def install(self, app_bundle): - subprocess.check_call( - [ - "xcrun", - "devicectl", - "device", - "install", - "app", - "--device", - self.uuid, - app_bundle, - ] - ) + subprocess.check_call([ + "xcrun", + "devicectl", + "device", + "install", + "app", + "--device", + self.uuid, + app_bundle, + ]) @staticmethod def all_devices(): @@ -106,7 +104,7 @@ class IosDeviceSimulator(IosDevice): env=None, wait_for_debugger=False, terminate_running_process=True, - **kwargs + **kwargs, ): # Put provided environment variables in `SIMCTL_CHILD_` so they # propagate into the simulator. @@ -139,7 +137,7 @@ class IosDeviceSimulator(IosDevice): bundle_id, *args, ], - **kwargs + **kwargs, ) def test_root(self, bundle_id): @@ -205,9 +203,13 @@ class IosDeviceSimulator(IosDevice): def stop_application(self, bundle_id): try: - subprocess.check_call( - ["xcrun", "simctl", "terminate", self.uuid, bundle_id] - ) + subprocess.check_call([ + "xcrun", + "simctl", + "terminate", + self.uuid, + bundle_id, + ]) except subprocess.CalledProcessError: pass diff --git a/testing/mozbase/mozdevice/mozdevice/version_codes.py b/testing/mozbase/mozdevice/mozdevice/version_codes.py @@ -7,6 +7,7 @@ VERSION CODES of the android releases. See https://developer.android.com/reference/android/os/Build.VERSION_CODES.html. """ + # Magic version number for a current development build, which has # not yet turned into an official release. CUR_DEVELOPMENT = 10000 diff --git a/testing/mozbase/mozfile/mozfile/mozfile.py b/testing/mozbase/mozfile/mozfile/mozfile.py @@ -434,19 +434,17 @@ def tree(directory, sort_key=lambda x: x.lower()): # add the files if filenames: last_file = filenames[-1] - retval.extend( - [ - ( - "%s%s%s" - % ( - "".join(indent), - files_end if filename == last_file else item_marker, - filename, - ) + retval.extend([ + ( + "%s%s%s" + % ( + "".join(indent), + files_end if filename == last_file else item_marker, + filename, ) - for index, filename in enumerate(filenames) - ] - ) + ) + for index, filename in enumerate(filenames) + ]) return "\n".join(retval) diff --git a/testing/mozbase/mozfile/tests/test_tempfile.py b/testing/mozbase/mozfile/tests/test_tempfile.py @@ -7,6 +7,7 @@ """ tests for mozfile.NamedTemporaryFile """ + import os import unittest diff --git a/testing/mozbase/mozfile/tests/test_url.py b/testing/mozbase/mozfile/tests/test_url.py @@ -3,6 +3,7 @@ """ tests for is_url """ + import unittest import mozunit diff --git a/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/__init__.py b/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/__init__.py @@ -5,6 +5,7 @@ """ mozgeckoprofiler has utilities to symbolicate and load gecko profiles. """ + from .profiling import save_gecko_profile, symbolicate_profile_json from .symbolication import ProfileSymbolicator from .viewgeckoprofile import view_gecko_profile diff --git a/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/profiling.py b/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/profiling.py @@ -38,36 +38,33 @@ def symbolicate_profile_json(profile_path, firefox_symbols_path): symbol_paths = {"FIREFOX": firefox_symbols_path, "WINDOWS": windows_symbol_path} - symbolicator = ProfileSymbolicator( - { - # Trace-level logging (verbose) - "enableTracing": 0, - # Fallback server if symbol is not found locally - "remoteSymbolServer": "https://symbolication.services.mozilla.com/symbolicate/v4", - # Maximum number of symbol files to keep in memory - "maxCacheEntries": 2000000, - # Frequency of checking for recent symbols to - # cache (in hours) - "prefetchInterval": 12, - # Oldest file age to prefetch (in hours) - "prefetchThreshold": 48, - # Maximum number of library versions to pre-fetch - # per library - "prefetchMaxSymbolsPerLib": 3, - # Default symbol lookup directories - "defaultApp": "FIREFOX", - "defaultOs": "WINDOWS", - # Paths to .SYM files, expressed internally as a - # mapping of app or platform names to directories - # Note: App & OS names from requests are converted - # to all-uppercase internally - "symbolPaths": symbol_paths, - } - ) + symbolicator = ProfileSymbolicator({ + # Trace-level logging (verbose) + "enableTracing": 0, + # Fallback server if symbol is not found locally + "remoteSymbolServer": "https://symbolication.services.mozilla.com/symbolicate/v4", + # Maximum number of symbol files to keep in memory + "maxCacheEntries": 2000000, + # Frequency of checking for recent symbols to + # cache (in hours) + "prefetchInterval": 12, + # Oldest file age to prefetch (in hours) + "prefetchThreshold": 48, + # Maximum number of library versions to pre-fetch + # per library + "prefetchMaxSymbolsPerLib": 3, + # Default symbol lookup directories + "defaultApp": "FIREFOX", + "defaultOs": "WINDOWS", + # Paths to .SYM files, expressed internally as a + # mapping of app or platform names to directories + # Note: App & OS names from requests are converted + # to all-uppercase internally + "symbolPaths": symbol_paths, + }) LOG.info( - "Symbolicating the performance profile... This could take a couple " - "of minutes." + "Symbolicating the performance profile... This could take a couple of minutes." ) try: @@ -82,7 +79,7 @@ def symbolicate_profile_json(profile_path, firefox_symbols_path): save_gecko_profile(profile, profile_path) except MemoryError: LOG.error( - "Ran out of memory while trying" f" to symbolicate profile {profile_path}" + f"Ran out of memory while trying to symbolicate profile {profile_path}" ) except Exception as e: LOG.error("Encountered an exception during profile symbolication") diff --git a/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/symbolication.py b/testing/mozbase/mozgeckoprofiler/mozgeckoprofiler/symbolication.py @@ -298,7 +298,6 @@ class ProfileSymbolicator: # Check if running in CI if "MOZ_AUTOMATION" in os.environ: - moz_fetch = os.environ["MOZ_FETCHES_DIR"] symbolicator_path = Path( moz_fetch, "symbolicator-cli", "symbolicator-cli.js" @@ -314,9 +313,11 @@ class ProfileSymbolicator: # Bug 2000026: Temporarily use fallback symbolication for --extra-profiler-run # since those tasks don't have the toolchains for symbolicator-cli symbolication yet. - if not self._validate_symbolication_deps( - [symbolicator_path, samply_path, node_path] - ): + if not self._validate_symbolication_deps([ + symbolicator_path, + samply_path, + node_path, + ]): LOG.info( "Symbolication dependencies not available, using fallback symbolication." ) @@ -327,7 +328,6 @@ class ProfileSymbolicator: breakpad_symbol_dir = self.options["symbolPaths"]["FIREFOX"] with tempfile.TemporaryDirectory() as work_dir: - unsym_profile = Path(work_dir, "unsym_profile.json") unsym_profile.write_text( json.dumps(profile_json, ensure_ascii=False), encoding="utf-8" diff --git a/testing/mozbase/mozhttpd/mozhttpd/__init__.py b/testing/mozbase/mozhttpd/mozhttpd/__init__.py @@ -41,6 +41,7 @@ content from the current directory, defines a single API endpoint httpd.start(block=True) """ + from .handlers import json_response from .mozhttpd import MozHttpd, Request, RequestHandler, main diff --git a/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py b/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py @@ -77,9 +77,11 @@ class RequestHandler(SimpleHTTPRequestHandler): def _try_handler(self, method): if self.log_requests: - self.request_log.append( - {"method": method, "path": self.request.path, "time": time.time()} - ) + self.request_log.append({ + "method": method, + "path": self.request.path, + "time": time.time(), + }) handlers = [ handler for handler in self.urlhandlers if handler["method"] == method diff --git a/testing/mozbase/mozhttpd/tests/api.py b/testing/mozbase/mozhttpd/tests/api.py @@ -310,11 +310,9 @@ def fixture_httpd_with_proxy_handler(docroot): httpd.start(block=False) port = httpd.httpd.server_port - proxy_support = ProxyHandler( - { - "http": f"http://127.0.0.1:{port:d}", - } - ) + proxy_support = ProxyHandler({ + "http": f"http://127.0.0.1:{port:d}", + }) install_opener(build_opener(proxy_support)) yield httpd diff --git a/testing/mozbase/mozinfo/mozinfo/mozinfo.py b/testing/mozbase/mozinfo/mozinfo/mozinfo.py @@ -150,12 +150,10 @@ elif processor == "arm" and bits == "64bit": processor = "aarch64" bits = re.search(r"(\d+)bit", bits).group(1) -info.update( - { - "processor": processor, - "bits": int(bits), - } -) +info.update({ + "processor": processor, + "bits": int(bits), +}) if info.get("arch", "") != "aarch64": info["arch"] = info["processor"] diff --git a/testing/mozbase/mozinfo/mozinfo/platforminfo.py b/testing/mozbase/mozinfo/mozinfo/platforminfo.py @@ -58,7 +58,6 @@ def android_api_to_os_version(api_version: str): class PlatformInfo: - variant_data = {} buildmap = { diff --git a/testing/mozbase/mozinstall/mozinstall/mozinstall.py b/testing/mozbase/mozinstall/mozinstall/mozinstall.py @@ -463,7 +463,7 @@ def install_cli(argv=sys.argv[1:]): "--destination", dest="dest", default=os.getcwd(), - help="Directory to install application into. " '[default: "%default"]', + help='Directory to install application into. [default: "%default"]', ) parser.add_option( "--app", diff --git a/testing/mozbase/mozlog/mozlog/commandline.py b/testing/mozbase/mozlog/mozlog/commandline.py @@ -14,22 +14,22 @@ from .structuredlog import StructuredLogger, set_default_logger log_formatters = { "raw": ( formatters.JSONFormatter, - "Raw structured log messages " "(provided by mozlog)", + "Raw structured log messages (provided by mozlog)", ), "unittest": ( formatters.UnittestFormatter, - "Unittest style output " "(provided by mozlog)", + "Unittest style output (provided by mozlog)", ), "xunit": ( formatters.XUnitFormatter, - "xUnit compatible XML " "(provided by mozlog)", + "xUnit compatible XML (provided by mozlog)", ), - "html": (formatters.HTMLFormatter, "HTML report " "(provided by mozlog)"), - "mach": (formatters.MachFormatter, "Human-readable output " "(provided by mozlog)"), - "tbpl": (formatters.TbplFormatter, "TBPL style log format " "(provided by mozlog)"), + "html": (formatters.HTMLFormatter, "HTML report (provided by mozlog)"), + "mach": (formatters.MachFormatter, "Human-readable output (provided by mozlog)"), + "tbpl": (formatters.TbplFormatter, "TBPL style log format (provided by mozlog)"), "grouped": ( formatters.GroupingFormatter, - "Grouped summary of test results " "(provided by mozlog)", + "Grouped summary of test results (provided by mozlog)", ), "errorsummary": (formatters.ErrorSummaryFormatter, argparse.SUPPRESS), } diff --git a/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py b/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py @@ -163,13 +163,11 @@ class ErrorSummaryFormatter(BaseFormatter): data["group"] = data["group"][0] self.groups[data["group"]]["status"] = "FAIL" else: - self.log( - { - "level": "ERROR", - "message": "Group '%s' was not found in known groups: %s. Please look at item: %s" - % (item["group"], self.groups, item), - } - ) + self.log({ + "level": "ERROR", + "message": "Group '%s' was not found in known groups: %s. Please look at item: %s" + % (item["group"], self.groups, item), + }) return self._output("log", data) def crash(self, item): diff --git a/testing/mozbase/mozlog/mozlog/formatters/grouping.py b/testing/mozbase/mozlog/mozlog/formatters/grouping.py @@ -155,7 +155,7 @@ class GroupingFormatter(base.BaseFormatter): def wrap_and_indent_lines(self, lines, indent): assert len(lines) > 0 - output = indent + "\u25B6 %s\n" % lines[0] + output = indent + "\u25b6 %s\n" % lines[0] for line in lines[1:-1]: output += indent + "\u2502 %s\n" % line if len(lines) > 1: diff --git a/testing/mozbase/mozlog/mozlog/formatters/html/html.py b/testing/mozbase/mozlog/mozlog/formatters/html/html.py @@ -84,12 +84,10 @@ class HTMLFormatter(base.BaseFormatter): if version_info.get("application_repository"): self.env["Gecko revision"] = html.a( version_info.get("application_changeset"), - href="/rev/".join( - [ - version_info.get("application_repository"), - version_info.get("application_changeset"), - ] - ), + href="/rev/".join([ + version_info.get("application_repository"), + version_info.get("application_changeset"), + ]), target="_blank", ) @@ -311,20 +309,16 @@ class HTMLFormatter(base.BaseFormatter): html.table( [ html.thead( - html.tr( - [ - html.th( - "Result", class_="sortable", col="result" - ), - html.th("Test", class_="sortable", col="name"), - html.th( - "Duration", - class_="sortable numeric", - col="duration", - ), - html.th("Links"), - ] - ), + html.tr([ + html.th("Result", class_="sortable", col="result"), + html.th("Test", class_="sortable", col="name"), + html.th( + "Duration", + class_="sortable numeric", + col="duration", + ), + html.th("Links"), + ]), id="results-table-head", ), html.tbody(self.result_rows, id="results-table-body"), diff --git a/testing/mozbase/mozlog/mozlog/formatters/html/xmlgen.py b/testing/mozbase/mozlog/mozlog/formatters/html/xmlgen.py @@ -100,24 +100,22 @@ class HtmlTag(Tag): class html(Namespace): __tagclass__ = HtmlTag __stickyname__ = True - __tagspec__ = dict( - [ - (x, 1) - for x in ( - "a,abbr,acronym,address,applet,area,b,bdo,big,blink," - "blockquote,body,br,button,caption,center,cite,code,col," - "colgroup,comment,dd,del,dfn,dir,div,dl,dt,em,embed," - "fieldset,font,form,frameset,h1,h2,h3,h4,h5,h6,head,html," - "i,iframe,img,input,ins,kbd,label,legend,li,link,listing," - "map,marquee,menu,meta,multicol,nobr,noembed,noframes," - "noscript,object,ol,optgroup,option,p,pre,q,s,script," - "select,small,span,strike,strong,style,sub,sup,table," - "tbody,td,textarea,tfoot,th,thead,title,tr,tt,u,ul,xmp," - "base,basefont,frame,hr,isindex,param,samp,var" - ).split(",") - if x - ] - ) + __tagspec__ = dict([ + (x, 1) + for x in ( + "a,abbr,acronym,address,applet,area,b,bdo,big,blink," + "blockquote,body,br,button,caption,center,cite,code,col," + "colgroup,comment,dd,del,dfn,dir,div,dl,dt,em,embed," + "fieldset,font,form,frameset,h1,h2,h3,h4,h5,h6,head,html," + "i,iframe,img,input,ins,kbd,label,legend,li,link,listing," + "map,marquee,menu,meta,multicol,nobr,noembed,noframes," + "noscript,object,ol,optgroup,option,p,pre,q,s,script," + "select,small,span,strike,strong,style,sub,sup,table," + "tbody,td,textarea,tfoot,th,thead,title,tr,tt,u,ul,xmp," + "base,basefont,frame,hr,isindex,param,samp,var" + ).split(",") + if x + ]) class Style: def __init__(self, **kw): @@ -245,24 +243,18 @@ class SimpleUnicodeVisitor: class HtmlVisitor(SimpleUnicodeVisitor): - single = dict( - [ - (x, 1) - for x in ("br,img,area,param,col,hr,meta,link,base," "input,frame").split( - "," - ) - ] - ) - inline = dict( - [ - (x, 1) - for x in ( - "a abbr acronym b basefont bdo big br cite code dfn em font " - "i img input kbd label q s samp select small span strike " - "strong sub sup textarea tt u var".split(" ") - ) - ] - ) + single = dict([ + (x, 1) + for x in ("br,img,area,param,col,hr,meta,link,base,input,frame").split(",") + ]) + inline = dict([ + (x, 1) + for x in ( + "a abbr acronym b basefont bdo big br cite code dfn em font " + "i img input kbd label q s samp select small span strike " + "strong sub sup textarea tt u var".split(" ") + ) + ]) def repr_attribute(self, attrs, name): if name == "class_": diff --git a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py @@ -250,9 +250,9 @@ class MachFormatter(base.BaseFormatter): for key in ("test", "subtest", "assert"): if not count[key]["unexpected"]: continue - status_str = ", ".join( - [f"{n} {s}" for s, n in sorted(count[key]["unexpected"].items())] - ) + status_str = ", ".join([ + f"{n} {s}" for s, n in sorted(count[key]["unexpected"].items()) + ]) rv.append( " {}: {} ({})".format( key, sum(count[key]["unexpected"].values()), status_str @@ -262,13 +262,11 @@ class MachFormatter(base.BaseFormatter): # Format intermittents if intermittents > 0: heading = "Known Intermittent Results" - rv.extend( - [ - "", - self.color_formatter.heading(heading), - self.color_formatter.heading("-" * len(heading)), - ] - ) + rv.extend([ + "", + self.color_formatter.heading(heading), + self.color_formatter.heading("-" * len(heading)), + ]) if count["subtest"]["count"]: for test_id, results in intermittent_logs.items(): test = self._get_file_name(test_id) @@ -291,13 +289,11 @@ class MachFormatter(base.BaseFormatter): else: # Format test failures heading = "Error Summary" - rv.extend( - [ - "", - self.color_formatter.heading(heading), - self.color_formatter.heading("-" * len(heading)), - ] - ) + rv.extend([ + "", + self.color_formatter.heading(heading), + self.color_formatter.heading("-" * len(heading)), + ]) if count["subtest"]["count"]: for test_id, results in logs.items(): test = self._get_file_name(test_id) @@ -448,7 +444,7 @@ class MachFormatter(base.BaseFormatter): ) status = self.color_formatter.log_test_status_pass("FAIL") - return "%s leakcheck: " "%s missing output line for total leaks!\n" % ( + return "%s leakcheck: %s missing output line for total leaks!\n" % ( status, data["process"], ) diff --git a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py @@ -61,9 +61,11 @@ class TbplFormatter(BaseFormatter): if subtract_context: count -= len(self.buffer) self.subtests_count = 0 - return self._log( - {"level": "INFO", "message": "." * count, "component": component} - ) + return self._log({ + "level": "INFO", + "message": "." * count, + "component": component, + }) @output_subtests def log(self, data): diff --git a/testing/mozbase/mozlog/mozlog/formatters/unittest.py b/testing/mozbase/mozlog/mozlog/formatters/unittest.py @@ -46,15 +46,13 @@ class UnittestFormatter(base.BaseFormatter): char = "X" elif data["count"] > data["max_expected"]: char = "F" - self.fails.append( - { - "test": data["test"], - "message": ( - "assertion count %i is greated than %i" - % (data["count"], data["max_expected"]) - ), - } - ) + self.fails.append({ + "test": data["test"], + "message": ( + "assertion count %i is greated than %i" + % (data["count"], data["max_expected"]) + ), + }) elif data["count"] > 0: char = "." else: @@ -64,9 +62,11 @@ class UnittestFormatter(base.BaseFormatter): def suite_end(self, data): self.end_time = data["time"] - summary = "\n".join( - [self.output_fails(), self.output_errors(), self.output_summary()] - ) + summary = "\n".join([ + self.output_fails(), + self.output_errors(), + self.output_summary(), + ]) return "\n%s\n" % summary def output_fails(self): diff --git a/testing/mozbase/mozlog/mozlog/formatters/xunit.py b/testing/mozbase/mozlog/mozlog/formatters/xunit.py @@ -97,15 +97,13 @@ class XUnitFormatter(base.BaseFormatter): self._create_result(data) def suite_end(self, data): - self.root.attrib.update( - { - "tests": str(self.tests_run), - "errors": str(self.errors), - "failures": str(self.failures), - "skips": str(self.skips), - "time": "%.2f" % ((data["time"] - self.suite_start_time) / 1000.0), - } - ) + self.root.attrib.update({ + "tests": str(self.tests_run), + "errors": str(self.errors), + "failures": str(self.failures), + "skips": str(self.skips), + "time": "%.2f" % ((data["time"] - self.suite_start_time) / 1000.0), + }) xml_string = ElementTree.tostring(self.root, encoding="utf8") # pretty printing can not be done from xml.etree from xml.dom import minidom diff --git a/testing/mozbase/mozlog/tests/test_formatters.py b/testing/mozbase/mozlog/tests/test_formatters.py @@ -331,9 +331,7 @@ def test_fail(get_logger, name, opts, expected): stack = """ SimpleTest.is@SimpleTest/SimpleTest.js:312:5 @caps/tests/mochitest/test_bug246699.html:53:1 -""".strip( - "\n" - ) +""".strip("\n") logger = get_logger(name, **opts) diff --git a/testing/mozbase/mozlog/tests/test_logger.py b/testing/mozbase/mozlog/tests/test_logger.py @@ -199,28 +199,22 @@ class TestStructuredLogging(unittest.TestCase): connection, self.logger, message_callback=self.message_callback, timeout=0.5 ) - message_string_one = json.dumps( - { - "_message": "socket message one", - "action": "test_message", - "_level": "DEBUG", - } - ) - message_string_two = json.dumps( - { - "_message": "socket message two", - "action": "test_message", - "_level": "DEBUG", - } - ) + message_string_one = json.dumps({ + "_message": "socket message one", + "action": "test_message", + "_level": "DEBUG", + }) + message_string_two = json.dumps({ + "_message": "socket message two", + "action": "test_message", + "_level": "DEBUG", + }) - message_string_three = json.dumps( - { - "_message": "socket message three", - "action": "test_message", - "_level": "DEBUG", - } - ) + message_string_three = json.dumps({ + "_message": "socket message three", + "action": "test_message", + "_level": "DEBUG", + }) message_string = ( message_string_one diff --git a/testing/mozbase/mozlog/tests/test_structured.py b/testing/mozbase/mozlog/tests/test_structured.py @@ -235,9 +235,11 @@ class TestSummaryHandler(BaseStructuredTest): class TestStructuredLog(BaseStructuredTest): def test_suite_start(self): self.logger.suite_start(["test"], "logtest") - self.assert_log_equals( - {"action": "suite_start", "name": "logtest", "tests": {"default": ["test"]}} - ) + self.assert_log_equals({ + "action": "suite_start", + "name": "logtest", + "tests": {"default": ["test"]}, + }) self.logger.suite_end() def test_suite_end(self): @@ -248,13 +250,11 @@ class TestStructuredLog(BaseStructuredTest): def test_add_subsuite(self): self.logger.suite_start([]) self.logger.add_subsuite("other") - self.assert_log_equals( - { - "action": "add_subsuite", - "name": "other", - "run_info": {"subsuite": "other"}, - } - ) + self.assert_log_equals({ + "action": "add_subsuite", + "name": "other", + "run_info": {"subsuite": "other"}, + }) self.logger.suite_end() def test_add_subsuite_duplicate(self): @@ -262,13 +262,11 @@ class TestStructuredLog(BaseStructuredTest): self.logger.add_subsuite("other") # This should be a no-op self.logger.add_subsuite("other") - self.assert_log_equals( - { - "action": "add_subsuite", - "name": "other", - "run_info": {"subsuite": "other"}, - } - ) + self.assert_log_equals({ + "action": "add_subsuite", + "name": "other", + "run_info": {"subsuite": "other"}, + }) self.assert_log_equals({"action": "suite_start", "tests": {"default": []}}) self.logger.suite_end() @@ -279,26 +277,22 @@ class TestStructuredLog(BaseStructuredTest): self.assert_log_equals({"action": "test_start", "test": "test1"}) self.logger.test_start(("test1", "==", "test1-ref"), path="path/to/test") - self.assert_log_equals( - { - "action": "test_start", - "test": ("test1", "==", "test1-ref"), - "path": "path/to/test", - } - ) + self.assert_log_equals({ + "action": "test_start", + "test": ("test1", "==", "test1-ref"), + "path": "path/to/test", + }) self.logger.suite_end() def test_start_inprogress(self): self.logger.suite_start([]) self.logger.test_start("test1") self.logger.test_start("test1") - self.assert_log_equals( - { - "action": "log", - "message": "test_start for test1 logged while in progress.", - "level": "ERROR", - } - ) + self.assert_log_equals({ + "action": "log", + "message": "test_start for test1 logged while in progress.", + "level": "ERROR", + }) self.logger.suite_end() def test_start_inprogress_subsuite(self): @@ -306,13 +300,11 @@ class TestStructuredLog(BaseStructuredTest): self.logger.add_subsuite("other") self.logger.test_start("test1") self.logger.test_start("test1", subsuite="other") - self.assert_log_equals( - { - "action": "test_start", - "test": "test1", - "subsuite": "other", - } - ) + self.assert_log_equals({ + "action": "test_start", + "test": "test1", + "subsuite": "other", + }) self.logger.suite_end() def test_status(self): @@ -321,15 +313,13 @@ class TestStructuredLog(BaseStructuredTest): self.logger.test_status( "test1", "subtest name", "fail", expected="FAIL", message="Test message" ) - self.assert_log_equals( - { - "action": "test_status", - "subtest": "subtest name", - "status": "FAIL", - "message": "Test message", - "test": "test1", - } - ) + self.assert_log_equals({ + "action": "test_status", + "subtest": "subtest name", + "status": "FAIL", + "message": "Test message", + "test": "test1", + }) self.logger.test_end("test1", "OK") self.logger.suite_end() @@ -337,15 +327,13 @@ class TestStructuredLog(BaseStructuredTest): self.logger.suite_start([]) self.logger.test_start("test1") self.logger.test_status("test1", "subtest name", "fail") - self.assert_log_equals( - { - "action": "test_status", - "subtest": "subtest name", - "status": "FAIL", - "expected": "PASS", - "test": "test1", - } - ) + self.assert_log_equals({ + "action": "test_status", + "subtest": "subtest name", + "status": "FAIL", + "expected": "PASS", + "test": "test1", + }) self.logger.test_end("test1", "OK") self.logger.suite_end() @@ -364,16 +352,14 @@ class TestStructuredLog(BaseStructuredTest): self.logger.test_status( "test1", "subtest name", "FAIL", expected="PASS", extra={"data": 42} ) - self.assert_log_equals( - { - "action": "test_status", - "subtest": "subtest name", - "status": "FAIL", - "expected": "PASS", - "test": "test1", - "extra": {"data": 42}, - } - ) + self.assert_log_equals({ + "action": "test_status", + "subtest": "subtest name", + "status": "FAIL", + "expected": "PASS", + "test": "test1", + "extra": {"data": 42}, + }) self.logger.test_end("test1", "OK") self.logger.suite_end() @@ -387,16 +373,14 @@ class TestStructuredLog(BaseStructuredTest): expected="PASS", stack="many\nlines\nof\nstack", ) - self.assert_log_equals( - { - "action": "test_status", - "subtest": "subtest name", - "status": "FAIL", - "expected": "PASS", - "test": "test1", - "stack": "many\nlines\nof\nstack", - } - ) + self.assert_log_equals({ + "action": "test_status", + "subtest": "subtest name", + "status": "FAIL", + "expected": "PASS", + "test": "test1", + "stack": "many\nlines\nof\nstack", + }) self.logger.test_end("test1", "OK") self.logger.suite_end() @@ -406,16 +390,14 @@ class TestStructuredLog(BaseStructuredTest): self.logger.test_status( "test1", "subtest name", "fail", known_intermittent=["FAIL"] ) - self.assert_log_equals( - { - "action": "test_status", - "subtest": "subtest name", - "status": "FAIL", - "expected": "PASS", - "known_intermittent": ["FAIL"], - "test": "test1", - } - ) + self.assert_log_equals({ + "action": "test_status", + "subtest": "subtest name", + "status": "FAIL", + "expected": "PASS", + "known_intermittent": ["FAIL"], + "test": "test1", + }) self.logger.test_end("test1", "OK") self.logger.suite_end() @@ -433,15 +415,13 @@ class TestStructuredLog(BaseStructuredTest): self.logger.test_status( "test1", "subtest name", "fail", message=None, stack=None ) - self.assert_log_equals( - { - "action": "test_status", - "subtest": "subtest name", - "status": "FAIL", - "expected": "PASS", - "test": "test1", - } - ) + self.assert_log_equals({ + "action": "test_status", + "subtest": "subtest name", + "status": "FAIL", + "expected": "PASS", + "test": "test1", + }) self.logger.test_end("test1", "OK") self.logger.suite_end() @@ -454,29 +434,25 @@ class TestStructuredLog(BaseStructuredTest): self.logger.suite_start([]) self.logger.test_start("test1") self.logger.test_end("test1", "fail", message="Test message") - self.assert_log_equals( - { - "action": "test_end", - "status": "FAIL", - "expected": "OK", - "message": "Test message", - "test": "test1", - } - ) + self.assert_log_equals({ + "action": "test_end", + "status": "FAIL", + "expected": "OK", + "message": "Test message", + "test": "test1", + }) self.logger.suite_end() def test_end_1(self): self.logger.suite_start([]) self.logger.test_start("test1") self.logger.test_end("test1", "PASS", expected="PASS", extra={"data": 123}) - self.assert_log_equals( - { - "action": "test_end", - "status": "PASS", - "extra": {"data": 123}, - "test": "test1", - } - ) + self.assert_log_equals({ + "action": "test_end", + "status": "PASS", + "extra": {"data": 123}, + "test": "test1", + }) self.logger.suite_end() def test_end_2(self): @@ -488,14 +464,12 @@ class TestStructuredLog(BaseStructuredTest): self.logger.test_end( "test1", "PASS", expected="PASS", stack="many\nlines\nof\nstack" ) - self.assert_log_equals( - { - "action": "test_end", - "status": "PASS", - "test": "test1", - "stack": "many\nlines\nof\nstack", - } - ) + self.assert_log_equals({ + "action": "test_end", + "status": "PASS", + "test": "test1", + "stack": "many\nlines\nof\nstack", + }) self.logger.suite_end() def test_end_no_start(self): @@ -518,23 +492,23 @@ class TestStructuredLog(BaseStructuredTest): ) ) self.logger.test_end("test1", "OK", subsuite="other") - self.assert_log_equals( - { - "action": "test_end", - "status": "OK", - "test": "test1", - "subsuite": "other", - } - ) + self.assert_log_equals({ + "action": "test_end", + "status": "OK", + "test": "test1", + "subsuite": "other", + }) self.logger.suite_end() def test_end_twice(self): self.logger.suite_start([]) self.logger.test_start("test2") self.logger.test_end("test2", "PASS", expected="PASS") - self.assert_log_equals( - {"action": "test_end", "status": "PASS", "test": "test2"} - ) + self.assert_log_equals({ + "action": "test_end", + "status": "PASS", + "test": "test2", + }) self.logger.test_end("test2", "PASS", expected="PASS") last_item = self.pop_last_item() self.assertEqual(last_item["action"], "log") @@ -584,9 +558,11 @@ class TestStructuredLog(BaseStructuredTest): def test_process(self): self.logger.process_output(1234, "test output") - self.assert_log_equals( - {"action": "process_output", "process": "1234", "data": "test output"} - ) + self.assert_log_equals({ + "action": "process_output", + "process": "1234", + "data": "test output", + }) def test_process_start(self): self.logger.process_start(1234) @@ -594,16 +570,20 @@ class TestStructuredLog(BaseStructuredTest): def test_process_exit(self): self.logger.process_exit(1234, 0) - self.assert_log_equals( - {"action": "process_exit", "process": "1234", "exitcode": 0} - ) + self.assert_log_equals({ + "action": "process_exit", + "process": "1234", + "exitcode": 0, + }) def test_log(self): for level in ["critical", "error", "warning", "info", "debug"]: getattr(self.logger, level)("message") - self.assert_log_equals( - {"action": "log", "level": level.upper(), "message": "message"} - ) + self.assert_log_equals({ + "action": "log", + "level": level.upper(), + "message": "message", + }) def test_logging_adapter(self): import logging @@ -620,9 +600,11 @@ class TestStructuredLog(BaseStructuredTest): try: for level in ["critical", "error", "warning", "info", "debug"]: getattr(logger, level)("message") - self.assert_log_equals( - {"action": "log", "level": level.upper(), "message": "message"} - ) + self.assert_log_equals({ + "action": "log", + "level": level.upper(), + "message": "message", + }) finally: logging.root.setLevel(old_level) @@ -672,9 +654,12 @@ class TestStructuredLog(BaseStructuredTest): log = structuredlog.StructuredLogger("test 1") log.add_handler(self.handler) log.info("line 1") - self.assert_log_equals( - {"action": "log", "level": "INFO", "message": "line 1", "source": "test 1"} - ) + self.assert_log_equals({ + "action": "log", + "level": "INFO", + "message": "line 1", + "source": "test 1", + }) log.shutdown() self.assert_log_equals({"action": "shutdown", "source": "test 1"}) with self.assertRaises(structuredlog.LoggerShutdownError): @@ -692,14 +677,12 @@ class TestStructuredLog(BaseStructuredTest): with structuredlog.StructuredLogger("test 2") as log: log.add_handler(self.handler) log.info("line 2") - self.assert_log_equals( - { - "action": "log", - "level": "INFO", - "message": "line 2", - "source": "test 2", - } - ) + self.assert_log_equals({ + "action": "log", + "level": "INFO", + "message": "line 2", + "source": "test 2", + }) self.assert_log_equals({"action": "shutdown", "source": "test 2"}) # shutdown prevents logging across instances @@ -713,27 +696,25 @@ class TestStructuredLog(BaseStructuredTest): class TestTypeConversions(BaseStructuredTest): def test_raw(self): self.logger.log_raw({"action": "suite_start", "tests": [1], "time": "1234"}) - self.assert_log_equals( - {"action": "suite_start", "tests": {"default": ["1"]}, "time": 1234} - ) + self.assert_log_equals({ + "action": "suite_start", + "tests": {"default": ["1"]}, + "time": 1234, + }) self.logger.suite_end() def test_tuple(self): self.logger.suite_start([]) - self.logger.test_start( - ( - b"\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90" - b"\x8d\x83\xf0\x90\x8d\x84".decode(), - 42, - "\u16a4", - ) - ) - self.assert_log_equals( - { - "action": "test_start", - "test": ("\U00010344\U00010334\U00010343\U00010344", "42", "\u16a4"), - } - ) + self.logger.test_start(( + b"\xf0\x90\x8d\x84\xf0\x90\x8c\xb4\xf0\x90" + b"\x8d\x83\xf0\x90\x8d\x84".decode(), + 42, + "\u16a4", + )) + self.assert_log_equals({ + "action": "test_start", + "test": ("\U00010344\U00010334\U00010343\U00010344", "42", "\u16a4"), + }) self.logger.suite_end() def test_non_string_messages(self): @@ -741,13 +722,11 @@ class TestTypeConversions(BaseStructuredTest): self.logger.info(1) self.assert_log_equals({"action": "log", "message": "1", "level": "INFO"}) self.logger.info([1, (2, "3"), "s", "s" + chr(255)]) - self.assert_log_equals( - { - "action": "log", - "message": "[1, (2, '3'), 's', 's\xff']", - "level": "INFO", - } - ) + self.assert_log_equals({ + "action": "log", + "message": "[1, (2, '3'), 's', 's\xff']", + "level": "INFO", + }) self.logger.suite_end() @@ -769,29 +748,27 @@ class TestTypeConversions(BaseStructuredTest): self.assert_log_equals({"action": "log", "message": "test", "level": "INFO"}) self.logger.suite_start([], run_info={}) - self.assert_log_equals( - {"action": "suite_start", "tests": {"default": []}, "run_info": {}} - ) + self.assert_log_equals({ + "action": "suite_start", + "tests": {"default": []}, + "run_info": {}, + }) self.logger.test_start(test="test1") self.logger.test_status("subtest1", "FAIL", test="test1", status="PASS") - self.assert_log_equals( - { - "action": "test_status", - "test": "test1", - "subtest": "subtest1", - "status": "PASS", - "expected": "FAIL", - } - ) + self.assert_log_equals({ + "action": "test_status", + "test": "test1", + "subtest": "subtest1", + "status": "PASS", + "expected": "FAIL", + }) self.logger.process_output(123, "data", "test") - self.assert_log_equals( - { - "action": "process_output", - "process": "123", - "command": "test", - "data": "data", - } - ) + self.assert_log_equals({ + "action": "process_output", + "process": "123", + "command": "test", + "data": "data", + }) self.assertRaises( TypeError, self.logger.test_status, @@ -832,14 +809,12 @@ class TestComponentFilter(BaseStructuredTest): component_logger.info("Test 1") self.assertFalse(self.handler.empty) - self.assert_log_equals( - { - "action": "log", - "level": "INFO", - "message": "Test 1", - "component": "test_component", - } - ) + self.assert_log_equals({ + "action": "log", + "level": "INFO", + "message": "Test 1", + "component": "test_component", + }) component_logger.debug("Test 2") self.assertTrue(self.handler.empty) @@ -848,14 +823,12 @@ class TestComponentFilter(BaseStructuredTest): component_logger.debug("Test 3") self.assertFalse(self.handler.empty) - self.assert_log_equals( - { - "action": "log", - "level": "DEBUG", - "message": "Test 3", - "component": "test_component", - } - ) + self.assert_log_equals({ + "action": "log", + "level": "DEBUG", + "message": "Test 3", + "component": "test_component", + }) def test_filter_default_component(self): component_logger = structuredlog.StructuredLogger( @@ -873,14 +846,12 @@ class TestComponentFilter(BaseStructuredTest): component_logger.debug("Test 2") self.assertFalse(self.handler.empty) - self.assert_log_equals( - { - "action": "log", - "level": "DEBUG", - "message": "Test 2", - "component": "test_component", - } - ) + self.assert_log_equals({ + "action": "log", + "level": "DEBUG", + "message": "Test 2", + "component": "test_component", + }) self.logger.component_filter = None @@ -896,9 +867,11 @@ class TestComponentFilter(BaseStructuredTest): self.logger.component_filter = filter_mutate self.logger.debug("Test") - self.assert_log_equals( - {"action": "log", "level": "DEBUG", "message": "FILTERED! Test"} - ) + self.assert_log_equals({ + "action": "log", + "level": "DEBUG", + "message": "FILTERED! Test", + }) self.logger.component_filter = None @@ -965,9 +938,10 @@ class TestCommandline(unittest.TestCase): def test_logging_errorlevel(self): parser = argparse.ArgumentParser() commandline.add_logging_group(parser) - args = parser.parse_args( - ["--log-tbpl=%s" % self.logfile.name, "--log-tbpl-level=error"] - ) + args = parser.parse_args([ + "--log-tbpl=%s" % self.logfile.name, + "--log-tbpl-level=error", + ]) logger = commandline.setup_logging("test_fmtopts", args, {}) logger.info("INFO message") logger.debug("DEBUG message") @@ -979,9 +953,10 @@ class TestCommandline(unittest.TestCase): def test_logging_debuglevel(self): parser = argparse.ArgumentParser() commandline.add_logging_group(parser) - args = parser.parse_args( - ["--log-tbpl=%s" % self.logfile.name, "--log-tbpl-level=debug"] - ) + args = parser.parse_args([ + "--log-tbpl=%s" % self.logfile.name, + "--log-tbpl-level=debug", + ]) logger = commandline.setup_logging("test_fmtopts", args, {}) logger.info("INFO message") logger.debug("DEBUG message") @@ -1036,14 +1011,12 @@ class TestBuffer(BaseStructuredTest): self.logger.test_status("test1", "sub1", status="PASS") # Even for buffered actions, the buffer does not interfere if # buffering is turned off. - self.assert_log_equals( - { - "action": "test_status", - "test": "test1", - "status": "PASS", - "subtest": "sub1", - } - ) + self.assert_log_equals({ + "action": "test_status", + "test": "test1", + "status": "PASS", + "subtest": "sub1", + }) self.logger.send_message("buffer", "on") self.logger.test_status("test1", "sub2", status="PASS") self.logger.test_status("test1", "sub3", status="PASS") @@ -1078,40 +1051,32 @@ class TestBuffer(BaseStructuredTest): self.assertEqual([4], self.logger.send_message("buffer", "flush")) # When the buffer is dumped, the failure is the last thing logged - self.assert_log_equals( - { - "action": "test_status", - "test": "test1", - "subtest": "sub8", - "status": "FAIL", - "expected": "PASS", - } - ) + self.assert_log_equals({ + "action": "test_status", + "test": "test1", + "subtest": "sub8", + "status": "FAIL", + "expected": "PASS", + }) # Three additional messages should have been retained for context - self.assert_log_equals( - { - "action": "test_status", - "test": "test1", - "status": "PASS", - "subtest": "sub7", - } - ) - self.assert_log_equals( - { - "action": "test_status", - "test": "test1", - "status": "PASS", - "subtest": "sub6", - } - ) - self.assert_log_equals( - { - "action": "test_status", - "test": "test1", - "status": "PASS", - "subtest": "sub5", - } - ) + self.assert_log_equals({ + "action": "test_status", + "test": "test1", + "status": "PASS", + "subtest": "sub7", + }) + self.assert_log_equals({ + "action": "test_status", + "test": "test1", + "status": "PASS", + "subtest": "sub6", + }) + self.assert_log_equals({ + "action": "test_status", + "test": "test1", + "status": "PASS", + "subtest": "sub5", + }) self.assert_log_equals({"action": "suite_start", "tests": {"default": []}}) diff --git a/testing/mozbase/moznetwork/moznetwork/moznetwork.py b/testing/mozbase/moznetwork/moznetwork/moznetwork.py @@ -127,12 +127,10 @@ def _parse_powershell(): "v1.0", "powershell.exe", ) - output = subprocess.check_output( - [ - cmd, - "(Get-NetIPAddress -AddressFamily IPv4 -AddressState Preferred | Format-List -Property IPAddress)", - ] - ).decode("ascii") + output = subprocess.check_output([ + cmd, + "(Get-NetIPAddress -AddressFamily IPv4 -AddressState Preferred | Format-List -Property IPAddress)", + ]).decode("ascii") ips = re.findall(r"IPAddress : (\d+.\d+.\d+.\d+)", output) for ip in ips: logger.debug("IPAddress: %s" % ip) diff --git a/testing/mozbase/mozpower/mozpower/intel_power_gadget.py b/testing/mozbase/mozpower/mozpower/intel_power_gadget.py @@ -818,12 +818,12 @@ class IPGResultsHandler: for measure in cut_results: if "watt" in measure.lower() and "limit" not in measure.lower(): # pylint --py3k W1619 - watt_usage[replace_measure_name(measure) + "-avg"] = sum( - [float(val) for val in cut_results[measure]] - ) / len(cut_results[measure]) - watt_usage[replace_measure_name(measure) + "-max"] = max( - [float(val) for val in cut_results[measure]] - ) + watt_usage[replace_measure_name(measure) + "-avg"] = sum([ + float(val) for val in cut_results[measure] + ]) / len(cut_results[measure]) + watt_usage[replace_measure_name(measure) + "-max"] = max([ + float(val) for val in cut_results[measure] + ]) # Get average CPU and GPU utilization average_utilization = {} @@ -841,9 +841,9 @@ class IPGResultsHandler: utilized_name = "gpu" # pylint --py3k W1619 - average_utilization[utilized_name] = sum( - [float(val) for val in cut_results[utilization]] - ) / len(cut_results[utilization]) + average_utilization[utilized_name] = sum([ + float(val) for val in cut_results[utilization] + ]) / len(cut_results[utilization]) # Get average and maximum CPU and GPU frequency frequency_info = {"cpu": {}, "gpu": {}} @@ -861,17 +861,17 @@ class IPGResultsHandler: fmeasure_name = "gpu" # pylint --py3k W1619 - frequency_info[fmeasure_name]["favg"] = sum( - [float(val) for val in cut_results[frequency_measure]] - ) / len(cut_results[frequency_measure]) + frequency_info[fmeasure_name]["favg"] = sum([ + float(val) for val in cut_results[frequency_measure] + ]) / len(cut_results[frequency_measure]) - frequency_info[fmeasure_name]["fmax"] = max( - [float(val) for val in cut_results[frequency_measure]] - ) + frequency_info[fmeasure_name]["fmax"] = max([ + float(val) for val in cut_results[frequency_measure] + ]) - frequency_info[fmeasure_name]["fmin"] = min( - [float(val) for val in cut_results[frequency_measure]] - ) + frequency_info[fmeasure_name]["fmin"] = min([ + float(val) for val in cut_results[frequency_measure] + ]) summarized_results = { "utilization": { diff --git a/testing/mozbase/mozpower/mozpower/mozpower.py b/testing/mozbase/mozpower/mozpower/mozpower.py @@ -77,7 +77,7 @@ class MozPower: android=False, logger_name="mozpower", output_file_path="power-testing", - **kwargs + **kwargs, ): """Initializes the MozPower object, detects OS and CPU (if not android), and instatiates the appropriate combo-dependent class for measurements. diff --git a/testing/mozbase/mozpower/tests/conftest.py b/testing/mozbase/mozpower/tests/conftest.py @@ -86,17 +86,15 @@ def mozpower_obj(): """Returns a MozPower object with subprocess.check_output and os.path.exists calls patched with side effects. """ - with mock.patch.object( - MozPower, "_get_os", return_value="Darwin" - ) as _, mock.patch.object( - MozPower, "_get_processor_info", return_value="GenuineIntel" - ) as _, mock.patch.object( - MacIntelPower, "get_ipg_path", return_value="/" - ) as _, mock.patch( - "subprocess.check_output" - ) as subprocess_mock, mock.patch( - "os.path.exists" - ) as os_mock: + with ( + mock.patch.object(MozPower, "_get_os", return_value="Darwin") as _, + mock.patch.object( + MozPower, "_get_processor_info", return_value="GenuineIntel" + ) as _, + mock.patch.object(MacIntelPower, "get_ipg_path", return_value="/") as _, + mock.patch("subprocess.check_output") as subprocess_mock, + mock.patch("os.path.exists") as os_mock, + ): subprocess_mock.side_effect = subprocess_side_effect os_mock.side_effect = os_side_effect diff --git a/testing/mozbase/mozpower/tests/test_mozpower.py b/testing/mozbase/mozpower/tests/test_mozpower.py @@ -22,11 +22,12 @@ def test_mozpower_oscpu_combo_missing_error(): when we can't find a OS, and CPU combination (and, therefore, cannot find a power measurer). """ - with mock.patch.object( - MozPower, "_get_os", return_value="Not-An-OS" - ) as _, mock.patch.object( - MozPower, "_get_processor_info", return_value="Not-A-Processor" - ) as _: + with ( + mock.patch.object(MozPower, "_get_os", return_value="Not-An-OS") as _, + mock.patch.object( + MozPower, "_get_processor_info", return_value="Not-A-Processor" + ) as _, + ): with pytest.raises(OsCpuComboMissingError): MozPower() @@ -99,13 +100,15 @@ def test_mozpower_oscpu_combo(mozpower_obj): def test_mozpower_measuring(mozpower_obj): """Tests that measurers are properly called with each method.""" - with mock.patch( - "mozpower.macintelpower.MacIntelPower.initialize_power_measurements" - ) as _, mock.patch( - "mozpower.macintelpower.MacIntelPower.finalize_power_measurements" - ) as _, mock.patch( - "mozpower.macintelpower.MacIntelPower.get_perfherder_data" - ) as _: + with ( + mock.patch( + "mozpower.macintelpower.MacIntelPower.initialize_power_measurements" + ) as _, + mock.patch( + "mozpower.macintelpower.MacIntelPower.finalize_power_measurements" + ) as _, + mock.patch("mozpower.macintelpower.MacIntelPower.get_perfherder_data") as _, + ): mozpower_obj.initialize_power_measurements() mozpower_obj.measurer.initialize_power_measurements.assert_called() @@ -120,13 +123,15 @@ def test_mozpower_measuring_with_no_measurer(mozpower_obj): """Tests that no errors occur when the measurer is None, and the initialize, finalize, and get_perfherder_data functions are called. """ - with mock.patch( - "mozpower.macintelpower.MacIntelPower.initialize_power_measurements" - ) as _, mock.patch( - "mozpower.macintelpower.MacIntelPower.finalize_power_measurements" - ) as _, mock.patch( - "mozpower.macintelpower.MacIntelPower.get_perfherder_data" - ) as _: + with ( + mock.patch( + "mozpower.macintelpower.MacIntelPower.initialize_power_measurements" + ) as _, + mock.patch( + "mozpower.macintelpower.MacIntelPower.finalize_power_measurements" + ) as _, + mock.patch("mozpower.macintelpower.MacIntelPower.get_perfherder_data") as _, + ): measurer = mozpower_obj.measurer mozpower_obj.measurer = None diff --git a/testing/mozbase/mozprocess/mozprocess/mozprocess.py b/testing/mozbase/mozprocess/mozprocess/mozprocess.py @@ -80,7 +80,7 @@ def run_and_wait( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=text, - **kwargs + **kwargs, ) def reader(fh, queue): diff --git a/testing/mozbase/mozprocess/mozprocess/winprocess.py b/testing/mozbase/mozprocess/mozprocess/winprocess.py @@ -384,7 +384,9 @@ SetInformationJobObject.errcheck = ErrCheckBool # CreateJobObject() CreateJobObjectProto = WINFUNCTYPE( - HANDLE, LPVOID, LPCWSTR # Return type # lpJobAttributes # lpName + HANDLE, + LPVOID, + LPCWSTR, # Return type # lpJobAttributes # lpName ) CreateJobObjectFlags = ((1, "lpJobAttributes", None), (1, "lpName", None)) @@ -397,7 +399,9 @@ CreateJobObject.errcheck = ErrCheckHandle # AssignProcessToJobObject() AssignProcessToJobObjectProto = WINFUNCTYPE( - BOOL, HANDLE, HANDLE # Return type # hJob # hProcess + BOOL, + HANDLE, + HANDLE, # Return type # hJob # hProcess ) AssignProcessToJobObjectFlags = ((1, "hJob"), (1, "hProcess")) AssignProcessToJobObject = AssignProcessToJobObjectProto( @@ -455,7 +459,9 @@ ResumeThread.errcheck = ErrCheckResumeThread # TerminateProcess() TerminateProcessProto = WINFUNCTYPE( - BOOL, HANDLE, UINT # Return type # hProcess # uExitCode + BOOL, + HANDLE, + UINT, # Return type # hProcess # uExitCode ) TerminateProcessFlags = ((1, "hProcess"), (1, "uExitCode", 127)) TerminateProcess = TerminateProcessProto( @@ -466,7 +472,9 @@ TerminateProcess.errcheck = ErrCheckBool # TerminateJobObject() TerminateJobObjectProto = WINFUNCTYPE( - BOOL, HANDLE, UINT # Return type # hJob # uExitCode + BOOL, + HANDLE, + UINT, # Return type # hJob # uExitCode ) TerminateJobObjectFlags = ((1, "hJob"), (1, "uExitCode", 127)) TerminateJobObject = TerminateJobObjectProto( diff --git a/testing/mozbase/mozprocess/tests/proctest.py b/testing/mozbase/mozprocess/tests/proctest.py @@ -53,10 +53,10 @@ class ProcTest(unittest.TestCase): if isalive: self.assertTrue( detected, - "Detected process is not running, " "process output: %s" % output, + "Detected process is not running, process output: %s" % output, ) else: self.assertTrue( not detected, - "Detected process is still running, " "process output: %s" % output, + "Detected process is still running, process output: %s" % output, ) diff --git a/testing/mozbase/mozprofile/mozprofile/cli.py b/testing/mozbase/mozprofile/mozprofile/cli.py @@ -10,6 +10,7 @@ The profile can be modified by passing in addons to install or preferences to se If no profile is specified, a new profile is created and the path of the resulting profile is printed. """ + import sys from optparse import OptionParser @@ -111,7 +112,7 @@ class MozProfileCLI: dest="prefs", action="append", default=[], - help="A preference to set. " "Must be a key-value pair separated by a ':'", + help="A preference to set. Must be a key-value pair separated by a ':'", ) parser.add_option( "--preferences", diff --git a/testing/mozbase/mozprofile/mozprofile/permissions.py b/testing/mozbase/mozprofile/mozprofile/permissions.py @@ -98,9 +98,9 @@ class Location: def isEqual(self, location): """compare scheme://host:port, but ignore options""" - return len( - [i for i in self.attrs if getattr(self, i) == getattr(location, i)] - ) == len(self.attrs) + return len([ + i for i in self.attrs if getattr(self, i) == getattr(location, i) + ]) == len(self.attrs) __eq__ = isEqual @@ -242,9 +242,10 @@ class Permissions: user_prefs.append(("network.trr.uri", trrUri)) user_prefs.append(("network.trr.bootstrapAddr", "127.0.0.1")) user_prefs.append(("network.dns.force_use_https_rr", True)) - user_prefs.append( - ("network.dns.https_rr.check_record_with_cname", False) - ) + user_prefs.append(( + "network.dns.https_rr.check_record_with_cname", + False, + )) else: user_prefs = self.pac_prefs(proxy) else: diff --git a/testing/mozbase/mozprofile/mozprofile/prefs.py b/testing/mozbase/mozprofile/mozprofile/prefs.py @@ -5,6 +5,7 @@ """ user preferences """ + import json import os import tokenize diff --git a/testing/mozbase/mozprofile/mozprofile/profile.py b/testing/mozbase/mozprofile/mozprofile/profile.py @@ -246,25 +246,19 @@ class Profile(BaseProfile): if platform.system() == "Darwin": assert len(self._allowlistpaths) <= 2 if len(self._allowlistpaths) == 2: - prefs_js.append( - ( - "security.sandbox.content.mac.testing_read_path2", - self._allowlistpaths[1], - ) - ) - prefs_js.append( - ( - "security.sandbox.content.mac.testing_read_path1", - self._allowlistpaths[0], - ) - ) + prefs_js.append(( + "security.sandbox.content.mac.testing_read_path2", + self._allowlistpaths[1], + )) + prefs_js.append(( + "security.sandbox.content.mac.testing_read_path1", + self._allowlistpaths[0], + )) else: - prefs_js.append( - ( - "security.sandbox.content.read_path_whitelist", - ",".join(self._allowlistpaths), - ) - ) + prefs_js.append(( + "security.sandbox.content.read_path_whitelist", + ",".join(self._allowlistpaths), + )) self.set_preferences(prefs_js, "prefs.js") self.set_preferences(user_js) @@ -421,20 +415,16 @@ class Profile(BaseProfile): prefs = Preferences.read_prefs(path) if prefs: prefs = dict(prefs) - parts.append( - ( - prefs_file, - "\n%s" - % ( - "\n".join( - [ - "%s: %s" % (key, format_value(key, prefs[key])) - for key in sorted(prefs.keys()) - ] - ) - ), - ) - ) + parts.append(( + prefs_file, + "\n%s" + % ( + "\n".join([ + "%s: %s" % (key, format_value(key, prefs[key])) + for key in sorted(prefs.keys()) + ]) + ), + )) # Currently hardcorded to 'network.proxy.autoconfig_url' # but could be generalized, possibly with a generalized (simple) @@ -462,12 +452,10 @@ class Profile(BaseProfile): ) splitline.append(lines[0][end:]) lines[0:1] = [i.strip() for i in splitline] - parts.append( - ( - "Network Proxy Autoconfig, %s" % (prefs_file), - "\n%s" % "\n".join(lines), - ) - ) + parts.append(( + "Network Proxy Autoconfig, %s" % (prefs_file), + "\n%s" % "\n".join(lines), + )) if return_parts: return parts diff --git a/testing/mozbase/mozprofile/mozprofile/view.py b/testing/mozbase/mozprofile/mozprofile/view.py @@ -7,6 +7,7 @@ """ script to view mozilla profiles """ + import optparse import os import sys diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py @@ -297,88 +297,74 @@ class Mitmproxy(Playback): if self.record_mode: # generate recording script paths - command.extend( - [ - "--save-stream-file", - os.path.normpath(self.recording.recording_path), - "--set", - "websocket=false", - ] - ) + command.extend([ + "--save-stream-file", + os.path.normpath(self.recording.recording_path), + "--set", + "websocket=false", + ]) if "inject_deterministic" in self.config.keys(): - command.extend( - [ - "--scripts", - os.path.join(mitm_folder, "scripts", "inject-deterministic.py"), - ] - ) + command.extend([ + "--scripts", + os.path.join(mitm_folder, "scripts", "inject-deterministic.py"), + ]) self.recording.set_metadata( "proxy_version", self.config["playback_version"] ) # playback mode elif len(self.playback_files) > 0: if self.config["playback_version"] in ["8.1.1", "11.0.0"]: - command.extend( - [ - "--set", - "websocket=false", - "--set", - "connection_strategy=lazy", - "--set", - "alt_server_replay_nopop=true", - "--set", - "alt_server_replay_kill_extra=true", - "--set", - "alt_server_replay_order_reversed=true", - "--set", - "tls_version_client_min=TLS1_2", - "--set", - "alt_server_replay={}".format( - ",".join( - [ - os.path.normpath(playback_file.recording_path) - for playback_file in self.playback_files - ] - ) - ), - "--scripts", - os.path.normpath( - os.path.join( - mitm_folder, "scripts", "alt-serverplayback.py" - ) - ), - ] - ) + command.extend([ + "--set", + "websocket=false", + "--set", + "connection_strategy=lazy", + "--set", + "alt_server_replay_nopop=true", + "--set", + "alt_server_replay_kill_extra=true", + "--set", + "alt_server_replay_order_reversed=true", + "--set", + "tls_version_client_min=TLS1_2", + "--set", + "alt_server_replay={}".format( + ",".join([ + os.path.normpath(playback_file.recording_path) + for playback_file in self.playback_files + ]) + ), + "--scripts", + os.path.normpath( + os.path.join(mitm_folder, "scripts", "alt-serverplayback.py") + ), + ]) elif self.config["playback_version"] in [ "4.0.4", "5.1.1", "6.0.2", ]: - command.extend( - [ - "--set", - "upstream_cert=false", - "--set", - "upload_dir=" + os.path.normpath(self.upload_dir), - "--set", - "websocket=false", - "--set", - "server_replay_files={}".format( - ",".join( - [ - os.path.normpath(playback_file.recording_path) - for playback_file in self.playback_files - ] - ) - ), - "--scripts", - os.path.normpath( - os.path.join( - mitm_folder, "scripts", "alternate-server-replay.py" - ) - ), - ] - ) + command.extend([ + "--set", + "upstream_cert=false", + "--set", + "upload_dir=" + os.path.normpath(self.upload_dir), + "--set", + "websocket=false", + "--set", + "server_replay_files={}".format( + ",".join([ + os.path.normpath(playback_file.recording_path) + for playback_file in self.playback_files + ]) + ), + "--scripts", + os.path.normpath( + os.path.join( + mitm_folder, "scripts", "alternate-server-replay.py" + ) + ), + ]) else: raise Exception("Mitmproxy version is unknown!") diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py @@ -70,9 +70,9 @@ def _remote_settings_changed(self, event, other_conn): # we can't use ctx.log in this layer print("Ignored remote settings upstream") return True - new_settings = dict( - [(key, cs.new_value) for (key, cs) in event.changed_settings.items()] - ) + new_settings = dict([ + (key, cs.new_value) for (key, cs) in event.changed_settings.items() + ]) self.connections[other_conn].safe_update_settings(new_settings) return True @@ -279,13 +279,11 @@ class AlternateServerPlayback: if ctx.options.upload_dir: parsed_url = urlparse(unquote(f.request.url)) self.netlocs[parsed_url.netloc][f.response.status_code] += 1 - self.calls.append( - { - "time": str(time.time()), - "url": f.request.url, - "response_status": f.response.status_code, - } - ) + self.calls.append({ + "time": str(time.time()), + "url": f.request.url, + "response_status": f.response.status_code, + }) except Exception as e: ctx.log.error("Could not generate response! Stopping playback process!") ctx.log.info(e) diff --git a/testing/mozbase/mozproxy/mozproxy/driver.py b/testing/mozbase/mozproxy/mozproxy/driver.py @@ -113,47 +113,41 @@ def main(): raise Exception("Please provide at least one recording file!") # Playback mode - proxy_service = get_playback( - { - "run_local": args.local, - "host": args.host, - "binary": args.binary, - "obj_path": args.objdir, - "platform": mozinfo.os, - "playback_tool": args.tool, - "playback_version": args.tool_version, - "playback_files": args.file, - "app": args.app, - "local_profile_dir": args.profiledir, - "verbose": args.verbose, - } - ) + proxy_service = get_playback({ + "run_local": args.local, + "host": args.host, + "binary": args.binary, + "obj_path": args.objdir, + "platform": mozinfo.os, + "playback_tool": args.tool, + "playback_version": args.tool_version, + "playback_files": args.file, + "app": args.app, + "local_profile_dir": args.profiledir, + "verbose": args.verbose, + }) if args.mode == "record": # Record mode if len(args.file) > 1: raise Exception("Please provide only one recording file!") LOG.info("Recording will be saved to: %s" % args.file) - proxy_service = get_playback( - { - "run_local": args.local, - "host": args.host, - "binary": args.binary, - "obj_path": args.objdir, - "platform": mozinfo.os, - "playback_tool": args.tool, - # bug 1883701 linux uses a different version for now - "playback_version": ( - "8.1.1" if mozinfo.isLinux else args.tool_version - ), - "record": True, - "recording_file": args.file[0], - "app": args.app, - "local_profile_dir": args.profiledir, - "verbose": args.verbose, - "inject_deterministic": args.deterministic, - } - ) + proxy_service = get_playback({ + "run_local": args.local, + "host": args.host, + "binary": args.binary, + "obj_path": args.objdir, + "platform": mozinfo.os, + "playback_tool": args.tool, + # bug 1883701 linux uses a different version for now + "playback_version": ("8.1.1" if mozinfo.isLinux else args.tool_version), + "record": True, + "recording_file": args.file[0], + "app": args.app, + "local_profile_dir": args.profiledir, + "verbose": args.verbose, + "inject_deterministic": args.deterministic, + }) LOG.info("Proxy settings %s" % proxy_service) proxy_service.start() LOG.info("Proxy running on port %d" % proxy_service.port) diff --git a/testing/mozbase/mozproxy/tests/test_command_line.py b/testing/mozbase/mozproxy/tests/test_command_line.py @@ -37,15 +37,13 @@ def _install_package(virtualenv_manager, package): # already installed in this venv, we can skip return - subprocess.check_call( - [ - virtualenv_manager.python_path, - "-m", - "pip", - "install", - package, - ] - ) + subprocess.check_call([ + virtualenv_manager.python_path, + "-m", + "pip", + "install", + package, + ]) def _kill_mozproxy(pid): diff --git a/testing/mozbase/mozrunner/mozrunner/devices/ios_device.py b/testing/mozbase/mozrunner/mozrunner/devices/ios_device.py @@ -41,23 +41,21 @@ def verify_ios_device( # FIXME: This should probably be happening as a build step, rather # than happening during verify_ios_device! print("Packaging GeckoTestBrowser...") - subprocess.check_call( - [ - "xcodebuild", - "-project", - os.path.join( - build_obj.topsrcdir, - "mobile/ios/GeckoTestBrowser/GeckoTestBrowser.xcodeproj", - ), - "-scheme", - "GeckoTestBrowser", - "-destination", - device.xcode_destination_specifier(), - "install", - "DSTROOT=" + build_obj.distdir, - "TOPOBJDIR=" + build_obj.topobjdir, - ] - ) + subprocess.check_call([ + "xcodebuild", + "-project", + os.path.join( + build_obj.topsrcdir, + "mobile/ios/GeckoTestBrowser/GeckoTestBrowser.xcodeproj", + ), + "-scheme", + "GeckoTestBrowser", + "-destination", + device.xcode_destination_specifier(), + "install", + "DSTROOT=" + build_obj.distdir, + "TOPOBJDIR=" + build_obj.topobjdir, + ]) print("Installing GeckoTestBrowser...") device.install( diff --git a/testing/mozbase/mozrunner/mozrunner/runners.py b/testing/mozbase/mozrunner/mozrunner/runners.py @@ -102,7 +102,7 @@ def FennecEmulatorRunner( serial=None, binary=None, app="org.mozilla.fennec", - **kwargs + **kwargs, ): """ Create a Fennec emulator runner. This can either start a new emulator diff --git a/testing/mozbase/mozserve/mozserve/servers.py b/testing/mozbase/mozserve/mozserve/servers.py @@ -115,7 +115,7 @@ class Http3Server: if "server listening" in msg: searchObj = re.search( r"HTTP3 server listening on ports ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+), ([0-9]+) and ([0-9]+)." - " EchConfig is @([\x00-\x7F]+)@", + " EchConfig is @([\x00-\x7f]+)@", msg, 0, ) diff --git a/testing/mozbase/mozshellutil/mozshellutil/__init__.py b/testing/mozbase/mozshellutil/mozshellutil/__init__.py @@ -6,4 +6,5 @@ """ This module contains utilities for parsing and quoting shell command lines. """ + from .shellutil import * diff --git a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py @@ -226,18 +226,16 @@ def _collect(pipe, poll_interval): swap_entry[sout_index] = swap_mem.sout - swap_last.sout swap_last = swap_mem - data.append( - ( - last_time, - measured_end_time, - io_diff, - net_io_diff, - cpu_diff, - cpu_percent, - list(virt_mem), - swap_entry, - ) - ) + data.append(( + last_time, + measured_end_time, + io_diff, + net_io_diff, + cpu_diff, + cpu_percent, + list(virt_mem), + swap_entry, + )) update_known_processes() @@ -255,19 +253,24 @@ def _collect(pipe, poll_interval): for pid, create_time, end_time, cmd, ppid in processes: if len(cmd) > 0: cmd[0] = os.path.basename(cmd[0]) - cmdline = " ".join( - [ - arg - for arg in cmd - if not arg.startswith("-D") - and not arg.startswith("-I") - and not arg.startswith("-W") - and not arg.startswith("-L") - ] - ) - pipe.send( - ("process", pid, create_time, end_time, cmdline, ppid, None, None) - ) + cmdline = " ".join([ + arg + for arg in cmd + if not arg.startswith("-D") + and not arg.startswith("-I") + and not arg.startswith("-W") + and not arg.startswith("-L") + ]) + pipe.send(( + "process", + pid, + create_time, + end_time, + cmdline, + ppid, + None, + None, + )) pipe.send(("done", None, None, None, None, None, None, None)) pipe.close() @@ -532,17 +535,15 @@ class SystemResourceMonitor: # We also can't recover, but output the data that caused the exception warnings.warn( "failed to read the received data: %s" - % str( - ( - start_time, - end_time, - io_diff, - cpu_diff, - cpu_percent, - virt_mem, - swap_mem, - ) - ) + % str(( + start_time, + end_time, + io_diff, + cpu_diff, + cpu_percent, + virt_mem, + swap_mem, + )) ) break @@ -680,14 +681,12 @@ class SystemResourceMonitor: if "write_time" in data: marker_data["write_time"] = data["write_time"] - self.markers.append( - ( - "sccache", - data["start_time"], - data["end_time"], - marker_data, - ) - ) + self.markers.append(( + "sccache", + data["start_time"], + data["end_time"], + marker_data, + )) except Exception as e: warnings.warn(f"Failed to parse sccache.log: {e}") @@ -697,17 +696,15 @@ class SystemResourceMonitor: num_markers = len(compilations) # Add as a duration marker if num_markers > 0: - self.markers.append( - ( - "sccache parsing", - parse_start, - parse_end, - { - "type": "Text", - "text": f"Parsed {num_markers} sccache entries from log", - }, - ) - ) + self.markers.append(( + "sccache parsing", + parse_start, + parse_end, + { + "type": "Text", + "text": f"Parsed {num_markers} sccache entries from log", + }, + )) # Methods to record events alongside the monitored data. @@ -2004,9 +2001,11 @@ class SystemResourceMonitor: "idle": "Idle %", }.items(): if field in valid_cpu_fields or field == "idle": - cpuData.append( - {"key": field + "_pct", "label": label, "format": "string"} - ) + cpuData.append({ + "key": field + "_pct", + "label": label, + "format": "string", + }) cpuGraphs = cpuSchema["graphs"] for field, color in { "softirq": "orange", diff --git a/testing/mozbase/moztest/moztest/resolve.py b/testing/mozbase/moztest/moztest/resolve.py @@ -956,21 +956,19 @@ class TestResolver(MozbuildObject): for root, dirs, paths in os.walk(test_path): for filename in fnmatch.filter(paths, "*.spec.js"): path = os.path.join(root, filename) - self._tests.append( - { - "path": os.path.abspath(path), - "flavor": "puppeteer", - "here": os.path.dirname(path), - "manifest": None, - "name": path, - "file_relpath": path, - "head": "", - "support-files": "", - "subsuite": "puppeteer", - "dir_relpath": os.path.dirname(path), - "srcdir_relpath": path, - } - ) + self._tests.append({ + "path": os.path.abspath(path), + "flavor": "puppeteer", + "here": os.path.dirname(path), + "manifest": None, + "name": path, + "file_relpath": path, + "head": "", + "support-files": "", + "subsuite": "puppeteer", + "dir_relpath": os.path.dirname(path), + "srcdir_relpath": path, + }) self._puppeteer_loaded = True @@ -987,21 +985,19 @@ class TestResolver(MozbuildObject): if "test" in root: for filename in fnmatch.filter(paths, "*.kt"): path = os.path.join(root, filename) - self._tests.append( - { - "path": os.path.abspath(path), - "flavor": "fenix", - "here": os.path.dirname(path), - "manifest": None, - "name": path, - "file_relpath": path, - "head": "", - "support-files": "", - "subsuite": "", - "dir_relpath": os.path.dirname(path), - "srcdir_relpath": path, - } - ) + self._tests.append({ + "path": os.path.abspath(path), + "flavor": "fenix", + "here": os.path.dirname(path), + "manifest": None, + "name": path, + "file_relpath": path, + "head": "", + "support-files": "", + "subsuite": "", + "dir_relpath": os.path.dirname(path), + "srcdir_relpath": path, + }) self._fenix_loaded = True @@ -1025,21 +1021,19 @@ class TestResolver(MozbuildObject): if "test" in root: for filename in fnmatch.filter(paths, "*.kt"): path = os.path.join(root, filename) - self._tests.append( - { - "path": os.path.abspath(path), - "flavor": "focus", - "here": os.path.dirname(path), - "manifest": None, - "name": path, - "file_relpath": path, - "head": "", - "support-files": "", - "subsuite": "", - "dir_relpath": os.path.dirname(path), - "srcdir_relpath": path, - } - ) + self._tests.append({ + "path": os.path.abspath(path), + "flavor": "focus", + "here": os.path.dirname(path), + "manifest": None, + "name": path, + "file_relpath": path, + "head": "", + "support-files": "", + "subsuite": "", + "dir_relpath": os.path.dirname(path), + "srcdir_relpath": path, + }) self._focus_loaded = True @@ -1058,21 +1052,19 @@ class TestResolver(MozbuildObject): if test_subdir_path in root: for filename in fnmatch.filter(paths, "*.kt"): path = os.path.join(root, filename) - self._tests.append( - { - "path": os.path.abspath(path), - "flavor": "android-components", - "here": os.path.dirname(path), - "manifest": None, - "name": path, - "file_relpath": path, - "head": "", - "support-files": "", - "subsuite": "", - "dir_relpath": os.path.dirname(path), - "srcdir_relpath": path, - } - ) + self._tests.append({ + "path": os.path.abspath(path), + "flavor": "android-components", + "here": os.path.dirname(path), + "manifest": None, + "name": path, + "file_relpath": path, + "head": "", + "support-files": "", + "subsuite": "", + "dir_relpath": os.path.dirname(path), + "srcdir_relpath": path, + }) self._ac_loaded = True @@ -1095,21 +1087,19 @@ class TestResolver(MozbuildObject): if "test" in root: for filename in fnmatch.filter(paths, "*.kt"): path = os.path.join(root, filename) - self._tests.append( - { - "path": os.path.abspath(path), - "flavor": "geckoview", - "here": os.path.dirname(path), - "manifest": None, - "name": path, - "file_relpath": path, - "head": "", - "support-files": "", - "subsuite": "", - "dir_relpath": os.path.dirname(path), - "srcdir_relpath": path, - } - ) + self._tests.append({ + "path": os.path.abspath(path), + "flavor": "geckoview", + "here": os.path.dirname(path), + "manifest": None, + "name": path, + "file_relpath": path, + "head": "", + "support-files": "", + "subsuite": "", + "dir_relpath": os.path.dirname(path), + "srcdir_relpath": path, + }) self._geckooview_junit_loaded = True diff --git a/testing/mozbase/moztest/tests/test_resolve.py b/testing/mozbase/moztest/tests/test_resolve.py @@ -95,109 +95,107 @@ def create_tests(topsrcdir): @pytest.fixture(scope="module") def all_tests(create_tests): - return create_tests( - *[ - ( - "apple/test_a11y.html", - { - "expected": "pass", - "manifest": "apple/a11y.toml", - "flavor": "a11y", - }, - ), - ( - "banana/currant/test_xpcshell_A.js", - { - "firefox-appdir": "browser", - "flavor": "xpcshell", - "head": "head_global.js head_helpers.js head_http.js", - }, - ), - ( - "banana/currant/test_xpcshell_B.js", - { - "firefox-appdir": "browser", - "flavor": "xpcshell", - "head": "head_global.js head_helpers.js head_http.js", - }, - ), - ( - "carrot/test_included.js", - { - "ancestor_manifest": "carrot/xpcshell-one.toml", - "manifest": "carrot/xpcshell-shared.toml", - "flavor": "xpcshell", - "stick": "one", - }, - ), - ( - "carrot/test_included.js", - { - "ancestor_manifest": "carrot/xpcshell-two.toml", - "manifest": "carrot/xpcshell-shared.toml", - "flavor": "xpcshell", - "stick": "two", - }, - ), - ( - "dragonfruit/elderberry/test_xpcshell_C.js", - { - "flavor": "xpcshell", - "generated-files": "head_update.js", - "head": "head_update.js", - "manifest": "dragonfruit/xpcshell.toml", - "reason": "busted", - "run-sequentially": "Launches application.", - "skip-if": "os == 'android'", - }, - ), - ( - "dragonfruit/elderberry/test_xpcshell_C.js", - { - "flavor": "xpcshell", - "generated-files": "head_update.js", - "head": "head_update.js head2.js", - "manifest": "dragonfruit/elderberry/xpcshell_updater.toml", - "reason": "don't work", - "run-sequentially": "Launches application.", - "skip-if": "os == 'android'", - }, - ), - ( - "fig/grape/src/TestInstrumentationA.java", - { - "flavor": "instrumentation", - "manifest": "fig/grape/instrumentation.toml", - "subsuite": "background", - }, - ), - ( - "fig/huckleberry/src/TestInstrumentationB.java", - { - "flavor": "instrumentation", - "manifest": "fig/huckleberry/instrumentation.toml", - "subsuite": "browser", - }, - ), - ( - "juniper/browser_chrome.js", - { - "flavor": "browser-chrome", - "manifest": "juniper/browser.toml", - "skip-if": "e10s # broken", - }, - ), - ( - "kiwi/browser_devtools.js", - { - "flavor": "browser-chrome", - "manifest": "kiwi/browser.toml", - "subsuite": "devtools", - "tags": "devtools", - }, - ), - ] - ) + return create_tests(*[ + ( + "apple/test_a11y.html", + { + "expected": "pass", + "manifest": "apple/a11y.toml", + "flavor": "a11y", + }, + ), + ( + "banana/currant/test_xpcshell_A.js", + { + "firefox-appdir": "browser", + "flavor": "xpcshell", + "head": "head_global.js head_helpers.js head_http.js", + }, + ), + ( + "banana/currant/test_xpcshell_B.js", + { + "firefox-appdir": "browser", + "flavor": "xpcshell", + "head": "head_global.js head_helpers.js head_http.js", + }, + ), + ( + "carrot/test_included.js", + { + "ancestor_manifest": "carrot/xpcshell-one.toml", + "manifest": "carrot/xpcshell-shared.toml", + "flavor": "xpcshell", + "stick": "one", + }, + ), + ( + "carrot/test_included.js", + { + "ancestor_manifest": "carrot/xpcshell-two.toml", + "manifest": "carrot/xpcshell-shared.toml", + "flavor": "xpcshell", + "stick": "two", + }, + ), + ( + "dragonfruit/elderberry/test_xpcshell_C.js", + { + "flavor": "xpcshell", + "generated-files": "head_update.js", + "head": "head_update.js", + "manifest": "dragonfruit/xpcshell.toml", + "reason": "busted", + "run-sequentially": "Launches application.", + "skip-if": "os == 'android'", + }, + ), + ( + "dragonfruit/elderberry/test_xpcshell_C.js", + { + "flavor": "xpcshell", + "generated-files": "head_update.js", + "head": "head_update.js head2.js", + "manifest": "dragonfruit/elderberry/xpcshell_updater.toml", + "reason": "don't work", + "run-sequentially": "Launches application.", + "skip-if": "os == 'android'", + }, + ), + ( + "fig/grape/src/TestInstrumentationA.java", + { + "flavor": "instrumentation", + "manifest": "fig/grape/instrumentation.toml", + "subsuite": "background", + }, + ), + ( + "fig/huckleberry/src/TestInstrumentationB.java", + { + "flavor": "instrumentation", + "manifest": "fig/huckleberry/instrumentation.toml", + "subsuite": "browser", + }, + ), + ( + "juniper/browser_chrome.js", + { + "flavor": "browser-chrome", + "manifest": "juniper/browser.toml", + "skip-if": "e10s # broken", + }, + ), + ( + "kiwi/browser_devtools.js", + { + "flavor": "browser-chrome", + "manifest": "kiwi/browser.toml", + "subsuite": "devtools", + "tags": "devtools", + }, + ), + ]) @pytest.fixture(scope="module") @@ -446,9 +444,11 @@ def test_resolve_metadata(resolver): assert suites == {"mochitest-browser-chrome"} assert tests == [] - suites, tests = resolver.resolve_metadata( - ["mochitest-a11y", "/browser", "xpcshell"] - ) + suites, tests = resolver.resolve_metadata([ + "mochitest-a11y", + "/browser", + "xpcshell", + ]) assert suites == {"mochitest-a11y", "xpcshell"} assert sorted(t["file_relpath"] for t in tests) == [ "juniper/browser_chrome.js", diff --git a/testing/mozbase/setup_development.py b/testing/mozbase/setup_development.py @@ -186,9 +186,9 @@ def main(args=sys.argv[1:]): packages = sorted(mozbase_packages) # ensure specified packages are in the list - assert set(packages).issubset( - mozbase_packages - ), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages) + assert set(packages).issubset(mozbase_packages), ( + "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages) + ) if options.list_dependencies: # list the package dependencies diff --git a/testing/mozbase/versioninfo.py b/testing/mozbase/versioninfo.py @@ -78,16 +78,14 @@ def changelog(args): to_ref = get_version_rev(to_ref) delim = "\x12\x59\x52\x99\x05" - changelog = run_hg( - [ - "log", - "-r", - "%s:children(%s)" % (to_ref, from_ref), - "--template={desc}%s" % delim, - "-M", - args.module, - ] - ).split(delim)[:-1] + changelog = run_hg([ + "log", + "-r", + "%s:children(%s)" % (to_ref, from_ref), + "--template={desc}%s" % delim, + "-M", + args.module, + ]).split(delim)[:-1] def prettify(desc): lines = desc.splitlines() @@ -138,7 +136,7 @@ def main(args=sys.argv[1:]): "--to", dest="to_ref", default=None, - help="Ending version or revision to list " "changes to. [defaults to tip]", + help="Ending version or revision to list changes to. [defaults to tip]", ) p_changelog.set_defaults(func=changelog) diff --git a/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py b/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py @@ -49,7 +49,7 @@ config = { "MOZ_CRASHREPORTER_NO_REPORT": "1", "LC_ALL": "C", ## 64 bit specific - "PATH": "/usr/local/bin:/bin:" "/usr/bin:/usr/local/sbin:/usr/sbin:/sbin", + "PATH": "/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin", ## }, "mozconfig_platform": "macosx64", diff --git a/testing/mozharness/mozharness/base/config.py b/testing/mozharness/mozharness/base/config.py @@ -345,7 +345,7 @@ class BaseConfig: "--dump-config", action="store_true", dest="dump_config", - help="List and dump the config generated from this run to " "a JSON file.", + help="List and dump the config generated from this run to a JSON file.", ) self.config_parser.add_option( "--dump-config-hierarchy", @@ -532,25 +532,21 @@ class BaseConfig: file_name = os.path.basename(cf) file_path = os.path.join(os.getcwd(), file_name) download_config_file(cf, file_path) - all_cfg_files_and_dicts.append( - ( + all_cfg_files_and_dicts.append(( + file_path, + parse_config_file( file_path, - parse_config_file( - file_path, - search_path=["."], - ), - ) - ) + search_path=["."], + ), + )) else: - all_cfg_files_and_dicts.append( - ( + all_cfg_files_and_dicts.append(( + cf, + parse_config_file( cf, - parse_config_file( - cf, - search_path=config_paths + [DEFAULT_CONFIG_PATH], - ), - ) - ) + search_path=config_paths + [DEFAULT_CONFIG_PATH], + ), + )) except Exception: if cf in options.opt_config_files: print("WARNING: optional config file not found %s" % cf) diff --git a/testing/mozharness/mozharness/base/diskutils.py b/testing/mozharness/mozharness/base/diskutils.py @@ -4,30 +4,31 @@ """Disk utility module, no mixins here! - examples: - 1) get disk size - from mozharness.base.diskutils import DiskInfo, DiskutilsError - ... - try: - DiskSize().get_size(path='/', unit='Mb') - except DiskutilsError as e: - # manage the exception e.g: log.error(e) - pass - log.info("%s" % di) - - - 2) convert disk size: - from mozharness.base.diskutils import DiskutilsError, convert_to - ... - file_size = <function that gets file size in bytes> - # convert file_size to GB - try: - file_size = convert_to(file_size, from_unit='bytes', to_unit='GB') - except DiskutilsError as e: - # manage the exception e.g: log.error(e) - pass +examples: +1) get disk size +from mozharness.base.diskutils import DiskInfo, DiskutilsError +... +try: + DiskSize().get_size(path='/', unit='Mb') +except DiskutilsError as e: + # manage the exception e.g: log.error(e) + pass +log.info("%s" % di) + + +2) convert disk size: +from mozharness.base.diskutils import DiskutilsError, convert_to +... +file_size = <function that gets file size in bytes> +# convert file_size to GB +try: + file_size = convert_to(file_size, from_unit='bytes', to_unit='GB') +except DiskutilsError as e: + # manage the exception e.g: log.error(e) + pass """ + import ctypes import logging import os diff --git a/testing/mozharness/mozharness/base/log.py b/testing/mozharness/mozharness/base/log.py @@ -626,7 +626,7 @@ class SimpleFileLogger(BaseLogger): log_format="%(asctime)s %(levelname)8s - %(message)s", logger_name="Simple", log_dir="logs", - **kwargs + **kwargs, ): """SimpleFileLogger constructor. Calls its superclass constructor, creates a new logger instance and log an init message. @@ -647,7 +647,7 @@ class SimpleFileLogger(BaseLogger): logger_name=logger_name, log_format=log_format, log_dir=log_dir, - **kwargs + **kwargs, ) self.new_logger() self.init_message() @@ -673,7 +673,7 @@ class MultiFileLogger(BaseLogger): log_format="%(asctime)s %(levelname)8s - %(message)s", log_dir="logs", log_to_raw=True, - **kwargs + **kwargs, ): """MultiFileLogger constructor. Calls its superclass constructor, creates a new logger instance and log an init message. @@ -697,7 +697,7 @@ class MultiFileLogger(BaseLogger): log_format=log_format, log_to_raw=log_to_raw, log_dir=log_dir, - **kwargs + **kwargs, ) self.new_logger() @@ -731,7 +731,7 @@ class ConsoleLogger(BaseLogger): log_format="%(levelname)8s - %(message)s", log_date_format="%H:%M:%S", logger_name="Console", - **kwargs + **kwargs, ): """ConsoleLogger constructor. Calls its superclass constructor, creates a new logger instance and log an init message. diff --git a/testing/mozharness/mozharness/base/parallel.py b/testing/mozharness/mozharness/base/parallel.py @@ -2,8 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""Generic ways to parallelize jobs. -""" +"""Generic ways to parallelize jobs.""" # ChunkingMixin {{{1 diff --git a/testing/mozharness/mozharness/base/python.py b/testing/mozharness/mozharness/base/python.py @@ -2,8 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""Python usage, esp. virtualenv. -""" +"""Python usage, esp. virtualenv.""" import errno import json @@ -160,9 +159,14 @@ class VirtualenvMixin: See the documentation for install_module for how the arguments are applied. """ - self._virtualenv_modules.append( - (name, url, method, requirements, optional, editable) - ) + self._virtualenv_modules.append(( + name, + url, + method, + requirements, + optional, + editable, + )) def query_virtualenv_path(self): """Determine the absolute path to the virtualenv.""" @@ -174,7 +178,7 @@ class VirtualenvMixin: p = self.config["virtualenv_path"] if not p: self.fatal( - "virtualenv_path config option not set; " "this should never happen" + "virtualenv_path config option not set; this should never happen" ) if os.path.isabs(p): @@ -202,13 +206,11 @@ class VirtualenvMixin: if self.site_packages_path: return self.site_packages_path python = self.query_python_path() - self.site_packages_path = self.get_output_from_command( - [ - python, - "-c", - "from sysconfig; print(sysconfig.get_paths()['purelib'])", - ] - ) + self.site_packages_path = self.get_output_from_command([ + python, + "-c", + "from sysconfig; print(sysconfig.get_paths()['purelib'])", + ]) return self.site_packages_path def package_versions( @@ -964,30 +966,23 @@ class ResourceMonitoringMixin(PerfherderResourceOptionsMixin): overall = [] if cpu_percent: - overall.append( - { - "name": "cpu_percent", - "value": cpu_percent, - } - ) - - overall.extend( - [ - {"name": "io_write_bytes", "value": io.write_bytes}, - {"name": "io.read_bytes", "value": io.read_bytes}, - {"name": "io_write_time", "value": io.write_time}, - {"name": "io_read_time", "value": io.read_time}, - ] - ) - - suites.append( - { - "name": "%s.overall" % perfherder_name, - "extraOptions": perfherder_options - + self.perfherder_resource_options(), - "subtests": overall, - } - ) + overall.append({ + "name": "cpu_percent", + "value": cpu_percent, + }) + + overall.extend([ + {"name": "io_write_bytes", "value": io.write_bytes}, + {"name": "io.read_bytes", "value": io.read_bytes}, + {"name": "io_write_time", "value": io.write_time}, + {"name": "io_read_time", "value": io.read_time}, + ]) + + suites.append({ + "name": "%s.overall" % perfherder_name, + "extraOptions": perfherder_options + self.perfherder_resource_options(), + "subtests": overall, + }) for phase in rm.phases.keys(): phase_duration = rm.phases[phase][1] - rm.phases[phase][0] @@ -999,24 +994,18 @@ class ResourceMonitoringMixin(PerfherderResourceOptionsMixin): ] cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False) if cpu_percent is not None: - subtests.append( - { - "name": "cpu_percent", - "value": rm.aggregate_cpu_percent( - phase=phase, per_cpu=False - ), - } - ) + subtests.append({ + "name": "cpu_percent", + "value": rm.aggregate_cpu_percent(phase=phase, per_cpu=False), + }) # We don't report I/O during each step because measured I/O # is system I/O and that I/O can be delayed (e.g. writes will # buffer before being flushed and recorded in our metrics). - suites.append( - { - "name": "%s.%s" % (perfherder_name, phase), - "subtests": subtests, - } - ) + suites.append({ + "name": "%s.%s" % (perfherder_name, phase), + "subtests": subtests, + }) data = { "framework": {"name": "job_resource_usage"}, diff --git a/testing/mozharness/mozharness/base/script.py b/testing/mozharness/mozharness/base/script.py @@ -2248,7 +2248,7 @@ class BaseScript(ScriptMixin, LogMixin): ) cfg_files_dump_config["not_from_cfg_file"] = not_from_file_dict self.action_message( - "Not from any config file (default_config, " "cmd line options, etc)" + "Not from any config file (default_config, cmd line options, etc)" ) self.info(pprint.pformat(not_from_file_dict)) @@ -2396,16 +2396,14 @@ class BaseScript(ScriptMixin, LogMixin): start = time.monotonic() self.run_action(action) end = time.monotonic() - perfherder_data["suites"].append( - { - "name": action, - "value": end - start, - "lowerIsBetter": True, - "unit": "s", - "shouldAlert": False, - "subtests": [], - } - ) + perfherder_data["suites"].append({ + "name": action, + "value": end - start, + "lowerIsBetter": True, + "unit": "s", + "shouldAlert": False, + "subtests": [], + }) except Exception: self.fatal("Uncaught exception: %s" % traceback.format_exc()) finally: diff --git a/testing/mozharness/mozharness/base/transfer.py b/testing/mozharness/mozharness/base/transfer.py @@ -2,8 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""Generic ways to upload + download files. -""" +"""Generic ways to upload + download files.""" import pprint diff --git a/testing/mozharness/mozharness/base/vcs/mercurial.py b/testing/mozharness/mozharness/base/vcs/mercurial.py @@ -2,8 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""Mercurial VCS support. -""" +"""Mercurial VCS support.""" import hashlib import os @@ -382,7 +381,7 @@ class MercurialVCS(ScriptMixin, LogMixin, TransferMixin): # only have 1 local copy of logical repo stores. if not share_base: raise VCSException( - "vcs share base not defined; " "refusing to operate sub-optimally" + "vcs share base not defined; refusing to operate sub-optimally" ) if not self.robustcheckout_path: diff --git a/testing/mozharness/mozharness/base/vcs/vcsbase.py b/testing/mozharness/mozharness/base/vcs/vcsbase.py @@ -2,8 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""Generic VCS support. -""" +"""Generic VCS support.""" import os import sys diff --git a/testing/mozharness/mozharness/lib/python/authentication.py b/testing/mozharness/mozharness/lib/python/authentication.py @@ -3,6 +3,7 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. """module for http authentication operations""" + import getpass import os diff --git a/testing/mozharness/mozharness/mozilla/automation.py b/testing/mozharness/mozharness/mozilla/automation.py @@ -2,8 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""Code to integration with automation. -""" +"""Code to integration with automation.""" try: import simplejson as json diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -2,11 +2,12 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -""" buildbase.py. +"""buildbase.py. provides a base class for fx desktop builds """ + import copy import json import os @@ -224,9 +225,10 @@ class BuildingConfig(BaseConfig): # stack variant cfg file on top of that, if it is present if variant_cfg_file: # take the whole config - all_config_dicts.append( - (variant_cfg_file, parse_config_file(variant_cfg_file)) - ) + all_config_dicts.append(( + variant_cfg_file, + parse_config_file(variant_cfg_file), + )) return all_config_dicts @@ -688,12 +690,10 @@ items from that key's value." os.path.join(dirs["abs_src_dir"], "toolchains.json"), ] if manifest_src: - cmd.extend( - [ - "--tooltool-manifest", - os.path.join(dirs["abs_src_dir"], manifest_src), - ] - ) + cmd.extend([ + "--tooltool-manifest", + os.path.join(dirs["abs_src_dir"], manifest_src), + ]) cache = c["env"].get("TOOLTOOL_CACHE") if cache: cmd.extend(["--cache-dir", cache]) @@ -707,9 +707,7 @@ items from that key's value." if mozbuild_path: self.mkdir_p(mozbuild_path) else: - self.warning( - "mozbuild_path could not be determined. skipping " "creating it." - ) + self.warning("mozbuild_path could not be determined. skipping creating it.") def preflight_build(self): """set up machine state for a complete build.""" @@ -732,9 +730,11 @@ items from that key's value." """Run mach static-analysis autotest, in order to make sure we dont regress""" self.preflight_build() self._run_mach_command_in_build_env(["configure"]) - self._run_mach_command_in_build_env( - ["static-analysis", "autotest", "--intree-tool"] - ) + self._run_mach_command_in_build_env([ + "static-analysis", + "autotest", + "--intree-tool", + ]) def _query_mach(self): return [sys.executable, "mach"] @@ -959,12 +959,10 @@ items from that key's value." } for name, duration in phases.items(): - data["subtests"].append( - { - "name": name, - "value": duration, - } - ) + data["subtests"].append({ + "name": name, + "value": duration, + }) return data @@ -1134,25 +1132,21 @@ items from that key's value." return alert if installer.endswith(".apk"): # Android - yield filter_alert( - { - "name": "installer size", - "value": installer_size, - "alertChangeType": "absolute", - "alertThreshold": (200 * 1024), - "subtests": size_measurements, - } - ) + yield filter_alert({ + "name": "installer size", + "value": installer_size, + "alertChangeType": "absolute", + "alertThreshold": (200 * 1024), + "subtests": size_measurements, + }) else: - yield filter_alert( - { - "name": "installer size", - "value": installer_size, - "alertChangeType": "absolute", - "alertThreshold": (100 * 1024), - "subtests": size_measurements, - } - ) + yield filter_alert({ + "name": "installer size", + "value": installer_size, + "alertChangeType": "absolute", + "alertThreshold": (100 * 1024), + "subtests": size_measurements, + }) def _get_sections(self, file, filter=None): """ @@ -1246,13 +1240,11 @@ items from that key's value." for k, v in list(section_details.items()): section_measurements.append({"name": k, "value": v}) lib_size += v - lib_details.append( - { - "name": lib_type, - "size": lib_size, - "sections": section_measurements, - } - ) + lib_details.append({ + "name": lib_type, + "size": lib_size, + "sections": section_measurements, + }) for lib_detail in lib_details: yield { @@ -1300,14 +1292,12 @@ items from that key's value." ) if warnings is not None: - perfherder_data["suites"].append( - { - "name": "compiler warnings", - "value": len(warnings.strip().splitlines()), - "alertThreshold": 100.0, - "subtests": [], - } - ) + perfherder_data["suites"].append({ + "name": "compiler warnings", + "value": len(warnings.strip().splitlines()), + "alertThreshold": 100.0, + "subtests": [], + }) build_metrics = self._load_build_resources() if build_metrics: diff --git a/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py b/testing/mozharness/mozharness/mozilla/firefox/autoconfig.py @@ -1,4 +1,4 @@ -""" This module helps modifying Firefox with autoconfig files.""" +"""This module helps modifying Firefox with autoconfig files.""" # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/testing/mozharness/mozharness/mozilla/secrets.py b/testing/mozharness/mozharness/mozilla/secrets.py @@ -2,8 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""Support for fetching secrets from the secrets API -""" +"""Support for fetching secrets from the secrets API""" import json import os diff --git a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py @@ -346,78 +346,61 @@ class SingleTestMixin: # code may run before the device is ready, so rely on configuration from mozinfo.platforminfo import android_api_to_os_version - mozinfo.update( - {"android_version": str(self.config.get("android_version", 34))} - ) - mozinfo.update( - { - "os_version": android_api_to_os_version( - self.config.get("android_version", 34) - ) - } - ) + mozinfo.update({ + "android_version": str(self.config.get("android_version", 34)) + }) + mozinfo.update({ + "os_version": android_api_to_os_version( + self.config.get("android_version", 34) + ) + }) mozinfo.update({"is_emulator": self.config.get("is_emulator", True)}) # variants mozinfo.update({"e10s": self.config.get("e10s", True)}) - mozinfo.update( - { - "fission": "fission.autostart=false" - not in self.config.get("extra_prefs", []) - } - ) + mozinfo.update({ + "fission": "fission.autostart=false" + not in self.config.get("extra_prefs", []) + }) mozinfo.update({"headless": self.config.get("headless", False)}) mozinfo.update({"a11y_checks": self.config.get("a11y_checks", False)}) - mozinfo.update( - { - "socketprocess_e10s": "media.peerconnection.mtransport_process=true" - in self.config.get("extra_prefs", []) - } - ) - mozinfo.update( - { - "socketprocess_networking": "network.http.network_access_on_socket_process.enabled=true" - in self.config.get("extra_prefs", []) - } - ) - mozinfo.update( - { - "swgl": "gfx.webrender.software=true" - in self.config.get("extra_prefs", []) - } - ) - mozinfo.update( - {"wmfme": "media-engine-compatible" in self.config.get("test_tags", [])} - ) - mozinfo.update( - { - "emewmf": "media.wmf.media-engine.enabled=2" - in self.config.get("extra_prefs", []) - } - ) + mozinfo.update({ + "socketprocess_e10s": "media.peerconnection.mtransport_process=true" + in self.config.get("extra_prefs", []) + }) + mozinfo.update({ + "socketprocess_networking": "network.http.network_access_on_socket_process.enabled=true" + in self.config.get("extra_prefs", []) + }) + mozinfo.update({ + "swgl": "gfx.webrender.software=true" in self.config.get("extra_prefs", []) + }) + mozinfo.update({ + "wmfme": "media-engine-compatible" in self.config.get("test_tags", []) + }) + mozinfo.update({ + "emewmf": "media.wmf.media-engine.enabled=2" + in self.config.get("extra_prefs", []) + }) mozinfo.update({"mda_gpu": "media-gpu" in self.config.get("test_tags", [])}) - mozinfo.update( - { - "nogpu": "layers.gpu-process.enabled=false" - in self.config.get("extra_prefs", []) - } - ) + mozinfo.update({ + "nogpu": "layers.gpu-process.enabled=false" + in self.config.get("extra_prefs", []) + }) mozinfo.update({"msix": "msix" in self.config.get("variant", "")}) - mozinfo.update( - {"vertical_tab": "vertical-tabs" in self.config.get("test_tags", [])} - ) - mozinfo.update( - {"inc_origin_init": "inc-origin-init" in self.config.get("test_tags", [])} - ) - mozinfo.update( - {"privateBrowsing": "privatebrowsing" in self.config.get("test_tags", [])} - ) - mozinfo.update( - { - "sessionHistoryInParent": "fission.disableSessionHistoryInParent=true" - in self.config.get("extra_prefs", []) - } - ) + mozinfo.update({ + "vertical_tab": "vertical-tabs" in self.config.get("test_tags", []) + }) + mozinfo.update({ + "inc_origin_init": "inc-origin-init" in self.config.get("test_tags", []) + }) + mozinfo.update({ + "privateBrowsing": "privatebrowsing" in self.config.get("test_tags", []) + }) + mozinfo.update({ + "sessionHistoryInParent": "fission.disableSessionHistoryInParent=true" + in self.config.get("extra_prefs", []) + }) mozinfo.update({"http2": self.config.get("useHttp2Server", False)}) mozinfo.update({"http3": self.config.get("useHttp3Server", False)}) mozinfo.update({"xorigin": self.config.get("enable_xorigin_tests", False)}) diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -774,21 +774,25 @@ class Raptor( if self.gecko_profile: gecko_results.append("--gecko-profile") if self.gecko_profile_interval: - gecko_results.extend( - ["--gecko-profile-interval", str(self.gecko_profile_interval)] - ) + gecko_results.extend([ + "--gecko-profile-interval", + str(self.gecko_profile_interval), + ]) if self.gecko_profile_entries: - gecko_results.extend( - ["--gecko-profile-entries", str(self.gecko_profile_entries)] - ) + gecko_results.extend([ + "--gecko-profile-entries", + str(self.gecko_profile_entries), + ]) if self.gecko_profile_features: - gecko_results.extend( - ["--gecko-profile-features", self.gecko_profile_features] - ) + gecko_results.extend([ + "--gecko-profile-features", + self.gecko_profile_features, + ]) if self.gecko_profile_threads: - gecko_results.extend( - ["--gecko-profile-threads", self.gecko_profile_threads] - ) + gecko_results.extend([ + "--gecko-profile-threads", + self.gecko_profile_threads, + ]) elif self.extra_profiler_run: gecko_results.append("--extra-profiler-run") return gecko_results @@ -918,7 +922,7 @@ class Raptor( self.chromium_dist_path = "/usr/bin/google-chrome" elif mac in self.platform_name(): self.chromium_dist_path = ( - "/Applications/Google Chrome.app/" "Contents/MacOS/Google Chrome" + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" ) else: self.error( @@ -1088,26 +1092,24 @@ class Raptor( if self.config.get("enable_marionette_trace", False): options.extend(["--enable-marionette-trace"]) if self.config.get("browser_cycles"): - options.extend( - ["--browser-cycles={}".format(self.config.get("browser_cycles"))] - ) + options.extend([ + "--browser-cycles={}".format(self.config.get("browser_cycles")) + ]) if self.config.get("test_bytecode_cache", False): options.extend(["--test-bytecode-cache"]) if self.config.get("collect_perfstats", False): options.extend(["--collect-perfstats"]) if self.config.get("extra_summary_methods"): - options.extend( - [ - f"--extra-summary-methods={method}" - for method in self.config.get("extra_summary_methods") - ] - ) + options.extend([ + f"--extra-summary-methods={method}" + for method in self.config.get("extra_summary_methods") + ]) if self.config.get("page_timeout"): options.extend([f"--page-timeout={self.page_timeout}"]) if self.config.get("post_startup_delay"): - options.extend( - [f"--post-startup-delay={self.config['post_startup_delay']}"] - ) + options.extend([ + f"--post-startup-delay={self.config['post_startup_delay']}" + ]) if ( self.config.get("screenshot_on_failure", False) or os.environ.get("MOZ_AUTOMATION", None) is not None @@ -1239,35 +1241,29 @@ class Raptor( # components are particularly fussy about python version. py3_minor = sys.version_info.minor if py3_minor <= 7: - modules.extend( - [ - "numpy==1.16.1", - "Pillow==6.1.0", - "scipy==1.2.3", - "pyssim==0.4", - "opencv-python==4.5.4.60", - ] - ) + modules.extend([ + "numpy==1.16.1", + "Pillow==6.1.0", + "scipy==1.2.3", + "pyssim==0.4", + "opencv-python==4.5.4.60", + ]) elif py3_minor <= 11: - modules.extend( - [ - "numpy==1.23.5", - "Pillow==9.2.0", - "scipy==1.9.3", - "pyssim==0.4", - "opencv-python==4.6.0.66", - ] - ) + modules.extend([ + "numpy==1.23.5", + "Pillow==9.2.0", + "scipy==1.9.3", + "pyssim==0.4", + "opencv-python==4.6.0.66", + ]) else: # python version >= 3.12 - modules.extend( - [ - "numpy==2.2.3", - "Pillow==11.1.0", - "scipy==1.15.2", - "pyssim==0.7", - "opencv-python==4.11.0.86", - ] - ) + modules.extend([ + "numpy==2.2.3", + "Pillow==11.1.0", + "scipy==1.15.2", + "pyssim==0.7", + "opencv-python==4.11.0.86", + ]) if self.run_local: self.setup_local_ffmpeg() diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -655,39 +655,31 @@ class Talos( webextension_dest = os.path.join(self.talos_path, "talos", "webextensions") if self.query_pagesets_name(): - tooltool_artifacts.append( - { - "name": self.pagesets_name, - "manifest": self.pagesets_name_manifest, - "dest": src_talos_pageset_dest, - } - ) - tooltool_artifacts.append( - { - "name": self.pagesets_name, - "manifest": self.pagesets_name_manifest, - "dest": src_talos_pageset_multidomain_dest, - "postprocess": self.replace_relative_iframe_paths, - } - ) + tooltool_artifacts.append({ + "name": self.pagesets_name, + "manifest": self.pagesets_name_manifest, + "dest": src_talos_pageset_dest, + }) + tooltool_artifacts.append({ + "name": self.pagesets_name, + "manifest": self.pagesets_name_manifest, + "dest": src_talos_pageset_multidomain_dest, + "postprocess": self.replace_relative_iframe_paths, + }) if self.query_benchmark_zip(): - tooltool_artifacts.append( - { - "name": self.benchmark_zip, - "manifest": self.benchmark_zip_manifest, - "dest": src_talos_pageset_dest, - } - ) + tooltool_artifacts.append({ + "name": self.benchmark_zip, + "manifest": self.benchmark_zip_manifest, + "dest": src_talos_pageset_dest, + }) if self.query_webextensions_zip(): - tooltool_artifacts.append( - { - "name": self.webextensions_zip, - "manifest": self.webextensions_zip_manifest, - "dest": webextension_dest, - } - ) + tooltool_artifacts.append({ + "name": self.webextensions_zip, + "manifest": self.webextensions_zip_manifest, + "dest": webextension_dest, + }) # now that have the suite name, check if artifact is required, if so download it # the --no-download option will override this diff --git a/testing/mozharness/mozharness/mozilla/tooltool.py b/testing/mozharness/mozharness/mozilla/tooltool.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """module for tooltool operations""" + import os import sys @@ -71,9 +72,10 @@ class TooltoolMixin: if self.topsrcdir: cmd.extend(["--tooltool-manifest", manifest]) - cmd.extend( - ["--artifact-manifest", os.path.join(self.topsrcdir, "toolchains.json")] - ) + cmd.extend([ + "--artifact-manifest", + os.path.join(self.topsrcdir, "toolchains.json"), + ]) else: cmd.extend(["fetch", "-m", manifest, "-o"]) diff --git a/testing/mozharness/mozharness/mozilla/vcstools.py b/testing/mozharness/mozharness/mozilla/vcstools.py @@ -6,6 +6,7 @@ Author: Armen Zambrano G. """ + import os from mozharness.base.script import PreScriptAction diff --git a/testing/mozharness/scripts/android_emulator_pgo.py b/testing/mozharness/scripts/android_emulator_pgo.py @@ -138,9 +138,9 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin): """ Install APKs on the device. """ - assert ( - self.installer_path is not None - ), "Either add installer_path to the config or use --installer-path." + assert self.installer_path is not None, ( + "Either add installer_path to the config or use --installer-path." + ) self.install_android_app(self.installer_path) self.info("Finished installing apps for %s" % self.device_serial) diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py @@ -413,13 +413,11 @@ class AndroidEmulatorTest( ) if self.java_code_coverage_enabled: - cmd.extend( - [ - "--enable-coverage", - "--coverage-output-dir", - self.java_coverage_output_dir, - ] - ) + cmd.extend([ + "--enable-coverage", + "--coverage-output-dir", + self.java_coverage_output_dir, + ]) if self.config.get("restartAfterFailure", False): cmd.append("--restartAfterFailure") @@ -508,9 +506,9 @@ class AndroidEmulatorTest( if install_needed is False: self.info("Skipping apk installation for %s" % self.test_suite) return - assert ( - self.installer_path is not None - ), "Either add installer_path to the config or use --installer-path." + assert self.installer_path is not None, ( + "Either add installer_path to the config or use --installer-path." + ) self.install_android_app(self.installer_path) self.info("Finished installing apps for %s" % self.device_serial) diff --git a/testing/mozharness/scripts/android_hardware_unittest.py b/testing/mozharness/scripts/android_hardware_unittest.py @@ -431,9 +431,9 @@ class AndroidHardwareTest( if install_needed is False: self.info("Skipping apk installation for %s" % self.test_suite) return - assert ( - self.installer_path is not None - ), "Either add installer_path to the config or use --installer-path." + assert self.installer_path is not None, ( + "Either add installer_path to the config or use --installer-path." + ) self.uninstall_android_app() self.install_android_app(self.installer_path) self.info("Finished installing apps for %s" % self.device_name) diff --git a/testing/mozharness/scripts/desktop_partner_repacks.py b/testing/mozharness/scripts/desktop_partner_repacks.py @@ -6,6 +6,7 @@ This script manages Desktop partner repacks for beta/release builds. """ + import os import sys diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py @@ -726,12 +726,10 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin, CodeCoverageM c.get("install_extension", []) ): fetches_dir = os.environ.get("MOZ_FETCHES_DIR", '""') - base_cmd.extend( - [ - f"--install-extension={os.path.join(fetches_dir, e)}" - for e in c["install_extension"] - ] - ) + base_cmd.extend([ + f"--install-extension={os.path.join(fetches_dir, e)}" + for e in c["install_extension"] + ]) # do not add --disable fission if we don't have --disable-e10s if c["disable_fission"] and suite_category not in [ @@ -778,14 +776,12 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin, CodeCoverageM "'{suite_category}' suite." ) elif c.get("total_chunks") and c.get("this_chunk"): - base_cmd.extend( - [ - "--total-chunks", - c["total_chunks"], - "--this-chunk", - c["this_chunk"], - ] - ) + base_cmd.extend([ + "--total-chunks", + c["total_chunks"], + "--this-chunk", + c["this_chunk"], + ]) if c.get("timeout_factor"): base_cmd.extend(["--timeout-factor", c["timeout_factor"]]) @@ -1061,16 +1057,14 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin, CodeCoverageM command = [] # Implies that underlying system is Linux. if os.environ.get("NEED_PULSEAUDIO") == "true": - command.extend( - [ - "pulseaudio", - "--daemonize", - "--log-level=4", - "--log-time=1", - "-vvvvv", - "--exit-idle-time=-1", - ] - ) + command.extend([ + "pulseaudio", + "--daemonize", + "--log-level=4", + "--log-time=1", + "-vvvvv", + "--exit-idle-time=-1", + ]) # Only run the initialization for Debian. # Ubuntu appears to have an alternate method of starting pulseaudio. diff --git a/testing/mozharness/scripts/does_it_crash.py b/testing/mozharness/scripts/does_it_crash.py @@ -2,10 +2,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -""" does_it_crash.py +"""does_it_crash.py - Runs a thing to see if it crashes within a set period. +Runs a thing to see if it crashes within a set period. """ + import os import signal import subprocess diff --git a/testing/mozharness/scripts/multil10n.py b/testing/mozharness/scripts/multil10n.py @@ -2,9 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""multil10n.py - -""" +"""multil10n.py""" import os import sys diff --git a/testing/mozharness/scripts/raptor_script.py b/testing/mozharness/scripts/raptor_script.py @@ -3,9 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -"""raptor - -""" +"""raptor""" import os import sys diff --git a/testing/mozharness/scripts/release/bouncer_check.py b/testing/mozharness/scripts/release/bouncer_check.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -""" bouncer_check.py +"""bouncer_check.py A script to check HTTP statuses of Bouncer products to be shipped. """ diff --git a/testing/mozharness/scripts/repackage.py b/testing/mozharness/scripts/repackage.py @@ -87,18 +87,14 @@ class Repackage(BaseScript): command = [sys.executable, "mach", "--log-no-times", "repackage"] command.extend([arg.format(**subst) for arg in repack_config["args"]]) for arg, filename in repack_config["inputs"].items(): - command.extend( - [ - f"--{arg}", - os.path.join(dirs["abs_input_dir"], filename), - ] - ) - command.extend( - [ - "--output", - os.path.join(dirs["abs_output_dir"], repack_config["output"]), - ] - ) + command.extend([ + f"--{arg}", + os.path.join(dirs["abs_input_dir"], filename), + ]) + command.extend([ + "--output", + os.path.join(dirs["abs_output_dir"], repack_config["output"]), + ]) self.run_command( command=command, cwd=dirs["abs_src_dir"], @@ -128,12 +124,10 @@ class Repackage(BaseScript): os.path.join(dirs["abs_src_dir"], "toolchains.json"), ] if manifest_src: - cmd.extend( - [ - "--tooltool-manifest", - os.path.join(dirs["abs_src_dir"], manifest_src), - ] - ) + cmd.extend([ + "--tooltool-manifest", + os.path.join(dirs["abs_src_dir"], manifest_src), + ]) cache = config.get("tooltool_cache") if cache: cmd.extend(["--cache-dir", cache]) diff --git a/testing/mozharness/scripts/talos_script.py b/testing/mozharness/scripts/talos_script.py @@ -2,9 +2,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -"""talos - -""" +"""talos""" import os import sys diff --git a/testing/mozharness/test/test_base_config.py b/testing/mozharness/test/test_base_config.py @@ -112,38 +112,32 @@ class TestParseConfigFile(unittest.TestCase): def test_optional_config_files_override_value(self): c = config.BaseConfig(initial_config_file="test/test.py") - c.parse_args( - [ - "--cfg", - "test/test_override.py,test/test_override2.py", - "--opt-cfg", - "test/test_optional.py", - ] - ) + c.parse_args([ + "--cfg", + "test/test_override.py,test/test_override2.py", + "--opt-cfg", + "test/test_optional.py", + ]) self.assertEqual(c._config["opt_override"], "new stuff") def test_optional_config_files_missing_config(self): c = config.BaseConfig(initial_config_file="test/test.py") - c.parse_args( - [ - "--cfg", - "test/test_override.py,test/test_override2.py", - "--opt-cfg", - "test/test_optional.py,does_not_exist.py", - ] - ) + c.parse_args([ + "--cfg", + "test/test_override.py,test/test_override2.py", + "--opt-cfg", + "test/test_optional.py,does_not_exist.py", + ]) self.assertEqual(c._config["opt_override"], "new stuff") def test_optional_config_files_keep_string(self): c = config.BaseConfig(initial_config_file="test/test.py") - c.parse_args( - [ - "--cfg", - "test/test_override.py,test/test_override2.py", - "--opt-cfg", - "test/test_optional.py", - ] - ) + c.parse_args([ + "--cfg", + "test/test_override.py,test/test_override2.py", + "--opt-cfg", + "test/test_optional.py", + ]) self.assertEqual(c._config["keep_string"], "don't change me") diff --git a/testing/mozharness/test/test_base_script.py b/testing/mozharness/test/test_base_script.py @@ -418,9 +418,11 @@ class TestHelperFunctions(unittest.TestCase): def test_get_output_from_command(self): self._create_temp_file() self.s = script.BaseScript(initial_config_file="test/test.json") - contents = self.s.get_output_from_command( - ["bash", "-c", "cat %s" % self.temp_file] - ) + contents = self.s.get_output_from_command([ + "bash", + "-c", + "cat %s" % self.temp_file, + ]) self.assertEqual( test_string, contents, diff --git a/testing/mozharness/test/test_base_vcs_mercurial.py b/testing/mozharness/test/test_base_vcs_mercurial.py @@ -81,15 +81,11 @@ class TestMakeAbsolute(unittest.TestCase): class TestHg(unittest.TestCase): def _init_hg_repo(self, hg_obj, repodir): - hg_obj.run_command( - [ - "bash", - os.path.join( - os.path.dirname(__file__), "helper_files", "init_hgrepo.sh" - ), - repodir, - ] - ) + hg_obj.run_command([ + "bash", + os.path.join(os.path.dirname(__file__), "helper_files", "init_hgrepo.sh"), + repodir, + ]) def setUp(self): self.tmpdir = tempfile.mkdtemp() diff --git a/testing/performance/android-resource/parse_resource_usage.py b/testing/performance/android-resource/parse_resource_usage.py @@ -15,60 +15,48 @@ def make_differential_metrics( metrics = [] # Setup memory differentials - metrics.extend( - [ - { - "name": f"{mem_type}-{category}-{differential_name}", - "unit": "Kb", - "values": [ - round(mem_usage - base_measures["mem"][mem_type][category], 2) - ], - } - for mem_type, mem_info in mem_measures.items() - for category, mem_usage in mem_info.items() - if category in base_measures["mem"].get(mem_type, {}) - ] - ) - metrics.extend( - [ - { - "name": f"{mem_type}-total-{differential_name}", - "unit": "Kb", - "values": [ - round( - sum(mem_info.values()) - - sum(base_measures["mem"][mem_type].values()), - 2, - ) - ], - } - for mem_type, mem_info in mem_measures.items() - ] - ) - - # Setup cpuTime differentials - metrics.extend( - [ - { - "name": f"cpuTime-{category}-{differential_name}", - "unit": "ms", - "values": [cpu_time - base_measures["cpu"][category]], - } - for category, cpu_time in cpu_measures.items() - if category in base_measures.get("cpu", {}) - ] - ) - metrics.append( + metrics.extend([ { - "name": f"cpuTime-total-{differential_name}", - "unit": "ms", + "name": f"{mem_type}-{category}-{differential_name}", + "unit": "Kb", + "values": [round(mem_usage - base_measures["mem"][mem_type][category], 2)], + } + for mem_type, mem_info in mem_measures.items() + for category, mem_usage in mem_info.items() + if category in base_measures["mem"].get(mem_type, {}) + ]) + metrics.extend([ + { + "name": f"{mem_type}-total-{differential_name}", + "unit": "Kb", "values": [ round( - sum(cpu_measures.values()) - sum(base_measures["cpu"].values()), 2 + sum(mem_info.values()) + - sum(base_measures["mem"][mem_type].values()), + 2, ) ], } - ) + for mem_type, mem_info in mem_measures.items() + ]) + + # Setup cpuTime differentials + metrics.extend([ + { + "name": f"cpuTime-{category}-{differential_name}", + "unit": "ms", + "values": [cpu_time - base_measures["cpu"][category]], + } + for category, cpu_time in cpu_measures.items() + if category in base_measures.get("cpu", {}) + ]) + metrics.append({ + "name": f"cpuTime-total-{differential_name}", + "unit": "ms", + "values": [ + round(sum(cpu_measures.values()) - sum(base_measures["cpu"].values()), 2) + ], + }) return metrics @@ -222,45 +210,37 @@ def main(): base_measures["cpu"] = cpu_measures base_measures["mem"] = mem_measures - perf_metrics.extend( - [ - { - "name": f"cpuTime-{category}-{measurement_time}", - "unit": "ms", - "values": [cpu_time], - } - for category, cpu_time in cpu_measures.items() - ] - ) - perf_metrics.append( + perf_metrics.extend([ { - "name": f"cpuTime-total-{measurement_time}", + "name": f"cpuTime-{category}-{measurement_time}", "unit": "ms", - "values": [round(sum(cpu_measures.values()), 2)], + "values": [cpu_time], + } + for category, cpu_time in cpu_measures.items() + ]) + perf_metrics.append({ + "name": f"cpuTime-total-{measurement_time}", + "unit": "ms", + "values": [round(sum(cpu_measures.values()), 2)], + }) + + perf_metrics.extend([ + { + "name": f"{mem_type}-{category}-{measurement_time}", + "unit": "Kb", + "values": [round(mem_usage, 2)], + } + for mem_type, mem_info in mem_measures.items() + for category, mem_usage in mem_info.items() + ]) + perf_metrics.extend([ + { + "name": f"{mem_type}-total-{measurement_time}", + "unit": "Kb", + "values": [round(sum(mem_info.values()), 2)], } - ) - - perf_metrics.extend( - [ - { - "name": f"{mem_type}-{category}-{measurement_time}", - "unit": "Kb", - "values": [round(mem_usage, 2)], - } - for mem_type, mem_info in mem_measures.items() - for category, mem_usage in mem_info.items() - ] - ) - perf_metrics.extend( - [ - { - "name": f"{mem_type}-total-{measurement_time}", - "unit": "Kb", - "values": [round(sum(mem_info.values()), 2)], - } - for mem_type, mem_info in mem_measures.items() - ] - ) + for mem_type, mem_info in mem_measures.items() + ]) if base_measures and run_background: if measurement_time == "10%": diff --git a/testing/performance/hooks_recording.py b/testing/performance/hooks_recording.py @@ -77,7 +77,7 @@ def before_iterations(kw): ): print( f"Skipping login test `{test.get('name')}` " - f"Because SCM = `{os.environ.get('MOZ_SCM_LEVEL') }`" + f"Because SCM = `{os.environ.get('MOZ_SCM_LEVEL')}`" f"and there is no secret available at this level" ) return False diff --git a/testing/performance/mobile-startup/android_startup_cmff_cvns.py b/testing/performance/mobile-startup/android_startup_cmff_cvns.py @@ -124,9 +124,12 @@ class Startup_test: if i % 10 == 0: screenshot_file = f"/sdcard/Download/{self.product}_iteration_{i}_startup_done_frame.png" self.device.shell(f"screencap -p {screenshot_file}") - self.device.command_output( - ["pull", "-a", screenshot_file, os.environ["TESTING_DIR"]] - ) + self.device.command_output([ + "pull", + "-a", + screenshot_file, + os.environ["TESTING_DIR"], + ]) self.device.stop_application(self.package_id) print(f"{self.test_name}: {str(test_measurements)}") # Bug 1934023 - create way to pass median and still have replicates available diff --git a/testing/performance/mobile-startup/android_startup_videoapplink.py b/testing/performance/mobile-startup/android_startup_videoapplink.py @@ -119,15 +119,13 @@ class ImageAnalzer: # Bug 1927548 - Recording command doesn't use mozdevice shell because the mozdevice shell # outputs an adbprocess obj whose adbprocess.proc.kill() does not work when called - recording = subprocess.Popen( - [ - "adb", - "shell", - "screenrecord", - "--bugreport", - video_location, - ] - ) + recording = subprocess.Popen([ + "adb", + "shell", + "screenrecord", + "--bugreport", + video_location, + ]) # Start Profilers if enabled. self.profiler.start() @@ -143,9 +141,12 @@ class ImageAnalzer: self.process_cpu_info(run) recording.kill() time.sleep(5) - self.device.command_output( - ["pull", "-a", video_location, os.environ["TESTING_DIR"]] - ) + self.device.command_output([ + "pull", + "-a", + video_location, + os.environ["TESTING_DIR"], + ]) time.sleep(4) video_location = str(pathlib.Path(os.environ["TESTING_DIR"], self.video_name)) @@ -216,7 +217,7 @@ class ImageAnalzer: video_timestamp = self.video.get(cv2.CAP_PROP_POS_MSEC) if video_timestamp > MAX_STARTUP_TIME: raise ValueError( - f"Startup time of {video_timestamp/1000}s exceeds max time of {MAX_STARTUP_TIME/1000}s" + f"Startup time of {video_timestamp / 1000}s exceeds max time of {MAX_STARTUP_TIME / 1000}s" ) return video_timestamp diff --git a/testing/platform_diff.py b/testing/platform_diff.py @@ -13,7 +13,6 @@ USER_AGENT = "mach-runner-diff/1.0" class PlatformDiff: - def __init__( self, command_context, @@ -93,9 +92,10 @@ class PlatformDiff: if line.startswith("+"): total_added.add(line.strip("+")) elif line.startswith("-"): - total_removed.add( - (line.strip("-"), self.transform_old_platform(line.strip("-"))) - ) + total_removed.add(( + line.strip("-"), + self.transform_old_platform(line.strip("-")), + )) total_removed_suggestion = set([r[1] for r in total_removed]) new_platforms = total_added - total_removed_suggestion diff --git a/testing/raptor/browsertime/support-scripts/browsertime_benchmark.py b/testing/raptor/browsertime/support-scripts/browsertime_benchmark.py @@ -47,14 +47,12 @@ class BenchmarkSupport(PageloadSupport): def modify_command(self, cmd, test): # Enable cpuTime, and wallclock-tracking metrics - cmd.extend( - [ - "--browsertime.cpuTime_test", - "true", - "--browsertime.wallclock_tracking_test", - "true", - ] - ) + cmd.extend([ + "--browsertime.cpuTime_test", + "true", + "--browsertime.wallclock_tracking_test", + "true", + ]) def handle_result(self, bt_result, raw_result, **kwargs): """Parse a result for the required results. @@ -273,9 +271,9 @@ class BenchmarkSupport(PageloadSupport): if not isinstance(value, Iterable): updated_metric = [value] # pylint: disable=W1633 - _subtests[metric]["replicates"].extend( - [round(x, 3) for x in updated_metric] - ) + _subtests[metric]["replicates"].extend([ + round(x, 3) for x in updated_metric + ]) vals = [] subtests = [] diff --git a/testing/raptor/browsertime/support-scripts/browsertime_pageload.py b/testing/raptor/browsertime/support-scripts/browsertime_pageload.py @@ -147,9 +147,9 @@ class PageloadSupport(BasePythonSupport): power_vals = raw_result.get("android").get("power", {}) if power_vals: - bt_result["measurements"].setdefault("powerUsage", []).extend( - [round(vals["powerUsage"] * (1 * 10**-6), 2) for vals in power_vals] - ) + bt_result["measurements"].setdefault("powerUsage", []).extend([ + round(vals["powerUsage"] * (1 * 10**-6), 2) for vals in power_vals + ]) def _process_measurements(self, suite, test, measurement_name, replicates): subtest = {} diff --git a/testing/raptor/browsertime/support-scripts/jetstream3.py b/testing/raptor/browsertime/support-scripts/jetstream3.py @@ -20,7 +20,6 @@ class JetStreamSupport(BasePythonSupport): score_tracker = {} for k, v in raw_result["extras"][0]["js3_res"]["tests"].items(): - score_tracker[k + "-" + "Geometric"] = v["metrics"]["Score"]["current"] for measure, metrics in v["tests"].items(): score_tracker[k + "-" + measure] = metrics["metrics"]["Time"]["current"] diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py @@ -137,12 +137,10 @@ class RaptorRunner(MozbuildObject): # We don't set `browsertime_ffmpeg` yet: it will need to be on the path. There is code # to configure the environment including the path in # `tools/browsertime/mach_commands.py` but integrating it here will take more effort. - self.config.update( - { - "browsertime_browsertimejs": browsertime.browsertime_path(), - "browsertime_vismet_script": browsertime.visualmetrics_path(), - } - ) + self.config.update({ + "browsertime_browsertimejs": browsertime.browsertime_path(), + "browsertime_vismet_script": browsertime.visualmetrics_path(), + }) def _get_browsertime_package(): with open( @@ -179,9 +177,9 @@ class RaptorRunner(MozbuildObject): # If ffmpeg doesn't exist in the .mozbuild directory, # then we should install btime_cache = os.path.join(self.config["mozbuild_path"], "browsertime") - if not os.path.exists(btime_cache) or not any( - ["ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache)] - ): + if not os.path.exists(btime_cache) or not any([ + "ffmpeg" in cache_dir for cache_dir in os.listdir(btime_cache) + ]): return True # If browsertime doesn't exist, install it diff --git a/testing/raptor/raptor/base_python_support.py b/testing/raptor/raptor/base_python_support.py @@ -273,9 +273,9 @@ class BasePythonSupport: power_usage_measurements.setdefault( "powerUsage", dict(default_power_settings) ).setdefault("replicates", []).extend( - __convert_from_pico_to_micro( - [vals["powerUsage"] for vals in power_vals] - ) + __convert_from_pico_to_micro([ + vals["powerUsage"] for vals in power_vals + ]) ) return power_usage_measurements diff --git a/testing/raptor/raptor/benchmark.py b/testing/raptor/raptor/benchmark.py @@ -104,18 +104,16 @@ class Benchmark: LOG.warning(f"Failed to stop benchmark server: {traceback.format_exc()}") def _full_clone(self, benchmark_repository, dest): - subprocess.check_call( - [ - "git", - "clone", - "-c", - "http.postBuffer=2147483648", - "-c", - "core.autocrlf=false", - benchmark_repository, - str(dest.resolve()), - ] - ) + subprocess.check_call([ + "git", + "clone", + "-c", + "http.postBuffer=2147483648", + "-c", + "core.autocrlf=false", + benchmark_repository, + str(dest.resolve()), + ]) def _get_benchmark_folder(self, benchmark_dest, run_local): if not run_local: @@ -131,19 +129,17 @@ class Benchmark: See bug 1804694. This method should only be used in CI, locally we can simply pull the whole repo. """ - subprocess.check_call( - [ - "git", - "clone", - "--depth", - "1", - "--filter", - "blob:none", - "--sparse", - benchmark_repository, - str(dest.resolve()), - ] - ) + subprocess.check_call([ + "git", + "clone", + "--depth", + "1", + "--filter", + "blob:none", + "--sparse", + benchmark_repository, + str(dest.resolve()), + ]) subprocess.check_call( [ "git", diff --git a/testing/raptor/raptor/browsertime/android.py b/testing/raptor/raptor/browsertime/android.py @@ -96,12 +96,10 @@ class BrowsertimeAndroid(PerftestAndroid, Browsertime): ] if self.config["app"] in CHROME_ANDROID_APPS: - args_list.extend( - [ - "--browser", - "chrome", - ] - ) + args_list.extend([ + "--browser", + "chrome", + ]) if self.config["app"] == "cstm-car-m": args_list.extend(["--chrome.android.package", "org.chromium.chrome"]) else: @@ -114,23 +112,19 @@ class BrowsertimeAndroid(PerftestAndroid, Browsertime): activity = "mozilla.telemetry.glean.debug.GleanDebugActivity" # all hardware we test on is android 11+ - args_list.extend( - [ - '--firefox.geckodriverArgs="--android-storage"', - '--firefox.geckodriverArgs="app"', - ] - ) - - args_list.extend( - [ - "--browser", - "firefox", - "--firefox.android.package", - self.config["binary"], - "--firefox.android.activity", - activity, - ] - ) + args_list.extend([ + '--firefox.geckodriverArgs="--android-storage"', + '--firefox.geckodriverArgs="app"', + ]) + + args_list.extend([ + "--browser", + "firefox", + "--firefox.android.package", + self.config["binary"], + "--firefox.android.activity", + activity, + ]) if self.config["app"] == "geckoview": # This is needed as geckoview is crashing on shutdown and is throwing marionette errors similar to 1768889 @@ -144,24 +138,23 @@ class BrowsertimeAndroid(PerftestAndroid, Browsertime): # special non-default one there if self.config.get("intent") is not None: args_list.extend(["--firefox.android.intentArgument=-a"]) - args_list.extend( - ["--firefox.android.intentArgument", self.config["intent"]] - ) + args_list.extend([ + "--firefox.android.intentArgument", + self.config["intent"], + ]) # Change glean ping names in all cases on Fenix - args_list.extend( - [ - "--firefox.android.intentArgument=--es", - "--firefox.android.intentArgument=startNext", - "--firefox.android.intentArgument=" + self.config["activity"], - "--firefox.android.intentArgument=--esa", - "--firefox.android.intentArgument=sourceTags", - "--firefox.android.intentArgument=automation", - "--firefox.android.intentArgument=--ez", - "--firefox.android.intentArgument=performancetest", - "--firefox.android.intentArgument=true", - ] - ) + args_list.extend([ + "--firefox.android.intentArgument=--es", + "--firefox.android.intentArgument=startNext", + "--firefox.android.intentArgument=" + self.config["activity"], + "--firefox.android.intentArgument=--esa", + "--firefox.android.intentArgument=sourceTags", + "--firefox.android.intentArgument=automation", + "--firefox.android.intentArgument=--ez", + "--firefox.android.intentArgument=performancetest", + "--firefox.android.intentArgument=true", + ]) args_list.extend(["--firefox.android.intentArgument=-d"]) args_list.extend(["--firefox.android.intentArgument", "about:blank"]) diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py @@ -171,9 +171,10 @@ class Browsertime(Perftest, metaclass=ABCMeta): # TODO: geckodriver/chromedriver from tasks. self.driver_paths = [] if self.browsertime_geckodriver: - self.driver_paths.extend( - ["--firefox.geckodriverPath", self.browsertime_geckodriver] - ) + self.driver_paths.extend([ + "--firefox.geckodriverPath", + self.browsertime_geckodriver, + ]) if self.browsertime_chromedriver and self.config["app"] in ( "chrome", "chrome-m", @@ -240,9 +241,10 @@ class Browsertime(Perftest, metaclass=ABCMeta): "being tested: %s" % self.browsertime_chromedriver ) - self.driver_paths.extend( - ["--chrome.chromedriverPath", self.browsertime_chromedriver] - ) + self.driver_paths.extend([ + "--chrome.chromedriverPath", + self.browsertime_chromedriver, + ]) # YTP tests fail in mozilla-release due to the `MOZ_DISABLE_NONLOCAL_CONNECTIONS` # environment variable. This logic changes this variable for the browsertime test @@ -442,19 +444,18 @@ class Browsertime(Perftest, metaclass=ABCMeta): if self.config["app"] in ("fenix",): # Fenix can take a lot of time to startup - browsertime_options.extend( - [ - "--browsertime.browserRestartTries", - "10", - "--timeouts.browserStart", - "180000", - ] - ) + browsertime_options.extend([ + "--browsertime.browserRestartTries", + "10", + "--timeouts.browserStart", + "180000", + ]) if test.get("secondary_url"): - browsertime_options.extend( - ["--browsertime.secondary_url", test.get("secondary_url")] - ) + browsertime_options.extend([ + "--browsertime.secondary_url", + test.get("secondary_url"), + ]) # These options can have multiple entries in a browsertime command MULTI_OPTS = [ @@ -500,20 +501,20 @@ class Browsertime(Perftest, metaclass=ABCMeta): if not extra_profiler_run: # must happen before --firefox.profileTemplate and --resultDir self.results_handler.remove_result_dir_for_test(test) - priority1_options.extend( - ["--resultDir", self.results_handler.result_dir_for_test(test)] - ) + priority1_options.extend([ + "--resultDir", + self.results_handler.result_dir_for_test(test), + ]) else: - priority1_options.extend( - [ - "--resultDir", - self.results_handler.result_dir_for_test_profiling(test), - ] - ) + priority1_options.extend([ + "--resultDir", + self.results_handler.result_dir_for_test_profiling(test), + ]) if self.profile is not None: - priority1_options.extend( - ["--firefox.profileTemplate", str(self.profile.profile)] - ) + priority1_options.extend([ + "--firefox.profileTemplate", + str(self.profile.profile), + ]) # This argument can have duplicates of the value "--firefox.env" so we do not need # to check if it conflicts @@ -529,22 +530,20 @@ class Browsertime(Perftest, metaclass=ABCMeta): browsertime_options.append("-vvv") if self.browsertime_video: - priority1_options.extend( - [ - "--video", - "true", - "--visualMetrics", - "true" if self.browsertime_visualmetrics else "false", - "--visualMetricsContentful", - "true", - "--visualMetricsPerceptual", - "true", - "--visualMetricsPortable", - "true", - "--videoParams.keepOriginalVideo", - "true", - ] - ) + priority1_options.extend([ + "--video", + "true", + "--visualMetrics", + "true" if self.browsertime_visualmetrics else "false", + "--visualMetricsContentful", + "true", + "--visualMetricsPerceptual", + "true", + "--visualMetricsPortable", + "true", + "--videoParams.keepOriginalVideo", + "true", + ]) if self.browsertime_no_ffwindowrecorder or self.config["app"] in ( "chrome-m", @@ -552,24 +551,20 @@ class Browsertime(Perftest, metaclass=ABCMeta): "custom-car", "cstm-car-m", ): - priority1_options.extend( - [ - "--firefox.windowRecorder", - "false", - "--xvfbParams.display", - "0", - ] - ) + priority1_options.extend([ + "--firefox.windowRecorder", + "false", + "--xvfbParams.display", + "0", + ]) LOG.info( "Using adb screenrecord for mobile, or ffmpeg on desktop for videos" ) else: - priority1_options.extend( - [ - "--firefox.windowRecorder", - "true", - ] - ) + priority1_options.extend([ + "--firefox.windowRecorder", + "true", + ]) LOG.info("Using Firefox Window Recorder for videos") else: priority1_options.extend(["--video", "false", "--visualMetrics", "false"]) @@ -649,12 +644,10 @@ class Browsertime(Perftest, metaclass=ABCMeta): self._init_gecko_profiling(test) priority1_options.append("--firefox.geckoProfiler") if self._expose_browser_profiler(self.config.get("extra_profiler_run"), test): - priority1_options.extend( - [ - "--firefox.geckoProfilerRecordingType", - "custom", - ] - ) + priority1_options.extend([ + "--firefox.geckoProfilerRecordingType", + "custom", + ]) for option, browsertime_option, default in ( ( "gecko_profile_features", @@ -732,9 +725,10 @@ class Browsertime(Perftest, metaclass=ABCMeta): as a safety precaution when doing a live login site. """ browsertime_options.extend(["--browsertime.testName", str(test.get("name"))]) - browsertime_options.extend( - ["--browsertime.liveSite", str(self.config["live_sites"])] - ) + browsertime_options.extend([ + "--browsertime.liveSite", + str(self.config["live_sites"]), + ]) login_required = self.is_live_login_site(test.get("name")) browsertime_options.extend(["--browsertime.loginRequired", str(login_required)]) @@ -823,7 +817,7 @@ class Browsertime(Perftest, metaclass=ABCMeta): break except ValueError: raise Exception( - f"Received a non-int value for the iterations: {cmd[i+1]}" + f"Received a non-int value for the iterations: {cmd[i + 1]}" ) bt_timeout = bt_timeout * iterations diff --git a/testing/raptor/raptor/browsertime/desktop.py b/testing/raptor/raptor/browsertime/desktop.py @@ -49,9 +49,11 @@ class BrowsertimeDesktop(PerftestDesktop, Browsertime): # Add this argument here, it's added by mozrunner # for raptor - chrome_args.extend( - ["--no-first-run", "--no-experiments", "--disable-site-isolation-trials"] - ) + chrome_args.extend([ + "--no-first-run", + "--no-experiments", + "--disable-site-isolation-trials", + ]) # Disable finch experiments chrome_args += ["--enable-benchmarking"] diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py @@ -256,20 +256,24 @@ def create_parser(mach_interface=False): default=None, help="How long to wait (ms) after browser start-up before starting the tests", ) - add_arg( - "--browser-cycles", - dest="browser_cycles", - type=int, - help="The number of times a cold load test is repeated (for cold load tests only, " - "where the browser is shutdown and restarted between test iterations)", - ), - add_arg( - "--project", - dest="project", - type=str, - default="mozilla-central", - help="Project name (try, mozilla-central, etc.)", - ), + ( + add_arg( + "--browser-cycles", + dest="browser_cycles", + type=int, + help="The number of times a cold load test is repeated (for cold load tests only, " + "where the browser is shutdown and restarted between test iterations)", + ), + ) + ( + add_arg( + "--project", + dest="project", + type=str, + default="mozilla-central", + help="Project name (try, mozilla-central, etc.)", + ), + ) add_arg( "--test-url-params", dest="test_url_params", diff --git a/testing/raptor/raptor/gecko_profile.py b/testing/raptor/raptor/gecko_profile.py @@ -5,6 +5,7 @@ """ Module to handle Gecko profiling. """ + import json import os import tempfile @@ -115,32 +116,30 @@ class GeckoProfile(RaptorProfiling): LOG.error("No profiles collected") return - symbolicator = ProfileSymbolicator( - { - # Trace-level logging (verbose) - "enableTracing": 0, - # Fallback server if symbol is not found locally - "remoteSymbolServer": "https://symbolication.services.mozilla.com/symbolicate/v4", - # Maximum number of symbol files to keep in memory - "maxCacheEntries": 2000000, - # Frequency of checking for recent symbols to - # cache (in hours) - "prefetchInterval": 12, - # Oldest file age to prefetch (in hours) - "prefetchThreshold": 48, - # Maximum number of library versions to pre-fetch - # per library - "prefetchMaxSymbolsPerLib": 3, - # Default symbol lookup directories - "defaultApp": "FIREFOX", - "defaultOs": "WINDOWS", - # Paths to .SYM files, expressed internally as a - # mapping of app or platform names to directories - # Note: App & OS names from requests are converted - # to all-uppercase internally - "symbolPaths": self.symbol_paths, - } - ) + symbolicator = ProfileSymbolicator({ + # Trace-level logging (verbose) + "enableTracing": 0, + # Fallback server if symbol is not found locally + "remoteSymbolServer": "https://symbolication.services.mozilla.com/symbolicate/v4", + # Maximum number of symbol files to keep in memory + "maxCacheEntries": 2000000, + # Frequency of checking for recent symbols to + # cache (in hours) + "prefetchInterval": 12, + # Oldest file age to prefetch (in hours) + "prefetchThreshold": 48, + # Maximum number of library versions to pre-fetch + # per library + "prefetchMaxSymbolsPerLib": 3, + # Default symbol lookup directories + "defaultApp": "FIREFOX", + "defaultOs": "WINDOWS", + # Paths to .SYM files, expressed internally as a + # mapping of app or platform names to directories + # Note: App & OS names from requests are converted + # to all-uppercase internally + "symbolPaths": self.symbol_paths, + }) if self.raptor_config.get("symbols_path") is not None: if mozfile.is_url(self.raptor_config["symbols_path"]): diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py @@ -200,15 +200,13 @@ def add_test_url_params(url, extra_params): final_query_string = unquote(urlencode(parsed_query_params, doseq=True)) # reconstruct test_url with the changed query string - return urlunsplit( - ( - parsed_url.scheme, - parsed_url.netloc, - parsed_url.path, - final_query_string, - parsed_url.fragment, - ) - ) + return urlunsplit(( + parsed_url.scheme, + parsed_url.netloc, + parsed_url.path, + final_query_string, + parsed_url.fragment, + )) def write_test_settings_json(args, test_details, oskey): @@ -268,9 +266,9 @@ def write_test_settings_json(args, test_details, oskey): "raptor-options" ]["lower_is_better"] else: - test_settings["raptor-options"][ - "subtest_lower_is_better" - ] = subtest_lower_is_better + test_settings["raptor-options"]["subtest_lower_is_better"] = ( + subtest_lower_is_better + ) if test_details.get("alert_change_type", None) is not None: test_settings["raptor-options"]["alert_change_type"] = test_details[ @@ -298,18 +296,16 @@ def write_test_settings_json(args, test_details, oskey): ) threads.extend(test_threads) - test_settings["raptor-options"].update( - { - "gecko_profile": True, - "gecko_profile_entries": int( - test_details.get("gecko_profile_entries", 1000000) - ), - "gecko_profile_interval": float( - test_details.get("gecko_profile_interval", 1) - ), - "gecko_profile_threads": ",".join(set(threads)), - } - ) + test_settings["raptor-options"].update({ + "gecko_profile": True, + "gecko_profile_entries": int( + test_details.get("gecko_profile_entries", 1000000) + ), + "gecko_profile_interval": float( + test_details.get("gecko_profile_interval", 1) + ), + "gecko_profile_threads": ",".join(set(threads)), + }) features = test_details.get("gecko_profile_features") if features: diff --git a/testing/raptor/raptor/output.py b/testing/raptor/raptor/output.py @@ -5,6 +5,7 @@ # some parts of this originally taken from /testing/talos/talos/output.py """output raptor test results""" + import copy import json import os @@ -148,13 +149,11 @@ class PerftestOutput(metaclass=ABCMeta): "subtests": subtests, } if data_set.get("summarize-values", True): - suite.update( - { - "lowerIsBetter": True, - "unit": data_set["unit"], - "alertThreshold": 2.0, - } - ) + suite.update({ + "lowerIsBetter": True, + "unit": data_set["unit"], + "alertThreshold": 2.0, + }) for result in self.results: if result["name"] == data_set["test"]: @@ -297,9 +296,9 @@ class PerftestOutput(metaclass=ABCMeta): self.summarized_results["os"] = {"name": self.os_name} self._set_platform_version() if self.os_platform_version: - self.summarized_results["os"][ - "platform_version" - ] = self.os_platform_version + self.summarized_results["os"]["platform_version"] = ( + self.os_platform_version + ) total_perfdata = 0 if output_perf_data: @@ -577,9 +576,9 @@ class PerftestOutput(metaclass=ABCMeta): if not isinstance(value, Iterable): updated_metric = [value] # pylint: disable=W1633 - _subtests[metric]["replicates"].extend( - [round(x, 3) for x in updated_metric] - ) + _subtests[metric]["replicates"].extend([ + round(x, 3) for x in updated_metric + ]) vals = [] subtests = [] @@ -742,9 +741,9 @@ class PerftestOutput(metaclass=ABCMeta): "replicates": [], } # pylint: disable=W1633 - _subtests[sub]["replicates"].extend( - [float(round(x, 3)) for x in replicates] - ) + _subtests[sub]["replicates"].extend([ + float(round(x, 3)) for x in replicates + ]) vals = [] for name, subtest_data in _subtests.items(): @@ -1011,9 +1010,9 @@ class PerftestOutput(metaclass=ABCMeta): "replicates": [], } # pylint: disable=W1633 - _subtests[sub]["replicates"].extend( - [float(round(x, 3)) for x in replicates] - ) + _subtests[sub]["replicates"].extend([ + float(round(x, 3)) for x in replicates + ]) vals = [] subtests = [] @@ -1084,9 +1083,9 @@ class PerftestOutput(metaclass=ABCMeta): "replicates": [], } # pylint: disable=W1633 - _subtests[sub]["replicates"].extend( - [float(round(x, 3)) for x in replicates] - ) + _subtests[sub]["replicates"].extend([ + float(round(x, 3)) for x in replicates + ]) subtests = [] vals = [] @@ -1165,9 +1164,9 @@ class PerftestOutput(metaclass=ABCMeta): "replicates": [], } # pylint: disable=W1633 - _subtests[sub]["replicates"].extend( - [float(round(x, 3)) for x in replicates] - ) + _subtests[sub]["replicates"].extend([ + float(round(x, 3)) for x in replicates + ]) vals = [] subtests = [] @@ -1414,12 +1413,10 @@ class RaptorOutput(PerftestOutput): subtests.append(new_subtest) elif test["type"] == "benchmark": - if any( - [ - "youtube-playback" in measurement - for measurement in test["measurements"].keys() - ] - ): + if any([ + "youtube-playback" in measurement + for measurement in test["measurements"].keys() + ]): subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test) elif "assorted-dom" in test["measurements"]: subtests, vals = self.parseAssortedDomOutput(test) diff --git a/testing/raptor/raptor/performance_tuning.py b/testing/raptor/raptor/performance_tuning.py @@ -135,14 +135,12 @@ class PerformanceTuner: # Pixel 6 perf tuning: Bug 1876545 # Samsung S21 perf tuning: Bug 1876546 if device_name == "SM-A515F": - commands.update( - { - "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "performance", - "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "performance", - "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1742000", - "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "2314000", - } - ) + commands.update({ + "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor": "performance", + "/sys/devices/system/cpu/cpufreq/policy4/scaling_governor": "performance", + "/sys/devices/system/cpu/cpufreq/policy0/scaling_min_freq": "1742000", + "/sys/devices/system/cpu/cpufreq/policy4/scaling_min_freq": "2314000", + }) else: self.log.info( f"CPU for device with ro.product.model '{device_name}' unknown, not scaling_governor" diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py @@ -609,16 +609,13 @@ class Perftest(metaclass=ABCMeta): playback_manifest = test.get("playback_pageset_manifest") playback_manifests = playback_manifest.split(",") - self.config.update( - { - "playback_tool": test.get("playback"), - "playback_version": test.get("playback_version", "8.1.1"), - "playback_files": [ - os.path.join(playback_dir, manifest) - for manifest in playback_manifests - ], - } - ) + self.config.update({ + "playback_tool": test.get("playback"), + "playback_version": test.get("playback_version", "8.1.1"), + "playback_files": [ + os.path.join(playback_dir, manifest) for manifest in playback_manifests + ], + }) LOG.info("test uses playback tool: %s " % self.config["playback_tool"]) diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py @@ -38,11 +38,9 @@ def main(args=sys.argv[1:]): args.extra_prefs = parse_preferences(args.extra_prefs or []) if args.enable_marionette_trace: - args.extra_prefs.update( - { - "remote.log.level": "Trace", - } - ) + args.extra_prefs.update({ + "remote.log.level": "Trace", + }) args.environment = dict(parse_key_value(args.environment or [], context="--setenv")) diff --git a/testing/raptor/raptor/raptor_profiling.py b/testing/raptor/raptor/raptor_profiling.py @@ -112,19 +112,18 @@ class RaptorProfiling: "The test ran in chimera mode but we found no cold " "and warm browsertime JSONs. Cannot collect profiles." ) - profile_locations.extend( - [("cold", results["cold"]), ("warm", results["warm"])] - ) + profile_locations.extend([ + ("cold", results["cold"]), + ("warm", results["warm"]), + ]) else: # When we don't run in chimera mode, it means that we # either ran a benchmark, scenario test or separate # warm/cold pageload tests. - profile_locations.append( - ( - __get_test_type(), - results["main"], - ) - ) + profile_locations.append(( + __get_test_type(), + results["main"], + )) for testtype, results_json in profile_locations: with open(results_json, encoding="utf-8") as f: @@ -135,12 +134,10 @@ class RaptorProfiling: for rel_profile_path in entry["files"][ self.profile_entry_string ]: - res.append( - { - "path": os.path.join(results_dir, rel_profile_path), - "type": testtype, - } - ) + res.append({ + "path": os.path.join(results_dir, rel_profile_path), + "type": testtype, + }) except KeyError: if self._is_extra_profiler_run: LOG.info("Failed to find profiles for extra profiler run.") @@ -150,12 +147,10 @@ class RaptorProfiling: # Raptor-webext stores its profiles in the self.temp_profile_dir # directory for profile in os.listdir(self.temp_profile_dir): - res.append( - { - "path": os.path.join(self.temp_profile_dir, profile), - "type": __get_test_type(), - } - ) + res.append({ + "path": os.path.join(self.temp_profile_dir, profile), + "type": __get_test_type(), + }) LOG.info("Found %s profiles: %s" % (len(res), str(res))) return res diff --git a/testing/raptor/raptor/results.py b/testing/raptor/raptor/results.py @@ -132,9 +132,11 @@ class PerftestResultsHandler(metaclass=ABCMeta): def add_image(self, screenshot, test_name, page_cycle): # add to results LOG.info("received screenshot") - self.images.append( - {"screenshot": screenshot, "test_name": test_name, "page_cycle": page_cycle} - ) + self.images.append({ + "screenshot": screenshot, + "test_name": test_name, + "page_cycle": page_cycle, + }) def add_page_timeout(self, test_name, page_url, page_cycle, pending_metrics): timeout_details = { @@ -246,15 +248,13 @@ class RaptorResultsHandler(PerftestResultsHandler): def add(self, new_result_json): LOG.info("received results in RaptorResultsHandler.add") new_result_json.setdefault("extra_options", []).extend( - self.build_extra_options( - [ - ( - self.conditioned_profile, - "condprof-%s" % self.conditioned_profile, - ), - (self.fission_enabled, "fission"), - ] - ) + self.build_extra_options([ + ( + self.conditioned_profile, + "condprof-%s" % self.conditioned_profile, + ), + (self.fission_enabled, "fission"), + ]) ) if self.live_sites: new_result_json.setdefault("tags", []).append("live") @@ -353,12 +353,10 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): self.browsertime_results_folders["browsertime_results"] ] if has_video_files: - target_subfolders.extend( - [ - self.browsertime_results_folders["videos_annotated"], - self.browsertime_results_folders["videos_original"], - ] - ) + target_subfolders.extend([ + self.browsertime_results_folders["videos_annotated"], + self.browsertime_results_folders["videos_original"], + ]) return target_subfolders # Default folder for unexpected files @@ -757,12 +755,10 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): def _extract_android_power_vals(): power_vals = raw_result.get("android").get("power", {}) if power_vals: - bt_result["measurements"].setdefault("powerUsage", []).extend( - [ - round(vals["powerUsage"] * (1 * 10**-6), 2) - for vals in power_vals - ] - ) + bt_result["measurements"].setdefault("powerUsage", []).extend([ + round(vals["powerUsage"] * (1 * 10**-6), 2) + for vals in power_vals + ]) if support_class: bt_result["custom_data"] = True diff --git a/testing/raptor/raptor/utils.py b/testing/raptor/raptor/utils.py @@ -34,7 +34,7 @@ def strtobool(value: str): if value in false_vals: return 0 - raise ValueError(f'Expected one of: {", ".join(true_vals + false_vals)}') + raise ValueError(f"Expected one of: {', '.join(true_vals + false_vals)}") def flatten(data, parent_dir, sep="/"): diff --git a/testing/raptor/test/test_chrome_trace.py b/testing/raptor/test/test_chrome_trace.py @@ -43,9 +43,9 @@ def test_browsertime_trace_collection(mock_log_info, mock_log_critical): test_config = {"name": "tp6", "type": "pageload"} try: profile = ChromeTrace(upload_dir, raptor_config, test_config) - assert ( - len(profile.collect_profiles()) == 2 - ), "We have two profiles for a cold & warm run" + assert len(profile.collect_profiles()) == 2, ( + "We have two profiles for a cold & warm run" + ) profile.output_trace() profile.clean() arcname = os.environ["RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"] diff --git a/testing/raptor/test/test_raptor.py b/testing/raptor/test/test_raptor.py @@ -200,7 +200,7 @@ def test_post_startup_delay( debug_mode=debug_mode, post_startup_delay=post_startup_delay, conditioned_profile=conditioned_profile, - **options + **options, ) assert perftest.post_startup_delay == expected_post_startup_delay diff --git a/testing/runcppunittests.py b/testing/runcppunittests.py @@ -290,22 +290,18 @@ def extract_unittests_from_args(args, environ, manifest_path): active_tests = mp.active_tests(exists=False, disabled=False, **environ) suffix = ".exe" if mozinfo.isWin else "" if binary_path: - tests.extend( - [ - ( - os.path.join(binary_path, test["relpath"] + suffix), - int(test.get("requesttimeoutfactor", 1)), - ) - for test in active_tests - ] - ) + tests.extend([ + ( + os.path.join(binary_path, test["relpath"] + suffix), + int(test.get("requesttimeoutfactor", 1)), + ) + for test in active_tests + ]) else: - tests.extend( - [ - (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1))) - for test in active_tests - ] - ) + tests.extend([ + (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1))) + for test in active_tests + ]) # Manually confirm that all tests named in the manifest exist. errors = False diff --git a/testing/talos/INSTALL.py b/testing/talos/INSTALL.py @@ -10,6 +10,7 @@ installation script for talos. This script: - sets up talos in development mode: `python setup.py develop` - downloads pageloader and packages to talos/page_load_test/pageloader.xpi """ + import os import subprocess import sys diff --git a/testing/talos/mach_commands.py b/testing/talos/mach_commands.py @@ -68,12 +68,10 @@ class TalosRunner(MozbuildObject): def make_config(self): default_actions = ["populate-webroot"] - default_actions.extend( - [ - "create-virtualenv", - "run-tests", - ] - ) + default_actions.extend([ + "create-virtualenv", + "run-tests", + ]) self.config = { "run_local": True, "talos_json": self.talos_json, diff --git a/testing/talos/talos/cmanager_mac.py b/testing/talos/talos/cmanager_mac.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """CounterManager for Mac OSX""" + import subprocess import sys diff --git a/testing/talos/talos/cmdline.py b/testing/talos/talos/cmdline.py @@ -264,7 +264,7 @@ def create_parser(mach_interface=False): add_arg( "--tppagecycles", type=int, - help="number of pageloader cycles to run for each page in" " the manifest", + help="number of pageloader cycles to run for each page in the manifest", ) add_arg( "--no-download", diff --git a/testing/talos/talos/config.py b/testing/talos/talos/config.py @@ -251,7 +251,7 @@ def setup_pdfpaint_test(config, test_instance): raise ConfigurationError( f"Chunk {chunk_number} contains no PDFs to test. " f"For {len(pdf_files)} PDFs, the max chunk is " - f"{int((len(pdf_files)-1)/pdfs_per_chunk)+1}." + f"{int((len(pdf_files) - 1) / pdfs_per_chunk) + 1}." ) with pdfpaint_manifest_path.open("w") as f: @@ -354,9 +354,9 @@ def get_test(config, global_overrides, counters, test_instance): if not isinstance(getattr(test_instance, key, None), list): setattr(test_instance, key, []) _counters = getattr(test_instance, key) - _counters.extend( - [counter for counter in counters if counter not in _counters] - ) + _counters.extend([ + counter for counter in counters if counter not in _counters + ]) return dict(test_instance.items()) diff --git a/testing/talos/talos/ffsetup.py b/testing/talos/talos/ffsetup.py @@ -5,6 +5,7 @@ """ Set up a browser environment before running a test. """ + import json import os import shutil @@ -139,12 +140,10 @@ class FFSetup: # set test preferences preferences = self.browser_config.get("preferences", {}).copy() if self.test_config.get("preferences"): - test_prefs = dict( - [ - (i, utils.parse_pref(j)) - for i, j in self.test_config["preferences"].items() - ] - ) + test_prefs = dict([ + (i, utils.parse_pref(j)) + for i, j in self.test_config["preferences"].items() + ]) preferences.update(test_prefs) for name, value in preferences.items(): @@ -216,7 +215,7 @@ class FFSetup: def _init_gecko_profile(self): upload_dir = os.getenv("MOZ_UPLOAD_DIR") if self.test_config.get("gecko_profile") and not upload_dir: - LOG.critical("Profiling ignored because MOZ_UPLOAD_DIR was not" " set") + LOG.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set") if upload_dir and self.test_config.get("gecko_profile"): self.gecko_profile = GeckoProfile( upload_dir, self.browser_config, self.test_config diff --git a/testing/talos/talos/gecko_profile.py b/testing/talos/talos/gecko_profile.py @@ -5,6 +5,7 @@ """ module to handle Gecko profiling. """ + import json import os import tempfile @@ -100,18 +101,16 @@ class GeckoProfile: # Set environment variables which will cause profiling to # start as early as possible. These are consumed by Gecko # itself, not by Talos JS code. - env.update( - { - "MOZ_PROFILER_STARTUP": "1", - # Temporary: Don't run Base Profiler, see bug 1630448. - # TODO: Remove when fix lands in bug 1648324 or bug 1648325. - "MOZ_PROFILER_STARTUP_NO_BASE": "1", - "MOZ_PROFILER_STARTUP_INTERVAL": str(self.option("interval")), - "MOZ_PROFILER_STARTUP_ENTRIES": str(self.option("entries")), - "MOZ_PROFILER_STARTUP_FEATURES": str(self.option("features")), - "MOZ_PROFILER_STARTUP_FILTERS": str(self.option("threads")), - } - ) + env.update({ + "MOZ_PROFILER_STARTUP": "1", + # Temporary: Don't run Base Profiler, see bug 1630448. + # TODO: Remove when fix lands in bug 1648324 or bug 1648325. + "MOZ_PROFILER_STARTUP_NO_BASE": "1", + "MOZ_PROFILER_STARTUP_INTERVAL": str(self.option("interval")), + "MOZ_PROFILER_STARTUP_ENTRIES": str(self.option("entries")), + "MOZ_PROFILER_STARTUP_FEATURES": str(self.option("features")), + "MOZ_PROFILER_STARTUP_FILTERS": str(self.option("threads")), + }) def _save_gecko_profile( self, cycle, symbolicator, missing_symbols_zip, profile_path @@ -143,32 +142,30 @@ class GeckoProfile: :param cycle: the number of the cycle of the test currently run. """ - symbolicator = ProfileSymbolicator( - { - # Trace-level logging (verbose) - "enableTracing": 0, - # Fallback server if symbol is not found locally - "remoteSymbolServer": "https://symbolication.services.mozilla.com/symbolicate/v4", - # Maximum number of symbol files to keep in memory - "maxCacheEntries": 2000000, - # Frequency of checking for recent symbols to - # cache (in hours) - "prefetchInterval": 12, - # Oldest file age to prefetch (in hours) - "prefetchThreshold": 48, - # Maximum number of library versions to pre-fetch - # per library - "prefetchMaxSymbolsPerLib": 3, - # Default symbol lookup directories - "defaultApp": "FIREFOX", - "defaultOs": "WINDOWS", - # Paths to .SYM files, expressed internally as a - # mapping of app or platform names to directories - # Note: App & OS names from requests are converted - # to all-uppercase internally - "symbolPaths": self.symbol_paths, - } - ) + symbolicator = ProfileSymbolicator({ + # Trace-level logging (verbose) + "enableTracing": 0, + # Fallback server if symbol is not found locally + "remoteSymbolServer": "https://symbolication.services.mozilla.com/symbolicate/v4", + # Maximum number of symbol files to keep in memory + "maxCacheEntries": 2000000, + # Frequency of checking for recent symbols to + # cache (in hours) + "prefetchInterval": 12, + # Oldest file age to prefetch (in hours) + "prefetchThreshold": 48, + # Maximum number of library versions to pre-fetch + # per library + "prefetchMaxSymbolsPerLib": 3, + # Default symbol lookup directories + "defaultApp": "FIREFOX", + "defaultOs": "WINDOWS", + # Paths to .SYM files, expressed internally as a + # mapping of app or platform names to directories + # Note: App & OS names from requests are converted + # to all-uppercase internally + "symbolPaths": self.symbol_paths, + }) if self.browser_config["symbols_path"]: if mozfile.is_url(self.browser_config["symbols_path"]): diff --git a/testing/talos/talos/heavy.py b/testing/talos/talos/heavy.py @@ -5,6 +5,7 @@ """ Downloads Heavy profiles from TaskCluster. """ + import datetime import functools import os diff --git a/testing/talos/talos/mainthreadio.py b/testing/talos/talos/mainthreadio.py @@ -30,29 +30,25 @@ KEY_RUN_COUNT = "RunCount" LEAKED_SYMLINK_PREFIX = "::\\{" -PATH_SUBSTITUTIONS = OrderedDict( - [ - ("profile", "{profile}"), - ("firefox", "{xre}"), - ("desktop", "{desktop}"), - ("fonts", "{fonts}"), - ("appdata", " {appdata}"), - ] -) -NAME_SUBSTITUTIONS = OrderedDict( - [ - ("installtime", "{time}"), - ("prefetch", "{prefetch}"), - ("thumbnails", "{thumbnails}"), - # {appdata}\locallow\mozilla\temp-{*} - ("temp-{", "{temp}"), - ("cltbld.", "{cltbld}"), - ("windows media player", "{media_player}"), - # regex order matters - (re.compile(r"{\w{8}-\w{4}-\w{4}-\w{4}-\w{12}}"), "{uuid}"), - (re.compile(r"{uuid}\.\d+\.ver\w+\.db"), "{uuid-db}"), - ] -) +PATH_SUBSTITUTIONS = OrderedDict([ + ("profile", "{profile}"), + ("firefox", "{xre}"), + ("desktop", "{desktop}"), + ("fonts", "{fonts}"), + ("appdata", " {appdata}"), +]) +NAME_SUBSTITUTIONS = OrderedDict([ + ("installtime", "{time}"), + ("prefetch", "{prefetch}"), + ("thumbnails", "{thumbnails}"), + # {appdata}\locallow\mozilla\temp-{*} + ("temp-{", "{temp}"), + ("cltbld.", "{cltbld}"), + ("windows media player", "{media_player}"), + # regex order matters + (re.compile(r"{\w{8}-\w{4}-\w{4}-\w{4}-\w{12}}"), "{uuid}"), + (re.compile(r"{uuid}\.\d+\.ver\w+\.db"), "{uuid-db}"), +]) TUPLE_EVENT_SOURCE_INDEX = 1 TUPLE_FILENAME_INDEX = 2 diff --git a/testing/talos/talos/output.py b/testing/talos/talos/output.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """output formats for Talos""" + # NOTE: we have a circular dependency with output.py when we import results import simplejson as json from mozlog import get_proxy_logger @@ -174,14 +175,12 @@ class Output: varray = [float(v) for v in vals] subtest["value"] = filter.mean(varray) if counter_subtests: - suites.append( - { - "name": test.name(), - "extraOptions": self.results.extra_options or [], - "subtests": counter_subtests, - "shouldAlert": test.test_config.get("suite_should_alert", True), - } - ) + suites.append({ + "name": test.name(), + "extraOptions": self.results.extra_options or [], + "subtests": counter_subtests, + "shouldAlert": test.test_config.get("suite_should_alert", True), + }) return test_results def output(self, results, results_url): diff --git a/testing/talos/talos/results.py b/testing/talos/talos/results.py @@ -8,6 +8,7 @@ objects and methods for parsing and serializing Talos results see https://wiki.mozilla.org/Buildbot/Talos/DataFormat """ + import csv import json import os @@ -511,12 +512,10 @@ class BrowserLogResults: values = dict(zip(header, row)) for i, mainthread_counter in enumerate(mainthread_counters): if int(values[mainthread_counter_keys[i]]) > 0: - counter_results.setdefault(mainthread_counter, []).append( - [ - int(values[mainthread_counter_keys[i]]), - values["filename"], - ] - ) + counter_results.setdefault(mainthread_counter, []).append([ + int(values[mainthread_counter_keys[i]]), + values["filename"], + ]) if session_store_counter in counter_results.keys(): filename = "etl_output_session_restore_stats.csv" diff --git a/testing/talos/talos/run_tests.py b/testing/talos/talos/run_tests.py @@ -140,9 +140,9 @@ def run_tests(config, browser_config): # so that the browser pref will be turned on (in ffsetup) if test.get("fnbpaint", False): LOG.info("Test is using firstNonBlankPaint, browser pref will be turned on") - test["preferences"][ - "dom.performance.time_to_non_blank_paint.enabled" - ] = True + test["preferences"]["dom.performance.time_to_non_blank_paint.enabled"] = ( + True + ) test["setup"] = utils.interpolate(test["setup"]) test["cleanup"] = utils.interpolate(test["cleanup"]) @@ -163,16 +163,13 @@ def run_tests(config, browser_config): browser_config["preferences"]["fission.autostart"] = False browser_config["preferences"]["network.proxy.type"] = 2 - browser_config["preferences"]["network.proxy.autoconfig_url"] = ( - """data:text/plain, + browser_config["preferences"]["network.proxy.autoconfig_url"] = """data:text/plain, function FindProxyForURL(url, host) { if (url.startsWith('http')) { return 'PROXY %s'; } return 'DIRECT'; -}""" - % browser_config["webserver"] - ) +}""" % browser_config["webserver"] # If --code-coverage files are expected, set flag in browser config so ffsetup knows # that it needs to delete any ccov files resulting from browser initialization @@ -182,8 +179,7 @@ function FindProxyForURL(url, host) { if config.get("code_coverage", False): if browser_config["develop"]: raise TalosError( - "Aborting: talos --code-coverage flag is only " - "supported in production" + "Aborting: talos --code-coverage flag is only supported in production" ) else: browser_config["code_coverage"] = True @@ -458,15 +454,13 @@ def make_comparison_result(base_and_reference_results): sub_test_name = base_and_reference_results.results[0].results[x]["page"] # populate our new comparison result with 'base' and 'ref' replicates - comp_results.append( - { - "index": 0, - "runs": [], - "page": sub_test_name, - "base_runs": base_result_runs, - "ref_runs": ref_result_runs, - } - ) + comp_results.append({ + "index": 0, + "runs": [], + "page": sub_test_name, + "base_runs": base_result_runs, + "ref_runs": ref_result_runs, + }) # now step thru each result, compare 'base' vs 'ref', and store the difference in 'runs' _index = 0 diff --git a/testing/talos/talos/talosconfig.py b/testing/talos/talos/talosconfig.py @@ -42,15 +42,13 @@ def generateTalosConfig(command_line, browser_config, test_config, pid=None): browser_config["xperf_stackwalk"] = test_config["xperf_stackwalk"] browser_config["processID"] = pid browser_config["approot"] = os.path.dirname(browser_config["browser_path"]) - bcontroller_vars.extend( - [ - "xperf_providers", - "xperf_user_providers", - "xperf_stackwalk", - "processID", - "approot", - ] - ) + bcontroller_vars.extend([ + "xperf_providers", + "xperf_user_providers", + "xperf_stackwalk", + "processID", + "approot", + ]) content = writeConfigFile(browser_config, bcontroller_vars) diff --git a/testing/talos/talos/ttest.py b/testing/talos/talos/ttest.py @@ -3,14 +3,15 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """A generic means of running an URL based browser test - follows the following steps - - creates a profile - - tests the profile - - gets metrics for the current test environment - - loads the url - - collects info on any counters while test runs - - waits for a 'dump' from the browser +follows the following steps + - creates a profile + - tests the profile + - gets metrics for the current test environment + - loads the url + - collects info on any counters while test runs + - waits for a 'dump' from the browser """ + import json import os import platform @@ -255,9 +256,9 @@ class TTest: ) # include global (cross-cycle) counters - test_results.all_counter_results.extend( - [{key: value} for key, value in global_counters.items()] - ) + test_results.all_counter_results.extend([ + {key: value} for key, value in global_counters.items() + ]) for c in test_results.all_counter_results: for key, value in c.items(): LOG.debug("COUNTER %r: %s" % (key, value)) diff --git a/testing/talos/talos/utils.py b/testing/talos/talos/utils.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Utility functions for Talos""" + import os import platform import re @@ -90,13 +91,14 @@ def tokenize(string, start, end): return [], -1 assert len(_start), "Could not find start token: '%s'" % start assert len(_end), "Could not find end token: '%s'" % end - assert len(_start) == len( - _end - ), "Unmatched number of tokens found: '%s' (%d) vs '%s' (%d)" % ( - start, - len(_start), - end, - len(_end), + assert len(_start) == len(_end), ( + "Unmatched number of tokens found: '%s' (%d) vs '%s' (%d)" + % ( + start, + len(_start), + end, + len(_end), + ) ) for i in range(len(_start)): assert _end[i] > _start[i], "End token '%s' occurs before start token '%s'" % ( diff --git a/testing/talos/talos/xtalos/etlparser.py b/testing/talos/talos/xtalos/etlparser.py @@ -393,7 +393,7 @@ def etlparser( allowlist = loadAllowlist(allowlist_file) - header = "filename, tid, stage, readcount, readbytes, writecount," " writebytes" + header = "filename, tid, stage, readcount, readbytes, writecount, writebytes" outFile.write(header + "\n") # Filter out stages, threads, and allowlisted files that we're not diff --git a/testing/talos/talos/xtalos/parse_xperf.py b/testing/talos/talos/xtalos/parse_xperf.py @@ -116,7 +116,7 @@ def main(args=sys.argv[1:]): stop_from_config( config_file=args.configFile, debug=args.debug_level >= xtalos.DEBUG_INFO, - **args.__dict__ + **args.__dict__, ) except xtalos.XTalosError as e: parser.error(str(e)) diff --git a/testing/talos/talos/xtalos/start_xperf.py b/testing/talos/talos/xtalos/start_xperf.py @@ -94,7 +94,7 @@ def main(args=sys.argv[1:]): start_from_config( config_file=args.configFile, debug=args.debug_level >= xtalos.DEBUG_INFO, - **args.__dict__ + **args.__dict__, ) except xtalos.XTalosError as e: parser.error(str(e)) diff --git a/testing/talos/talos/xtalos/xtalos.py b/testing/talos/talos/xtalos/xtalos.py @@ -43,7 +43,7 @@ class XtalosOptions(argparse.ArgumentParser): "-x", "--xperf", dest="xperf_path", - help="location of xperf tool, defaults to" " 'xperf.exe'", + help="location of xperf tool, defaults to 'xperf.exe'", ) defaults["xperf_path"] = "xperf.exe" @@ -51,7 +51,7 @@ class XtalosOptions(argparse.ArgumentParser): "-e", "--etl_filename", dest="etl_filename", - help="Name of the .etl file to work with." " Defaults to 'output.etl'", + help="Name of the .etl file to work with. Defaults to 'output.etl'", ) defaults["etl_filename"] = "test.etl" @@ -69,7 +69,7 @@ class XtalosOptions(argparse.ArgumentParser): "-o", "--output-file", dest="outputFile", - help="Filename to write all output to, default" " is stdout", + help="Filename to write all output to, default is stdout", ) defaults["outputFile"] = "etl_output.csv" @@ -86,7 +86,7 @@ class XtalosOptions(argparse.ArgumentParser): "--user-providers", dest="xperf_user_providers", action="append", - help="user mode xperf providers to collect data" " from", + help="user mode xperf providers to collect data from", ) defaults["xperf_user_providers"] = [] @@ -103,8 +103,7 @@ class XtalosOptions(argparse.ArgumentParser): "-c", "--config-file", dest="configFile", - help="Name of the json config file with test run" - " and browser information", + help="Name of the json config file with test run and browser information", ) defaults["configFile"] = None @@ -121,7 +120,7 @@ class XtalosOptions(argparse.ArgumentParser): "--all-stages", dest="all_stages", action="store_true", - help="Include all stages in file I/O output," "not just startup", + help="Include all stages in file I/O output,not just startup", ) defaults["all_stages"] = False @@ -130,7 +129,7 @@ class XtalosOptions(argparse.ArgumentParser): "--all-threads", dest="all_threads", action="store_true", - help="Include all threads in file I/O output," " not just main", + help="Include all threads in file I/O output, not just main", ) defaults["all_threads"] = False @@ -147,7 +146,7 @@ class XtalosOptions(argparse.ArgumentParser): self.add_argument( "--error-filename", dest="error_filename", - help="Filename to store the failures detected" " while runnning the test", + help="Filename to store the failures detected while runnning the test", ) defaults["error_filename"] = None diff --git a/testing/testinfo.py b/testing/testinfo.py @@ -399,17 +399,13 @@ class TestInfoReport(TestInfo): def get_testinfoall_index_url(self): import taskcluster - index = taskcluster.Index( - { - "rootUrl": "https://firefox-ci-tc.services.mozilla.com", - } - ) + index = taskcluster.Index({ + "rootUrl": "https://firefox-ci-tc.services.mozilla.com", + }) route = "gecko.v2.mozilla-central.latest.source.test-info-all" - queue = taskcluster.Queue( - { - "rootUrl": "https://firefox-ci-tc.services.mozilla.com", - } - ) + queue = taskcluster.Queue({ + "rootUrl": "https://firefox-ci-tc.services.mozilla.com", + }) task_id = index.findTask(route)["taskId"] artifacts = queue.listLatestArtifacts(task_id)["artifacts"] diff --git a/testing/tps/create_venv.py b/testing/tps/create_venv.py @@ -133,9 +133,11 @@ def main(): activate(target) # Install TPS in environment - subprocess.check_call( - [os.path.join(target, python_env), os.path.join(here, "setup.py"), "install"] - ) + subprocess.check_call([ + os.path.join(target, python_env), + os.path.join(here, "setup.py"), + "install", + ]) # Get the path to tests and extensions directory by checking check where # the tests and extensions directories are located diff --git a/testing/tps/tps/testrunner.py b/testing/tps/tps/testrunner.py @@ -482,14 +482,12 @@ class TPSTestRunner: if not self.addonversion: self.addonversion = result["addonversion"] - self.results.append( - { - "state": result["state"], - "name": result["name"], - "message": result["message"], - "logdata": result["logdata"], - } - ) + self.results.append({ + "state": result["state"], + "name": result["name"], + "message": result["message"], + "logdata": result["logdata"], + }) if result["state"] == "TEST-PASS": self.numpassed += 1 else: diff --git a/testing/update/test_apply_update.py b/testing/update/test_apply_update.py @@ -55,9 +55,9 @@ class TestApplyUpdate(MarionetteTestCase): if environ.get("BALROG_STAGING"): print("staging") patch_url = root[0][0].get("URL") - assert ( - f"{target_ver}" in patch_url - ), f"{target_ver} not in patch url: {patch_url}" + assert f"{target_ver}" in patch_url, ( + f"{target_ver} not in patch url: {patch_url}" + ) patch_response = requests.get(patch_url) patch_response.raise_for_status() return True diff --git a/testing/update/test_background_update.py b/testing/update/test_background_update.py @@ -78,9 +78,9 @@ class TestBackgroundUpdate(MarionetteTestCase): if environ.get("BALROG_STAGING"): root = get_update_server_response(update_url, 1) patch_url = root[0][0].get("URL") - assert ( - f"/{target_vers[-1]}-candidates" in patch_url - ), f'"/{target_vers[-1]}-candidates not in patch url: {patch_url}' + assert f"/{target_vers[-1]}-candidates" in patch_url, ( + f'"/{target_vers[-1]}-candidates not in patch url: {patch_url}' + ) patch_response = requests.get(patch_url) patch_response.raise_for_status() return True diff --git a/testing/web-platform/interop.py b/testing/web-platform/interop.py @@ -252,11 +252,11 @@ def get_expected_failures(path: str) -> Mapping[str, set[Optional[str]]]: continue if len(entry) > 2: raise ValueError( - f"{path}:{i+1} expected at most two columns, got {len(entry)}" + f"{path}:{i + 1} expected at most two columns, got {len(entry)}" ) if entry[0][0] != "/": raise ValueError( - f'{path}:{i+1} "{entry[0]}" is not a valid test id (must start with "/")' + f'{path}:{i + 1} "{entry[0]}" is not a valid test id (must start with "/")' ) test_id = entry[0] if test_id not in expected_failures: @@ -265,7 +265,7 @@ def get_expected_failures(path: str) -> Mapping[str, set[Optional[str]]]: subtest_id = entry[1] if subtest_id == "": print( - f"Warning: {path}:{i+1} got empty string subtest id, remove the trailing comma to make this apply to the full test" + f"Warning: {path}:{i + 1} got empty string subtest id, remove the trailing comma to make this apply to the full test" ) expected_failures[test_id].add(subtest_id) else: diff --git a/testing/web-platform/mach_commands_base.py b/testing/web-platform/mach_commands_base.py @@ -103,5 +103,5 @@ class WebPlatformTestsRunner: logger=logger, src_root=self.setup.topsrcdir, obj_root=self.setup.topobjdir, - **kwargs + **kwargs, ) diff --git a/testing/web-platform/manifestdownload.py b/testing/web-platform/manifestdownload.py @@ -127,7 +127,7 @@ def taskcluster_url(logger, commits): return tc_index_url + artifact_path logger.info( - "Can't find a commit-specific manifest so just using the most " "recent one" + "Can't find a commit-specific manifest so just using the most recent one" ) return default diff --git a/testing/web-platform/metasummary.py b/testing/web-platform/metasummary.py @@ -171,7 +171,7 @@ def create_parser(): parser = argparse.ArgumentParser() parser.add_argument("--out-dir", help="Directory to store output files") parser.add_argument( - "--meta-dir", help="Directory containing wpt-metadata " "checkout to update." + "--meta-dir", help="Directory containing wpt-metadata checkout to update." ) return parser diff --git a/testing/web-platform/mozilla/tests/geolocation-API/dummy.py b/testing/web-platform/mozilla/tests/geolocation-API/dummy.py @@ -4,13 +4,11 @@ import json def main(request, response): response.headers.set(b"Content-Type", b"application/json") response.headers.set(b"Cache-Control", b"no-cache") - response.content = json.dumps( - { - "status": "OK", - "location": { - "lat": 37.41857, - "lng": -122.08769, - }, - "accuracy": 42, - } - ).encode("utf-8") + response.content = json.dumps({ + "status": "OK", + "location": { + "lat": 37.41857, + "lng": -122.08769, + }, + "accuracy": 42, + }).encode("utf-8") diff --git a/testing/web-platform/mozilla/tests/webdriver/bidi/browsing_context/navigate/https_upgrade.py b/testing/web-platform/mozilla/tests/webdriver/bidi/browsing_context/navigate/https_upgrade.py @@ -21,18 +21,16 @@ def http_with_https_port_url(server_config): return _http_with_https_port_url -@pytest.mark.capabilities( - { - "moz:firefoxOptions": { - "prefs": { - # Allow HTTPS upgrades for localhost and custom ports - "dom.security.https_first_for_custom_ports": True, - "dom.security.https_first_for_local_addresses": True, - "dom.security.https_first_for_unknown_suffixes": True, - }, +@pytest.mark.capabilities({ + "moz:firefoxOptions": { + "prefs": { + # Allow HTTPS upgrades for localhost and custom ports + "dom.security.https_first_for_custom_ports": True, + "dom.security.https_first_for_local_addresses": True, + "dom.security.https_first_for_unknown_suffixes": True, }, - } -) + }, +}) async def test_no_https_first_upgrade(bidi_session, new_tab, http_with_https_port_url): page = http_with_https_port_url("/webdriver/tests/support/html/default.html") diff --git a/testing/web-platform/mozilla/tests/webdriver/bidi/emulation/set_locale_override/contexts.py b/testing/web-platform/mozilla/tests/webdriver/bidi/emulation/set_locale_override/contexts.py @@ -5,15 +5,13 @@ pytestmark = pytest.mark.asyncio pytest_plugins = "tests.bidi.emulation.conftest" -@pytest.mark.capabilities( - { - "moz:firefoxOptions": { - "prefs": { - "dom.ipc.processCount": 1, - }, +@pytest.mark.capabilities({ + "moz:firefoxOptions": { + "prefs": { + "dom.ipc.processCount": 1, }, - } -) + }, +}) async def test_locale_override_isolated_in_browsing_context( bidi_session, another_locale, assert_locale_against_value, some_locale ): diff --git a/testing/web-platform/mozilla/tests/webdriver/bidi/emulation/set_timezone_override/contexts.py b/testing/web-platform/mozilla/tests/webdriver/bidi/emulation/set_timezone_override/contexts.py @@ -5,15 +5,13 @@ pytestmark = pytest.mark.asyncio pytest_plugins = "tests.bidi.emulation.conftest" -@pytest.mark.capabilities( - { - "moz:firefoxOptions": { - "prefs": { - "dom.ipc.processCount": 1, - }, +@pytest.mark.capabilities({ + "moz:firefoxOptions": { + "prefs": { + "dom.ipc.processCount": 1, }, - } -) + }, +}) async def test_timezone_override_isolated_in_browsing_context( bidi_session, get_current_timezone, some_timezone, another_timezone ): diff --git a/testing/web-platform/mozilla/tests/webdriver/bidi/network/add_data_collector/max_total_data_size.py b/testing/web-platform/mozilla/tests/webdriver/bidi/network/add_data_collector/max_total_data_size.py @@ -55,15 +55,13 @@ async def send_request(wait_for_event, inline, fetch, wait_for_future_safe): return _send_request -@pytest.mark.capabilities( - { - "moz:firefoxOptions": { - "prefs": { - "remote.network.maxTotalDataSize": MAX_TOTAL_SIZE, - }, +@pytest.mark.capabilities({ + "moz:firefoxOptions": { + "prefs": { + "remote.network.maxTotalDataSize": MAX_TOTAL_SIZE, }, - } -) + }, +}) @pytest.mark.parametrize( "mode", [ diff --git a/testing/web-platform/mozilla/tests/webdriver/classic/navigate_to/https_upgrade.py b/testing/web-platform/mozilla/tests/webdriver/classic/navigate_to/https_upgrade.py @@ -25,19 +25,17 @@ def http_with_https_port_url(server_config): return _http_with_https_port_url -@pytest.mark.capabilities( - { - "pageLoadStrategy": "eager", - "moz:firefoxOptions": { - "prefs": { - # Allow HTTPS upgrades for localhost and custom ports - "dom.security.https_first_for_custom_ports": True, - "dom.security.https_first_for_local_addresses": True, - "dom.security.https_first_for_unknown_suffixes": True, - }, +@pytest.mark.capabilities({ + "pageLoadStrategy": "eager", + "moz:firefoxOptions": { + "prefs": { + # Allow HTTPS upgrades for localhost and custom ports + "dom.security.https_first_for_custom_ports": True, + "dom.security.https_first_for_local_addresses": True, + "dom.security.https_first_for_unknown_suffixes": True, }, - } -) + }, +}) def test_no_https_first_upgrade(session, http_with_https_port_url): page = http_with_https_port_url("/webdriver/tests/support/html/default.html") diff --git a/testing/web-platform/mozilla/tests/webdriver/harness/detect_crash.py b/testing/web-platform/mozilla/tests/webdriver/harness/detect_crash.py @@ -55,12 +55,12 @@ def run_crash_test(configuration, geckodriver, crash_callback): # Check that for both Marionette and Remote Agent the annotations are present extra_data = read_extra_file(file_map[".extra"]) - assert ( - extra_data.get("Marionette") == "1" - ), "Marionette entry is missing or invalid" - assert ( - extra_data.get("RemoteAgent") == "1" - ), "RemoteAgent entry is missing or invalid" + assert extra_data.get("Marionette") == "1", ( + "Marionette entry is missing or invalid" + ) + assert extra_data.get("RemoteAgent") == "1", ( + "RemoteAgent entry is missing or invalid" + ) # Remove original minidump files from the profile directory remove_files(profile_minidump_path, file_map.values()) @@ -105,8 +105,8 @@ def verify_minidump_files(directory): } missing_extensions = required_extensions - file_map.keys() - assert ( - not missing_extensions - ), f"Missing required files with extensions: {missing_extensions}" + assert not missing_extensions, ( + f"Missing required files with extensions: {missing_extensions}" + ) return file_map diff --git a/testing/web-platform/mozilla/tests/websockets/handlers/mozilla_push_dummy_wsh.py b/testing/web-platform/mozilla/tests/websockets/handlers/mozilla_push_dummy_wsh.py @@ -68,16 +68,14 @@ class DummyEndpointHandler(BaseHTTPRequestHandler): } msgutil.send_message( self.server.websocket_request, - json.dumps( - { - "messageType": "notification", - "channelID": query["channelID"][0], - "data": base64.urlsafe_b64encode(post_body).decode(), - "headers": headers if len(post_body) > 0 else None, - # without a version string the push client thinks it's a duplicate - "version": str(uuid4()), - } - ), + json.dumps({ + "messageType": "notification", + "channelID": query["channelID"][0], + "data": base64.urlsafe_b64encode(post_body).decode(), + "headers": headers if len(post_body) > 0 else None, + # without a version string the push client thinks it's a duplicate + "version": str(uuid4()), + }), ) @@ -123,37 +121,31 @@ def web_socket_transfer_data(request): if messageType == "hello": msgutil.send_message( request, - json.dumps( - { - "messageType": "hello", - "uaid": UAID, - "status": 200, - "use_webpush": True, - } - ), + json.dumps({ + "messageType": "hello", + "uaid": UAID, + "status": 200, + "use_webpush": True, + }), ) elif messageType == "register": channelID = message["channelID"] msgutil.send_message( request, - json.dumps( - { - "messageType": "register", - "uaid": UAID, - "channelID": channelID, - "status": 200, - "pushEndpoint": f"https://web-platform.test:{port}/push_endpoint?channelID={channelID}", - } - ), + json.dumps({ + "messageType": "register", + "uaid": UAID, + "channelID": channelID, + "status": 200, + "pushEndpoint": f"https://web-platform.test:{port}/push_endpoint?channelID={channelID}", + }), ) elif messageType == "unregister": msgutil.send_message( request, - json.dumps( - { - "messageType": "unregister", - "channelID": message["channelID"], - "status": 200, - } - ), + json.dumps({ + "messageType": "unregister", + "channelID": message["channelID"], + "status": 200, + }), ) diff --git a/testing/web-platform/update/upstream.py b/testing/web-platform/update/upstream.py @@ -54,19 +54,18 @@ def rewrite_patch(patch, strip_dir): def rewrite_message(patch): if patch.message.bug is not None: - return "\n".join( - [ - patch.message.summary, - patch.message.body, - "", - "Upstreamed from https://bugzilla.mozilla.org/show_bug.cgi?id=%s [ci skip]" - % patch.message.bug, # noqa E501 - ] - ) + return "\n".join([ + patch.message.summary, + patch.message.body, + "", + "Upstreamed from https://bugzilla.mozilla.org/show_bug.cgi?id=%s [ci skip]" + % patch.message.bug, # noqa E501 + ]) - return "\n".join( - [patch.message.full_summary, "%s\n[ci skip]\n" % patch.message.body] - ) + return "\n".join([ + patch.message.full_summary, + "%s\n[ci skip]\n" % patch.message.body, + ]) class SyncToUpstream(Step): @@ -93,9 +92,13 @@ class SyncToUpstream(Step): state.sync_tree = GitTree(root=state.sync["path"]) kwargs = state.kwargs - with state.push( - ["local_tree", "sync_tree", "tests_path", "metadata_path", "sync"] - ): + with state.push([ + "local_tree", + "sync_tree", + "tests_path", + "metadata_path", + "sync", + ]): state.token = kwargs["token"] runner = SyncToUpstreamRunner(self.logger, state) runner.run() diff --git a/testing/webcompat/client.py b/testing/webcompat/client.py @@ -1523,9 +1523,9 @@ class Client: ) time.sleep(0.5) without_scrollbar = trending_list.screenshot() - assert ( - with_scrollbar == without_scrollbar - ), "scrollbar does not cover any text" + assert with_scrollbar == without_scrollbar, ( + "scrollbar does not cover any text" + ) def test_for_fastclick(self, element): # FastClick cancels touchend, breaking default actions on Fenix. diff --git a/testing/webcompat/fixtures.py b/testing/webcompat/fixtures.py @@ -219,11 +219,9 @@ async def test_failed_check(request): and request.node.rep_call.failed ): session = request.node.funcargs["session"] - file_name = f'{request.node.nodeid}_failure_{datetime.today().strftime("%Y-%m-%d_%H:%M")}.png'.replace( + file_name = f"{request.node.nodeid}_failure_{datetime.today().strftime('%Y-%m-%d_%H:%M')}.png".replace( "/", "_" - ).replace( - "::", "__" - ) + ).replace("::", "__") dest_dir = request.config.getoption("failure_screenshots_dir") try: await take_screenshot(session, file_name, dest_dir=dest_dir) @@ -328,13 +326,11 @@ def install_addon(session, addon_file_path): @pytest.fixture(scope="function") async def session(driver, request, test_config): caps = driver.capabilities(request, test_config) - caps.update( - { - "acceptInsecureCerts": True, - "webSocketUrl": True, - "unhandledPromptBehavior": "dismiss", - } - ) + caps.update({ + "acceptInsecureCerts": True, + "webSocketUrl": True, + "unhandledPromptBehavior": "dismiss", + }) caps = {"alwaysMatch": caps} print(caps) diff --git a/testing/webcompat/interventions/tests/test_1902464_workhub_transcribeme_com.py b/testing/webcompat/interventions/tests/test_1902464_workhub_transcribeme_com.py @@ -25,13 +25,11 @@ async def does_unsupported_popup_appear(client, credentials): sign_in.click() for _ in range(10): - captcha, unsupported, expired = client.await_first_element_of( - [ - client.css(CAPTCHA_CSS), - client.css(UNSUPPORTED_CSS), - client.css(EXPIRED_CSS), - ] - ) + captcha, unsupported, expired = client.await_first_element_of([ + client.css(CAPTCHA_CSS), + client.css(UNSUPPORTED_CSS), + client.css(EXPIRED_CSS), + ]) if captcha: print("\a") # beep to let the user know to do the captcha client.await_element_hidden(client.css(CAPTCHA_CSS), timeout=90) diff --git a/testing/xpcshell/remotexpcshelltests.py b/testing/xpcshell/remotexpcshelltests.py @@ -512,15 +512,13 @@ class XPCShellRemote(xpcshell.XPCShellTests): f.write("#!/system/bin/sh\n") for envkey, envval in self.env.items(): f.write("export %s=%s\n" % (envkey, envval)) - f.writelines( - [ - "cd $1\n", - "echo xpcw: cd $1\n", - "shift\n", - 'echo xpcw: xpcshell "$@"\n', - '%s/xpcshell "$@"\n' % self.remoteBinDir, - ] - ) + f.writelines([ + "cd $1\n", + "echo xpcw: cd $1\n", + "shift\n", + 'echo xpcw: xpcshell "$@"\n', + '%s/xpcshell "$@"\n' % self.remoteBinDir, + ]) remoteWrapper = posixpath.join(self.remoteBinDir, "xpcw") self.device.push(localWrapper, remoteWrapper) self.device.chmod(remoteWrapper) @@ -530,8 +528,7 @@ class XPCShellRemote(xpcshell.XPCShellTests): test.selectedProcess = RemoteProcessMonitor.pickUnusedProcess() if test.selectedProcess == -1: self.log.error( - "TEST-UNEXPECTED-FAIL | remotexpcshelltests.py | " - "no more free processes" + "TEST-UNEXPECTED-FAIL | remotexpcshelltests.py | no more free processes" ) test.start() diff --git a/testing/xpcshell/runxpcshelltests.py b/testing/xpcshell/runxpcshelltests.py @@ -70,13 +70,11 @@ TBPL_RETRY = 4 # defined in mozharness # are needed for custom CPU/memory configurations NUM_THREADS = int(cpu_count() * 2.5) -EXPECTED_LOG_ACTIONS = set( - [ - "crash_reporter_init", - "test_status", - "log", - ] -) +EXPECTED_LOG_ACTIONS = set([ + "crash_reporter_init", + "test_status", + "log", +]) # -------------------------------------------------------------- # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900 @@ -740,9 +738,10 @@ class XPCShellTestThread(Thread): if "message" in line: line["message"] = self.fix_text_output(line["message"]) if "xpcshell_process" in line: - line["thread"] = " ".join( - [current_thread().name, line["xpcshell_process"]] - ) + line["thread"] = " ".join([ + current_thread().name, + line["xpcshell_process"], + ]) else: line["thread"] = current_thread().name self.log.log_raw(line) @@ -913,15 +912,17 @@ class XPCShellTestThread(Thread): # 3) Arguments for the test file self.command.extend(self.buildCmdTestFile(path)) self.command.extend(["-e", 'const _TEST_NAME = "%s";' % name]) - self.command.extend( - ["-e", 'const _EXPECTED = "%s";' % self.test_object["expected"]] - ) + self.command.extend([ + "-e", + 'const _EXPECTED = "%s";' % self.test_object["expected"], + ]) # 4) Arguments for code coverage if self.jscovdir: - self.command.extend( - ["-e", 'const _JSCOV_DIR = "%s";' % self.jscovdir.replace("\\", "/")] - ) + self.command.extend([ + "-e", + 'const _JSCOV_DIR = "%s";' % self.jscovdir.replace("\\", "/"), + ]) # 5) Runtime arguments if "debug" in self.test_object: @@ -1455,9 +1456,10 @@ class XPCShellTests: elif "LD_LIBRARY_PATH" not in self.env or self.env["LD_LIBRARY_PATH"] is None: self.env["LD_LIBRARY_PATH"] = self.xrePath else: - self.env["LD_LIBRARY_PATH"] = ":".join( - [self.xrePath, self.env["LD_LIBRARY_PATH"]] - ) + self.env["LD_LIBRARY_PATH"] = ":".join([ + self.xrePath, + self.env["LD_LIBRARY_PATH"], + ]) usingASan = "asan" in self.mozInfo and self.mozInfo["asan"] usingTSan = "tsan" in self.mozInfo and self.mozInfo["tsan"] diff --git a/testing/xpcshell/selftest.py b/testing/xpcshell/selftest.py @@ -1494,9 +1494,9 @@ add_test({ Check that the manifest entry overrides the explicit default. """ self.writeFile("test_notHeadlessWhenFalseInManifest.js", HEADLESS_FALSE) - self.writeManifest( - [("test_notHeadlessWhenFalseInManifest.js", "headless = false")] - ) + self.writeManifest([ + ("test_notHeadlessWhenFalseInManifest.js", "headless = false") + ]) self.assertTestResult(True, headless=True) diff --git a/testing/xpcshell/xpcshellcommandline.py b/testing/xpcshell/xpcshellcommandline.py @@ -206,7 +206,7 @@ def add_common_arguments(parser): "--jsdebugger", dest="jsDebugger", action="store_true", - help="Waits for a devtools JS debugger to connect before " "starting the test.", + help="Waits for a devtools JS debugger to connect before starting the test.", ) parser.add_argument( "--jsdebugger-port", @@ -450,7 +450,7 @@ def add_remote_arguments(parser): action="store", type=str, dest="remoteTestRoot", - help="Remote directory to use as test root " "(eg. /data/local/tmp/test_root).", + help="Remote directory to use as test root (eg. /data/local/tmp/test_root).", ) diff --git a/toolkit/components/antitracking/bouncetrackingprotection/test/marionette/test_bouncetracking_storage_persistence.py b/toolkit/components/antitracking/bouncetrackingprotection/test/marionette/test_bouncetracking_storage_persistence.py @@ -12,12 +12,10 @@ from marionette_harness import MarionetteTestCase class BounceTrackingStoragePersistenceTestCase(MarionetteTestCase): def setUp(self): super().setUp() - self.marionette.enforce_gecko_prefs( - { - "privacy.bounceTrackingProtection.enabled": True, - "privacy.bounceTrackingProtection.enableTestMode": True, - } - ) + self.marionette.enforce_gecko_prefs({ + "privacy.bounceTrackingProtection.enabled": True, + "privacy.bounceTrackingProtection.enableTestMode": True, + }) self.marionette.set_context("chrome") self.populate_state() diff --git a/toolkit/components/antitracking/test/marionette/test_trackingprotection_allow_list_prefs_persistence.py b/toolkit/components/antitracking/test/marionette/test_trackingprotection_allow_list_prefs_persistence.py @@ -13,14 +13,12 @@ class TrackingProtectionAllowListPreferenceTestCase(MarionetteTestCase): super().setUp() # Simulate the state after ETP Strict user upgrades, and UrlClassifierExceptionsListService # has ran to ensure the baseline and convenience preferences are set to False. - self.marionette.enforce_gecko_prefs( - { - "browser.contentblocking.category": "strict", - "privacy.trackingprotection.allow_list.baseline.enabled": False, - "privacy.trackingprotection.allow_list.convenience.enabled": False, - "privacy.trackingprotection.allow_list.hasMigratedCategoryPrefs": True, - } - ) + self.marionette.enforce_gecko_prefs({ + "browser.contentblocking.category": "strict", + "privacy.trackingprotection.allow_list.baseline.enabled": False, + "privacy.trackingprotection.allow_list.convenience.enabled": False, + "privacy.trackingprotection.allow_list.hasMigratedCategoryPrefs": True, + }) self.marionette.set_context("chrome") diff --git a/toolkit/components/cleardata/tests/marionette/test_moved_origin_directory_cleanup.py b/toolkit/components/cleardata/tests/marionette/test_moved_origin_directory_cleanup.py @@ -12,14 +12,12 @@ from marionette_harness import MarionetteTestCase class MovedOriginDirectoryCleanupTestCase(MarionetteTestCase): def setUp(self): super().setUp() - self.marionette.enforce_gecko_prefs( - { - "privacy.sanitize.sanitizeOnShutdown": True, - "privacy.clearOnShutdown.offlineApps": True, - "dom.quotaManager.backgroundTask.enabled": False, - "browser.sanitizer.loglevel": "All", - } - ) + self.marionette.enforce_gecko_prefs({ + "privacy.sanitize.sanitizeOnShutdown": True, + "privacy.clearOnShutdown.offlineApps": True, + "dom.quotaManager.backgroundTask.enabled": False, + "browser.sanitizer.loglevel": "All", + }) self.moved_origin_directory = ( Path(self.marionette.profile_path) / "storage" / "to-be-removed" / "foo" ) diff --git a/toolkit/components/extensions/webidl-api/GenerateWebIDLBindings.py b/toolkit/components/extensions/webidl-api/GenerateWebIDLBindings.py @@ -480,12 +480,10 @@ class WebIDLHelpers: if group and group not in api_event.schema_groups: return "" attrs = ["Replaceable", "SameObject"] - return "\n".join( - [ - " [%s]" % ", ".join(attrs), - " readonly attribute ExtensionEventManager %s;" % api_event.name, - ] - ) + return "\n".join([ + " [%s]" % ", ".join(attrs), + " readonly attribute ExtensionEventManager %s;" % api_event.name, + ]) if schema_group is not None: return generate_webidl(schema_group) @@ -510,12 +508,10 @@ class WebIDLHelpers: lines = [] for fn_params in api_fun.iter_multiple_webidl_signatures_params(group): params = ", ".join(cls.webidl_method_params(api_fun, group, fn_params)) - lines.extend( - [ - " [%s]" % ", ".join(attrs), - " %s %s(%s);" % (retval_type, api_fun.name, params), - ] - ) + lines.extend([ + " [%s]" % ", ".join(attrs), + " %s %s(%s);" % (retval_type, api_fun.name, params), + ]) return "\n".join(lines) if schema_group is not None: @@ -1318,16 +1314,18 @@ class APINamespace: def get_boilerplate_cpp_header(self): template = self.root.jinja_env.get_template("ExtensionAPI.h.in") webidl_props = WebIDLHelpers.to_template_props(self) - return template.render( - {"webidl_name": webidl_props["webidl_name"], "api_namespace": self.name} - ) + return template.render({ + "webidl_name": webidl_props["webidl_name"], + "api_namespace": self.name, + }) def get_boilerplate_cpp(self): template = self.root.jinja_env.get_template("ExtensionAPI.cpp.in") webidl_props = WebIDLHelpers.to_template_props(self) - return template.render( - {"webidl_name": webidl_props["webidl_name"], "api_namespace": self.name} - ) + return template.render({ + "webidl_name": webidl_props["webidl_name"], + "api_namespace": self.name, + }) def dump(self, schema_group=None): """ diff --git a/toolkit/components/extensions/webidl-api/test/test_json_schema_parsing.py b/toolkit/components/extensions/webidl-api/test/test_json_schema_parsing.py @@ -22,10 +22,9 @@ def test_parse_simple_single_api_namespace(write_jsonschema_fixtures): - parse a simple namespace that includes one permission, type, function and event """ - schema_dir = write_jsonschema_fixtures( - { - "test_api.json": dedent( - """ + schema_dir = write_jsonschema_fixtures({ + "test_api.json": dedent( + """ // Single line comments added before the JSON data are tolerated // and ignored. [ @@ -62,9 +61,8 @@ def test_parse_simple_single_api_namespace(write_jsonschema_fixtures): } ] """ - ), - } - ) + ), + }) schemas = Schemas() schemas.load_schemas(schema_dir, "toolkit") @@ -107,23 +105,19 @@ def test_parse_error_on_types_without_id_or_extend( """ Test parsing types without id or $extend raise an error while parsing. """ - schema_dir = write_jsonschema_fixtures( - { - "test_broken_types.json": json.dumps( - [ + schema_dir = write_jsonschema_fixtures({ + "test_broken_types.json": json.dumps([ + { + **base_schema(), + "namespace": "testBrokenTypeAPI", + "types": [ { - **base_schema(), - "namespace": "testBrokenTypeAPI", - "types": [ - { - # type with no "id2 or "$ref" properties - } - ], + # type with no "id2 or "$ref" properties } - ] - ) - } - ) + ], + } + ]) + }) schemas = Schemas() schemas.load_schemas(schema_dir, "toolkit") @@ -139,31 +133,27 @@ def test_parse_ignores_unsupported_types(base_schema, write_jsonschema_fixtures) """ Test parsing types without id or $extend raise an error while parsing. """ - schema_dir = write_jsonschema_fixtures( - { - "test_broken_types.json": json.dumps( - [ + schema_dir = write_jsonschema_fixtures({ + "test_broken_types.json": json.dumps([ + { + **base_schema(), + "namespace": "testUnsupportedTypesAPI", + "types": [ { - **base_schema(), - "namespace": "testUnsupportedTypesAPI", - "types": [ - { - "id": "AnUnsupportedType", - "type": "string", - "unsupported": True, - }, - { - # missing id or $ref shouldn't matter - # no parsing error expected. - "unsupported": True, - }, - {"id": "ASupportedType", "type": "string"}, - ], - } - ] - ) - } - ) + "id": "AnUnsupportedType", + "type": "string", + "unsupported": True, + }, + { + # missing id or $ref shouldn't matter + # no parsing error expected. + "unsupported": True, + }, + {"id": "ASupportedType", "type": "string"}, + ], + } + ]) + }) schemas = Schemas() schemas.load_schemas(schema_dir, "toolkit") diff --git a/toolkit/components/extensions/webidl-api/test/test_webidl_from_json_schema.py b/toolkit/components/extensions/webidl-api/test/test_webidl_from_json_schema.py @@ -33,10 +33,9 @@ def test_ambiguous_stub_mappings(write_jsonschema_fixtures): - mapped to "AsyncAmbiguous" stub per WEBEXT_STUBS_MAPPING python script config """ - schema_dir = write_jsonschema_fixtures( - { - "test_api.json": dedent( - """ + schema_dir = write_jsonschema_fixtures({ + "test_api.json": dedent( + """ [ { "namespace": "testAPINamespace", @@ -65,9 +64,8 @@ def test_ambiguous_stub_mappings(write_jsonschema_fixtures): } ] """ - ) - } - ) + ) + }) assert "testAPINamespace.configuredAsAmbiguousMethod" not in WEBEXT_STUBS_MAPPING # NOTE: mocked config reverted in the teardown_method pytest hook. @@ -86,23 +84,19 @@ def test_ambiguous_stub_mappings(write_jsonschema_fixtures): assert isinstance(fnAmbiguousBySchema, APIFunction) generated_webidl = WebIDLHelpers.to_webidl_definition(fnAmbiguousBySchema, None) - expected_webidl = "\n".join( - [ - ' [Throws, WebExtensionStub="AsyncAmbiguous"]', - " any jsonSchemaAmbiguousMethod(any... args);", - ] - ) + expected_webidl = "\n".join([ + ' [Throws, WebExtensionStub="AsyncAmbiguous"]', + " any jsonSchemaAmbiguousMethod(any... args);", + ]) assert generated_webidl == expected_webidl fnAmbiguousByConfig = apiNs.functions.get("configuredAsAmbiguousMethod") assert isinstance(fnAmbiguousByConfig, APIFunction) generated_webidl = WebIDLHelpers.to_webidl_definition(fnAmbiguousByConfig, None) - expected_webidl = "\n".join( - [ - ' [Throws, WebExtensionStub="AsyncAmbiguous"]', - " any configuredAsAmbiguousMethod(any... args);", - ] - ) + expected_webidl = "\n".join([ + ' [Throws, WebExtensionStub="AsyncAmbiguous"]', + " any configuredAsAmbiguousMethod(any... args);", + ]) assert generated_webidl == expected_webidl diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/js.py b/toolkit/components/glean/build_scripts/glean_parser_ext/js.py @@ -167,9 +167,9 @@ def write_metrics( template_filename, ) - assert ( - INDEX_BITS + TYPE_BITS + ID_BITS <= ENTRY_WIDTH - ), "INDEX_BITS, TYPE_BITS, or ID_BITS are larger than allowed" + assert INDEX_BITS + TYPE_BITS + ID_BITS <= ENTRY_WIDTH, ( + "INDEX_BITS, TYPE_BITS, or ID_BITS are larger than allowed" + ) get_metric_id = generate_metric_ids(objs, options) # Mapping from a metric's identifier to the entry (metric ID | type id | index) @@ -223,12 +223,12 @@ def write_metrics( # TODO: Merge the method here that creates the metric_type_ids dict with `type_ids_and_categories` in `util.py` - https://bugzilla.mozilla.org/show_bug.cgi?id=1999421 other_type_ids = list(type_ids_and_categories(objs)[0].items()) for i, v in enumerate(metric_type_ids): - assert ( - v[1][1] == other_type_ids[i][0] - ), f"Metric {v[1][1]} is at index {i} in `glean_parser_ext/js.py`, but {other_type_ids[i][0]} is at that index in `glean_parser_ext/util.py`" - assert ( - v[1][0] == other_type_ids[i][1]["id"] - ), f"Metric {v[1][1]} has type_id {v[1][0]} in `glean_parser_ext/js.py`, but its type_id in `glean_parser_ext/util.py` is {other_type_ids[i][1]['id']}" + assert v[1][1] == other_type_ids[i][0], ( + f"Metric {v[1][1]} is at index {i} in `glean_parser_ext/js.py`, but {other_type_ids[i][0]} is at that index in `glean_parser_ext/util.py`" + ) + assert v[1][0] == other_type_ids[i][1]["id"], ( + f"Metric {v[1][1]} has type_id {v[1][0]} in `glean_parser_ext/js.py`, but its type_id in `glean_parser_ext/util.py` is {other_type_ids[i][1]['id']}" + ) # Create a lookup table for the metric categories only category_string_table = category_string_table.writeToString("gCategoryStringTable") diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/metrics_header_names.py b/toolkit/components/glean/build_scripts/glean_parser_ext/metrics_header_names.py @@ -39,9 +39,7 @@ def convert_yaml_path_to_header_name(filepath): path_components = ["geckoview", "metrics"] path_components = "_".join(path_components).split("_") - return "".join( - [ - path_component[0].upper() + path_component[1:] - for path_component in path_components - ] - ) + return "".join([ + path_component[0].upper() + path_component[1:] + for path_component in path_components + ]) diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py b/toolkit/components/glean/build_scripts/glean_parser_ext/run_glean_parser.py @@ -124,9 +124,9 @@ def parse(args, interesting_yamls=None): with open(cache_file, "rb") as cache: cached_objects, cached_options = pickle.load(cache) objects.update(cached_objects) - assert ( - options is None or cached_options == options - ), "consistent options" + assert options is None or cached_options == options, ( + "consistent options" + ) options = options or cached_options return objects, options @@ -372,12 +372,10 @@ def output_gifft_map(output_fd, probe_type, all_objs, cpp_fd, options): sys.exit(1) ids_to_probes[get_metric_id(metric)] = info # If we don't support a mirror for this metric type: build error. - elif not any( - [ - metric.type in types_for_probe - for types_for_probe in GIFFT_TYPES.values() - ] - ): + elif not any([ + metric.type in types_for_probe + for types_for_probe in GIFFT_TYPES.values() + ]): print( f"Glean metric {category_name}.{metric.name} is of type {metric.type}" " which can't be mirrored (we don't know how).", diff --git a/toolkit/components/glean/build_scripts/glean_parser_ext/util.py b/toolkit/components/glean/build_scripts/glean_parser_ext/util.py @@ -5,6 +5,7 @@ """ Utility functions for the glean_parser-based code generator """ + import copy from hashlib import sha1 diff --git a/toolkit/components/glean/tests/pytest/test_yaml_indices.py b/toolkit/components/glean/tests/pytest/test_yaml_indices.py @@ -33,9 +33,9 @@ def test_yamls_sorted(): continue yamls_to_test = metrics_index.__dict__[name] - assert ( - sorted(yamls_to_test) == yamls_to_test - ), f"{name} must be be lexicographically sorted." + assert sorted(yamls_to_test) == yamls_to_test, ( + f"{name} must be be lexicographically sorted." + ) if __name__ == "__main__": diff --git a/toolkit/components/nimbus/generate/generate_feature_manifest.py b/toolkit/components/nimbus/generate/generate_feature_manifest.py @@ -224,20 +224,18 @@ def generate_platform_feature_manifest(fd, input_file): write_fm_headers(fd) def file_structure(data): - return "\n".join( - [ - "#ifndef mozilla_NimbusFeaturesManifest_h", - "#define mozilla_NimbusFeaturesManifest_h", - "#include <utility>", - '#include "mozilla/Maybe.h"', - '#include "nsStringFwd.h"', - "namespace mozilla {", - platform_feature_manifest_array(data), - '#include "./lib/NimbusFeatureManifest.inc.h"', - "} // namespace mozilla", - "#endif // mozilla_NimbusFeaturesManifest_h", - ] - ) + return "\n".join([ + "#ifndef mozilla_NimbusFeaturesManifest_h", + "#define mozilla_NimbusFeaturesManifest_h", + "#include <utility>", + '#include "mozilla/Maybe.h"', + '#include "nsStringFwd.h"', + "namespace mozilla {", + platform_feature_manifest_array(data), + '#include "./lib/NimbusFeatureManifest.inc.h"', + "} // namespace mozilla", + "#endif // mozilla_NimbusFeaturesManifest_h", + ]) try: with open(input_file, encoding="utf-8") as yaml_input: diff --git a/toolkit/components/telemetry/build_scripts/gen_event_data.py b/toolkit/components/telemetry/build_scripts/gen_event_data.py @@ -165,21 +165,17 @@ def generate_JSON_definitions(output, *filenames): if category not in event_definitions: event_definitions[category] = OrderedDict() - event_definitions[category][event.name] = OrderedDict( - { - "methods": event.methods, - "objects": event.objects, - "extra_keys": event.extra_keys, - "record_on_release": ( - True if event.dataset_short == "opt-out" else False - ), - # We don't expire dynamic-builtin scalars: they're only meant for - # use in local developer builds anyway. They will expire when rebuilding. - "expires": event.expiry_version, - "expired": False, - "products": event.products, - } - ) + event_definitions[category][event.name] = OrderedDict({ + "methods": event.methods, + "objects": event.objects, + "extra_keys": event.extra_keys, + "record_on_release": (True if event.dataset_short == "opt-out" else False), + # We don't expire dynamic-builtin scalars: they're only meant for + # use in local developer builds anyway. They will expire when rebuilding. + "expires": event.expiry_version, + "expired": False, + "products": event.products, + }) json.dump(event_definitions, output, sort_keys=True) diff --git a/toolkit/components/telemetry/build_scripts/gen_scalar_data.py b/toolkit/components/telemetry/build_scripts/gen_scalar_data.py @@ -181,22 +181,18 @@ def generate_JSON_definitions(output, *filenames): if category not in scalar_definitions: scalar_definitions[category] = OrderedDict() - scalar_definitions[category][scalar.name] = OrderedDict( - { - "kind": scalar.nsITelemetry_kind, - "keyed": scalar.keyed, - "keys": scalar.keys, - "record_on_release": ( - True if scalar.dataset_short == "opt-out" else False - ), - # We don't expire dynamic-builtin scalars: they're only meant for - # use in local developer builds anyway. They will expire when rebuilding. - "expired": False, - "stores": scalar.record_into_store, - "expires": scalar.expires, - "products": scalar.products, - } - ) + scalar_definitions[category][scalar.name] = OrderedDict({ + "kind": scalar.nsITelemetry_kind, + "keyed": scalar.keyed, + "keys": scalar.keys, + "record_on_release": (True if scalar.dataset_short == "opt-out" else False), + # We don't expire dynamic-builtin scalars: they're only meant for + # use in local developer builds anyway. They will expire when rebuilding. + "expired": False, + "stores": scalar.record_into_store, + "expires": scalar.expires, + "products": scalar.products, + }) json.dump(scalar_definitions, output) diff --git a/toolkit/components/telemetry/build_scripts/mach_commands.py b/toolkit/components/telemetry/build_scripts/mach_commands.py @@ -22,9 +22,7 @@ GLEAN_METRIC_TEMPLATE = """ notification_emails:{emails_alias}{emails_list} expires: {expiry} {extra}telemetry_mirror: {legacy_enum} -""".strip( - "\n" -) +""".strip("\n") EXTRA_KEY_DESCRIPTION_INDENT = DESCRIPTION_INDENT + " " VALUE_EXTRA_DESCRIPTION = "The `value` of the event. Mirrors to the Legacy Telemetry event's `value` parameter." @@ -33,9 +31,7 @@ EXTRA_KEY_TEMPLATE = """ description: > {description} type: string -""".strip( - "\n" -) +""".strip("\n") @Command( diff --git a/toolkit/components/telemetry/build_scripts/mozparsers/parse_histograms.py b/toolkit/components/telemetry/build_scripts/mozparsers/parse_histograms.py @@ -41,7 +41,7 @@ ALWAYS_ALLOWED_KEYS = [ ] BASE_DOC_URL = ( - "https://firefox-source-docs.mozilla.org/toolkit/components/" "telemetry/telemetry/" + "https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/telemetry/" ) HISTOGRAMS_DOC_URL = BASE_DOC_URL + "collection/histograms.html" SCALARS_DOC_URL = BASE_DOC_URL + "collection/scalars.html" diff --git a/toolkit/components/telemetry/build_scripts/mozparsers/parse_scalars.py b/toolkit/components/telemetry/build_scripts/mozparsers/parse_scalars.py @@ -459,7 +459,7 @@ def load_scalars(filename, strict_type_checks=True): ParserError("Error opening " + filename + ": " + str(e)).handle_now() except ValueError as e: ParserError( - f"Error parsing scalars in {filename}: {e}" f".\nSee: {BASE_DOC_URL}" + f"Error parsing scalars in {filename}: {e}.\nSee: {BASE_DOC_URL}" ).handle_now() scalar_list = [] diff --git a/toolkit/components/telemetry/build_scripts/mozparsers/parse_user_interactions.py b/toolkit/components/telemetry/build_scripts/mozparsers/parse_user_interactions.py @@ -204,8 +204,7 @@ def load_user_interactions(filename): ParserError("Error opening " + filename + ": " + str(e)).handle_now() except ValueError as e: ParserError( - f"Error parsing UserInteractions in {filename}: {e}" - f".\nSee: {BASE_DOC_URL}" + f"Error parsing UserInteractions in {filename}: {e}.\nSee: {BASE_DOC_URL}" ).handle_now() user_interaction_list = [] diff --git a/toolkit/components/telemetry/tests/integration/tests/conftest.py b/toolkit/components/telemetry/tests/integration/tests/conftest.py @@ -99,15 +99,15 @@ class Browser: self.addon_ids = [] def disable_telemetry(self): - self.marionette.instance.profile.set_persistent_preferences( - {"datareporting.healthreport.uploadEnabled": False} - ) + self.marionette.instance.profile.set_persistent_preferences({ + "datareporting.healthreport.uploadEnabled": False + }) self.marionette.set_pref("datareporting.healthreport.uploadEnabled", False) def enable_telemetry(self): - self.marionette.instance.profile.set_persistent_preferences( - {"datareporting.healthreport.uploadEnabled": True} - ) + self.marionette.instance.profile.set_persistent_preferences({ + "datareporting.healthreport.uploadEnabled": True + }) self.marionette.set_pref("datareporting.healthreport.uploadEnabled", True) def get_client_id(self): diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_testcase.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/fog_testcase.py @@ -43,22 +43,20 @@ class FOGTestCase(TelemetryTestCase): f"Submitting to FOG ping server at {self.fog_ping_server.url}" ) - self.marionette.enforce_gecko_prefs( - { - "telemetry.fog.test.localhost_port": self.fog_ping_server.port, - # Enable FOG logging. 5 means "Verbose". See - # https://firefox-source-docs.mozilla.org/xpcom/logging.html - # for details. - "logging.config.clear_on_startup": False, - "logging.config.sync": True, - "logging.fog::*": 5, - "logging.fog_control::*": 5, - "logging.glean::*": 5, - "logging.glean_core::*": 5, - # Slow down the user inactivity timeout (bug 1690728) - "dom.events.user_interaction_interval": 60000, - } - ) + self.marionette.enforce_gecko_prefs({ + "telemetry.fog.test.localhost_port": self.fog_ping_server.port, + # Enable FOG logging. 5 means "Verbose". See + # https://firefox-source-docs.mozilla.org/xpcom/logging.html + # for details. + "logging.config.clear_on_startup": False, + "logging.config.sync": True, + "logging.fog::*": 5, + "logging.fog_control::*": 5, + "logging.glean::*": 5, + "logging.glean_core::*": 5, + # Slow down the user inactivity timeout (bug 1690728) + "dom.events.user_interaction_interval": 60000, + }) def tearDown(self, *args, **kwargs): super().tearDown(*args, **kwargs) diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runner.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runner.py @@ -27,37 +27,35 @@ class TelemetryTestRunner(BaseMarionetteTestRunner): prefs["fission.autostart"] = False # Set Firefox Client Telemetry specific preferences - prefs.update( - { - # Clear the region detection url to - # * avoid net access in tests - # * stabilize browser.search.region to avoid extra subsessions (bug 1579840#c40) - "browser.region.network.url": "", - # Disable smart sizing because it changes prefs at startup. (bug 1547750) - "browser.cache.disk.smart_size.enabled": False, - "toolkit.telemetry.server": f"{SERVER_URL}/pings", - "telemetry.fog.test.localhost_port": -1, - "toolkit.telemetry.initDelay": 1, - "toolkit.telemetry.minSubsessionLength": 0, - "datareporting.healthreport.uploadEnabled": True, - "datareporting.policy.dataSubmissionEnabled": True, - "datareporting.policy.dataSubmissionPolicyBypassNotification": True, - "datareporting.usage.uploadEnabled": True, - "toolkit.telemetry.log.level": "Trace", - "toolkit.telemetry.log.dump": True, - "toolkit.telemetry.send.overrideOfficialCheck": True, - "toolkit.telemetry.testing.disableFuzzingDelay": True, - # Disable Normandy to avoid extra subsessions due to Experiment - # activation in tests (bug 1641571) - "app.normandy.enabled": False, - # Disable Normandy a little harder (bug 1608807). - # This should also disable Nimbus. - "app.shield.optoutstudies.enabled": False, - # Bug 1789727: Keep the screenshots extension disabled to avoid - # disabling the addon resulting in extra subsessions - "screenshots.browser.component.enabled": False, - } - ) + prefs.update({ + # Clear the region detection url to + # * avoid net access in tests + # * stabilize browser.search.region to avoid extra subsessions (bug 1579840#c40) + "browser.region.network.url": "", + # Disable smart sizing because it changes prefs at startup. (bug 1547750) + "browser.cache.disk.smart_size.enabled": False, + "toolkit.telemetry.server": f"{SERVER_URL}/pings", + "telemetry.fog.test.localhost_port": -1, + "toolkit.telemetry.initDelay": 1, + "toolkit.telemetry.minSubsessionLength": 0, + "datareporting.healthreport.uploadEnabled": True, + "datareporting.policy.dataSubmissionEnabled": True, + "datareporting.policy.dataSubmissionPolicyBypassNotification": True, + "datareporting.usage.uploadEnabled": True, + "toolkit.telemetry.log.level": "Trace", + "toolkit.telemetry.log.dump": True, + "toolkit.telemetry.send.overrideOfficialCheck": True, + "toolkit.telemetry.testing.disableFuzzingDelay": True, + # Disable Normandy to avoid extra subsessions due to Experiment + # activation in tests (bug 1641571) + "app.normandy.enabled": False, + # Disable Normandy a little harder (bug 1608807). + # This should also disable Nimbus. + "app.shield.optoutstudies.enabled": False, + # Bug 1789727: Keep the screenshots extension disabled to avoid + # disabling the addon resulting in extra subsessions + "screenshots.browser.component.enabled": False, + }) super().__init__(prefs=prefs, **kwargs) diff --git a/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py b/toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py @@ -45,16 +45,16 @@ class TelemetryTestCase(WindowManagerMixin, MarionetteTestCase): def disable_telemetry(self): """Disable the Firefox Data Collection and Use in the current browser.""" - self.marionette.instance.profile.set_persistent_preferences( - {"datareporting.healthreport.uploadEnabled": False} - ) + self.marionette.instance.profile.set_persistent_preferences({ + "datareporting.healthreport.uploadEnabled": False + }) self.marionette.set_pref("datareporting.healthreport.uploadEnabled", False) def enable_telemetry(self): """Enable the Firefox Data Collection and Use in the current browser.""" - self.marionette.instance.profile.set_persistent_preferences( - {"datareporting.healthreport.uploadEnabled": True} - ) + self.marionette.instance.profile.set_persistent_preferences({ + "datareporting.healthreport.uploadEnabled": True + }) self.marionette.set_pref("datareporting.healthreport.uploadEnabled", True) @contextlib.contextmanager diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_usage_reporting.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_usage_reporting.py @@ -20,16 +20,16 @@ class TestUsageReporting(FOGTestCase): def disable_usage_reporting(self): """Disable usage reporting in the current browser.""" - self.marionette.instance.profile.set_persistent_preferences( - {"datareporting.usage.uploadEnabled": False} - ) + self.marionette.instance.profile.set_persistent_preferences({ + "datareporting.usage.uploadEnabled": False + }) self.marionette.set_pref("datareporting.usage.uploadEnabled", False) def enable_usage_reporting(self): """Enable usage reporting in the current browser.""" - self.marionette.instance.profile.set_persistent_preferences( - {"datareporting.usage.uploadEnabled": True} - ) + self.marionette.instance.profile.set_persistent_preferences({ + "datareporting.usage.uploadEnabled": True + }) self.marionette.set_pref("datareporting.usage.uploadEnabled", True) def test_usage_reporting_independent_from_telemetry(self): @@ -273,17 +273,17 @@ class TestUsageReporting(FOGTestCase): last_pid = self.marionette.process_id # New profiles should not inherit the general preference. - self.marionette.enforce_gecko_prefs( - {"datareporting.healthreport.uploadEnabled": False} - ) + self.marionette.enforce_gecko_prefs({ + "datareporting.healthreport.uploadEnabled": False + }) self.assertIs(healthreportEnabled(), False) self.assertIs(usageEnabled(), True) self.assertNotEqual(last_pid, self.marionette.process_id) last_pid = self.marionette.process_id - self.marionette.enforce_gecko_prefs( - {"datareporting.healthreport.uploadEnabled": True} - ) + self.marionette.enforce_gecko_prefs({ + "datareporting.healthreport.uploadEnabled": True + }) self.assertIs(healthreportEnabled(), True) self.assertIs(usageEnabled(), True) self.assertNotEqual(last_pid, self.marionette.process_id) diff --git a/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_user_activity.py b/toolkit/components/telemetry/tests/marionette/tests/client/test_fog_user_activity.py @@ -56,12 +56,10 @@ class TestClientActivity(FOGTestCase): ) # First test that restarting the browser sends a "active" ping - received_pings = sorted( - [ - ping0["request_url"]["doc_type"], - ping1["request_url"]["doc_type"], - ] - ) + received_pings = sorted([ + ping0["request_url"]["doc_type"], + ping1["request_url"]["doc_type"], + ]) self.assertEqual(expected_pings, received_pings) if ping0["request_url"]["doc_type"] == "baseline": self.assertEqual("active", ping0["payload"]["ping_info"]["reason"]) @@ -96,24 +94,20 @@ class TestClientActivity(FOGTestCase): ping_server=self.fog_ping_server, ) - received_pings = sorted( - [ - ping2["request_url"]["doc_type"], - ping3["request_url"]["doc_type"], - ] - ) + received_pings = sorted([ + ping2["request_url"]["doc_type"], + ping3["request_url"]["doc_type"], + ]) self.assertEqual(expected_pings, received_pings) if ping2["request_url"]["doc_type"] == "baseline": self.assertEqual("active", ping2["payload"]["ping_info"]["reason"]) if ping3["request_url"]["doc_type"] == "baseline": self.assertEqual("active", ping3["payload"]["ping_info"]["reason"]) - received_pings = sorted( - [ - ping4["request_url"]["doc_type"], - ping5["request_url"]["doc_type"], - ] - ) + received_pings = sorted([ + ping4["request_url"]["doc_type"], + ping5["request_url"]["doc_type"], + ]) self.assertEqual(expected_pings, received_pings) if ping4["request_url"]["doc_type"] == "baseline": self.assertEqual("inactive", ping4["payload"]["ping_info"]["reason"]) diff --git a/toolkit/components/telemetry/tests/python/test_mirrors.py b/toolkit/components/telemetry/tests/python/test_mirrors.py @@ -72,9 +72,9 @@ def mirroring_metrics(objs): hasattr(metric, "telemetry_mirror") and metric.telemetry_mirror is not None ): - assert ( - metric.type in MIRROR_TYPES.keys() - ), f"{metric.type} is not a GIFFT-supported type." + assert metric.type in MIRROR_TYPES.keys(), ( + f"{metric.type} is not a GIFFT-supported type." + ) yield metric @@ -89,14 +89,16 @@ def ensure_compatible_event(metric, probe): # mirror's value parameter. if key == "value": continue - assert ( - key in probe.extra_keys - ), f"Key {key} not in mirrored event probe {probe.identifier}. Be sure to add it." + assert key in probe.extra_keys, ( + f"Key {key} not in mirrored event probe {probe.identifier}. Be sure to add it." + ) else: assert ( metric.allowed_extra_keys == probe.extra_keys or metric.allowed_extra_keys == sorted(probe.extra_keys + ["value"]) - ), f"Metric {metric.identifier()}'s extra keys {metric.allowed_extra_keys} are not the same as probe {probe.identifier}'s extras {probe.extra_keys}." + ), ( + f"Metric {metric.identifier()}'s extra keys {metric.allowed_extra_keys} are not the same as probe {probe.identifier}'s extras {probe.extra_keys}." + ) # Histograms are compatible with metrics if they are @@ -104,42 +106,46 @@ def ensure_compatible_event(metric, probe): # * of a suitable `kind` (e.g. "linear", "exponential", or "enumerated") def ensure_compatible_histogram(metric, probe): if metric.type == "counter": - assert ( - probe.kind() == "count" - ), f"Metric {metric.identifier()} is a `counter` mapping to a histogram, but {probe.name()} isn't a 'count' Histogram (is '{probe.kind()}')." + assert probe.kind() == "count", ( + f"Metric {metric.identifier()} is a `counter` mapping to a histogram, but {probe.name()} isn't a 'count' Histogram (is '{probe.kind()}')." + ) return elif metric.type == "labeled_counter": if probe.kind() == "boolean": assert metric.ordered_labels == [ "false", "true", - ], f"Metric {metric.identifier()} is a `labeled_counter` mapping to a boolean histogram, but it doesn't have labels ['false', 'true'] (has {metric.ordered_labels} instead)." + ], ( + f"Metric {metric.identifier()} is a `labeled_counter` mapping to a boolean histogram, but it doesn't have labels ['false', 'true'] (has {metric.ordered_labels} instead)." + ) elif probe.kind() == "count": - assert ( - probe.keyed() - ), f"Metric {metric.identifier()} is a `labeled_counter` mapping to un-keyed 'count' histogram {probe.name()}." + assert probe.keyed(), ( + f"Metric {metric.identifier()} is a `labeled_counter` mapping to un-keyed 'count' histogram {probe.name()}." + ) elif probe.kind() == "categorical": - assert ( - metric.ordered_labels == probe.labels() - ), f"Metric {metric.identifier()} is a `labeled_counter` mapping to categorical histogram {probe.name()}, but the labels don't match." + assert metric.ordered_labels == probe.labels(), ( + f"Metric {metric.identifier()} is a `labeled_counter` mapping to categorical histogram {probe.name()}, but the labels don't match." + ) else: - assert ( - False - ), f"Metric {metric.identifier()} is a `labeled_counter` mapping to a histogram, but {probe.name()} isn't a 'boolean, keyed 'count', or 'categorical' Histogram (is '{probe.kind()}')." + assert False, ( + f"Metric {metric.identifier()} is a `labeled_counter` mapping to a histogram, but {probe.name()} isn't a 'boolean, keyed 'count', or 'categorical' Histogram (is '{probe.kind()}')." + ) return elif metric.type == "dual_labeled_counter": - assert ( - probe.keyed() - ), f"Metric {metric.identifier()} must mirror to a keyed histogram." + assert probe.keyed(), ( + f"Metric {metric.identifier()} must mirror to a keyed histogram." + ) if probe.kind() == "boolean": assert metric.ordered_categories == [ "false", "true", - ], f"Metric {metric.identifier()} is a `dual_labeled_counter` mapping to a keyed boolean histogram, but it doesn't have labels ['false', 'true'] (has {metric.ordered_labels} instead)." + ], ( + f"Metric {metric.identifier()} is a `dual_labeled_counter` mapping to a keyed boolean histogram, but it doesn't have labels ['false', 'true'] (has {metric.ordered_labels} instead)." + ) elif probe.kind() == "categorical": - assert ( - metric.ordered_categories == probe.labels() - ), f"Metric {metric.identifier()} is a `dual_labeled_counter` mapping to keyed categorical histogram {probe.name()}, but the labels don't match." + assert metric.ordered_categories == probe.labels(), ( + f"Metric {metric.identifier()} is a `dual_labeled_counter` mapping to keyed categorical histogram {probe.name()}, but the labels don't match." + ) return assert probe.kind() in [ @@ -161,10 +167,12 @@ def ensure_compatible_histogram(metric, probe): metric.range_min == 0 and metric.histogram_type == metrics.HistogramType.linear and metric.bucket_count == n_values_plus_one - ), f"Metric {metric.identifier()} mapping to enumerated histogram {probe.name()} must have a range that starts at 0 (is {metric.range_min}), must have `linear` bucket allocation (is {metric.histogram_type}), and must have one more bucket than the probe's n_values (is {metric.bucket_count}, should be {n_values_plus_one})." - assert ( - hasattr(metric, "labeled") and metric.labeled - ) == probe.keyed(), f"Metric {metric.identifier()}'s labeledness must match mirrored histogram probe {probe.name()}'s keyedness." + ), ( + f"Metric {metric.identifier()} mapping to enumerated histogram {probe.name()} must have a range that starts at 0 (is {metric.range_min}), must have `linear` bucket allocation (is {metric.histogram_type}), and must have one more bucket than the probe's n_values (is {metric.bucket_count}, should be {n_values_plus_one})." + ) + assert (hasattr(metric, "labeled") and metric.labeled) == probe.keyed(), ( + f"Metric {metric.identifier()}'s labeledness must match mirrored histogram probe {probe.name()}'s keyedness." + ) # Scalars are compatible with metrics if they are @@ -174,9 +182,9 @@ def ensure_compatible_scalar(metric, probe): mirror_should_be_keyed = ( hasattr(metric, "labeled") and metric.labeled ) or metric.type in ["string_list", "rate"] - assert ( - mirror_should_be_keyed == probe.keyed - ), f"Metric {metric.identifier()}'s type ({metric.type}) must have appropriate keyedness in the mirrored scalar probe {probe.label}." + assert mirror_should_be_keyed == probe.keyed, ( + f"Metric {metric.identifier()}'s type ({metric.type}) must have appropriate keyedness in the mirrored scalar probe {probe.label}." + ) TYPE_MAP = { "boolean": "boolean", @@ -193,9 +201,9 @@ def ensure_compatible_scalar(metric, probe): "labeled_quantity": "uint", "rate": "uint", } - assert ( - TYPE_MAP[metric.type] == probe.kind - ), f"Metric {metric.identifier()}'s type ({metric.type}) requires a mirror probe scalar of kind '{TYPE_MAP[metric.type]}' which doesn't match mirrored scalar probe {probe.label}'s kind ({probe.kind})" + assert TYPE_MAP[metric.type] == probe.kind, ( + f"Metric {metric.identifier()}'s type ({metric.type}) requires a mirror probe scalar of kind '{TYPE_MAP[metric.type]}' which doesn't match mirrored scalar probe {probe.label}'s kind ({probe.kind})" + ) class TestTelemetryMirrors(unittest.TestCase): @@ -215,9 +223,9 @@ class TestTelemetryMirrors(unittest.TestCase): assert not util.report_validation_errors(objs) hgrams = list( - parse_histograms.from_files( - [path.join(TELEMETRY_ROOT_PATH, "Histograms.json")] - ) + parse_histograms.from_files([ + path.join(TELEMETRY_ROOT_PATH, "Histograms.json") + ]) ) scalars = list( @@ -265,12 +273,12 @@ class TestTelemetryMirrors(unittest.TestCase): ensure_compatible_scalar(metric, scalar) break else: - assert ( - False - ), f"mirror probe type {MIRROR_TYPES[metric.type]} isn't recognized." - assert ( - found - ), f"Mirror {metric.telemetry_mirror} not found for metric {metric.identifier()}" + assert False, ( + f"mirror probe type {MIRROR_TYPES[metric.type]} isn't recognized." + ) + assert found, ( + f"Mirror {metric.telemetry_mirror} not found for metric {metric.identifier()}" + ) # Step 3: Forbid unmirrored-to probes for event in events: diff --git a/toolkit/components/telemetry/tests/python/test_no_expired_probes.py b/toolkit/components/telemetry/tests/python/test_no_expired_probes.py @@ -38,9 +38,9 @@ class TestNoExpiredProbes(unittest.TestCase): app_version_major = app_version.split(".", 1)[0] - hgrams = parse_histograms.from_files( - [path.join(TELEMETRY_ROOT_PATH, "Histograms.json")] - ) + hgrams = parse_histograms.from_files([ + path.join(TELEMETRY_ROOT_PATH, "Histograms.json") + ]) for hgram in hgrams: if hgram.name().startswith("TELEMETRY_TEST_"): # We ignore test histograms which are permitted to be expired. diff --git a/toolkit/components/translations/bergamot-translator/build-bergamot.py b/toolkit/components/translations/bergamot-translator/build-bergamot.py @@ -123,9 +123,9 @@ def fetch_bergamot_source(translations_repo: Path | None): maybe_remove_repo_path() if translations_repo: - assert ( - translations_repo.is_dir() - ), f"The translations repo must be a directory: {translations_repo}" + assert translations_repo.is_dir(), ( + f"The translations repo must be a directory: {translations_repo}" + ) logger.info(f"Using local mozilla/translations repo: {translations_repo}") diff --git a/toolkit/components/translations/bergamot-translator/upload-bergamot.py b/toolkit/components/translations/bergamot-translator/upload-bergamot.py @@ -190,7 +190,7 @@ class RemoteSettings: print("📦 Packages in the workspace:") for record in workspace_records["data"]: if record["name"] == "bergamot-translator": - print(f' - bergamot-translator@{record["version"]}') + print(f" - bergamot-translator@{record['version']}") print(f"✅ Version {self.version} does not conflict, ready for uploading.") diff --git a/toolkit/crashreporter/annotations/generate.py b/toolkit/crashreporter/annotations/generate.py @@ -194,7 +194,7 @@ def emit_java(annotations, output_name): package, klass, is_kotlin = derive_package_and_class(output_name) enum = ",\n".join( - f"/** {javadoc_sanitize(data['description'])} */\n{name}(\"{name}\", \"{data.get('scope', 'client')}\")" + f'/** {javadoc_sanitize(data["description"])} */\n{name}("{name}", "{data.get("scope", "client")}")' for (name, data) in annotations ) diff --git a/toolkit/crashreporter/crashping/conversions.py b/toolkit/crashreporter/crashping/conversions.py @@ -50,7 +50,7 @@ def main(output, annotations_path): args = "" if glean_type == "string_list": - args += f", \"{glean['delimiter']}\"" + args += f', "{glean["delimiter"]}"' output.write( f'#[allow(non_upper_case_globals)] pub const {name}: Annotation = convert!({namespace}::{metric} = {conversion}("{name}"{args}));\n' diff --git a/toolkit/crashreporter/crashping/glean_metrics.py b/toolkit/crashreporter/crashping/glean_metrics.py @@ -50,20 +50,18 @@ def generate_string(annotations): make_bugzilla_url(bug) for bug in definition.get("bugs", ["1950749"]) ] - glean.update( - { - "description": definition["description"].strip(), - "notification_emails": [ - "crash-reporting-wg@mozilla.org", - "stability@mozilla.org", - ], - "bugs": bug_urls, - "data_reviews": bug_urls.copy(), # Copy so the produced yaml doesn't use anchors - "data_sensitivity": ["technical"], - "expires": "never", - "send_in_pings": ["crash"], - } - ) + glean.update({ + "description": definition["description"].strip(), + "notification_emails": [ + "crash-reporting-wg@mozilla.org", + "stability@mozilla.org", + ], + "bugs": bug_urls, + "data_reviews": bug_urls.copy(), # Copy so the produced yaml doesn't use anchors + "data_sensitivity": ["technical"], + "expires": "never", + "send_in_pings": ["crash"], + }) output.setdefault(metric_category, {})[metric_name] = glean diff --git a/toolkit/crashreporter/tools/symbolstore.py b/toolkit/crashreporter/tools/symbolstore.py @@ -394,7 +394,7 @@ def validate_install_manifests(install_manifest_args): bits = arg.split(",") if len(bits) != 2: raise ValueError( - "Invalid format for --install-manifest: " "specify manifest,target_dir" + "Invalid format for --install-manifest: specify manifest,target_dir" ) manifest_file, destination = [os.path.abspath(b) for b in bits] if not os.path.isfile(manifest_file): @@ -603,30 +603,24 @@ class Dumper: ] if buildconfig.substs.get("MOZ_APP_VENDOR") is not None: - cmdline.extend( - [ - "--extra-info", - "VENDOR " + buildconfig.substs["MOZ_APP_VENDOR"], - ] - ) + cmdline.extend([ + "--extra-info", + "VENDOR " + buildconfig.substs["MOZ_APP_VENDOR"], + ]) if buildconfig.substs.get("MOZ_APP_BASENAME") is not None: - cmdline.extend( - [ - "--extra-info", - "PRODUCTNAME " + buildconfig.substs["MOZ_APP_BASENAME"], - ] - ) + cmdline.extend([ + "--extra-info", + "PRODUCTNAME " + buildconfig.substs["MOZ_APP_BASENAME"], + ]) # Add the build ID if it's present try: buildid = get_buildid() - cmdline.extend( - [ - "--extra-info", - "BUILDID " + buildid, - ] - ) + cmdline.extend([ + "--extra-info", + "BUILDID " + buildid, + ]) except Exception: pass @@ -933,15 +927,13 @@ class Dumper_Linux(Dumper): file_dbg = file + ".dbg" if ( subprocess.call([self.objcopy, "--only-keep-debug", file, file_dbg]) == 0 - and subprocess.call( - [ - self.objcopy, - "--remove-section", - ".gnu_debuglink", - "--add-gnu-debuglink=%s" % file_dbg, - file, - ] - ) + and subprocess.call([ + self.objcopy, + "--remove-section", + ".gnu_debuglink", + "--add-gnu-debuglink=%s" % file_dbg, + file, + ]) == 0 ): rel_path = os.path.join(debug_file, guid, debug_file + ".dbg") @@ -993,13 +985,11 @@ class Dumper_Mac(Dumper): cmdline = [self.dump_syms] cmdline.extend(arch.split()) - cmdline.extend( - [ - "--inlines", - "-j", - "2", - ] - ) + cmdline.extend([ + "--inlines", + "-j", + "2", + ]) if self.include_moz_extra_info: cmdline.extend(self.dump_syms_extra_info()) diff --git a/toolkit/crashreporter/tools/unit-symbolstore.py b/toolkit/crashreporter/tools/unit-symbolstore.py @@ -349,13 +349,11 @@ if target_platform() == "WINNT": self.add_test_files(test_files) # mock calls to `dump_syms`, `hg parent` and # `hg showconfig paths.default` - mock_Popen.return_value.stdout = iter( - [ - "MODULE os x86 %s %s" % ("X" * 33, test_files[0]), - "FILE 0 %s" % sourcefile, - "PUBLIC xyz 123", - ] - ) + mock_Popen.return_value.stdout = iter([ + "MODULE os x86 %s %s" % ("X" * 33, test_files[0]), + "FILE 0 %s" % sourcefile, + "PUBLIC xyz 123", + ]) mock_Popen.return_value.wait.return_value = 0 mock_communicate = mock_Popen.return_value.communicate mock_communicate.side_effect = [ diff --git a/toolkit/library/libxul.so-gdb.py b/toolkit/library/libxul.so-gdb.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" GDB Python customization auto-loader for libxul """ +"""GDB Python customization auto-loader for libxul""" import re from os.path import abspath, dirname, exists diff --git a/toolkit/xre/dllservices/mozglue/gen_dll_blocklist_defs.py b/toolkit/xre/dllservices/mozglue/gen_dll_blocklist_defs.py @@ -447,7 +447,7 @@ class Version: for component in arg: if not isinstance(component, int) or component < 0 or component > 0xFFFF: raise ValueError( - "Each version component must be a 16-bit " "unsigned integer" + "Each version component must be a 16-bit unsigned integer" ) def build_long(self, args): @@ -699,12 +699,10 @@ class LspBlocklistEntry(DllBlocklistEntry): # We dump the entire contents of Guids on the first call, and then # clear it. Remaining invocations of this method are no-ops. if LspBlocklistEntry.Guids: - result = ",\n".join( - [ - self.as_c_struct(guid, names) - for guid, names in LspBlocklistEntry.Guids.items() - ] - ) + result = ",\n".join([ + self.as_c_struct(guid, names) + for guid, names in LspBlocklistEntry.Guids.items() + ]) print(result, file=output) LspBlocklistEntry.Guids.clear() diff --git a/toolkit/xre/test/marionette/gen_win32k_tests.py b/toolkit/xre/test/marionette/gen_win32k_tests.py @@ -29,9 +29,7 @@ def set_default(d): output.write( """ if self.default_is is not {}: - return\n""".format( - "True" if d == "T" else "False" - ) + return\n""".format("True" if d == "T" else "False") ) @@ -96,9 +94,7 @@ def print_assertion(assertion): experimentStatus=ExperimentStatus.{}, pref={}, enrollmentStatusPref=ExperimentStatus.{}, - )\n""".format( - *assertion - ) + )\n""".format(*assertion) ) diff --git a/toolkit/xre/test/marionette/xdg_config/test_xdg_config_legacy_existing.py b/toolkit/xre/test/marionette/xdg_config/test_xdg_config_legacy_existing.py @@ -9,7 +9,6 @@ from xdg_config_home_test_case import XdgConfigHomeTestCase class TestXdgConfigHomeLegacyExisting(XdgConfigHomeTestCase): - def setUp(self): assert "XDG_CONFIG_HOME" not in self._env.keys() self._env.update({"XDG_CONFIG_HOME": f"{self.homedir}/.config-test/"}) diff --git a/toolkit/xre/test/marionette/xdg_config/test_xdg_config_legacy_forced.py b/toolkit/xre/test/marionette/xdg_config/test_xdg_config_legacy_forced.py @@ -7,7 +7,6 @@ from xdg_config_home_test_case import XdgConfigHomeTestCase class TestXdgConfigHomeLegacy(XdgConfigHomeTestCase): - def setUp(self): assert "MOZ_LEGACY_HOME" not in self._env.keys() self._env.update({"MOZ_LEGACY_HOME": "1"}) diff --git a/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new.py b/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new.py @@ -7,7 +7,6 @@ from xdg_config_home_test_case import XdgConfigHomeTestCase class TestXdgConfigHomeNew(XdgConfigHomeTestCase): - def test_profile_dir(self): self.client.navigate(self.about_support) diff --git a/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new_env.py b/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new_env.py @@ -7,7 +7,6 @@ from xdg_config_home_test_case import XdgConfigHomeTestCase class TestXdgConfigHomeNewEnv(XdgConfigHomeTestCase): - def setUp(self): assert "XDG_CONFIG_HOME" not in self._env.keys() self._env.update({"XDG_CONFIG_HOME": f"{self.homedir}/mozXDG-config-dir"}) diff --git a/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new_env_invalid.py b/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new_env_invalid.py @@ -7,7 +7,6 @@ from xdg_config_home_test_case import XdgConfigHomeTestCase class TestXdgConfigHomeNewEnvInvalid(XdgConfigHomeTestCase): - def setUp(self): assert "XDG_CONFIG_HOME" not in self._env.keys() self._env.update({"XDG_CONFIG_HOME": "$HOME/mozXDG-config-dir"}) diff --git a/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new_existing.py b/toolkit/xre/test/marionette/xdg_config/test_xdg_config_new_existing.py @@ -9,7 +9,6 @@ from xdg_config_home_test_case import XdgConfigHomeTestCase class TestXdgConfigHomeNewExisting(XdgConfigHomeTestCase): - def setUp(self): assert "XDG_CONFIG_HOME" not in self._env.keys() self._env.update({"XDG_CONFIG_HOME": f"{self.homedir}/.config-test/"}) diff --git a/toolkit/xre/test/marionette/xdg_config/xdg_config_home_test_case.py b/toolkit/xre/test/marionette/xdg_config/xdg_config_home_test_case.py @@ -61,9 +61,9 @@ class XdgConfigHomeTestCase(MarionetteTestCase): self.process_handler = subprocess.Popen(self._cmd, env=self._env) def assert_safe_homedir(self): - assert ( - "mozrunner-xdg_config-test" in self.homedir - ), f"HOME is not real user's home: {self.homedir}" + assert "mozrunner-xdg_config-test" in self.homedir, ( + f"HOME is not real user's home: {self.homedir}" + ) def get_home_root(self): rv = tempfile.mkdtemp(prefix="{}.".format("run"), dir=self.tmproot) diff --git a/tools/browsertime/mach_commands.py b/tools/browsertime/mach_commands.py @@ -454,15 +454,13 @@ def activate_browsertime_virtualenv(command_context, *args, **kwargs): "opencv-python==%s" % OPENCV_VERSION, ): if _need_install(command_context, dep): - subprocess.check_call( - [ - command_context.virtualenv_manager.python_path, - "-m", - "pip", - "install", - dep, - ] - ) + subprocess.check_call([ + command_context.virtualenv_manager.python_path, + "-m", + "pip", + "install", + dep, + ]) def check(command_context): @@ -548,9 +546,10 @@ def extra_default_args(command_context, args=[]): if not specifies_binaryPath: try: - extra_args.extend( - ("--firefox.binaryPath", command_context.get_binary_path()) - ) + extra_args.extend(( + "--firefox.binaryPath", + command_context.get_binary_path(), + )) except BinaryNotFoundException as e: command_context.log( logging.ERROR, diff --git a/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py b/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py @@ -38,6 +38,7 @@ Created on Apr 11, 2012 @author: mrmiller """ + import argparse import concurrent.futures import errno diff --git a/tools/fuzzing/smoke/js.py b/tools/fuzzing/smoke/js.py @@ -2,9 +2,10 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" Hello I am a fake jsshell for testing purpose. +"""Hello I am a fake jsshell for testing purpose. Add more features! """ + import argparse import sys diff --git a/tools/fuzzing/smoke/smoke.py b/tools/fuzzing/smoke/smoke.py @@ -1,7 +1,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" Smoke test script for Fuzzing +"""Smoke test script for Fuzzing This script can be used to perform simple calls using `jsshell` or whatever other tools you may add. @@ -15,6 +15,7 @@ Everything included in this directory will be added in tools if you need. They will be located in `$MOZ_FETCHES_DIR` and follow the same directory structure than the source tree. """ + import os import os.path import shlex diff --git a/tools/fuzzing/smoke/test_grizzly.py b/tools/fuzzing/smoke/test_grizzly.py @@ -33,9 +33,9 @@ def test_grizzly_smoke(relaunch): ) if MOZ_AUTOMATION: - assert Path( - ffbin - ).exists(), "Missing Firefox build. Build it, or set GECKO_BINARY_PATH" + assert Path(ffbin).exists(), ( + "Missing Firefox build. Build it, or set GECKO_BINARY_PATH" + ) elif not Path(ffbin).exists(): pytest.skip("Missing Firefox build. Build it, or set GECKO_BINARY_PATH") diff --git a/tools/lint/android/lints.py b/tools/lint/android/lints.py @@ -454,9 +454,7 @@ def read_lint_report(config, subdir, tasks=[], **lintargs): open( os.path.join(reports, "lint", file), ) - ).get( - "runs", [{}] - )[0] + ).get("runs", [{}])[0] issues = data.get("results", []) rules = data.get("tool", {}).get("driver", {}).get("rules", []) diff --git a/tools/lint/cargo-audit/__init__.py b/tools/lint/cargo-audit/__init__.py @@ -275,19 +275,16 @@ def setup(root, log, **lintargs) -> int: installed_version = get_audit_version() if not installed_version or installed_version != desired_version: - - output = run_process( - [ - "cargo", - "install", - "--locked", - "--version", - desired_version, - "--color", - "never", - "cargo-audit", - ] - ) + output = run_process([ + "cargo", + "install", + "--locked", + "--version", + desired_version, + "--color", + "never", + "cargo-audit", + ]) if not which("cargo-audit") or get_audit_version() != desired_version: log.error(f"Could not install cargo-audit:\n{output}") return 1 diff --git a/tools/lint/eslint/__init__.py b/tools/lint/eslint/__init__.py @@ -69,9 +69,10 @@ def lint(paths, config, binary=None, fix=None, rules=[], setup=None, **lintargs) if not lintargs.get("formatonly", False): exclude_args = [] for path in config.get("exclude", []): - exclude_args.extend( - ["--ignore-pattern", os.path.relpath(path, lintargs["root"])] - ) + exclude_args.extend([ + "--ignore-pattern", + os.path.relpath(path, lintargs["root"]), + ]) for rule in rules: extra_args.extend(["--rule", rule]) @@ -188,15 +189,13 @@ def run(cmd_args, config): fixed = fixed + 1 for err in errors: - err.update( - { - "hint": err.get("fix"), - "level": "error" if err["severity"] == 2 else "warning", - "lineno": err.get("line") or 0, - "path": obj["filePath"], - "rule": err.get("ruleId"), - } - ) + err.update({ + "hint": err.get("fix"), + "level": "error" if err["severity"] == 2 else "warning", + "lineno": err.get("line") or 0, + "path": obj["filePath"], + "rule": err.get("ruleId"), + }) results.append(result.from_config(config, **err)) return {"results": results, "fixed": fixed} diff --git a/tools/lint/eslint/prettier_utils.py b/tools/lint/eslint/prettier_utils.py @@ -63,7 +63,7 @@ def run_prettier(cmd_args, config, fix): "rule": "prettier", "lineno": 0, "column": 0, - } + }, ) ) @@ -103,7 +103,7 @@ def run_prettier(cmd_args, config, fix): "rule": "prettier", "lineno": 0, "column": 0, - } + }, ) ) diff --git a/tools/lint/fluent-lint/__init__.py b/tools/lint/fluent-lint/__init__.py @@ -128,8 +128,7 @@ class Linter(visitor.Visitor): "node_can_be_resource_comment" ] and ( # This is the root node, empty space is allowed, comments are allowed - node_name - in {"Resource", "Span", "Comment"} + node_name in {"Resource", "Span", "Comment"} ) if self.debug_print_json: @@ -472,9 +471,9 @@ def get_exclusions(root): ) as f: exclusions = list(yaml.safe_load_all(f))[0] for error_type in exclusions: - exclusions[error_type]["files"] = set( - [mozpath.join(root, x) for x in exclusions[error_type]["files"]] - ) + exclusions[error_type]["files"] = set([ + mozpath.join(root, x) for x in exclusions[error_type]["files"] + ]) return exclusions diff --git a/tools/lint/glean-parser/__init__.py b/tools/lint/glean-parser/__init__.py @@ -125,8 +125,9 @@ def lint(paths, config, fix=None, **lintargs): f" {metric_name}:", group_files ) else: - lineno, file_path = None, ( - group_files[0] if group_files else None + lineno, file_path = ( + None, + (group_files[0] if group_files else None), ) if file_path: diff --git a/tools/lint/includes/std.py b/tools/lint/includes/std.py @@ -119,7 +119,7 @@ api = { "u16string", "u32string", "wstring", - "pmr::string" "pmr::u8string", + "pmr::stringpmr::u8string", "pmr::u16string", "pmr::u32string", "pmr::wstring", diff --git a/tools/lint/libpref/__init__.py b/tools/lint/libpref/__init__.py @@ -95,15 +95,13 @@ def check_against(path, pref_names): def check_value_for_pref(some_pref, some_value, path): errors = [] if some_pref["value"] == some_value: - errors.append( - { - "path": path, - "message": some_pref["raw"], - "lineno": some_pref["line"], - "hint": "Remove the duplicate pref or add it to IGNORE_PREFS.", - "level": "error", - } - ) + errors.append({ + "path": path, + "message": some_pref["raw"], + "lineno": some_pref["line"], + "hint": "Remove the duplicate pref or add it to IGNORE_PREFS.", + "level": "error", + }) return errors @@ -115,14 +113,12 @@ def read_prefs(path): for lineno, line in enumerate(source, start=1): match = PATTERN.match(line) if match: - prefs.append( - { - "name": match.group("pref"), - "value": evaluate_pref(match.group("val")), - "line": lineno, - "raw": line, - } - ) + prefs.append({ + "name": match.group("pref"), + "value": evaluate_pref(match.group("val")), + "line": lineno, + "raw": line, + }) return prefs diff --git a/tools/lint/mach_commands.py b/tools/lint/mach_commands.py @@ -50,13 +50,11 @@ def get_global_excludes(**lintargs): topsrcdir = lintargs["root"] # exclude top level paths that look like objdirs - excludes.extend( - [ - name - for name in os.listdir(topsrcdir) - if name.startswith("obj") and os.path.isdir(name) - ] - ) + excludes.extend([ + name + for name in os.listdir(topsrcdir) + if name.startswith("obj") and os.path.isdir(name) + ]) if lintargs.get("include_third-party"): # For some linters, we want to include the thirdparty code too. @@ -175,7 +173,7 @@ def eslint(command_context, paths, extra_args=[], **kwargs): linters=["eslint"], paths=paths, argv=extra_args, - **kwargs + **kwargs, ) @@ -204,7 +202,7 @@ def prettier(command_context, paths, extra_args=[], **kwargs): linters=["eslint", "stylelint"], paths=paths, argv=extra_args, - **kwargs + **kwargs, ) diff --git a/tools/lint/node-licenses/__init__.py b/tools/lint/node-licenses/__init__.py @@ -77,7 +77,7 @@ def lint(paths, config, binary=None, skip_reinstall=False, **lintargs): "path": path, "message": "Unable to install node_modules for this package, try running 'npm ci' in the directory to debug", "level": "error", - } + }, ) ) diff --git a/tools/lint/perfdocs/framework_gatherers.py b/tools/lint/perfdocs/framework_gatherers.py @@ -349,7 +349,7 @@ class RaptorGatherer(FrameworkGatherer): for idx, description in enumerate(matcher): if description["name"] != title: - result += f" {idx+1}. **{description['name']}**\n\n" + result += f" {idx + 1}. **{description['name']}**\n\n" if "owner" in description.keys(): result += f" **Owner**: {description['owner']}\n\n" if test_description: @@ -415,7 +415,7 @@ class RaptorGatherer(FrameworkGatherer): if match_run_on_projects( {"project": x}, task["run_on_projects"] ) - else "\u274C" + else "\u274c" ) for x in BRANCHES ] @@ -505,9 +505,9 @@ class MozperftestGatherer(FrameworkGatherer): cleaned_name = si["name"].replace(".", "") self.script_infos[cleaned_name] = si - self._test_list.setdefault(suite_name.replace("\\", "/"), {}).update( - {cleaned_name: {"path": str(path)}} - ) + self._test_list.setdefault(suite_name.replace("\\", "/"), {}).update({ + cleaned_name: {"path": str(path)} + }) return self._test_list @@ -635,7 +635,7 @@ class TalosGatherer(FrameworkGatherer): if match_run_on_projects( {"project": x}, task["run_on_projects"] ) - else "\u274C" + else "\u274c" ) for x in BRANCHES ] diff --git a/tools/lint/perfdocs/generator.py b/tools/lint/perfdocs/generator.py @@ -139,22 +139,18 @@ class Generator: # For static `.rst` file for static_file in framework["static"]: if static_file.endswith("rst"): - frameworks_info[yaml_content["name"]]["static"].append( - { - "file": static_file, - "content": read_file( - pathlib.Path(framework["path"], static_file), - stringify=True, - ), - } - ) + frameworks_info[yaml_content["name"]]["static"].append({ + "file": static_file, + "content": read_file( + pathlib.Path(framework["path"], static_file), + stringify=True, + ), + }) else: - frameworks_info[yaml_content["name"]]["static"].append( - { - "file": static_file, - "content": pathlib.Path(framework["path"], static_file), - } - ) + frameworks_info[yaml_content["name"]]["static"].append({ + "file": static_file, + "content": pathlib.Path(framework["path"], static_file), + }) return frameworks_info @@ -281,12 +277,10 @@ class Generator: # (used to output a linter warning for regeneration) files = [] for entry in self._perfdocs_tree: - files.extend( - [ - pathlib.Path(entry["path"], entry["yml"]), - pathlib.Path(entry["path"], entry["rst"]), - ] - ) + files.extend([ + pathlib.Path(entry["path"], entry["yml"]), + pathlib.Path(entry["path"], entry["rst"]), + ]) return files # Throw a warning if there's no need for generating diff --git a/tools/lint/rst/__init__.py b/tools/lint/rst/__init__.py @@ -26,17 +26,13 @@ RSTCHECK_NOT_FOUND = """ Could not find rstcheck! Install rstcheck and try again. $ pip install -U --require-hashes -r {} -""".strip().format( - rstcheck_requirements_file -) +""".strip().format(rstcheck_requirements_file) RSTCHECK_INSTALL_ERROR = """ Unable to install required version of rstcheck Try to install it manually with: $ pip install -U --require-hashes -r {} -""".strip().format( - rstcheck_requirements_file -) +""".strip().format(rstcheck_requirements_file) RSTCHECK_FORMAT_REGEX = re.compile(r"(.*):(.*): \(.*/([0-9]*)\) (.*)$") diff --git a/tools/lint/stylelint/__init__.py b/tools/lint/stylelint/__init__.py @@ -83,9 +83,10 @@ def lint(paths, config, binary=None, fix=None, rules=[], setup=None, **lintargs) if not lintargs.get("formatonly", False): exclude_args = [] for path in config.get("exclude", []): - exclude_args.extend( - ["--ignore-pattern", os.path.relpath(path, lintargs["root"])] - ) + exclude_args.extend([ + "--ignore-pattern", + os.path.relpath(path, lintargs["root"]), + ]) # Default to $topsrcdir/.stylelintrc.js, but allow override in stylelint.yml stylelint_rc = config.get("stylelint-rc", ".stylelintrc.js") @@ -205,15 +206,13 @@ def run(cmd_args, config, fix): # All mozlint formatters that include the error message also already # separately include the rule id, so that leads to duplication. Fix: msg = msg.replace("(" + err.get("rule") + ")", "").strip() - err.update( - { - "message": msg, - "level": err.get("severity") or "error", - "lineno": err.get("line") or 0, - "path": obj["source"], - "rule": err.get("rule") or "parseError", - } - ) + err.update({ + "message": msg, + "level": err.get("severity") or "error", + "lineno": err.get("line") or 0, + "path": obj["source"], + "rule": err.get("rule") or "parseError", + }) results.append(result.from_config(config, **err)) return {"results": results, "fixed": fixed} diff --git a/tools/lint/test/conftest.py b/tools/lint/test/conftest.py @@ -279,25 +279,38 @@ def perfdocs_sample(): raptor_suitedir.mkdir(parents=True, exist_ok=True) raptor_another_suitedir.mkdir(parents=True, exist_ok=True) - with temp_file( - "perftest.toml", tempdir=suite_dir, content='["perftest_sample.js"]' - ) as tmpmanifest, temp_file( - "raptor_example1.ini", tempdir=raptor_suitedir, content=SAMPLE_INI - ) as tmpexample1manifest, temp_file( - "raptor_example2.ini", tempdir=raptor_another_suitedir, content=SAMPLE_INI - ) as tmpexample2manifest, temp_file( - "perftest_sample.js", tempdir=suite_dir, content=SAMPLE_TEST - ) as tmptest, temp_file( - "config.yml", tempdir=perfdocs_dir, content=SAMPLE_CONFIG - ) as tmpconfig, temp_file( - "config_2.yml", tempdir=perfdocs_dir, content=DYNAMIC_SAMPLE_CONFIG - ) as tmpconfig_2, temp_file( - "config_metrics.yml", tempdir=perfdocs_dir, content=SAMPLE_METRICS_CONFIG - ) as tmpconfig_metrics, temp_file( - "index.rst", - tempdir=perfdocs_dir, - content="{metrics_rst_name}{documentation}", - ) as tmpindex: + with ( + temp_file( + "perftest.toml", tempdir=suite_dir, content='["perftest_sample.js"]' + ) as tmpmanifest, + temp_file( + "raptor_example1.ini", tempdir=raptor_suitedir, content=SAMPLE_INI + ) as tmpexample1manifest, + temp_file( + "raptor_example2.ini", + tempdir=raptor_another_suitedir, + content=SAMPLE_INI, + ) as tmpexample2manifest, + temp_file( + "perftest_sample.js", tempdir=suite_dir, content=SAMPLE_TEST + ) as tmptest, + temp_file( + "config.yml", tempdir=perfdocs_dir, content=SAMPLE_CONFIG + ) as tmpconfig, + temp_file( + "config_2.yml", tempdir=perfdocs_dir, content=DYNAMIC_SAMPLE_CONFIG + ) as tmpconfig_2, + temp_file( + "config_metrics.yml", + tempdir=perfdocs_dir, + content=SAMPLE_METRICS_CONFIG, + ) as tmpconfig_metrics, + temp_file( + "index.rst", + tempdir=perfdocs_dir, + content="{metrics_rst_name}{documentation}", + ) as tmpindex, + ): yield { "top_dir": tmpdir, "manifest": {"path": tmpmanifest}, diff --git a/tools/lint/test/test_condprof_addons.py b/tools/lint/test/test_condprof_addons.py @@ -43,8 +43,11 @@ def test_get_missing_xpi_msg(lint, paths): def test_xpi_missing_from_firefox_addons_tar(lint, paths): fixture_customizations = paths("with-missing-xpi.json") - with linter_module_mocks(), linter_class_mocks( - get_firefox_addons_tar_names=mock.Mock(return_value=list()), + with ( + linter_module_mocks(), + linter_class_mocks( + get_firefox_addons_tar_names=mock.Mock(return_value=list()), + ), ): logger_mock = mock.Mock() lint(fixture_customizations, logger=logger_mock) @@ -69,8 +72,11 @@ def test_xpi_all_found_in_firefox_addons_tar(lint, paths): } ) - with linter_module_mocks(), linter_class_mocks( - get_firefox_addons_tar_names=get_tarnames_mock, read_json=read_json_mock + with ( + linter_module_mocks(), + linter_class_mocks( + get_firefox_addons_tar_names=get_tarnames_mock, read_json=read_json_mock + ), ): logger_mock = mock.Mock() # Compute a fake condprof customization path, the content is @@ -94,8 +100,9 @@ def test_lint_error_on_missing_or_invalid_firefoxaddons_fetch_task( read_json_mock = mock.Mock(return_value=dict()) read_yaml_mock = mock.Mock(return_value=dict()) # Verify that an explicit linter error is reported if the fetch task is not found. - with linter_module_mocks(), linter_class_mocks( - read_json=read_json_mock, read_yaml=read_yaml_mock + with ( + linter_module_mocks(), + linter_class_mocks(read_json=read_json_mock, read_yaml=read_yaml_mock), ): logger_mock = mock.Mock() fixture_customizations = paths("fake-condprof-config.json") @@ -162,11 +169,15 @@ def test_get_xpi_list_from_downloaded_tar(lint, paths): # Verify that when executed locally on a developer machine, the tar archive is downloaded # and the list of xpi files included in it returned by the helper method. - with tempfile.TemporaryDirectory() as tempdir, linter_module_mocks( - MOZ_AUTOMATION=0, - tempdir=tempdir, - ), linter_class_mocks( - download_firefox_addons_tar=download_firefox_addons_tar_mock, + with ( + tempfile.TemporaryDirectory() as tempdir, + linter_module_mocks( + MOZ_AUTOMATION=0, + tempdir=tempdir, + ), + linter_class_mocks( + download_firefox_addons_tar=download_firefox_addons_tar_mock, + ), ): condprof_addons = importlib.import_module("condprof-addons") logger_mock = mock.Mock() @@ -186,8 +197,9 @@ def test_get_xpi_list_from_downloaded_tar(lint, paths): def test_error_on_downloading_tar(requests_get_mock, lint, paths): # Verify that when executed locally and the tar archive fails to download # the linter does report an explicit linting error with the http error included. - with tempfile.TemporaryDirectory() as tempdir, linter_module_mocks( - MOZ_AUTOMATION=0, tempdir=tempdir + with ( + tempfile.TemporaryDirectory() as tempdir, + linter_module_mocks(MOZ_AUTOMATION=0, tempdir=tempdir), ): condprof_addons = importlib.import_module("condprof-addons") logger_mock = mock.Mock() @@ -218,8 +230,9 @@ def test_error_on_downloading_tar(requests_get_mock, lint, paths): def test_error_on_opening_tar(requests_get_mock, lint, paths): # Verify that when executed locally and the tar archive fails to open # the linter does report an explicit linting error with the tarfile error included. - with tempfile.TemporaryDirectory() as tempdir, linter_module_mocks( - MOZ_AUTOMATION=0, tempdir=tempdir + with ( + tempfile.TemporaryDirectory() as tempdir, + linter_module_mocks(MOZ_AUTOMATION=0, tempdir=tempdir), ): condprof_addons = importlib.import_module("condprof-addons") logger_mock = mock.Mock() @@ -261,11 +274,14 @@ def test_lint_all_customization_files_when_linting_browsertime_yml( "addons": {"an-extension": "http://localhost/ext/an-extension.xpi"} } ) - with linter_module_mocks( - customizations_path="fake-customizations-dir", - ), linter_class_mocks( - get_firefox_addons_tar_names=get_tarnames_mock, - read_json=read_json_mock, + with ( + linter_module_mocks( + customizations_path="fake-customizations-dir", + ), + linter_class_mocks( + get_firefox_addons_tar_names=get_tarnames_mock, + read_json=read_json_mock, + ), ): logger_mock = mock.Mock() importlib.import_module("condprof-addons") diff --git a/tools/lint/test/test_includes.py b/tools/lint/test/test_includes.py @@ -39,9 +39,9 @@ def test_lint_api_yml(lint): mfbt_dir = os.path.join(topsrcdir, "mfbt") for header, categories in description.items(): header_path = os.path.join(mfbt_dir, header) - assert os.path.exists( - header_path - ), f"{header} described in {api_yaml}, but missing in mfbt/" + assert os.path.exists(header_path), ( + f"{header} described in {api_yaml}, but missing in mfbt/" + ) with open(header_path) as fd: header_content = fd.read() @@ -55,9 +55,9 @@ def test_lint_api_yml(lint): symbol_found = re.search( category_re[category].format(symbol), header_content ) - assert ( - symbol_found - ), f"{symbol} described as a {category} available in {header}, but cannot be found there" + assert symbol_found, ( + f"{symbol} described as a {category} available in {header}, but cannot be found there" + ) def test_lint_mfbt_includes(lint, paths): diff --git a/tools/lint/test/test_perfdocs_generation.py b/tools/lint/test/test_perfdocs_generation.py @@ -83,18 +83,19 @@ def test_perfdocs_generator_generate_perfdocs_metrics_pass( "perfdocs.framework_gatherers.RaptorGatherer.get_test_list" ) as m: m.return_value = sample_gatherer_result - with perfdocs_sample["config"].open("w") as f1, perfdocs_sample[ - "config_metrics" - ].open("r") as f2: + with ( + perfdocs_sample["config"].open("w") as f1, + perfdocs_sample["config_metrics"].open("r") as f2, + ): # Overwrite content of config.yml with metrics config f1.write(f2.read()) verifier = Verifier(top_dir) verifier.validate_tree() - verifier._gatherer.framework_gatherers["raptor"]._descriptions = ( - sample_test_list_result - ) + verifier._gatherer.framework_gatherers[ + "raptor" + ]._descriptions = sample_test_list_result generator = Generator(verifier, generate=True, workspace=top_dir) with temp_file( diff --git a/tools/lint/test/test_perfdocs_helpers.py b/tools/lint/test/test_perfdocs_helpers.py @@ -150,9 +150,12 @@ def test_table_builder_add_row_too_long(): table_specifications["indent"], ) with pytest.raises(MismatchedRowLengthsException) as error: - table.add_row( - ["big ones", "small ones", "some as big as your head!", "(and bigger)"] - ) + table.add_row([ + "big ones", + "small ones", + "some as big as your head!", + "(and bigger)", + ]) assert ( str(error.value) == "Number of items in a row must must number of columns defined." @@ -170,9 +173,12 @@ def test_table_builder_add_rows_type_error(): table_specifications["indent"], ) with pytest.raises(TypeError) as error: - table.add_rows( - ["big ones", "small ones", "some as big as your head!", "(and bigger)"] - ) + table.add_rows([ + "big ones", + "small ones", + "some as big as your head!", + "(and bigger)", + ]) assert str(error.value) == "add_rows() requires a two-dimensional list of strings." @@ -187,9 +193,11 @@ def test_table_builder_validate(): table_specifications["indent"], ) table.add_row(["big ones", "small ones", "some as big as your head!"]) - table.add_row( - ["Give 'em a twist", "A flick of the wrist", "That's what the showman said!"] - ) + table.add_row([ + "Give 'em a twist", + "A flick of the wrist", + "That's what the showman said!", + ]) table = table.finish_table() print(table) assert ( diff --git a/tools/lint/wpt/wpt.py b/tools/lint/wpt/wpt.py @@ -37,7 +37,7 @@ def lint(files, config, **kwargs): if files == [tests_dir]: print( - "No specific files specified, running the full wpt lint" " (this is slow)", + "No specific files specified, running the full wpt lint (this is slow)", file=sys.stderr, ) files = ["--all"] diff --git a/tools/mach_commands.py b/tools/mach_commands.py @@ -176,15 +176,13 @@ class PypiBasedTool: print(release) # there is one, so install it. Note that install_pip_package # does not work here, so just run pip directly. - subprocess.check_call( - [ - cmd.virtualenv_manager.python_path, - "-m", - "pip", - "install", - f"{self.pypi_name}=={release}", - ] - ) + subprocess.check_call([ + cmd.virtualenv_manager.python_path, + "-m", + "pip", + "install", + f"{self.pypi_name}=={release}", + ]) print( "%s was updated to version %s. please" " re-run your command." % (self.pypi_name, release) diff --git a/tools/moztreedocs/__init__.py b/tools/moztreedocs/__init__.py @@ -186,12 +186,10 @@ class _SphinxManager: # tree (Bug 1557020). The page is no longer referenced within the index # tree, thus we shall check categorisation only if complete tree is being rebuilt. if app.srcdir == self.topsrcdir: - indexes = set( - [ - os.path.normpath(os.path.join(p, "index")) - for p in toplevel_trees.keys() - ] - ) + indexes = set([ + os.path.normpath(os.path.join(p, "index")) + for p in toplevel_trees.keys() + ]) # Format categories like indexes cats = "\n".join(CATEGORIES.values()).split("\n") # Remove heading spaces diff --git a/tools/moztreedocs/mach_commands.py b/tools/moztreedocs/mach_commands.py @@ -474,7 +474,7 @@ def _s3_upload(root, project, unique_id, version=None): def generate_telemetry_docs(command_context): args = [ sys.executable, - "-m" "glean_parser", + "-mglean_parser", "translate", "-f", "markdown", @@ -489,9 +489,9 @@ def generate_telemetry_docs(command_context): for handler in Registrar.command_handlers.values() if handler.metrics_path is not None ] - args.extend( - [os.path.join(command_context.topsrcdir, path) for path in set(metrics_paths)] - ) + args.extend([ + os.path.join(command_context.topsrcdir, path) for path in set(metrics_paths) + ]) subprocess.check_call(args) diff --git a/tools/rb/find_leakers.py b/tools/rb/find_leakers.py @@ -50,9 +50,7 @@ def process_log(log_lines): ignore, operation, count, - ) = log_line.strip("\r\n").split( - " " - )[:5] + ) = log_line.strip("\r\n").split(" ")[:5] # for AddRef/Release `count' is the refcount, # for Ctor/Dtor it's the size. diff --git a/tools/signing/macos/mach_commands.py b/tools/signing/macos/mach_commands.py @@ -131,8 +131,7 @@ from mozbuild.base import MachCommandConditions as conditions default=None, type=str, dest="p12_password_file_arg", - help="The rcodesign pkcs12 password file, passed to rcodesign without " - "validation.", + help="The rcodesign pkcs12 password file, passed to rcodesign without validation.", ) def macos_sign( command_context, @@ -175,7 +174,7 @@ def macos_sign( logging.ERROR, "macos-sign", {}, - "ERROR: p12 password file with no p12 file, " "use both or neither", + "ERROR: p12 password file with no p12 file, use both or neither", ) sys.exit(1) if p12_file_arg is not None and p12_password_file_arg is None: @@ -183,7 +182,7 @@ def macos_sign( logging.ERROR, "macos-sign", {}, - "ERROR: p12 file with no p12 password file, " "use both or neither", + "ERROR: p12 file with no p12 password file, use both or neither", ) sys.exit(1) if p12_file_arg is not None and p12_password_file_arg is not None: @@ -197,8 +196,7 @@ def macos_sign( logging.ERROR, "macos-sign", {}, - "ERROR: pkcs12 signing not supported with " - "native codesign, only rcodesign", + "ERROR: pkcs12 signing not supported with native codesign, only rcodesign", ) sys.exit(1) @@ -218,7 +216,7 @@ def macos_sign( logging.ERROR, "macos-sign", {}, - "ERROR: rcodesign requires pkcs12 or " "ad-hoc signing", + "ERROR: rcodesign requires pkcs12 or ad-hoc signing", ) sys.exit(1) @@ -228,7 +226,7 @@ def macos_sign( logging.ERROR, "macos-sign", {}, - "ERROR: both ad-hoc and pkcs12 signing " "requested", + "ERROR: both ad-hoc and pkcs12 signing requested", ) sys.exit(1) @@ -242,7 +240,7 @@ def macos_sign( logging.ERROR, "macos-sign", {}, - "ERROR: " "Production entitlements and self-signing are " "not compatible", + "ERROR: Production entitlements and self-signing are not compatible", ) sys.exit(1) diff --git a/tools/tryselect/selectors/coverage.py b/tools/tryselect/selectors/coverage.py @@ -174,15 +174,13 @@ def download_coverage_mapping(base_revision): r.raw.decode_content = True shutil.copyfileobj(r.raw, f) - subprocess.check_call( - [ - "tar", - "-xJf", - CHUNK_MAPPING_ARCHIVE, - "-C", - os.path.dirname(CHUNK_MAPPING_FILE), - ] - ) + subprocess.check_call([ + "tar", + "-xJf", + CHUNK_MAPPING_ARCHIVE, + "-C", + os.path.dirname(CHUNK_MAPPING_FILE), + ]) os.remove(CHUNK_MAPPING_ARCHIVE) assert os.path.isfile(CHUNK_MAPPING_FILE) with open(CHUNK_MAPPING_TAG_FILE, "w") as f: @@ -354,10 +352,10 @@ def filter_tasks_by_chunks(tasks, chunks): ): continue - assert ( - selected_task is None - ), "Only one task should be selected for a given platform-chunk couple ({} - {}), {} and {} were selected".format( # noqa - platform, chunk, selected_task, task + assert selected_task is None, ( + "Only one task should be selected for a given platform-chunk couple ({} - {}), {} and {} were selected".format( # noqa + platform, chunk, selected_task, task + ) ) selected_task = task diff --git a/tools/tryselect/selectors/perf.py b/tools/tryselect/selectors/perf.py @@ -563,9 +563,9 @@ class PerfParser(CompareParser): # Disable the variant combination if none of them # are found in the suite - disable_variant = not any( - [variant.value in suite_variants for variant in variant_combination] - ) + disable_variant = not any([ + variant.value in suite_variants for variant in variant_combination + ]) for platform in Platforms: if disable_variant: @@ -628,20 +628,18 @@ class PerfParser(CompareParser): if BASE_CATEGORY_NAME not in variant_combination: # Make sure that all portions of the variant combination # target at least one of the suites in the category - tmp_variant_combination = set( - [v.value for v in variant_combination] - ) + tmp_variant_combination = set([ + v.value for v in variant_combination + ]) for suite in Suites: if suite.value not in category_info["suites"]: continue - tmp_variant_combination = tmp_variant_combination - set( - [ - variant.value - for variant in variant_combination - if variant.value - in PerfParser.suites[suite.value]["variants"] - ] - ) + tmp_variant_combination = tmp_variant_combination - set([ + variant.value + for variant in variant_combination + if variant.value + in PerfParser.suites[suite.value]["variants"] + ]) if tmp_variant_combination: # If it's not empty, then some variants # are non-existent @@ -1032,8 +1030,7 @@ class PerfParser(CompareParser): if len(mwu_task) > 1 or len(mwu_task) == 0: raise InvalidRegressionDetectorQuery( - f"Expected 1 task from change detector " - f"query, but found {len(mwu_task)}" + f"Expected 1 task from change detector query, but found {len(mwu_task)}" ) selected_tasks |= set(mwu_task) diff --git a/tools/tryselect/selectors/perf_preview.py b/tools/tryselect/selectors/perf_preview.py @@ -4,7 +4,6 @@ """This script is intended to be called through fzf as a preview formatter.""" - import argparse import json import os diff --git a/tools/tryselect/selectors/preview.py b/tools/tryselect/selectors/preview.py @@ -4,7 +4,6 @@ """This script is intended to be called through fzf as a preview formatter.""" - import argparse import os import sys diff --git a/tools/tryselect/selectors/release.py b/tools/tryselect/selectors/release.py @@ -117,13 +117,11 @@ def run( elif release_type == "esr": release_type += str(version.major_number) task_config = {"version": 2, "parameters": try_config_params or {}} - task_config["parameters"].update( - { - "target_tasks_method": TARGET_TASKS[tasks], - "optimize_target_tasks": True, - "release_type": release_type, - } - ) + task_config["parameters"].update({ + "target_tasks_method": TARGET_TASKS[tasks], + "optimize_target_tasks": True, + "release_type": release_type, + }) with open( os.path.join(vcs.path, "taskcluster/kinds/merge-automation/kind.yml") diff --git a/tools/tryselect/test/test_auto.py b/tools/tryselect/test/test_auto.py @@ -12,9 +12,10 @@ def test_strategy_validation(): args = parser.parse_args(["--strategy", "relevant_tests"]) assert args.strategy == "gecko_taskgraph.optimize:tryselect.relevant_tests" - args = parser.parse_args( - ["--strategy", "gecko_taskgraph.optimize:experimental.relevant_tests"] - ) + args = parser.parse_args([ + "--strategy", + "gecko_taskgraph.optimize:experimental.relevant_tests", + ]) assert args.strategy == "gecko_taskgraph.optimize:experimental.relevant_tests" with pytest.raises(SystemExit): diff --git a/tools/tryselect/test/test_mozharness_integration.py b/tools/tryselect/test/test_mozharness_integration.py @@ -74,9 +74,11 @@ def all_suites(): all_suites.append({"flavor": flavor, "srcdir_relpath": "test"}) for flavor, subsuite in _test_subsuites: - all_suites.append( - {"flavor": flavor, "subsuite": subsuite, "srcdir_relpath": "test"} - ) + all_suites.append({ + "flavor": flavor, + "subsuite": subsuite, + "srcdir_relpath": "test", + }) return all_suites @@ -96,7 +98,7 @@ def generate_suites_from_config(path): for category in sorted(config["suite_definitions"]): key = f"all_{category}_suites" if key not in config: - yield category, + yield (category,) continue for suite in sorted(config[f"all_{category}_suites"]): diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py @@ -880,9 +880,9 @@ def test_category_expansion( assert len(expanded_cats) == expected_counts assert not any([expanded_cats.get(ucat, None) is not None for ucat in missing]) - assert all( - [expanded_cats.get(ucat, None) is not None for ucat in unique_categories.keys()] - ) + assert all([ + expanded_cats.get(ucat, None) is not None for ucat in unique_categories.keys() + ]) # Ensure that the queries are as expected for cat_name, cat_query in unique_categories.items(): @@ -1052,25 +1052,31 @@ def test_category_expansion_with_non_pgo_flag(category_options, call_counts): ) @pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") def test_full_run(options, call_counts, log_ind, expected_log_message): - with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( - "tryselect.selectors.perf.run_fzf" - ) as fzf, mock.patch( - "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() - ), mock.patch( - "tryselect.selectors.perf.LogProcessor.revision", - new_callable=mock.PropertyMock, - return_value="revision", - ) as logger, mock.patch( - "tryselect.selectors.perf.PerfParser.check_cached_revision", - ) as ccr, mock.patch( - "tryselect.selectors.perf.PerfParser.save_revision_treeherder" - ) as srt, mock.patch( - "tryselect.selectors.perf.print", - ) as perf_print, mock.patch( - "tryselect.selectors.perf.PerfParser.set_categories_for_test" - ) as tests_mock, mock.patch( - "tryselect.selectors.perf.requests" - ) as requests_mock: + with ( + mock.patch("tryselect.selectors.perf.push_to_try") as ptt, + mock.patch("tryselect.selectors.perf.run_fzf") as fzf, + mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), + mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, + mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision", + ) as ccr, + mock.patch( + "tryselect.selectors.perf.PerfParser.save_revision_treeherder" + ) as srt, + mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print, + mock.patch( + "tryselect.selectors.perf.PerfParser.set_categories_for_test" + ) as tests_mock, + mock.patch("tryselect.selectors.perf.requests") as requests_mock, + ): def test_mock_func(*args, **kwargs): """Used for testing any --test functionality.""" @@ -1170,25 +1176,31 @@ def test_full_run(options, call_counts, log_ind, expected_log_message): ) @pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") def test_full_run_lando(options, call_counts, log_ind, expected_log_message): - with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( - "tryselect.selectors.perf.run_fzf" - ) as fzf, mock.patch( - "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() - ), mock.patch( - "tryselect.selectors.perf.LogProcessor.revision", - new_callable=mock.PropertyMock, - return_value="revision", - ) as logger, mock.patch( - "tryselect.selectors.perf.PerfParser.check_cached_revision", - ) as ccr, mock.patch( - "tryselect.selectors.perf.PerfParser.save_revision_treeherder" - ) as srt, mock.patch( - "tryselect.selectors.perf.print", - ) as perf_print, mock.patch( - "tryselect.selectors.perf.PerfParser.set_categories_for_test" - ) as tests_mock, mock.patch( - "tryselect.selectors.perf.requests" - ) as requests_mock: + with ( + mock.patch("tryselect.selectors.perf.push_to_try") as ptt, + mock.patch("tryselect.selectors.perf.run_fzf") as fzf, + mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), + mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, + mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision", + ) as ccr, + mock.patch( + "tryselect.selectors.perf.PerfParser.save_revision_treeherder" + ) as srt, + mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print, + mock.patch( + "tryselect.selectors.perf.PerfParser.set_categories_for_test" + ) as tests_mock, + mock.patch("tryselect.selectors.perf.requests") as requests_mock, + ): def test_mock_func(*args, **kwargs): """Used for testing any --test functionality.""" @@ -1295,19 +1307,22 @@ def test_change_detection_task_injection_failure( ): setup_perfparser() - with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( - "tryselect.selectors.perf.run_fzf" - ) as fzf, mock.patch( - "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() - ), mock.patch( - "tryselect.selectors.perf.LogProcessor.revision", - new_callable=mock.PropertyMock, - return_value="revision", - ) as logger, mock.patch( - "tryselect.selectors.perf.PerfParser.check_cached_revision" - ) as ccr, mock.patch( - "tryselect.selectors.perf.print", - ) as perf_print: + with ( + mock.patch("tryselect.selectors.perf.push_to_try") as ptt, + mock.patch("tryselect.selectors.perf.run_fzf") as fzf, + mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), + mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, + mock.patch("tryselect.selectors.perf.PerfParser.check_cached_revision") as ccr, + mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print, + ): fzf_side_effects = [ ["", ["Benchmarks linux"]], ["", TASKS], @@ -1391,9 +1406,10 @@ def test_category_rules(query, should_fail): ], ) def test_apk_upload(apk_name, apk_content, should_fail, failure_message): - with mock.patch("tryselect.selectors.perf.subprocess") as _, mock.patch( - "tryselect.selectors.perf.shutil" - ) as _: + with ( + mock.patch("tryselect.selectors.perf.subprocess") as _, + mock.patch("tryselect.selectors.perf.shutil") as _, + ): temp_dir = None try: temp_dir = tempfile.mkdtemp() @@ -1504,12 +1520,11 @@ def test_apk_upload(apk_name, apk_content, should_fail, failure_message): def test_check_cached_revision( args, load_data, return_value, call_counts, exists_cache_file ): - with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( - "tryselect.selectors.perf.json.dump" - ) as dump, mock.patch( - "tryselect.selectors.perf.pathlib.Path.is_file" - ) as is_file, mock.patch( - "tryselect.selectors.perf.pathlib.Path.open" + with ( + mock.patch("tryselect.selectors.perf.json.load") as load, + mock.patch("tryselect.selectors.perf.json.dump") as dump, + mock.patch("tryselect.selectors.perf.pathlib.Path.is_file") as is_file, + mock.patch("tryselect.selectors.perf.pathlib.Path.open"), ): load.return_value = load_data is_file.return_value = exists_cache_file @@ -1536,12 +1551,11 @@ def test_check_cached_revision( ], ) def test_save_revision_treeherder(args, call_counts, exists_cache_file): - with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( - "tryselect.selectors.perf.json.dump" - ) as dump, mock.patch( - "tryselect.selectors.perf.pathlib.Path.is_file" - ) as is_file, mock.patch( - "tryselect.selectors.perf.pathlib.Path.open" + with ( + mock.patch("tryselect.selectors.perf.json.load") as load, + mock.patch("tryselect.selectors.perf.json.dump") as dump, + mock.patch("tryselect.selectors.perf.pathlib.Path.is_file") as is_file, + mock.patch("tryselect.selectors.perf.pathlib.Path.open"), ): is_file.return_value = exists_cache_file @@ -1561,7 +1575,7 @@ def test_save_revision_treeherder(args, call_counts, exists_cache_file): [1, 0, 0, 1], ( "\n\n----------------------------------------------------------------------------------------------\n" - f"You have selected {MAX_PERF_TASKS+1} total test runs! (selected tasks({MAX_PERF_TASKS+1}) * rebuild" + f"You have selected {MAX_PERF_TASKS + 1} total test runs! (selected tasks({MAX_PERF_TASKS + 1}) * rebuild" f" count(1) \nThese tests won't be triggered as the current maximum for a single ./mach try " f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance." "\n----------------------------------------------------------------------------------------------\n\n" @@ -1607,23 +1621,28 @@ def test_max_perf_tasks( ): setup_perfparser() - with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( - "tryselect.selectors.perf.print", - ) as perf_print, mock.patch( - "tryselect.selectors.perf.LogProcessor.revision", - new_callable=mock.PropertyMock, - return_value="revision", - ), mock.patch( - "tryselect.selectors.perf.PerfParser.perf_push_to_try", - new_callable=mock.MagicMock, - ) as perf_push_to_try_mock, mock.patch( - "tryselect.selectors.perf.PerfParser.get_perf_tasks" - ) as get_perf_tasks_mock, mock.patch( - "tryselect.selectors.perf.PerfParser.get_tasks" - ) as get_tasks_mock, mock.patch( - "tryselect.selectors.perf.run_fzf" - ) as fzf, mock.patch( - "tryselect.selectors.perf.fzf_bootstrap", return_value=mock.MagicMock() + with ( + mock.patch("tryselect.selectors.perf.push_to_try") as ptt, + mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print, + mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ), + mock.patch( + "tryselect.selectors.perf.PerfParser.perf_push_to_try", + new_callable=mock.MagicMock, + ) as perf_push_to_try_mock, + mock.patch( + "tryselect.selectors.perf.PerfParser.get_perf_tasks" + ) as get_perf_tasks_mock, + mock.patch("tryselect.selectors.perf.PerfParser.get_tasks") as get_tasks_mock, + mock.patch("tryselect.selectors.perf.run_fzf") as fzf, + mock.patch( + "tryselect.selectors.perf.fzf_bootstrap", return_value=mock.MagicMock() + ), ): tasks = ["a-task"] * total_tasks get_tasks_mock.return_value = tasks @@ -1692,15 +1711,17 @@ def test_build_category_description(): ], ) def test_preview_description(options, call_count): - with mock.patch("tryselect.selectors.perf.PerfParser.perf_push_to_try"), mock.patch( - "tryselect.selectors.perf.fzf_bootstrap" - ), mock.patch( - "tryselect.selectors.perf.PerfParser.get_perf_tasks" - ) as get_perf_tasks, mock.patch( - "tryselect.selectors.perf.PerfParser.get_tasks" - ), mock.patch( - "tryselect.selectors.perf.PerfParser.build_category_description" - ) as bcd: + with ( + mock.patch("tryselect.selectors.perf.PerfParser.perf_push_to_try"), + mock.patch("tryselect.selectors.perf.fzf_bootstrap"), + mock.patch( + "tryselect.selectors.perf.PerfParser.get_perf_tasks" + ) as get_perf_tasks, + mock.patch("tryselect.selectors.perf.PerfParser.get_tasks"), + mock.patch( + "tryselect.selectors.perf.PerfParser.build_category_description" + ) as bcd, + ): get_perf_tasks.return_value = [], [], [] run(**options) @@ -1718,11 +1739,12 @@ def test_preview_description(options, call_count): taskfile = option[option.index("-t") + 1] - with mock.patch("tryselect.selectors.perf_preview.open"), mock.patch( - "tryselect.selectors.perf_preview.pathlib.Path.open" - ), mock.patch("tryselect.selectors.perf_preview.json.load") as load, mock.patch( - "tryselect.selectors.perf_preview.print" - ) as preview_print: + with ( + mock.patch("tryselect.selectors.perf_preview.open"), + mock.patch("tryselect.selectors.perf_preview.pathlib.Path.open"), + mock.patch("tryselect.selectors.perf_preview.json.load") as load, + mock.patch("tryselect.selectors.perf_preview.print") as preview_print, + ): load.return_value = {line: "test description"} plain_display(taskfile, description, line) @@ -1746,12 +1768,12 @@ def test_preview_description(options, call_count): ], ) def test_test_selection(tests, tasks_found, categories_produced): - with mock.patch( - "tryselect.selectors.perfselector.classification.pathlib" - ), mock.patch( - "tryselect.selectors.perfselector.classification.json" - ) as mocked_json, mock.patch( - "tryselect.selectors.perfselector.classification.ScriptInfo" + with ( + mock.patch("tryselect.selectors.perfselector.classification.pathlib"), + mock.patch( + "tryselect.selectors.perfselector.classification.json" + ) as mocked_json, + mock.patch("tryselect.selectors.perfselector.classification.ScriptInfo"), ): def mocked_json_load(*args, **kwargs): @@ -1782,12 +1804,15 @@ def test_test_selection(tests, tasks_found, categories_produced): ], ) def test_perftest_test_selection(tests, tasks_found, categories_produced): - with mock.patch("pathlib.Path.is_file", return_value=True), mock.patch( - "tryselect.selectors.perfselector.classification.ScriptInfo" - ) as mock_script_info, mock.patch( - "mozperftest.argparser.PerftestArgumentParser.parse_known_args" - ) as mock_parse_args: - + with ( + mock.patch("pathlib.Path.is_file", return_value=True), + mock.patch( + "tryselect.selectors.perfselector.classification.ScriptInfo" + ) as mock_script_info, + mock.patch( + "mozperftest.argparser.PerftestArgumentParser.parse_known_args" + ) as mock_parse_args, + ): mock_si_instance = mock_script_info.return_value mock_si_instance.get.return_value = "background-resource" mock_si_instance.script = pathlib.Path( diff --git a/tools/tryselect/test/test_presets.py b/tools/tryselect/test/test_presets.py @@ -36,7 +36,7 @@ def skip_taskgraph_generation(monkeypatch, tg): @pytest.mark.xfail( - strict=False, reason="Bug 1635204: " "test_shared_presets[sample-suites] is flaky" + strict=False, reason="Bug 1635204: test_shared_presets[sample-suites] is flaky" ) def test_shared_presets(run_mach, shared_name, shared_preset): """This test makes sure that we don't break any of the in-tree presets when diff --git a/tools/tryselect/test/test_selectors.py b/tools/tryselect/test/test_selectors.py @@ -73,26 +73,24 @@ def target_task_set(): @pytest.fixture def full_task_set(target_task_set): full_task_set = deepcopy(target_task_set) - full_task_set.update( - { - "test/bar-opt": { - "kind": "test", - "label": "test/bar-opt", - "attributes": {}, - "task": {}, - "optimization": {}, - "dependencies": {}, - }, - "test/bar-debug": { - "kind": "test", - "label": "test/bar-debug", - "attributes": {}, - "task": {}, - "optimization": {}, - "dependencies": {}, - }, - } - ) + full_task_set.update({ + "test/bar-opt": { + "kind": "test", + "label": "test/bar-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {}, + }, + "test/bar-debug": { + "kind": "test", + "label": "test/bar-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {}, + }, + }) return full_task_set @@ -686,8 +684,8 @@ def full_task_set(target_task_set): "fuzzy", [ ["try", "fuzzy", "--save", "foo", "-q", "'foo", "--rebuild", "5"], - ["try", "fuzzy", "--preset", "foo", "-q" "'build"], - ["try", "fuzzy", "--preset", "foo", "-xq" "'opt"], + ["try", "fuzzy", "--preset", "foo", "-q'build"], + ["try", "fuzzy", "--preset", "foo", "-xq'opt"], ], dedent( """ diff --git a/tools/tryselect/test/test_task_configs.py b/tools/tryselect/test/test_task_configs.py @@ -127,7 +127,7 @@ TASK_CONFIG_TESTS = { ( [ "--worker-override", - "b-linux=worker/pool" "--worker-suffix", + "b-linux=worker/pool--worker-suffix", "b-linux=-dev", ], SystemExit, diff --git a/tools/tryselect/util/fzf.py b/tools/tryselect/util/fzf.py @@ -317,9 +317,9 @@ def format_header(): def run_fzf(cmd, tasks): env = dict(os.environ) - env.update( - {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])} - ) + env.update({ + "PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p]) + }) # Make sure fzf uses Windows' shell rather than MozillaBuild bash or # whatever our caller uses, since it doesn't quote the arguments properly # and thus windows paths like: C:\moz\foo end up as C:mozfoo... diff --git a/tools/ts/mach_commands.py b/tools/ts/mach_commands.py @@ -160,9 +160,11 @@ def subs(ctx): def tsc(ctx, *args): - return ctx._sub_mach( - ["node", os.path.join("node_modules", "typescript", "bin", "tsc"), *args] - ) + return ctx._sub_mach([ + "node", + os.path.join("node_modules", "typescript", "bin", "tsc"), + *args, + ]) def node(ctx, script, *args): diff --git a/tools/update-programs/mach_commands.py b/tools/update-programs/mach_commands.py @@ -15,7 +15,6 @@ $ ./mach update serve -v https://archive.mozilla.org/pub/firefox/candidates/135. ``` """ - import json import logging import os diff --git a/tools/update-verify/scripts/async_download.py b/tools/update-verify/scripts/async_download.py @@ -105,7 +105,7 @@ async def fetch_url(url, path, connector): break fd.write(chunk) result = _result(response) - log.info(f'Finished downloading {url}\n{result["headers"]}') + log.info(f"Finished downloading {url}\n{result['headers']}") return result except ( @@ -345,7 +345,7 @@ async def _download_from_config(verifyConfig): with open(cacheIndexPath) as cache: for ln, url in enumerate(cache.readlines()): line = url.replace("\n", "") - log.info(f"Line {ln+1}: {line}") + log.info(f"Line {ln + 1}: {line}") def download_from_config(verifyConfig): diff --git a/xpcom/base/ErrorList.py b/xpcom/base/ErrorList.py @@ -1381,9 +1381,7 @@ enum class nsresult : uint32_t {} }}; -""".format( - ",\n".join(items) - ) +""".format(",\n".join(items)) ) items = [] @@ -1397,9 +1395,7 @@ const nsresult ; #endif // ErrorList_h__ -""".format( - ",\n".join(items) - ) +""".format(",\n".join(items)) ) diff --git a/xpcom/components/gen_static_components.py b/xpcom/components/gen_static_components.py @@ -272,7 +272,7 @@ class ModuleEntry: if len(self.interfaces) > 255: raise Exception( - "JS service %s may not have more than 255 " "interfaces" % self.js_name + "JS service %s may not have more than 255 interfaces" % self.js_name ) self.interfaces_offset = len(interfaces) @@ -322,8 +322,7 @@ class ModuleEntry: for prop in ("init_method", "legacy_constructor", "headers"): if getattr(self, prop): error( - "JavaScript components may not specify a '%s' " - "property" % prop + "JavaScript components may not specify a '%s' property" % prop ) elif self.external: if self.constructor or self.legacy_constructor: @@ -349,7 +348,7 @@ class ModuleEntry: ) if self.overridable and not self.contract_ids: - error("Overridable components must specify at least one contract " "ID") + error("Overridable components must specify at least one contract ID") @property def contract_id(self): @@ -940,7 +939,7 @@ def gen_substs(manifests): lower_entry=lambda entry: entry.to_cxx(), return_type="const StaticModule*", return_entry=( - "return entry.CID().Equals(aKey) && entry.Active()" " ? &entry : nullptr;" + "return entry.CID().Equals(aKey) && entry.Active() ? &entry : nullptr;" ), key_type="const nsID&", key_bytes="reinterpret_cast<const char*>(&aKey)", diff --git a/xpcom/ds/StaticAtoms.py b/xpcom/ds/StaticAtoms.py @@ -2662,20 +2662,18 @@ def generate_nsgkatomlist_h(output, *ignore): "#undef small\n" "#endif\n\n" "// GK_ATOM(identifier, string, hash, is_ascii_lower, gecko_type, atom_type)\n" - + "".join( - [ - 'GK_ATOM(%s, "%s", 0x%08x, %s, %s, %s)\n' - % ( - a.ident, - a.string, - a.hash, - str(a.is_ascii_lowercase).lower(), - a.ty, - a.atom_type, - ) - for a in STATIC_ATOMS - ] - ) + + "".join([ + 'GK_ATOM(%s, "%s", 0x%08x, %s, %s, %s)\n' + % ( + a.ident, + a.string, + a.hash, + str(a.is_ascii_lowercase).lower(), + a.ty, + a.atom_type, + ) + for a in STATIC_ATOMS + ]) ) diff --git a/xpcom/ds/tools/make_dafsa.py b/xpcom/ds/tools/make_dafsa.py @@ -170,6 +170,7 @@ The bytes in the generated array has the following meaning: 9: 0x62 <char> label character 0x62 -> match "b" 10: 0x82 <return_value> 0x82 & 0x0F -> return 2 """ + import struct import sys diff --git a/xpcom/ds/tools/perfecthash.py b/xpcom/ds/tools/perfecthash.py @@ -227,32 +227,26 @@ class CGHelper: if idx and idx % 16 == 0: # 16 bases per line bases += "\n " bases += "%4d," % base - return ( - textwrap.dedent( - """\ + return textwrap.dedent( + """\ static const %s %s[] = { %s }; """ - ) - % (self.basis_ty(), name, bases) - ) + ) % (self.basis_ty(), name, bases) def gen_entries(self, lower_entry): """Generate code for an entries table""" entries = self._indent( ",\n".join(lower_entry(entry).rstrip() for entry in self.phf.entries) ) - return ( - textwrap.dedent( - """\ + return textwrap.dedent( + """\ const %s %s[] = { %s }; """ - ) - % (self.entry_type, self.entries_name, entries) - ) + ) % (self.entry_type, self.entries_name, entries) def gen_getter( self, @@ -280,9 +274,8 @@ class CGHelper: if return_type is None: return_type = "const %s&" % self.entry_type - return ( - textwrap.dedent( - """ + return textwrap.dedent( + """ %(return_type)s %(name)s(%(key_type)s aKey) { @@ -295,18 +288,16 @@ class CGHelper: %(return_entry)s } """ - ) - % { - "name": name, - "basis_table": self._indent(self.basis_table()), - "entries_name": self.entries_name, - "return_type": return_type, - "return_entry": self._indent(return_entry), - "key_type": key_type, - "key_bytes": key_bytes, - "key_length": key_length, - } - ) + ) % { + "name": name, + "basis_table": self._indent(self.basis_table()), + "entries_name": self.entries_name, + "return_type": return_type, + "return_entry": self._indent(return_entry), + "key_type": key_type, + "key_bytes": key_bytes, + "key_length": key_length, + } def gen_jslinearstr_getter( self, name, return_type=None, return_entry="return entry;" @@ -322,16 +313,15 @@ class CGHelper: can be used for additional checks, e.g. for keys not in the table.""" - assert all( - b <= 0x7F for e in self.phf.entries for b in self.phf.key(e) - ), "non-ASCII key" + assert all(b <= 0x7F for e in self.phf.entries for b in self.phf.key(e)), ( + "non-ASCII key" + ) if return_type is None: return_type = "const %s&" % self.entry_type - return ( - textwrap.dedent( - """ + return textwrap.dedent( + """ %(return_type)s %(name)s(JSLinearString* aKey) { @@ -355,12 +345,10 @@ class CGHelper: } } """ - ) - % { - "name": name, - "basis_table": self._indent(self.basis_table()), - "entries_name": self.entries_name, - "return_type": return_type, - "return_entry": self._indent(return_entry, 2), - } - ) + ) % { + "name": name, + "basis_table": self._indent(self.basis_table()), + "entries_name": self.entries_name, + "return_type": return_type, + "return_entry": self._indent(return_entry, 2), + } diff --git a/xpcom/idl-parser/xpidl/header.py b/xpcom/idl-parser/xpidl/header.py @@ -557,15 +557,13 @@ def write_interface(iface, fd): else: implclass = "_MYCLASS_" - names.update( - { - "defname": defname, - "macroname": iface.name.upper(), - "name": iface.name, - "iid": iface.attributes.uuid, - "implclass": implclass, - } - ) + names.update({ + "defname": defname, + "macroname": iface.name.upper(), + "name": iface.name, + "iid": iface.attributes.uuid, + "implclass": implclass, + }) fd.write(iface_header % names) diff --git a/xpcom/idl-parser/xpidl/jsonxpt.py b/xpcom/idl-parser/xpidl/jsonxpt.py @@ -157,9 +157,9 @@ def build_interface(iface): if iface.namemap is None: raise Exception("Interface was not resolved.") - assert ( - iface.attributes.scriptable - ), "Don't generate XPT info for non-scriptable interfaces" + assert iface.attributes.scriptable, ( + "Don't generate XPT info for non-scriptable interfaces" + ) # State used while building an interface consts = [] @@ -169,23 +169,19 @@ def build_interface(iface): needs_scriptable = set() def build_const(c): - consts.append( - { - "name": c.name, - "type": get_type(c.basetype, ""), - "value": c.getValue(), # All of our consts are numbers - } - ) + consts.append({ + "name": c.name, + "type": get_type(c.basetype, ""), + "value": c.getValue(), # All of our consts are numbers + }) def build_cenum(b): for var in b.variants: - consts.append( - { - "name": var.name, - "type": get_type(b, "in"), - "value": var.value, - } - ) + consts.append({ + "name": var.name, + "type": get_type(b, "in"), + "value": var.value, + }) def build_method(m, needs_scriptable=None): params = [] @@ -288,9 +284,9 @@ def build_typelib(idl): def link(typelibs): """Link a list of typelibs together into a single typelib""" linked = list(itertools.chain.from_iterable(typelibs)) - assert len(set(iface["name"] for iface in linked)) == len( - linked - ), "Multiple typelibs containing the same interface were linked together" + assert len(set(iface["name"] for iface in linked)) == len(linked), ( + "Multiple typelibs containing the same interface were linked together" + ) return linked diff --git a/xpcom/idl-parser/xpidl/xpidl.py b/xpcom/idl-parser/xpidl/xpidl.py @@ -48,12 +48,10 @@ def attlistToIDL(attlist): attlist = list(attlist) attlist.sort(key=lambda a: a[0]) - return "[%s] " % ",".join( - [ - "%s%s" % (name, value is not None and "(%s)" % value or "") - for name, value, aloc in attlist - ] - ) + return "[%s] " % ",".join([ + "%s%s" % (name, value is not None and "(%s)" % value or "") + for name, value, aloc in attlist + ]) _paramsHardcode = { @@ -83,12 +81,10 @@ def paramAttlistToIDL(attlist): sorted.extend(attlist) - return "[%s] " % ", ".join( - [ - "%s%s" % (name, value is not None and " (%s)" % value or "") - for name, value, aloc in sorted - ] - ) + return "[%s] " % ", ".join([ + "%s%s" % (name, value is not None and " (%s)" % value or "") + for name, value, aloc in sorted + ]) def unaliasType(t): @@ -783,9 +779,9 @@ class WebIDL: # interfaces. # TODO: More explicit compile-time checks? - assert ( - parent.webidlconfig is not None - ), "WebIDL declarations require passing webidlconfig to resolve." + assert parent.webidlconfig is not None, ( + "WebIDL declarations require passing webidlconfig to resolve." + ) # Resolve our native name according to the WebIDL configs. config = parent.webidlconfig.get(self.name, {}) @@ -1398,14 +1394,14 @@ class Attribute: elif attr_name == "setter_can_run_script": if self.explicit_setter_can_run_script: raise IDLError( - "Redundant setter_can_run_script annotation " "on attribute", + "Redundant setter_can_run_script annotation on attribute", aloc, ) self.explicit_setter_can_run_script = True elif attr_name == "getter_can_run_script": if self.explicit_getter_can_run_script: raise IDLError( - "Redundant getter_can_run_script annotation " "on attribute", + "Redundant getter_can_run_script annotation on attribute", aloc, ) self.explicit_getter_can_run_script = True