commit 2a0706bab36d2c5511eb4324b4909542fd7be306
parent 10182f819e6840972402924add4e8a1278760dee
Author: Julien Cristau <jcristau@mozilla.com>
Date: Tue, 25 Nov 2025 16:53:49 +0000
Bug 2002274 - mozharness: stop uploading resource-usage.json artifact. r=florian
The same info is in profile_resource-usage.json.
Differential Revision: https://phabricator.services.mozilla.com/D274009
Diffstat:
3 files changed, 16 insertions(+), 137 deletions(-)
diff --git a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py
@@ -1153,129 +1153,8 @@ class SystemResourceMonitor:
return max(values)
- def as_dict(self):
- """Convert the recorded data to a dict, suitable for serialization.
-
- The returned dict has the following keys:
-
- version - Integer version number being rendered. Currently 2.
- cpu_times_fields - A list of the names of the CPU times fields.
- io_fields - A list of the names of the I/O fields.
- virt_fields - A list of the names of the virtual memory fields.
- swap_fields - A list of the names of the swap memory fields.
- samples - A list of dicts containing low-level measurements.
- events - A list of lists representing point events. The inner list
- has 2 elements, the float wall time of the event and the string
- event name.
- phases - A list of dicts describing phases. Each phase looks a lot
- like an entry from samples (see below). Some phases may not have
- data recorded against them, so some keys may be None.
- overall - A dict representing overall resource usage. This resembles
- a sample entry.
- system - Contains additional information about the system including
- number of processors and amount of memory.
-
- Each entry in the sample list is a dict with the following keys:
-
- start - Float wall time this measurement began on.
- end - Float wall time this measurement ended on.
- io - List of numerics for I/O values.
- virt - List of numerics for virtual memory values.
- swap - List of numerics for swap memory values.
- cpu_percent - List of floats representing CPU percent on each core.
- cpu_times - List of lists. Main list is each core. Inner lists are
- lists of floats representing CPU times on that core.
- cpu_percent_mean - Float of mean CPU percent across all cores.
- cpu_times_sum - List of floats representing the sum of CPU times
- across all cores.
- cpu_times_total - Float representing the sum of all CPU times across
- all cores. This is useful for calculating the percent in each CPU
- time.
- """
-
- o = dict(
- version=2,
- cpu_times_fields=list(self._cpu_times_type._fields),
- io_fields=list(self._io_type._fields),
- virt_fields=list(self._virt_type._fields),
- swap_fields=list(self._swap_type._fields),
- samples=[],
- phases=[],
- system={},
- )
-
- def populate_derived(e):
- if e["cpu_percent_cores"]:
- # pylint --py3k W1619
- e["cpu_percent_mean"] = sum(e["cpu_percent_cores"]) / len(
- e["cpu_percent_cores"]
- )
- else:
- e["cpu_percent_mean"] = None
-
- if e["cpu_times"]:
- e["cpu_times_sum"] = [0.0] * self._cpu_times_len
- for i in range(0, self._cpu_times_len):
- e["cpu_times_sum"][i] = sum(core[i] for core in e["cpu_times"])
-
- e["cpu_times_total"] = sum(e["cpu_times_sum"])
-
- def phase_entry(name, start, end):
- e = dict(
- name=name,
- start=start,
- end=end,
- duration=end - start,
- cpu_percent_cores=self.aggregate_cpu_percent(phase=name),
- cpu_times=[list(c) for c in self.aggregate_cpu_times(phase=name)],
- io=list(self.aggregate_io(phase=name)),
- )
- populate_derived(e)
- return e
-
- for m in self.measurements:
- e = dict(
- start=m.start,
- end=m.end,
- io=list(m.io),
- virt=list(m.virt),
- swap=list(m.swap),
- cpu_percent_cores=list(m.cpu_percent),
- cpu_times=list(list(cpu) for cpu in m.cpu_times),
- )
-
- populate_derived(e)
- o["samples"].append(e)
-
- if o["samples"]:
- o["start"] = o["samples"][0]["start"]
- o["end"] = o["samples"][-1]["end"]
- o["duration"] = o["end"] - o["start"]
- o["overall"] = phase_entry(None, o["start"], o["end"])
- else:
- o["start"] = None
- o["end"] = None
- o["duration"] = None
- o["overall"] = None
-
- o["events"] = [list(ev) for ev in self.events]
-
- for phase, v in self.phases.items():
- o["phases"].append(phase_entry(phase, v[0], v[1]))
-
- if have_psutil:
- o["system"].update(
- dict(
- cpu_logical_count=psutil.cpu_count(logical=True),
- cpu_physical_count=psutil.cpu_count(logical=False),
- swap_total=psutil.swap_memory()[0],
- vmem_total=psutil.virtual_memory()[0],
- )
- )
-
- return o
-
def as_profile(self):
+ """Convert the recorded data to an object suitable for import into the firefox profiler"""
profile_time = time.monotonic()
start_time = self.start_time
profile = {
diff --git a/testing/mozbase/mozsystemmonitor/tests/test_resource_monitor.py b/testing/mozbase/mozsystemmonitor/tests/test_resource_monitor.py
@@ -149,7 +149,7 @@ class TestResourceMonitor(unittest.TestCase):
v = monitor.max_memory_percent()
self.assertIsInstance(v, float)
- def test_as_dict(self):
+ def test_as_profile(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
@@ -166,16 +166,20 @@ class TestResourceMonitor(unittest.TestCase):
time.sleep(0.4)
monitor.stop()
- d = monitor.as_dict()
-
- self.assertEqual(d["version"], 2)
- self.assertEqual(len(d["events"]), 2)
- self.assertEqual(len(d["phases"]), 2)
- self.assertIn("system", d)
- self.assertIsInstance(d["system"], dict)
- self.assertIsInstance(d["overall"], dict)
- self.assertIn("duration", d["overall"])
- self.assertIn("cpu_times", d["overall"])
+ d = monitor.as_profile()
+
+ self.assertEqual(len(d["threads"]), 1)
+ self.assertIn("markers", d["threads"][0])
+ self.assertIn("data", d["threads"][0]["markers"])
+ markers = d["threads"][0]["markers"]["data"]
+ self.assertTrue(
+ any(m["type"] == "Phase" and m["phase"] == "phase1" for m in markers)
+ )
+ self.assertTrue(
+ any(m["type"] == "Phase" and m["phase"] == "phase2" for m in markers)
+ )
+ self.assertIn({"type": "Text", "text": "foo"}, markers)
+ self.assertIn({"type": "Text", "text": "bar"}, markers)
if __name__ == "__main__":
diff --git a/testing/mozharness/mozharness/base/python.py b/testing/mozharness/mozharness/base/python.py
@@ -890,10 +890,6 @@ class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
try:
if not os.path.exists(upload_dir):
os.makedirs(upload_dir)
- with open(os.path.join(upload_dir, "resource-usage.json"), "w") as fh:
- json.dump(
- self._resource_monitor.as_dict(), fh, sort_keys=True, indent=4
- )
with open(
os.path.join(upload_dir, "profile_resource-usage.json"), "w"
) as fh:
