get_concurrent_links.py (7539B)
1 #!/usr/bin/env python3 2 # Copyright 2014 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # This script computs the number of concurrent links we want to run in the build 7 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP. 8 9 import argparse 10 import multiprocessing 11 import os 12 import re 13 import subprocess 14 import sys 15 16 sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..')) 17 import gn_helpers 18 19 20 def _GetMemoryMaxInCurrentCGroup(explanation): 21 with open("/proc/self/cgroup") as cgroup: 22 lines = cgroup.readlines() 23 if len(lines) >= 1: 24 cgroupname = lines[0].strip().split(':')[-1] 25 memmax = '/sys/fs/cgroup' + cgroupname + '/memory.max' 26 if os.path.exists(memmax): 27 with open(memmax) as f: 28 data = f.read().strip() 29 explanation.append(f'# cgroup {cgroupname} memory.max={data}') 30 try: 31 return int(data) 32 except ValueError as ex: 33 explanation.append(f'# cgroup memory.max exception {ex}') 34 return None 35 explanation.append(f'# cgroup memory.max not found') 36 return None 37 38 39 def _GetCPUCountFromCurrentCGroup(explanation): 40 with open("/proc/self/cgroup") as cgroup: 41 lines = cgroup.readlines() 42 if len(lines) >= 1: 43 cgroupname = lines[0].strip().split(':')[-1] 44 cpuset = '/sys/fs/cgroup' + cgroupname + '/cpuset.cpus' 45 if os.path.exists(cpuset): 46 with open(cpuset) as f: 47 data = f.read().strip() 48 explanation.append(f'# cgroup {cgroupname} cpuset.cpus={data}') 49 try: 50 return _CountCPUs(data) 51 except ValueError as ex: 52 explanation.append(f'# cgroup cpuset.cpus exception {ex}') 53 return None 54 explanation.append(f'# cgroup cpuset.cpus not found') 55 return None 56 57 58 def _CountCPUs(cpuset): 59 n = 0 60 for s in cpuset.split(','): 61 r = s.split('-') 62 if len(r) == 1 and int(r[0]) >= 0: 63 n += 1 64 continue 65 elif len(r) == 2: 66 n += int(r[1]) - int(r[0]) + 1 67 else: 68 # wrong range? 69 return 0 70 return n 71 72 73 def _GetTotalMemoryInBytes(explanation): 74 if sys.platform in ('win32', 'cygwin'): 75 import ctypes 76 77 class MEMORYSTATUSEX(ctypes.Structure): 78 _fields_ = [ 79 ("dwLength", ctypes.c_ulong), 80 ("dwMemoryLoad", ctypes.c_ulong), 81 ("ullTotalPhys", ctypes.c_ulonglong), 82 ("ullAvailPhys", ctypes.c_ulonglong), 83 ("ullTotalPageFile", ctypes.c_ulonglong), 84 ("ullAvailPageFile", ctypes.c_ulonglong), 85 ("ullTotalVirtual", ctypes.c_ulonglong), 86 ("ullAvailVirtual", ctypes.c_ulonglong), 87 ("sullAvailExtendedVirtual", ctypes.c_ulonglong), 88 ] 89 90 stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) 91 ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) 92 return stat.ullTotalPhys 93 elif sys.platform.startswith('linux'): 94 if os.path.exists("/proc/self/cgroup"): 95 memmax = _GetMemoryMaxInCurrentCGroup(explanation) 96 if memmax: 97 return memmax 98 if os.path.exists("/proc/meminfo"): 99 with open("/proc/meminfo") as meminfo: 100 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') 101 for line in meminfo: 102 match = memtotal_re.match(line) 103 if not match: 104 continue 105 return float(match.group(1)) * 2**10 106 elif sys.platform == 'darwin': 107 try: 108 return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) 109 except Exception: 110 return 0 111 # TODO(scottmg): Implement this for other platforms. 112 return 0 113 114 115 def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, thin_lto_type, 116 secondary_per_link_gb, override_ram_in_gb): 117 explanation = [] 118 explanation.append( 119 'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format( 120 per_link_gb, reserve_gb, secondary_per_link_gb)) 121 if override_ram_in_gb: 122 mem_total_gb = override_ram_in_gb 123 else: 124 mem_total_gb = float(_GetTotalMemoryInBytes(explanation)) / 2**30 125 adjusted_mem_total_gb = max(0, mem_total_gb - reserve_gb) 126 127 # Ensure that there is at least as many links allocated for the secondary as 128 # there is for the primary. The secondary link usually uses fewer gbs. 129 mem_cap = int( 130 max(1, adjusted_mem_total_gb / (per_link_gb + secondary_per_link_gb))) 131 132 cpu_count = None 133 if sys.platform.startswith('linux'): 134 try: 135 if os.path.exists('/proc/self/cgroup'): 136 cpu_count = _GetCPUCountFromCurrentCGroup(explanation) 137 except Exception as ex: 138 explanation.append(f'# cpu_count from cgroup exception {ex}') 139 if not cpu_count: 140 try: 141 cpu_count = multiprocessing.cpu_count() 142 explanation.append(f'# cpu_count from multiprocessing {cpu_count}') 143 except Exception as ex: 144 cpu_count = 1 145 explanation.append(f'# cpu_count from multiprocessing exception {ex}') 146 147 # A local LTO links saturate all cores, but only for some amount of the link. 148 cpu_cap = cpu_count 149 if thin_lto_type is not None: 150 assert thin_lto_type == 'local' 151 cpu_cap = min(cpu_count, 6) 152 153 explanation.append(f'cpu_count={cpu_count} cpu_cap={cpu_cap} ' + 154 f'mem_total_gb={mem_total_gb:.1f}GiB ' + 155 f'adjusted_mem_total_gb={adjusted_mem_total_gb:.1f}GiB') 156 157 num_links = min(mem_cap, cpu_cap) 158 if num_links == cpu_cap: 159 if cpu_cap == cpu_count: 160 reason = 'cpu_count' 161 else: 162 reason = 'cpu_cap (thinlto)' 163 else: 164 reason = 'RAM' 165 166 # static link see too many open files if we have many concurrent links. 167 # ref: http://b/233068481 168 if num_links > 30: 169 num_links = 30 170 reason = 'nofile' 171 172 explanation.append('concurrent_links={} (reason: {})'.format( 173 num_links, reason)) 174 175 # Use remaining RAM for a secondary pool if needed. 176 if secondary_per_link_gb: 177 mem_remaining = adjusted_mem_total_gb - num_links * per_link_gb 178 secondary_size = int(max(0, mem_remaining / secondary_per_link_gb)) 179 if secondary_size > cpu_count: 180 secondary_size = cpu_count 181 reason = 'cpu_count' 182 else: 183 reason = 'mem_remaining={:.1f}GiB'.format(mem_remaining) 184 explanation.append('secondary_size={} (reason: {})'.format( 185 secondary_size, reason)) 186 else: 187 secondary_size = 0 188 189 return num_links, secondary_size, explanation 190 191 192 def main(): 193 parser = argparse.ArgumentParser() 194 parser.add_argument('--mem_per_link_gb', type=int, default=8) 195 parser.add_argument('--reserve_mem_gb', type=int, default=0) 196 parser.add_argument('--secondary_mem_per_link', type=int, default=0) 197 parser.add_argument('--override-ram-in-gb-for-testing', type=float, default=0) 198 parser.add_argument('--thin-lto') 199 options = parser.parse_args() 200 201 primary_pool_size, secondary_pool_size, explanation = ( 202 _GetDefaultConcurrentLinks(options.mem_per_link_gb, 203 options.reserve_mem_gb, options.thin_lto, 204 options.secondary_mem_per_link, 205 options.override_ram_in_gb_for_testing)) 206 if options.override_ram_in_gb_for_testing: 207 print('primary={} secondary={} explanation={}'.format( 208 primary_pool_size, secondary_pool_size, explanation)) 209 else: 210 sys.stdout.write( 211 gn_helpers.ToGNString({ 212 'primary_pool_size': primary_pool_size, 213 'secondary_pool_size': secondary_pool_size, 214 'explanation': explanation, 215 })) 216 return 0 217 218 219 if __name__ == '__main__': 220 sys.exit(main())