ipdl.py (12792B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 import json 5 import optparse 6 import os 7 import sys 8 from configparser import RawConfigParser 9 from io import StringIO 10 from multiprocessing import Manager 11 12 import ipdl 13 from ipdl.ast import SYNC 14 from ipdl.exporter import JSONExporter 15 16 17 class WorkerPool: 18 per_process_context = None 19 20 def __init__( 21 self, 22 manager, 23 files, 24 asts, 25 headersdir, 26 cppdir, 27 segmentCapacityDict, 28 allmessages, 29 allprotocols, 30 allmessageprognames, 31 allsyncmessages, 32 alljsonobjs, 33 *, 34 processes=None, 35 ): 36 if processes is None: 37 processes = os.cpu_count() or 1 38 processes = min(processes, 8) 39 self.n = len(files) 40 self.asts = asts 41 self.pool = manager.Pool( 42 initializer=WorkerPool._init_worker, 43 initargs=( 44 files, 45 asts, 46 headersdir, 47 cppdir, 48 segmentCapacityDict, 49 allmessages, 50 allprotocols, 51 allmessageprognames, 52 allsyncmessages, 53 alljsonobjs, 54 ), 55 processes=processes, 56 ) 57 58 def run(self): 59 self.pool.map(WorkerPool._run_worker, range(self.n)) 60 61 @staticmethod 62 def _init_worker(*state): 63 WorkerPool.per_process_context = state 64 65 @staticmethod 66 def _run_worker(index): 67 ( 68 files, 69 asts, 70 headersdir, 71 cppdir, 72 segmentCapacityDict, 73 allmessages, 74 allprotocols, 75 allmessageprognames, 76 allsyncmessages, 77 alljsonobjs, 78 ) = WorkerPool.per_process_context 79 ast = asts[index] 80 ipdl.gencxx(files[index], ast, headersdir, cppdir, segmentCapacityDict) 81 82 if ast.protocol: 83 allmessages[ast.protocol.name] = ipdl.genmsgenum(ast) 84 allprotocols.append(ast.protocol.name) 85 86 alljsonobjs.append(JSONExporter.protocolToObject(ast.protocol)) 87 88 # e.g. PContent::RequestMemoryReport (not prefixed or suffixed.) 89 for md in ast.protocol.messageDecls: 90 allmessageprognames.append("%s::%s" % (md.namespace, md.decl.progname)) 91 92 if md.sendSemantics is SYNC: 93 allsyncmessages.append( 94 "%s__%s" % (ast.protocol.name, md.prettyMsgName()) 95 ) 96 97 98 def main(): 99 def log(minv, fmt, *args): 100 if _verbosity >= minv: 101 print(fmt % args) 102 103 # process command line 104 105 op = optparse.OptionParser(usage="ipdl.py [options] IPDLfiles...") 106 op.add_option( 107 "-I", 108 "--include", 109 dest="includedirs", 110 default=[], 111 action="append", 112 help="Additional directory to search for included protocol specifications", 113 ) 114 op.add_option( 115 "-s", 116 "--sync-msg-list", 117 dest="syncMsgList", 118 default="sync-messages.ini", 119 help="Config file listing allowed sync messages", 120 ) 121 op.add_option( 122 "-m", 123 "--msg-metadata", 124 dest="msgMetadata", 125 default="message-metadata.ini", 126 help="Predicted message sizes for reducing serialization malloc overhead.", 127 ) 128 op.add_option( 129 "-v", 130 "--verbose", 131 dest="verbosity", 132 default=1, 133 action="count", 134 help="Verbose logging (specify -vv or -vvv for very verbose logging)", 135 ) 136 op.add_option( 137 "-q", 138 "--quiet", 139 dest="verbosity", 140 action="store_const", 141 const=0, 142 help="Suppress logging output", 143 ) 144 op.add_option( 145 "-d", 146 "--outheaders-dir", 147 dest="headersdir", 148 default=".", 149 help="""Directory into which C++ headers will be generated. 150 A protocol Foo in the namespace bar will cause the headers 151 dir/bar/Foo.h, dir/bar/FooParent.h, and dir/bar/FooParent.h 152 to be generated""", 153 ) 154 op.add_option( 155 "-o", 156 "--outcpp-dir", 157 dest="cppdir", 158 default=".", 159 help="""Directory into which C++ sources will be generated 160 A protocol Foo in the namespace bar will cause the sources 161 cppdir/FooParent.cpp, cppdir/FooChild.cpp 162 to be generated""", 163 ) 164 op.add_option( 165 "-F", 166 "--file-list", 167 dest="file_list_file", 168 default=None, 169 help="""A file containing IPDL files to parse. This will be 170 merged with files provided on the commandline.""", 171 ) 172 173 options, cmdline_files = op.parse_args() 174 _verbosity = options.verbosity 175 syncMsgList = options.syncMsgList 176 headersdir = options.headersdir 177 cppdir = options.cppdir 178 includedirs = [os.path.abspath(incdir) for incdir in options.includedirs] 179 180 files = [] 181 182 if options.file_list_file is not None: 183 with open(options.file_list_file) as f: 184 files.extend(f.read().splitlines()) 185 186 files.extend(cmdline_files) 187 188 if not files: 189 op.error("No IPDL files specified") 190 191 ipcmessagestartpath = os.path.join(headersdir, "IPCMessageStart.h") 192 ipc_msgtype_name_path = os.path.join(cppdir, "IPCMessageTypeName.cpp") 193 194 log(2, 'Generated C++ headers will be generated relative to "%s"', headersdir) 195 log(2, 'Generated C++ sources will be generated in "%s"', cppdir) 196 197 def normalizedFilename(f): 198 if f == "-": 199 return "<stdin>" 200 return f 201 202 log(2, "Reading sync message list") 203 parser = RawConfigParser() 204 parser.read_file(open(options.syncMsgList)) 205 syncMsgList = parser.sections() 206 207 for section in syncMsgList: 208 if not parser.get(section, "description"): 209 print( 210 "Error: Sync message %s lacks a description" % section, file=sys.stderr 211 ) 212 sys.exit(1) 213 214 # Read message metadata. Right now we only have 'segment_capacity' 215 # for the standard segment size used for serialization. 216 log(2, "Reading message metadata...") 217 msgMetadataConfig = RawConfigParser() 218 msgMetadataConfig.read_file(open(options.msgMetadata)) 219 220 manager = Manager() 221 segmentCapacityDict = manager.dict() 222 allmessages = manager.dict() 223 allsyncmessages = manager.list() 224 allmessageprognames = manager.list() 225 allprotocols = manager.list() 226 alljsonobjs = manager.list() 227 228 for msgName in msgMetadataConfig.sections(): 229 if msgMetadataConfig.has_option(msgName, "segment_capacity"): 230 capacity = msgMetadataConfig.get(msgName, "segment_capacity") 231 segmentCapacityDict[msgName] = capacity 232 233 # First pass: parse and type-check all protocols 234 asts = [] 235 for f in files: 236 log(2, os.path.basename(f)) 237 filename = normalizedFilename(f) 238 if f == "-": 239 fd = sys.stdin 240 else: 241 fd = open(f) 242 243 specstring = fd.read() 244 fd.close() 245 246 ast = ipdl.parse(specstring, filename, includedirs=includedirs) 247 if ast is None: 248 print("Specification could not be parsed.", file=sys.stderr) 249 sys.exit(1) 250 251 log(2, "checking types") 252 if not ipdl.typecheck(ast): 253 print("Specification is not well typed.", file=sys.stderr) 254 sys.exit(1) 255 256 if not ipdl.checkSyncMessage(ast, syncMsgList): 257 print( 258 "Error: New sync IPC messages must be reviewed by an IPC peer and recorded in %s" 259 % options.syncMsgList, 260 file=sys.stderr, 261 ) # NOQA: E501 262 sys.exit(1) 263 264 asts.append(ast) 265 266 if not ipdl.checkFixedSyncMessages(parser): 267 # Errors have alraedy been printed to stderr, just exit 268 sys.exit(1) 269 270 # Second pass: generate code 271 pool = WorkerPool( 272 manager, 273 files, 274 asts, 275 headersdir, 276 cppdir, 277 segmentCapacityDict, 278 allmessages, 279 allprotocols, 280 allmessageprognames, 281 allsyncmessages, 282 alljsonobjs, 283 ) 284 pool.run() 285 286 if cppdir is not None: 287 # Sort to ensure deterministic output 288 alljsonobjs = list(alljsonobjs) 289 alljsonobjs.sort(key=lambda p: p["name"]) 290 ipdl.writeifmodified( 291 json.dumps({"protocols": alljsonobjs}, indent=2), 292 os.path.join(cppdir, "protocols.json"), 293 ) 294 295 allprotocols.sort() 296 allsyncmessages.sort() 297 298 # Check if we have undefined message names in segmentCapacityDict. 299 # This is a fool-proof of the 'message-metadata.ini' file. 300 undefinedMessages = set(segmentCapacityDict.keys()) - set(allmessageprognames) 301 if len(undefinedMessages) > 0: 302 print( 303 "Error: Undefined message names in message-metadata.ini:", file=sys.stderr 304 ) 305 print(undefinedMessages, file=sys.stderr) 306 sys.exit(1) 307 308 ipcmsgstart = StringIO() 309 310 print( 311 """ 312 // CODE GENERATED by ipdl.py. Do not edit. 313 314 #ifndef IPCMessageStart_h 315 #define IPCMessageStart_h 316 317 enum IPCMessageStart { 318 """, 319 file=ipcmsgstart, 320 ) 321 322 for name in allprotocols: 323 print(" %sMsgStart," % name, file=ipcmsgstart) 324 325 print( 326 """ 327 LastMsgIndex 328 }; 329 330 static_assert(LastMsgIndex <= 65536, "need to update IPC_MESSAGE_MACRO"); 331 332 #endif // ifndef IPCMessageStart_h 333 """, 334 file=ipcmsgstart, 335 ) 336 337 ipc_msgtype_name = StringIO() 338 print( 339 """ 340 // CODE GENERATED by ipdl.py. Do not edit. 341 #include <cstdint> 342 343 #include "mozilla/ipc/ProtocolUtils.h" 344 #include "IPCMessageStart.h" 345 346 using std::uint32_t; 347 348 namespace { 349 350 enum IPCMessages { 351 """, 352 file=ipc_msgtype_name, 353 ) 354 355 for protocol in sorted(allmessages.keys()): 356 for msg, num in allmessages[protocol].idnums: 357 if num: 358 print(" %s = %s," % (msg, num), file=ipc_msgtype_name) 359 elif not msg.endswith("End"): 360 print(" %s__%s," % (protocol, msg), file=ipc_msgtype_name) 361 362 print( 363 """ 364 }; 365 366 } // anonymous namespace 367 368 namespace IPC { 369 370 bool IPCMessageTypeIsSync(uint32_t aMessageType) 371 { 372 switch (aMessageType) { 373 """, 374 file=ipc_msgtype_name, 375 ) 376 377 for msg in allsyncmessages: 378 print(" case %s:" % msg, file=ipc_msgtype_name) 379 380 print( 381 """ return true; 382 default: 383 return false; 384 } 385 } 386 387 const char* StringFromIPCMessageType(uint32_t aMessageType) 388 { 389 switch (aMessageType) { 390 """, 391 file=ipc_msgtype_name, 392 ) 393 394 for protocol in sorted(allmessages.keys()): 395 for msg, num in allmessages[protocol].idnums: 396 if num or msg.endswith("End"): 397 continue 398 print( 399 """ 400 case %s__%s: 401 return "%s::%s";""" 402 % (protocol, msg, protocol, msg), 403 file=ipc_msgtype_name, 404 ) 405 406 print( 407 """ 408 case DATA_PIPE_CLOSED_MESSAGE_TYPE: 409 return "DATA_PIPE_CLOSED_MESSAGE"; 410 case DATA_PIPE_BYTES_CONSUMED_MESSAGE_TYPE: 411 return "DATA_PIPE_BYTES_CONSUMED_MESSAGE"; 412 case ACCEPT_INVITE_MESSAGE_TYPE: 413 return "ACCEPT_INVITE_MESSAGE"; 414 case REQUEST_INTRODUCTION_MESSAGE_TYPE: 415 return "REQUEST_INTRODUCTION_MESSAGE"; 416 case INTRODUCE_MESSAGE_TYPE: 417 return "INTRODUCE_MESSAGE"; 418 case BROADCAST_MESSAGE_TYPE: 419 return "BROADCAST_MESSAGE"; 420 case EVENT_MESSAGE_TYPE: 421 return "EVENT_MESSAGE"; 422 case IMPENDING_SHUTDOWN_MESSAGE_TYPE: 423 return "IMPENDING_SHUTDOWN"; 424 case BUILD_IDS_MATCH_MESSAGE_TYPE: 425 return "BUILD_IDS_MATCH_MESSAGE"; 426 case BUILD_ID_MESSAGE_TYPE: 427 return "BUILD_ID_MESSAGE"; 428 case CHANNEL_OPENED_MESSAGE_TYPE: 429 return "CHANNEL_OPENED_MESSAGE"; 430 case SHMEM_DESTROYED_MESSAGE_TYPE: 431 return "SHMEM_DESTROYED_MESSAGE"; 432 case SHMEM_CREATED_MESSAGE_TYPE: 433 return "SHMEM_CREATED_MESSAGE"; 434 case GOODBYE_MESSAGE_TYPE: 435 return "GOODBYE_MESSAGE"; 436 case CANCEL_MESSAGE_TYPE: 437 return "CANCEL_MESSAGE"; 438 default: 439 return "<unknown IPC msg name>"; 440 } 441 } 442 443 } // namespace IPC 444 445 namespace mozilla { 446 namespace ipc { 447 448 const char* ProtocolIdToName(IPCMessageStart aId) { 449 switch (aId) { 450 """, 451 file=ipc_msgtype_name, 452 ) 453 454 for name in allprotocols: 455 print(" case %sMsgStart:" % name, file=ipc_msgtype_name) 456 print(' return "%s";' % name, file=ipc_msgtype_name) 457 458 print( 459 """ 460 default: 461 return "<unknown protocol id>"; 462 } 463 } 464 465 } // namespace ipc 466 } // namespace mozilla 467 """, 468 file=ipc_msgtype_name, 469 ) 470 471 ipdl.writeifmodified(ipcmsgstart.getvalue(), ipcmessagestartpath) 472 ipdl.writeifmodified(ipc_msgtype_name.getvalue(), ipc_msgtype_name_path) 473 474 475 if __name__ == "__main__": 476 main()