database.py (8238B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 # You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 # This modules provides functionality for dealing with code completion. 6 7 import os 8 from collections import OrderedDict, defaultdict 9 10 import mozpack.path as mozpath 11 from mozshellutil import quote as shell_quote 12 13 from mozbuild.backend.common import CommonBackend 14 from mozbuild.frontend.data import ( 15 ComputedFlags, 16 DirectoryTraversal, 17 PerSourceFlag, 18 Sources, 19 VariablePassthru, 20 ) 21 from mozbuild.util import expand_variables 22 23 24 class CompileDBBackend(CommonBackend): 25 def _init(self): 26 CommonBackend._init(self) 27 28 # The database we're going to dump out to. 29 self._db = OrderedDict() 30 31 # The cache for per-directory flags 32 self._flags = {} 33 34 self._envs = {} 35 self._local_flags = defaultdict(dict) 36 self._per_source_flags = defaultdict(list) 37 38 def _build_cmd(self, cmd, filename, unified): 39 cmd = list(cmd) 40 if unified is None: 41 cmd.append(filename) 42 else: 43 cmd.append(unified) 44 45 return cmd 46 47 def consume_object(self, obj): 48 # Those are difficult directories, that will be handled later. 49 if obj.relsrcdir in ( 50 "build/unix/elfhack", 51 "build/unix/elfhack/inject", 52 "build/clang-plugin", 53 "build/clang-plugin/tests", 54 ): 55 return True 56 57 consumed = CommonBackend.consume_object(self, obj) 58 59 if consumed: 60 return True 61 62 if isinstance(obj, DirectoryTraversal): 63 self._envs[obj.objdir] = obj.config 64 65 elif isinstance(obj, Sources): 66 # For other sources, include each source file. 67 for f in obj.files: 68 self._build_db_line( 69 obj.objdir, obj.relsrcdir, obj.config, f, obj.canonical_suffix 70 ) 71 72 elif isinstance(obj, VariablePassthru): 73 for var in ("MOZBUILD_CMFLAGS", "MOZBUILD_CMMFLAGS"): 74 if var in obj.variables: 75 self._local_flags[obj.objdir][var] = obj.variables[var] 76 77 elif isinstance(obj, PerSourceFlag): 78 self._per_source_flags[obj.file_name].extend(obj.flags) 79 80 elif isinstance(obj, ComputedFlags): 81 for var, flags in obj.get_flags(): 82 self._local_flags[obj.objdir]["COMPUTED_%s" % var] = flags 83 84 return True 85 86 def consume_finished(self): 87 CommonBackend.consume_finished(self) 88 89 db = [] 90 91 for (directory, filename, unified), cmd in self._db.items(): 92 env = self._envs[directory] 93 cmd = self._build_cmd(cmd, filename, unified) 94 variables = { 95 "DIST": mozpath.join(env.topobjdir, "dist"), 96 "DEPTH": env.topobjdir, 97 "MOZILLA_DIR": env.topsrcdir, 98 "topsrcdir": env.topsrcdir, 99 "topobjdir": env.topobjdir, 100 } 101 variables.update(self._local_flags[directory]) 102 c = [] 103 for a in cmd: 104 accum = "" 105 for word in expand_variables(a, variables).split(): 106 # We can't just split() the output of expand_variables since 107 # there can be spaces enclosed by quotes, e.g. '"foo bar"'. 108 # Handle that case by checking whether there are an even 109 # number of double-quotes in the word and appending it to 110 # the accumulator if not. Meanwhile, shlex.split() and 111 # mozshellutil.split() aren't able to properly handle 112 # this and break in various ways, so we can't use something 113 # off-the-shelf. 114 has_quote = bool(word.count('"') % 2) 115 if accum and has_quote: 116 c.append(accum + " " + word) 117 accum = "" 118 elif accum and not has_quote: 119 accum += " " + word 120 elif not accum and has_quote: 121 accum = word 122 else: 123 c.append(word) 124 # Tell clangd to keep parsing to the end of a file, regardless of 125 # how many errors are encountered. (Unified builds mean that we 126 # encounter a lot of errors parsing some files.) 127 c.insert(-1, "-ferror-limit=0") 128 129 per_source_flags = self._per_source_flags.get(filename) 130 if per_source_flags is not None: 131 c.extend(per_source_flags) 132 db.append({ 133 "directory": directory, 134 "command": shell_quote(*c), 135 "file": mozpath.join(directory, filename), 136 }) 137 138 import json 139 140 outputfile = self._outputfile_path() 141 with self._write_file(outputfile) as jsonout: 142 json.dump(db, jsonout, indent=0) 143 144 def _outputfile_path(self): 145 # Output the database (a JSON file) to objdir/compile_commands.json 146 return os.path.join(self.environment.topobjdir, "compile_commands.json") 147 148 def _process_unified_sources_without_mapping(self, obj): 149 for f in list(sorted(obj.files)): 150 self._build_db_line( 151 obj.objdir, obj.relsrcdir, obj.config, f, obj.canonical_suffix 152 ) 153 154 def _process_unified_sources(self, obj): 155 if not obj.have_unified_mapping: 156 return self._process_unified_sources_without_mapping(obj) 157 158 # For unified sources, only include the unified source file. 159 # Note that unified sources are never used for host sources. 160 for f in obj.unified_source_mapping: 161 self._build_db_line( 162 obj.objdir, obj.relsrcdir, obj.config, f[0], obj.canonical_suffix 163 ) 164 for entry in f[1]: 165 self._build_db_line( 166 obj.objdir, 167 obj.relsrcdir, 168 obj.config, 169 entry, 170 obj.canonical_suffix, 171 unified=f[0], 172 ) 173 174 def _handle_idl_manager(self, idl_manager): 175 pass 176 177 def _handle_ipdl_sources( 178 self, 179 ipdl_dir, 180 sorted_ipdl_sources, 181 sorted_nonstatic_ipdl_sources, 182 sorted_static_ipdl_sources, 183 ): 184 pass 185 186 def _handle_webidl_build( 187 self, 188 bindings_dir, 189 unified_source_mapping, 190 webidls, 191 expected_build_output_files, 192 global_define_files, 193 ): 194 for f in unified_source_mapping: 195 self._build_db_line(bindings_dir, None, self.environment, f[0], ".cpp") 196 197 COMPILERS = { 198 ".c": "CC", 199 ".cpp": "CXX", 200 ".m": "CC", 201 ".mm": "CXX", 202 } 203 204 CFLAGS = { 205 ".c": "CFLAGS", 206 ".cpp": "CXXFLAGS", 207 ".m": "CFLAGS", 208 ".mm": "CXXFLAGS", 209 } 210 211 def _get_compiler_args(self, cenv, canonical_suffix): 212 if canonical_suffix not in self.COMPILERS: 213 return None 214 return cenv.substs[self.COMPILERS[canonical_suffix]].split() 215 216 def _build_db_line( 217 self, objdir, reldir, cenv, filename, canonical_suffix, unified=None 218 ): 219 compiler_args = self._get_compiler_args(cenv, canonical_suffix) 220 if compiler_args is None: 221 return 222 db = self._db.setdefault( 223 (objdir, filename, unified), 224 compiler_args + ["-o", "/dev/null", "-c"], 225 ) 226 reldir = reldir or mozpath.relpath(objdir, cenv.topobjdir) 227 228 def append_var(name): 229 value = cenv.substs.get(name) 230 if not value: 231 return 232 if isinstance(value, str): 233 value = value.split() 234 db.extend(value) 235 236 db.append("$(COMPUTED_%s)" % self.CFLAGS[canonical_suffix]) 237 if canonical_suffix == ".m": 238 append_var("OS_COMPILE_CMFLAGS") 239 db.append("$(MOZBUILD_CMFLAGS)") 240 elif canonical_suffix == ".mm": 241 append_var("OS_COMPILE_CMMFLAGS") 242 db.append("$(MOZBUILD_CMMFLAGS)")