test_manifests.py (15404B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import os 6 7 import mozunit 8 9 from mozpack.copier import FileCopier, FileRegistry 10 from mozpack.manifests import InstallManifest, UnreadableInstallManifest 11 from mozpack.test.test_files import TestWithTmpDir 12 13 14 class TestInstallManifest(TestWithTmpDir): 15 def test_construct(self): 16 m = InstallManifest() 17 self.assertEqual(len(m), 0) 18 19 def test_malformed(self): 20 f = self.tmppath("manifest") 21 open(f, "w").write("junk\n") 22 with self.assertRaises(UnreadableInstallManifest): 23 InstallManifest(f) 24 25 def test_adds(self): 26 m = InstallManifest() 27 m.add_link("s_source", "s_dest") 28 m.add_copy("c_source", "c_dest") 29 m.add_required_exists("e_dest") 30 m.add_optional_exists("o_dest") 31 m.add_pattern_link("ps_base", "ps/*", "ps_dest") 32 m.add_pattern_copy("pc_base", "pc/**", "pc_dest") 33 m.add_preprocess("p_source", "p_dest", "p_source.pp") 34 m.add_content("content", "content") 35 36 self.assertEqual(len(m), 8) 37 self.assertIn("s_dest", m) 38 self.assertIn("c_dest", m) 39 self.assertIn("p_dest", m) 40 self.assertIn("e_dest", m) 41 self.assertIn("o_dest", m) 42 self.assertIn("content", m) 43 44 with self.assertRaises(ValueError): 45 m.add_link("s_other", "s_dest") 46 47 with self.assertRaises(ValueError): 48 m.add_copy("c_other", "c_dest") 49 50 with self.assertRaises(ValueError): 51 m.add_preprocess("p_other", "p_dest", "p_other.pp") 52 53 with self.assertRaises(ValueError): 54 m.add_required_exists("e_dest") 55 56 with self.assertRaises(ValueError): 57 m.add_optional_exists("o_dest") 58 59 with self.assertRaises(ValueError): 60 m.add_pattern_link("ps_base", "ps/*", "ps_dest") 61 62 with self.assertRaises(ValueError): 63 m.add_pattern_copy("pc_base", "pc/**", "pc_dest") 64 65 with self.assertRaises(ValueError): 66 m.add_content("content", "content") 67 68 def _get_test_manifest(self): 69 m = InstallManifest() 70 m.add_link(self.tmppath("s_source"), "s_dest") 71 m.add_copy(self.tmppath("c_source"), "c_dest") 72 m.add_preprocess( 73 self.tmppath("p_source"), 74 "p_dest", 75 self.tmppath("p_source.pp"), 76 "#", 77 {"FOO": "BAR", "BAZ": "QUX"}, 78 ) 79 m.add_required_exists("e_dest") 80 m.add_optional_exists("o_dest") 81 m.add_pattern_link("ps_base", "*", "ps_dest") 82 m.add_pattern_copy("pc_base", "**", "pc_dest") 83 m.add_content("the content\non\nmultiple lines", "content") 84 85 return m 86 87 def test_serialization(self): 88 m = self._get_test_manifest() 89 90 p = self.tmppath("m") 91 m.write(path=p) 92 self.assertTrue(os.path.isfile(p)) 93 94 with open(p) as fh: 95 c = fh.read() 96 97 self.assertEqual(c.count("\n"), 9) 98 99 lines = c.splitlines() 100 self.assertEqual(len(lines), 9) 101 102 self.assertEqual(lines[0], "5") 103 104 m2 = InstallManifest(path=p) 105 self.assertEqual(m, m2) 106 p2 = self.tmppath("m2") 107 m2.write(path=p2) 108 109 with open(p2) as fh: 110 c2 = fh.read() 111 112 self.assertEqual(c, c2) 113 114 def test_populate_registry(self): 115 m = self._get_test_manifest() 116 r = FileRegistry() 117 m.populate_registry(r) 118 119 self.assertEqual(len(r), 6) 120 self.assertEqual( 121 r.paths(), ["c_dest", "content", "e_dest", "o_dest", "p_dest", "s_dest"] 122 ) 123 124 def test_pattern_expansion(self): 125 source = self.tmppath("source") 126 os.mkdir(source) 127 os.mkdir("%s/base" % source) 128 os.mkdir("%s/base/foo" % source) 129 130 with open("%s/base/foo/file1" % source, "a"): 131 pass 132 133 with open("%s/base/foo/file2" % source, "a"): 134 pass 135 136 m = InstallManifest() 137 m.add_pattern_link("%s/base" % source, "**", "dest") 138 139 c = FileCopier() 140 m.populate_registry(c) 141 self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"]) 142 143 def test_write_expand_pattern(self): 144 source = self.tmppath("source") 145 os.mkdir(source) 146 os.mkdir("%s/base" % source) 147 os.mkdir("%s/base/foo" % source) 148 149 with open("%s/base/foo/file1" % source, "a"): 150 pass 151 152 with open("%s/base/foo/file2" % source, "a"): 153 pass 154 155 m = InstallManifest() 156 m.add_pattern_link("%s/base" % source, "**", "dest") 157 158 track = self.tmppath("track") 159 m.write(path=track, expand_pattern=True) 160 161 m = InstallManifest(path=track) 162 self.assertEqual( 163 sorted(dest for dest in m._dests), ["dest/foo/file1", "dest/foo/file2"] 164 ) 165 166 def test_or(self): 167 m1 = self._get_test_manifest() 168 orig_length = len(m1) 169 m2 = InstallManifest() 170 m2.add_link("s_source2", "s_dest2") 171 m2.add_copy("c_source2", "c_dest2") 172 173 m1 |= m2 174 175 self.assertEqual(len(m2), 2) 176 self.assertEqual(len(m1), orig_length + 2) 177 178 self.assertIn("s_dest2", m1) 179 self.assertIn("c_dest2", m1) 180 181 def test_copier_application(self): 182 dest = self.tmppath("dest") 183 os.mkdir(dest) 184 185 to_delete = self.tmppath("dest/to_delete") 186 with open(to_delete, "a"): 187 pass 188 189 with open(self.tmppath("s_source"), "w") as fh: 190 fh.write("symlink!") 191 192 with open(self.tmppath("c_source"), "w") as fh: 193 fh.write("copy!") 194 195 with open(self.tmppath("p_source"), "w") as fh: 196 fh.write("#define FOO 1\npreprocess!") 197 198 with open(self.tmppath("dest/e_dest"), "a"): 199 pass 200 201 with open(self.tmppath("dest/o_dest"), "a"): 202 pass 203 204 m = self._get_test_manifest() 205 c = FileCopier() 206 m.populate_registry(c) 207 result = c.copy(dest) 208 209 self.assertTrue(os.path.exists(self.tmppath("dest/s_dest"))) 210 self.assertTrue(os.path.exists(self.tmppath("dest/c_dest"))) 211 self.assertTrue(os.path.exists(self.tmppath("dest/p_dest"))) 212 self.assertTrue(os.path.exists(self.tmppath("dest/e_dest"))) 213 self.assertTrue(os.path.exists(self.tmppath("dest/o_dest"))) 214 self.assertTrue(os.path.exists(self.tmppath("dest/content"))) 215 self.assertFalse(os.path.exists(to_delete)) 216 217 with open(self.tmppath("dest/s_dest")) as fh: 218 self.assertEqual(fh.read(), "symlink!") 219 220 with open(self.tmppath("dest/c_dest")) as fh: 221 self.assertEqual(fh.read(), "copy!") 222 223 with open(self.tmppath("dest/p_dest")) as fh: 224 self.assertEqual(fh.read(), "preprocess!") 225 226 self.assertEqual( 227 result.updated_files, 228 set( 229 self.tmppath(p) 230 for p in ("dest/s_dest", "dest/c_dest", "dest/p_dest", "dest/content") 231 ), 232 ) 233 self.assertEqual( 234 result.existing_files, 235 set([self.tmppath("dest/e_dest"), self.tmppath("dest/o_dest")]), 236 ) 237 self.assertEqual(result.removed_files, {to_delete}) 238 self.assertEqual(result.removed_directories, set()) 239 240 def test_preprocessor(self): 241 manifest = self.tmppath("m") 242 deps = self.tmppath("m.pp") 243 dest = self.tmppath("dest") 244 include = self.tmppath("p_incl") 245 246 with open(include, "w") as fh: 247 fh.write("#define INCL\n") 248 time = os.path.getmtime(include) - 3 249 os.utime(include, (time, time)) 250 251 with open(self.tmppath("p_source"), "w") as fh: 252 fh.write("#ifdef FOO\n#if BAZ == QUX\nPASS1\n#endif\n#endif\n") 253 fh.write("#ifdef DEPTEST\nPASS2\n#endif\n") 254 fh.write("#include p_incl\n#ifdef INCLTEST\nPASS3\n#endif\n") 255 time = os.path.getmtime(self.tmppath("p_source")) - 3 256 os.utime(self.tmppath("p_source"), (time, time)) 257 258 # Create and write a manifest with the preprocessed file, then apply it. 259 # This should write out our preprocessed file. 260 m = InstallManifest() 261 m.add_preprocess( 262 self.tmppath("p_source"), "p_dest", deps, "#", {"FOO": "BAR", "BAZ": "QUX"} 263 ) 264 m.write(path=manifest) 265 266 m = InstallManifest(path=manifest) 267 c = FileCopier() 268 m.populate_registry(c) 269 c.copy(dest) 270 271 self.assertTrue(os.path.exists(self.tmppath("dest/p_dest"))) 272 273 with open(self.tmppath("dest/p_dest")) as fh: 274 self.assertEqual(fh.read(), "PASS1\n") 275 276 # Create a second manifest with the preprocessed file, then apply it. 277 # Since this manifest does not exist on the disk, there should not be a 278 # dependency on it, and the preprocessed file should not be modified. 279 m2 = InstallManifest() 280 m2.add_preprocess( 281 self.tmppath("p_source"), "p_dest", deps, "#", {"DEPTEST": True} 282 ) 283 c = FileCopier() 284 m2.populate_registry(c) 285 result = c.copy(dest) 286 287 self.assertFalse(self.tmppath("dest/p_dest") in result.updated_files) 288 self.assertTrue(self.tmppath("dest/p_dest") in result.existing_files) 289 290 # Write out the second manifest, then load it back in from the disk. 291 # This should add the dependency on the manifest file, so our 292 # preprocessed file should be regenerated with the new defines. 293 # We also set the mtime on the destination file back, so it will be 294 # older than the manifest file. 295 m2.write(path=manifest) 296 time = os.path.getmtime(manifest) - 1 297 os.utime(self.tmppath("dest/p_dest"), (time, time)) 298 m2 = InstallManifest(path=manifest) 299 c = FileCopier() 300 m2.populate_registry(c) 301 self.assertTrue(c.copy(dest)) 302 303 with open(self.tmppath("dest/p_dest")) as fh: 304 self.assertEqual(fh.read(), "PASS2\n") 305 306 # Set the time on the manifest back, so it won't be picked up as 307 # modified in the next test 308 time = os.path.getmtime(manifest) - 1 309 os.utime(manifest, (time, time)) 310 311 # Update the contents of a file included by the source file. This should 312 # cause the destination to be regenerated. 313 with open(include, "w") as fh: 314 fh.write("#define INCLTEST\n") 315 316 time = os.path.getmtime(include) - 1 317 os.utime(self.tmppath("dest/p_dest"), (time, time)) 318 c = FileCopier() 319 m2.populate_registry(c) 320 self.assertTrue(c.copy(dest)) 321 322 with open(self.tmppath("dest/p_dest")) as fh: 323 self.assertEqual(fh.read(), "PASS2\nPASS3\n") 324 325 def test_preprocessor_dependencies(self): 326 manifest = self.tmppath("m") 327 deps = self.tmppath("m.pp") 328 dest = self.tmppath("dest") 329 source = self.tmppath("p_source") 330 destfile = self.tmppath("dest/p_dest") 331 include = self.tmppath("p_incl") 332 os.mkdir(dest) 333 334 with open(source, "w") as fh: 335 fh.write("#define SRC\nSOURCE\n") 336 time = os.path.getmtime(source) - 3 337 os.utime(source, (time, time)) 338 339 with open(include, "w") as fh: 340 fh.write("INCLUDE\n") 341 time = os.path.getmtime(source) - 3 342 os.utime(include, (time, time)) 343 344 # Create and write a manifest with the preprocessed file. 345 m = InstallManifest() 346 m.add_preprocess(source, "p_dest", deps, "#", {"FOO": "BAR", "BAZ": "QUX"}) 347 m.write(path=manifest) 348 349 time = os.path.getmtime(source) - 5 350 os.utime(manifest, (time, time)) 351 352 # Now read the manifest back in, and apply it. This should write out 353 # our preprocessed file. 354 m = InstallManifest(path=manifest) 355 c = FileCopier() 356 m.populate_registry(c) 357 self.assertTrue(c.copy(dest)) 358 359 with open(destfile) as fh: 360 self.assertEqual(fh.read(), "SOURCE\n") 361 362 # Next, modify the source to #INCLUDE another file. 363 with open(source, "w") as fh: 364 fh.write("SOURCE\n#include p_incl\n") 365 time = os.path.getmtime(source) - 1 366 os.utime(destfile, (time, time)) 367 368 # Apply the manifest, and confirm that it also reads the newly included 369 # file. 370 m = InstallManifest(path=manifest) 371 c = FileCopier() 372 m.populate_registry(c) 373 c.copy(dest) 374 375 with open(destfile) as fh: 376 self.assertEqual(fh.read(), "SOURCE\nINCLUDE\n") 377 378 # Set the time on the source file back, so it won't be picked up as 379 # modified in the next test. 380 time = os.path.getmtime(source) - 1 381 os.utime(source, (time, time)) 382 383 # Now, modify the include file (but not the original source). 384 with open(include, "w") as fh: 385 fh.write("INCLUDE MODIFIED\n") 386 time = os.path.getmtime(include) - 1 387 os.utime(destfile, (time, time)) 388 389 # Apply the manifest, and confirm that the change to the include file 390 # is detected. That should cause the preprocessor to run again. 391 m = InstallManifest(path=manifest) 392 c = FileCopier() 393 m.populate_registry(c) 394 c.copy(dest) 395 396 with open(destfile) as fh: 397 self.assertEqual(fh.read(), "SOURCE\nINCLUDE MODIFIED\n") 398 399 # ORing an InstallManifest should copy file dependencies 400 m = InstallManifest() 401 m |= InstallManifest(path=manifest) 402 c = FileCopier() 403 m.populate_registry(c) 404 e = c._files["p_dest"] 405 self.assertEqual(e.extra_depends, [manifest]) 406 407 def test_add_entries_from(self): 408 source = self.tmppath("source") 409 os.mkdir(source) 410 os.mkdir("%s/base" % source) 411 os.mkdir("%s/base/foo" % source) 412 413 with open("%s/base/foo/file1" % source, "a"): 414 pass 415 416 with open("%s/base/foo/file2" % source, "a"): 417 pass 418 419 m = InstallManifest() 420 m.add_pattern_link("%s/base" % source, "**", "dest") 421 422 p = InstallManifest() 423 p.add_entries_from(m) 424 self.assertEqual(len(p), 1) 425 426 c = FileCopier() 427 p.populate_registry(c) 428 self.assertEqual(c.paths(), ["dest/foo/file1", "dest/foo/file2"]) 429 430 q = InstallManifest() 431 q.add_entries_from(m, base="target") 432 self.assertEqual(len(q), 1) 433 434 d = FileCopier() 435 q.populate_registry(d) 436 self.assertEqual(d.paths(), ["target/dest/foo/file1", "target/dest/foo/file2"]) 437 438 # Some of the values in an InstallManifest include destination 439 # information that is present in the keys. Verify that we can 440 # round-trip serialization. 441 r = InstallManifest() 442 r.add_entries_from(m) 443 r.add_entries_from(m, base="target") 444 self.assertEqual(len(r), 2) 445 446 temp_path = self.tmppath("temp_path") 447 r.write(path=temp_path) 448 449 s = InstallManifest(path=temp_path) 450 e = FileCopier() 451 s.populate_registry(e) 452 453 self.assertEqual( 454 e.paths(), 455 [ 456 "dest/foo/file1", 457 "dest/foo/file2", 458 "target/dest/foo/file1", 459 "target/dest/foo/file2", 460 ], 461 ) 462 463 464 if __name__ == "__main__": 465 mozunit.main()