build.py (8634B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 from copy import deepcopy 6 from taskgraph.transforms.base import TransformSequence 7 8 transforms = TransformSequence() 9 10 11 @transforms.add 12 def set_build_attributes(config, jobs): 13 """ 14 Set the build_platform and build_type attributes based on the job name. 15 """ 16 for job in jobs: 17 build_platform, build_type = job["name"].split("/") 18 attributes = job.setdefault("attributes", {}) 19 attributes.update( 20 { 21 "build_platform": build_platform, 22 "build_type": build_type, 23 } 24 ) 25 26 yield job 27 28 29 EXTRA_COMPILERS = { 30 "clang-10": {"CC": "clang-10", "CCC": "clang++-10"}, 31 "clang-18": {"CC": "clang-18", "CCC": "clang++-18"}, 32 # gcc-4.6 introduced nullptr. 33 "gcc-4.4": {"CC": "gcc-4.4", "CCC": "g++-4.4", "NSS_DISABLE_GTESTS": "1"}, 34 # gcc-4.8 has incomplete c++11 support 35 "gcc-4.8": {"CC": "gcc-4.8", "CCC": "g++-4.8", "NSS_DISABLE_GTESTS": "1"}, 36 "gcc-5": {"CC": "gcc-5", "CCC": "g++-5"}, 37 "gcc-11": {"CC": "gcc-11", "CCC": "g++-11"}, 38 } 39 40 41 @transforms.add 42 def add_variants(config, jobs): 43 for job in jobs: 44 attributes = job["attributes"] 45 46 # nspr 47 if not any(attributes.get(attr) for attr in ("make", "asan", "fuzz")): 48 nspr_job = deepcopy(job) 49 nspr_job["name"] = f"{attributes['build_platform']}-nspr/{attributes['build_type']}" 50 nspr_job["description"]+= " (NSPR only)" 51 nspr_job["attributes"]["nspr"] = True 52 yield nspr_job 53 54 # base build 55 build_job = deepcopy(job) 56 build_job["attributes"].setdefault("certs", True) 57 yield build_job 58 59 # fips 60 if attributes.get("make"): 61 fips_build = deepcopy(job) 62 fips_build["name"] = f"{attributes['build_platform']}-fips/{attributes['build_type']}" 63 fips_build["description"] += " w/ NSS_FORCE_FIPS" 64 fips_build.setdefault("worker", {}).setdefault("env", {})["NSS_FORCE_FIPS"] = "1" 65 fips_build["attributes"]["make-fips"] = True 66 fips_build["attributes"]["certs"] = True 67 yield fips_build 68 69 if "linux" in attributes["build_platform"]: 70 # more compilers 71 if not attributes.get("asan") and not attributes.get("fuzz"): 72 for cc in EXTRA_COMPILERS: 73 if cc == "gcc-4.4" and not (attributes.get("make") and job["attributes"]["build_platform"].startswith("linux64")): 74 # Use the old Makefile-based build system, GYP doesn't have a proper GCC 75 # version check for __int128 support. It's mainly meant to cover RHEL6. 76 continue 77 if cc == "gcc-4.8" and not attributes.get("make"): 78 # Use -Ddisable-intelhw_sha=1, GYP doesn't have a proper GCC version 79 # check for Intel SHA support. 80 continue 81 cc_job = deepcopy(job) 82 cc_job["name"] += f"-{cc}" 83 cc_job["description"] += f" w/ {cc}" 84 cc_job["attributes"]["cc"] = cc 85 cc_job.setdefault("worker", {}).setdefault("env", {}).update(EXTRA_COMPILERS[cc]) 86 yield cc_job 87 88 # modular 89 if attributes.get("make"): 90 modular_job = deepcopy(job) 91 modular_job["attributes"]["modular"] = True 92 modular_job.setdefault("worker", {}).setdefault("env", {})["NSS_BUILD_MODULAR"] = "1" 93 modular_job["name"] += "-modular" 94 modular_job["description"] += " w/ modular builds" 95 yield modular_job 96 97 # dbm 98 if not attributes.get("make") and not attributes.get("fuzz"): 99 dbm_job = deepcopy(job) 100 dbm_job["attributes"]["dbm"] = True 101 dbm_job["attributes"].setdefault("certs", True) 102 dbm_job["name"] += "-dbm" 103 dbm_job["description"] += " w/ legacy-db" 104 yield dbm_job 105 106 if attributes.get("fuzz"): 107 tlsfuzz_job = deepcopy(job) 108 tlsfuzz_job["attributes"]["tlsfuzz"] = True 109 tlsfuzz_job["name"] = job["name"].replace("-fuzz", "-tlsfuzz") 110 tlsfuzz_job["description"] = job["description"].replace("fuzz", "TLS fuzz") 111 yield tlsfuzz_job 112 113 114 @transforms.add 115 def set_attributes_defaults(config, jobs): 116 for job in jobs: 117 attributes = job["attributes"] 118 for attr in ("make", "asan", "make-fips", "fuzz", "certs", "nspr", "cc", "dbm", "modular", "tlsfuzz"): 119 attributes.setdefault(attr, False) 120 yield job 121 122 123 @transforms.add 124 def set_make_command(config, jobs): 125 for job in jobs: 126 if not job["attributes"].get("make"): 127 yield job 128 continue 129 platform = job["attributes"]["build_platform"] 130 if "win" in platform: 131 script = "${VCS_PATH}/nss/automation/taskcluster/windows/build.sh" 132 else: 133 script = "${VCS_PATH}/nss/automation/taskcluster/scripts/build.sh" 134 if "64" in job["attributes"]["build_platform"]: 135 job["worker"].setdefault("env", {})["USE_64"] = "1" 136 job["run"]["command"] = script 137 yield job 138 139 140 @transforms.add 141 def set_gyp_command(config, jobs): 142 for job in jobs: 143 attributes = job["attributes"] 144 if attributes.get("make"): 145 yield job 146 continue 147 platform = attributes["build_platform"] 148 if "win" in platform: 149 script = "${VCS_PATH}/nss/automation/taskcluster/windows/build_gyp.sh" 150 else: 151 script = "${VCS_PATH}/nss/automation/taskcluster/scripts/build_gyp.sh" 152 command = script + " --python=python3" 153 if "64" not in platform: 154 command += " -t ia32" 155 if attributes["build_type"] in ("opt", "opt-static"): 156 command += " --opt" 157 if "fips" in attributes["build_type"]: 158 command += " --enable-fips" 159 if attributes.get("asan"): 160 command += " --ubsan --asan" 161 if attributes.get("nspr"): 162 command += " --nspr-only --nspr-test-build --nspr-test-run" 163 if attributes.get("static"): 164 command += " --static -Ddisable_libpkix=1" 165 if attributes.get("fuzz"): 166 command += " --disable-tests -Ddisable_libpkix=1 --fuzz" 167 job.setdefault("worker", {}).setdefault("env", {}).update({ 168 "ASAN_OPTIONS": "allocator_may_return_null=1:detect_stack_use_after_return=1", 169 "UBSAN_OPTIONS": "print_stacktrace=1", 170 "NSS_DISABLE_ARENA_FREE_LIST": "1", 171 "NSS_DISABLE_UNLOAD": "1", 172 "CC": "clang", 173 "CCC": "clang++", 174 }) 175 if attributes.get("tlsfuzz"): 176 command += "=tls" 177 else: 178 command += " && nss/automation/taskcluster/scripts/build_cryptofuzz.sh" 179 if "64" not in platform: 180 command += " --i386" 181 job["run"]["command"] = command 182 yield job 183 184 @transforms.add 185 def set_docker_image(config, jobs): 186 for job in jobs: 187 if "linux" not in job["attributes"]["build_platform"]: 188 yield job 189 continue 190 191 if job["attributes"].get("cc") == "gcc-4.4": 192 image = "gcc-4.4" 193 elif job["attributes"].get("cc"): 194 image = "builds" 195 elif job["attributes"].get("fuzz"): 196 image = "fuzz" 197 else: 198 image = "base" 199 job["worker"]["docker-image"] = {"in-tree": image} 200 yield job 201 202 203 def get_job_symbol(job): 204 if job["attributes"].get("nspr"): 205 return "NSPR" 206 if job["attributes"].get("cc"): 207 return f"Builds({job['attributes']['cc']})" 208 if job["attributes"].get("tlsfuzz"): 209 return "TLS(B)" 210 if job["attributes"].get("dbm"): 211 return "DBM(B)" 212 if job["attributes"].get("make-fips"): 213 return "FIPS(B)" 214 if job["attributes"].get("modular"): 215 return "Builds(modular)" 216 return "B" 217 218 @transforms.add 219 def set_treeherder_symbol(config, jobs): 220 for job in jobs: 221 job["treeherder"].update({ 222 "symbol": get_job_symbol(job), 223 "platform": f'{job["attributes"]["build_platform"]}/{job["attributes"]["build_type"]}', 224 }) 225 yield job