vendor-libwebrtc.py (21063B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 import argparse 5 import datetime 6 import os 7 import shutil 8 import stat 9 import subprocess 10 import sys 11 import tarfile 12 13 import dateutil 14 import requests 15 16 THIRDPARTY_USED_IN_FIREFOX = [ 17 "crc32c", 18 "pffft", 19 "rnnoise", 20 ] 21 22 LIBWEBRTC_DIR = os.path.normpath("third_party/libwebrtc") 23 24 25 # Files in this list are excluded. 26 def get_excluded_files(): 27 return [ 28 ".clang-format", 29 ".git-blame-ignore-revs", 30 ".gitignore", 31 "CODE_OF_CONDUCT.md", 32 "ENG_REVIEW_OWNERS", 33 "PRESUBMIT.py", 34 "README.chromium", 35 "WATCHLISTS", 36 "codereview.settings", 37 "license_template.txt", 38 "native-api.md", 39 "presubmit_test.py", 40 "presubmit_test_mocks.py", 41 "pylintrc", 42 "api/location.h", 43 "modules/desktop_capture/linux/x11/x_error_trap.cc", 44 "modules/desktop_capture/linux/x11/x_error_trap.h", 45 "rtc_base/trace_event.h", 46 ] 47 48 49 # Directories in this list are excluded. Directories are handled 50 # separately from files so that script 'filter_git_changes.py' can use 51 # different regex handling for directory paths. 52 def get_excluded_dirs(): 53 return [ 54 "build_overrides", 55 # Only the camera code under sdk/android/api/org/webrtc is used, so 56 # we remove sdk/android and add back the specific files we want. 57 "sdk/android", 58 ] 59 60 61 # Paths in this list are included even if their parent directory is 62 # excluded in get_excluded_dirs() 63 def get_included_path_overrides(): 64 return [ 65 "sdk/android/src/java/org/webrtc/NativeLibrary.java", 66 "sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java", 67 "sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java", 68 "sdk/android/src/java/org/webrtc/BitrateAdjuster.java", 69 "sdk/android/src/java/org/webrtc/MediaCodecWrapperFactory.java", 70 "sdk/android/src/java/org/webrtc/WebRtcClassLoader.java", 71 "sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java", 72 "sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java", 73 "sdk/android/src/java/org/webrtc/audio/WebRtcAudioManager.java", 74 "sdk/android/src/java/org/webrtc/audio/LowLatencyAudioBufferManager.java", 75 "sdk/android/src/java/org/webrtc/audio/WebRtcAudioUtils.java", 76 "sdk/android/src/java/org/webrtc/audio/WebRtcAudioEffects.java", 77 "sdk/android/src/java/org/webrtc/audio/VolumeLogger.java", 78 "sdk/android/src/java/org/webrtc/NativeCapturerObserver.java", 79 "sdk/android/src/java/org/webrtc/MediaCodecWrapper.java", 80 "sdk/android/src/java/org/webrtc/CalledByNative.java", 81 "sdk/android/src/java/org/webrtc/Histogram.java", 82 "sdk/android/src/java/org/webrtc/EglBase10Impl.java", 83 "sdk/android/src/java/org/webrtc/EglBase14Impl.java", 84 "sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java", 85 "sdk/android/src/java/org/webrtc/AndroidVideoDecoder.java", 86 "sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java", 87 "sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java", 88 "sdk/android/src/java/org/webrtc/VideoCodecMimeType.java", 89 "sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java", 90 "sdk/android/src/java/org/webrtc/VideoDecoderWrapper.java", 91 "sdk/android/src/java/org/webrtc/JNILogging.java", 92 "sdk/android/src/java/org/webrtc/CameraCapturer.java", 93 "sdk/android/src/java/org/webrtc/CameraSession.java", 94 "sdk/android/src/java/org/webrtc/H264Utils.java", 95 "sdk/android/src/java/org/webrtc/Empty.java", 96 "sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java", 97 "sdk/android/src/java/org/webrtc/Camera1Session.java", 98 "sdk/android/src/java/org/webrtc/JniCommon.java", 99 "sdk/android/src/java/org/webrtc/NV12Buffer.java", 100 "sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java", 101 "sdk/android/src/java/org/webrtc/GlGenericDrawer.java", 102 "sdk/android/src/java/org/webrtc/RefCountDelegate.java", 103 "sdk/android/src/java/org/webrtc/Camera2Session.java", 104 "sdk/android/src/java/org/webrtc/MediaCodecUtils.java", 105 "sdk/android/src/java/org/webrtc/CalledByNativeUnchecked.java", 106 "sdk/android/src/java/org/webrtc/VideoEncoderWrapper.java", 107 "sdk/android/src/java/org/webrtc/NV21Buffer.java", 108 "sdk/android/api/org/webrtc/RendererCommon.java", 109 "sdk/android/api/org/webrtc/RenderSynchronizer.java", 110 "sdk/android/api/org/webrtc/YuvHelper.java", 111 "sdk/android/api/org/webrtc/LibvpxVp9Encoder.java", 112 "sdk/android/api/org/webrtc/Metrics.java", 113 "sdk/android/api/org/webrtc/CryptoOptions.java", 114 "sdk/android/api/org/webrtc/MediaConstraints.java", 115 "sdk/android/api/org/webrtc/YuvConverter.java", 116 "sdk/android/api/org/webrtc/JavaI420Buffer.java", 117 "sdk/android/api/org/webrtc/VideoDecoder.java", 118 "sdk/android/api/org/webrtc/WrappedNativeVideoDecoder.java", 119 "sdk/android/api/org/webrtc/Camera2Enumerator.java", 120 "sdk/android/api/org/webrtc/SurfaceTextureHelper.java", 121 "sdk/android/api/org/webrtc/EglBase10.java", 122 "sdk/android/api/org/webrtc/DataChannel.java", 123 "sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java", 124 "sdk/android/api/org/webrtc/audio/AudioDeviceModule.java", 125 "sdk/android/api/org/webrtc/SessionDescription.java", 126 "sdk/android/api/org/webrtc/GlUtil.java", 127 "sdk/android/api/org/webrtc/VideoSource.java", 128 "sdk/android/api/org/webrtc/AudioTrack.java", 129 "sdk/android/api/org/webrtc/EglRenderer.java", 130 "sdk/android/api/org/webrtc/EglThread.java", 131 "sdk/android/api/org/webrtc/VideoEncoder.java", 132 "sdk/android/api/org/webrtc/VideoCapturer.java", 133 "sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java", 134 "sdk/android/api/org/webrtc/AudioSource.java", 135 "sdk/android/api/org/webrtc/GlRectDrawer.java", 136 "sdk/android/api/org/webrtc/StatsReport.java", 137 "sdk/android/api/org/webrtc/CameraVideoCapturer.java", 138 "sdk/android/api/org/webrtc/NetEqFactoryFactory.java", 139 "sdk/android/api/org/webrtc/AudioProcessingFactory.java", 140 "sdk/android/api/org/webrtc/Camera2Capturer.java", 141 "sdk/android/api/org/webrtc/ScreenCapturerAndroid.java", 142 "sdk/android/api/org/webrtc/RefCounted.java", 143 "sdk/android/api/org/webrtc/VideoEncoderFallback.java", 144 "sdk/android/api/org/webrtc/AudioEncoderFactoryFactory.java", 145 "sdk/android/api/org/webrtc/EglBase14.java", 146 "sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java", 147 "sdk/android/api/org/webrtc/VideoEncoderFactory.java", 148 "sdk/android/api/org/webrtc/StatsObserver.java", 149 "sdk/android/api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java", 150 "sdk/android/api/org/webrtc/Camera1Capturer.java", 151 "sdk/android/api/org/webrtc/AddIceObserver.java", 152 "sdk/android/api/org/webrtc/SurfaceViewRenderer.java", 153 "sdk/android/api/org/webrtc/CameraEnumerator.java", 154 "sdk/android/api/org/webrtc/CameraEnumerationAndroid.java", 155 "sdk/android/api/org/webrtc/VideoDecoderFallback.java", 156 "sdk/android/api/org/webrtc/FileVideoCapturer.java", 157 "sdk/android/api/org/webrtc/NativeLibraryLoader.java", 158 "sdk/android/api/org/webrtc/Camera1Enumerator.java", 159 "sdk/android/api/org/webrtc/NativePeerConnectionFactory.java", 160 "sdk/android/api/org/webrtc/LibaomAv1Encoder.java", 161 "sdk/android/api/org/webrtc/BuiltinAudioEncoderFactoryFactory.java", 162 "sdk/android/api/org/webrtc/AudioDecoderFactoryFactory.java", 163 "sdk/android/api/org/webrtc/FecControllerFactoryFactoryInterface.java", 164 "sdk/android/api/org/webrtc/VideoFrameBufferType.java", 165 "sdk/android/api/org/webrtc/SdpObserver.java", 166 "sdk/android/api/org/webrtc/Predicate.java", 167 "sdk/android/api/org/webrtc/VideoFileRenderer.java", 168 "sdk/android/api/org/webrtc/WrappedNativeVideoEncoder.java", 169 "sdk/android/api/org/webrtc/LibvpxVp8Encoder.java", 170 "sdk/android/api/org/webrtc/DtmfSender.java", 171 "sdk/android/api/org/webrtc/VideoTrack.java", 172 "sdk/android/api/org/webrtc/LibvpxVp8Decoder.java", 173 "sdk/android/api/org/webrtc/GlShader.java", 174 "sdk/android/api/org/webrtc/FrameEncryptor.java", 175 "sdk/android/api/org/webrtc/EglBase.java", 176 "sdk/android/api/org/webrtc/VideoProcessor.java", 177 "sdk/android/api/org/webrtc/SSLCertificateVerifier.java", 178 "sdk/android/api/org/webrtc/VideoSink.java", 179 "sdk/android/api/org/webrtc/MediaSource.java", 180 "sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java", 181 "sdk/android/api/org/webrtc/VideoCodecInfo.java", 182 "sdk/android/api/org/webrtc/FrameDecryptor.java", 183 "sdk/android/api/org/webrtc/VideoDecoderFactory.java", 184 "sdk/android/api/org/webrtc/TextureBufferImpl.java", 185 "sdk/android/api/org/webrtc/VideoFrame.java", 186 "sdk/android/api/org/webrtc/IceCandidateErrorEvent.java", 187 "sdk/android/api/org/webrtc/CapturerObserver.java", 188 "sdk/android/api/org/webrtc/MediaStreamTrack.java", 189 "sdk/android/api/org/webrtc/GlTextureFrameBuffer.java", 190 "sdk/android/api/org/webrtc/TurnCustomizer.java", 191 "sdk/android/api/org/webrtc/TimestampAligner.java", 192 "sdk/android/api/org/webrtc/BuiltinAudioDecoderFactoryFactory.java", 193 "sdk/android/api/org/webrtc/LibvpxVp9Decoder.java", 194 "sdk/android/api/org/webrtc/SurfaceEglRenderer.java", 195 "sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java", 196 "sdk/android/api/org/webrtc/VideoCodecStatus.java", 197 "sdk/android/api/org/webrtc/Dav1dDecoder.java", 198 "sdk/android/api/org/webrtc/VideoFrameDrawer.java", 199 "sdk/android/api/org/webrtc/CallSessionFileRotatingLogSink.java", 200 "sdk/android/api/org/webrtc/EncodedImage.java", 201 ] 202 203 204 def make_github_url(repo, commit): 205 if not repo.endswith("/"): 206 repo += "/" 207 return repo + "archive/" + commit + ".tar.gz" 208 209 210 def make_googlesource_url(target, commit): 211 if target == "libwebrtc": 212 return "https://webrtc.googlesource.com/src.git/+archive/" + commit + ".tar.gz" 213 elif target == "build": 214 return ( 215 "https://chromium.googlesource.com/chromium/src/build/+archive/" 216 + commit 217 + ".tar.gz" 218 ) 219 elif target == "third_party": 220 return ( 221 "https://chromium.googlesource.com/chromium/src/third_party/+archive/" 222 + commit 223 + ".tar.gz" 224 ) 225 226 227 def fetch(target, url): 228 print(f"Fetching commit from {url}") 229 req = requests.get(url) 230 if req.status_code == 200: 231 with open(target + ".tar.gz", "wb") as f: 232 f.write(req.content) 233 else: 234 print( 235 f"Hit status code {req.status_code} fetching commit. Aborting.", 236 file=sys.stderr, 237 ) 238 sys.exit(1) 239 with open(os.path.join(LIBWEBRTC_DIR, "README.mozilla.last-vendor"), "w") as f: 240 # write the the command line used 241 f.write(f"# ./mach python {' '.join(sys.argv[0:])}\n") 242 f.write( 243 f"{target} updated from commit {url} on {datetime.datetime.now(dateutil.tz.tzutc()).isoformat()}.\n" 244 ) 245 246 247 def fetch_local(target, path, commit): 248 target_archive = target + ".tar.gz" 249 cp = subprocess.run( 250 ["git", "archive", "-o", target_archive, commit], cwd=path, check=False 251 ) 252 if cp.returncode != 0: 253 print( 254 f"Hit return code {cp.returncode} fetching commit. Aborting.", 255 file=sys.stderr, 256 ) 257 sys.exit(1) 258 259 with open(os.path.join(LIBWEBRTC_DIR, "README.mozilla.last-vendor"), "w") as f: 260 # write the the command line used 261 f.write(f"# ./mach python {' '.join(sys.argv[0:])}\n") 262 f.write( 263 f"{target} updated from {path} commit {commit} on {datetime.datetime.now(dateutil.tz.tzutc()).isoformat()}.\n" 264 ) 265 shutil.move(os.path.join(path, target_archive), target_archive) 266 267 268 def validate_tar_member(member, path): 269 def _is_within_directory(directory, target): 270 real_directory = os.path.realpath(directory) 271 real_target = os.path.realpath(target) 272 prefix = os.path.commonprefix([real_directory, real_target]) 273 return prefix == real_directory 274 275 member_path = os.path.join(path, member.name) 276 if not _is_within_directory(path, member_path): 277 raise Exception("Attempted path traversal in tar file: " + member.name) 278 if member.issym(): 279 link_path = os.path.join(os.path.dirname(member_path), member.linkname) 280 if not _is_within_directory(path, link_path): 281 raise Exception("Attempted link path traversal in tar file: " + member.name) 282 if member.mode & (stat.S_ISUID | stat.S_ISGID): 283 raise Exception("Attempted setuid or setgid in tar file: " + member.name) 284 285 286 def safe_extract(tar, path=".", *, numeric_owner=False): 287 def _files(tar, path): 288 for member in tar: 289 validate_tar_member(member, path) 290 yield member 291 292 # Once all our platforms are using Python 3.12 or newer, we can 293 # remove the conditionality of the filter argument and always 294 # include it. Currently I see: 295 # ubuntu 20.04 - Python 3.8.10 (probably no longer regularly used) 296 # 22.04 - Python 3.10.12 297 # 24.04 - Python 3.12.3 298 # macOS Sequoia 15.3.2 - Python 3.9.6 (the real issue) 299 tar.extractall( 300 path, 301 members=_files(tar, path), 302 numeric_owner=numeric_owner, 303 **({} if sys.version_info < (3, 12) else dict(filter="tar")), 304 ) 305 306 307 def unpack(target): 308 target_archive = target + ".tar.gz" 309 target_path = "tmp-" + target 310 try: 311 shutil.rmtree(target_path) 312 except FileNotFoundError: 313 pass 314 with tarfile.open(target_archive) as t: 315 safe_extract(t, path=target_path) 316 317 if target == "libwebrtc": 318 # use the top level directories from the tarfile and 319 # delete those directories in LIBWEBRTC_DIR 320 libwebrtc_used_in_firefox = os.listdir(target_path) 321 for path in libwebrtc_used_in_firefox: 322 try: 323 shutil.rmtree(os.path.join(LIBWEBRTC_DIR, path)) 324 except FileNotFoundError: 325 pass 326 except NotADirectoryError: 327 pass 328 329 unused_libwebrtc_in_firefox = get_excluded_files() + get_excluded_dirs() 330 forced_used_in_firefox = get_included_path_overrides() 331 332 # adjust target_path if GitHub packaging is involved 333 if not os.path.exists(os.path.join(target_path, libwebrtc_used_in_firefox[0])): 334 # GitHub packs everything inside a separate directory 335 target_path = os.path.join(target_path, os.listdir(target_path)[0]) 336 337 # remove any entries found in unused_libwebrtc_in_firefox from the 338 # tarfile 339 for path in unused_libwebrtc_in_firefox: 340 if os.path.isdir(os.path.join(target_path, path)): 341 shutil.rmtree(os.path.join(target_path, path)) 342 else: 343 os.remove(os.path.join(target_path, path)) 344 345 # move remaining top level entries from the tarfile to LIBWEBRTC_DIR 346 for path in os.listdir(target_path): 347 shutil.move( 348 os.path.join(target_path, path), os.path.join(LIBWEBRTC_DIR, path) 349 ) 350 351 # An easy, but inefficient way to accomplish including specific 352 # files from directories otherwise removed. Re-extract the tar 353 # file, and only copy over the exact files requested. 354 shutil.rmtree(target_path) 355 with tarfile.open(target_archive) as t: 356 safe_extract(t, path=target_path) 357 358 # Copy the force included files. Note: the instinctual action 359 # is to do this prior to removing the excluded paths to avoid 360 # reextracting the tar file. However, this causes errors due to 361 # pre-existing paths when other directories are moved out of the 362 # tar file in the "move all the top level entries from the 363 # tarfile" phase above. 364 for path in forced_used_in_firefox: 365 dest_path = os.path.join(LIBWEBRTC_DIR, path) 366 dir_path = os.path.dirname(dest_path) 367 if not os.path.exists(dir_path): 368 os.makedirs(dir_path) 369 shutil.move(os.path.join(target_path, path), dest_path) 370 371 elif target == "build": 372 # adjust target_path if GitHub packaging is involved 373 if not os.path.exists(os.path.join(target_path, "linux")): 374 # GitHub packs everything inside a separate directory 375 target_path = os.path.join(target_path, os.listdir(target_path)[0]) 376 377 build_used_in_firefox = os.listdir(target_path) 378 for path in build_used_in_firefox: 379 try: 380 shutil.rmtree(os.path.join(LIBWEBRTC_DIR, path)) 381 except FileNotFoundError: 382 pass 383 except NotADirectoryError: 384 pass 385 386 for path in os.listdir(target_path): 387 shutil.move( 388 os.path.join(target_path, path), 389 os.path.join(LIBWEBRTC_DIR, path), 390 ) 391 392 elif target == "third_party": 393 # Only delete the THIRDPARTY_USED_IN_FIREFOX paths from 394 # LIBWEBRTC_DIR/third_party to avoid deleting directories that 395 # we use to trampoline to libraries already in mozilla's tree. 396 for path in THIRDPARTY_USED_IN_FIREFOX: 397 try: 398 shutil.rmtree(os.path.join(LIBWEBRTC_DIR, path)) 399 except FileNotFoundError: 400 pass 401 except NotADirectoryError: 402 pass 403 404 # adjust target_path if GitHub packaging is involved 405 if not os.path.exists(os.path.join(target_path, THIRDPARTY_USED_IN_FIREFOX[0])): 406 # GitHub packs everything inside a separate directory 407 target_path = os.path.join(target_path, os.listdir(target_path)[0]) 408 409 for path in THIRDPARTY_USED_IN_FIREFOX: 410 shutil.move( 411 os.path.join(target_path, path), 412 os.path.join(LIBWEBRTC_DIR, path), 413 ) 414 415 elif target == "abseil-cpp": 416 # adjust target_path if GitHub packaging is involved 417 if not os.path.exists(os.path.join(target_path, "abseil-cpp")): 418 # GitHub packs everything inside a separate directory 419 target_path = os.path.join(target_path, os.listdir(target_path)[0]) 420 421 abseil_path = os.path.join(target_path, "abseil-cpp") 422 423 abseil_used_in_firefox = os.listdir(abseil_path) 424 for path in abseil_used_in_firefox: 425 try: 426 shutil.rmtree(os.path.join(LIBWEBRTC_DIR, path)) 427 except FileNotFoundError: 428 pass 429 except NotADirectoryError: 430 pass 431 432 for path in os.listdir(abseil_path): 433 shutil.move( 434 os.path.join(target_path, target, path), 435 os.path.join(LIBWEBRTC_DIR, path), 436 ) 437 438 439 def cleanup(target): 440 os.remove(target + ".tar.gz") 441 shutil.rmtree("tmp-" + target) 442 443 444 if __name__ == "__main__": 445 parser = argparse.ArgumentParser(description="Update libwebrtc") 446 parser.add_argument( 447 "target", choices=("libwebrtc", "build", "third_party", "abseil-cpp") 448 ) 449 group = parser.add_mutually_exclusive_group(required=True) 450 group.add_argument("--from-github", type=str) 451 group.add_argument("--from-googlesource", action="store_true", default=False) 452 group.add_argument("--from-local", type=str) 453 parser.add_argument("--commit", type=str, default="master") 454 parser.add_argument("--skip-fetch", action="store_true", default=False) 455 parser.add_argument("--skip-cleanup", action="store_true", default=False) 456 args = parser.parse_args() 457 458 # the default for LIBWEBRTC_DIR is set for target libwebrtc 459 if args.target == "build": 460 LIBWEBRTC_DIR = os.path.normpath("third_party/chromium/build") 461 elif args.target == "third_party": 462 LIBWEBRTC_DIR = os.path.join(LIBWEBRTC_DIR, "third_party") 463 elif args.target == "abseil-cpp": 464 LIBWEBRTC_DIR = os.path.normpath("third_party/abseil-cpp") 465 466 os.makedirs(LIBWEBRTC_DIR, exist_ok=True) 467 468 if not args.skip_fetch: 469 if args.from_github: 470 fetch(args.target, make_github_url(args.from_github, args.commit)) 471 elif args.from_googlesource: 472 fetch(args.target, make_googlesource_url(args.target, args.commit)) 473 elif args.from_local: 474 fetch_local(args.target, args.from_local, args.commit) 475 unpack(args.target) 476 if not args.skip_cleanup: 477 cleanup(args.target)