ci.sh (51251B)
1 #!/usr/bin/env bash 2 # Copyright (c) the JPEG XL Project Authors. All rights reserved. 3 # 4 # Use of this source code is governed by a BSD-style 5 # license that can be found in the LICENSE file. 6 7 # Continuous integration helper module. This module is meant to be called from 8 # workflows during the continuous integration build, as well as from the 9 # command line for developers. 10 11 set -eu 12 13 OS=`uname -s` 14 15 SELF=$(realpath "$0") 16 MYDIR=$(dirname "${SELF}") 17 18 ### Environment parameters: 19 TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-256}" 20 BENCHMARK_NUM_THREADS="${BENCHMARK_NUM_THREADS:-0}" 21 BUILD_CONFIG=${BUILD_CONFIG:-} 22 CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo} 23 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-} 24 CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-} 25 CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-} 26 CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-} 27 SKIP_BUILD="${SKIP_BUILD:-0}" 28 SKIP_TEST="${SKIP_TEST:-0}" 29 FASTER_MSAN_BUILD="${FASTER_MSAN_BUILD:-0}" 30 TARGETS="${TARGETS:-all doc}" 31 TEST_SELECTOR="${TEST_SELECTOR:-}" 32 BUILD_TARGET="${BUILD_TARGET:-}" 33 ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}" 34 if [[ -n "${BUILD_TARGET}" ]]; then 35 BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}" 36 else 37 BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}" 38 fi 39 # Whether we should post a message in the MR when the build fails. 40 POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}" 41 # By default, do a lightweight debian HWY package build. 42 HWY_PKG_OPTIONS="${HWY_PKG_OPTIONS:---set-envvar=HWY_EXTRA_CONFIG=-DBUILD_TESTING=OFF -DHWY_ENABLE_EXAMPLES=OFF -DHWY_ENABLE_CONTRIB=OFF}" 43 44 # Set default compilers to clang if not already set 45 export CC=${CC:-clang} 46 export CXX=${CXX:-clang++} 47 48 # Time limit for the "fuzz" command in seconds (0 means no limit). 49 FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}" 50 51 SANITIZER="none" 52 53 54 if [[ "${BUILD_TARGET%%-*}" == "x86_64" || 55 "${BUILD_TARGET%%-*}" == "i686" ]]; then 56 # Default to building all targets, even if compiler baseline is SSE4 57 HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_EMU128} 58 else 59 HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-} 60 fi 61 62 # Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS 63 CMAKE_FLAGS=${CMAKE_FLAGS:-} 64 CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}" 65 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}" 66 67 CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-} 68 CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-} 69 CMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH:-} 70 CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-} 71 CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-} 72 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-} 73 74 if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then 75 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -msimd128" 76 CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -msimd128" 77 CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128" 78 fi 79 80 if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then 81 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2" 82 CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2" 83 fi 84 85 if [[ -z "${BUILD_CONFIG}" ]]; then 86 TOOLS_DIR="${BUILD_DIR}/tools" 87 else 88 TOOLS_DIR="${BUILD_DIR}/tools/${BUILD_CONFIG}" 89 fi 90 91 if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then 92 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}" 93 fi 94 95 # Version inferred from the CI variables. 96 CI_COMMIT_SHA=${GITHUB_SHA:-} 97 JPEGXL_VERSION=${JPEGXL_VERSION:-} 98 99 # Benchmark parameters 100 STORE_IMAGES=${STORE_IMAGES:-1} 101 BENCHMARK_CORPORA="${MYDIR}/third_party/corpora" 102 103 # Local flags passed to sanitizers. 104 UBSAN_FLAGS=( 105 -fsanitize=alignment 106 -fsanitize=bool 107 -fsanitize=bounds 108 -fsanitize=builtin 109 -fsanitize=enum 110 -fsanitize=float-cast-overflow 111 -fsanitize=float-divide-by-zero 112 -fsanitize=integer-divide-by-zero 113 -fsanitize=null 114 -fsanitize=object-size 115 -fsanitize=pointer-overflow 116 -fsanitize=return 117 -fsanitize=returns-nonnull-attribute 118 -fsanitize=shift-base 119 -fsanitize=shift-exponent 120 -fsanitize=unreachable 121 -fsanitize=vla-bound 122 123 -fno-sanitize-recover=undefined 124 # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning. 125 -fsanitize-recover=alignment 126 ) 127 # -fsanitize=function doesn't work on aarch64 and arm. 128 if [[ "${BUILD_TARGET%%-*}" != "aarch64" && 129 "${BUILD_TARGET%%-*}" != "arm" ]]; then 130 UBSAN_FLAGS+=( 131 -fsanitize=function 132 ) 133 fi 134 if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then 135 UBSAN_FLAGS+=( 136 -fsanitize=signed-integer-overflow 137 ) 138 fi 139 140 CLANG_TIDY_BIN_CANDIDATES=( 141 clang-tidy 142 clang-tidy-6.0 143 clang-tidy-7 144 clang-tidy-8 145 clang-tidy-9 146 clang-tidy-10 147 clang-tidy-11 148 clang-tidy-12 149 clang-tidy-13 150 clang-tidy-14 151 clang-tidy-15 152 clang-tidy-16 153 clang-tidy-17 154 clang-tidy-18 155 ) 156 157 CLANG_TIDY_BIN=${CLANG_TIDY_BIN:-$(which ${CLANG_TIDY_BIN_CANDIDATES[@]} 2>/dev/null | tail -n 1)} 158 # Default to "cat" if "colordiff" is not installed or if stdout is not a tty. 159 if [[ -t 1 ]]; then 160 COLORDIFF_BIN=$(which colordiff cat 2>/dev/null | head -n 1) 161 else 162 COLORDIFF_BIN="cat" 163 fi 164 FIND_BIN=$(which gfind find 2>/dev/null | head -n 1) 165 # "false" will disable wine64 when not installed. This won't allow 166 # cross-compiling. 167 WINE_BIN=$(which wine64 false 2>/dev/null | head -n 1) 168 169 CLANG_VERSION="${CLANG_VERSION:-}" 170 # Detect the clang version suffix and store it in CLANG_VERSION. For example, 171 # "6.0" for clang 6 or "7" for clang 7. 172 detect_clang_version() { 173 if [[ -n "${CLANG_VERSION}" ]]; then 174 return 0 175 fi 176 local clang_version=$("${CC:-clang}" --version | head -n1) 177 clang_version=${clang_version#"Debian "} 178 clang_version=${clang_version#"Ubuntu "} 179 local llvm_tag 180 case "${clang_version}" in 181 "clang version 6."*) 182 CLANG_VERSION="6.0" 183 ;; 184 "clang version "*) 185 # Any other clang version uses just the major version number. 186 local suffix="${clang_version#clang version }" 187 CLANG_VERSION="${suffix%%.*}" 188 ;; 189 "emcc"*) 190 # We can't use asan or msan in the emcc case. 191 ;; 192 *) 193 echo "Unknown clang version: ${clang_version}" >&2 194 return 1 195 esac 196 } 197 198 # Temporary files cleanup hooks. 199 CLEANUP_FILES=() 200 cleanup() { 201 if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then 202 rm -fr "${CLEANUP_FILES[@]}" 203 fi 204 } 205 206 # Executed on exit. 207 on_exit() { 208 local retcode="$1" 209 # Always cleanup the CLEANUP_FILES. 210 cleanup 211 } 212 213 trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT 214 215 216 # These variables are populated when calling merge_request_commits(). 217 218 # The current hash at the top of the current branch or merge request branch (if 219 # running from a merge request pipeline). 220 MR_HEAD_SHA="" 221 # The common ancestor between the current commit and the tracked branch, such 222 # as main. This includes a list 223 MR_ANCESTOR_SHA="" 224 225 # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA. 226 merge_request_commits() { 227 { set +x; } 2>/dev/null 228 # GITHUB_SHA is the current reference being build in GitHub Actions. 229 if [[ -n "${GITHUB_SHA:-}" ]]; then 230 # GitHub normally does a checkout of a merge commit on a shallow repository 231 # by default. We want to get a bit more of the history to be able to diff 232 # changes on the Pull Request if needed. This fetches 10 more commits which 233 # should be enough given that PR normally should have 1 commit. 234 git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10 235 if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then 236 MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null || 237 echo "${GITHUB_SHA}")" 238 else 239 MR_HEAD_SHA="${GITHUB_SHA}" 240 fi 241 else 242 MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "HEAD") 243 fi 244 245 if [[ -n "${GITHUB_BASE_REF:-}" ]]; then 246 # Pull request workflow in GitHub Actions. GitHub checkout action uses 247 # "origin" as the remote for the git checkout. 248 git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}" 249 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD) 250 else 251 # We are in a local branch, not a pull request workflow. 252 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true) 253 fi 254 255 if [[ -z "${MR_ANCESTOR_SHA}" ]]; then 256 echo "Warning, not tracking any branch, using the last commit in HEAD.">&2 257 # This prints the return value with just HEAD. 258 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^") 259 else 260 # GitHub runs the pipeline on a merge commit, no need to look for the common 261 # ancestor in that case. 262 if [[ -z "${GITHUB_BASE_REF:-}" ]]; then 263 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base \ 264 "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}") 265 fi 266 fi 267 set -x 268 } 269 270 271 # Set up and export the environment variables needed by the child processes. 272 export_env() { 273 if [[ "${BUILD_TARGET}" == *mingw32 ]]; then 274 # Wine needs to know the paths to the mingw dlls. These should be 275 # separated by ';'. 276 WINEPATH=$("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \ 277 | grep -F 'libraries: =' | cut -f 2- -d '=' | tr ':' ';') 278 # We also need our own libraries in the wine path. 279 local real_build_dir=$(realpath "${BUILD_DIR}") 280 # Some library .dll dependencies are installed in /bin: 281 export WINEPATH="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin" 282 283 local prefix="${BUILD_DIR}/wineprefix" 284 mkdir -p "${prefix}" 285 export WINEPREFIX=$(realpath "${prefix}") 286 fi 287 # Sanitizers need these variables to print and properly format the stack 288 # traces: 289 LLVM_SYMBOLIZER=$("${CC:-clang}" -print-prog-name=llvm-symbolizer || true) 290 if [[ -n "${LLVM_SYMBOLIZER}" ]]; then 291 export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 292 export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 293 export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 294 fi 295 } 296 297 cmake_configure() { 298 export_env 299 300 if [[ "${STACK_SIZE:-0}" == 1 ]]; then 301 # Dump the stack size of each function in the .stack_sizes section for 302 # analysis. 303 CMAKE_C_FLAGS+=" -fstack-size-section" 304 CMAKE_CXX_FLAGS+=" -fstack-size-section" 305 fi 306 307 local args=( 308 -B"${BUILD_DIR}" -H"${MYDIR}" 309 -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" 310 -G Ninja 311 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" 312 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" 313 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" 314 -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}" 315 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" 316 -DJPEGXL_VERSION="${JPEGXL_VERSION}" 317 -DSANITIZER="${SANITIZER}" 318 # These are not enabled by default in cmake. 319 -DJPEGXL_ENABLE_VIEWERS=ON 320 -DJPEGXL_ENABLE_PLUGINS=ON 321 -DJPEGXL_ENABLE_DEVTOOLS=ON 322 # We always use libfuzzer in the ci.sh wrapper. 323 -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer" 324 ) 325 if [[ "${BUILD_TARGET}" != *mingw32 ]]; then 326 args+=( 327 -DJPEGXL_WARNINGS_AS_ERRORS=ON 328 ) 329 fi 330 if [[ -n "${BUILD_TARGET}" ]]; then 331 local system_name="Linux" 332 if [[ "${BUILD_TARGET}" == *mingw32 ]]; then 333 # When cross-compiling with mingw the target must be set to Windows and 334 # run programs with wine. 335 system_name="Windows" 336 args+=( 337 -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}" 338 # Normally CMake automatically defines MINGW=1 when building with the 339 # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling 340 # with clang. 341 -DMINGW=1 342 ) 343 fi 344 # EMSCRIPTEN toolchain sets the right values itself 345 if [[ "${BUILD_TARGET}" != wasm* ]]; then 346 # If set, BUILD_TARGET must be the target triplet such as 347 # x86_64-unknown-linux-gnu. 348 args+=( 349 -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}" 350 -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}" 351 # Only the first element of the target triplet. 352 -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}" 353 -DCMAKE_SYSTEM_NAME="${system_name}" 354 -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" 355 ) 356 else 357 args+=( 358 # sjpeg confuses WASM SIMD with SSE. 359 -DSJPEG_ENABLE_SIMD=OFF 360 # Building shared libs is not very useful for WASM. 361 -DBUILD_SHARED_LIBS=OFF 362 ) 363 fi 364 args+=( 365 # These are needed to make googletest work when cross-compiling. 366 -DCMAKE_CROSSCOMPILING=1 367 -DHAVE_STD_REGEX=0 368 -DHAVE_POSIX_REGEX=0 369 -DHAVE_GNU_POSIX_REGEX=0 370 -DHAVE_STEADY_CLOCK=0 371 -DHAVE_THREAD_SAFETY_ATTRIBUTES=0 372 ) 373 if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then 374 # find_package() will look in this prefix for libraries. 375 CMAKE_FIND_ROOT_PATH="/usr/${BUILD_TARGET}" 376 fi 377 if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then 378 CMAKE_PREFIX_PATH="/usr/${BUILD_TARGET}" 379 fi 380 # Use pkg-config for the target. If there's no pkg-config available for the 381 # target we can set the PKG_CONFIG_PATH to the appropriate path in most 382 # linux distributions. 383 local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true) 384 if [[ -z "${pkg_config}" ]]; then 385 pkg_config=$(which pkg-config) 386 export PKG_CONFIG_LIBDIR="/usr/${BUILD_TARGET}/lib/pkgconfig" 387 fi 388 if [[ -n "${pkg_config}" ]]; then 389 args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}") 390 fi 391 fi 392 if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then 393 args+=( 394 -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}" 395 ) 396 fi 397 if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then 398 args+=( 399 -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}" 400 ) 401 fi 402 if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then 403 args+=( 404 -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" 405 ) 406 fi 407 if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then 408 args+=( 409 -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}" 410 ) 411 fi 412 if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then 413 args+=( 414 -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}" 415 ) 416 fi 417 if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then 418 args+=( 419 -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}" 420 ) 421 fi 422 if [[ "${BUILD_TARGET}" == wasm* ]]; then 423 emcmake cmake "${args[@]}" "$@" 424 else 425 cmake "${args[@]}" "$@" 426 fi 427 } 428 429 cmake_build_and_test() { 430 if [[ "${SKIP_BUILD}" -eq "1" ]]; then 431 return 0 432 fi 433 # gtest_discover_tests() runs the test binaries to discover the list of tests 434 # at build time, which fails under qemu. 435 ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- $TARGETS 436 # Pack test binaries if requested. 437 if [[ "${PACK_TEST:-}" == "1" ]]; then 438 (cd "${BUILD_DIR}" 439 ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*' 440 # gtest / gtest_main shared libs 441 ${FIND_BIN} lib/ -name 'libg*.so*' 442 ${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*' 443 ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \ 444 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6" 445 du -h "${BUILD_DIR}/tests.tar.xz" 446 # Pack coverage data if also available. 447 touch "${BUILD_DIR}/gcno.sentinel" 448 (cd "${BUILD_DIR}"; echo gcno.sentinel; ${FIND_BIN} -name '*gcno') | \ 449 tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \ 450 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6" 451 fi 452 453 if [[ "${SKIP_TEST}" -ne "1" ]]; then 454 (cd "${BUILD_DIR}" 455 export UBSAN_OPTIONS=print_stacktrace=1 456 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 457 ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure) 458 fi 459 } 460 461 # Configure the build to strip unused functions. This considerably reduces the 462 # output size, specially for tests which only use a small part of the whole 463 # library. 464 strip_dead_code() { 465 # Emscripten does tree shaking without any extra flags. 466 if [[ "${BUILD_TARGET}" == wasm* ]]; then 467 return 0 468 fi 469 # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively 470 # discard all unreachable code, reducing the code size. For this to work, we 471 # need to also pass --no-export-dynamic to prevent it from exporting all the 472 # internal symbols (like functions) making them all reachable and thus not a 473 # candidate for removal. 474 CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections" 475 CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections" 476 if [[ "${OS}" == "Darwin" ]]; then 477 CMAKE_EXE_LINKER_FLAGS+=" -dead_strip" 478 CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip" 479 else 480 CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic" 481 CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic" 482 fi 483 } 484 485 ### Externally visible commands 486 487 cmd_debug() { 488 CMAKE_BUILD_TYPE="DebugOpt" 489 cmake_configure "$@" 490 cmake_build_and_test 491 } 492 493 cmd_release() { 494 CMAKE_BUILD_TYPE="Release" 495 strip_dead_code 496 cmake_configure "$@" 497 cmake_build_and_test 498 } 499 500 cmd_opt() { 501 CMAKE_BUILD_TYPE="RelWithDebInfo" 502 CMAKE_CXX_FLAGS+=" -DJXL_IS_DEBUG_BUILD" 503 cmake_configure "$@" 504 cmake_build_and_test 505 } 506 507 cmd_coverage() { 508 # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests. 509 TEST_STACK_LIMIT="none" 510 511 cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@" 512 513 if [[ "${SKIP_TEST}" -ne "1" ]]; then 514 # If we didn't run the test we also don't print a coverage report. 515 cmd_coverage_report 516 fi 517 } 518 519 cmd_coverage_report() { 520 LLVM_COV=$("${CC:-clang}" -print-prog-name=llvm-cov) 521 local real_build_dir=$(realpath "${BUILD_DIR}") 522 local gcovr_args=( 523 -r "${real_build_dir}" 524 --gcov-executable "${LLVM_COV} gcov" 525 # Only print coverage information for the libjxl directories. The rest 526 # is not part of the code under test. 527 --filter '.*jxl/.*' 528 --exclude '.*_gbench.cc' 529 --exclude '.*_test.cc' 530 --exclude '.*_testonly..*' 531 --exclude '.*_debug.*' 532 --exclude '.*test_utils..*' 533 --object-directory "${real_build_dir}" 534 ) 535 536 ( 537 cd "${real_build_dir}" 538 gcovr "${gcovr_args[@]}" --html --html-details \ 539 --output="${real_build_dir}/coverage.html" 540 gcovr "${gcovr_args[@]}" --print-summary | 541 tee "${real_build_dir}/coverage.txt" 542 gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml" 543 ) 544 } 545 546 cmd_test() { 547 export_env 548 # Unpack tests if needed. 549 if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then 550 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz" 551 fi 552 if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then 553 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz" 554 fi 555 (cd "${BUILD_DIR}" 556 export UBSAN_OPTIONS=print_stacktrace=1 557 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 558 ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure "$@") 559 } 560 561 cmd_gbench() { 562 export_env 563 (cd "${BUILD_DIR}" 564 export UBSAN_OPTIONS=print_stacktrace=1 565 lib/jxl_gbench \ 566 --benchmark_counters_tabular=true \ 567 --benchmark_out_format=json \ 568 --benchmark_out=gbench.json "$@" 569 ) 570 } 571 572 cmd_asanfuzz() { 573 CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 574 CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 575 cmd_asan -DJPEGXL_ENABLE_FUZZERS=ON "$@" 576 } 577 578 cmd_msanfuzz() { 579 # Install msan if needed before changing the flags. 580 detect_clang_version 581 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 582 # TODO(eustas): why libc++abi.a is bad? 583 if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then 584 # Install msan libraries for this version if needed or if an older version 585 # with libc++abi was installed. 586 cmd_msan_install 587 fi 588 589 CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 590 CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 591 cmd_msan -DJPEGXL_ENABLE_FUZZERS=ON "$@" 592 } 593 594 cmd_asan() { 595 SANITIZER="asan" 596 CMAKE_C_FLAGS+=" -g -DADDRESS_SANITIZER \ 597 -fsanitize=address ${UBSAN_FLAGS[@]}" 598 CMAKE_CXX_FLAGS+=" -g -DADDRESS_SANITIZER \ 599 -fsanitize=address ${UBSAN_FLAGS[@]}" 600 strip_dead_code 601 cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF 602 cmake_build_and_test 603 } 604 605 cmd_tsan() { 606 SANITIZER="tsan" 607 local tsan_args=( 608 -g 609 -DTHREAD_SANITIZER 610 -fsanitize=thread 611 ) 612 CMAKE_C_FLAGS+=" ${tsan_args[@]}" 613 CMAKE_CXX_FLAGS+=" ${tsan_args[@]}" 614 615 cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF 616 cmake_build_and_test 617 } 618 619 cmd_msan() { 620 SANITIZER="msan" 621 detect_clang_version 622 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 623 if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then 624 # Install msan libraries for this version if needed or if an older version 625 # with libc++abi was installed. 626 cmd_msan_install 627 fi 628 629 local msan_c_flags=( 630 -fsanitize=memory 631 -fno-omit-frame-pointer 632 633 -g 634 -DMEMORY_SANITIZER 635 636 # Force gtest to not use the cxxbai. 637 -DGTEST_HAS_CXXABI_H_=0 638 ) 639 if [[ "${FASTER_MSAN_BUILD}" -ne "1" ]]; then 640 msan_c_flags=( 641 "${msan_c_flags[@]}" 642 -fsanitize-memory-track-origins 643 ) 644 fi 645 646 local msan_cxx_flags=( 647 "${msan_c_flags[@]}" 648 649 # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused 650 # in those cases. Ignore the warning. 651 -Wno-unused-command-line-argument 652 -stdlib=libc++ 653 654 # We include the libc++ from the msan directory instead, so we don't want 655 # the std includes. 656 -nostdinc++ 657 -cxx-isystem"${msan_prefix}/include/c++/v1" 658 ) 659 660 local msan_linker_flags=( 661 -L"${msan_prefix}"/lib 662 -Wl,-rpath -Wl,"${msan_prefix}"/lib/ 663 ) 664 665 CMAKE_C_FLAGS+=" ${msan_c_flags[@]}" 666 CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]}" 667 CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 668 CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 669 CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 670 strip_dead_code 671 672 # MSAN share of stack size is non-negligible. 673 TEST_STACK_LIMIT="none" 674 675 # TODO(eustas): investigate why fuzzers do not link when MSAN libc++ is used 676 cmake_configure "$@" \ 677 -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \ 678 -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_WARNINGS_AS_ERRORS=OFF \ 679 -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}" \ 680 -DJPEGXL_ENABLE_FUZZERS=OFF 681 cmake_build_and_test 682 } 683 684 # Install libc++ libraries compiled with msan in the msan_prefix for the current 685 # compiler version. 686 cmd_msan_install() { 687 local tmpdir=$(mktemp -d) 688 CLEANUP_FILES+=("${tmpdir}") 689 local msan_root="${HOME}/.msan" 690 mkdir -p "${msan_root}" 691 # Detect the llvm to install: 692 export CC="${CC:-clang}" 693 export CXX="${CXX:-clang++}" 694 detect_clang_version 695 # Allow overriding the LLVM checkout. 696 local llvm_root="${LLVM_ROOT:-}" 697 if [ -z "${llvm_root}" ]; then 698 declare -A llvm_tag_by_version=( 699 ["6.0"]="6.0.1" 700 ["7"]="7.1.0" 701 ["8"]="8.0.1" 702 ["9"]="9.0.2" 703 ["10"]="10.0.1" 704 ["11"]="11.1.0" 705 ["12"]="12.0.1" 706 ["13"]="13.0.1" 707 ["14"]="14.0.6" 708 ["15"]="15.0.7" 709 ["16"]="16.0.6" 710 ["17"]="17.0.6" 711 ["18"]="18.1.6" 712 ) 713 local llvm_tag="${CLANG_VERSION}.0.0" 714 if [[ -n "${llvm_tag_by_version["${CLANG_VERSION}"]}" ]]; then 715 llvm_tag=${llvm_tag_by_version["${CLANG_VERSION}"]} 716 fi 717 llvm_tag="llvmorg-${llvm_tag}" 718 local llvm_targz="${msan_root}/${llvm_tag}.tar.gz" 719 if [ ! -f "${llvm_targz}" ]; then 720 curl -L --show-error -o "${llvm_targz}" \ 721 "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz" 722 fi 723 tar -C "${tmpdir}" -zxf "${llvm_targz}" 724 llvm_root="${tmpdir}/llvm-project-${llvm_tag}" 725 fi 726 727 local msan_prefix="${msan_root}/${CLANG_VERSION}" 728 rm -rf "${msan_prefix}" 729 730 local TARGET_OPTS="" 731 if [[ -n "${BUILD_TARGET}" ]]; then 732 TARGET_OPTS=" \ 733 -DCMAKE_C_COMPILER_TARGET=\"${BUILD_TARGET}\" \ 734 -DCMAKE_CXX_COMPILER_TARGET=\"${BUILD_TARGET}\" \ 735 -DCMAKE_SYSTEM_PROCESSOR=\"${BUILD_TARGET%%-*}\" \ 736 " 737 fi 738 739 local build_dir="${tmpdir}/build-llvm" 740 mkdir -p "${build_dir}" 741 cd ${llvm_root} 742 cmake -B"${build_dir}" \ 743 -G Ninja \ 744 -S runtimes \ 745 -DCMAKE_BUILD_TYPE=Release \ 746 -DLLVM_USE_SANITIZER=Memory \ 747 -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind;compiler-rt" \ 748 -DLIBCXXABI_ENABLE_SHARED=ON \ 749 -DLIBCXXABI_ENABLE_STATIC=OFF \ 750 -DLIBCXX_ENABLE_SHARED=ON \ 751 -DLIBCXX_ENABLE_STATIC=OFF \ 752 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \ 753 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \ 754 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \ 755 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \ 756 -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \ 757 -DLLVM_PATH="${llvm_root}/llvm" \ 758 -DLLVM_CONFIG_PATH="$(which llvm-config-${CLANG_VERSION} llvm-config | head -n1)" \ 759 ${TARGET_OPTS} 760 cmake --build "${build_dir}" 761 ninja -C "${build_dir}" install 762 } 763 764 # Internal build step shared between all cmd_ossfuzz_* commands. 765 _cmd_ossfuzz() { 766 local sanitizer="$1" 767 shift 768 mkdir -p "${BUILD_DIR}" 769 local real_build_dir=$(realpath "${BUILD_DIR}") 770 771 # oss-fuzz defines three directories: 772 # * /work, with the working directory to do re-builds 773 # * /src, with the source code to build 774 # * /out, with the output directory where to copy over the built files. 775 # We use $BUILD_DIR as the /work and the script directory as the /src. The 776 # /out directory is ignored as developers are used to look for the fuzzers in 777 # $BUILD_DIR/tools/ directly. 778 779 if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then 780 sudo docker run --rm -i \ 781 --user $(id -u):$(id -g) \ 782 -v "${real_build_dir}":/work \ 783 gcr.io/oss-fuzz-base/msan-libs-builder \ 784 bash -c "cp -r /msan /work" 785 fi 786 787 # Args passed to ninja. These will be evaluated as a string separated by 788 # spaces. 789 local jpegxl_extra_args="$@" 790 791 sudo docker run --rm -i \ 792 -e JPEGXL_UID=$(id -u) \ 793 -e JPEGXL_GID=$(id -g) \ 794 -e FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" \ 795 -e SANITIZER="${sanitizer}" \ 796 -e ARCHITECTURE=x86_64 \ 797 -e FUZZING_LANGUAGE=c++ \ 798 -e MSAN_LIBS_PATH="/work/msan" \ 799 -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \ 800 -v "${MYDIR}":/src/libjxl \ 801 -v "${MYDIR}/tools/scripts/ossfuzz-build.sh":/src/build.sh \ 802 -v "${real_build_dir}":/work \ 803 gcr.io/oss-fuzz/libjxl 804 } 805 806 cmd_ossfuzz_asan() { 807 _cmd_ossfuzz address "$@" 808 } 809 cmd_ossfuzz_msan() { 810 _cmd_ossfuzz memory "$@" 811 } 812 cmd_ossfuzz_ubsan() { 813 _cmd_ossfuzz undefined "$@" 814 } 815 816 cmd_ossfuzz_ninja() { 817 [[ -e "${BUILD_DIR}/build.ninja" ]] 818 local real_build_dir=$(realpath "${BUILD_DIR}") 819 820 if [[ -e "${BUILD_DIR}/msan" ]]; then 821 echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2 822 exit 1 823 fi 824 825 sudo docker run --rm -i \ 826 --user $(id -u):$(id -g) \ 827 -v "${MYDIR}":/src/libjxl \ 828 -v "${real_build_dir}":/work \ 829 gcr.io/oss-fuzz/libjxl \ 830 ninja -C /work "$@" 831 } 832 833 cmd_fast_benchmark() { 834 local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar" 835 local small_corpus_url="https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar" 836 mkdir -p "${BENCHMARK_CORPORA}" 837 if [ -f "${small_corpus_tar}" ]; then 838 curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" "${small_corpus_url}" 839 else 840 curl --show-error -o "${small_corpus_tar}" "${small_corpus_url}" 841 fi 842 843 local tmpdir=$(mktemp -d) 844 CLEANUP_FILES+=("${tmpdir}") 845 tar -xf "${small_corpus_tar}" -C "${tmpdir}" 846 847 run_benchmark "${tmpdir}" 1048576 848 } 849 850 cmd_benchmark() { 851 local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar" 852 mkdir -p "${BENCHMARK_CORPORA}" 853 curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \ 854 "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar" 855 856 local tmpdir=$(mktemp -d) 857 CLEANUP_FILES+=("${tmpdir}") 858 tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}" 859 860 local sem_id="jpegxl_benchmark-$$" 861 local nprocs=$(nproc --all || echo 1) 862 images=() 863 local filename 864 while IFS= read -r filename; do 865 # This removes the './' 866 filename="${filename:2}" 867 local mode 868 if [[ "${filename:0:4}" == "srgb" ]]; then 869 mode="RGB_D65_SRG_Rel_SRG" 870 elif [[ "${filename:0:5}" == "adobe" ]]; then 871 mode="RGB_D65_Ado_Rel_Ado" 872 else 873 echo "Unknown image colorspace: ${filename}" >&2 874 exit 1 875 fi 876 png_filename="${filename%.ppm}.png" 877 png_filename=$(echo "${png_filename}" | tr '/' '_') 878 sem --bg --id "${sem_id}" -j"${nprocs}" -- \ 879 "${TOOLS_DIR}/decode_and_encode" \ 880 "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}" 881 images+=( "${png_filename}" ) 882 done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f) 883 sem --id "${sem_id}" --wait 884 885 # We need about 10 GiB per thread on these images. 886 run_benchmark "${tmpdir}" 10485760 887 } 888 889 get_mem_available() { 890 if [[ "${OS}" == "Darwin" ]]; then 891 echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}') 892 elif [[ "${OS}" == MINGW* ]]; then 893 echo $(vmstat | tail -n 1 | awk '{print $4 * 4}') 894 else 895 echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}') 896 fi 897 } 898 899 run_benchmark() { 900 local src_img_dir="$1" 901 local mem_per_thread="${2:-10485760}" 902 903 local output_dir="${BUILD_DIR}/benchmark_results" 904 mkdir -p "${output_dir}" 905 906 if [[ "${OS}" == MINGW* ]]; then 907 src_img_dir=`cygpath -w "${src_img_dir}"` 908 fi 909 910 local num_threads=1 911 if [[ ${BENCHMARK_NUM_THREADS} -gt 0 ]]; then 912 num_threads=${BENCHMARK_NUM_THREADS} 913 else 914 # The memory available at the beginning of the benchmark run in kB. The number 915 # of threads depends on the available memory, and the passed memory per 916 # thread. We also add a 2 GiB of constant memory. 917 local mem_available="$(get_mem_available)" 918 # Check that we actually have a MemAvailable value. 919 [[ -n "${mem_available}" ]] 920 num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} )) 921 if [[ ${num_threads} -le 0 ]]; then 922 num_threads=1 923 fi 924 fi 925 926 local benchmark_args=( 927 --input "${src_img_dir}/*.png" 928 --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2 929 --output_dir "${output_dir}" 930 --show_progress 931 --num_threads="${num_threads}" 932 --decode_reps=11 933 --encode_reps=11 934 ) 935 if [[ "${STORE_IMAGES}" == "1" ]]; then 936 benchmark_args+=(--save_decompressed --save_compressed) 937 fi 938 ( 939 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 940 "${TOOLS_DIR}/benchmark_xl" "${benchmark_args[@]}" | \ 941 tee "${output_dir}/results.txt" 942 943 # Check error code for benchmark_xl command. This will exit if not. 944 return ${PIPESTATUS[0]} 945 ) 946 } 947 948 # Helper function to wait for the CPU temperature to cool down on ARM. 949 wait_for_temp() { 950 { set +x; } 2>/dev/null 951 local temp_limit=${1:-38000} 952 if [[ -z "${THERMAL_FILE:-}" ]]; then 953 echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \ 954 "to read the temperature from. This is normally set in the runner." >&2 955 exit 1 956 fi 957 local org_temp=$(cat "${THERMAL_FILE}") 958 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then 959 echo -n "Waiting for temp to get down from ${org_temp}... " 960 fi 961 local temp="${org_temp}" 962 local secs=0 963 while [[ "${temp}" -ge "${temp_limit}" ]]; do 964 sleep 1 965 temp=$(cat "${THERMAL_FILE}") 966 echo -n "${temp} " 967 secs=$((secs + 1)) 968 if [[ ${secs} -ge 5 ]]; then 969 break 970 fi 971 done 972 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then 973 echo "Done, temp=${temp}" 974 fi 975 set -x 976 } 977 978 # Helper function to set the cpuset restriction of the current process. 979 cmd_cpuset() { 980 [[ "${SKIP_CPUSET:-}" != "1" ]] || return 0 981 local newset="$1" 982 local mycpuset=$(cat /proc/self/cpuset) 983 mycpuset="/dev/cpuset${mycpuset}" 984 # Check that the directory exists: 985 [[ -d "${mycpuset}" ]] 986 if [[ -e "${mycpuset}/cpuset.cpus" ]]; then 987 echo "${newset}" >"${mycpuset}/cpuset.cpus" 988 else 989 echo "${newset}" >"${mycpuset}/cpus" 990 fi 991 } 992 993 # Return the encoding/decoding speed from the Stats output. 994 _speed_from_output() { 995 local speed="$1" 996 local unit="${2:-MP/s}" 997 if [[ "${speed}" == *"${unit}"* ]]; then 998 speed="${speed%% ${unit}*}" 999 speed="${speed##* }" 1000 echo "${speed}" 1001 fi 1002 } 1003 1004 1005 # Run benchmarks on ARM for the big and little CPUs. 1006 cmd_arm_benchmark() { 1007 # Flags used for cjxl encoder with .png inputs 1008 local jxl_png_benchmarks=( 1009 # Lossy options: 1010 "--epf=0 --distance=1.0 --speed=cheetah" 1011 "--epf=2 --distance=1.0 --speed=cheetah" 1012 "--epf=0 --distance=8.0 --speed=cheetah" 1013 "--epf=1 --distance=8.0 --speed=cheetah" 1014 "--epf=2 --distance=8.0 --speed=cheetah" 1015 "--epf=3 --distance=8.0 --speed=cheetah" 1016 "--modular -Q 90" 1017 "--modular -Q 50" 1018 # Lossless options: 1019 "--modular" 1020 "--modular -E 0 -I 0" 1021 "--modular -P 5" 1022 "--modular --responsive=1" 1023 # Near-lossless options: 1024 "--epf=0 --distance=0.3 --speed=fast" 1025 "--modular -Q 97" 1026 ) 1027 1028 # Flags used for cjxl encoder with .jpg inputs. These should do lossless 1029 # JPEG recompression (of pixels or full jpeg). 1030 local jxl_jpeg_benchmarks=( 1031 "--num_reps=3" 1032 ) 1033 1034 local images=( 1035 "testdata/jxl/flower/flower.png" 1036 ) 1037 1038 local jpg_images=( 1039 "testdata/jxl/flower/flower.png.im_q85_420.jpg" 1040 ) 1041 1042 if [[ "${SKIP_CPUSET:-}" == "1" ]]; then 1043 # Use a single cpu config in this case. 1044 local cpu_confs=("?") 1045 else 1046 # Otherwise the CPU config comes from the environment: 1047 local cpu_confs=( 1048 "${RUNNER_CPU_LITTLE}" 1049 "${RUNNER_CPU_BIG}" 1050 # The CPU description is something like 3-7, so these configurations only 1051 # take the first CPU of the group. 1052 "${RUNNER_CPU_LITTLE%%-*}" 1053 "${RUNNER_CPU_BIG%%-*}" 1054 ) 1055 # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will 1056 # be ignored but still evaluated when calling cmd_cpuset. 1057 [[ -n "${RUNNER_CPU_ALL}" ]] 1058 fi 1059 1060 local jpg_dirname="third_party/corpora/jpeg" 1061 mkdir -p "${jpg_dirname}" 1062 local jpg_qualities=( 50 80 95 ) 1063 for src_img in "${images[@]}"; do 1064 for q in "${jpg_qualities[@]}"; do 1065 local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg" 1066 convert -sampling-factor 1x1 -quality "${q}" \ 1067 "${src_img}" "${jpeg_name}" 1068 jpg_images+=("${jpeg_name}") 1069 done 1070 done 1071 1072 local output_dir="${BUILD_DIR}/benchmark_results" 1073 mkdir -p "${output_dir}" 1074 local runs_file="${output_dir}/runs.txt" 1075 1076 if [[ ! -e "${runs_file}" ]]; then 1077 echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" | 1078 tee -a "${runs_file}" 1079 fi 1080 1081 mkdir -p "${BUILD_DIR}/arm_benchmark" 1082 local flags 1083 local src_img 1084 for src_img in "${jpg_images[@]}" "${images[@]}"; do 1085 local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ') 1086 local enc_binaries=("${TOOLS_DIR}/cjxl") 1087 local src_ext="${src_img##*.}" 1088 for enc_binary in "${enc_binaries[@]}"; do 1089 local enc_binary_base=$(basename "${enc_binary}") 1090 1091 # Select the list of flags to use for the current encoder/image pair. 1092 local img_benchmarks 1093 if [[ "${src_ext}" == "jpg" ]]; then 1094 img_benchmarks=("${jxl_jpeg_benchmarks[@]}") 1095 else 1096 img_benchmarks=("${jxl_png_benchmarks[@]}") 1097 fi 1098 1099 for flags in "${img_benchmarks[@]}"; do 1100 # Encoding step. 1101 local enc_file_hash="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}" 1102 enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ') 1103 local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl" 1104 1105 for cpu_conf in "${cpu_confs[@]}"; do 1106 cmd_cpuset "${cpu_conf}" 1107 # nproc returns the number of active CPUs, which is given by the cpuset 1108 # mask. 1109 local num_threads="$(nproc)" 1110 1111 echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}" 1112 local enc_output 1113 if [[ "${flags}" == *"modular"* ]]; then 1114 # We don't benchmark encoding speed in this case. 1115 if [[ ! -f "${enc_file}" ]]; then 1116 cmd_cpuset "${RUNNER_CPU_ALL:-}" 1117 "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" 1118 mv "${enc_file}.tmp" "${enc_file}" 1119 cmd_cpuset "${cpu_conf}" 1120 fi 1121 enc_output=" ?? MP/s" 1122 else 1123 wait_for_temp 1124 enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \ 1125 2>&1 | tee /dev/stderr | grep -F "MP/s [") 1126 mv "${enc_file}.tmp" "${enc_file}" 1127 fi 1128 local enc_speed=$(_speed_from_output "${enc_output}") 1129 local enc_size=$(stat -c "%s" "${enc_file}") 1130 1131 echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}" 1132 1133 local dec_output 1134 wait_for_temp 1135 dec_output=$("${TOOLS_DIR}/djxl" "${enc_file}" \ 1136 --num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr | 1137 grep -E "M[BP]/s \[") 1138 local img_size=$(echo "${dec_output}" | cut -f 1 -d ',') 1139 local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ') 1140 local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ') 1141 local img_size_px=$(( ${img_size_x} * ${img_size_y} )) 1142 local dec_speed=$(_speed_from_output "${dec_output}") 1143 1144 # For JPEG lossless recompression modes (where the original is a JPEG) 1145 # decode to JPG as well. 1146 local jpeg_dec_mps_speed="" 1147 local jpeg_dec_mbs_speed="" 1148 if [[ "${src_ext}" == "jpg" ]]; then 1149 wait_for_temp 1150 local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg" 1151 dec_output=$("${TOOLS_DIR}/djxl" "${enc_file}" \ 1152 "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \ 1153 tee /dev/stderr | grep -E "M[BP]/s \[") 1154 local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}") 1155 local jpeg_dec_mbs_speed=$(_speed_from_output "${dec_output}" MB/s) 1156 if ! cmp --quiet "${src_img}" "${dec_file}"; then 1157 # Add a start at the end to signal that the files are different. 1158 jpeg_dec_mbs_speed+="*" 1159 fi 1160 fi 1161 1162 # Record entry in a tab-separated file. 1163 local src_img_base=$(basename "${src_img}") 1164 echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" | 1165 tee -a "${runs_file}" 1166 done 1167 done 1168 done 1169 done 1170 cmd_cpuset "${RUNNER_CPU_ALL:-}" 1171 cat "${runs_file}" 1172 1173 } 1174 1175 # Generate a corpus and run the fuzzer on that corpus. 1176 cmd_fuzz() { 1177 local corpus_dir=$(realpath "${BUILD_DIR}/fuzzer_corpus") 1178 local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash") 1179 mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}" 1180 # Generate step. 1181 "${TOOLS_DIR}/fuzzer_corpus" "${corpus_dir}" 1182 # Run step: 1183 local nprocs=$(nproc --all || echo 1) 1184 ( 1185 cd "${TOOLS_DIR}" 1186 djxl_fuzzer "${fuzzer_crash_dir}" "${corpus_dir}" \ 1187 -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \ 1188 -artifact_prefix="${fuzzer_crash_dir}/" 1189 ) 1190 } 1191 1192 # Runs the linters (clang-format, build_cleaner, buildirier) on the pending CLs. 1193 cmd_lint() { 1194 merge_request_commits 1195 { set +x; } 2>/dev/null 1196 local versions=(${1:-16 15 14 13 12 11 10 9 8 7 6.0}) 1197 local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format) 1198 local tmpdir=$(mktemp -d) 1199 CLEANUP_FILES+=("${tmpdir}") 1200 1201 local ret=0 1202 local build_patch="${tmpdir}/build_cleaner.patch" 1203 if ! "${MYDIR}/tools/scripts/build_cleaner.py" >"${build_patch}"; then 1204 ret=1 1205 echo "build_cleaner.py findings:" >&2 1206 "${COLORDIFF_BIN}" <"${build_patch}" 1207 echo "Run \`tools/scripts/build_cleaner.py --update\` to apply them" >&2 1208 fi 1209 1210 # It is ok, if buildifier is not installed. 1211 if which buildifier >/dev/null; then 1212 local buildifier_patch="${tmpdir}/buildifier.patch" 1213 local bazel_files=`git -C "${MYDIR}" ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"` 1214 set -x 1215 buildifier -d ${bazel_files} >"${buildifier_patch}"|| true 1216 { set +x; } 2>/dev/null 1217 if [ -s "${buildifier_patch}" ]; then 1218 ret=1 1219 echo 'buildifier have found some problems in Bazel build files:' >&2 1220 "${COLORDIFF_BIN}" <"${buildifier_patch}" 1221 echo 'To fix them run (from the base directory):' >&2 1222 echo ' buildifier `git ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`' >&2 1223 fi 1224 fi 1225 1226 # It is ok, if spell-checker is not installed. 1227 if which typos >/dev/null; then 1228 local src_ext="bazel|bzl|c|cc|cmake|gni|h|html|in|java|js|m|md|nix|py|rst|sh|ts|txt|yaml|yml" 1229 local sources=`git -C "${MYDIR}" ls-files | grep -E "\.(${src_ext})$"` 1230 typos -c "${MYDIR}/tools/scripts/typos.toml" ${sources} 1231 else 1232 echo "Consider installing https://github.com/crate-ci/typos for spell-checking" 1233 fi 1234 1235 local installed=() 1236 local clang_patch 1237 local clang_format 1238 for clang_format in "${clang_format_bins[@]}"; do 1239 if ! which "${clang_format}" >/dev/null; then 1240 continue 1241 fi 1242 installed+=("${clang_format}") 1243 local tmppatch="${tmpdir}/${clang_format}.patch" 1244 # We include in this linter all the changes including the uncommitted changes 1245 # to avoid printing changes already applied. 1246 set -x 1247 # Ignoring the error that git-clang-format outputs. 1248 git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \ 1249 --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true 1250 { set +x; } 2>/dev/null 1251 if grep -E '^--- ' "${tmppatch}" | grep -v 'a/third_party' >/dev/null; then 1252 if [[ -n "${LINT_OUTPUT:-}" ]]; then 1253 cp "${tmppatch}" "${LINT_OUTPUT}" 1254 fi 1255 clang_patch="${tmppatch}" 1256 else 1257 echo "clang-format check OK" >&2 1258 return ${ret} 1259 fi 1260 done 1261 1262 if [[ ${#installed[@]} -eq 0 ]]; then 1263 echo "You must install clang-format for \"git clang-format\"" >&2 1264 exit 1 1265 fi 1266 1267 # clang-format is installed but found problems. 1268 echo "clang-format findings:" >&2 1269 "${COLORDIFF_BIN}" < "${clang_patch}" 1270 1271 echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \ 1272 "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \ 1273 "directory to apply them." >&2 1274 exit 1 1275 } 1276 1277 # Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs 1278 # clang-tidy over all the source files instead. 1279 cmd_tidy() { 1280 local what="${1:-}" 1281 1282 if [[ -z "${CLANG_TIDY_BIN}" ]]; then 1283 echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2 1284 exit 1 1285 fi 1286 1287 local git_args=() 1288 if [[ "${what}" == "all" ]]; then 1289 git_args=(ls-files) 1290 shift 1291 else 1292 merge_request_commits 1293 git_args=( 1294 diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}" 1295 "${MR_HEAD_SHA}" 1296 ) 1297 fi 1298 1299 # Clang-tidy needs the compilation database generated by cmake. 1300 if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then 1301 # Generate the build options in debug mode, since we need the debug asserts 1302 # enabled for the clang-tidy analyzer to use them. 1303 CMAKE_BUILD_TYPE="Debug" 1304 cmake_configure 1305 # Build the autogen targets to generate the .h files from the .ui files. 1306 local autogen_targets=( 1307 $(ninja -C "${BUILD_DIR}" -t targets | grep -F _autogen: | 1308 cut -f 1 -d :) 1309 ) 1310 if [[ ${#autogen_targets[@]} != 0 ]]; then 1311 ninja -C "${BUILD_DIR}" "${autogen_targets[@]}" 1312 fi 1313 fi 1314 1315 cd "${MYDIR}" 1316 local nprocs=$(nproc --all || echo 1) 1317 local ret=0 1318 if ! parallel -j"${nprocs}" --keep-order -- \ 1319 "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \ 1320 < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \ 1321 >"${BUILD_DIR}/clang-tidy.txt"; then 1322 ret=1 1323 fi 1324 { set +x; } 2>/dev/null 1325 echo "Findings statistics:" >&2 1326 grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \ 1327 | uniq -c >&2 1328 1329 if [[ $ret -ne 0 ]]; then 1330 cat >&2 <<EOF 1331 Errors found, see ${BUILD_DIR}/clang-tidy.txt for details. 1332 To automatically fix them, run: 1333 1334 SKIP_TEST=1 ./ci.sh debug 1335 ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$') 1336 EOF 1337 fi 1338 1339 return ${ret} 1340 } 1341 1342 # Print stats about all the packages built in ${BUILD_DIR}/debs/. 1343 cmd_debian_stats() { 1344 { set +x; } 2>/dev/null 1345 local debsdir="${BUILD_DIR}/debs" 1346 local f 1347 while IFS='' read -r -d '' f; do 1348 echo "=====================================================================" 1349 echo "Package $f:" 1350 dpkg --info $f 1351 dpkg --contents $f 1352 done < <(find "${BUILD_DIR}/debs" -maxdepth 1 -mindepth 1 -type f \ 1353 -name '*.deb' -print0) 1354 } 1355 1356 build_debian_pkg() { 1357 local srcdir="$1" 1358 local srcpkg="$2" 1359 local options="${3:-}" 1360 1361 local debsdir="${BUILD_DIR}/debs" 1362 local builddir="${debsdir}/${srcpkg}" 1363 1364 # debuild doesn't have an easy way to build out of tree, so we make a copy 1365 # of with all symlinks on the first level. 1366 mkdir -p "${builddir}" 1367 for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do 1368 if [[ ! -L "${builddir}/$f" ]]; then 1369 rm -f "${builddir}/$f" 1370 ln -s "${srcdir}/$f" "${builddir}/$f" 1371 fi 1372 done 1373 ( 1374 cd "${builddir}" 1375 debuild "${options}" -b -uc -us 1376 ) 1377 } 1378 1379 cmd_debian_build() { 1380 local srcpkg="${1:-}" 1381 1382 case "${srcpkg}" in 1383 jpeg-xl) 1384 build_debian_pkg "${MYDIR}" "jpeg-xl" 1385 ;; 1386 highway) 1387 build_debian_pkg "${MYDIR}/third_party/highway" "highway" "${HWY_PKG_OPTIONS}" 1388 ;; 1389 *) 1390 echo "ERROR: Must pass a valid source package name to build." >&2 1391 ;; 1392 esac 1393 } 1394 1395 get_version() { 1396 local varname=$1 1397 local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1) 1398 [[ -n "${line}" ]] 1399 line="${line#set(${varname} }" 1400 line="${line%)}" 1401 echo "${line}" 1402 } 1403 1404 cmd_bump_version() { 1405 local newver="${1:-}" 1406 1407 if ! which dch >/dev/null; then 1408 echo "Missing dch\nTo install it run:\n sudo apt install devscripts" 1409 exit 1 1410 fi 1411 1412 if [[ -z "${newver}" ]]; then 1413 local major=$(get_version JPEGXL_MAJOR_VERSION) 1414 local minor=$(get_version JPEGXL_MINOR_VERSION) 1415 local patch=0 1416 minor=$(( ${minor} + 1)) 1417 else 1418 local major="${newver%%.*}" 1419 newver="${newver#*.}" 1420 local minor="${newver%%.*}" 1421 newver="${newver#${minor}}" 1422 local patch="${newver#.}" 1423 if [[ -z "${patch}" ]]; then 1424 patch=0 1425 fi 1426 fi 1427 1428 newver="${major}.${minor}.${patch}" 1429 1430 echo "Bumping version to ${newver} (${major}.${minor}.${patch})" 1431 sed -E \ 1432 -e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \ 1433 -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \ 1434 -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \ 1435 -i lib/CMakeLists.txt 1436 sed -E \ 1437 -e "s/(LIBJXL_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}.${patch}\"/" \ 1438 -e "s/(LIBJXL_ABI_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}\"/" \ 1439 -i .github/workflows/conformance.yml 1440 1441 # Update lib.gni 1442 tools/scripts/build_cleaner.py --update 1443 1444 # Mark the previous version as "unstable". 1445 DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release '' 1446 DEBCHANGE_RELEASE_HEURISTIC=log dch -M \ 1447 --newversion "${newver}" \ 1448 "Bump JPEG XL version to ${newver}." 1449 } 1450 1451 # Check that the AUTHORS file contains the email of the committer. 1452 cmd_authors() { 1453 merge_request_commits 1454 local emails 1455 local names 1456 readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}") 1457 readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}") 1458 for i in "${!names[@]}"; do 1459 echo "Checking name '${names[$i]}' with email '${emails[$i]}' ..." 1460 "${MYDIR}"/tools/scripts/check_author.py "${emails[$i]}" "${names[$i]}" 1461 done 1462 } 1463 1464 main() { 1465 local cmd="${1:-}" 1466 if [[ -z "${cmd}" ]]; then 1467 cat >&2 <<EOF 1468 Use: $0 CMD 1469 1470 Where cmd is one of: 1471 opt Build and test a Release with symbols build. 1472 debug Build and test a Debug build (NDEBUG is not defined). 1473 release Build and test a striped Release binary without debug information. 1474 asan Build and test an ASan (AddressSanitizer) build. 1475 msan Build and test an MSan (MemorySanitizer) build. Needs to have msan 1476 c++ libs installed with msan_install first. 1477 tsan Build and test a TSan (ThreadSanitizer) build. 1478 asanfuzz Build and test an ASan (AddressSanitizer) build for fuzzing. 1479 msanfuzz Build and test an MSan (MemorySanitizer) build for fuzzing. 1480 test Run the tests build by opt, debug, release, asan or msan. Useful when 1481 building with SKIP_TEST=1. 1482 gbench Run the Google benchmark tests. 1483 fuzz Generate the fuzzer corpus and run the fuzzer on it. Useful after 1484 building with asan or msan. 1485 benchmark Run the benchmark over the default corpus. 1486 fast_benchmark Run the benchmark over the small corpus. 1487 1488 coverage Build and run tests with coverage support. Runs coverage_report as 1489 well. 1490 coverage_report Generate HTML, XML and text coverage report after a coverage 1491 run. 1492 1493 lint Run the linter checks on the current commit or merge request. 1494 tidy Run clang-tidy on the current commit or merge request. 1495 authors Check that the last commit's author is listed in the AUTHORS file. 1496 1497 msan_install Install the libc++ libraries required to build in msan mode. This 1498 needs to be done once. 1499 1500 debian_build <srcpkg> Build the given source package. 1501 debian_stats Print stats about the built packages. 1502 1503 oss-fuzz commands: 1504 ossfuzz_asan Build the local source inside oss-fuzz docker with asan. 1505 ossfuzz_msan Build the local source inside oss-fuzz docker with msan. 1506 ossfuzz_ubsan Build the local source inside oss-fuzz docker with ubsan. 1507 ossfuzz_ninja Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra 1508 parameters are passed to ninja, for example "djxl_fuzzer" will 1509 only build that ninja target. Use for faster build iteration 1510 after one of the ossfuzz_*san commands. 1511 1512 You can pass some optional environment variables as well: 1513 - BUILD_DIR: The output build directory (by default "$$repo/build") 1514 - BUILD_TARGET: The target triplet used when cross-compiling. 1515 - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS. 1516 - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package. 1517 - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only). 1518 - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds. 1519 - LINT_OUTPUT: Path to the output patch from the "lint" command. 1520 - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark. 1521 - SKIP_BUILD=1: Skip the build stage, cmake configure only. 1522 - SKIP_TEST=1: Skip the test stage. 1523 - STORE_IMAGES=0: Makes the benchmark discard the computed images. 1524 - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB. 1525 - TEST_SELECTOR: pass additional arguments to ctest, e.g. "-R .Resample.". 1526 - STACK_SIZE=1: Generate binaries with the .stack_sizes sections. 1527 1528 These optional environment variables are forwarded to the cmake call as 1529 parameters: 1530 - CMAKE_BUILD_TYPE 1531 - CMAKE_C_FLAGS 1532 - CMAKE_CXX_FLAGS 1533 - CMAKE_C_COMPILER_LAUNCHER 1534 - CMAKE_CXX_COMPILER_LAUNCHER 1535 - CMAKE_CROSSCOMPILING_EMULATOR 1536 - CMAKE_FIND_ROOT_PATH 1537 - CMAKE_EXE_LINKER_FLAGS 1538 - CMAKE_MAKE_PROGRAM 1539 - CMAKE_MODULE_LINKER_FLAGS 1540 - CMAKE_SHARED_LINKER_FLAGS 1541 - CMAKE_TOOLCHAIN_FILE 1542 1543 Example: 1544 BUILD_DIR=/tmp/build $0 opt 1545 EOF 1546 exit 1 1547 fi 1548 1549 cmd="cmd_${cmd}" 1550 shift 1551 set -x 1552 "${cmd}" "$@" 1553 } 1554 1555 main "$@"