diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..d4da4bf5d --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +workdir* +__pycache__/ +tools/benchd/*.json +tools/report_df/out/ +cov_out/ +*.pkl +*.html diff --git a/docker/Dockerfile b/docker/Dockerfile index e3dd49ab1..7cebd9e59 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,7 @@ -FROM ubuntu:18.04 +## +# Stage: build core magma +## +FROM ubuntu:24.04 AS magma_core # TODO remove sudo for user "magma" to avoid unwanted priv escalation from # other attack vectors. @@ -11,32 +14,34 @@ RUN apt-get update && apt-get install -y sudo ARG magma_root=./ ## Path variables inside the container -ENV MAGMA_R /magma -ENV OUT /magma_out -ENV SHARED /magma_shared - -ENV CC /usr/bin/gcc -ENV CXX /usr/bin/g++ -ENV LD /usr/bin/ld -ENV AR /usr/bin/ar -ENV AS /usr/bin/as -ENV NM /usr/bin/nm -ENV RANLIB /usr/bin/ranlib - -ARG USER_ID=1000 -ARG GROUP_ID=1000 +ENV MAGMA_R=/magma +ENV OUT=/magma_out +ENV COV=/magma_cov +ENV SHARED=/magma_shared + +ENV CC=/usr/bin/gcc +ENV CXX=/usr/bin/g++ +ENV LD=/usr/bin/ld +ENV AR=/usr/bin/ar +ENV AS=/usr/bin/as +ENV NM=/usr/bin/nm +ENV RANLIB=/usr/bin/ranlib + +ARG USER_ID=1001 +ARG GROUP_ID=1001 RUN mkdir -p /home && \ groupadd -g ${GROUP_ID} magma && \ useradd -l -u ${USER_ID} -K UMASK=0000 -d /home -g magma magma && \ chown magma:magma /home RUN echo "magma:amgam" | chpasswd && usermod -a -G sudo magma +RUN echo "magma ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers -RUN mkdir -p ${SHARED} ${OUT} && \ - chown magma:magma ${SHARED} ${OUT} && \ - chmod 744 ${SHARED} ${OUT} +RUN mkdir -p ${SHARED} ${OUT} ${COV} && \ + chown magma:magma ${SHARED} ${OUT} ${COV} && \ + chmod 744 ${SHARED} ${OUT} ${COV} ARG magma_path=magma -ENV MAGMA ${MAGMA_R}/${magma_path} +ENV MAGMA=${MAGMA_R}/${magma_path} USER root:root RUN mkdir -p ${MAGMA} && chown magma:magma ${MAGMA} COPY --chown=magma:magma ${magma_root}/${magma_path} ${MAGMA}/ @@ -46,7 +51,7 @@ RUN ${MAGMA}/prebuild.sh ARG fuzzer_name ARG fuzzer_path=fuzzers/${fuzzer_name} -ENV FUZZER ${MAGMA_R}/${fuzzer_path} +ENV FUZZER=${MAGMA_R}/${fuzzer_path} USER root:root RUN mkdir -p ${FUZZER} && chown magma:magma ${FUZZER} COPY --chown=magma:magma ${magma_root}/${fuzzer_path} ${FUZZER}/ @@ -54,16 +59,20 @@ RUN ${FUZZER}/preinstall.sh USER magma:magma RUN ${FUZZER}/fetch.sh RUN ${FUZZER}/build.sh +RUN if [ -f ${FUZZER}/postinstall.sh ]; then /bin/bash ${FUZZER}/postinstall.sh; fi ARG target_name ARG target_path=targets/${target_name} -ENV TARGET ${MAGMA_R}/${target_path} +ENV TARGET_NAME=${target_name} +ENV TARGET=${MAGMA_R}/${target_path} +ARG target_version +ENV TARGET_VERSION=${target_version} USER root:root RUN mkdir -p ${TARGET} && chown magma:magma ${TARGET} COPY --chown=magma:magma ${magma_root}/${target_path} ${TARGET}/ RUN ${TARGET}/preinstall.sh USER magma:magma -RUN ${TARGET}/fetch.sh +RUN ${MAGMA}/fetch_target.sh RUN ${MAGMA}/apply_patches.sh ## Configuration parameters @@ -78,11 +87,74 @@ ARG CANARIES_FLAG=${canaries:+-DMAGMA_ENABLE_CANARIES} ARG FIXES_FLAG=${fixes:+-DMAGMA_ENABLE_FIXES} ARG BUILD_FLAGS="-include ${MAGMA}/src/canary.h ${CANARIES_FLAG} ${FIXES_FLAG} ${ISAN_FLAG} ${HARDEN_FLAG} -g -O0" -ENV CFLAGS ${BUILD_FLAGS} -ENV CXXFLAGS ${BUILD_FLAGS} -ENV LIBS -l:magma.o -lrt -ENV LDFLAGS -L"${OUT}" -g +ENV CFLAGS=${BUILD_FLAGS} +ENV CXXFLAGS=${BUILD_FLAGS} +ENV LIBS="-l:magma.o -lrt" +ENV LDFLAGS="-L${OUT} -g" + +ARG source_coverage +ENV SOURCE_COVERAGE=${source_coverage:+1} +ENV SOURCE_COVERAGE_FLAGS=${source_coverage:+"-fprofile-instr-generate -fcoverage-mapping"} RUN ${FUZZER}/instrument.sh -ENTRYPOINT "${MAGMA}/run.sh" +ENTRYPOINT ["/bin/bash", "-c", "${MAGMA}/run.sh"] + + +## +# Stage: build magma PoCs +## +FROM ubuntu:24.04 AS magma_pocs + +RUN apt-get update && apt-get install -y sudo g++ git + +ARG magma_root=./ +ARG poc_target_name +ARG poc_target_version +ARG poc_bug +ENV MAGMA_R=/magma +ENV OUT=/magma_out +ENV CC=/usr/bin/gcc +ENV CXX=/usr/bin/g++ + +# set up user and group +ARG USER_ID=1001 +ARG GROUP_ID=1001 + +RUN mkdir -p /home && \ + groupadd -g ${GROUP_ID} magma && \ + useradd -l -u ${USER_ID} -K UMASK=0000 -d /home -g magma magma && \ + chown magma:magma /home +RUN echo "magma:amgam" | chpasswd && usermod -a -G sudo magma +RUN echo "magma ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +RUN mkdir -p ${OUT} && chown magma:magma ${OUT} && chmod 744 ${OUT} + +# copy over magma source code +ARG magma_path=magma +ENV MAGMA=${MAGMA_R}/${magma_path} + +USER root:root +RUN mkdir -p ${MAGMA} && chown magma:magma ${MAGMA} +COPY --chown=magma:magma ${magma_root}/${magma_path} ${MAGMA}/ + +# build a clean install of the target with a specific bug +ARG target_path=targets/${poc_target_name} +ENV TARGET_NAME=${poc_target_name} +ENV TARGET=${MAGMA_R}/${target_path} +ENV TARGET_VERSION=${poc_target_version} +ENV BUG_PATH=${TARGET}/patches/bugs/${poc_bug}.patch + +USER root:root +RUN mkdir -p ${TARGET} && chown magma:magma ${TARGET} +COPY --chown=magma:magma ${magma_root}/${target_path} ${TARGET}/ +RUN ${TARGET}/preinstall.sh +RUN ${MAGMA}/fetch_target.sh +RUN ${MAGMA}/apply_patches.sh ${BUG_PATH} +RUN ${TARGET}/build_poc.sh + +# copy over the PoC input +ARG CRASH_CONTAINER_PATH=/test/crash_input +ARG CRASH_INPUT_PATH=${magma_root}/${target_path}/pocs/${poc_bug}.crash + +COPY --chown=magma:magma ${CRASH_INPUT_PATH} ${CRASH_CONTAINER_PATH} diff --git a/fuzzers/aflplusplus/build.sh b/fuzzers/aflplusplus/build.sh index 9f7f9d22b..f2be00c23 100755 --- a/fuzzers/aflplusplus/build.sh +++ b/fuzzers/aflplusplus/build.sh @@ -19,4 +19,4 @@ export PYTHON_INCLUDE=/ make -j$(nproc) || exit 1 make -C utils/aflpp_driver || exit 1 -mkdir -p "$OUT/afl" "$OUT/cmplog" +mkdir -p "$OUT/afl" "$OUT/cmplog" "$COV/afl" diff --git a/fuzzers/aflplusplus/coverage.sh b/fuzzers/aflplusplus/coverage.sh new file mode 100644 index 000000000..4022d930b --- /dev/null +++ b/fuzzers/aflplusplus/coverage.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +## +# Pre-requirements: +# - env SHARED: path to directory shared with host (to store results) +## + +export DIRECTORY_TO_SEARCH=$SHARED/findings/default/queue +chmod -R o+rx $SHARED/findings/default +export PATTERN_TO_MATCH="id:*" +cp $COV/afl/$PROGRAM $COV \ No newline at end of file diff --git a/fuzzers/aflplusplus/fetch.sh b/fuzzers/aflplusplus/fetch.sh index 4caa9d481..810a1a455 100755 --- a/fuzzers/aflplusplus/fetch.sh +++ b/fuzzers/aflplusplus/fetch.sh @@ -6,8 +6,11 @@ set -e # - env FUZZER: path to fuzzer work dir ## +# Currently points to the first commit of 2025 +AFLPLUSPLUS_STABLE_HASH=1ddfb1fec2b8aa99886a5de35c07e8f2a7bd8b98 + git clone --no-checkout https://github.com/AFLplusplus/AFLplusplus "$FUZZER/repo" -git -C "$FUZZER/repo" checkout 458eb0813a6f7d63eed97f18696bca8274533123 +git -C "$FUZZER/repo" checkout $AFLPLUSPLUS_STABLE_HASH # Fix: CMake-based build systems fail with duplicate (of main) or undefined references (of LLVMFuzzerTestOneInput) sed -i '{s/^int main/__attribute__((weak)) &/}' $FUZZER/repo/utils/aflpp_driver/aflpp_driver.c @@ -24,15 +27,15 @@ EOF patch -p1 -d "$FUZZER/repo" << EOF --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c -@@ -53,7 +53,7 @@ - #include "hash.h" +@@ -65,7 +65,7 @@ #endif + // AFL++ shared memory fuzz cases -int __afl_sharedmem_fuzzing = 1; +int __afl_sharedmem_fuzzing = 0; - extern unsigned int * __afl_fuzz_len; + extern unsigned int *__afl_fuzz_len; extern unsigned char *__afl_fuzz_ptr; - + @@ -111,7 +111,8 @@ extern unsigned int * __afl_fuzz_len; __attribute__((weak)) int LLVMFuzzerInitialize(int *argc, char ***argv); diff --git a/fuzzers/aflplusplus/instrument.sh b/fuzzers/aflplusplus/instrument.sh index 5fdfd1978..6c5929136 100755 --- a/fuzzers/aflplusplus/instrument.sh +++ b/fuzzers/aflplusplus/instrument.sh @@ -7,6 +7,8 @@ set -e # - env TARGET: path to target work dir # - env MAGMA: path to Magma support files # - env OUT: path to directory where artifacts are stored +# - env SOURCE_COVERAGE: if source-based code coverage is enabled +# - env COV: path to directory where artifacts for source-base code coverage are stored # - env CFLAGS and CXXFLAGS must be set to link against Magma instrumentation ## @@ -18,6 +20,7 @@ export LIBS="$LIBS -lc++ -lc++abi $FUZZER/repo/utils/aflpp_driver/libAFLDriver.a # AFL++'s driver is compiled against libc++ export CXXFLAGS="$CXXFLAGS -stdlib=libc++" +export LDFLAGS="$LDFLAGS -stdlib=libc++" # Build the AFL-only instrumented version ( @@ -44,3 +47,14 @@ export CXXFLAGS="$CXXFLAGS -stdlib=libc++" # NOTE: We pass $OUT directly to the target build.sh script, since the artifact # itself is the fuzz target. In the case of Angora, we might need to # replace $OUT by $OUT/fast and $OUT/track, for instance. + +if [ ! -z $SOURCE_COVERAGE ]; then + export CC=clang + export CXX=clang++ + export CFLAGS="$SOURCE_COVERAGE_FLAGS" + export CXXFLAGS="$SOURCE_COVERAGE_FLAGS" + export LDFLAGS="$SOURCE_COVERAGE_FLAGS" + export LIBS="" + export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + OUT=$COV/afl TARGET=$COV "$TARGET/build.sh" +fi diff --git a/fuzzers/aflplusplus/preinstall.sh b/fuzzers/aflplusplus/preinstall.sh index e15163feb..59c6b53bc 100755 --- a/fuzzers/aflplusplus/preinstall.sh +++ b/fuzzers/aflplusplus/preinstall.sh @@ -1,35 +1,40 @@ #!/bin/bash set -e +export LLVM_VERSION=16 + apt-get update && \ - apt-get install -y make clang-9 llvm-9-dev libc++-9-dev libc++abi-9-dev \ - build-essential git wget gcc-7-plugin-dev + apt-get install -y make build-essential git wget \ + clang-$LLVM_VERSION llvm-$LLVM_VERSION-dev \ + libc++-$LLVM_VERSION-dev libc++abi-$LLVM_VERSION-dev \ + gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev update-alternatives \ - --install /usr/lib/llvm llvm /usr/lib/llvm-9 20 \ - --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-9 \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-9 \ - --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-9 \ - --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-9 \ - --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-9 \ - --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-9 \ - --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-9 \ - --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-9 \ - --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-9 \ - --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-9 \ - --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-9 \ - --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-9 \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-9 \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-9 \ - --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-9 \ - --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-9 \ - --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-9 \ - --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-9 \ - --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-9 \ - --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-9 \ - --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-9 + --install /usr/lib/llvm llvm /usr/lib/llvm-$LLVM_VERSION 20 \ + --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-$LLVM_VERSION \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-$LLVM_VERSION \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-$LLVM_VERSION \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-$LLVM_VERSION \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-$LLVM_VERSION \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-$LLVM_VERSION \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-$LLVM_VERSION \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-$LLVM_VERSION \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-$LLVM_VERSION \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-$LLVM_VERSION \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-$LLVM_VERSION \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-$LLVM_VERSION \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-$LLVM_VERSION \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-$LLVM_VERSION \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-$LLVM_VERSION \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-$LLVM_VERSION \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-$LLVM_VERSION \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-$LLVM_VERSION \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-$LLVM_VERSION \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-$LLVM_VERSION \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-$LLVM_VERSION \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-$LLVM_VERSION update-alternatives \ - --install /usr/bin/clang clang /usr/bin/clang-9 20 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-9 \ - --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-9 + --install /usr/bin/clang clang /usr/bin/clang-$LLVM_VERSION 20 \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-$LLVM_VERSION diff --git a/fuzzers/aflplusplus/run.sh b/fuzzers/aflplusplus/run.sh index 8a369e3a9..a645289f5 100755 --- a/fuzzers/aflplusplus/run.sh +++ b/fuzzers/aflplusplus/run.sh @@ -17,13 +17,19 @@ fi mkdir -p "$SHARED/findings" -flag_cmplog=(-m none -c "$OUT/cmplog/$PROGRAM") +# TODO: Figure out why cmplog gets stuck infinitely on PHP +if [[ "$TARGET" == */php ]]; then + flag_cmplog=() +else + flag_cmplog=(-m none -c "$OUT/cmplog/$PROGRAM") +fi export AFL_SKIP_CPUFREQ=1 export AFL_NO_AFFINITY=1 export AFL_NO_UI=1 export AFL_MAP_SIZE=256000 export AFL_DRIVER_DONT_DEFER=1 +export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 "$FUZZER/repo/afl-fuzz" -i "$TARGET/corpus/$PROGRAM" -o "$SHARED/findings" \ "${flag_cmplog[@]}" -d \ diff --git a/fuzzers/aflplusplus/runonce.sh b/fuzzers/aflplusplus/runonce.sh index 1f269e3a4..732592f97 100755 --- a/fuzzers/aflplusplus/runonce.sh +++ b/fuzzers/aflplusplus/runonce.sh @@ -8,6 +8,7 @@ # - env OUT: path to directory where artifacts are stored # - env PROGRAM: name of program to run (should be found in $OUT) # - env ARGS: extra arguments to pass to the program +# - env SOURCE_COVERAGE: if source-based code coverage is enabled ## export TIMELIMIT=0.1s @@ -24,5 +25,8 @@ if [ -z "$args" ]; then args="'$1'" fi +if [ ! -z $SOURCE_COVERAGE ]; then + export LD_PRELOAD=$OUT/source_coverage.so +fi timeout -s KILL --preserve-status $TIMELIMIT bash -c \ "run_limited '$OUT/afl/$PROGRAM' $args" diff --git a/fuzzers/honggfuzz/coverage.sh b/fuzzers/honggfuzz/coverage.sh new file mode 100644 index 000000000..5489bba5b --- /dev/null +++ b/fuzzers/honggfuzz/coverage.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +## +# Pre-requirements: +# - env SHARED: path to directory shared with host (to store results) +## + +export DIRECTORY_TO_SEARCH=$SHARED/output +chmod -R o+rx $SHARED/output +export PATTERN_TO_MATCH="*" \ No newline at end of file diff --git a/fuzzers/honggfuzz/fetch.sh b/fuzzers/honggfuzz/fetch.sh index 5307403d7..ad4e461cc 100755 --- a/fuzzers/honggfuzz/fetch.sh +++ b/fuzzers/honggfuzz/fetch.sh @@ -6,21 +6,8 @@ set -e # - env FUZZER: path to fuzzer work dir ## -git clone --no-checkout https://github.com/google/honggfuzz.git "$FUZZER/repo" -git -C "$FUZZER/repo" checkout fc6b818c1276056bc565d07edec6ada784cd1670 +# Currently points to the first commit of 2025 +HONGGFUZZ_STABLE_HASH=974db6a90f0efcf6b1171cf355a960fed7b1302d -patch -p1 -d "$FUZZER/repo" << EOF ---- a/linux/trace.c -+++ b/linux/trace.c -@@ -232,8 +232,8 @@ struct user_regs_struct { - #endif /* defined(__ANDROID__) */ - - #if defined(__clang__) --_Pragma("clang Diagnostic push\n"); --_Pragma("clang Diagnostic ignored \"-Woverride-init\"\n"); -+_Pragma("clang diagnostic push\n"); -+_Pragma("clang diagnostic ignored \"-Woverride-init\""); - #endif - - static struct { -EOF +git clone --no-checkout https://github.com/google/honggfuzz.git "$FUZZER/repo" +git -C "$FUZZER/repo" checkout $HONGGFUZZ_STABLE_HASH diff --git a/fuzzers/honggfuzz/instrument.sh b/fuzzers/honggfuzz/instrument.sh index f2ef1dcfa..8137b341b 100755 --- a/fuzzers/honggfuzz/instrument.sh +++ b/fuzzers/honggfuzz/instrument.sh @@ -7,6 +7,8 @@ set -e # - env TARGET: path to target work dir # - env MAGMA: path to Magma support files # - env OUT: path to directory where artifacts are stored +# - env SOURCE_COVERAGE: if source-based code coverage is enabled +# - env COV: path to directory where artifacts for source-base code coverage are stored # - env CFLAGS and CXXFLAGS must be set to link against Magma instrumentation ## @@ -18,4 +20,15 @@ export CXX="$FUZZER/repo/hfuzz_cc/hfuzz-clang++" # NOTE: We pass $OUT directly to the target build.sh script, since the artifact # itself is the fuzz target. In the case of Angora, we might need to -# replace $OUT by $OUT/fast and $OUT/track, for instance. \ No newline at end of file +# replace $OUT by $OUT/fast and $OUT/track, for instance. + +if [ ! -z $SOURCE_COVERAGE ]; then + export CC=clang + export CXX=clang++ + export CFLAGS="$SOURCE_COVERAGE_FLAGS" + export CXXFLAGS="$SOURCE_COVERAGE_FLAGS" + export LDFLAGS="$SOURCE_COVERAGE_FLAGS" + export LIBS="" + export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + OUT=$COV TARGET=$COV "$TARGET/build.sh" +fi diff --git a/fuzzers/honggfuzz/postinstall.sh b/fuzzers/honggfuzz/postinstall.sh new file mode 100755 index 000000000..6fb2c5594 --- /dev/null +++ b/fuzzers/honggfuzz/postinstall.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# This postinstall script is needed for installing some additional dependencies for some targets (poppler). +# The reason why this needs to be done after the honggfuzz build is because the libc++ packages from the llvm +# PPA bring in libunwind-16 with them, this breaks the honggfuzz build. These packages could be installed +# in the preinstall for poppler instead but that could break other fuzzers since this is a honggfuzz specific +# dependency. + +export LLVM_VERSION=16 + +sudo apt-get install -y \ + libclang-rt-$LLVM_VERSION-dev libc++-$LLVM_VERSION-dev libc++abi-$LLVM_VERSION-dev diff --git a/fuzzers/honggfuzz/preinstall.sh b/fuzzers/honggfuzz/preinstall.sh index af78a1c09..3c342187d 100755 --- a/fuzzers/honggfuzz/preinstall.sh +++ b/fuzzers/honggfuzz/preinstall.sh @@ -1,35 +1,38 @@ #!/bin/bash set -e -apt-get update && \ - apt-get install -y make git wget binutils-dev libunwind-dev \ - clang-9 llvm-9-dev build-essential libblocksruntime-dev +export LLVM_VERSION=16 +apt-get update && \ + apt-get install -y make git wget binutils-dev build-essential \ + libblocksruntime-dev libunwind-dev clang-$LLVM_VERSION + update-alternatives \ - --install /usr/lib/llvm llvm /usr/lib/llvm-9 20 \ - --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-9 \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-9 \ - --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-9 \ - --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-9 \ - --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-9 \ - --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-9 \ - --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-9 \ - --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-9 \ - --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-9 \ - --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-9 \ - --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-9 \ - --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-9 \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-9 \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-9 \ - --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-9 \ - --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-9 \ - --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-9 \ - --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-9 \ - --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-9 \ - --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-9 \ - --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-9 + --install /usr/lib/llvm llvm /usr/lib/llvm-$LLVM_VERSION 20 \ + --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-$LLVM_VERSION \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-$LLVM_VERSION \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-$LLVM_VERSION \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-$LLVM_VERSION \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-$LLVM_VERSION \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-$LLVM_VERSION \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-$LLVM_VERSION \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-$LLVM_VERSION \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-$LLVM_VERSION \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-$LLVM_VERSION \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-$LLVM_VERSION \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-$LLVM_VERSION \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-$LLVM_VERSION \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-$LLVM_VERSION \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-$LLVM_VERSION \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-$LLVM_VERSION \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-$LLVM_VERSION \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-$LLVM_VERSION \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-$LLVM_VERSION \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-$LLVM_VERSION \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-$LLVM_VERSION \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-$LLVM_VERSION update-alternatives \ - --install /usr/bin/clang clang /usr/bin/clang-9 20 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-9 \ - --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-9 \ No newline at end of file + --install /usr/bin/clang clang /usr/bin/clang-$LLVM_VERSION 20 \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-$LLVM_VERSION \ No newline at end of file diff --git a/fuzzers/honggfuzz/runonce.sh b/fuzzers/honggfuzz/runonce.sh index e4e29ad62..a5abd14ba 100755 --- a/fuzzers/honggfuzz/runonce.sh +++ b/fuzzers/honggfuzz/runonce.sh @@ -8,6 +8,7 @@ # - env OUT: path to directory where artifacts are stored # - env PROGRAM: name of program to run (should be found in $OUT) # - env ARGS: extra arguments to pass to the program +# - env SOURCE_COVERAGE: if source-based code coverage is enabled ## export TIMELIMIT=0.1s @@ -25,5 +26,8 @@ if [ -z "$args" ]; then args="'$1'" fi +if [ ! -z $SOURCE_COVERAGE ]; then + export LD_PRELOAD=$OUT/source_coverage.so +fi timeout -s KILL --preserve-status $TIMELIMIT bash -c \ "run_limited '$OUT/$PROGRAM' $args" \ No newline at end of file diff --git a/fuzzers/libfuzzer/build.sh b/fuzzers/libfuzzer/build.sh index 740c40c31..243e95175 100755 --- a/fuzzers/libfuzzer/build.sh +++ b/fuzzers/libfuzzer/build.sh @@ -9,8 +9,8 @@ set -e # We need the version of LLVM which has the LLVMFuzzerRunDriver exposed cd "$FUZZER/repo/compiler-rt/lib/fuzzer" for f in *.cpp; do - clang++ -stdlib=libstdc++ -fPIC -O2 -std=c++11 $f -c & + clang++ -stdlib=libc++ -fPIC -O2 -std=c++20 $f -c & done && wait ar r "$OUT/libFuzzer.a" *.o -clang++ $CXXFLAGS -std=c++11 -c "$FUZZER/src/driver.cpp" -fPIC -o "$OUT/driver.o" \ No newline at end of file +clang++ $CXXFLAGS -std=c++20 -c "$FUZZER/src/driver.cpp" -fPIC -o "$OUT/driver.o" \ No newline at end of file diff --git a/fuzzers/libfuzzer/coverage.sh b/fuzzers/libfuzzer/coverage.sh new file mode 100644 index 000000000..9d0c8b40a --- /dev/null +++ b/fuzzers/libfuzzer/coverage.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +## +# Pre-requirements: +# - env SHARED: path to directory shared with host (to store results) +## + +export DIRECTORY_TO_SEARCH=$SHARED/corpus +chmod -R o+rx $SHARED/corpus +export PATTERN_TO_MATCH="*" diff --git a/fuzzers/libfuzzer/fetch.sh b/fuzzers/libfuzzer/fetch.sh index cea097035..fc85196ee 100755 --- a/fuzzers/libfuzzer/fetch.sh +++ b/fuzzers/libfuzzer/fetch.sh @@ -6,5 +6,19 @@ set -e # - env FUZZER: path to fuzzer work dir ## -git clone --no-checkout https://github.com/llvm/llvm-project.git "$FUZZER/repo" -git -C "$FUZZER/repo" checkout 29cc50e17a6800ca75cd23ed85ae1ddf3e3dcc14 \ No newline at end of file +# This fetch script fetches only the compiler-rt source code from the LLVM project which +# is much more efficient than cloning the entire LLVM project which takes ~12 minutes. +# It needs latest git version to support sparse checkout. + +# Currently points to the first commit of 2025 +COMPLIER_RT_STABLE_HASH=4b577830033066cfd1b2acf4fcf39950678b27bd + +git clone --depth 1 --filter=blob:none --sparse \ + https://github.com/llvm/llvm-project.git "$FUZZER/repo" + +pushd "$FUZZER/repo" +git sparse-checkout init --cone +git sparse-checkout set compiler-rt/lib/fuzzer +git fetch --depth 1 origin $COMPLIER_RT_STABLE_HASH +git checkout $COMPLIER_RT_STABLE_HASH +popd diff --git a/fuzzers/libfuzzer/instrument.sh b/fuzzers/libfuzzer/instrument.sh index 8d17752db..47adab59b 100755 --- a/fuzzers/libfuzzer/instrument.sh +++ b/fuzzers/libfuzzer/instrument.sh @@ -7,6 +7,8 @@ set -e # - env TARGET: path to target work dir # - env MAGMA: path to Magma support files # - env OUT: path to directory where artifacts are stored +# - env SOURCE_COVERAGE: if source-based code coverage is enabled +# - env COV: path to directory where artifacts for source-base code coverage are stored # - env CFLAGS and CXXFLAGS must be set to link against Magma instrumentation ## @@ -17,7 +19,7 @@ export CFLAGS="$CFLAGS -fsanitize=fuzzer-no-link" export CXXFLAGS="$CXXFLAGS -fsanitize=fuzzer-no-link" export LDFLAGS="$LDFLAGS -fsanitize=fuzzer-no-link" -export LIBS="$LIBS -l:driver.o $OUT/libFuzzer.a -lstdc++" +export LIBS="$LIBS -l:driver.o $OUT/libFuzzer.a -lc++ -lc++abi" "$MAGMA/build.sh" "$TARGET/build.sh" @@ -25,3 +27,14 @@ export LIBS="$LIBS -l:driver.o $OUT/libFuzzer.a -lstdc++" # NOTE: We pass $OUT directly to the target build.sh script, since the artifact # itself is the fuzz target. In the case of Angora, we might need to # replace $OUT by $OUT/fast and $OUT/track, for instance. + +if [ ! -z $SOURCE_COVERAGE ]; then + export CC=clang + export CXX=clang++ + export CFLAGS="$SOURCE_COVERAGE_FLAGS" + export CXXFLAGS="$SOURCE_COVERAGE_FLAGS" + export LDFLAGS="$SOURCE_COVERAGE_FLAGS" + export LIBS="" + export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + OUT=$COV TARGET=$COV "$TARGET/build.sh" +fi diff --git a/fuzzers/libfuzzer/preinstall.sh b/fuzzers/libfuzzer/preinstall.sh index 715b9e2aa..0f08e6fad 100755 --- a/fuzzers/libfuzzer/preinstall.sh +++ b/fuzzers/libfuzzer/preinstall.sh @@ -1,42 +1,40 @@ #!/bin/bash set -e +LLVM_VERSION=16 + apt-get update && \ apt-get install -y make build-essential wget git -apt-get install -y apt-utils apt-transport-https ca-certificates gnupg - -echo deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main >> /etc/apt/sources.list -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - -apt-get update && \ - apt-get install -y clang-11 +apt-get install -y clang-$LLVM_VERSION \ + libc++-$LLVM_VERSION-dev libc++abi-$LLVM_VERSION-dev update-alternatives \ - --install /usr/lib/llvm llvm /usr/lib/llvm-11 20 \ - --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-11 \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-11 \ - --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-11 \ - --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-11 \ - --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-11 \ - --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-11 \ - --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-11 \ - --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-11 \ - --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-11 \ - --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-11 \ - --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-11 \ - --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-11 \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-11 \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-11 \ - --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-11 \ - --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-11 \ - --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-11 \ - --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-11 \ - --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-11 \ - --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-11 \ - --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-11 + --install /usr/lib/llvm llvm /usr/lib/llvm-$LLVM_VERSION 20 \ + --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-$LLVM_VERSION \ + --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-$LLVM_VERSION \ + --slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-$LLVM_VERSION \ + --slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-$LLVM_VERSION \ + --slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-$LLVM_VERSION \ + --slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-$LLVM_VERSION \ + --slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-$LLVM_VERSION \ + --slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-$LLVM_VERSION \ + --slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-$LLVM_VERSION \ + --slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-$LLVM_VERSION \ + --slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-$LLVM_VERSION \ + --slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-$LLVM_VERSION \ + --slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-$LLVM_VERSION \ + --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-$LLVM_VERSION \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-$LLVM_VERSION \ + --slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-$LLVM_VERSION \ + --slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-$LLVM_VERSION \ + --slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-$LLVM_VERSION \ + --slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-$LLVM_VERSION \ + --slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-$LLVM_VERSION \ + --slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-$LLVM_VERSION \ + --slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-$LLVM_VERSION update-alternatives \ - --install /usr/bin/clang clang /usr/bin/clang-11 20 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-11 \ - --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-11 + --install /usr/bin/clang clang /usr/bin/clang-$LLVM_VERSION 20 \ + --slave /usr/bin/clang++ clang++ /usr/bin/clang++-$LLVM_VERSION \ + --slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-$LLVM_VERSION diff --git a/fuzzers/libfuzzer/run.sh b/fuzzers/libfuzzer/run.sh index 951fb5aca..e95051416 100755 --- a/fuzzers/libfuzzer/run.sh +++ b/fuzzers/libfuzzer/run.sh @@ -11,9 +11,10 @@ # - env FUZZARGS: extra arguments to pass to the fuzzer ## -mkdir -p "$SHARED/findings" +mkdir -p "$SHARED/findings" "$SHARED/corpus" +cp $TARGET/corpus/$PROGRAM/* "$SHARED/corpus" "$OUT/$PROGRAM" -rss_limit_mb=100 \ -fork=1 -ignore_timeouts=1 -ignore_crashes=1 -ignore_ooms=1 \ -artifact_prefix="$SHARED/findings/" $FUZZARGS \ - "$TARGET/corpus/$PROGRAM" $ARGS + "$SHARED/corpus" $ARGS diff --git a/fuzzers/libfuzzer/runonce.sh b/fuzzers/libfuzzer/runonce.sh index 6ffea3182..07f4c0e1a 100755 --- a/fuzzers/libfuzzer/runonce.sh +++ b/fuzzers/libfuzzer/runonce.sh @@ -8,6 +8,7 @@ # - env OUT: path to directory where artifacts are stored # - env PROGRAM: name of program to run (should be found in $OUT) # - env ARGS: extra arguments to pass to the program +# - env SOURCE_COVERAGE: if source-based code coverage is enabled ## export TIMELIMIT=0.1s @@ -26,5 +27,8 @@ if [ -z "$args" ]; then args="'$1'" fi +if [ ! -z $SOURCE_COVERAGE ]; then + export LD_PRELOAD=$OUT/source_coverage.so +fi timeout -s KILL --preserve-status $TIMELIMIT bash -c \ "run_limited '$OUT/$PROGRAM' $args" \ No newline at end of file diff --git a/gen_target_releases.py b/gen_target_releases.py new file mode 100644 index 000000000..c741da060 --- /dev/null +++ b/gen_target_releases.py @@ -0,0 +1,121 @@ +#!/usr/bin/python3 + +""" +This script is used to generate a list of releases for a target. The results are written +to a file at path `targets//releases`. The releases file is used to point to +a specific target version while building it using the tools/captain/run.sh script. +""" + +import os +import subprocess +import sys + +from datetime import datetime + +subprocess.check_call([sys.executable, "-m", "pip", "install", "gitpython"]) + +from git import Repo # type: ignore # noqa: E402 + +START_YEAR = 2022 +# If START_YEAR=2022, END_YEAR year can be 2025, 2028, 2031, etc. in steps of 3. +END_YEAR = START_YEAR + (((datetime.now().year - START_YEAR) // 3) * 3) + +links = { + "libpng": "https://github.com/pnggroup/libpng.git", + "libsndfile": "https://github.com/libsndfile/libsndfile.git", + "libtiff": "https://gitlab.com/libtiff/libtiff.git", + "libxml2": "https://gitlab.gnome.org/GNOME/libxml2.git", + "lua": "https://github.com/lua/lua.git", + "openssl": "https://github.com/openssl/openssl.git", + "php": "https://github.com/php/php-src.git", + "poppler": "https://gitlab.freedesktop.org/poppler/poppler.git", + "sqlite3": "https://github.com/sqlite/sqlite", +} + +repos = [] + + +def clone_repo(repo_url): + repo_name = repo_url.split("/")[-1].replace(".git", "") + local_path = os.path.join(os.getcwd(), repo_name) + repos.append(local_path) + + repo = None + if not os.path.exists(local_path): + repo = Repo.clone_from(repo_url, local_path) + print(f"Cloned {repo_url}") + else: + repo = Repo(local_path) + + return repo + + +def get_tags(repo): + # get all tags available + year_to_tag = {} + for tag in repo.tags: + tag_date = tag.tag.tagged_date if tag.tag else tag.commit.committed_date + year = datetime.fromtimestamp(tag_date).year + if year not in year_to_tag: + year_to_tag[year] = tag.name + + year_to_tag_sorted = dict(sorted(year_to_tag.items())) + min_year = list(year_to_tag_sorted.keys())[0] + + # fill the gaps + last_tag = 0 + for i in range(min_year, END_YEAR): + if i in year_to_tag_sorted: + last_tag = year_to_tag_sorted[i] + else: + year_to_tag_sorted[i] = last_tag + + year_to_tag_sorted = dict(sorted(year_to_tag_sorted.items())) + return year_to_tag_sorted + + +def get_first_commit_of_year(repo, year): + since = f"{year}-01-01" + until = f"{year}-12-31" + commits = list(repo.iter_commits(rev="HEAD", since=since, until=until)) + return commits[-1].hexsha + + +for target, link in links.items(): + path_to_releases = os.path.join("targets", target, "releases") + + if isinstance(link, str): # git repo + repo = clone_repo(link) + year_to_tag = get_tags(repo) + + latest_year = max(year_to_tag.keys()) + stable_hash = get_first_commit_of_year(repo, END_YEAR) + + print(f"Got tags and latest commit of {target}") + releases = [ + f'{target}_PIONEER="{link}"\n', + f'{target}_PIONEER_STABLE_COMMIT="{stable_hash}"\n', + ] + for i in range(START_YEAR, END_YEAR): + releases.append(f'{target}_LEGACY_{i}="{link}"\n') + releases.append(f'{target}_LEGACY_{i}_TAG="{year_to_tag[i]}"\n') + else: # predefined + assert isinstance(link, dict) + releases = [ + f'{target}_PIONEER="{link[END_YEAR]}"\n', + ] + for i in range(START_YEAR, END_YEAR): + if isinstance(link[i], str): + releases.append(f'{target}_LEGACY_{i}="{link[i]}"\n') + else: # git repo, + tag or commit + releases.append(f'{target}_LEGACY_{i}="{link[i][0]}"\n') + releases.append(f'{target}_LEGACY_{i}_TAG="{link[i][1]}"\n') + + with open(path_to_releases, "w") as f: + for line in releases: + f.write(line) + print(f"Wrote to {path_to_releases}") + +# clean +all_repos = " ".join(repos) +print(f"rm -rf {all_repos}") diff --git a/magma/apply_patches.sh b/magma/apply_patches.sh index 38989b114..01abf750e 100755 --- a/magma/apply_patches.sh +++ b/magma/apply_patches.sh @@ -6,11 +6,19 @@ set -e # - env TARGET: path to target work dir ## +BUGS_PATH="${1:-"$TARGET/patches/bugs"}" + # TODO filter patches by target config.yaml -find "$TARGET/patches/setup" "$TARGET/patches/bugs" -name "*.patch" | \ +find "$TARGET/patches/setup" $BUGS_PATH -name "*.patch" | \ while read patch; do echo "Applying $patch" name=${patch##*/} name=${name%.patch} sed "s/%MAGMA_BUG%/$name/g" "$patch" | patch -p1 -d "$TARGET/repo" done + +find "$TARGET/patches/setup" -name "*.patch" | \ +while read patch; do + echo "Applying $patch" + cat $patch | patch -p1 -d "$COV/repo" +done diff --git a/magma/coverage.sh b/magma/coverage.sh new file mode 100755 index 000000000..8d9962bb7 --- /dev/null +++ b/magma/coverage.sh @@ -0,0 +1,47 @@ +#!/bin/bash -ex + +## +# Pre-requirements: +# - env FUZZER: path to fuzzer work dir +# - env TARGET: path to target work dir +# - env MAGMA: path to Magma support files +# - env SHARED: path to directory shared with host (to store results) +# - env PROGRAM: name of program to run (should be found in $OUT) +# - env COV: path to directory where artifacts for source-base code coverage are stored +## + +# load $DIRECTORY_TO_SEARCH and $PATTERN_TO_MATCH +source "$FUZZER/coverage.sh" + +# replay +seeds=() +readarray -d '' seeds < <(find $DIRECTORY_TO_SEARCH -type f -name "$PATTERN_TO_MATCH" -print0) +rm -f $COV/*profdata +for seed in ${seeds[@]}; do + seed_name=$(basename $seed) + export LLVM_PROFILE_FILE=$COV/$seed_name.profdata + export OUT=$COV + $FUZZER/runonce.sh $seed || continue +done + +python3 $MAGMA/coverage_overtime.py $DIRECTORY_TO_SEARCH $COV $COV/$PROGRAM 60 --output $SHARED/coverage_overtime.txt +chmod o+rx $SHARED/coverage_overtime.txt + +export TARGET_NAME=$(basename $TARGET) + +# filter valid profdata files before merging +VALID_PROFDATA=() +for f in "$COV"/*.profdata; do + if llvm-profdata show "$f" > /dev/null 2>&1; then + VALID_PROFDATA+=("$f") + fi +done +# merge +llvm-profdata merge -output=$COV/$TARGET_NAME.profraw "${VALID_PROFDATA[@]}" +# show +llvm-cov show -format=html -output-dir=$SHARED/coverage-reports \ + -instr-profile $COV/$TARGET_NAME.profraw $COV/$PROGRAM +llvm-cov export -format=text -summary-only \ + -instr-profile $COV/$TARGET_NAME.profraw $COV/$PROGRAM > $SHARED/coverage-reports.json +chmod -R o+rx $SHARED/coverage-reports +echo "Dumped the source-based code coverage" diff --git a/magma/coverage_overtime.py b/magma/coverage_overtime.py new file mode 100644 index 000000000..5af0f5057 --- /dev/null +++ b/magma/coverage_overtime.py @@ -0,0 +1,138 @@ +import os +import argparse +import subprocess +import json + + +def get_timestamp(filepath): + return os.path.getmtime(filepath) + + +def is_valid_profdata(filepath): + try: + result = subprocess.run( + ["llvm-profdata", "show", filepath], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return result.returncode == 0 + except Exception: + return False + + +def merge_profdata_to_profraw(corpus_dir, profdata_path, timestamp): + """ + Given a single .profdata file, find all .profdata files in the same directory + that have a timestamp <= current file, and merge them into a .profraw. + """ + cov_dir = os.path.dirname(profdata_path) + + target_name = os.path.splitext(os.path.basename(profdata_path))[0] + profraw_path = os.path.join(cov_dir, f"{target_name}.profraw") + + # Collect all valid profdata files up to current timestamp + profdata_files = [] + for f in os.listdir(corpus_dir): + path = os.path.join(cov_dir, f"{f}.profdata") + if ( + os.path.exists(path) + and os.path.getmtime(os.path.join(corpus_dir, f)) <= timestamp + ): + if is_valid_profdata(path): + profdata_files.append(path) + assert profdata_path in profdata_files + + if not profdata_files: + raise FileNotFoundError("No .profdata files found to merge.") + + cmd_merge = ["llvm-profdata", "merge", "-output", profraw_path] + profdata_files + result = subprocess.run(cmd_merge, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"llvm-profdata merge failed:\n{result.stderr}") + + return profraw_path + + +def generate_json_report(profraw_path, binary_path, output_json_path): + cmd_cov = [ + "llvm-cov", + "export", + "-format=text", + "-summary-only", + f"-instr-profile={profraw_path}", + binary_path, + ] + result_cov = subprocess.run(cmd_cov, capture_output=True, text=True) + if result_cov.returncode != 0: + raise RuntimeError(f"llvm-cov export failed:\n{result_cov.stderr}") + + with open(output_json_path, "w") as f: + f.write(result_cov.stdout) + + +def parse_coverage_from_json(json_path): + with open(json_path) as f: + data = json.load(f) + totals = data.get("data", [{}])[0].get("totals", {}) + branches = totals.get("branches", {}) + covered = branches.get("covered", 0) + percent = branches.get("percent", 0.0) + return covered, percent + + +def main(corpus_dir, profdata_dir, binary_path, output_txt, interval): + results = [] + last_sampled_time = 0 + + file_infos = [] + for root, _, files in os.walk(profdata_dir): + for file in sorted(files): + if file.endswith(".profdata"): + profdata_path = os.path.join(root, file) + testcase_path = os.path.join(corpus_dir, os.path.splitext(file)[0]) + timestamp = get_timestamp(testcase_path) + file_infos.append((profdata_path, timestamp)) + + file_infos.sort(key=lambda x: x[1]) + for profdata_path, timestamp in file_infos: + try: + # skip a few and always include the last one + if ( + timestamp - last_sampled_time >= interval + or timestamp == file_infos[-1][1] + ): + json_output = os.path.splitext(profdata_path)[0] + ".json" + profraw_path = merge_profdata_to_profraw( + corpus_dir, profdata_path, timestamp + ) + generate_json_report(profraw_path, binary_path, json_output) + covered, percent = parse_coverage_from_json(json_output) + results.append((timestamp, covered, round(percent, 2))) + last_sampled_time = timestamp + except Exception as e: + print(f"Error processing {profdata_path}: {e}") + + if len(results) == 0: + return + with open(output_txt, "w") as out: + out.write("timestamp,covered,percent\n") + for ts, covered, pct in sorted(results, key=lambda x: x[0]): + out.write(f"{ts},{covered},{pct}\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate coverage summary from .profdata files." + ) + parser.add_argument("corpus_dir", help="Directory containing test cases") + parser.add_argument("profdata_dir", help="Directory containing .profdata files") + parser.add_argument("binary_path", help="Path to the instrumented binary") + parser.add_argument("interval", help="Sample every `interval` seconds", type=int) + parser.add_argument( + "--output", default="coverage_overtime.txt", help="Output summary .txt file" + ) + + args = parser.parse_args() + main( + args.corpus_dir, args.profdata_dir, args.binary_path, args.output, args.interval + ) diff --git a/magma/fetch_target.sh b/magma/fetch_target.sh new file mode 100755 index 000000000..8e5885dcf --- /dev/null +++ b/magma/fetch_target.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +## +# Pre-requirements: +# - env TARGET: path to target work dir +# - env TARGET_NAME: name of the target +# - env TARGET_VERSION: version of target from releases (PIONEER = latest) +## + +source "$TARGET"/releases + +to_fetch_var="$TARGET_NAME"_"$TARGET_VERSION" +to_fetch=${!to_fetch_var} +to_checkout_var="$TARGET_NAME"_"$TARGET_VERSION"_"STABLE_COMMIT" +to_checkout=${!to_checkout_var} + +git_hosts=( + github.com + gitlab.com + gitlab.gnome.org + gitlab.freedesktop.org +) +host=$(echo "$to_fetch" | awk -F/ '{print $3}') + +if [[ " ${git_hosts[*]} " == *" $host "* ]]; then + git init "$TARGET/repo" + cd "$TARGET/repo" && \ + git remote add origin "$to_fetch" && \ + git fetch --depth 1 origin "$to_checkout" && \ + git checkout "$to_checkout" +elif [[ "$to_fetch" =~ \.tar\.gz(\?|$) ]]; then + wget -O "$TARGET"/repo.tar.gz "$to_fetch" + mkdir "$TARGET"/repo + tar -xf "$TARGET"/repo.tar.gz --strip-components=1 -C "$TARGET"/repo +else + echo "Unsupported link: $to_fetch" + exit +fi + +cp -r "$TARGET"/repo "$COV"/repo +cp -r "$TARGET"/src "$COV"/src || echo "No such file or directory" diff --git a/magma/prebuild.sh b/magma/prebuild.sh index 602ba9d77..3f8e4374d 100755 --- a/magma/prebuild.sh +++ b/magma/prebuild.sh @@ -5,6 +5,7 @@ set -e # Pre-requirements: # - env MAGMA: path to Magma support files # - env OUT: path to directory where artifacts are stored +# - env COV: path to directory where artifacts for source-base code coverage are stored # - env SHARED: path to directory shared with host (to store results) ## @@ -17,3 +18,6 @@ $CC $CFLAGS -g -O0 -D"MAGMA_STORAGE=\"$MAGMA_STORAGE\"" "$MAGMA/src/monitor.c" \ "$OUT/pre_storage.o" -I "$MAGMA/src/" -o "$OUT/monitor" $LDFLAGS $LIBS rm "$OUT/pre_storage.o" + +$CC $CFLAGS -shared "$MAGMA/src/source_coverage.c" \ + -fPIC -o $COV/source_coverage.so -ldl diff --git a/magma/run.sh b/magma/run.sh index d6fcdd53e..2a3bda22e 100755 --- a/magma/run.sh +++ b/magma/run.sh @@ -13,6 +13,7 @@ # - env TIMEOUT: time to run the campaign # - env MAGMA: path to Magma support files # + env LOGSIZE: size (in bytes) of log file to generate (default: 1 MiB) +# - env SOURCE_COVERAGE: if source-based code coverage is enabled ## # set default max log size to 1 MiB @@ -46,7 +47,9 @@ fi # launch the fuzzer in parallel with the monitor rm -f "$MONITOR/tmp"* +shopt -s nullglob polls=("$MONITOR"/*) +shopt -u nullglob if [ ${#polls[@]} -eq 0 ]; then counter=0 else @@ -71,6 +74,7 @@ echo "Campaign launched at $(date '+%F %R')" timeout $TIMEOUT "$FUZZER/run.sh" | \ multilog n2 s$LOGSIZE "$SHARED/log" +chmod o+rx $SHARED/log if [ -f "$SHARED/log/current" ]; then cat "$SHARED/log/current" fi @@ -78,3 +82,7 @@ fi echo "Campaign terminated at $(date '+%F %R')" kill $(jobs -p) + +if [ ! -z $SOURCE_COVERAGE ]; then + $MAGMA/coverage.sh +fi diff --git a/magma/src/canary.h b/magma/src/canary.h index 572f8b22b..ceb1b03ff 100644 --- a/magma/src/canary.h +++ b/magma/src/canary.h @@ -1,3 +1,4 @@ +#ifndef __ASSEMBLER__ #ifndef CANARY_H_ #define CANARY_H_ #ifdef __cplusplus @@ -21,3 +22,4 @@ extern void magma_log(const char *bug, int condition); } #endif #endif +#endif /* __ASSEMBLER__ */ diff --git a/magma/src/source_coverage.c b/magma/src/source_coverage.c new file mode 100644 index 000000000..0b0db1709 --- /dev/null +++ b/magma/src/source_coverage.c @@ -0,0 +1,33 @@ +#define _GNU_SOURCE +#include +#include +#include +#include + +static void clang_source_coverage_handler(int signum) { + fprintf(stderr, "DEBUG: Clang source coverage signal handler invoked (sig: %d).\n", signum); + return exit(0); +} + +static void clang_source_coverage_init(void) { + signal(SIGSEGV, clang_source_coverage_handler); + signal(SIGILL, clang_source_coverage_handler); + signal(SIGSTOP, clang_source_coverage_handler); + signal(SIGABRT, clang_source_coverage_handler); + fprintf(stderr, "DEBUG: Clang source coverage signal handler installed.\n"); +} + +int __libc_start_main( + int (*main)(int, char **, char **), + int argc, + char **argv, + int (*init)(int, char **, char **), + void (*fini)(void), + void (*rtld_fini)(void), + void *stack_end) +{ + typeof(&__libc_start_main) orig = dlsym(RTLD_NEXT, "__libc_start_main"); + + clang_source_coverage_init(); + return orig(main, argc, argv, init, fini, rtld_fini, stack_end); +} \ No newline at end of file diff --git a/targets/libpng/build.sh b/targets/libpng/build.sh index 9d2dd0095..22d7192dc 100755 --- a/targets/libpng/build.sh +++ b/targets/libpng/build.sh @@ -9,7 +9,7 @@ set -e ## if [ ! -d "$TARGET/repo" ]; then - echo "fetch.sh must be executed first." + echo "fetch_target.sh must be executed first." exit 1 fi @@ -26,4 +26,4 @@ cp .libs/libpng16.a "$OUT/" $CXX $CXXFLAGS -std=c++11 -I. \ contrib/oss-fuzz/libpng_read_fuzzer.cc \ -o $OUT/libpng_read_fuzzer \ - $LDFLAGS .libs/libpng16.a $LIBS -lz \ No newline at end of file + $LDFLAGS .libs/libpng16.a $LIBS $LIB_FUZZING_ENGINE -lz diff --git a/targets/libpng/build_poc.sh b/targets/libpng/build_poc.sh new file mode 100755 index 000000000..3f0c2b8d3 --- /dev/null +++ b/targets/libpng/build_poc.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch_target.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" +autoreconf -f -i +./configure --disable-shared +make -j$(nproc) clean +make -j$(nproc) +make install diff --git a/targets/libpng/fetch.sh b/targets/libpng/fetch.sh deleted file mode 100755 index 09772565c..000000000 --- a/targets/libpng/fetch.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -git clone --no-checkout https://github.com/glennrp/libpng.git \ - "$TARGET/repo" -git -C "$TARGET/repo" checkout a37d4836519517bdce6cb9d956092321eca3e73b \ No newline at end of file diff --git a/targets/libpng/patches/bugs/PNG002.patch b/targets/libpng/patches/bugs/PNG002.patch index 88e84d169..9a3566afb 100644 --- a/targets/libpng/patches/bugs/PNG002.patch +++ b/targets/libpng/patches/bugs/PNG002.patch @@ -46,18 +46,17 @@ index 57b9399..2aba9d7 100644 /* READ APIs * --------- diff --git a/pngerror.c b/pngerror.c -index ec3a709..fed5cdc 100644 +index aa0ae58e1..7671b6cf2 100644 --- a/pngerror.c +++ b/pngerror.c -@@ -951,6 +951,10 @@ png_safe_execute(png_imagep image_in, int (*function)(png_voidp), png_voidp arg) - result = function(arg); +@@ -948,6 +948,10 @@ png_safe_execute(png_imagep image, int (*function)(png_voidp), png_voidp arg) + return result; } +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", (image->flags & PNG_IMAGE_FLAG_INVALID) != 0); +#endif + + /* On png_error, return via longjmp, pop the jmpbuf, and free the image. */ image->opaque->error_buf = saved_error_buf; - - /* And do the cleanup prior to any failure return. */ - + png_image_free(image); diff --git a/targets/libpng/patches/bugs/PNG006.patch b/targets/libpng/patches/bugs/PNG006.patch index de55580a5..7413165b7 100644 --- a/targets/libpng/patches/bugs/PNG006.patch +++ b/targets/libpng/patches/bugs/PNG006.patch @@ -12,18 +12,17 @@ index e92008d..8d3cecb 100644 /* Free any eXIf entry */ if (((mask & PNG_FREE_EXIF) & info_ptr->free_me) != 0) { -diff --git a/pngrutil.c b/pngrutil.c -index d5fa08c..2a08277 100644 ---- a/pngrutil.c -+++ b/pngrutil.c -@@ -2059,7 +2059,9 @@ png_handle_eXIf(png_structrp png_ptr, png_inforp info_ptr, png_uint_32 length) - return; - } +diff --git a/pngset.c b/pngset.c +index 3e63c2724..e73a670a6 100644 +--- a/pngset.c ++++ b/pngset.c +@@ -350,7 +350,9 @@ png_set_eXIf_1(png_const_structrp png_ptr, png_inforp info_ptr, + info_ptr->num_exif = num_exif; + info_ptr->exif = new_exif; +#ifdef MAGMA_ENABLE_FIXES info_ptr->free_me |= PNG_FREE_EXIF; +#endif - - info_ptr->eXIf_buf = png_voidcast(png_bytep, - png_malloc_warn(png_ptr, length)); - + info_ptr->valid |= PNG_INFO_eXIf; + } + #endif /* eXIf */ diff --git a/targets/libpng/pocs/PNG001.crash b/targets/libpng/pocs/PNG001.crash new file mode 100644 index 000000000..9b6552674 --- /dev/null +++ b/targets/libpng/pocs/PNG001.crash @@ -0,0 +1,2 @@ +# TODO: Replace this with a real crash and investigate how to to cause this crash +# and add the corresponding bug command diff --git a/targets/libpng/releases b/targets/libpng/releases new file mode 100644 index 000000000..484098e90 --- /dev/null +++ b/targets/libpng/releases @@ -0,0 +1,8 @@ +libpng_PIONEER="https://github.com/pnggroup/libpng.git" +libpng_PIONEER_STABLE_COMMIT="6f786ff0f6bf4b971a7159d2b4a95e8a0770892f" +libpng_LEGACY_2022="https://github.com/pnggroup/libpng.git" +libpng_LEGACY_2022_TAG="v1.6.38" +libpng_LEGACY_2023="https://github.com/pnggroup/libpng.git" +libpng_LEGACY_2023_TAG="v1.6.40" +libpng_LEGACY_2024="https://github.com/pnggroup/libpng.git" +libpng_LEGACY_2024_TAG="v1.6.41" diff --git a/targets/libsndfile/build.sh b/targets/libsndfile/build.sh index 89a808d93..0bd34e293 100755 --- a/targets/libsndfile/build.sh +++ b/targets/libsndfile/build.sh @@ -9,12 +9,12 @@ set -e ## if [ ! -d "$TARGET/repo" ]; then - echo "fetch.sh must be executed first." + echo "fetch_target.sh must be executed first." exit 1 fi cd "$TARGET/repo" -./autogen.sh +autoreconf -f -i ./configure --disable-shared --enable-ossfuzzers make -j$(nproc) clean make -j$(nproc) ossfuzz/sndfile_fuzzer diff --git a/targets/libsndfile/build_poc.sh b/targets/libsndfile/build_poc.sh new file mode 100755 index 000000000..3f0c2b8d3 --- /dev/null +++ b/targets/libsndfile/build_poc.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch_target.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" +autoreconf -f -i +./configure --disable-shared +make -j$(nproc) clean +make -j$(nproc) +make install diff --git a/targets/libsndfile/fetch.sh b/targets/libsndfile/fetch.sh deleted file mode 100755 index a7cdbd05d..000000000 --- a/targets/libsndfile/fetch.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -git clone --no-checkout https://github.com/libsndfile/libsndfile.git \ - "$TARGET/repo" -git -C "$TARGET/repo" checkout 86c9f9eb7022d186ad4d0689487e7d4f04ce2b29 \ No newline at end of file diff --git a/targets/libsndfile/patches/setup/libsndfile.patch b/targets/libsndfile/patches/graveyard/libsndfile.patch similarity index 87% rename from targets/libsndfile/patches/setup/libsndfile.patch rename to targets/libsndfile/patches/graveyard/libsndfile.patch index cd052c765..207772c91 100644 --- a/targets/libsndfile/patches/setup/libsndfile.patch +++ b/targets/libsndfile/patches/graveyard/libsndfile.patch @@ -1,3 +1,6 @@ +// All these updates are already fixed in the latest version of libsndfile. +// This patch is no longer needed. + --- a/Makefile.am +++ b/Makefile.am @@ -450,7 +450,7 @@ diff --git a/targets/libsndfile/pocs/SND001.crash b/targets/libsndfile/pocs/SND001.crash new file mode 100644 index 000000000..0459ec759 Binary files /dev/null and b/targets/libsndfile/pocs/SND001.crash differ diff --git a/targets/libsndfile/preinstall.sh b/targets/libsndfile/preinstall.sh index 5853c68fa..e429a8822 100755 --- a/targets/libsndfile/preinstall.sh +++ b/targets/libsndfile/preinstall.sh @@ -3,4 +3,4 @@ apt-get update && \ apt-get install -y git make autoconf autogen automake build-essential libasound2-dev \ libflac-dev libogg-dev libtool libvorbis-dev libopus-dev libmp3lame-dev \ - libmpg123-dev pkg-config python + libmpg123-dev pkg-config python-is-python3 diff --git a/targets/libsndfile/releases b/targets/libsndfile/releases new file mode 100644 index 000000000..ae7556fd0 --- /dev/null +++ b/targets/libsndfile/releases @@ -0,0 +1,8 @@ +libsndfile_PIONEER="https://github.com/libsndfile/libsndfile.git" +libsndfile_PIONEER_STABLE_COMMIT="2251737b3b175925684ec0d37029ff4cb521d302" +libsndfile_LEGACY_2022="https://github.com/libsndfile/libsndfile.git" +libsndfile_LEGACY_2022_TAG="1.1.0" +libsndfile_LEGACY_2023="https://github.com/libsndfile/libsndfile.git" +libsndfile_LEGACY_2023_TAG="1.2.1" +libsndfile_LEGACY_2024="https://github.com/libsndfile/libsndfile.git" +libsndfile_LEGACY_2024_TAG="1.2.1" diff --git a/targets/libtiff/build.sh b/targets/libtiff/build.sh index fad7e494b..861db23dd 100755 --- a/targets/libtiff/build.sh +++ b/targets/libtiff/build.sh @@ -9,10 +9,13 @@ set -e ## if [ ! -d "$TARGET/repo" ]; then - echo "fetch.sh must be executed first." + echo "fetch_target.sh must be executed first." exit 1 fi +cp "$TARGET/src/tiff_read_rgba_fuzzer.cc" \ + "$TARGET/repo/contrib/oss-fuzz/tiff_read_rgba_fuzzer.cc" + WORK="$TARGET/work" rm -rf "$WORK" mkdir -p "$WORK" @@ -29,4 +32,4 @@ cp "$WORK/bin/tiffcp" "$OUT/" $CXX $CXXFLAGS -std=c++11 -I$WORK/include \ contrib/oss-fuzz/tiff_read_rgba_fuzzer.cc -o $OUT/tiff_read_rgba_fuzzer \ $WORK/lib/libtiffxx.a $WORK/lib/libtiff.a -lz -ljpeg -Wl,-Bstatic -llzma -Wl,-Bdynamic \ - $LDFLAGS $LIBS + $LDFLAGS $LIBS $LIB_FUZZING_ENGINE diff --git a/targets/libtiff/build_poc.sh b/targets/libtiff/build_poc.sh new file mode 100755 index 000000000..8006064d4 --- /dev/null +++ b/targets/libtiff/build_poc.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch_target.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" +./autogen.sh +./configure --disable-shared --enable-tools +make -j$(nproc) clean +make -j$(nproc) +make install diff --git a/targets/libtiff/fetch.sh b/targets/libtiff/fetch.sh deleted file mode 100755 index f6e3c7c23..000000000 --- a/targets/libtiff/fetch.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -git clone --no-checkout https://gitlab.com/libtiff/libtiff.git \ - "$TARGET/repo" -git -C "$TARGET/repo" checkout c145a6c14978f73bb484c955eb9f84203efcb12e - -cp "$TARGET/src/tiff_read_rgba_fuzzer.cc" \ - "$TARGET/repo/contrib/oss-fuzz/tiff_read_rgba_fuzzer.cc" diff --git a/targets/libtiff/patches/bugs/TIF001.patch b/targets/libtiff/patches/bugs/TIF001.patch index 66adb3a7a..13f58d45f 100644 --- a/targets/libtiff/patches/bugs/TIF001.patch +++ b/targets/libtiff/patches/bugs/TIF001.patch @@ -1,16 +1,15 @@ diff --git a/libtiff/tif_predict.c b/libtiff/tif_predict.c -index b775663..98d7c6b 100644 +index 386b5fe8..23b3be07 100644 --- a/libtiff/tif_predict.c +++ b/libtiff/tif_predict.c -@@ -282,12 +282,17 @@ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc) - tmsize_t stride = PredictorState(tif)->stride; +@@ -344,11 +344,16 @@ static int horAcc8(TIFF *tif, uint8_t *cp0, tmsize_t cc) + tmsize_t stride = PredictorState(tif)->stride; - unsigned char* cp = (unsigned char*) cp0; + unsigned char *cp = (unsigned char *)cp0; +#ifdef MAGMA_ENABLE_FIXES - if((cc%stride)!=0) + if ((cc % stride) != 0) { - TIFFErrorExt(tif->tif_clientdata, "horAcc8", - "%s", "(cc%stride)!=0"); + TIFFErrorExtR(tif, "horAcc8", "%s", "(cc%stride)!=0"); return 0; } +#endif @@ -18,17 +17,16 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%stride)!=0); +#endif - if (cc > stride) { - /* -@@ -351,12 +356,17 @@ horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc) - uint16* wp = (uint16*) cp0; - tmsize_t wc = cc / 2; + if (cc > stride) + { +@@ -414,11 +419,16 @@ static int horAcc16(TIFF *tif, uint8_t *cp0, tmsize_t cc) + uint16_t *wp = (uint16_t *)cp0; + tmsize_t wc = cc / 2; +#ifdef MAGMA_ENABLE_FIXES - if((cc%(2*stride))!=0) + if ((cc % (2 * stride)) != 0) { - TIFFErrorExt(tif->tif_clientdata, "horAcc16", - "%s", "cc%(2*stride))!=0"); + TIFFErrorExtR(tif, "horAcc16", "%s", "cc%(2*stride))!=0"); return 0; } +#endif @@ -36,17 +34,16 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%(2*stride))!=0); +#endif - if (wc > stride) { - wc -= stride; -@@ -386,12 +396,17 @@ horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc) - uint32* wp = (uint32*) cp0; - tmsize_t wc = cc / 4; + if (wc > stride) + { +@@ -451,11 +461,16 @@ static int horAcc32(TIFF *tif, uint8_t *cp0, tmsize_t cc) + uint32_t *wp = (uint32_t *)cp0; + tmsize_t wc = cc / 4; +#ifdef MAGMA_ENABLE_FIXES - if((cc%(4*stride))!=0) + if ((cc % (4 * stride)) != 0) { - TIFFErrorExt(tif->tif_clientdata, "horAcc32", - "%s", "cc%(4*stride))!=0"); + TIFFErrorExtR(tif, "horAcc32", "%s", "cc%(4*stride))!=0"); return 0; } +#endif @@ -54,17 +51,16 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%(4*stride))!=0); +#endif - if (wc > stride) { - wc -= stride; -@@ -416,12 +431,17 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc) - uint8 *cp = (uint8 *) cp0; - uint8 *tmp; + if (wc > stride) + { +@@ -515,11 +530,16 @@ static int fpAcc(TIFF *tif, uint8_t *cp0, tmsize_t cc) + uint8_t *cp = (uint8_t *)cp0; + uint8_t *tmp; +#ifdef MAGMA_ENABLE_FIXES - if(cc%(bps*stride)!=0) + if (cc % (bps * stride) != 0) { - TIFFErrorExt(tif->tif_clientdata, "fpAcc", - "%s", "cc%(bps*stride))!=0"); + TIFFErrorExtR(tif, "fpAcc", "%s", "cc%(bps*stride))!=0"); return 0; } +#endif @@ -72,37 +68,34 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%(bps*stride))!=0); +#endif - tmp = (uint8 *)_TIFFmalloc(cc); - if (!tmp) -@@ -486,12 +506,19 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s) - if ((*sp->decodetile)(tif, op0, occ0, s)) { - tmsize_t rowsize = sp->rowsize; - assert(rowsize > 0); -+ + tmp = (uint8_t *)_TIFFmallocExt(tif, cc); + if (!tmp) +@@ -590,12 +610,17 @@ static int PredictorDecodeTile(TIFF *tif, uint8_t *op0, tmsize_t occ0, + { + tmsize_t rowsize = sp->rowsize; + assert(rowsize > 0); +#ifdef MAGMA_ENABLE_FIXES - if((occ0%rowsize) !=0) + if ((occ0 % rowsize) != 0) { - TIFFErrorExt(tif->tif_clientdata, "PredictorDecodeTile", - "%s", "occ0%rowsize != 0"); + TIFFErrorExtR(tif, "PredictorDecodeTile", "%s", + "occ0%rowsize != 0"); return 0; } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", (occ0%rowsize)!=0); +#endif -+ - assert(sp->decodepfunc != NULL); - while (occ0 > 0) { - if( !(*sp->decodepfunc)(tif, op0, rowsize) ) -@@ -512,12 +539,17 @@ horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc) - tmsize_t stride = sp->stride; - unsigned char* cp = (unsigned char*) cp0; + assert(sp->decodepfunc != NULL); + while (occ0 > 0) + { +@@ -617,11 +642,16 @@ static int horDiff8(TIFF *tif, uint8_t *cp0, tmsize_t cc) + tmsize_t stride = sp->stride; + unsigned char *cp = (unsigned char *)cp0; +#ifdef MAGMA_ENABLE_FIXES - if((cc%stride)!=0) + if ((cc % stride) != 0) { - TIFFErrorExt(tif->tif_clientdata, "horDiff8", - "%s", "(cc%stride)!=0"); + TIFFErrorExtR(tif, "horDiff8", "%s", "(cc%stride)!=0"); return 0; } +#endif @@ -110,17 +103,16 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%stride)!=0); +#endif - if (cc > stride) { - cc -= stride; -@@ -567,12 +599,17 @@ horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc) - uint16 *wp = (uint16*) cp0; - tmsize_t wc = cc/2; + if (cc > stride) + { +@@ -696,11 +726,16 @@ static int horDiff16(TIFF *tif, uint8_t *cp0, tmsize_t cc) + uint16_t *wp = (uint16_t *)cp0; + tmsize_t wc = cc / 2; +#ifdef MAGMA_ENABLE_FIXES - if((cc%(2*stride))!=0) + if ((cc % (2 * stride)) != 0) { - TIFFErrorExt(tif->tif_clientdata, "horDiff8", - "%s", "(cc%(2*stride))!=0"); + TIFFErrorExtR(tif, "horDiff8", "%s", "(cc%(2*stride))!=0"); return 0; } +#endif @@ -128,17 +120,16 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%(2*stride))!=0); +#endif - if (wc > stride) { - wc -= stride; -@@ -607,12 +644,17 @@ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc) - uint32 *wp = (uint32*) cp0; - tmsize_t wc = cc/4; + if (wc > stride) + { +@@ -738,11 +773,16 @@ static int horDiff32(TIFF *tif, uint8_t *cp0, tmsize_t cc) + uint32_t *wp = (uint32_t *)cp0; + tmsize_t wc = cc / 4; +#ifdef MAGMA_ENABLE_FIXES - if((cc%(4*stride))!=0) + if ((cc % (4 * stride)) != 0) { - TIFFErrorExt(tif->tif_clientdata, "horDiff32", - "%s", "(cc%(4*stride))!=0"); + TIFFErrorExtR(tif, "horDiff32", "%s", "(cc%(4*stride))!=0"); return 0; } +#endif @@ -146,17 +137,16 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%(4*stride))!=0); +#endif - if (wc > stride) { - wc -= stride; -@@ -652,12 +694,17 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc) - uint8 *cp = (uint8 *) cp0; - uint8 *tmp; + if (wc > stride) + { +@@ -821,11 +861,16 @@ static int fpDiff(TIFF *tif, uint8_t *cp0, tmsize_t cc) + uint8_t *cp = (uint8_t *)cp0; + uint8_t *tmp; +#ifdef MAGMA_ENABLE_FIXES - if((cc%(bps*stride))!=0) + if ((cc % (bps * stride)) != 0) { - TIFFErrorExt(tif->tif_clientdata, "fpDiff", - "%s", "(cc%(bps*stride))!=0"); + TIFFErrorExtR(tif, "fpDiff", "%s", "(cc%(bps*stride))!=0"); return 0; } +#endif @@ -164,19 +154,18 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc%(bps*stride))!=0); +#endif - tmp = (uint8 *)_TIFFmalloc(cc); - if (!tmp) -@@ -730,6 +777,8 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s) + tmp = (uint8_t *)_TIFFmallocExt(tif, cc); + if (!tmp) +@@ -900,12 +945,19 @@ static int PredictorEncodeTile(TIFF *tif, uint8_t *bp0, tmsize_t cc0, - rowsize = sp->rowsize; - assert(rowsize > 0); + rowsize = sp->rowsize; + assert(rowsize > 0); + +#ifdef MAGMA_ENABLE_FIXES - if((cc0%rowsize)!=0) + if ((cc0 % rowsize) != 0) { - TIFFErrorExt(tif->tif_clientdata, "PredictorEncodeTile", -@@ -737,6 +786,11 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s) - _TIFFfree( working_copy ); + TIFFErrorExtR(tif, "PredictorEncodeTile", "%s", "(cc0%rowsize)!=0"); + _TIFFfreeExt(tif, working_copy); return 0; } +#endif @@ -184,6 +173,6 @@ index b775663..98d7c6b 100644 + MAGMA_LOG("%MAGMA_BUG%", (cc0%rowsize)!=0); +#endif + - while (cc > 0) { - (*sp->encodepfunc)(tif, bp, rowsize); - cc -= rowsize; + while (cc > 0) + { + (*sp->encodepfunc)(tif, bp, rowsize); diff --git a/targets/libtiff/patches/bugs/TIF002.patch b/targets/libtiff/patches/bugs/TIF002.patch index 444e1ddc4..cec6c70f5 100644 --- a/targets/libtiff/patches/bugs/TIF002.patch +++ b/targets/libtiff/patches/bugs/TIF002.patch @@ -1,22 +1,23 @@ diff --git a/libtiff/tif_pixarlog.c b/libtiff/tif_pixarlog.c -index b1e48d9..5901551 100644 +index 56cf416a..e65291b7 100644 --- a/libtiff/tif_pixarlog.c +++ b/libtiff/tif_pixarlog.c -@@ -804,12 +804,17 @@ PixarLogDecode(TIFF* tif, uint8* op, tmsize_t occ, uint16 s) - TIFFErrorExt(tif->tif_clientdata, module, "ZLib cannot deal with buffers this size"); - return (0); - } +@@ -883,6 +883,7 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) + memset(op, 0, (size_t)occ); + return (0); + } +#ifdef MAGMA_ENABLE_FIXES - /* Check that we will not fill more than what was allocated */ - if ((tmsize_t)sp->stream.avail_out > sp->tbuf_size) - { - TIFFErrorExt(tif->tif_clientdata, module, "sp->stream.avail_out > sp->tbuf_size"); - return (0); - } + /* Check that we will not fill more than what was allocated */ + if ((tmsize_t)sp->stream.avail_out > sp->tbuf_size) + { +@@ -890,6 +891,10 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) + memset(op, 0, (size_t)occ); + return (0); + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", (tmsize_t)sp->stream.avail_out > sp->tbuf_size); +#endif - do { - int state = inflate(&sp->stream, Z_PARTIAL_FLUSH); - if (state == Z_STREAM_END) { + do + { + int state = inflate(&sp->stream, Z_PARTIAL_FLUSH); diff --git a/targets/libtiff/patches/bugs/TIF003.patch b/targets/libtiff/patches/bugs/TIF003.patch index 43e4eefd5..57f5ab012 100644 --- a/targets/libtiff/patches/bugs/TIF003.patch +++ b/targets/libtiff/patches/bugs/TIF003.patch @@ -1,19 +1,25 @@ diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c -index 1f5362a..0e8a1c6 100644 +index 964f119a..6d9a3030 100644 --- a/libtiff/tif_read.c +++ b/libtiff/tif_read.c -@@ -492,7 +492,14 @@ static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16 - rowsperstrip=td->td_rowsperstrip; - if (rowsperstrip>td->td_imagelength) - rowsperstrip=td->td_imagelength; +@@ -501,6 +501,7 @@ static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF *tif, uint32_t strip, + rowsperstrip = td->td_rowsperstrip; + if (rowsperstrip > td->td_imagelength) + rowsperstrip = td->td_imagelength; +#ifdef MAGMA_ENABLE_FIXES - stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip); + if (rowsperstrip == 0) + { + TIFFErrorExtR(tif, module, "rowsperstrip is zero"); +@@ -508,6 +509,12 @@ static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF *tif, uint32_t strip, + } + stripsperplane = + TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip); +#else + stripsperplane=((td->td_imagelength+rowsperstrip-1)/rowsperstrip); +#endif +#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", stripsperplane == 0); ++ MAGMA_LOG("%MAGMA_BUG%", rowsperstrip == 0); +#endif - stripinplane=(strip%stripsperplane); - if( pplane ) *pplane=(uint16)(strip/stripsperplane); - rows=td->td_imagelength-stripinplane*rowsperstrip; + stripinplane = (strip % stripsperplane); + if (pplane) + *pplane = (uint16_t)(strip / stripsperplane); diff --git a/targets/libtiff/patches/bugs/TIF004.patch b/targets/libtiff/patches/bugs/TIF004.patch index ed7c1355c..57fbe6a06 100644 --- a/targets/libtiff/patches/bugs/TIF004.patch +++ b/targets/libtiff/patches/bugs/TIF004.patch @@ -1,25 +1,37 @@ diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c -index ad3e1e7..5abc4a9 100644 +index f94d2a4e..3898558d 100644 --- a/libtiff/tif_ojpeg.c +++ b/libtiff/tif_ojpeg.c -@@ -789,15 +789,20 @@ OJPEGDecode(TIFF* tif, uint8* buf, tmsize_t cc, uint16 s) - static const char module[]="OJPEGDecode"; - OJPEGState* sp=(OJPEGState*)tif->tif_data; - (void)s; +@@ -841,6 +841,7 @@ static int OJPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) + static const char module[] = "OJPEGDecode"; + OJPEGState *sp = (OJPEGState *)tif->tif_data; + (void)s; +#ifdef MAGMA_ENABLE_FIXES - if( !sp->decoder_ok ) - { - TIFFErrorExt(tif->tif_clientdata,module,"Cannot decode: decoder not correctly initialized"); - return 0; - } - if( sp->error_in_raw_data_decoding ) - { - return 0; - } + if (!sp->decoder_ok) + { + memset(buf, 0, (size_t)cc); +@@ -848,6 +849,7 @@ static int OJPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) + "Cannot decode: decoder not correctly initialized"); + return 0; + } ++#endif + if (sp->libjpeg_session_active == 0) + { + memset(buf, 0, (size_t)cc); +@@ -862,11 +864,16 @@ static int OJPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) + "Cannot decode: libjpeg_session_active == 0"); + return 0; + } ++#ifdef MAGMA_ENABLE_FIXES + if (sp->error_in_raw_data_decoding) + { + memset(buf, 0, (size_t)cc); + return 0; + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", sp->decoder_ok == 0); +#endif - if (sp->libjpeg_jpeg_query_style==0) - { - if (OJPEGDecodeRaw(tif,buf,cc)==0) + if (sp->libjpeg_jpeg_query_style == 0) + { + if (OJPEGDecodeRaw(tif, buf, cc) == 0) diff --git a/targets/libtiff/patches/bugs/TIF005.patch b/targets/libtiff/patches/bugs/TIF005.patch index a65840568..6679b3d38 100644 --- a/targets/libtiff/patches/bugs/TIF005.patch +++ b/targets/libtiff/patches/bugs/TIF005.patch @@ -1,24 +1,24 @@ diff --git a/libtiff/tif_luv.c b/libtiff/tif_luv.c -index 6a63ead..780107e 100644 +index 021756d5..6eb43723 100644 --- a/libtiff/tif_luv.c +++ b/libtiff/tif_luv.c -@@ -1587,7 +1587,9 @@ LogLuvClose(TIFF* tif) - * before they have been recorded in the file, we reset them here. - * Note: this is really a nasty approach. See PixarLogClose - */ +@@ -1663,7 +1663,9 @@ static void LogLuvClose(TIFF *tif) + * before they have been recorded in the file, we reset them here. + * Note: this is really a nasty approach. See PixarLogClose + */ +#ifdef MAGMA_ENABLE_FIXES - if( sp->encoder_state ) + if (sp->encoder_state) +#endif - { - /* See PixarLogClose. Might avoid issues with tags whose size depends - * on those below, but not completely sure this is enough. */ -@@ -1596,6 +1598,9 @@ LogLuvClose(TIFF* tif) - td->td_bitspersample = 16; - td->td_sampleformat = SAMPLEFORMAT_INT; - } + { + /* See PixarLogClose. Might avoid issues with tags whose size depends + * on those below, but not completely sure this is enough. */ +@@ -1672,6 +1674,9 @@ static void LogLuvClose(TIFF *tif) + td->td_bitspersample = 16; + td->td_sampleformat = SAMPLEFORMAT_INT; + } +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", sp->encoder_state == 0); +#endif } - static void + static void LogLuvCleanup(TIFF *tif) diff --git a/targets/libtiff/patches/bugs/TIF006.patch b/targets/libtiff/patches/bugs/TIF006.patch index 224cfc6c2..893061f6f 100644 --- a/targets/libtiff/patches/bugs/TIF006.patch +++ b/targets/libtiff/patches/bugs/TIF006.patch @@ -1,27 +1,26 @@ diff --git a/libtiff/tif_pixarlog.c b/libtiff/tif_pixarlog.c -index b1e48d9..1c14581 100644 +index 5ed921eb..956514e4 100644 --- a/libtiff/tif_pixarlog.c +++ b/libtiff/tif_pixarlog.c -@@ -1265,8 +1265,10 @@ PixarLogClose(TIFF* tif) - * readers that don't know about PixarLog, or how to set - * the PIXARLOGDATFMT pseudo-tag. - */ +@@ -1449,8 +1449,9 @@ static void PixarLogClose(TIFF *tif) + * readers that don't know about PixarLog, or how to set + * the PIXARLOGDATFMT pseudo-tag. + */ - -- if (sp->state&PLSTATE_INIT) { +- if (sp->state & PLSTATE_INIT) +#ifdef MAGMA_ENABLE_FIXES + if (sp->state&PLSTATE_INIT) +#endif -+ { - /* We test the state to avoid an issue such as in - * http://bugzilla.maptools.org/show_bug.cgi?id=2604 - * What appends in that case is that the bitspersample is 1 and -@@ -1278,6 +1280,9 @@ PixarLogClose(TIFF* tif) - td->td_bitspersample = 8; - td->td_sampleformat = SAMPLEFORMAT_UINT; - } + { + /* We test the state to avoid an issue such as in + * http://bugzilla.maptools.org/show_bug.cgi?id=2604 +@@ -1463,6 +1464,9 @@ static void PixarLogClose(TIFF *tif) + td->td_bitspersample = 8; + td->td_sampleformat = SAMPLEFORMAT_UINT; + } +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", (sp->state&PLSTATE_INIT) == 0); +#endif } - static void + static void PixarLogCleanup(TIFF *tif) diff --git a/targets/libtiff/patches/bugs/TIF007.patch b/targets/libtiff/patches/bugs/TIF007.patch index bab902d18..37f443a11 100644 --- a/targets/libtiff/patches/bugs/TIF007.patch +++ b/targets/libtiff/patches/bugs/TIF007.patch @@ -1,8 +1,8 @@ diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c -index 1b300b0..b04612d 100644 +index cea569d5..f733bc07 100644 --- a/libtiff/tif_dirread.c +++ b/libtiff/tif_dirread.c -@@ -5773,7 +5773,11 @@ static void allocChoppedUpStripArrays(TIFF* tif, uint32 nstrips, +@@ -7373,7 +7373,11 @@ static void allocChoppedUpStripArrays(TIFF *tif, uint32_t nstrips, if (stripbytes > bytecount) stripbytes = bytecount; newcounts[i] = stripbytes; @@ -14,15 +14,14 @@ index 1b300b0..b04612d 100644 offset += stripbytes; bytecount -= stripbytes; } -@@ -5848,9 +5852,18 @@ ChopUpSingleUncompressedStrip(TIFF* tif) - */ - if (rowsperstrip >= td->td_rowsperstrip) - return; +@@ -7448,9 +7452,19 @@ static void ChopUpSingleUncompressedStrip(TIFF *tif) + */ + if (rowsperstrip >= td->td_rowsperstrip) + return; +#ifdef MAGMA_ENABLE_FIXES - nstrips = TIFFhowmany_32(td->td_imagelength, rowsperstrip); - if( nstrips == 0 ) - return; -- + nstrips = TIFFhowmany_32(td->td_imagelength, rowsperstrip); + if (nstrips == 0) + return; +#else + uint64_t nstrips64 = TIFFhowmany_64(bytecount, stripbytes); + if ((nstrips64==0)||(nstrips64>0xFFFFFFFF)) /* something is wonky, do nothing. */ @@ -32,5 +31,6 @@ index 1b300b0..b04612d 100644 +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", nstrips > TIFFhowmany_32(td->td_imagelength, rowsperstrip)); +#endif - /* If we are going to allocate a lot of memory, make sure that the */ - /* file is as big as needed */ + + /* If we are going to allocate a lot of memory, make sure that the */ + /* file is as big as needed */ diff --git a/targets/libtiff/patches/bugs/TIF008.patch b/targets/libtiff/patches/bugs/TIF008.patch index 9a0ca8ba1..1e478804c 100644 --- a/targets/libtiff/patches/bugs/TIF008.patch +++ b/targets/libtiff/patches/bugs/TIF008.patch @@ -1,31 +1,34 @@ diff --git a/libtiff/tif_next.c b/libtiff/tif_next.c -index 0ba61ae..ba8b3cc 100644 +index f000574e..34cf0782 100644 --- a/libtiff/tif_next.c +++ b/libtiff/tif_next.c -@@ -122,15 +122,26 @@ NeXTDecode(TIFF* tif, uint8* buf, tmsize_t occ, uint16 s) - * bounds, potentially resulting in a security - * issue. - */ +@@ -137,11 +137,21 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) + * bounds, potentially resulting in a security + * issue. + */ +#ifdef MAGMA_ENABLE_FIXES - while (n-- > 0 && npixels < imagewidth && op_offset < scanline) + while (n-- > 0 && npixels < imagewidth && + op_offset < scanline) +#else -+ while (n-- > 0 && npixels < imagewidth) ++ while (n-- > 0 && npixels < imagewidth) +#endif -+ { ++ { +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", op_offset >= scanline); +#endif - SETPIXEL(op, grey); -+ } - if (npixels >= imagewidth) - break; + SETPIXEL(op, grey); ++ } + if (npixels >= imagewidth) + break; +#ifdef MAGMA_ENABLE_FIXES - if (op_offset >= scanline ) { - TIFFErrorExt(tif->tif_clientdata, module, "Invalid data for scanline %"PRIu32, - tif->tif_row); - return (0); - } + if (op_offset >= scanline) + { + TIFFErrorExtR(tif, module, +@@ -149,6 +159,7 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) + tif->tif_row); + return (0); + } +#endif - if (cc == 0) - goto bad; - n = *bp++; + if (cc == 0) + goto bad; + n = *bp++; diff --git a/targets/libtiff/patches/bugs/TIF009.patch b/targets/libtiff/patches/bugs/TIF009.patch index f7c0e31e1..58e4a2b08 100644 --- a/targets/libtiff/patches/bugs/TIF009.patch +++ b/targets/libtiff/patches/bugs/TIF009.patch @@ -1,33 +1,50 @@ diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c -index 9e4d306..98d6e1a 100644 +index 0187e76f..ab075030 100644 --- a/libtiff/tif_dirwrite.c +++ b/libtiff/tif_dirwrite.c -@@ -2026,14 +2026,28 @@ TIFFWriteDirectoryTagTransferfunction(TIFF* tif, uint32* ndir, TIFFDirEntry* dir - n=3; - if (n==3) - { -+#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", tif->tif_dir.td_transferfunction[2] == NULL); -+#endif +@@ -2137,6 +2137,7 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, + /* clang-format on */ + + /* Check for proper number of transferfunctions */ +#ifdef MAGMA_ENABLE_FIXES - if (tif->tif_dir.td_transferfunction[2] == NULL || - !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[2],m*sizeof(uint16_t))) + for (int i = 0; i < n; i++) + { + if (tif->tif_dir.td_transferfunction[i] == NULL) +@@ -2147,6 +2148,29 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, + return (1); /* Not an error; only tag is not written. */ + } + } +#else -+ if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[2],m*sizeof(uint16_t))) ++ if (n == 3) ++ { ++#ifdef MAGMA_ENABLE_CANARIES ++ MAGMA_LOG("%MAGMA_BUG%", tif->tif_dir.td_transferfunction[2] == NULL); +#endif - n=2; - } - if (n==2) - { ++ if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0], ++ tif->tif_dir.td_transferfunction[2], ++ m*sizeof(uint16_t))) ++ n=2; ++ } ++ if (n == 2) ++ { +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", tif->tif_dir.td_transferfunction[1] == NULL); +#endif ++ if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0], ++ tif->tif_dir.td_transferfunction[1], ++ m*sizeof(uint16_t))) ++ n=1; ++ } ++#endif +#ifdef MAGMA_ENABLE_FIXES - if (tif->tif_dir.td_transferfunction[1] == NULL || - !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[1],m*sizeof(uint16_t))) -+#else -+ if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[1],m*sizeof(uint16_t))) + /* + * Check if the table can be written as a single column, + * or if it must be written as 3 columns. Note that we +@@ -2163,6 +2187,7 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, + m * sizeof(uint16_t))) + n = 1; + } +#endif - n=1; - } - if (n==0) + if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ + { + EvaluateIFDdatasizeWrite(tif, n * m, 2, ndir); diff --git a/targets/libtiff/patches/bugs/TIF010.patch b/targets/libtiff/patches/bugs/TIF010.patch index f1f81e684..181ccdfb7 100644 --- a/targets/libtiff/patches/bugs/TIF010.patch +++ b/targets/libtiff/patches/bugs/TIF010.patch @@ -1,21 +1,21 @@ diff --git a/libtiff/tif_lzw.c b/libtiff/tif_lzw.c -index 21064f2..fb63a80 100644 +index 05ac8aa1..b9b0e9b9 100644 --- a/libtiff/tif_lzw.c +++ b/libtiff/tif_lzw.c -@@ -764,9 +764,16 @@ LZWDecodeCompat(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s) - len = codep->length; - tp = op + len; - do { +@@ -973,9 +973,16 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) + tp = op + len; + do + { +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", tp <= op); // or use == instead of <= so the bug is reported once only +#endif - *--tp = codep->value; - codep = codep->next; + *--tp = codep->value; + codep = codep->next; +#ifdef MAGMA_ENABLE_FIXES - } while (codep && tp > op); + } while (codep && tp > op); +#else + } while (codep); +#endif - assert(occ >= len); - op += len; - occ -= len; + assert(occ >= len); + op += len; + occ -= len; diff --git a/targets/libtiff/patches/bugs/TIF011.patch b/targets/libtiff/patches/bugs/TIF011.patch index 963f10502..ae751b772 100644 --- a/targets/libtiff/patches/bugs/TIF011.patch +++ b/targets/libtiff/patches/bugs/TIF011.patch @@ -1,13 +1,15 @@ diff --git a/libtiff/tif_print.c b/libtiff/tif_print.c -index a073794..0a0e691 100644 +index 2b7fd176..3ceb7836 100644 --- a/libtiff/tif_print.c +++ b/libtiff/tif_print.c -@@ -544,9 +544,18 @@ TIFFPrintDirectory(TIFF* tif, FILE* fd, long flags) - uint16 i; - fprintf(fd, " %2ld: %5"PRIu16, - l, td->td_transferfunction[0][l]); +@@ -584,10 +584,19 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) + uint16_t i; + fprintf(fd, " %2ld: %5" PRIu16, l, + td->td_transferfunction[0][l]); +#ifdef MAGMA_ENABLE_FIXES - for (i = 1; i < td->td_samplesperpixel - td->td_extrasamples && i < 3; i++) + for (i = 1; + i < td->td_samplesperpixel - td->td_extrasamples && i < 3; + i++) +#else + for (i = 1; i < td->td_samplesperpixel; i++) +#endif @@ -15,9 +17,8 @@ index a073794..0a0e691 100644 +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(i == sizeof(td->td_transferfunction) / sizeof(*td->td_transferfunction), i == (td->td_samplesperpixel - td->td_extrasamples))); +#endif - fprintf(fd, " %5"PRIu16, - td->td_transferfunction[i][l]); + fprintf(fd, " %5" PRIu16, td->td_transferfunction[i][l]); + } - fputc('\n', fd); - } - } else + fputc('\n', fd); + } + } diff --git a/targets/libtiff/patches/bugs/TIF012.patch b/targets/libtiff/patches/bugs/TIF012.patch index 0c3b807fb..dca83cbbe 100644 --- a/targets/libtiff/patches/bugs/TIF012.patch +++ b/targets/libtiff/patches/bugs/TIF012.patch @@ -1,38 +1,41 @@ +diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c +index 47ad21e0..06689f24 100644 --- a/libtiff/tif_dir.c +++ b/libtiff/tif_dir.c -@@ -117,7 +117,7 @@ setExtraSamples(TIFF* tif, va_list ap, uint32* v) - return 0; - } - } +@@ -164,7 +164,7 @@ static int setExtraSamples(TIFF *tif, va_list ap, uint32_t *v) + return 0; + } + } - +#ifdef MAGMA_ENABLE_FIXES - if ( td->td_transferfunction[0] != NULL && (td->td_samplesperpixel - *v > 1) && - !(td->td_samplesperpixel - td->td_extrasamples > 1)) - { -@@ -128,6 +128,11 @@ setExtraSamples(TIFF* tif, va_list ap, uint32* v) - _TIFFfree(td->td_transferfunction[0]); - td->td_transferfunction[0] = NULL; - } + if (td->td_transferfunction[0] != NULL && + (td->td_samplesperpixel - *v > 1) && + !(td->td_samplesperpixel - td->td_extrasamples > 1)) +@@ -177,6 +177,11 @@ static int setExtraSamples(TIFF *tif, va_list ap, uint32_t *v) + _TIFFfreeExt(tif, td->td_transferfunction[0]); + td->td_transferfunction[0] = NULL; + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_AND(td->td_transferfunction[0] != NULL, MAGMA_AND(td->td_samplesperpixel - *v > 1, \ + !(td->td_samplesperpixel - td->td_extrasamples > 1)))); +#endif - td->td_extrasamples = (uint16) *v; - _TIFFsetShortArray(&td->td_sampleinfo, va, td->td_extrasamples); -@@ -270,6 +275,7 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap) - goto badvalue; - if( v != td->td_samplesperpixel ) - { -+#ifdef MAGMA_ENABLE_FIXES - /* See http://bugzilla.maptools.org/show_bug.cgi?id=2500 */ - if( td->td_sminsamplevalue != NULL ) + td->td_extrasamples = (uint16_t)*v; + _TIFFsetShortArrayExt(tif, &td->td_sampleinfo, va, td->td_extrasamples); +@@ -324,6 +329,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) + goto badvalue; + if (v != td->td_samplesperpixel) { -@@ -301,6 +307,13 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap) - _TIFFfree(td->td_transferfunction[0]); ++#ifdef MAGMA_ENABLE_FIXES + /* See http://bugzilla.maptools.org/show_bug.cgi?id=2500 */ + if (td->td_sminsamplevalue != NULL) + { +@@ -359,6 +365,13 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) + TIFFClrFieldBit(tif, FIELD_TRANSFERFUNCTION); + _TIFFfreeExt(tif, td->td_transferfunction[0]); td->td_transferfunction[0] = NULL; - } + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(td->td_sminsamplevalue != NULL, \ @@ -40,6 +43,5 @@ + MAGMA_AND(td->td_transferfunction[0] != NULL, MAGMA_AND(v - td->td_extrasamples > 1, \ + !(td->td_samplesperpixel - td->td_extrasamples > 1)))))); +#endif - } - td->td_samplesperpixel = (uint16) v; - break; + } + td->td_samplesperpixel = (uint16_t)v; diff --git a/targets/libtiff/patches/bugs/TIF013.patch b/targets/libtiff/patches/bugs/TIF013.patch index 19c575c1c..e4f9a13e5 100644 --- a/targets/libtiff/patches/bugs/TIF013.patch +++ b/targets/libtiff/patches/bugs/TIF013.patch @@ -1,26 +1,27 @@ diff --git a/libtiff/tif_jbig.c b/libtiff/tif_jbig.c -index 7ffe885..a2bc7bf 100644 +index 7e455ad1..78d4c384 100644 --- a/libtiff/tif_jbig.c +++ b/libtiff/tif_jbig.c -@@ -106,12 +106,21 @@ static int JBIGDecode(TIFF* tif, uint8* buffer, tmsize_t size, uint16 s) - } - else if( (tmsize_t)decodedSize > size ) - { +@@ -106,13 +106,22 @@ static int JBIGDecode(TIFF *tif, uint8_t *buffer, tmsize_t size, uint16_t s) + } + else if ((tmsize_t)decodedSize > size) + { +#ifdef MAGMA_ENABLE_FIXES - TIFFErrorExt(tif->tif_clientdata, "JBIG", - "Decoded %lu bytes, whereas %"TIFF_SSIZE_FORMAT" were requested", - decodedSize, size); - jbg_dec_free(&decoder); - return 0; + TIFFErrorExtR(tif, "JBIG", + "Decoded %lu bytes, whereas %" TIFF_SSIZE_FORMAT + " were requested", + decodedSize, size); + jbg_dec_free(&decoder); + return 0; +#else + TIFFWarningExt(tif->tif_clientdata, "JBIG", + "Decoded %lu bytes, whereas %lu were requested", + decodedSize, (unsigned long)size); +#endif - } + } +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", (tmsize_t)decodedSize > size); +#endif - pImage = jbg_dec_getimage(&decoder, 0); - _TIFFmemcpy(buffer, pImage, decodedSize); - jbg_dec_free(&decoder); + pImage = jbg_dec_getimage(&decoder, 0); + _TIFFmemcpy(buffer, pImage, decodedSize); + jbg_dec_free(&decoder); diff --git a/targets/libtiff/patches/bugs/TIF014.patch b/targets/libtiff/patches/bugs/TIF014.patch index 5e84cbc57..3b009802a 100644 --- a/targets/libtiff/patches/bugs/TIF014.patch +++ b/targets/libtiff/patches/bugs/TIF014.patch @@ -1,16 +1,18 @@ diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c -index 1b300b0..47d6992 100644 +index cea569d5..0c973c32 100644 --- a/libtiff/tif_dirread.c +++ b/libtiff/tif_dirread.c -@@ -5852,3 +5852,4 @@ ChopUpSingleUncompressedStrip(TIFF* tif) - /* If we are going to allocate a lot of memory, make sure that the */ - /* file is as big as needed */ +@@ -7454,12 +7454,20 @@ static void ChopUpSingleUncompressedStrip(TIFF *tif) + + /* If we are going to allocate a lot of memory, make sure that the */ + /* file is as big as needed */ +#ifdef MAGMA_ENABLE_FIXES - if( tif->tif_mode == O_RDONLY && -@@ -5861,6 +5862,13 @@ ChopUpSingleUncompressedStrip(TIFF* tif) - { - return; - } + if (tif->tif_mode == O_RDONLY && nstrips > 1000000 && + (offset >= TIFFGetFileSize(tif) || + stripbytes > (TIFFGetFileSize(tif) - offset) / (nstrips - 1))) + { + return; + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_AND(tif->tif_mode == O_RDONLY, \ @@ -19,5 +21,5 @@ index 1b300b0..47d6992 100644 + stripbytes * (nstrips - 1) > (TIFFGetFileSize(tif) - offset))))); +#endif - allocChoppedUpStripArrays(tif, nstrips, stripbytes, rowsperstrip); + allocChoppedUpStripArrays(tif, nstrips, stripbytes, rowsperstrip); } diff --git a/targets/libtiff/patches/setup/fix_autogen.patch b/targets/libtiff/patches/setup/fix_autogen.patch new file mode 100644 index 000000000..82a72fddf --- /dev/null +++ b/targets/libtiff/patches/setup/fix_autogen.patch @@ -0,0 +1,39 @@ +diff --git a/autogen.sh b/autogen.sh +index 2e6c6bd0..32ab8ace 100755 +--- a/autogen.sh ++++ b/autogen.sh +@@ -4,27 +4,19 @@ retval=0 + + set -x + +-case `uname` in +- Darwin*) +- glibtoolize --force --copy || retval=$? +- ;; +- *) +- libtoolize --force --copy || retval=$? +- ;; +-esac +-aclocal -I ./m4 || retval=$? +-autoheader || retval=$? +-automake --foreign --add-missing --copy || retval=$? +-autoconf || retval=$? ++autoreconf --install --force || retval=$? ++ + # Get latest config.guess and config.sub from upstream master since +-# these are often out of date. ++# these are often out of date. This requires network connectivity and ++# sometimes the site is down, a failure here does not result in ++# failure of the whole script. + for file in config.guess config.sub + do + echo "$0: getting $file..." +- wget -q --timeout=5 -O config/$file.tmp \ ++ wget --timeout=5 -O config/$file.tmp \ + "https://git.savannah.gnu.org/cgit/config.git/plain/${file}" \ + && mv -f config/$file.tmp config/$file \ +- && chmod a+x config/$file || retval=$? ++ && chmod a+x config/$file + rm -f config/$file.tmp + done + diff --git a/targets/libtiff/patches/setup/libtiff.patch b/targets/libtiff/patches/setup/libtiff.patch index 246a3852e..c620f50be 100644 --- a/targets/libtiff/patches/setup/libtiff.patch +++ b/targets/libtiff/patches/setup/libtiff.patch @@ -1,11 +1,15 @@ --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -702,8 +702,6 @@ - add_subdirectory(port) - add_subdirectory(libtiff) - add_subdirectory(tools) --add_subdirectory(test) --add_subdirectory(contrib) +@@ -151,12 +151,6 @@ + if(tiff-tools) + add_subdirectory(tools) + endif() +-if(tiff-tests) +- add_subdirectory(test) +-endif() +-if(tiff-contrib) +- add_subdirectory(contrib) +-endif() add_subdirectory(build) - add_subdirectory(man) - add_subdirectory(html) + if(tiff-docs) + add_subdirectory(doc) diff --git a/targets/libtiff/pocs/TIF005.crash b/targets/libtiff/pocs/TIF005.crash new file mode 100644 index 000000000..f65bcf901 Binary files /dev/null and b/targets/libtiff/pocs/TIF005.crash differ diff --git a/targets/libtiff/pocs/TIF007.crash b/targets/libtiff/pocs/TIF007.crash new file mode 100644 index 000000000..c9a2bc295 Binary files /dev/null and b/targets/libtiff/pocs/TIF007.crash differ diff --git a/targets/libtiff/pocs/TIF008.crash b/targets/libtiff/pocs/TIF008.crash new file mode 100644 index 000000000..6f1f6960d Binary files /dev/null and b/targets/libtiff/pocs/TIF008.crash differ diff --git a/targets/libtiff/pocs/TIF009.crash b/targets/libtiff/pocs/TIF009.crash new file mode 100644 index 000000000..5e9427853 Binary files /dev/null and b/targets/libtiff/pocs/TIF009.crash differ diff --git a/targets/libtiff/releases b/targets/libtiff/releases new file mode 100644 index 000000000..5fdef5414 --- /dev/null +++ b/targets/libtiff/releases @@ -0,0 +1,8 @@ +libtiff_PIONEER="https://gitlab.com/libtiff/libtiff.git" +libtiff_PIONEER_STABLE_COMMIT="7eace5c973d25be3da3d548962aa648d5795183d" +libtiff_LEGACY_2022="https://gitlab.com/libtiff/libtiff.git" +libtiff_LEGACY_2022_TAG="v3.4beta018" +libtiff_LEGACY_2023="https://gitlab.com/libtiff/libtiff.git" +libtiff_LEGACY_2023_TAG="v4.5.1" +libtiff_LEGACY_2024="https://gitlab.com/libtiff/libtiff.git" +libtiff_LEGACY_2024_TAG="v4.7.0" diff --git a/targets/libxml2/build.sh b/targets/libxml2/build.sh index fb216a882..836529e0c 100755 --- a/targets/libxml2/build.sh +++ b/targets/libxml2/build.sh @@ -9,7 +9,7 @@ set -e ## if [ ! -d "$TARGET/repo" ]; then - echo "fetch.sh must be executed first." + echo "fetch_target.sh must be executed first." exit 1 fi @@ -28,5 +28,5 @@ cp xmllint "$OUT/" for fuzzer in libxml2_xml_read_memory_fuzzer libxml2_xml_reader_for_file_fuzzer; do $CXX $CXXFLAGS -std=c++11 -Iinclude/ -I"$TARGET/src/" \ "$TARGET/src/$fuzzer.cc" -o "$OUT/$fuzzer" \ - .libs/libxml2.a $LDFLAGS $LIBS -lz -llzma + .libs/libxml2.a $LDFLAGS $LIBS $LIB_FUZZING_ENGINE -lz -llzma -include limits done diff --git a/targets/libxml2/build_poc.sh b/targets/libxml2/build_poc.sh new file mode 100755 index 000000000..ee88d49c0 --- /dev/null +++ b/targets/libxml2/build_poc.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch_target.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" +./autogen.sh \ + --with-http=no \ + --with-python=no \ + --with-lzma=yes \ + --with-threads=no \ + --disable-shared +make -j$(nproc) clean +make -j$(nproc) all +make install diff --git a/targets/libxml2/fetch.sh b/targets/libxml2/fetch.sh deleted file mode 100755 index cbe48b2c7..000000000 --- a/targets/libxml2/fetch.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -git clone --no-checkout https://gitlab.gnome.org/GNOME/libxml2.git \ - "$TARGET/repo" -git -C "$TARGET/repo" checkout ec6e3efb06d7b15cf5a2328fabd3845acea4c815 diff --git a/targets/libxml2/patches/bugs/XML007.patch b/targets/libxml2/patches/bugs/XML007.patch deleted file mode 100644 index d8712965d..000000000 --- a/targets/libxml2/patches/bugs/XML007.patch +++ /dev/null @@ -1,31 +0,0 @@ ---- a/HTMLparser.c -+++ b/HTMLparser.c -@@ -5392,6 +5392,9 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - avail = in->length - (in->cur - in->base); - else - avail = (ptrdiff_t)xmlBufUse(in->buf->buffer) - - (in->cur - in->base); -+#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", avail != (in->end - in->cur)); -+#endif - if ((avail == 0) && (terminate)) { - htmlAutoCloseOnEnd(ctxt); - if ((ctxt->nameNr == 0) && (ctxt->instate != XML_PARSER_EOF)) { ---- a/parser.c -+++ b/parser.c -@@ -12491,12 +12491,14 @@ xmlHaltParser(xmlParserCtxtPtr ctxt) { - ctxt->input->free((xmlChar *) ctxt->input->base); - ctxt->input->free = NULL; - } -+#ifdef MAGMA_ENABLE_FIXES - if (ctxt->input->buf != NULL) { - xmlFreeParserInputBuffer(ctxt->input->buf); - ctxt->input->buf = NULL; - } -- ctxt->input->cur = BAD_CAST""; - ctxt->input->length = 0; -+#endif -+ ctxt->input->cur = BAD_CAST""; - ctxt->input->base = ctxt->input->cur; - ctxt->input->end = ctxt->input->cur; - } diff --git a/targets/libxml2/patches/bugs/XML011.patch b/targets/libxml2/patches/bugs/XML011.patch index c53537ad8..e56b1edad 100644 --- a/targets/libxml2/patches/bugs/XML011.patch +++ b/targets/libxml2/patches/bugs/XML011.patch @@ -1,19 +1,19 @@ diff --git a/xmlstring.c b/xmlstring.c -index 8d2e06f..5715bb0 100644 +index 5f03e07a..899e96cc 100644 --- a/xmlstring.c +++ b/xmlstring.c -@@ -457,13 +457,21 @@ xmlStrncat(xmlChar *cur, const xmlChar *add, int len) { +@@ -465,13 +465,21 @@ xmlStrncat(xmlChar *cur, const xmlChar *add, int len) { return(xmlStrndup(add, len)); size = xmlStrlen(cur); +#ifdef MAGMA_ENABLE_FIXES - if (size < 0) + if ((size < 0) || (size > INT_MAX - len)) return(NULL); +#endif - ret = (xmlChar *) xmlRealloc(cur, (size + len + 1) * sizeof(xmlChar)); + ret = (xmlChar *) xmlRealloc(cur, (size_t) size + len + 1); if (ret == NULL) { - xmlErrMemory(NULL, NULL); - return(cur); + xmlFree(cur); + return(NULL); } +#ifdef MAGMA_ENABLE_CANARIES + // whenever size < 0, the previous reallocation results in allocating a @@ -21,10 +21,10 @@ index 8d2e06f..5715bb0 100644 + // elements into it, which will cause a heap buffer overflow. + MAGMA_LOG("%MAGMA_BUG%", size < 0); +#endif - memcpy(&ret[size], add, len * sizeof(xmlChar)); + memcpy(&ret[size], add, len); ret[size + len] = 0; return(ret); -@@ -488,8 +496,10 @@ xmlStrncatNew(const xmlChar *str1, const xmlChar *str2, int len) { +@@ -496,8 +504,10 @@ xmlStrncatNew(const xmlChar *str1, const xmlChar *str2, int len) { if (len < 0) { len = xmlStrlen(str2); @@ -33,25 +33,23 @@ index 8d2e06f..5715bb0 100644 return(NULL); +#endif } - if ((str2 == NULL) || (len == 0)) - return(xmlStrdup(str1)); -@@ -497,13 +507,19 @@ xmlStrncatNew(const xmlChar *str1, const xmlChar *str2, int len) { + if (str1 == NULL) return(xmlStrndup(str2, len)); +@@ -505,11 +515,17 @@ xmlStrncatNew(const xmlChar *str1, const xmlChar *str2, int len) { + return(xmlStrdup(str1)); size = xmlStrlen(str1); +#ifdef MAGMA_ENABLE_FIXES - if (size < 0) + if ((size < 0) || (size > INT_MAX - len)) return(NULL); +#endif - ret = (xmlChar *) xmlMalloc((size + len + 1) * sizeof(xmlChar)); - if (ret == NULL) { - xmlErrMemory(NULL, NULL); - return(xmlStrndup(str1, size)); - } + ret = (xmlChar *) xmlMalloc((size_t) size + len + 1); + if (ret == NULL) + return(NULL); +#ifdef MAGMA_ENABLE_CANARIES + // ditto ;) + MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(size < 0, len < 0)); +#endif - memcpy(ret, str1, size * sizeof(xmlChar)); - memcpy(&ret[size], str2, len * sizeof(xmlChar)); + memcpy(ret, str1, size); + memcpy(&ret[size], str2, len); ret[size + len] = 0; diff --git a/targets/libxml2/patches/bugs/XML012.patch b/targets/libxml2/patches/bugs/XML012.patch index 12fd8b056..9e1c642f6 100644 --- a/targets/libxml2/patches/bugs/XML012.patch +++ b/targets/libxml2/patches/bugs/XML012.patch @@ -1,51 +1,42 @@ diff --git a/parser.c b/parser.c -index 3f31329..55b4bb4 100644 +index ad035f3c..4a438f4b 100644 --- a/parser.c +++ b/parser.c -@@ -3349,6 +3349,7 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { - int c; - int count = 0; +@@ -3520,6 +3520,7 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { + XML_MAX_TEXT_LENGTH : + XML_MAX_NAME_LENGTH; size_t startPosition = 0; + const xmlChar *end; - #ifdef DEBUG - nbParseNCNameComplex++; -@@ -3359,6 +3360,7 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { + ret.name = NULL; + ret.hashValue = 0; +@@ -3528,6 +3529,7 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { + * Handler for more complex cases */ - GROW; startPosition = CUR_PTR - BASE_PTR; + end = ctxt->input->cur; - c = CUR_CHAR(l); + c = xmlCurrentChar(ctxt, &l); if ((c == ' ') || (c == '>') || (c == '/') || /* accelerators */ (!xmlIsNameStartChar(ctxt, c) || (c == ':'))) { -@@ -3380,6 +3382,7 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { - } - len += l; +@@ -3539,13 +3541,21 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { + if (len <= INT_MAX - l) + len += l; NEXTL(l); -+ end = ctxt->input->cur; - c = CUR_CHAR(l); - if (c == 0) { - count = 0; -@@ -3393,6 +3396,7 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { - if (ctxt->instate == XML_PARSER_EOF) - return(NULL); - ctxt->input->cur += l; -+ end = ctxt->input->cur; - c = CUR_CHAR(l); - } ++ end = ctxt->input->cur; + c = xmlCurrentChar(ctxt, &l); } -@@ -3401,7 +3405,14 @@ xmlParseNCNameComplex(xmlParserCtxtPtr ctxt) { + if (len > maxLength) { xmlFatalErr(ctxt, XML_ERR_NAME_TOO_LONG, "NCName"); - return(NULL); + return(ret); } +#ifdef MAGMA_ENABLE_FIXES - return(xmlDictLookup(ctxt->dict, (BASE_PTR + startPosition), len)); + ret = xmlDictLookupHashed(ctxt->dict, (BASE_PTR + startPosition), len); +#else +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", (end - len) != (BASE_PTR + startPosition)); +#endif -+ return(xmlDictLookup(ctxt->dict, (end - len), len)); ++ ret = xmlDictLookupHashed(ctxt->dict, (end - len), len); +#endif - } - - /** + if (ret.name == NULL) + xmlErrMemory(ctxt); + return(ret); diff --git a/targets/libxml2/patches/bugs/XML014.patch b/targets/libxml2/patches/bugs/XML014.patch deleted file mode 100644 index 750b2bd4e..000000000 --- a/targets/libxml2/patches/bugs/XML014.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/HTMLparser.c b/HTMLparser.c -index d6971b5..7c82c11 100644 ---- a/HTMLparser.c -+++ b/HTMLparser.c -@@ -2473,8 +2473,13 @@ htmlParseName(htmlParserCtxtPtr ctxt) { - (*in == ':') || (*in == '.')) - in++; - -+#ifdef MAGMA_ENABLE_FIXES - if (in == ctxt->input->end) - return(NULL); -+#endif -+#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", in >= ctxt->input->end); -+#endif - - if ((*in > 0) && (*in < 0x80)) { - count = in - ctxt->input->cur; diff --git a/targets/libxml2/patches/bugs/XML015.patch b/targets/libxml2/patches/bugs/XML015.patch index 38009d194..100cf2a1c 100644 --- a/targets/libxml2/patches/bugs/XML015.patch +++ b/targets/libxml2/patches/bugs/XML015.patch @@ -1,23 +1,39 @@ +# Codebase versions used for the following explaination: +# 1. Old code: htmlParseNameComplex() in HTMLparser.c (commit: ec6e3efb06d7b15cf5a2328fabd3845acea4c815) +# 2. New code: htmlParseData() in HTMLparser.c (commit: 41c10c0cec2ac072897c7d8df0e87fdf4b715586) + +# In the old code's name-parser, hitting a non-ASCII character could trigger an encoding switch +# and buffer reallocation. The if (base != old_base) + recursive call was the only place we ever +# restarted the ASCII fast-path, so disabling it re-introduced the "continue with stale pointers" bug. + +# In the new unified parser, that same "encoding switch -> must restart locals" logic is centralized +# in the goto restart;. Wrapping only the goto restart; under MAGMA_ENABLE_FIXES in theory shoul make +# the new parser behave just like the old fallback did: it sometimes fails to re-sync after a buffer move, +# leading us to scan off the end of a freed chunk. + diff --git a/HTMLparser.c b/HTMLparser.c -index d6971b5..15da2ae 100644 +index b9e176b3..cd4eace9 100644 --- a/HTMLparser.c +++ b/HTMLparser.c -@@ -2519,6 +2519,7 @@ htmlParseNameComplex(xmlParserCtxtPtr ctxt) { - len += l; - NEXTL(l); - c = CUR_CHAR(l); +@@ -2612,6 +2612,7 @@ static xmlChar * + htmlParseData(htmlParserCtxtPtr ctxt, htmlAsciiMask mask, + int comment, int refs, int maxLength) { + xmlParserInputPtr input = ctxt->input; ++ const xmlChar *bug_base = input->base; + xmlChar *ret = NULL; + xmlChar *buffer; + xmlChar utf8Char[4]; +@@ -2807,8 +2808,12 @@ htmlParseData(htmlParserCtxtPtr ctxt, htmlAsciiMask mask, + xmlFree(guess); + } + input->flags |= XML_INPUT_HAS_ENCODING; +- +#ifdef MAGMA_ENABLE_FIXES - if (ctxt->input->base != base) { - /* - * We changed encoding from an unknown encoding -@@ -2526,6 +2527,10 @@ htmlParseNameComplex(xmlParserCtxtPtr ctxt) { - */ - return(htmlParseNameComplex(ctxt)); - } + goto restart; +#endif +#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", ctxt->input->base != base); ++ MAGMA_LOG("%MAGMA_BUG%", ctxt->input->base != bug_base); +#endif - } + } - if (ctxt->input->cur - ctxt->input->base < len) { + size = htmlValidateUtf8(ctxt, in, avail); diff --git a/targets/libxml2/patches/bugs/XML017.patch b/targets/libxml2/patches/bugs/XML017.patch index f17d70a73..2ced2fb08 100644 --- a/targets/libxml2/patches/bugs/XML017.patch +++ b/targets/libxml2/patches/bugs/XML017.patch @@ -1,13 +1,13 @@ diff --git a/parser.c b/parser.c -index 3f31329..4aea5b8 100644 +index ad035f3c..7bbfaa7f 100644 --- a/parser.c +++ b/parser.c -@@ -8311,10 +8311,20 @@ xmlParseInternalSubset(xmlParserCtxtPtr ctxt) { +@@ -8385,10 +8385,20 @@ xmlParseInternalSubset(xmlParserCtxtPtr ctxt) { /* * We should be at the end of the DOCTYPE declaration. */ +#ifdef MAGMA_ENABLE_FIXES - if (RAW != '>') { + if ((ctxt->wellFormed) && (RAW != '>')) { xmlFatalErr(ctxt, XML_ERR_DOCTYPE_NOT_FINISHED, NULL); return; } diff --git a/targets/libxml2/patches/bugs/XML002.patch b/targets/libxml2/patches/graveyard/XML002.patch similarity index 82% rename from targets/libxml2/patches/bugs/XML002.patch rename to targets/libxml2/patches/graveyard/XML002.patch index 594802b46..c668fd39d 100644 --- a/targets/libxml2/patches/bugs/XML002.patch +++ b/targets/libxml2/patches/graveyard/XML002.patch @@ -1,3 +1,5 @@ +# This bug is applied on functions that are now deprecated, therefore it does not make sense to include +# it in magma anymore since it will likely not be triggered at all by the latest version of libxml2 diff --git a/valid.c b/valid.c index b1cfede..49e6251 100644 --- a/valid.c diff --git a/targets/libxml2/patches/bugs/XML003.patch b/targets/libxml2/patches/graveyard/XML003.patch similarity index 85% rename from targets/libxml2/patches/bugs/XML003.patch rename to targets/libxml2/patches/graveyard/XML003.patch index ad87561fe..af3f142ad 100644 --- a/targets/libxml2/patches/bugs/XML003.patch +++ b/targets/libxml2/patches/graveyard/XML003.patch @@ -1,3 +1,5 @@ +# This bug is applied on functions that are now deprecated, therefore it does not make sense to include +# it in magma anymore since it will likely not be triggered at all by the latest version of libxml2 diff --git a/parser.c b/parser.c index 04aa717..579a9a7 100644 --- a/parser.c diff --git a/targets/libxml2/patches/bugs/XML005.patch b/targets/libxml2/patches/graveyard/XML005.patch similarity index 90% rename from targets/libxml2/patches/bugs/XML005.patch rename to targets/libxml2/patches/graveyard/XML005.patch index c3c773823..9eb450a1e 100644 --- a/targets/libxml2/patches/bugs/XML005.patch +++ b/targets/libxml2/patches/graveyard/XML005.patch @@ -1,3 +1,5 @@ +# This bug is applied on functions that are now deprecated, therefore it does not make sense to include +# it in magma anymore since it will likely not be triggered at all by the latest version of libxml2 --- a/xmlmemory.c +++ b/xmlmemory.c @@ -172,12 +172,17 @@ xmlMallocLoc(size_t size, const char * file, int line) diff --git a/targets/libxml2/patches/graveyard/XML007.patch b/targets/libxml2/patches/graveyard/XML007.patch new file mode 100644 index 000000000..b930b4606 --- /dev/null +++ b/targets/libxml2/patches/graveyard/XML007.patch @@ -0,0 +1,52 @@ +# Codebase versions used for the following explaination: +# 1. Old code: htmlParseTryOrFinish() in HTMLparser.c, xmlHaltParser() in parser.c +# (commit: ec6e3efb06d7b15cf5a2328fabd3845acea4c815) +# 2. New code: htmlParseTryOrFinish() in HTMLparser.c, xmlHaltParser() in parserInternals.c +# (commit: 41c10c0cec2ac072897c7d8df0e87fdf4b715586) + +# In the earlier HTML parser, xmlHaltParser() itself freed the input stream. So removing that if in +# just this function was enough to introduce the dangling-pointer bug in exactly one place. + +# In the new parser: +# 1. All buffer freeing is moved into the generic destructor xmlFreeInputStream(), which is called +# when an input stream is popped or the parser context is torn down, not in haltParser. +# 2. xmlHaltParser() just sets instate=EOF and disableSAX and htmlParseTryOrFinish() never calls +# xmlFreeInputStream() directly. +# 3. Therefore, there is no code path where we can skip the free and introduce the bug in a specific way. +# We could skip the free in xmlFreeInputStream() but this will affect every parser and entity pop. +# Because buffer freeing happens in a shared teardown routine outside of the haltParser, we cannot +# reintroduce the old bug in a contained way. + +# Therefore it is not possible to re-introduce this bug due to complex structural changes. + +--- a/HTMLparser.c ++++ b/HTMLparser.c +@@ -5392,6 +5392,9 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { + avail = in->length - (in->cur - in->base); + else + avail = (ptrdiff_t)xmlBufUse(in->buf->buffer) - + (in->cur - in->base); ++#ifdef MAGMA_ENABLE_CANARIES ++ MAGMA_LOG("%MAGMA_BUG%", avail != (in->end - in->cur)); ++#endif + if ((avail == 0) && (terminate)) { + htmlAutoCloseOnEnd(ctxt); + if ((ctxt->nameNr == 0) && (ctxt->instate != XML_PARSER_EOF)) { +--- a/parser.c ++++ b/parser.c +@@ -12491,12 +12491,14 @@ xmlHaltParser(xmlParserCtxtPtr ctxt) { + ctxt->input->free((xmlChar *) ctxt->input->base); + ctxt->input->free = NULL; + } ++#ifdef MAGMA_ENABLE_FIXES + if (ctxt->input->buf != NULL) { + xmlFreeParserInputBuffer(ctxt->input->buf); + ctxt->input->buf = NULL; + } +- ctxt->input->cur = BAD_CAST""; + ctxt->input->length = 0; ++#endif ++ ctxt->input->cur = BAD_CAST""; + ctxt->input->base = ctxt->input->cur; + ctxt->input->end = ctxt->input->cur; + } diff --git a/targets/libxml2/patches/bugs/XML008.patch b/targets/libxml2/patches/graveyard/XML008.patch similarity index 80% rename from targets/libxml2/patches/bugs/XML008.patch rename to targets/libxml2/patches/graveyard/XML008.patch index 142bfaf10..4be9a1dba 100644 --- a/targets/libxml2/patches/bugs/XML008.patch +++ b/targets/libxml2/patches/graveyard/XML008.patch @@ -1,3 +1,5 @@ +# This bug is applied on functions that are now deprecated, therefore it does not make sense to include +# it in magma anymore since it will likely not be triggered at all by the latest version of libxml2 diff --git a/parser.c b/parser.c index 3f31329..b3e1429 100644 --- a/parser.c diff --git a/targets/libxml2/patches/bugs/XML009.patch b/targets/libxml2/patches/graveyard/XML009.patch similarity index 84% rename from targets/libxml2/patches/bugs/XML009.patch rename to targets/libxml2/patches/graveyard/XML009.patch index 169b3ed17..bd37fde25 100644 --- a/targets/libxml2/patches/bugs/XML009.patch +++ b/targets/libxml2/patches/graveyard/XML009.patch @@ -1,3 +1,5 @@ +# This bug is applied on functions that are now deprecated, therefore it does not make sense to include +# it in magma anymore since it will likely not be triggered at all by the latest version of libxml2 diff --git a/parser.c b/parser.c index 3f31329..ff89bb7 100644 --- a/parser.c diff --git a/targets/libxml2/patches/bugs/XML010.patch b/targets/libxml2/patches/graveyard/XML010.patch similarity index 85% rename from targets/libxml2/patches/bugs/XML010.patch rename to targets/libxml2/patches/graveyard/XML010.patch index e772b5547..742f4aba6 100644 --- a/targets/libxml2/patches/bugs/XML010.patch +++ b/targets/libxml2/patches/graveyard/XML010.patch @@ -1,3 +1,5 @@ +# This bug is applied on functions that are now deprecated, therefore it does not make sense to include +# it in magma anymore since it will likely not be triggered at all by the latest version of libxml2 diff --git a/parser.c b/parser.c index 3f31329..db7f7d1 100644 --- a/parser.c diff --git a/targets/libxml2/patches/bugs/XML013.patch b/targets/libxml2/patches/graveyard/XML013.patch similarity index 66% rename from targets/libxml2/patches/bugs/XML013.patch rename to targets/libxml2/patches/graveyard/XML013.patch index e8bfd15fd..8328b757b 100644 --- a/targets/libxml2/patches/bugs/XML013.patch +++ b/targets/libxml2/patches/graveyard/XML013.patch @@ -1,3 +1,21 @@ +# Codebase versions used for the following explaination: +# 1. Old code: htmlParseSystemLiteral() in HTMLparser.c (commit: ec6e3efb06d7b15cf5a2328fabd3845acea4c815) +# 2. New code: htmlParseDoctypeLiteral() in HTMLparser.c (commit: 41c10c0cec2ac072897c7d8df0e87fdf4b715586) + +# The new parser always passes everything through htmlParseData() which allocates its buffer, +# copies into it, and then returns a heap-allocated string. So we have no way to get at the raw +# input buffer start or length without re-introducing a bit of the old logic. To reproduce the +# old "copy straight from CUR_PTR" bug, we must: +# - Remember where in the input buffer you began (i.e. capture q = input->cur), +# - Let htmlParseData() run, +# - Discard its result and do our own xmlStrndup(q, input->cur - q). + +# Alternatives like trying to introduce that bug inside htmlParseData() would force us to re-implement +# the literal-scanning loop there (resurrecting all of the old byte-by-byte code), or require exposing +# internal buffers/pointers, which does not make sense with the unified function. + +# Therefore, it is not possible to re-introduce this bug without ressurecting old code. + diff --git a/HTMLparser.c b/HTMLparser.c index de624f8d..00b2fb1c 100644 --- a/HTMLparser.c diff --git a/targets/libxml2/patches/graveyard/XML014.patch b/targets/libxml2/patches/graveyard/XML014.patch new file mode 100644 index 000000000..1318be862 --- /dev/null +++ b/targets/libxml2/patches/graveyard/XML014.patch @@ -0,0 +1,39 @@ +# Codebase versions used for the following explaination: +# 1. Old code: htmlParseName() in HTMLparser.c (commit: ec6e3efb06d7b15cf5a2328fabd3845acea4c815) +# 2. New code: htmlParseData() in HTMLparser.c (commit: 41c10c0cec2ac072897c7d8df0e87fdf4b715586) + +# In the old code there was a two-phase "fast ASCII name" logic in htmlParseName which scanned forward +# over purely ASCII name characters up until either: +# a. It hit a non-name character (success), or +# b. It hit the very end of the in-memory buffer (in == end) (bail out, return NULL), +# and only if it bailed out did the parser refill the buffer and fall back to the slower, complex path. +# Because that fast path was a standalone function, re-introducing its exact "if you hit the buffer end, +# just return NULL” behavior could be done by dropping the if (in == end) return NULL; in htmlParseName. + +# In the new code there is no separate htmlParseName fast path. Every piece of HTML text is handled by +# the one big htmlParseData loop. Which never does a simple return NULL when it reaches input->end; +# instead it watches the avail = end - cur count and, before it ever risks running off the end, calls +# xmlParserGrow() to refill the buffer and then resumes scanning. Because name-character checking is just one +# branch inside a giant, mask-driven state machine, there is no isolated place where "we are in a name scan and +# we hit buffer end-return NULL" can be injected without turning the unified loop back into a two-phase design. + +# Therefore, it is not possible to re-introduce this bug without ressurecting old code. + +diff --git a/HTMLparser.c b/HTMLparser.c +index d6971b5..7c82c11 100644 +--- a/HTMLparser.c ++++ b/HTMLparser.c +@@ -2473,8 +2473,13 @@ htmlParseName(htmlParserCtxtPtr ctxt) { + (*in == ':') || (*in == '.')) + in++; + ++#ifdef MAGMA_ENABLE_FIXES + if (in == ctxt->input->end) + return(NULL); ++#endif ++#ifdef MAGMA_ENABLE_CANARIES ++ MAGMA_LOG("%MAGMA_BUG%", in >= ctxt->input->end); ++#endif + + if ((*in > 0) && (*in < 0x80)) { + count = in - ctxt->input->cur; diff --git a/targets/libxml2/patches/setup/libxml2.patch b/targets/libxml2/patches/graveyard/libxml2.patch similarity index 82% rename from targets/libxml2/patches/setup/libxml2.patch rename to targets/libxml2/patches/graveyard/libxml2.patch index e2faa48c6..2db92bcab 100644 --- a/targets/libxml2/patches/setup/libxml2.patch +++ b/targets/libxml2/patches/graveyard/libxml2.patch @@ -1,3 +1,5 @@ +// No longer needed, XML_COPTS is not used in the updated libxml2 +// anywhere and build is successful without it. diff --git a/Makefile.am b/Makefile.am index 05d1671..75ea675 100644 --- a/Makefile.am diff --git a/targets/libxml2/pocs/XML001.crash b/targets/libxml2/pocs/XML001.crash new file mode 100644 index 000000000..9b6552674 --- /dev/null +++ b/targets/libxml2/pocs/XML001.crash @@ -0,0 +1,2 @@ +# TODO: Replace this with a real crash and investigate how to to cause this crash +# and add the corresponding bug command diff --git a/targets/libxml2/preinstall.sh b/targets/libxml2/preinstall.sh index ca3a3715d..4cb4dec5a 100755 --- a/targets/libxml2/preinstall.sh +++ b/targets/libxml2/preinstall.sh @@ -1,5 +1,5 @@ #!/bin/bash apt-get update && \ - apt-get install -y git make autoconf automake libtool pkg-config zlib1g-dev \ - liblzma-dev + apt-get install -y git make autoconf libtool pkg-config zlib1g-dev \ + liblzma-dev automake diff --git a/targets/libxml2/releases b/targets/libxml2/releases new file mode 100644 index 000000000..34c42cf4f --- /dev/null +++ b/targets/libxml2/releases @@ -0,0 +1,8 @@ +libxml2_PIONEER="https://gitlab.gnome.org/GNOME/libxml2.git" +libxml2_PIONEER_STABLE_COMMIT="41c10c0cec2ac072897c7d8df0e87fdf4b715586" +libxml2_LEGACY_2022="https://gitlab.gnome.org/GNOME/libxml2.git" +libxml2_LEGACY_2022_TAG="v2.10.0" +libxml2_LEGACY_2023="https://gitlab.gnome.org/GNOME/libxml2.git" +libxml2_LEGACY_2023_TAG="v2.10.4" +libxml2_LEGACY_2024="https://gitlab.gnome.org/GNOME/libxml2.git" +libxml2_LEGACY_2024_TAG="v2.11.7" diff --git a/targets/lua/build_poc.sh b/targets/lua/build_poc.sh new file mode 100755 index 000000000..891bbb25a --- /dev/null +++ b/targets/lua/build_poc.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch_target.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" +make -j$(nproc) clean +make -j$(nproc) liblua.a +make -j$(nproc) lua + +mv liblua.a /usr/local/bin/ +mv lua /usr/local/bin/ diff --git a/targets/lua/fetch.sh b/targets/lua/fetch.sh deleted file mode 100755 index 8a42e3976..000000000 --- a/targets/lua/fetch.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -git clone --no-checkout https://github.com/lua/lua.git "$TARGET/repo" -git -C "$TARGET/repo" checkout dbdc74dc5502c2e05e1c1e2ac894943f418c8431 \ No newline at end of file diff --git a/targets/lua/patches/bugs/LUA001.patch b/targets/lua/patches/bugs/LUA001.patch index a6045eef4..01a08f968 100644 --- a/targets/lua/patches/bugs/LUA001.patch +++ b/targets/lua/patches/bugs/LUA001.patch @@ -1,25 +1,25 @@ diff --git a/ldebug.c b/ldebug.c -index 8e3657a9..8b3a61ea 100644 +index 8b4bd546..528a7196 100644 --- a/ldebug.c +++ b/ldebug.c -@@ -181,8 +181,16 @@ static const char *upvalname (const Proto *p, int uv) { +@@ -184,8 +184,16 @@ static const char *upvalname (const Proto *p, int uv) { static const char *findvararg (CallInfo *ci, int n, StkId *pos) { - if (clLvalue(s2v(ci->func))->p->is_vararg) { + if (clLvalue(s2v(ci->func.p))->p->flag & PF_ISVARARG) { int nextra = ci->u.l.nextraargs; +#ifdef MAGMA_ENABLE_FIXES if (n >= -nextra) { /* 'n' is negative */ - *pos = ci->func - nextra - (n + 1); + *pos = ci->func.p - nextra - (n + 1); +#else + if (n <= nextra) { +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", INT_MAX - nextra <= (n - 1)); +#endif -+ *pos = ci->func - nextra + (n - 1); ++ *pos = ci->func.p - nextra + (n - 1); +#endif return "(vararg)"; /* generic name for any vararg */ } } -@@ -195,7 +203,11 @@ const char *luaG_findlocal (lua_State *L, CallInfo *ci, int n, StkId *pos) { +@@ -198,7 +206,11 @@ const char *luaG_findlocal (lua_State *L, CallInfo *ci, int n, StkId *pos) { const char *name = NULL; if (isLua(ci)) { if (n < 0) /* access to vararg values? */ diff --git a/targets/lua/patches/bugs/LUA003.patch b/targets/lua/patches/bugs/LUA003.patch index dc4d66770..0eba662ce 100644 --- a/targets/lua/patches/bugs/LUA003.patch +++ b/targets/lua/patches/bugs/LUA003.patch @@ -1,5 +1,5 @@ diff --git a/liolib.c b/liolib.c -index b08397da..49d72eeb 100644 +index 98ff3d0d..e082349a 100644 --- a/liolib.c +++ b/liolib.c @@ -291,7 +291,13 @@ static int io_popen (lua_State *L) { @@ -13,6 +13,6 @@ index b08397da..49d72eeb 100644 + MAGMA_LOG("%MAGMA_BUG%", !l_checkmodep(mode)); +#endif +#endif + errno = 0; p->f = l_popen(L, filename, mode); p->closef = &io_pclose; - return (p->f == NULL) ? luaL_fileresult(L, 0, filename) : 1; diff --git a/targets/lua/patches/bugs/LUA004.patch b/targets/lua/patches/bugs/LUA004.patch index baf5d8848..fc76e2876 100644 --- a/targets/lua/patches/bugs/LUA004.patch +++ b/targets/lua/patches/bugs/LUA004.patch @@ -1,8 +1,8 @@ diff --git a/ldebug.c b/ldebug.c -index 8e3657a9..a4b64532 100644 +index 09ec197c..71b7ff45 100644 --- a/ldebug.c +++ b/ldebug.c -@@ -130,6 +130,10 @@ LUA_API void lua_sethook (lua_State *L, lua_Hook func, int mask, int count) { +@@ -133,6 +133,10 @@ LUA_API void lua_sethook (lua_State *L, lua_Hook func, int mask, int count) { mask = 0; func = NULL; } @@ -13,17 +13,17 @@ index 8e3657a9..a4b64532 100644 L->hook = func; L->basehookcount = count; resethookcount(L); -@@ -830,7 +834,9 @@ static int changedline (const Proto *p, int oldpc, int newpc) { +@@ -920,7 +924,9 @@ int luaG_tracecall (lua_State *L) { int luaG_traceexec (lua_State *L, const Instruction *pc) { CallInfo *ci = L->ci; - lu_byte mask = L->hookmask; + lu_byte mask = cast_byte(L->hookmask); +#ifdef MAGMA_ENABLE_FIXES const Proto *p = ci_func(ci)->p; +#endif int counthook; if (!(mask & (LUA_MASKLINE | LUA_MASKCOUNT))) { /* no hooks? */ ci->u.l.trap = 0; /* don't need to stop again */ -@@ -852,15 +858,32 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { +@@ -942,15 +948,32 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { if (counthook) luaD_hook(L, LUA_HOOKCOUNT, -1, 0, 0); /* call count hook */ if (mask & LUA_MASKLINE) { @@ -57,24 +57,24 @@ index 8e3657a9..a4b64532 100644 if (L->status == LUA_YIELD) { /* did hook yield? */ if (counthook) diff --git a/ldo.c b/ldo.c -index 7135079b..2589be9b 100644 +index f825d959..a146cf5a 100644 --- a/ldo.c +++ b/ldo.c -@@ -368,7 +368,11 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { +@@ -446,7 +446,11 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { int delta = 0; /* correction for vararg functions */ int ftransfer; if (isLua(ci)) { +#ifdef MAGMA_ENABLE_FIXES Proto *p = ci_func(ci)->p; +#else -+ Proto *p = clLvalue(s2v(ci->func))->p; ++ Proto *p = clLvalue(s2v(ci->func.p))->p; +#endif - if (p->is_vararg) + if (p->flag & PF_ISVARARG) delta = ci->u.l.nextraargs + p->numparams + 1; } -@@ -377,8 +381,13 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { +@@ -455,8 +459,13 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { luaD_hook(L, LUA_HOOKRET, -1, ftransfer, nres); /* call it */ - ci->func -= delta; + ci->func.p -= delta; } +#ifdef MAGMA_ENABLE_FIXES if (isLua(ci = ci->previous)) @@ -87,10 +87,10 @@ index 7135079b..2589be9b 100644 diff --git a/lstate.c b/lstate.c -index c5e3b437..07524a8e 100644 +index 0e1cb01e..008ba707 100644 --- a/lstate.c +++ b/lstate.c -@@ -262,7 +262,9 @@ static void preinit_thread (lua_State *L, global_State *g) { +@@ -253,7 +253,9 @@ static void preinit_thread (lua_State *L, global_State *g) { L->openupval = NULL; L->status = LUA_OK; L->errfunc = 0; @@ -101,20 +101,20 @@ index c5e3b437..07524a8e 100644 diff --git a/lstate.h b/lstate.h -index c1283bb6..bc0eadc4 100644 +index 1c81b6ed..fe01f0c1 100644 --- a/lstate.h +++ b/lstate.h -@@ -309,6 +309,9 @@ struct lua_State { - StkId top; /* first free slot in the stack */ +@@ -343,6 +343,9 @@ struct lua_State { + StkIdRel top; /* first free slot in the stack */ global_State *l_G; CallInfo *ci; /* call info for current function */ +#ifndef MAGMA_ENABLE_FIXES + const Instruction *oldpc; +#endif - StkId stack_last; /* end of stack (last element + 1) */ - StkId stack; /* stack base */ + StkIdRel stack_last; /* end of stack (last element + 1) */ + StkIdRel stack; /* stack base */ UpVal *openupval; /* list of open upvalues in this stack */ -@@ -320,7 +323,9 @@ struct lua_State { +@@ -354,7 +357,9 @@ struct lua_State { volatile lua_Hook hook; ptrdiff_t errfunc; /* current error handling function (stack index) */ l_uint32 nCcalls; /* number of nested (non-yieldable | C) calls */ @@ -125,10 +125,10 @@ index c1283bb6..bc0eadc4 100644 int hookcount; volatile l_signalT hookmask; diff --git a/lvm.c b/lvm.c -index c9729bcc..572b6a23 100644 +index b6b18a69..0a7134ff 100644 --- a/lvm.c +++ b/lvm.c -@@ -1820,7 +1820,11 @@ void luaV_execute (lua_State *L, CallInfo *ci) { +@@ -1895,7 +1895,11 @@ void luaV_execute (lua_State *L, CallInfo *ci) { ProtectNT(luaT_adjustvarargs(L, GETARG_A(i), ci, cl->p)); if (l_unlikely(trap)) { /* previous "Protect" updated trap */ luaD_hookcall(L, ci); diff --git a/targets/lua/pocs/LUA002.crash b/targets/lua/pocs/LUA002.crash new file mode 100644 index 000000000..29e1baef7 Binary files /dev/null and b/targets/lua/pocs/LUA002.crash differ diff --git a/targets/lua/pocs/LUA004.crash b/targets/lua/pocs/LUA004.crash new file mode 100644 index 000000000..7df267fc5 Binary files /dev/null and b/targets/lua/pocs/LUA004.crash differ diff --git a/targets/lua/releases b/targets/lua/releases new file mode 100644 index 000000000..9b7d32b28 --- /dev/null +++ b/targets/lua/releases @@ -0,0 +1,8 @@ +lua_PIONEER="https://github.com/lua/lua.git" +lua_PIONEER_STABLE_COMMIT="1ec251e091302515e54aa81d965840a5de4be0a1" +lua_LEGACY_2022="https://github.com/lua/lua.git" +lua_LEGACY_2022_TAG="v5.4.4" +lua_LEGACY_2023="https://github.com/lua/lua.git" +lua_LEGACY_2023_TAG="v5.4.5" +lua_LEGACY_2024="https://github.com/lua/lua.git" +lua_LEGACY_2024_TAG="v5.4.7" diff --git a/targets/openssl/build.sh b/targets/openssl/build.sh index 965f44cfe..0bd8cdc4f 100755 --- a/targets/openssl/build.sh +++ b/targets/openssl/build.sh @@ -13,7 +13,10 @@ if [ ! -d "$TARGET/repo" ]; then exit 1 fi -# build the libpng library +# An extra target specific step from previously used openssl/fetch.sh +cp "$TARGET/src/abilist.txt" "$TARGET/repo/abilist.txt" + +# build the openssl library cd "$TARGET/repo" CONFIGURE_FLAGS="" @@ -23,6 +26,7 @@ fi # the config script supports env var LDLIBS instead of LIBS export LDLIBS="$LIBS" +export LDFLAGS="$LDFLAGS $LIB_FUZZING_ENGINE" ./config --debug enable-fuzz-libfuzzer enable-fuzz-afl disable-tests -DPEDANTIC \ -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION no-shared no-module \ @@ -31,7 +35,9 @@ export LDLIBS="$LIBS" $CFLAGS -fno-sanitize=alignment $CONFIGURE_FLAGS make -j$(nproc) clean -make -j$(nproc) LDCMD="$CXX $CXXFLAGS" +set +e +make -j$(nproc) -k LDCMD="$CXX $CXXFLAGS" +set -e fuzzers=$(find fuzz -executable -type f '!' -name \*.py '!' -name \*-test '!' -name \*.pl) for f in $fuzzers; do diff --git a/targets/openssl/build_poc.sh b/targets/openssl/build_poc.sh new file mode 100755 index 000000000..48bd5c0b3 --- /dev/null +++ b/targets/openssl/build_poc.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch_target.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" +cp "$TARGET/src/abilist.txt" "$TARGET/repo/abilist.txt" + +./config --debug disable-tests -DPEDANTIC \ + -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION no-shared no-module \ + enable-tls1_3 enable-rc5 enable-md2 enable-ec_nistp_64_gcc_128 enable-ssl3 \ + enable-ssl3-method enable-nextprotoneg enable-weak-ssl-ciphers \ + +make -j$(nproc) clean +make -j$(nproc) +make install diff --git a/targets/openssl/fetch.sh b/targets/openssl/fetch.sh deleted file mode 100755 index 10e48e47b..000000000 --- a/targets/openssl/fetch.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -git clone --no-checkout https://github.com/openssl/openssl.git \ - "$TARGET/repo" -git -C "$TARGET/repo" checkout 3bd5319b5d0df9ecf05c8baba2c401ad8e3ba130 - -cp "$TARGET/src/abilist.txt" "$TARGET/repo/abilist.txt" diff --git a/targets/openssl/patches/bugs/SSL008.patch b/targets/openssl/patches/bugs/SSL008.patch index 7597c980a..e7c5f1ff3 100644 --- a/targets/openssl/patches/bugs/SSL008.patch +++ b/targets/openssl/patches/bugs/SSL008.patch @@ -22,7 +22,7 @@ ckey = ssl_generate_pkey(s, skey); +#ifdef MAGMA_ENABLE_FIXES if (ckey == NULL) { - SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE); + SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB); goto err; } +#endif diff --git a/targets/openssl/patches/bugs/SSL015.patch b/targets/openssl/patches/bugs/SSL015.patch index 50d3487bf..7712b6f58 100644 --- a/targets/openssl/patches/bugs/SSL015.patch +++ b/targets/openssl/patches/bugs/SSL015.patch @@ -64,25 +64,23 @@ index 2cf62b62cd..f53a5e0082 100644 ctx_tmp = EVP_MD_CTX_new(); if (ctx_tmp == NULL) { PKCS7err(PKCS7_F_PKCS7_DATAFINAL, ERR_R_MALLOC_FAILURE); -@@ -920,7 +960,8 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, - int ret = 0, i; - STACK_OF(X509) *cert; - X509 *x509; +@@ -1007,6 +1007,7 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, + STACK_OF(X509_CRL) *crls; + X509 *signer; +#ifdef MAGMA_ENABLE_FIXES if (p7 == NULL) { ERR_raise(ERR_LIB_PKCS7, PKCS7_R_INVALID_NULL_POINTER); return 0; -@@ -930,7 +970,12 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, +@@ -1016,6 +1017,11 @@ int PKCS7_dataVerify(X509_STORE *cert_store, X509_STORE_CTX *ctx, BIO *bio, ERR_raise(ERR_LIB_PKCS7, PKCS7_R_NO_CONTENT); return 0; } - +#else +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(p7 == NULL, p7->d.ptr)); +#endif +#endif + if (PKCS7_type_is_signed(p7)) { - cert = p7->d.sign->cert; - } else if (PKCS7_type_is_signedAndEnveloped(p7)) { + untrusted = p7->d.sign->cert; diff --git a/targets/openssl/patches/bugs/SSL019.patch b/targets/openssl/patches/bugs/SSL019.patch index e141bc46e..a062d1ad7 100644 --- a/targets/openssl/patches/bugs/SSL019.patch +++ b/targets/openssl/patches/bugs/SSL019.patch @@ -1,25 +1,25 @@ diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c -index b4675f3c8c..cfd1b5d415 100644 +index 0930222edb..22c9e52f46 100644 --- a/ssl/record/rec_layer_s3.c +++ b/ssl/record/rec_layer_s3.c -@@ -1469,11 +1469,20 @@ int ssl3_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf, +@@ -791,11 +791,20 @@ int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type, memcpy(buf, &(rr->data[rr->off]), n); buf += n; +#ifdef MAGMA_ENABLE_FIXES if (peek) { /* Mark any zero length record as consumed CVE-2016-6305 */ - if (SSL3_RECORD_get_length(rr) == 0) - SSL3_RECORD_set_read(rr); + if (rr->length == 0 && !ssl_release_record(s, rr, 0)) + return -1; } else { +#else +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_AND(peek, \ -+ MAGMA_AND(SSL3_RECORD_get_length(rr) == 0, \ -+ !SSL3_RECORD_is_read(rr)))); ++ MAGMA_AND(rr->length == 0, \ ++ !ssl_release_record(s, rr, 0)))); +#endif + if (!peek) { +#endif - if (s->options & SSL_OP_CLEANSE_PLAINTEXT) - OPENSSL_cleanse(&(rr->data[rr->off]), n); - SSL3_RECORD_sub_length(rr, n); + if (!ssl_release_record(s, rr, n)) + return -1; + } diff --git a/targets/openssl/patches/bugs/SSL020.patch b/targets/openssl/patches/bugs/SSL020.patch index 3c45d793f..c09850a84 100644 --- a/targets/openssl/patches/bugs/SSL020.patch +++ b/targets/openssl/patches/bugs/SSL020.patch @@ -1,14 +1,13 @@ diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c -index 923cf5b5cd..6ed2362391 100644 +index 5a7201b7a2..56f9bdbf8b 100644 --- a/ssl/t1_lib.c +++ b/ssl/t1_lib.c -@@ -1550,11 +1556,18 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL *s, const unsigned char *etick, +@@ -3184,10 +3184,17 @@ SSL_TICKET_STATUS tls_decrypt_ticket(SSL_CONNECTION *s, } /* Sanity check ticket length: must exceed keyname + IV + HMAC */ +#ifdef MAGMA_ENABLE_FIXES - if (eticklen <= - TLSEXT_KEYNAME_LENGTH + EVP_CIPHER_CTX_get_iv_length(ctx) + mlen) { + if (eticklen <= TLSEXT_KEYNAME_LENGTH + ivlen + mlen) { +#else + if (eticklen < 48) { +#endif @@ -16,7 +15,7 @@ index 923cf5b5cd..6ed2362391 100644 goto end; } +#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", eticklen <= TLSEXT_KEYNAME_LENGTH + EVP_CIPHER_CTX_get_iv_length(ctx) + mlen); ++ MAGMA_LOG("%MAGMA_BUG%", eticklen <= TLSEXT_KEYNAME_LENGTH + ivlen + mlen); +#endif eticklen -= mlen; /* Check HMAC of encrypted ticket */ diff --git a/targets/openssl/pocs/SSL001.crash b/targets/openssl/pocs/SSL001.crash new file mode 100644 index 000000000..9b6552674 --- /dev/null +++ b/targets/openssl/pocs/SSL001.crash @@ -0,0 +1,2 @@ +# TODO: Replace this with a real crash and investigate how to to cause this crash +# and add the corresponding bug command diff --git a/targets/openssl/releases b/targets/openssl/releases new file mode 100644 index 000000000..b47549f44 --- /dev/null +++ b/targets/openssl/releases @@ -0,0 +1,8 @@ +openssl_PIONEER="https://github.com/openssl/openssl.git" +openssl_PIONEER_STABLE_COMMIT="32476957ead4151dceaf873306fc7e79cd262812" +openssl_LEGACY_2022="https://github.com/openssl/openssl.git" +openssl_LEGACY_2022_TAG="OpenSSL_1_1_1n" +openssl_LEGACY_2023="https://github.com/openssl/openssl.git" +openssl_LEGACY_2023_TAG="OpenSSL_1_1_1t" +openssl_LEGACY_2024="https://github.com/openssl/openssl.git" +openssl_LEGACY_2024_TAG="openssl-3.0.13" diff --git a/targets/php/build.sh b/targets/php/build.sh index 7207c0b8f..c941c5821 100755 --- a/targets/php/build.sh +++ b/targets/php/build.sh @@ -13,6 +13,11 @@ if [ ! -d "$TARGET/repo" ]; then exit 1 fi +# The source code of oniguruma is cloned in preinstall.sh. Do a cp + sudo rm +# over mv so that the script has the correct permissions to build oniguruma. +cp -r "$TARGET/oniguruma" "$TARGET/repo/oniguruma" && \ + sudo rm -rf "$TARGET/oniguruma" + cd "$TARGET/repo" export ONIG_CFLAGS="-I$PWD/oniguruma/src" export ONIG_LIBS="-L$PWD/oniguruma/src/.libs -l:libonig.a" @@ -24,9 +29,17 @@ export EXTRA_CXXFLAGS="$CXXFLAGS -fno-sanitize=object-size" unset CFLAGS unset CXXFLAGS +# These need to be exported to build latest PHP with honggfuzz +export CFLAGS="-fPIE" +export CXXFLAGS="-fPIE" +export LDFLAGS="$LDFLAGS -pie" + #build the php library ./buildconf -LIB_FUZZING_ENGINE="-Wall" ./configure \ + +# Note: the acv_cv_func_fork=yes is needed to avoid the fork() +# check in configure which fails when using the hongfuzz compiler +LIB_FUZZING_ENGINE="-Wall $LIB_FUZZING_ENGINE" ./configure \ --disable-all \ --enable-option-checking=fatal \ --enable-fuzzer \ @@ -37,7 +50,8 @@ LIB_FUZZING_ENGINE="-Wall" ./configure \ --without-pcre-jit \ --disable-phpdbg \ --disable-cgi \ - --with-pic + --with-pic \ + ac_cv_func_fork=yes make -j$(nproc) clean diff --git a/targets/php/build_poc.sh b/targets/php/build_poc.sh new file mode 100755 index 000000000..5bb112c0a --- /dev/null +++ b/targets/php/build_poc.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch_target.sh must be executed first." + exit 1 +fi + +cp -r "$TARGET/oniguruma" "$TARGET/repo/oniguruma" && \ + sudo rm -rf "$TARGET/oniguruma" + +cd "$TARGET/repo" + +export ONIG_CFLAGS="-I$PWD/oniguruma/src" +export ONIG_LIBS="-L$PWD/oniguruma/src/.libs -l:libonig.a" +export CFLAGS="-fPIE" +export CXXFLAGS="-fPIE" +export LDFLAGS="$LDFLAGS -pie" + +#build the php library +./buildconf + +./configure \ + --disable-all \ + --enable-option-checking=fatal \ + --enable-exif \ + --enable-phar \ + --enable-intl \ + --enable-mbstring \ + --without-pcre-jit \ + --disable-phpdbg \ + --disable-cgi \ + --with-pic \ + ac_cv_func_fork=yes + +make -j$(nproc) clean + +# build oniguruma and link statically +pushd oniguruma +autoreconf -vfi +./configure --disable-shared +make -j$(nproc) +popd + +make -j$(nproc) +make install diff --git a/targets/php/fetch.sh b/targets/php/fetch.sh deleted file mode 100755 index dc3cbd845..000000000 --- a/targets/php/fetch.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -git clone --no-checkout https://github.com/php/php-src.git \ - "$TARGET/repo" -git -C "$TARGET/repo" checkout bc39abe8c3c492e29bc5d60ca58442040bbf063b - -git clone --no-checkout https://github.com/kkos/oniguruma.git \ - "$TARGET/repo/oniguruma" -git -C "$TARGET/repo/oniguruma" checkout 227ec0bd690207812793c09ad70024707c405376 \ No newline at end of file diff --git a/targets/php/patches/bugs/PHP012.patch b/targets/php/patches/bugs/PHP012.patch index 7960edcdd..9a152b672 100644 --- a/targets/php/patches/bugs/PHP012.patch +++ b/targets/php/patches/bugs/PHP012.patch @@ -1,18 +1,19 @@ diff --git a/ext/standard/http_fopen_wrapper.c b/ext/standard/http_fopen_wrapper.c -index ae9b915101..c3d64424f3 100644 +index fff6af5e214..9c63f400d3e 100644 --- a/ext/standard/http_fopen_wrapper.c +++ b/ext/standard/http_fopen_wrapper.c -@@ -712,12 +712,25 @@ finish: - tmp_line, response_code); - } - } +@@ -723,6 +723,7 @@ static php_stream *php_stream_url_wrap_http_ex(php_stream_wrapper *wrapper, + tmp_line, response_code); + } + } + #ifdef MAGMA_ENABLE_FIXES - if (tmp_line_len >= 1 && tmp_line[tmp_line_len - 1] == '\n') { - --tmp_line_len; - if (tmp_line_len >= 1 &&tmp_line[tmp_line_len - 1] == '\r') { - --tmp_line_len; - } - } + if (tmp_line_len >= 1 && tmp_line[tmp_line_len - 1] == '\n') { + --tmp_line_len; + if (tmp_line_len >= 1 &&tmp_line[tmp_line_len - 1] == '\r') { +@@ -733,6 +734,18 @@ static php_stream *php_stream_url_wrap_http_ex(php_stream_wrapper *wrapper, + char *line = php_stream_get_line(stream, NULL, 0, NULL); + efree(line); + } + #else + #ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", tmp_line_len < 1); @@ -25,6 +26,6 @@ index ae9b915101..c3d64424f3 100644 + } + } + #endif - ZVAL_STRINGL(&http_response, tmp_line, tmp_line_len); - zend_hash_next_index_insert(Z_ARRVAL_P(response_header), &http_response); - } else { + ZVAL_STRINGL(&http_response, tmp_line, tmp_line_len); + zend_hash_next_index_insert(Z_ARRVAL_P(response_header), &http_response); + } else { diff --git a/targets/php/patches/setup/setup.patch b/targets/php/patches/setup/setup.patch index c4e1a2282..05217e36f 100644 --- a/targets/php/patches/setup/setup.patch +++ b/targets/php/patches/setup/setup.patch @@ -1,68 +1,66 @@ +diff --git a/build/php.m4 b/build/php.m4 +index 53684cf011..467ee8a968 100644 --- a/build/php.m4 +++ b/build/php.m4 -@@ -252,8 +252,9 @@ dnl Append to the array which has been dynamically chosen at m4 time. +@@ -272,8 +272,9 @@ dnl Append to the array which has been dynamically chosen at m4 time. dnl Choose the right compiler/flags/etc. for the source-file. case $ac_src in - *.c[)] ac_comp="$b_c_pre $ac_inc $b_c_meta $3 -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; -- *.s[)] ac_comp="$b_c_pre $ac_inc $b_c_meta $3 -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; -- *.S[)] ac_comp="$b_c_pre $ac_inc $b_c_meta $3 -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; + *.c[)] ac_comp="$b_c_pre $ac_inc $b_c_meta m4_normalize(m4_expand([$3])) -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; +- *.s[)] ac_comp="$b_c_pre $ac_inc $b_c_meta m4_normalize(m4_expand([$3])) -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; +- *.S[)] ac_comp="$b_c_pre $ac_inc $b_c_meta m4_normalize(m4_expand([$3])) -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; + *.s|*.S[)] + b_c_meta_clean='$(COMMON_FLAGS) $(CFLAGS_CLEAN)'; -+ ac_comp="$b_c_pre $ac_inc $b_c_meta_clean $3 -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; - *.cpp|*.cc|*.cxx[)] ac_comp="$b_cxx_pre $ac_inc $b_cxx_meta $3 -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_cxx_post" ;; ++ ac_comp="$b_c_pre $ac_inc $b_c_meta_clean m4_normalize(m4_expand([$3])) -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_c_post" ;; + *.cpp|*.cc|*.cxx[)] ac_comp="$b_cxx_pre $ac_inc $b_cxx_meta m4_normalize(m4_expand([$3])) -c $ac_srcdir$ac_src -o $ac_bdir$ac_obj.$b_lo $b_cxx_post" ;; esac ---- a/sapi/fuzzer/config.m4 -+++ b/sapi/fuzzer/config.m4 -@@ -60,7 +60,7 @@ if test "$PHP_FUZZER" != "no"; then - ]) - else - FUZZING_LIB="$LIB_FUZZING_ENGINE" -- FUZZING_CC="$CXX -stdlib=libc++" -+ AX_CHECK_COMPILE_FLAG([-stdlib=libstdc++], [FUZZING_CC="$CXX -stdlib=libstdc++"], [FUZZING_CC="$CXX"]) - fi - PHP_SUBST(FUZZING_LIB) - PHP_SUBST(FUZZING_CC) ---- a/ext/standard/string.c -+++ b/ext/standard/string.c -@@ -3682,10 +3682,14 @@ - # include "Zend/zend_cpuinfo.h" - - ZEND_INTRIN_SSE4_2_FUNC_DECL(zend_string *php_addslashes_sse42(zend_string *str)); -+ -+__attribute__((used)) - zend_string *php_addslashes_default(zend_string *str); - - ZEND_INTRIN_SSE4_2_FUNC_DECL(void php_stripslashes_sse42(zend_string *str)); -+ -+__attribute__((used)) - void php_stripslashes_default(zend_string *str); - - # if ZEND_INTRIN_SSE4_2_FUNC_PROTO - PHPAPI zend_string *php_addslashes(zend_string *str) __attribute__((ifunc("resolve_addslashes"))); +@@ -2402,7 +2403,7 @@ AC_CACHE_CHECK([for $1], [php_var], + [__builtin_clz], [return $1(1) ? 1 : 0;], + [__builtin_clzl], [return $1(1) ? 1 : 0;], + [__builtin_clzll], [return $1(1) ? 1 : 0;], +- [__builtin_cpu_init], [$1();], ++ [__builtin_cpu_init], [return ($1(), 1) ? 1 : 0;], + [__builtin_cpu_supports], [return $1("sse")? 1 : 0;], + [__builtin_ctzl], [return $1(2L) ? 1 : 0;], + [__builtin_ctzll], [return $1(2LL) ? 1 : 0;], +diff --git a/ext/standard/base64.c b/ext/standard/base64.c +index 8d4feee849..3c0b73a213 100644 --- a/ext/standard/base64.c +++ b/ext/standard/base64.c -@@ -376,8 +376,12 @@ +@@ -403,7 +403,10 @@ ZEND_INTRIN_SSSE3_FUNC_DECL(zend_string *php_base64_encode_ssse3(const unsigned ZEND_INTRIN_SSSE3_FUNC_DECL(zend_string *php_base64_decode_ex_ssse3(const unsigned char *str, size_t length, bool strict)); # endif -+ +__attribute__((used)) - zend_string *php_base64_encode_default(const unsigned char *str, size_t length); + zend_string *php_base64_encode_default(const unsigned char *str, size_t length, zend_long flags); + +__attribute__((used)) zend_string *php_base64_decode_ex_default(const unsigned char *str, size_t length, bool strict); - # if (ZEND_INTRIN_AVX2_FUNC_PROTO || ZEND_INTRIN_SSSE3_FUNC_PROTO) - PHPAPI zend_string *php_base64_encode(const unsigned char *str, size_t length) __attribute__((ifunc("resolve_base64_encode"))); ---- a/build/php.m4 -+++ b/build/php.m4 -@@ -2638,7 +2638,7 @@ - AC_MSG_CHECKING([for __builtin_cpu_init]) + # if (defined(ZEND_INTRIN_AVX2_FUNC_PROTO) || defined(ZEND_INTRIN_SSSE3_FUNC_PROTO) || defined(BASE64_INTRIN_AVX512_FUNC_PROTO) || defined(BASE64_INTRIN_AVX512_VBMI_FUNC_PROTO)) +diff --git a/ext/standard/string.c b/ext/standard/string.c +index b15a24a098..457f5833eb 100644 +--- a/ext/standard/string.c ++++ b/ext/standard/string.c +@@ -3837,6 +3837,8 @@ PHPAPI zend_string *php_addcslashes(zend_string *str, const char *what, size_t w + # include "Zend/zend_cpuinfo.h" - AC_LINK_IFELSE([AC_LANG_PROGRAM([], [[ -- return __builtin_cpu_init()? 1 : 0; -+ return (__builtin_cpu_init(), 1) ? 1 : 0; - ]])], [ - have_builtin_cpu_init=1 - AC_MSG_RESULT([yes]) + ZEND_INTRIN_SSE4_2_FUNC_DECL(zend_string *php_addslashes_sse42(zend_string *str)); ++ ++__attribute__((used)) + zend_string *php_addslashes_default(zend_string *str); + + # ifdef ZEND_INTRIN_SSE4_2_FUNC_PROTO +diff --git a/sapi/fuzzer/config.m4 b/sapi/fuzzer/config.m4 +index 21a44cd6d8..0bc19cecf0 100644 +--- a/sapi/fuzzer/config.m4 ++++ b/sapi/fuzzer/config.m4 +@@ -42,7 +42,7 @@ if test "$PHP_FUZZER" != "no"; then + [AC_MSG_ERROR([Compiler doesn't support -fsanitize=fuzzer-no-link])]) + ], [ + FUZZING_LIB=$LIB_FUZZING_ENGINE +- FUZZING_CC="$CXX -stdlib=libc++" ++ AX_CHECK_COMPILE_FLAG([-stdlib=libstdc++], [FUZZING_CC="$CXX -stdlib=libstdc++"], [FUZZING_CC="$CXX"]) + ]) + PHP_SUBST([FUZZING_LIB]) + PHP_SUBST([FUZZING_CC]) diff --git a/targets/php/pocs/PHP011.crash b/targets/php/pocs/PHP011.crash new file mode 100644 index 000000000..d1f147203 Binary files /dev/null and b/targets/php/pocs/PHP011.crash differ diff --git a/targets/php/preinstall.sh b/targets/php/preinstall.sh index 07b1eab7a..02d9dbbf1 100755 --- a/targets/php/preinstall.sh +++ b/targets/php/preinstall.sh @@ -1,5 +1,11 @@ #!/bin/bash apt-get update && \ - apt-get install -y git make autoconf automake libtool bison re2c pkg-config \ - libicu-dev + apt-get install -y git make autoconf automake libtool bison pkg-config \ + libicu-dev re2c + +# Dependency for PHP: oniguruma +git clone https://github.com/kkos/oniguruma.git \ + "$TARGET/oniguruma" + +cp -r $TARGET/oniguruma $COV/ diff --git a/targets/php/releases b/targets/php/releases new file mode 100644 index 000000000..2e0325e3c --- /dev/null +++ b/targets/php/releases @@ -0,0 +1,8 @@ +php_PIONEER="https://github.com/php/php-src.git" +php_PIONEER_STABLE_COMMIT="37190df4239cc0b4f93eb02d1748fb35ad02f63f" +php_LEGACY_2022="https://github.com/php/php-src.git" +php_LEGACY_2022_TAG="php-7.4.28" +php_LEGACY_2023="https://github.com/php/php-src.git" +php_LEGACY_2023_TAG="php-8.0.27" +php_LEGACY_2024="https://github.com/php/php-src.git" +php_LEGACY_2024_TAG="php-8.1.28" diff --git a/targets/poppler/build.sh b/targets/poppler/build.sh index 973b39c50..1e4b485b3 100755 --- a/targets/poppler/build.sh +++ b/targets/poppler/build.sh @@ -16,25 +16,18 @@ fi export WORK="$TARGET/work" rm -rf "$WORK" mkdir -p "$WORK" -mkdir -p "$WORK/lib" "$WORK/include" - -pushd "$TARGET/freetype2" -./autogen.sh -./configure --prefix="$WORK" --disable-shared PKG_CONFIG_PATH="$WORK/lib/pkgconfig" -make -j$(nproc) clean -make -j$(nproc) -make install - mkdir -p "$WORK/poppler" cd "$WORK/poppler" rm -rf * -EXTRA="" -test -n "$AR" && EXTRA="$EXTRA -DCMAKE_AR=$AR" -test -n "$RANLIB" && EXTRA="$EXTRA -DCMAKE_RANLIB=$RANLIB" - cmake "$TARGET/repo" \ - $EXTRA \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DCMAKE_CXX_EXTENSIONS=OFF \ + -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ + -DCMAKE_EXE_LINKER_FLAGS="$LDFLAGS $LIBS" \ + -DCMAKE_SHARED_LINKER_FLAGS="$LDFLAGS $LIBS" \ + -DFREETYPE_LIBRARY=/usr/lib/x86_64-linux-gnu/libfreetype.so \ -DCMAKE_BUILD_TYPE=debug \ -DBUILD_SHARED_LIBS=OFF \ -DFONT_CONFIGURATION=generic \ @@ -44,26 +37,21 @@ cmake "$TARGET/repo" \ -DENABLE_LIBPNG=ON \ -DENABLE_LIBTIFF=ON \ -DENABLE_LIBJPEG=ON \ - -DENABLE_SPLASH=ON \ - -DENABLE_UTILS=ON \ + -DENABLE_BOOST=ON \ -DWITH_Cairo=ON \ - -DENABLE_CMS=none \ + -DENABLE_UTILS=ON \ -DENABLE_LIBCURL=OFF \ -DENABLE_GLIB=OFF \ -DENABLE_GOBJECT_INTROSPECTION=OFF \ -DENABLE_QT5=OFF \ - -DENABLE_LIBCURL=OFF \ - -DWITH_NSS3=OFF \ - -DFREETYPE_INCLUDE_DIRS="$WORK/include/freetype2" \ - -DFREETYPE_LIBRARY="$WORK/lib/libfreetype.a" \ - -DICONV_LIBRARIES="/usr/lib/x86_64-linux-gnu/libc.so" \ - -DCMAKE_EXE_LINKER_FLAGS_INIT="$LIBS" + -DENABLE_QT6=OFF \ + -DENABLE_NSS3=OFF \ + -DENABLE_GPGME=OFF make -j$(nproc) poppler poppler-cpp pdfimages pdftoppm -EXTRA="" cp "$WORK/poppler/utils/"{pdfimages,pdftoppm} "$OUT/" -$CXX $CXXFLAGS -std=c++11 -I"$WORK/poppler/cpp" -I"$TARGET/repo/cpp" \ +$CXX $CXXFLAGS -std=c++20 \ + -I"$WORK/poppler/cpp" -I"$TARGET/repo/cpp" \ "$TARGET/src/pdf_fuzzer.cc" -o "$OUT/pdf_fuzzer" \ - "$WORK/poppler/cpp/libpoppler-cpp.a" "$WORK/poppler/libpoppler.a" \ - "$WORK/lib/libfreetype.a" $LDFLAGS $LIBS -ljpeg -lz \ - -lopenjp2 -lpng -ltiff -llcms2 -lm -lpthread -pthread + "$WORK/poppler/cpp/libpoppler-cpp.a" "$WORK/poppler/libpoppler.a" "/usr/lib/x86_64-linux-gnu/libfreetype.so" \ + $LDFLAGS $LIBS $LIB_FUZZING_ENGINE -ljpeg -lz -lopenjp2 -lpng -ltiff -llcms2 -lm -lpthread -pthread diff --git a/targets/poppler/build_poc.sh b/targets/poppler/build_poc.sh new file mode 100755 index 000000000..b9e15083b --- /dev/null +++ b/targets/poppler/build_poc.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" +cmake \ + -DBUILD_SHARED_LIBS=OFF \ + -DFONT_CONFIGURATION=generic \ + -DBUILD_GTK_TESTS=OFF \ + -DBUILD_QT5_TESTS=OFF \ + -DBUILD_CPP_TESTS=OFF \ + -DENABLE_LIBPNG=ON \ + -DENABLE_LIBTIFF=ON \ + -DENABLE_LIBJPEG=ON \ + -DENABLE_BOOST=ON \ + -DWITH_Cairo=ON \ + -DENABLE_UTILS=ON \ + -DENABLE_LIBCURL=OFF \ + -DENABLE_GLIB=OFF \ + -DENABLE_GOBJECT_INTROSPECTION=OFF \ + -DENABLE_QT5=OFF \ + -DENABLE_QT6=OFF \ + -DENABLE_NSS3=OFF \ + -DENABLE_GPGME=OFF + +make -j$(nproc) poppler poppler-cpp pdfimages pdftoppm +make install diff --git a/targets/poppler/patches/bugs/PDF002.patch b/targets/poppler/patches/bugs/PDF002.patch index ecccd0e47..0cb2b68db 100644 --- a/targets/poppler/patches/bugs/PDF002.patch +++ b/targets/poppler/patches/bugs/PDF002.patch @@ -1,20 +1,21 @@ diff --git a/poppler/JPEG2000Stream.cc b/poppler/JPEG2000Stream.cc -index 8e6902f..0fe7a62 100644 +index 67d28a8c..a678d3d8 100644 --- a/poppler/JPEG2000Stream.cc +++ b/poppler/JPEG2000Stream.cc -@@ -219,8 +219,15 @@ void JPXStream::init() +@@ -242,9 +242,16 @@ void JPXStream::init() } int bufSize = BUFFER_INITIAL_SIZE; +#ifdef MAGMA_ENABLE_FIXES - if (oLen.isInt() && oLen.getInt() > 0) + if (oLen.isInt() && oLen.getInt() > 0) { bufSize = oLen.getInt(); + } +#else +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", oLen.getInt() < 0); +#endif + if (oLen.isInt()) bufSize = oLen.getInt(); +#endif - + bool indexed = false; if (cspace.isArray() && cspace.arrayGetLength() > 0) { diff --git a/targets/poppler/patches/bugs/PDF003.patch b/targets/poppler/patches/bugs/PDF003.patch index e7558d44c..387697ec2 100644 --- a/targets/poppler/patches/bugs/PDF003.patch +++ b/targets/poppler/patches/bugs/PDF003.patch @@ -45,13 +45,13 @@ index 850eb10..7a89884 100644 if (colToByte(gray) == 0) invert_bits = 0x00; diff --git a/poppler/GfxState.h b/poppler/GfxState.h -index 7a6e4031..caa5c164 100644 +index 6e988fd3..ae25e3fd 100644 --- a/poppler/GfxState.h +++ b/poppler/GfxState.h -@@ -1192,6 +1192,7 @@ public: +@@ -1242,6 +1242,7 @@ public: // Get the color space. - GfxColorSpace *getColorSpace() { return colorSpace; } + GfxColorSpace *getColorSpace() { return colorSpace.get(); } + GfxColorSpace *getColorSpace2() { return colorSpace2; } // Get stream decoding info. diff --git a/targets/poppler/patches/bugs/PDF004.patch b/targets/poppler/patches/bugs/PDF004.patch index 267e15b4c..3e76d2db1 100644 --- a/targets/poppler/patches/bugs/PDF004.patch +++ b/targets/poppler/patches/bugs/PDF004.patch @@ -1,14 +1,15 @@ diff --git a/splash/SplashXPathScanner.cc b/splash/SplashXPathScanner.cc -index 042a6ef..d1ab6b1 100644 +index 3d7db3ac..e7390903 100644 --- a/splash/SplashXPathScanner.cc +++ b/splash/SplashXPathScanner.cc -@@ -455,8 +455,13 @@ void SplashXPathScanner::clipAALine(SplashBitmap *aaBuf, +@@ -449,9 +449,14 @@ void SplashXPathScanner::clipAALine(SplashBitmap *aaBuf, const int *x0, const in xx = *x0 * splashAASize; if (yy >= yyMin && yy <= yyMax) { const int intersectionIndex = splashAASize * y + yy - yMin; +#ifdef MAGMA_ENABLE_FIXES - if (unlikely(intersectionIndex < 0 || (unsigned)intersectionIndex >= allIntersections.size())) + if (unlikely(intersectionIndex < 0 || (unsigned)intersectionIndex >= allIntersections.size())) { break; + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(intersectionIndex < 0, (unsigned)intersectionIndex >= allIntersections.size())); diff --git a/targets/poppler/patches/bugs/PDF006.patch b/targets/poppler/patches/bugs/PDF006.patch index 1c2b72939..cb80cd0fd 100644 --- a/targets/poppler/patches/bugs/PDF006.patch +++ b/targets/poppler/patches/bugs/PDF006.patch @@ -1,19 +1,19 @@ diff --git a/splash/Splash.cc b/splash/Splash.cc -index 654608c..2405541 100644 +index 3d2576a7..d7119801 100644 --- a/splash/Splash.cc +++ b/splash/Splash.cc -@@ -5862,6 +5862,7 @@ SplashError Splash::blitTransparent(SplashBitmap *src, int xSrc, int ySrc, +@@ -5871,6 +5871,7 @@ SplashError Splash::blitTransparent(SplashBitmap *src, int xSrc, int ySrc, int x return splashErrZeroImage; } +#ifdef MAGMA_ENABLE_FIXES - if (src->getWidth() - xSrc < width) + if (src->getWidth() - xSrc < width) { width = src->getWidth() - xSrc; - -@@ -5879,6 +5880,15 @@ SplashError Splash::blitTransparent(SplashBitmap *src, int xSrc, int ySrc, - - if (height < 0) + } +@@ -5894,6 +5895,15 @@ SplashError Splash::blitTransparent(SplashBitmap *src, int xSrc, int ySrc, int x + if (height < 0) { height = 0; + } +#else +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(src->getWidth() - xSrc < width, \ diff --git a/targets/poppler/patches/bugs/PDF007.patch b/targets/poppler/patches/bugs/PDF007.patch index f90b9fbd7..62f25fe8c 100644 --- a/targets/poppler/patches/bugs/PDF007.patch +++ b/targets/poppler/patches/bugs/PDF007.patch @@ -1,10 +1,10 @@ diff --git a/poppler/Stream.cc b/poppler/Stream.cc -index 99ccbd2..581dcd5 100644 +index 8aee314d..515872f1 100644 --- a/poppler/Stream.cc +++ b/poppler/Stream.cc -@@ -496,9 +496,14 @@ unsigned char *ImageStream::getLine() { +@@ -667,9 +667,14 @@ unsigned char *ImageStream::getLine() } - + int readChars = str->doGetChars(inputLineSize, inputLine); +#ifdef MAGMA_ENABLE_FIXES if (unlikely(readChars == -1)) { @@ -14,6 +14,6 @@ index 99ccbd2..581dcd5 100644 +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", readChars < 0); +#endif - for (; readChars < inputLineSize; readChars++) + for (; readChars < inputLineSize; readChars++) { inputLine[readChars] = EOF; - if (nBits == 1) { + } diff --git a/targets/poppler/patches/bugs/PDF009.patch b/targets/poppler/patches/bugs/PDF009.patch index b7805b969..542c5a260 100644 --- a/targets/poppler/patches/bugs/PDF009.patch +++ b/targets/poppler/patches/bugs/PDF009.patch @@ -1,14 +1,15 @@ diff --git a/poppler/Parser.cc b/poppler/Parser.cc -index 0cb1f7f..d273504 100644 +index ce0f1fd2..c808a031 100644 --- a/poppler/Parser.cc +++ b/poppler/Parser.cc -@@ -283,8 +283,15 @@ Stream *Parser::makeStream(Object &&dict, unsigned char *fileKey, +@@ -307,9 +307,16 @@ Stream *Parser::makeStream(Object &&dict, const unsigned char *fileKey, CryptAlg // When building the xref we can't use it so use this // kludge for broken PDF files: just add 5k to the length, and // hope its enough +#ifdef MAGMA_ENABLE_FIXES - if (length < LLONG_MAX - pos - 5000) + if (length < LLONG_MAX - pos - 5000) { length += 5000; + } +#else +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", length >= LLONG_MAX - pos - 5000); diff --git a/targets/poppler/patches/bugs/PDF015.patch b/targets/poppler/patches/bugs/PDF015.patch index 46263a300..6da5cf29e 100644 --- a/targets/poppler/patches/bugs/PDF015.patch +++ b/targets/poppler/patches/bugs/PDF015.patch @@ -1,19 +1,19 @@ diff --git a/poppler/FileSpec.cc b/poppler/FileSpec.cc -index be262ad4..445931f3 100644 +index 62782603..985483f8 100644 --- a/poppler/FileSpec.cc +++ b/poppler/FileSpec.cc -@@ -97,9 +97,13 @@ bool EmbFile::save(const char *path) { - bool EmbFile::save2(FILE *f) { +@@ -104,9 +104,14 @@ bool EmbFile::save2(FILE *f) + { int c; +#ifdef MAGMA_ENABLE_FIXES - if (unlikely(!m_objStr.isStream())) + if (unlikely(!m_objStr.isStream())) { return false; -- + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", !m_objStr.isStream()); +#endif + m_objStr.streamReset(); while ((c = m_objStr.streamGetChar()) != EOF) { - fputc(c, f); diff --git a/targets/poppler/patches/bugs/PDF017.patch b/targets/poppler/patches/bugs/PDF017.patch index 5f0d3be9a..730dd42c2 100644 --- a/targets/poppler/patches/bugs/PDF017.patch +++ b/targets/poppler/patches/bugs/PDF017.patch @@ -1,10 +1,10 @@ diff --git a/fofi/FoFiTrueType.cc b/fofi/FoFiTrueType.cc -index 24624352..6a48384b 100644 +index c6cb8233..59647e4a 100644 --- a/fofi/FoFiTrueType.cc +++ b/fofi/FoFiTrueType.cc -@@ -1227,16 +1227,28 @@ void FoFiTrueType::cvtSfnts(FoFiOutputFunc outputFunc, +@@ -1373,15 +1373,27 @@ void FoFiTrueType::cvtSfnts(FoFiOutputFunc outputFunc, void *outputStream, const if ((j = seekTable(t42Tables[i].tag)) >= 0 && checkRegion(tables[j].offset, tables[j].len)) { - dumpString(file + tables[j].offset, tables[j].len, outputFunc, outputStream); + dumpString(std::span(file + tables[j].offset, tables[j].len), outputFunc, outputStream); } else if (needVerticalMetrics && i == t42VheaTable) { +#ifdef MAGMA_ENABLE_FIXES if (unlikely(length > (int)sizeof(vheaTab))) { @@ -16,18 +16,17 @@ index 24624352..6a48384b 100644 + MAGMA_LOG("%MAGMA_BUG%", length > (int)sizeof(vheaTab)); +#endif +#endif - dumpString(vheaTab, length, outputFunc, outputStream); + dumpString(vheaTab, outputFunc, outputStream); } else if (needVerticalMetrics && i == t42VmtxTable) { +#ifdef MAGMA_ENABLE_FIXES - if (unlikely(length > vmtxTabLength)) { + if (unlikely(length > (int)vmtxTab.size())) { error(errSyntaxWarning, -1, "length bigger than vmtxTab size"); - length = vmtxTabLength; } +#else +#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", length > vmtxTabLength); ++ MAGMA_LOG("%MAGMA_BUG%", length > (int)vmtxTab.size()); +#endif +#endif - dumpString(vmtxTab, length, outputFunc, outputStream); + dumpString(vmtxTab, outputFunc, outputStream); } } diff --git a/targets/poppler/patches/bugs/PDF018.patch b/targets/poppler/patches/bugs/PDF018.patch index de19cbff3..d57707732 100644 --- a/targets/poppler/patches/bugs/PDF018.patch +++ b/targets/poppler/patches/bugs/PDF018.patch @@ -1,11 +1,11 @@ diff --git a/poppler/Annot.cc b/poppler/Annot.cc -index 37165e38..095a9d09 100644 +index 0b2733b9..22df46a9 100644 --- a/poppler/Annot.cc +++ b/poppler/Annot.cc -@@ -5897,7 +5897,14 @@ void AnnotInk::draw(Gfx *gfx, bool printing) { +@@ -6624,7 +6624,14 @@ void AnnotInk::draw(Gfx *gfx, bool printing) + appearBBox->setBorderWidth(std::max(1., border->getWidth())); - for (int i = 0; i < inkListLength; ++i) { - const AnnotPath *path = inkList[i]; + for (const auto &path : inkList) { +#ifdef MAGMA_ENABLE_FIXES if (path && path->getCoordsLength() != 0) { +#else diff --git a/targets/poppler/patches/bugs/PDF019.patch b/targets/poppler/patches/bugs/PDF019.patch index 35af5de31..e224087b4 100644 --- a/targets/poppler/patches/bugs/PDF019.patch +++ b/targets/poppler/patches/bugs/PDF019.patch @@ -1,14 +1,18 @@ diff --git a/poppler/JBIG2Stream.cc b/poppler/JBIG2Stream.cc -index a861da21..fd38c198 100644 +index 0dffc111..35872893 100644 --- a/poppler/JBIG2Stream.cc +++ b/poppler/JBIG2Stream.cc -@@ -902,9 +902,14 @@ void JBIG2Bitmap::combine(JBIG2Bitmap *bitmap, int x, int y, +@@ -809,12 +809,17 @@ void JBIG2Bitmap::combine(JBIG2Bitmap *bitmap, int x, int y, unsigned int combOp oneByte = x0 == ((x1 - 1) & ~7); for (yy = y0; yy < y1; ++yy) { +#ifdef MAGMA_ENABLE_FIXES - if (unlikely((y + yy >= h) || (y + yy < 0))) + if (unlikely(checkedAdd(y, yy, &yyy))) { continue; + } + if (unlikely((yyy >= h) || (yyy < 0))) { + continue; + } +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(y + yy >= h, y + yy < 0)); @@ -16,4 +20,3 @@ index a861da21..fd38c198 100644 // one byte per line -- need to mask both left and right side if (oneByte) { - if (x >= 0) { diff --git a/targets/poppler/patches/bugs/PDF022.patch b/targets/poppler/patches/bugs/PDF022.patch index 8de7648b3..c7ddec588 100644 --- a/targets/poppler/patches/bugs/PDF022.patch +++ b/targets/poppler/patches/bugs/PDF022.patch @@ -1,8 +1,8 @@ diff --git a/poppler/GfxState.cc b/poppler/GfxState.cc -index 9aca67b3..8d5e89f2 100644 +index aa7b592d..7c863bb5 100644 --- a/poppler/GfxState.cc +++ b/poppler/GfxState.cc -@@ -2701,16 +2701,24 @@ void GfxSeparationColorSpace::getGray(const GfxColor *color, GfxGray *gray) cons +@@ -2665,16 +2665,24 @@ void GfxSeparationColorSpace::getGray(const GfxColor *color, GfxGray *gray) cons GfxColor color2; int i; @@ -26,83 +26,4 @@ index 9aca67b3..8d5e89f2 100644 +#endif } - void GfxSeparationColorSpace::getRGB(const GfxColor *color, GfxRGB *rgb) const { -@@ -2719,24 +2727,32 @@ void GfxSeparationColorSpace::getRGB(const GfxColor *color, GfxRGB *rgb) const { - GfxColor color2; - int i; - -+#ifdef MAGMA_ENABLE_FIXES - if (alt->getMode() == csDeviceGray && name->cmp("Black") == 0) { - rgb->r = clip01(gfxColorComp1 - color->c[0]); - rgb->g = clip01(gfxColorComp1 - color->c[0]); - rgb->b = clip01(gfxColorComp1 - color->c[0]); - } else { -+#else -+#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", MAGMA_AND(alt->getMode() == csDeviceGray, name->cmp("Black") == 0)); -+#endif -+#endif - x = colToDbl(color->c[0]); - func->transform(&x, c); - const int altNComps = alt->getNComps(); - for (i = 0; i < altNComps; ++i) { - color2.c[i] = dblToCol(c[i]); - } - if (unlikely(altNComps > func->getOutputSize())) { - for (i = func->getOutputSize(); i < altNComps; ++i) { - color2.c[i] = 0; - } - } - alt->getRGB(&color2, rgb); -+#ifdef MAGMA_ENABLE_FIXES - } -+#endif - } - - void GfxSeparationColorSpace::getCMYK(const GfxColor *color, GfxCMYK *cmyk) const { -@@ -2745,34 +2761,45 @@ void GfxSeparationColorSpace::getCMYK(const GfxColor *color, GfxCMYK *cmyk) cons - GfxColor color2; - int i; - -+#ifdef MAGMA_ENABLE_FIXES - if (name->cmp("Black") == 0) { - cmyk->c = 0; - cmyk->m = 0; - cmyk->y = 0; - cmyk->k = color->c[0]; - } else if (name->cmp("Cyan") == 0) { - cmyk->c = color->c[0]; - cmyk->m = 0; - cmyk->y = 0; - cmyk->k = 0; - } else if (name->cmp("Magenta") == 0) { - cmyk->c = 0; - cmyk->m = color->c[0]; - cmyk->y = 0; - cmyk->k = 0; - } else if (name->cmp("Yellow") == 0) { - cmyk->c = 0; - cmyk->m = 0; - cmyk->y = color->c[0]; - cmyk->k = 0; - } else { -+#else -+#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR(name->cmp("Black") == 0, \ -+ MAGMA_OR(name->cmp("Cyan") == 0, \ -+ MAGMA_OR(name->cmp("Magenta") == 0, \ -+ name->cmp("Yellow") == 0)))); -+#endif -+#endif - x = colToDbl(color->c[0]); - func->transform(&x, c); - for (i = 0; i < alt->getNComps(); ++i) { - color2.c[i] = dblToCol(c[i]); - } - alt->getCMYK(&color2, cmyk); -+#ifdef MAGMA_ENABLE_FIXES - } -+#endif - } - - void GfxSeparationColorSpace::getDeviceN(const GfxColor *color, GfxColor *deviceN) const { + void GfxSeparationColorSpace::getRGB(const GfxColor *color, GfxRGB *rgb) const diff --git a/targets/poppler/pocs/PDF002.crash b/targets/poppler/pocs/PDF002.crash new file mode 100644 index 000000000..127d2310f Binary files /dev/null and b/targets/poppler/pocs/PDF002.crash differ diff --git a/targets/poppler/pocs/PDF010.crash b/targets/poppler/pocs/PDF010.crash new file mode 100644 index 000000000..37f9f2b35 Binary files /dev/null and b/targets/poppler/pocs/PDF010.crash differ diff --git a/targets/poppler/pocs/PDF014.crash b/targets/poppler/pocs/PDF014.crash new file mode 100644 index 000000000..d387c1a24 Binary files /dev/null and b/targets/poppler/pocs/PDF014.crash differ diff --git a/targets/poppler/pocs/PDF018.crash b/targets/poppler/pocs/PDF018.crash new file mode 100644 index 000000000..1884b1b1e Binary files /dev/null and b/targets/poppler/pocs/PDF018.crash differ diff --git a/targets/poppler/pocs/PDF021.crash b/targets/poppler/pocs/PDF021.crash new file mode 100644 index 000000000..2be1405f4 Binary files /dev/null and b/targets/poppler/pocs/PDF021.crash differ diff --git a/targets/poppler/preinstall.sh b/targets/poppler/preinstall.sh index 276002d0a..9701de9cc 100755 --- a/targets/poppler/preinstall.sh +++ b/targets/poppler/preinstall.sh @@ -1,6 +1,6 @@ #!/bin/bash +set -e -apt-get update && \ - apt-get install -y git make autoconf automake libtool pkg-config cmake \ - zlib1g-dev libjpeg-dev libopenjp2-7-dev libpng-dev libcairo2-dev \ - libtiff-dev liblcms2-dev libboost-dev \ No newline at end of file +apt-get install -y git make autoconf automake libtool pkg-config zlib1g-dev \ + libjpeg-dev libopenjp2-7-dev libpng-dev libpixman-1-dev liblcms2-dev \ + cmake libtiff-dev libboost-dev libcairo2-dev libfreetype6 libfreetype6-dev diff --git a/targets/poppler/releases b/targets/poppler/releases new file mode 100644 index 000000000..eedbe4582 --- /dev/null +++ b/targets/poppler/releases @@ -0,0 +1,8 @@ +poppler_PIONEER="https://gitlab.freedesktop.org/poppler/poppler.git" +poppler_PIONEER_STABLE_COMMIT="221914fb062253b879c9fbf5a58e89330e679313" +poppler_LEGACY_2022="https://gitlab.freedesktop.org/poppler/poppler.git" +poppler_LEGACY_2022_TAG="poppler-22.01.0" +poppler_LEGACY_2023="https://gitlab.freedesktop.org/poppler/poppler.git" +poppler_LEGACY_2023_TAG="poppler-23.01.0" +poppler_LEGACY_2024="https://gitlab.freedesktop.org/poppler/poppler.git" +poppler_LEGACY_2024_TAG="poppler-24.01.0" diff --git a/targets/sqlite3/build.sh b/targets/sqlite3/build.sh index 08a3e611c..7d935d9f4 100755 --- a/targets/sqlite3/build.sh +++ b/targets/sqlite3/build.sh @@ -30,10 +30,10 @@ export CFLAGS="$CFLAGS -DSQLITE_MAX_LENGTH=128000000 \ "$TARGET/repo"/configure --disable-shared --enable-rtree make clean -make -j$(nproc) +make -j$(nproc) LDFLAGS.configure="$LIBS $LDFLAGS" make sqlite3.c $CC $CFLAGS -I. \ "$TARGET/repo/test/ossfuzz.c" "./sqlite3.o" \ -o "$OUT/sqlite3_fuzz" \ - $LDFLAGS $LIBS -pthread -ldl -lm + $LDFLAGS $LIBS $LIB_FUZZING_ENGINE -pthread -ldl -lm diff --git a/targets/sqlite3/build_poc.sh b/targets/sqlite3/build_poc.sh new file mode 100755 index 000000000..0df175b9a --- /dev/null +++ b/targets/sqlite3/build_poc.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +## +# Globally installs the target in the image +# Pre-requirements: +# - env TARGET: path to target work dir +# - env CC, CXX +## + +if [ ! -d "$TARGET/repo" ]; then + echo "fetch.sh must be executed first." + exit 1 +fi + +cd "$TARGET/repo" + +export CFLAGS="$CFLAGS -DSQLITE_MAX_LENGTH=128000000 \ + -DSQLITE_MAX_SQL_LENGTH=128000000 \ + -DSQLITE_MAX_MEMORY=25000000 \ + -DSQLITE_PRINTF_PRECISION_LIMIT=1048576 \ + -DSQLITE_DEBUG=1 \ + -DSQLITE_MAX_PAGE_COUNT=16384" + +./configure --disable-shared --enable-rtree +make clean +make -j$(nproc) +make sqlite3.c +make install diff --git a/targets/sqlite3/fetch.sh b/targets/sqlite3/fetch.sh deleted file mode 100755 index bce98c328..000000000 --- a/targets/sqlite3/fetch.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -## -# Pre-requirements: -# - env TARGET: path to target work dir -## - -curl "https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=8c432642572c8c4b" \ - -o "$OUT/sqlite.tar.gz" && \ -mkdir -p "$TARGET/repo" && \ -tar -C "$TARGET/repo" --strip-components=1 -xzf "$OUT/sqlite.tar.gz" \ No newline at end of file diff --git a/targets/sqlite3/patches/bugs/SQL001.patch b/targets/sqlite3/patches/bugs/SQL001.patch index 0390a2af4..8f4a39b9f 100644 --- a/targets/sqlite3/patches/bugs/SQL001.patch +++ b/targets/sqlite3/patches/bugs/SQL001.patch @@ -1,16 +1,18 @@ +diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c +index 5e0959aa8e..b0c59321ce 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c -@@ -456,9 +456,16 @@ static int fts5HashEntrySort( +@@ -463,9 +463,16 @@ static int fts5HashEntrySort( for(iSlot=0; iSlotnSlot; iSlot++){ Fts5HashEntry *pIter; for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ +#ifdef MAGMA_ENABLE_FIXES if( pTerm==0 - || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) ){ +#else +#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", MAGMA_AND(pTerm != 0, pIter->nKey+1 < nTerm)); ++ MAGMA_LOG("%MAGMA_BUG%", MAGMA_AND(pTerm != 0, pIter->nKey < nTerm)); +#endif + if(pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) { +#endif diff --git a/targets/sqlite3/patches/bugs/SQL002.patch b/targets/sqlite3/patches/bugs/SQL002.patch index b51ec4a1c..74bf1834a 100644 --- a/targets/sqlite3/patches/bugs/SQL002.patch +++ b/targets/sqlite3/patches/bugs/SQL002.patch @@ -1,20 +1,23 @@ diff --git a/src/select.c b/src/select.c -index a9ec1e852..0ce67e935 100644 +index cf25c8e678..53cc27c4cc 100644 --- a/src/select.c +++ b/src/select.c -@@ -5008,9 +5008,16 @@ static int selectExpander(Walker *pWalker, Select *p){ +@@ -6140,10 +6140,17 @@ static int selectExpander(Walker *pWalker, Select *p){ /* Process NATURAL keywords, and ON and USING clauses of joins. */ -- if( pParse->nErr || db->mallocFailed || sqliteProcessJoin(pParse, p) ){ +- assert( db->mallocFailed==0 || pParse->nErr!=0 ); +- if( pParse->nErr || sqlite3ProcessJoin(pParse, p) ){ - return WRC_Abort; +- } ++ + if( +#ifdef MAGMA_ENABLE_FIXES -+ pParse->nErr || ++ pParse->nErr || +#endif -+ db->mallocFailed || sqliteProcessJoin(pParse, p) ){ ++ db->mallocFailed || sqlite3ProcessJoin(pParse, p) ){ + return WRC_Abort; - } ++ } +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", pParse->nErr); +#endif diff --git a/targets/sqlite3/patches/bugs/SQL003.patch b/targets/sqlite3/patches/bugs/SQL003.patch index 0e36c3da6..de3b23de7 100644 --- a/targets/sqlite3/patches/bugs/SQL003.patch +++ b/targets/sqlite3/patches/bugs/SQL003.patch @@ -1,20 +1,21 @@ diff --git a/src/select.c b/src/select.c -index a9ec1e852..ad955b5de 100644 +index cf25c8e678..be9913db44 100644 --- a/src/select.c +++ b/src/select.c -@@ -3827,10 +3827,15 @@ static int flattenSubquery( - if( pSubSrc->nSrc>1 /* (3a) */ - || isAgg /* (3b) */ - || IsVirtual(pSubSrc->a[0].pTab) /* (3c) */ +@@ -4431,11 +4431,16 @@ static int flattenSubquery( + if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ + if( pSubSrc->nSrc>1 /* (3a) */ + || IsVirtual(pSubSrc->a[0].pSTab) /* (3b) */ +#ifdef MAGMA_ENABLE_FIXES - || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + || (p->selFlags & SF_Distinct)!=0 /* (3d) */ +#endif + || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ return 0; } +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", (p->selFlags & SF_Distinct)!=0); +#endif + isOuterJoin = 1; } - #ifdef SQLITE_EXTRA_IFNULLROW - else if( iFrom>0 && !isAgg ){ + diff --git a/targets/sqlite3/patches/bugs/SQL006.patch b/targets/sqlite3/patches/bugs/SQL006.patch index cfdd52943..b5e62c3ae 100644 --- a/targets/sqlite3/patches/bugs/SQL006.patch +++ b/targets/sqlite3/patches/bugs/SQL006.patch @@ -1,16 +1,17 @@ diff --git a/src/select.c b/src/select.c -index a9ec1e852..8c8c9d0bf 100644 +index cf25c8e678..9521977683 100644 --- a/src/select.c +++ b/src/select.c -@@ -6123,10 +6123,17 @@ int sqlite3Select( - */ - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct +@@ -7975,10 +7975,17 @@ int sqlite3Select( + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && OptimizationEnabled(db, SQLITE_GroupByOrder) +-#ifndef SQLITE_OMIT_WINDOWFUNC +#ifdef MAGMA_ENABLE_FIXES - #ifndef SQLITE_OMIT_WINDOWFUNC ++ #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 ++ #endif #endif -+#endif ){ +#ifdef MAGMA_ENABLE_CANARIES +#ifndef SQLITE_OMIT_WINDOWFUNC @@ -19,4 +20,4 @@ index a9ec1e852..8c8c9d0bf 100644 +#endif p->selFlags &= ~SF_Distinct; pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); - p->selFlags |= SF_Aggregate; + if( pGroupBy ){ diff --git a/targets/sqlite3/patches/bugs/SQL007.patch b/targets/sqlite3/patches/bugs/SQL007.patch index d87a574e0..944262e48 100644 --- a/targets/sqlite3/patches/bugs/SQL007.patch +++ b/targets/sqlite3/patches/bugs/SQL007.patch @@ -1,13 +1,13 @@ diff --git a/src/build.c b/src/build.c -index 73e4cea7c..593b31617 100644 +index a5deb54fc6..8d9de5bc72 100644 --- a/src/build.c +++ b/src/build.c -@@ -2236,10 +2236,15 @@ void sqlite3EndTable( +@@ -2626,10 +2626,15 @@ void sqlite3EndTable( ** table itself. So mark it read-only. */ if( db->init.busy ){ +#ifdef MAGMA_ENABLE_FIXES - if( pSelect ){ + if( pSelect || (!IsOrdinaryTable(p) && db->init.newTnum) ){ sqlite3ErrorMsg(pParse, ""); return; } diff --git a/targets/sqlite3/patches/bugs/SQL012.patch b/targets/sqlite3/patches/bugs/SQL012.patch deleted file mode 100644 index 79ccf77c9..000000000 --- a/targets/sqlite3/patches/bugs/SQL012.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/src/pragma.c b/src/pragma.c -index c5b5bb667..a882e2307 100644 ---- a/src/pragma.c -+++ b/src/pragma.c -@@ -1645,9 +1645,17 @@ void sqlite3Pragma( - if( j==pTab->iPKey ) continue; - if( pTab->aCol[j].notNull==0 ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); -+#ifndef MAGMA_ENABLE_FIXES -+#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", sqlite3VdbeGetOp(v,-1)->opcode!=OP_Column); -+#endif -+#else - if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ -+#endif - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); -+#ifdef MAGMA_ENABLE_FIXES - } -+#endif - jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); - zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, - pTab->aCol[j].zName); diff --git a/targets/sqlite3/patches/bugs/SQL014.patch b/targets/sqlite3/patches/bugs/SQL014.patch index 722fce9cd..980cf066d 100644 --- a/targets/sqlite3/patches/bugs/SQL014.patch +++ b/targets/sqlite3/patches/bugs/SQL014.patch @@ -1,18 +1,18 @@ diff --git a/src/select.c b/src/select.c -index b2dd5f149..61c4cb9f9 100644 +index cf25c8e678..4fcb84841c 100644 --- a/src/select.c +++ b/src/select.c -@@ -2833,8 +2833,12 @@ static int multiSelect( +@@ -3170,8 +3170,12 @@ static int multiSelect( } #endif } +#ifdef MAGMA_ENABLE_FIXES if( pParse->nErr ) goto multi_select_end; -- +- +#endif +#ifdef MAGMA_ENABLE_CANARIES + MAGMA_LOG("%MAGMA_BUG%", pParse->nErr != 0); +#endif - /* Compute collating sequences used by + /* Compute collating sequences used by ** temporary tables needed to implement the compound select. ** Attach the KeyInfo structure to all temporary tables. diff --git a/targets/sqlite3/patches/bugs/SQL015.patch b/targets/sqlite3/patches/bugs/SQL015.patch index 081253068..3a590a0ba 100644 --- a/targets/sqlite3/patches/bugs/SQL015.patch +++ b/targets/sqlite3/patches/bugs/SQL015.patch @@ -1,19 +1,21 @@ diff --git a/src/resolve.c b/src/resolve.c -index e90e8a768..654bf133e 100644 +index d6a5144af8..eb37ee893c 100644 --- a/src/resolve.c +++ b/src/resolve.c -@@ -182,22 +182,37 @@ static int areDoubleQuotedStringsEnabled(sqlite3 *db, NameContext *pTopNC){ +@@ -179,24 +179,38 @@ static int areDoubleQuotedStringsEnabled(sqlite3 *db, NameContext *pTopNC){ Bitmask sqlite3ExprColUsed(Expr *pExpr){ int n; Table *pExTab; + Bitmask result; n = pExpr->iColumn; + assert( ExprUseYTab(pExpr) ); pExTab = pExpr->y.pTab; assert( pExTab!=0 ); + assert( n < pExTab->nCol ); +#ifdef MAGMA_ENABLE_FIXES if( (pExTab->tabFlags & TF_HasGenerated)!=0 - && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 + && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 ){ testcase( pExTab->nCol==BMS-1 ); testcase( pExTab->nCol==BMS ); @@ -33,8 +35,7 @@ index e90e8a768..654bf133e 100644 + MAGMA_LOG("%MAGMA_BUG%", + MAGMA_AND((pExTab->tabFlags & TF_HasGenerated)!=0, + MAGMA_AND((pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0, -+ MAGMA_AND(pExTab->nCol >= BMS, -+ result < ALLBITS))) ++ MAGMA_AND(pExTab->nCol >= BMS, result < ALLBITS))) + ); +#endif +#endif diff --git a/targets/sqlite3/patches/bugs/SQL017.patch b/targets/sqlite3/patches/bugs/SQL017.patch index 92c6b6848..e47dd0cbe 100644 --- a/targets/sqlite3/patches/bugs/SQL017.patch +++ b/targets/sqlite3/patches/bugs/SQL017.patch @@ -1,49 +1,51 @@ diff --git a/src/expr.c b/src/expr.c -index 8587f5ec7..225ee6737 100644 +index 86c966683c..6ccde06009 100644 --- a/src/expr.c +++ b/src/expr.c -@@ -5514,11 +5514,24 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ - testcase( pExpr->op==TK_GE ); - /* The y.pTab=0 assignment in wherecode.c always happens after the +@@ -6670,13 +6670,26 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ ** impliesNotNullRow() test */ + assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); + assert( pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); +#ifdef MAGMA_ENABLE_FIXES - if( (pLeft->op==TK_COLUMN && ALWAYS(pLeft->y.pTab!=0) - && IsVirtual(pLeft->y.pTab)) - || (pRight->op==TK_COLUMN && ALWAYS(pRight->y.pTab!=0) - && IsVirtual(pRight->y.pTab)) + if( (pLeft->op==TK_COLUMN + && ALWAYS(pLeft->y.pTab!=0) + && IsVirtual(pLeft->y.pTab)) + || (pRight->op==TK_COLUMN + && ALWAYS(pRight->y.pTab!=0) + && IsVirtual(pRight->y.pTab)) - ){ -+ ) ++ ) +#else +#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR( -+ MAGMA_AND(pLeft->op==TK_COLUMN, pLeft->y.pTab == 0), -+ MAGMA_AND(pLeft->op!=TK_COLUMN, -+ MAGMA_AND(pRight->op==TK_COLUMN, pRight->y.pTab == 0)))); ++ MAGMA_LOG("%MAGMA_BUG%", MAGMA_OR( ++ MAGMA_AND(pLeft->op==TK_COLUMN, pLeft->y.pTab == 0), ++ MAGMA_AND(pLeft->op!=TK_COLUMN, ++ MAGMA_AND(pRight->op==TK_COLUMN, pRight->y.pTab == 0)))); +#endif + if( (pLeft->op==TK_COLUMN && IsVirtual(pLeft->y.pTab)) -+ || (pRight->op==TK_COLUMN && IsVirtual(pRight->y.pTab)) ++ || (pRight->op==TK_COLUMN && IsVirtual(pRight->y.pTab)) + ) +#endif + { return WRC_Prune; } - } + /* no break */ deliberate_fall_through diff --git a/src/sqliteInt.h b/src/sqliteInt.h -index 7a00b576a..9e7d21b7f 100644 +index b8c9136a59..d0e9ce8178 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h -@@ -2153,9 +2153,20 @@ struct Table { +@@ -2471,9 +2471,20 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE - # define IsVirtual(X) ((X)->nModuleArg) + # define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB) +#ifdef MAGMA_ENABLE_FIXES # define ExprIsVtab(X) \ - ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->nModuleArg) + ((X)->op==TK_COLUMN && (X)->y.pTab->eTabType==TABTYP_VTAB) #else +#ifdef MAGMA_ENABLE_CANARIES +# define ExprIsVtab(X) \ + (MAGMA_LOG_V("%MAGMA_BUG%", MAGMA_AND((X)->op==TK_COLUMN, (X)->y.pTab == 0)), \ -+ (X)->op==TK_COLUMN && (X)->y.pTab->nModuleArg) ++ (X)->op==TK_COLUMN && (X)->y.pTab->eTabType) +#else +# define ExprIsVtab(X) \ + ((X)->op==TK_COLUMN && (X)->y.pTab==0) diff --git a/targets/sqlite3/patches/bugs/SQL019.patch b/targets/sqlite3/patches/bugs/SQL019.patch deleted file mode 100644 index ec04a97ad..000000000 --- a/targets/sqlite3/patches/bugs/SQL019.patch +++ /dev/null @@ -1,32 +0,0 @@ -diff --git a/src/printf.c b/src/printf.c -index fc77f68df..a22223d75 100644 ---- a/src/printf.c -+++ b/src/printf.c -@@ -522,8 +522,12 @@ void sqlite3_str_vappendf( - prefix = flag_prefix; - } - if( xtype==etGENERIC && precision>0 ) precision--; -+#ifdef MAGMA_ENABLE_FIXES - testcase( precision>0xfff ); - idx = precision & 0xfff; -+#else -+ idx = precision; -+#endif - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } - if( xtype==etFLOAT ){ -@@ -585,7 +589,14 @@ void sqlite3_str_vappendf( - } - { - i64 szBufNeeded; /* Size of a temporary buffer needed */ -+#ifdef MAGMA_ENABLE_FIXES -+ szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; -+#else -+#ifdef MAGMA_ENABLE_CANARIES -+ MAGMA_LOG("%MAGMA_BUG%", MAX(e2,0) + precision > (((int)-1)&0x7fffffff) - 15 - width); -+#endif - szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; -+#endif - if( szBufNeeded > etBUFSIZE ){ - bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); - if( bufpt==0 ) return; diff --git a/targets/sqlite3/patches/graveyard/SQL012.patch b/targets/sqlite3/patches/graveyard/SQL012.patch new file mode 100644 index 000000000..19b07a9a6 --- /dev/null +++ b/targets/sqlite3/patches/graveyard/SQL012.patch @@ -0,0 +1,46 @@ +# Codebase versions used for the following explaination: +# 1. Old code: sqlite3Pragma in pragma.c (tarball: https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=8c432642572c8c4b) +# 2. New code: sqlite3Pragma in pragma.c (commit: 66985fb8fff34b7d46980202b9a245f3bb9db26a) + +# The logic behind this bug is to disable the following guard in the sqlite3Pragma functionality: +# - Only set the P5 flag if the last VDBE instruction was an OP_Column; otherwise, leave it unmodified. +# But in the updated code, in addtion to new logic that handles virtual tables, there is this snippet +# (https://github.com/sqlite/sqlite/blob/d7324103b196c572a98724a5658970b4000b8c39/src/pragma.c#L1861): + +# /* make sure there actually is a real column to fetch before emitting the two preparatory opcodes */ +# if( mxCol>=0 ){ +# // OP_Column will parse the entire record header, which sanity-checks the blob format +# // and fills the column cache for cursor iDataCur. +# sqlite3VdbeAddOp3(v, OP_Column, iDataCur, mxCol, 3); +# // sqlite3VdbeTypeofColumn(v,3) emits OP_Typeof, which reads that freshly-cached column into register +# sqlite3VdbeTypeofColumn(v, 3); +# // After these two ops, the “last opcode” is always OP_Column. +# } + +# The only time OP_Column ever appears is deep inside the “NaN-fixup” path (p1<0 case), and they no +# longer bother checking "did I just emit an OP_Column?" since it is unconditionally emitted. + +# Therefore the patch is no longer applicable on the current codebase and cannot be updated without backporting. + +diff --git a/src/pragma.c b/src/pragma.c +index c5b5bb667..a882e2307 100644 +--- a/src/pragma.c ++++ b/src/pragma.c +@@ -1645,9 +1645,17 @@ void sqlite3Pragma( + if( j==pTab->iPKey ) continue; + if( pTab->aCol[j].notNull==0 ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); ++#ifndef MAGMA_ENABLE_FIXES ++#ifdef MAGMA_ENABLE_CANARIES ++ MAGMA_LOG("%MAGMA_BUG%", sqlite3VdbeGetOp(v,-1)->opcode!=OP_Column); ++#endif ++#else + if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ ++#endif + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); ++#ifdef MAGMA_ENABLE_FIXES + } ++#endif + jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pTab->aCol[j].zName); diff --git a/targets/sqlite3/patches/graveyard/SQL019.patch b/targets/sqlite3/patches/graveyard/SQL019.patch new file mode 100644 index 000000000..2e2da17c5 --- /dev/null +++ b/targets/sqlite3/patches/graveyard/SQL019.patch @@ -0,0 +1,48 @@ +# Codebase versions used for the following explaination: +# 1. Old code: sqlite3_str_vappendf in printf.c (tarball: https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=8c432642572c8c4b) +# 2. New code: sqlite3FpDecode in util.c (commit: 66985fb8fff34b7d46980202b9a245f3bb9db26a) + +# In this bug, if one passed a huge precision, we would get a correspondingly huge idx and an equally +# large loop of while(idx >= 10). +# However, in the new code, the rounding logic is moved to sqlite3FpDecode in src/util.c and also isolated +# from the precision value itself. +# (https://github.com/sqlite/sqlite/blob/d7324103b196c572a98724a5658970b4000b8c39/src/util.c#L1017) +# There is no idx = precision; while(idx>=10){} construct in the new floating-point decoder. sqlite3FpDecode +# uses the iRound and mxRound variables for the rounding process. So, even if precision is huge, iRound is +# forced down to 16 or 26, and we you never iterate more than that many digits. Furthermore, While +# szBufNeeded can grow, it only affects allocation size and not the core decode/round performance or buffer safety. + +# Therefore the patch is no longer applicable on the current codebase and cannot be updated without backporting. + +diff --git a/src/printf.c b/src/printf.c +index fc77f68df..a22223d75 100644 +--- a/src/printf.c ++++ b/src/printf.c +@@ -522,8 +522,12 @@ void sqlite3_str_vappendf( + prefix = flag_prefix; + } + if( xtype==etGENERIC && precision>0 ) precision--; ++#ifdef MAGMA_ENABLE_FIXES + testcase( precision>0xfff ); + idx = precision & 0xfff; ++#else ++ idx = precision; ++#endif + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ +@@ -585,7 +589,14 @@ void sqlite3_str_vappendf( + } + { + i64 szBufNeeded; /* Size of a temporary buffer needed */ ++#ifdef MAGMA_ENABLE_FIXES ++ szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; ++#else ++#ifdef MAGMA_ENABLE_CANARIES ++ MAGMA_LOG("%MAGMA_BUG%", MAX(e2,0) + precision > (((int)-1)&0x7fffffff) - 15 - width); ++#endif + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; ++#endif + if( szBufNeeded > etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); + if( bufpt==0 ) return; diff --git a/targets/sqlite3/patches/setup/tclexec-ignore-stderr.patch b/targets/sqlite3/patches/graveyard/tclexec-ignore-stderr.patch similarity index 72% rename from targets/sqlite3/patches/setup/tclexec-ignore-stderr.patch rename to targets/sqlite3/patches/graveyard/tclexec-ignore-stderr.patch index 5c204e7ae..b80fd10d1 100644 --- a/targets/sqlite3/patches/setup/tclexec-ignore-stderr.patch +++ b/targets/sqlite3/patches/graveyard/tclexec-ignore-stderr.patch @@ -1,3 +1,6 @@ +// This was a bug fix for v3.37, it is already fixed in v3.50 so +// the patch is no longer needed. + --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -55,7 +55,7 @@ diff --git a/targets/sqlite3/pocs/SQL018.crash b/targets/sqlite3/pocs/SQL018.crash new file mode 100644 index 000000000..32e3753ce --- /dev/null +++ b/targets/sqlite3/pocs/SQL018.crash @@ -0,0 +1,4 @@ +--i) AS ( + SELECT 0 EXCEPT SELECT 0 ORDER BY 1 COLLATE """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""��""""""""""""""hsql_tert 15.1 { + WITH RECURSIVE + c(x) AS (VALUES(1) diff --git a/targets/sqlite3/releases b/targets/sqlite3/releases new file mode 100644 index 000000000..03349ee16 --- /dev/null +++ b/targets/sqlite3/releases @@ -0,0 +1,8 @@ +sqlite3_PIONEER="https://github.com/sqlite/sqlite" +sqlite3_PIONEER_STABLE_COMMIT="66985fb8fff34b7d46980202b9a245f3bb9db26a" +sqlite3_LEGACY_2022="https://github.com/sqlite/sqlite" +sqlite3_LEGACY_2022_TAG="relese" +sqlite3_LEGACY_2023="https://github.com/sqlite/sqlite" +sqlite3_LEGACY_2023_TAG="version-3.41.0" +sqlite3_LEGACY_2024="https://github.com/sqlite/sqlite" +sqlite3_LEGACY_2024_TAG="version-3.44.3" diff --git a/tools/captain/build.sh b/tools/captain/build.sh index 939950029..1807a8cb7 100755 --- a/tools/captain/build.sh +++ b/tools/captain/build.sh @@ -40,14 +40,17 @@ fi if [ ! -z $HARDEN ]; then harden_flag="--build-arg harden=1" fi +if [ ! -z $SOURCE_COVERAGE ]; then + coverage_flag="--build-arg source_coverage=1" +fi set -x docker build -t "$IMG_NAME" \ + --target magma_core \ --build-arg fuzzer_name="$FUZZER" \ --build-arg target_name="$TARGET" \ - --build-arg USER_ID=$(id -u $USER) \ - --build-arg GROUP_ID=$(id -g $USER) \ - $mode_flag $isan_flag $harden_flag \ + --build-arg target_version="$TARGET_VERSION" \ + $mode_flag $isan_flag $harden_flag $coverage_flag \ -f "$MAGMA/docker/Dockerfile" "$MAGMA" set +x diff --git a/tools/captain/captainrc b/tools/captain/captainrc index 466d21647..cbbd05d97 100644 --- a/tools/captain/captainrc +++ b/tools/captain/captainrc @@ -65,6 +65,15 @@ POLL=5 # (default: unset) # POC_EXTRACT=1 +# [SOURCE_COVERAGE]: if set, instrument the target for the source-based code +# coverage and obtain the coverage by replaying the corpus (default: unset) +# SOURCE_COVERAGE=1 + +# [MAGMA_DEBUG]: if set, container entrypoint is switched to a shell. The fuzzer, +# target and magma folders are mounted as volumes in a separate workdir. Useful +# for investigating build issues. +# (default: unset) +# MAGMA_DEBUG=1 ### ## Campaigns to run @@ -89,3 +98,17 @@ FUZZERS=(afl aflfast moptafl aflplusplus fairfuzz honggfuzz) # [fuzzer_CAMPAIGN_WORKERS]: overrides the global CAMPAIGN_WORKERS setting # afl_CAMPAIGN_WORKERS=3 + +# [TARGET_VERSION]: target version to be chosen from its releases file +TARGET_VERSION=PIONEER + +# [POC_MODE]: wether or not to build PoCs for the targets specified in POC_TARGETS +# This mode will not run fuzzing campaigns, but it will build targets and generate +# the specified bug PoCs using crash files provided in the target folders. +# (default: 0) +# POC_MODE=1 + +# [POC_TARGETS]: for which targets the PoCs should be built. Specific bug configs +# can be provided in the pocs/.poc file +# (default: no targets) +# POC_TARGETS=(libpng libsndfile libtiff libxml2 lua openssl php poppler sqlite3) diff --git a/tools/captain/poc/build.sh b/tools/captain/poc/build.sh new file mode 100755 index 000000000..21e5d18ef --- /dev/null +++ b/tools/captain/poc/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +## +# Build a PoC image for a specific target and bug. +# Should only be called by tools/captain/poc/run.sh. +# Pre-requirements: +# - env POC_TARGET: PoC target name (from POC_TARGETS in captainrc) +# - env BUG: PoC bug name (from target_BUGS in tools/captain/poc_configs/target.conf) +## + +if [ -z $POC_TARGET ] || [ -z $BUG ]; then + echo '$POC_TARGET and $BUG must be specified as environment variables.' + exit 1 +fi + +IMG_NAME="magma/$POC_TARGET/${BUG,,}" +MAGMA=${MAGMA:-"$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" >/dev/null 2>&1 && pwd)"} +source "$MAGMA/tools/captain/common.sh" + +set -x +docker build -t "$IMG_NAME" \ + --target magma_pocs \ + --build-arg poc_target_name="$POC_TARGET" \ + --build-arg poc_target_version="$TARGET_VERSION" \ + --build-arg poc_bug="$BUG" \ + --build-arg CRASH_PATH="$CRASH_INPUT" \ + -f "$MAGMA/docker/Dockerfile" "$MAGMA" +set +x + +echo "$IMG_NAME" diff --git a/tools/captain/poc/configs/libpng.conf b/tools/captain/poc/configs/libpng.conf new file mode 100644 index 000000000..1daa7d158 --- /dev/null +++ b/tools/captain/poc/configs/libpng.conf @@ -0,0 +1,17 @@ +### +# Configuration parameters for libtiff PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which libtiff bugs to build PoCs for. default: None +## +libpng_BUGS=(PNG001) + +## +# Commands to reproduce PoC for each specified bug +## +PNG001_COMMAND="libpng16-config --version" # TODO: replace with actual command diff --git a/tools/captain/poc/configs/libsndfile.conf b/tools/captain/poc/configs/libsndfile.conf new file mode 100644 index 000000000..61f5e395b --- /dev/null +++ b/tools/captain/poc/configs/libsndfile.conf @@ -0,0 +1,17 @@ +### +# Configuration parameters for libtiff PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which libtiff bugs to build PoCs for. default: None +## +libsndfile_BUGS=(SND001) + +## +# Commands to reproduce PoC for each specified bug +## +SND001_COMMAND="sndfile-info ${CRASH_INPUT}" diff --git a/tools/captain/poc/configs/libtiff.conf b/tools/captain/poc/configs/libtiff.conf new file mode 100644 index 000000000..685e54051 --- /dev/null +++ b/tools/captain/poc/configs/libtiff.conf @@ -0,0 +1,20 @@ +### +# Configuration parameters for libtiff PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which libtiff bugs to build PoCs for. default: None +## +libtiff_BUGS=(TIF005 TIF007 TIF008 TIF009) + +## +# Commands to reproduce PoC for each specified bug +## +TIF005_COMMAND="tiffcp ${CRASH_INPUT} ${CRASH_OUTPUT}" +TIF007_COMMAND="tiffinfo -D ${CRASH_INPUT} ${CRASH_OUTPUT}" +TIF008_COMMAND="tiffcp ${CRASH_INPUT} ${CRASH_OUTPUT}" +TIF009_COMMAND="tiffcp ${CRASH_INPUT} ${CRASH_OUTPUT}" diff --git a/tools/captain/poc/configs/libxml2.conf b/tools/captain/poc/configs/libxml2.conf new file mode 100644 index 000000000..e89a05757 --- /dev/null +++ b/tools/captain/poc/configs/libxml2.conf @@ -0,0 +1,17 @@ +### +# Configuration parameters for libtiff PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which libtiff bugs to build PoCs for. default: None +## +libxml2_BUGS=(XML001) + +## +# Commands to reproduce PoC for each specified bug +## +XML001_COMMAND="xml2-config --version" diff --git a/tools/captain/poc/configs/lua.conf b/tools/captain/poc/configs/lua.conf new file mode 100644 index 000000000..2fb6aa574 --- /dev/null +++ b/tools/captain/poc/configs/lua.conf @@ -0,0 +1,18 @@ +### +# Configuration parameters for poppler PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which poppler bugs to build PoCs for. default: None +## +lua_BUGS=(LUA002 LUA004) + +## +# Commands to reproduce PoC for each specified bug +## +LUA002_COMMAND="lua ${CRASH_INPUT}" +LUA004_COMMAND="lua ${CRASH_INPUT}" diff --git a/tools/captain/poc/configs/openssl.conf b/tools/captain/poc/configs/openssl.conf new file mode 100644 index 000000000..62ab2fce0 --- /dev/null +++ b/tools/captain/poc/configs/openssl.conf @@ -0,0 +1,17 @@ +### +# Configuration parameters for poppler PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which poppler bugs to build PoCs for. default: None +## +openssl_BUGS=(SSL001) + +## +# Commands to reproduce PoC for each specified bug +## +SSL001_COMMAND="openssl -v" diff --git a/tools/captain/poc/configs/php.conf b/tools/captain/poc/configs/php.conf new file mode 100644 index 000000000..96ea2ccae --- /dev/null +++ b/tools/captain/poc/configs/php.conf @@ -0,0 +1,17 @@ +### +# Configuration parameters for poppler PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which poppler bugs to build PoCs for. default: None +## +php_BUGS=(PHP011) + +## +# Commands to reproduce PoC for each specified bug +## +PHP011_COMMAND="php -l ${CRASH_INPUT}" diff --git a/tools/captain/poc/configs/poppler.conf b/tools/captain/poc/configs/poppler.conf new file mode 100644 index 000000000..0b27f0d2f --- /dev/null +++ b/tools/captain/poc/configs/poppler.conf @@ -0,0 +1,21 @@ +### +# Configuration parameters for poppler PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which poppler bugs to build PoCs for. default: None +## +poppler_BUGS=(PDF002 PDF010 PDF014 PDF018 PDF021) + +## +# Commands to reproduce PoC for each specified bug +## +PDF002_COMMAND="pdfimages ${CRASH_INPUT} ${CRASH_OUTPUT_NULL}" +PDF010_COMMAND="pdftoppm -mono -cropbox ${CRASH_INPUT}" +PDF014_COMMAND="pdftoppm -mono -cropbox ${CRASH_INPUT}" +PDF018_COMMAND="pdfimages ${CRASH_INPUT} ${CRASH_OUTPUT_NULL}" +PDF021_COMMAND="pdfimages ${CRASH_INPUT} ${CRASH_OUTPUT_NULL}" diff --git a/tools/captain/poc/configs/sqlite3.conf b/tools/captain/poc/configs/sqlite3.conf new file mode 100644 index 000000000..c80fc69b9 --- /dev/null +++ b/tools/captain/poc/configs/sqlite3.conf @@ -0,0 +1,17 @@ +### +# Configuration parameters for poppler PoCs +# Variables available to use in bug commands: +# - env CRASH_INPUT: /test/crash_input (from run_pocs.sh) +# - env CRASH_OUTPUT: /test/crash_output (from run_pocs.sh) +# - env CRASH_OUTPUT_NULL: /dev/null (from run_pocs.sh) +### + +## +# Which poppler bugs to build PoCs for. default: None +## +sqlite3_BUGS=(SQL018) + +## +# Commands to reproduce PoC for each specified bug +## +SQL018_COMMAND="sqlite3 :memory: < ${CRASH_INPUT}" diff --git a/tools/captain/poc/run.sh b/tools/captain/poc/run.sh new file mode 100755 index 000000000..54f19386a --- /dev/null +++ b/tools/captain/poc/run.sh @@ -0,0 +1,127 @@ +#!/bin/bash -e + +## +# Schedule the building and running of PoC images defined in POC_TARGETS +# [NOTE] This script depends on variabled from tools/captain/run.sh and +# should only be from by it. +# Pre-requirements: +# - env variables and exports from tools/captain/run.sh +## + +if [[ -z "${POC_TARGETS}" ]]; then + echo_time "POC_TARGETS is not set or empty. Nothing to do." + exit 0 +fi + +CRASH_INPUT="/test/crash_input" +CRASH_OUTPUT=/test/crash_output +CRASH_OUTPUT_NULL=/dev/null + +export POC_SUBDIR="poc" +export POC_LOGDIR="${LOGDIR}/${POC_SUBDIR}" +export POC_CACHEDIR="${CACHEDIR}/${POC_SUBDIR}" + +mkdir -p $POC_LOGDIR +mkdir -p $POC_CACHEDIR + +start_poc_run() +{ + launch_poc_run() + { + echo_time "Container $POC_TARGET/$BUG/$CACHECID started on CPU $AFFINITY" + if [ ! -z "$MAGMA_DEBUG" ]; then + "$MAGMA/tools/captain/$POC_SUBDIR/start.sh" + else + "$MAGMA/tools/captain/$POC_SUBDIR/start.sh" &> \ + "${POC_LOGDIR}/${POC_TARGET}_${BUG}_${CACHECID}_container.log" + fi + echo_time "Container $POC_TARGET/$BUG/$CACHECID stopped" + } + export -f launch_poc_run + + while : ; do + export POC_RUN_CACHEDIR="$POC_CACHEDIR/$POC_TARGET/$BUG" + export CACHECID=$(mutex $MUX_CID get_next_cid "$POC_RUN_CACHEDIR") + + errno_lock=69 + SHELL=/bin/bash flock -xnF -E $errno_lock "${POC_RUN_CACHEDIR}/${CACHECID}" \ + -c launch_poc_run || \ + if [ $? -eq $errno_lock ]; then + continue + fi + break + done +} +export -f start_poc_run + +start_ex_poc() +{ + release_workers() + { + IFS=',' + read -a workers <<< "$AFFINITY" + unset IFS + for i in "${workers[@]}"; do + rm -rf "$LOCKDIR/magma_cpu_$i" + done + } + trap release_workers EXIT + + start_poc_run + exit 0 +} +export -f start_ex_poc + + +if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Only the first PoC target is chosen." + POC_TARGETS=(${POC_TARGETS[0]}) +fi + +# schedule PoC builds +for POC_TARGET in "${POC_TARGETS[@]}"; do + export POC_TARGET + source "$MAGMA/tools/captain/$POC_SUBDIR/configs/$POC_TARGET.conf" + + unset BUGS + BUGS=($(get_var_or_default $POC_TARGET 'BUGS')) + + if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Only the first bug is chosen." + BUGS=(${BUGS[0]}) + fi + + if [[ -z "${BUGS}" ]]; then + echo_time "BUGS for $POC_TARGET is not set or empty. Nothing to do." + continue + fi + + for BUG in "${BUGS[@]}"; do + export BUG + + export BUG_COMMAND="$(get_var_or_default $BUG 'COMMAND')" + if [ -z "$BUG_COMMAND" ]; then + echo_time "No command defined for bug '$BUG', skipping." + continue + fi + + # build the Docker image + IMG_NAME="magma/$POC_TARGET/${BUG,,}" + echo_time "Building $IMG_NAME" + if ! "$MAGMA/tools/captain/$POC_SUBDIR/build.sh" &> \ + "${POC_LOGDIR}/${POC_TARGET}_${BUG}_build.log"; then + echo_time "Failed to build $IMG_NAME. Check build log for info." + continue + fi + + echo_time "Running PoCs for $POC_TARGET" + export NUMWORKERS="$(get_var_or_default $FUZZER 'CAMPAIGN_WORKERS')" + export AFFINITY=$(allocate_workers) + if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Don't run start_ex_poc as a daemon." + start_ex_poc + else + start_ex_poc & + fi + done +done diff --git a/tools/captain/poc/start.sh b/tools/captain/poc/start.sh new file mode 100755 index 000000000..327ba329e --- /dev/null +++ b/tools/captain/poc/start.sh @@ -0,0 +1,67 @@ +#!/bin/bash -e + +## +# Start a PoC container for a specific target and bug. +# Should only be called by tools/captain/poc/run.sh. +# Pre-requirements: +# - env POC_TARGET: target name (from targets/) +# - env BUG: name of the bug to reproduce (e.g. TIF009) +# - env BUG_COMMAND: the command to run in the container to reproduce the bug +# - env AFFINITY: the CPU to bind the container to (default: no affinity) +## + +cleanup() { + if [ ! -t 1 ]; then + docker rm -f $container_id &> /dev/null + fi + exit 0 +} + +trap cleanup EXIT SIGINT SIGTERM + +if [ -z "$POC_TARGET" ] || [ -z "$BUG" ] || [ -z "$BUG_COMMAND" ]; then + echo '$POC_TARGET, $BUG, and $BUG_COMMAND must be specified as' \ + 'environment variables.' + exit 1 +fi + +MAGMA=${MAGMA:-"$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" >/dev/null 2>&1 \ + && pwd)"} +export MAGMA +source "$MAGMA/tools/captain/common.sh" + +IMG_NAME="magma/$POC_TARGET/${BUG,,}" +RUN_COMMAND="$BUG_COMMAND; rc=\$?; echo \"[CONTAINER EXIT CODE] \$rc\"; exit \$rc" + +if [ ! -z $AFFINITY ]; then + flag_aff="--cpuset-cpus=$AFFINITY --env=AFFINITY=$AFFINITY" +fi + +if [ ! -z "$MAGMA_DEBUG" ]; then + flag_volume+=" --volume=$MAGMA/magma:/magma/magma" + flag_volume+=" --volume=$MAGMA/targets/$POC_TARGET:/magma/targets/$POC_TARGET/workdir" +fi + +if [ -t 1 ]; then + docker run -it --rm $flag_volume $flag_aff \ + --env=POC_TARGET=$POC_TARGET \ + --env=BUG=$BUG \ + --entrypoint=/bin/bash \ + "$IMG_NAME" +else + container_id=$( + docker run -dt $flag_volume $flag_aff \ + --env=POC_TARGET=$POC_TARGET \ + --env=BUG=$BUG \ + --network=none \ + --entrypoint=sh \ + "$IMG_NAME" \ + -c "$RUN_COMMAND" + ) + container_id=$(cut -c-12 <<< $container_id) + echo_time "Container for $POC_TARGET/$BUG started in $container_id" + docker logs -f "$container_id" & + exit_code=$(docker wait $container_id) + exit $exit_code +fi + diff --git a/tools/captain/run.sh b/tools/captain/run.sh index 5d54e90f5..6a135358e 100755 --- a/tools/captain/run.sh +++ b/tools/captain/run.sh @@ -106,10 +106,19 @@ start_campaign() mkdir -p "$SHARED" && chmod 777 "$SHARED" echo_time "Container $FUZZER/$TARGET/$PROGRAM/$ARCID started on CPU $AFFINITY" - "$MAGMA"/tools/captain/start.sh &> \ - "${LOGDIR}/${FUZZER}_${TARGET}_${PROGRAM}_${ARCID}_container.log" + if [ ! -z "$MAGMA_DEBUG" ]; then + "$MAGMA"/tools/captain/start.sh + else + "$MAGMA"/tools/captain/start.sh &> \ + "${LOGDIR}/${FUZZER}_${TARGET}_${PROGRAM}_${ARCID}_container.log" + fi echo_time "Container $FUZZER/$TARGET/$PROGRAM/$ARCID stopped" + if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Skip to extract the PoC." + unset POC_EXTRACT + fi + if [ ! -z $POC_EXTRACT ]; then "$MAGMA"/tools/captain/extract.sh fi @@ -234,11 +243,29 @@ cleanup() trap cleanup EXIT +# If PoC mode is enabled, only build PoCs for specified targets +if [ "$POC_MODE" = "1" ]; then + cd "$MAGMA/tools/captain/poc" && \ + . ./run.sh + exit 0 +fi + +if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Only the first fuzzer is chosen." + FUZZERS=(${FUZZERS[0]}) +fi + # schedule campaigns for FUZZER in "${FUZZERS[@]}"; do export FUZZER TARGETS=($(get_var_or_default $FUZZER 'TARGETS')) + + if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Only the first target is chosen." + TARGETS=(${TARGETS[0]}) + fi + for TARGET in "${TARGETS[@]}"; do export TARGET @@ -254,15 +281,31 @@ for FUZZER in "${FUZZERS[@]}"; do fi PROGRAMS=($(get_var_or_default $FUZZER $TARGET 'PROGRAMS')) + + if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Only the first program is chosen." + PROGRAMS=(${PROGRAMS[0]}) + fi + for PROGRAM in "${PROGRAMS[@]}"; do export PROGRAM export ARGS="$(get_var_or_default $FUZZER $TARGET $PROGRAM 'ARGS')" + + if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Only the first instance is running." + REPEAT=1 + fi echo_time "Starting campaigns for $PROGRAM $ARGS" for ((i=0; i<$REPEAT; i++)); do export NUMWORKERS="$(get_var_or_default $FUZZER 'CAMPAIGN_WORKERS')" export AFFINITY=$(allocate_workers) - start_ex & + if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Magma debug is enabled. Don't run start_ex as a daemon." + start_ex + else + start_ex & + fi done done done diff --git a/tools/captain/start.sh b/tools/captain/start.sh index e91a76f50..d68ac23a9 100755 --- a/tools/captain/start.sh +++ b/tools/captain/start.sh @@ -42,6 +42,12 @@ if [ ! -z $AFFINITY ]; then flag_aff="--cpuset-cpus=$AFFINITY --env=AFFINITY=$AFFINITY" fi +if [ ! -z "$MAGMA_DEBUG" ]; then + echo_time "Old entrypoint: $ENTRYPOINT" + ENTRYPOINT="/bin/bash" + echo_time "New entrypoint: $ENTRYPOINT" +fi + if [ ! -z "$ENTRYPOINT" ]; then flag_ep="--entrypoint=$ENTRYPOINT" fi @@ -51,8 +57,15 @@ if [ ! -z "$SHARED" ]; then flag_volume="--volume=$SHARED:/magma_shared" fi + +if [ ! -z "$MAGMA_DEBUG" ]; then + flag_volume+=" --volume=$MAGMA/magma:/magma/magma" + flag_volume+=" --volume=$MAGMA/fuzzers/$FUZZER:/magma/fuzzers/$FUZZER/workdir" + flag_volume+=" --volume=$MAGMA/targets/$TARGET:/magma/targets/$TARGET/workdir" +fi + if [ -t 1 ]; then - docker run -it $flag_volume \ + docker run -it --rm $flag_volume \ --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ --env=PROGRAM="$PROGRAM" --env=ARGS="$ARGS" \ --env=FUZZARGS="$FUZZARGS" --env=POLL="$POLL" --env=TIMEOUT="$TIMEOUT" \ diff --git a/tools/captain/tests/aflplusplus.auto b/tools/captain/tests/aflplusplus.auto new file mode 100644 index 000000000..eed59adf6 --- /dev/null +++ b/tools/captain/tests/aflplusplus.auto @@ -0,0 +1,24 @@ +### +## Configuration parameters +### + +source captainrc + +WORKDIR=workdir_aflplusplus +REPEAT=1 +TIMEOUT=10m + + +### +## Campaigns to run +### + +FUZZERS=( + aflplusplus +) + +aflplusplus_TARGETS=(libpng) + +### +## This script should finish in 1*10=10 minutes (one core). +### diff --git a/tools/captain/tests/aflplusplus.debug b/tools/captain/tests/aflplusplus.debug new file mode 100644 index 000000000..dd81dbd5a --- /dev/null +++ b/tools/captain/tests/aflplusplus.debug @@ -0,0 +1,24 @@ +### +## Configuration parameters +### + +source captainrc + +WORKDIR=workdir_aflplusplus +REPEAT=1 +TIMEOUT=10m +MAGMA_DEBUG=1 + +### +## Campaigns to run +### + +FUZZERS=( + aflplusplus +) + +aflplusplus_TARGETS=(libpng) + +### +## This script should finish in 1*10=10 minutes (one core). +### diff --git a/tools/captain/tests/report_2025.auto b/tools/captain/tests/report_2025.auto new file mode 100644 index 000000000..74d093d7b --- /dev/null +++ b/tools/captain/tests/report_2025.auto @@ -0,0 +1,28 @@ +### +## Configuration parameters +### + +source captainrc + +WORKDIR=workdir_report_2025 +REPEAT=10 +TIMEOUT=24h +ISAN=1 +SOURCE_COVERAGE=1 + + +### +## Campaigns to run +### + +FUZZERS=( + aflplusplus + honggfuzz + libfuzzer +) + +# By default, all targets (21 programs) will be fuzzed + +### +## This script should finish in 3*21*24 hours (one core). +### diff --git a/tools/captain/tests/report_2025.debug b/tools/captain/tests/report_2025.debug new file mode 100644 index 000000000..7a86907c4 --- /dev/null +++ b/tools/captain/tests/report_2025.debug @@ -0,0 +1,28 @@ +### +## Configuration parameters +### + +source captainrc + +WORKDIR=workdir_report_2025 +REPEAT=1 +TIMEOUT=10m +ISAN=1 +SOURCE_COVERAGE=1 +MAGMA_DEBUG=1 + +### +## Campaigns to run +### + +FUZZERS=( + aflplusplus + honggfuzz + libfuzzer +) + +# By default, all targets (21 programs) will be fuzzed + +### +## This script should finish in 3*21*24 hours (one core). +### diff --git a/tools/report_df/BenchmarkData.py b/tools/report_df/BenchmarkData.py index 8d14c9ecc..af9393ac0 100644 --- a/tools/report_df/BenchmarkData.py +++ b/tools/report_df/BenchmarkData.py @@ -3,7 +3,7 @@ import numpy as np import sys import json -from collections import Mapping, Iterable +from collections.abc import Mapping, Iterable INDEX_NAMES = ['Fuzzer', 'Target','Program','Campaign','Metric','BugID'] @@ -48,7 +48,7 @@ def update_dict(d, u): d[k] = v return d - print("Load json") + print(f"Loading benchmark data from '{filename}'...") with open(filename) as f: json_data = json.load(f) @@ -67,7 +67,7 @@ def update_dict(d, u): # save configuration parameters self._config = json_data.get('config', {}) - self._version = json_data.get('version', 'v1.0') + self._version = json_data.get('version', 'v1.3') @property def frame(self): diff --git a/tools/report_df/CoverageData.py b/tools/report_df/CoverageData.py new file mode 100644 index 000000000..010601808 --- /dev/null +++ b/tools/report_df/CoverageData.py @@ -0,0 +1,371 @@ +""" +This script can be used standalone to generate an HTML coverage report from +workdir/ar directory: + python CoverageData.py --root_dir ../captain//ar --cache + +It can alsoe be imported to get coverage data for markdown report: + from CoverageData import get_md_report_data + cov_data = get_md_report_data() +""" +import os +import json +import tarfile +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd +import plotly.express as px +import plotly.graph_objects as go + +ROOT_DIR = "../captain/workdir/ar" +OUT_DIR = "out/cov" +OUT_FILE = f"{OUT_DIR}/coverage.html" +USE_CACHE = False +CACHE_PATH = f"{OUT_DIR}/final_coverage.pkl" +COVERAGE_OVERTIME_CACHE_PATH = f"{OUT_DIR}/coverage_overtime.pkl" +TARGETS = [] + +IMG_DIR = f"{OUT_DIR}/assets" +IMG_FMT = "svg" + +records = [] +records_overtime = [] +figures = [] + + +def _load_cov_data(target_dirs: list): + # Step 1: Load data into DataFrame + for root, _, files in os.walk(ROOT_DIR): + if "coverage-reports.json" not in files and "ball.tar" in files: + tar_path = os.path.join(root, "ball.tar") + try: + with tarfile.open(tar_path) as tar: + tar.extractall(path=root) + print(f"Extracted ball.tar in {root}") + files = os.listdir(root) # refresh the file list after extraction + if "coverage-reports.json" not in files: + print(f"Failed to obtain coverage reports of {root}") + except Exception as e: + print(f"Failed to extract {tar_path}: {e}") + if "coverage-reports.json" in files or "coverage_overtime.txt" in files: + try: + parts = os.path.normpath(root).split(os.sep) + fuzzer, target, program, run = parts[-4:] + except ValueError: + continue + if "coverage-reports.json" in files: + json_path = os.path.join(root, "coverage-reports.json") + html_path = os.path.join(root, "coverage-reports/index.html") + parts = os.path.normpath(json_path).split(os.sep) + with open(json_path, "r") as f: + try: + data = json.load(f) + totals = data["data"][0]["totals"]["branches"] + record = { + "fuzzer": "afl++" if fuzzer == "aflplusplus" else fuzzer, + "target": target, + "program": program, + "run": run, + "percent": totals["percent"], + "covered": totals["covered"], + "count": totals["count"], + "html_path": html_path, + } + if not len(target_dirs) or target in target_dirs: + records.append(record) + except Exception as e: + print(f"Error parsing {json_path}: {e}") + if "coverage_overtime.txt" in files: + txt_path = os.path.join(root, "coverage_overtime.txt") + try: + df = pd.read_csv(txt_path) + df["timestamp"] = pd.to_numeric(df["timestamp"], errors="coerce") + df["timestamp"] = pd.to_datetime( + df["timestamp"], unit="s", errors="coerce" + ) + df = df.dropna(subset=["timestamp"]) + + df["timestamp"] = df["timestamp"].dt.floor("s") + df = df.drop_duplicates(subset="timestamp", keep="last") + df = df.sort_values("timestamp").reset_index(drop=True) + df = df.ffill() + + start_time = df["timestamp"].iloc[0] + df["timestamp"] = (df["timestamp"] - start_time).dt.total_seconds() + + df["fuzzer"] = "afl++" if fuzzer == "aflplusplus" else fuzzer + df["target"] = target + df["program"] = program + df["run"] = run + if not len(target_dirs) or target in target_dirs: + records_overtime.append(df) + except Exception as e: + print(f"Failed to load {txt_path}: {e}") + + try: + df = pd.DataFrame(records) + df_overtime = pd.concat(records_overtime, ignore_index=True) + return df, df_overtime + except Exception as e: + print(f"Error creating DataFrame: {e}. Is the target directory correct?") + exit(1) + + +def get_overtime_stats( + fuzzer_group: pd.DataFrame, time_grid: np.ndarray, stat_type: str +): + interpolated_runs = [] + for _, run_group in fuzzer_group.groupby("run"): + run_group = run_group.sort_values("timestamp") + interp = np.interp(time_grid, run_group["timestamp"], run_group[stat_type]) + interpolated_runs.append(interp) + + interp_runs_arr = np.array(interpolated_runs) + return { + "median": np.median(interp_runs_arr, axis=0), + "min": np.min(interp_runs_arr, axis=0), + "max": np.max(interp_runs_arr, axis=0), + } + + +def add_overtime_fuzzer_line( + figure: go.Figure, time_grid: np.ndarray, stats: dict, fuzzer: str +): + figure.add_trace( + go.Scatter( + x=time_grid, + y=stats["median"], + mode="markers" if len(time_grid) == 1 else "lines", + name=f"{fuzzer} avg", + line=dict(width=2), + ) + ) + figure.add_trace( + go.Scatter( + x=np.concatenate([time_grid, time_grid[::-1]]), + y=np.concatenate([stats["min"], stats["max"][::-1]]), + fill="toself", + fillcolor="rgba(0,100,200,0.1)", + line=dict(color="rgba(255,255,255,0)"), + hoverinfo="skip", + name=f"{fuzzer} range", + showlegend=False, + ) + ) + + +def fig_to_img_tag(fig: go.Figure, name: str) -> str: + safe = "".join(c if c.isalnum() or c in "-_." else "_" for c in name) + out_path = os.path.join(IMG_DIR, f"{safe}.{IMG_FMT}") + fig.write_image(out_path, format=IMG_FMT, scale=2) + out_path = out_path.replace(f"{OUT_DIR}/", "") + return f'{safe}' + + +def gen_figures(df: pd.DataFrame, df_overtime: pd.DataFrame): + for (target, program), group in df.groupby(["target", "program"]): + # 1. Coverage Percentage Figure (Y = percent) + plot_df = group[["fuzzer", "percent"]].dropna() + fig_percent = px.box(plot_df, x="fuzzer", y="percent", points="all") + + # 2. Covered Branches Figure (Y = covered) + plot_df = group[["fuzzer", "covered"]].dropna() + fig_covered = px.box(plot_df, x="fuzzer", y="covered", points="all") + + df_overtime_program = df_overtime.query( + "target == @target and program == @program" + ) + time_grid = np.arange(0, df_overtime_program["timestamp"].max() + 1, 1) + + fig_percent_overtime = go.Figure() + fig_covered_overtime = go.Figure() + for fuzzer, fuzzer_group in df_overtime_program.groupby("fuzzer"): + # 3. Coverage Overtime Figure (Y = percent) + stats_p = get_overtime_stats(fuzzer_group, time_grid, "percent") + add_overtime_fuzzer_line(fig_percent_overtime, time_grid, stats_p, fuzzer) + + # 4. Covered Branches Figure (Y = covered) + stats_c = get_overtime_stats(fuzzer_group, time_grid, "covered") + add_overtime_fuzzer_line(fig_covered_overtime, time_grid, stats_c, fuzzer) + + fig_percent_overtime.update_layout(xaxis_title="Time", yaxis_title="Percent") + fig_covered_overtime.update_layout(xaxis_title="Time", yaxis_title="Covered") + + # Collect both figures for display + links = group.apply( + lambda row: f'{row["fuzzer"]} / {row["run"]}', + axis=1, + ).tolist() + + base = f"{target}_{program}" + img_percent = fig_to_img_tag(fig_percent, f"{base}_percent_box") + img_covered = fig_to_img_tag(fig_covered, f"{base}_covered_box") + + if not df_overtime_program.empty and pd.notna( + df_overtime_program["timestamp"].max() + ): + img_percent_overtime = fig_to_img_tag( + fig_percent_overtime, f"{base}_percent_overtime" + ) + img_covered_overtime = fig_to_img_tag( + fig_covered_overtime, f"{base}_covered_overtime" + ) + else: + img_percent_overtime = "No overtime data" + img_covered_overtime = "No overtime data" + + figures.append( + { + "key": f"{target}/{program}", + "fig_percent": img_percent, + "fig_covered": img_covered, + "fig_percent_overtime": img_percent_overtime, + "fig_covered_overtime": img_covered_overtime, + "links": links, + } + ) + print(f"Generated figures for {target}/{program} with {len(group)} runs.") + + +def plot_figures(): + html_parts = [ + "Coverage Summary", + """ + + """, + "", + "

Coverage Summary

", + "", + ] + + for fig in figures: + html_parts.append(f"") + html_parts.append("") + html_parts.append(f"") + html_parts.append(f"") + html_parts.append("") + html_parts.append(f"") + html_parts.append(f"") + html_parts.append("") + html_parts.append("") + + html_parts.append("

{fig['key']}

{fig['fig_percent']}{fig['fig_covered']}
{fig['fig_percent_overtime']}{fig['fig_covered_overtime']}
    ") + for link in fig["links"]: + html_parts.append(f"
  • {link}
    • ") + html_parts.append("
") + + with open(OUT_FILE, "w") as f: + f.write("\n".join(html_parts)) + print("HTML report generated.") + + +def gen_cov_report(df: pd.DataFrame, df_overtime: pd.DataFrame): + gen_figures(df, df_overtime) + plot_figures() + + +def init_dirs(): + out_dir = Path(OUT_DIR) + out_dir.mkdir(parents=True, exist_ok=True) + + img_dir = Path(IMG_DIR) + img_dir.mkdir(parents=True, exist_ok=True) + for ext in ("*.png", "*.svg"): + for f in img_dir.glob(ext): + f.unlink() + + +def get_args(): + parser = argparse.ArgumentParser( + description="Generate HTML coverage report.(Google chrome or chromium is needed to export images.)" + ) + parser.add_argument( + "--root_dir", help="Root directory containing coverage data", default=ROOT_DIR + ) + parser.add_argument("--out_file", help="Output HTML file name", default=OUT_FILE) + parser.add_argument( + "--target", + help="Specify one or more targets (default=all)", + nargs="+", + default=TARGETS, + ) + parser.add_argument( + "--cache", + help="Use cached data if available", + action="store_true", + default=USE_CACHE, + ) + return parser.parse_args() + + +def load_data(): + init_dirs() + + if USE_CACHE and os.path.exists(CACHE_PATH): + print("Loading cached DataFrame...") + df = pd.read_pickle(CACHE_PATH) + df_overtime = pd.read_pickle(COVERAGE_OVERTIME_CACHE_PATH) + else: + print(f"Scanning {ROOT_DIR} for coverage-reports.json ...") + df, df_overtime = _load_cov_data(TARGETS) + df.to_pickle(CACHE_PATH) + df_overtime.to_pickle(COVERAGE_OVERTIME_CACHE_PATH) + print("Saved DataFrame to cache.") + + return df, df_overtime + +def get_md_report_data(): + df, df_overtime = load_data() + gen_figures(df, df_overtime) + print("Generated figures for markdown report.") + + cov_data = {} + + for figure in figures: + parent, child = figure["key"].split('/') + if parent not in cov_data: + cov_data[parent] = {} + cov_data[parent][child] = figure + + return cov_data + +if __name__ == "__main__": + args = get_args() + + ROOT_DIR = args.root_dir + OUT_FILE = args.out_file + USE_CACHE = args.cache + TARGETS = args.target + + df, df_overtime = load_data() + gen_cov_report(df, df_overtime) diff --git a/tools/report_df/DataProcessing.py b/tools/report_df/DataProcessing.py index dec3c6aa5..da7973ef9 100644 --- a/tools/report_df/DataProcessing.py +++ b/tools/report_df/DataProcessing.py @@ -20,7 +20,11 @@ def average_time_to_metric_data(bd,metric) : #Select only bugs that satisfy the metric and then calculate the mean time by grouping df = bd.get_frame() - average_time = df.iloc[df.index.get_level_values('Metric') == metric].mean(level=['Fuzzer','Target','Program']) + df_sel = df.loc[df.index.get_level_values('Metric') == metric] + average_time = ( + df_sel.groupby(level=['Fuzzer', 'Target', 'Program'])['Time'].mean() + ) + return average_time def expected_time_to_trigger_data(bd) : @@ -125,7 +129,7 @@ def compute_p_values(series): index = pd.MultiIndex.from_product([fuzzer_label,fuzzer_label],names=['Outer','Inner']) #Creation of the p_value Dataframe with the previously computed index. #Index has to been reset such that the multi index values can be passed as argument to the lambda function - p_values = pd.DataFrame(index=index,columns=['p_value']).fillna(np.nan) + p_values = pd.DataFrame(index=index,columns=['p_value'], dtype=float).fillna(np.nan) p_values = p_values.apply(lambda f: two_sided_test(*f.name,fuzzer_data), axis=1) #Unstacking the Dataframe gives it the expected 2 dimensional shape for a p_value array return p_values.unstack() @@ -141,7 +145,7 @@ def two_sided_test(f1, f2, fuzzer_data): return ss.mannwhitneyu(fuzzer_data[f1],fuzzer_data[f2], alternative='two-sided').pvalue df = bd.frame - df = df.iloc[df.index.get_level_values('Metric') == metric] + df = df.loc[df.index.get_level_values('Metric') == metric] #Extract the number of unique bugs per campaign unique_bugs = df.reset_index().groupby(['Fuzzer','Target','Campaign'])['BugID'].nunique() #Unstack and stack back to fill the missing campaign values in case there is @@ -149,7 +153,7 @@ def two_sided_test(f1, f2, fuzzer_data): #This is needed because the Mann-Whitney U-test requires the same number of #samples for both sample sets. unique_bugs = unique_bugs.unstack('Campaign', fill_value=0) \ - .stack(level=0) + .stack(level=0, future_stack=True) agg = unique_bugs.groupby(['Fuzzer', 'Target']) \ .apply(lambda d: pd.DataFrame([d.mean(), d.std()], @@ -175,7 +179,7 @@ def number_of_unique_bugs_found_data(bd): df = bd.frame #Extracting all found bugs - df_triggered = df.iloc[df.index.get_level_values('Metric') == Metric.TRIGGERED.value] + df_triggered = df.loc[df.index.get_level_values('Metric') == Metric.TRIGGERED.value] #Reseting the index is necessary to get the number of unique bugs triggered by each fuzzer num_trigg = df_triggered.reset_index().groupby(['Fuzzer'])['BugID'].nunique().to_frame() num_trigg.columns = ['Bugs'] @@ -203,7 +207,7 @@ def bug_list(bd,fuzzer,target,metric): df = bd.get_frame() #Extracting the bug fullfilling the metric by putting there metric times into a list - df_bugs = df.iloc[df.index.get_level_values('Metric') == metric] + df_bugs = df.loc[df.index.get_level_values('Metric') == metric] df_bugs = df_bugs.loc[fuzzer,target].groupby('BugID')['Time'].apply(list) #Preparing the new index to be the bugs index = df_bugs.index.tolist() @@ -232,7 +236,7 @@ def get_step_value(fuzzer,x_values, campaigns): """ #Extracting data for the correct fuzzer - df_fuzz = campaigns.iloc[campaigns.index.get_level_values('Fuzzer') == fuzzer] + df_fuzz = campaigns.loc[campaigns.index.get_level_values('Fuzzer') == fuzzer] campaigns_data = df_fuzz.groupby(['Campaign'])['Time'].apply(lambda x : sorted(list(x))) num_campaigns = len(campaigns_data.index) @@ -251,8 +255,8 @@ def step_val(series,x): df = bd.get_frame() - df_metric = df.iloc[df.index.get_level_values('Metric') == metric] - df_lib = df_metric.iloc[df_metric.index.get_level_values('Target') == target] + df_metric = df.loc[df.index.get_level_values('Metric') == metric] + df_lib = df_metric.loc[df_metric.index.get_level_values('Target') == target] #For each unique BugID in each campaign in multiple Programs, only retain the smallest time to metric df_lib = df_lib.groupby(['Fuzzer','Target','Campaign','BugID']).min() @@ -310,7 +314,7 @@ def fillmissing(group, supergroup_name): 'Metric': metric, 'BugID': bug }) - group = group.append(new_row, ignore_index=True) + group = pd.concat([group, new_row.to_frame().T], ignore_index=True) return group name = group.name @@ -329,26 +333,31 @@ def fillmissing(group, supergroup_name): 'Metric': 'triggered' }), ] - group = group.append(new_rows, ignore_index=True) - - group = group.groupby('Fuzzer').apply(fillmissing, name).reset_index(drop=True) - - subgroups = group.groupby(['Fuzzer','Metric']).apply(fit_kmf_one, name, N) + group = pd.concat([group, pd.DataFrame(new_rows)], ignore_index=True) + + group = group.groupby('Fuzzer')[ + ['Fuzzer', 'Target', 'Program', 'Campaign', 'Metric', 'BugID', 'Time'] + ].apply(fillmissing, name, include_groups=False).reset_index(drop=True) + + subgroups = group.groupby(['Fuzzer','Metric'])[ + ['Fuzzer', 'Target', 'Program', 'Campaign', 'Metric', 'BugID', 'Time'] + ].apply(fit_kmf_one, name, N, include_groups=False) return subgroups df = bd.frame N = df.reset_index().groupby(['Fuzzer', 'Target', 'Program'])['Campaign'].nunique() kmf = df.reset_index() \ - .groupby(['Target', 'Program', 'BugID']) \ - .apply(fit_kmf_all, N) + .groupby(['Target', 'Program', 'BugID'])[ + ['Fuzzer', 'Target', 'Program', 'Campaign', 'Metric', 'BugID', 'Time'] + ].apply(fit_kmf_all, N, include_groups=False) # get the mean survival time for every (target, program, bug, fuzzer, metric) tuple - means = kmf.applymap(lambda k: restricted_mean_survival_time(k, bd.duration)) + means = kmf.map(lambda k: restricted_mean_survival_time(k, bd.duration)) # re-arrange the dataframe such that the columns are the metrics - means = means.stack(level=0) + means = means.stack(level=0, future_stack=True) # for every (target, bug, fuzzer) tuple, select the row corresponding to the program where the bug was triggered earliest means = means.loc[means.groupby(['Target', 'BugID', 'Fuzzer'])[Metric.TRIGGERED.value].idxmin()] # re-arrange dataframe so that index is (target, bug) and columns are (fuzzer, metric) - means = means.droplevel('Program').stack().unstack(-2).unstack() + means = means.droplevel('Program').stack(future_stack=True).unstack(-2).unstack() return kmf, means diff --git a/tools/report_df/MatplotlibPlotter.py b/tools/report_df/MatplotlibPlotter.py index 94ddfea56..e7b1b91c3 100644 --- a/tools/report_df/MatplotlibPlotter.py +++ b/tools/report_df/MatplotlibPlotter.py @@ -39,7 +39,7 @@ def expected_time_to_trigger(bd, outdir): fuzzer_label = list(ett.columns) bug_label = list(ett.index) annotations = ett.copy() - annotations[fuzzer_label] = annotations[fuzzer_label].applymap(lambda x: pp_time(x)) + annotations[fuzzer_label] = annotations[fuzzer_label].map(pp_time) fig, ax = plt.subplots(figsize=(10,10)) plt.yticks(rotation=0) #Norm factor has to been precomputed @@ -109,7 +109,7 @@ def unique_bugs_per_target(bd, outdir, metric, libraries=None, symmetric=False, for ax in axs.flat[len(libraries):]: fig.delaxes(ax) - fig.tight_layout(pad=2.0) + fig.set_constrained_layout(True) sigmatrix, path = output(outdir, 'plot', 'summary_signplot.svg') fig.savefig(path, bbox_inches='tight') @@ -266,26 +266,25 @@ def series_to_mask(series, df): mask = mask.unstack().apply(lambda x: x == x.name) return mask.reindex_like(df) - uniq_min = df.stack(level=0).groupby(level=0) \ - .apply(lambda x: x[x == x.min()].count() == 1).stack() - uniq_max = df.stack(level=0).groupby(level=0) \ - .apply(lambda x: x[x == x.max()].count() == 1).stack() + uniq_min = df.stack(level=0, future_stack=True).groupby(level=0) \ + .apply(lambda x: x[x == x.min()].count() == 1).stack(future_stack=True) + uniq_max = df.stack(level=0, future_stack=True).groupby(level=0) \ + .apply(lambda x: x[x == x.max()].count() == 1).stack(future_stack=True) # filter out entries which do not need to be highlighted - mins = df.stack().groupby(level=0, as_index=False) \ - .idxmin(axis=1).droplevel(0) - mins = mins[uniq_min == True] - maxs = df.stack().groupby(level=0, as_index=False) \ - .idxmax(axis=1).droplevel(0) - maxs = maxs[uniq_max == True] + mins = df.stack(level=1, future_stack=True).idxmin(axis=1) + mins = mins[uniq_min] + + maxs = df.stack(level=1, future_stack=True).idxmax(axis=1) + maxs = maxs[uniq_max] survivals = df == bd.duration - color_df = series_to_mask(mins, df.stack()).unstack() \ - .applymap(lambda x: 'background-color: #bfe8c2' if x else None) - color_df.update(series_to_mask(maxs, df.stack()).unstack() \ - .applymap(lambda x: 'background-color: #e2ea67' if x else None)) + color_df = series_to_mask(mins, df.stack(future_stack=True)).unstack() \ + .map(lambda x: 'background-color: #bfe8c2' if x else None) + color_df.update(series_to_mask(maxs, df.stack(future_stack=True)).unstack() \ + .map(lambda x: 'background-color: #e2ea67' if x else None)) color_df.update(survivals \ - .applymap(lambda x: 'background-color: #efa2a2' if x else None)) + .map(lambda x: 'background-color: #efa2a2' if x else None)) color_df.fillna('', inplace=True) color_df = color_df.reindex_like(df) return color_df @@ -293,7 +292,7 @@ def series_to_mask(series, df): # adjust dataframe for better presentation means = means.droplevel('Target') - agg = means.stack(0).groupby('BugID') \ + agg = means.stack(0, future_stack=True).groupby('BugID') \ .apply(lambda x: pd.Series( { Metric.REACHED.value: x[Metric.REACHED.value].mean(), @@ -303,13 +302,21 @@ def series_to_mask(series, df): agg.columns = pd.MultiIndex.from_product([['Aggregate'], [Metric.REACHED.value, Metric.TRIGGERED.value]]) means = means.join(agg) - means = means.stack() \ - .sort_values( - by='Fuzzer', - ascending=False, - axis='columns', - key=lambda idx: [means[(f, Metric.TRIGGERED.value)][means[(f, Metric.TRIGGERED.value)] < bd.duration].count() for f in idx]) \ - .unstack() + stacked = means.stack(future_stack=True) + + def _sort_key(cols): + out = [] + for f in cols: + if f == 'Aggregate': + out.append((-1,)) + else: + cnt = (means[(f, Metric.TRIGGERED.value)] < bd.duration).sum() + out.append((cnt,)) + return out + + stacked = stacked.sort_index(axis='columns', key=_sort_key, ascending=False) + means = stacked.unstack() + means.sort_values(by=('Aggregate', Metric.TRIGGERED.value), inplace=True) means.drop(columns='Aggregate', level=0, inplace=True) @@ -390,17 +397,17 @@ def series_to_mask(series, df): hiliter.template = style_tpl heatmap.template = style_tpl - table_html = re.sub(r'colspan=(\d+)', r'colspan="\1"', styler.render()) + table_html = re.sub(r'colspan=(\d+)', r'colspan="\1"', styler.to_html()) table_name, path = output(outdir, 'data', 'mean_survival.html') with open(path, 'w') as f: f.write(table_html) - hiliter_css = '\n'.join(hiliter.render().split('\n')[1:-1]) + '}' + hiliter_css = '\n'.join(hiliter.to_html().split('\n')[1:-1]) + '}' hiliter_name, path = output(outdir, 'css', 'survival_hiliter.css') with open(path, 'w') as f: f.write(hiliter_css) - heatmap_css = '\n'.join(heatmap.render().split('\n')[1:-1]) + '}' + heatmap_css = '\n'.join(heatmap.to_html().split('\n')[1:-1]) + '}' heatmap_name, path = output(outdir, 'css', 'survival_heatmap.css') with open(path, 'w') as f: f.write(heatmap_css) diff --git a/tools/report_df/README.md b/tools/report_df/README.md new file mode 100644 index 000000000..45be12a1c --- /dev/null +++ b/tools/report_df/README.md @@ -0,0 +1,86 @@ +# Steps to generate magma reports + +1. Create and activate a virtualenv (optional) + +2. Install required packages: + ``` + $ cd tools && pip3 install -r requirements.txt + ``` + +3. Generate benchmark JSON data: + ``` + $ python3 benchd/exp2json.py -h + usage: exp2json.py [-h] [--workers WORKERS] [-v] workdir outfile + + Collects data from the experiment workdir and outputs a summary as a JSON file. + + positional arguments: + workdir The path to the Captain tool output workdir. + outfile The file to which the output will be written, or - for stdout. + + options: + -h, --help show this help message and exit + --workers WORKERS The number of concurrent processes to launch. + -v, --verbose Controls the verbosity of messages. -v prints info. -vv prints debug. Default: warnings and higher. + ``` + + For example, + ``` + $ cd benchd + $ python3 exp2json.py --workers $(nproc) /path/to/workdir out.json + ``` + +4. Generate all reports: + ``` + $ python3 report_df/main.py -h + usage: main.py [-h] [-v] [-t TRIALS] [-d DURATION] json outdir + + Creates detailed plots from experiment summary and generates a report for the Magma website. + + positional arguments: + json The experiment summary JSON file generated by the benchd tool. + outdir The path to the directory where webpage output and hierarchy will be stored. + + options: + -h, --help show this help message and exit + -v, --verbose Controls the verbosity of messages. -v prints info. -vv prints debug. Default: warnings and + higher. + -t TRIALS, --trials TRIALS + Number of trials to consider in the analysis. Default: 10 + -d DURATION, --duration DURATION + Duration in days for the benchmark analysis. Default: 7 + ``` + + For example, + ``` + $ cd report_df + $ python3 report_df/main.py ../benchd/out.json out --trials 5 --duration 1 + ``` + Trials and duration are equal to the `REPEAT` and `TIMEOUT` values set in the fuzzing campaign. + + *Note: Coverage reports need Chrome for generating images with `plotly`. If not already present run, `plotly_get_chrome` from the commandline in the virtualenv after installing `kaledio`.* + + +## Generating only coverage reports + +It is possible to generate only coverage reports as a separate HTML file. + +``` +$ python3 report_df/CoverageData.py -h +usage: CoverageData.py [-h] [--root_dir ROOT_DIR] [--out_file OUT_FILE] [--target TARGET [TARGET ...]] [--cache] + +Generate HTML coverage report.(Google chrome or chromium is needed to export images.) + +options: + -h, --help show this help message and exit + --root_dir ROOT_DIR Root directory containing coverage data + --out_file OUT_FILE Output HTML file name + --target TARGET [TARGET ...] + Specify one or more targets (default=all) + --cache Use cached data if available +``` + +For example, +``` +$ python3 CoverageData.py --target libpng lua --cache +``` diff --git a/tools/report_df/ReportGeneration.py b/tools/report_df/ReportGeneration.py index 3e90da812..66f30021f 100644 --- a/tools/report_df/ReportGeneration.py +++ b/tools/report_df/ReportGeneration.py @@ -1,6 +1,7 @@ import jinja2 import MatplotlibPlotter from Metric import Metric +from CoverageData import get_md_report_data import os import errno @@ -42,6 +43,11 @@ def generate_fuzzer_page(bd, base, env, fuzzer, **kwargs): html = template.render(base_template=base, fuzzer=fuzzer, **kwargs) return html +def generate_coverage_page(base, env, **kwargs): + template = env.get_template('coverage_template.md') + html = template.render(base_template=base, **kwargs) + return html + def generate_report(bd, outdir, report_title="Report", **kwargs): def ensure_dir(path): try: @@ -55,11 +61,14 @@ def ensure_dir(path): ensure_dir(os.path.join(outdir, 'plot')) ensure_dir(os.path.join(outdir, 'fuzzers')) ensure_dir(os.path.join(outdir, 'targets')) + ensure_dir(os.path.join(outdir, 'cov')) boxplots = MatplotlibPlotter.bug_metric_boxplot(bd, outdir) + print("Generated boxplots.") uniq_bugs, sigmatrix = MatplotlibPlotter.unique_bugs_per_target(bd, outdir, Metric.TRIGGERED.value) ett = MatplotlibPlotter.expected_time_to_trigger(bd, outdir) survplots, survlegend, survtable, hiliter_css, heatmap_css = MatplotlibPlotter.bug_survival_plots(bd, outdir) + print("Generated survival plots.") ppool = locals() env = jinja2.Environment(loader=jinja2.ChoiceLoader( @@ -98,4 +107,10 @@ def ensure_dir(path): for target, html in targets.items(): with open(os.path.join(outdir, 'targets', f'{target}.md'), 'w') as f: - f.write(html) \ No newline at end of file + f.write(html) + + print(f"Generated markdown pages at path '{outdir}'.\n\n") + + with open(os.path.join(outdir, 'cov', 'coverage.md'), 'w') as f: + cov_data = get_md_report_data() + f.write(generate_coverage_page(base_template, env, cov_data=cov_data)) \ No newline at end of file diff --git a/tools/report_df/main.py b/tools/report_df/main.py index 7b6dd338b..db8527ecb 100644 --- a/tools/report_df/main.py +++ b/tools/report_df/main.py @@ -23,6 +23,10 @@ def parse_args(): help=("Controls the verbosity of messages. " "-v prints info. -vv prints debug. Default: warnings and higher.") ) + parser.add_argument('-t', '--trials', type=int, default=10, + help="Number of trials to consider in the analysis. Default: 10") + parser.add_argument('-d', '--duration', type=int, default=7, + help="Duration in days for the benchmark analysis. Default: 7") return parser.parse_args() def configure_verbosity(level): @@ -38,7 +42,11 @@ def configure_verbosity(level): def main(): args = parse_args() configure_verbosity(args.verbose) - bd = BenchmarkData(args.json, config={'duration': 7 * 24 * 60 * 60, 'trials': 10}) + # Convert duration from days to seconds + duration_seconds = args.duration * 24 * 60 * 60 + bd = BenchmarkData(args.json, config={'duration': duration_seconds, 'trials': args.trials}) + + # Generate main reports generate_report(bd, args.outdir) if __name__ == '__main__': diff --git a/tools/report_df/requirements.txt b/tools/report_df/requirements.txt deleted file mode 100644 index 608d708db..000000000 --- a/tools/report_df/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -pandas>=1.1.0 -lifelines>=0.25.2 -scipy>=1.4.1 -seaborn>=0.11.0 -scikit-posthocs>=0.6.4 \ No newline at end of file diff --git a/tools/report_df/templates/coverage_template.md b/tools/report_df/templates/coverage_template.md new file mode 100644 index 000000000..d006e9505 --- /dev/null +++ b/tools/report_df/templates/coverage_template.md @@ -0,0 +1,38 @@ +{% extends base_template %} +{% block title -%} +Coverage summary +{%- endblock %} + +{% block body %} +
+

Coverage summary

+

+ This page summarizes the coverage data collected during fuzzing. In Magma v1.3 this feature + is enabled for 3 fuzzers AFL++, Honggfuzz and libFuzzer. For each target and its programs, +

    +
  1. The overall percentage of branches covered
    2. +
  2. The overall number of branches covered
    3. +
  3. The percentage of branches covered measured over time
    4. +
  4. The number of branches covered measured over time
    5. +
+ is plotted to compare the fuzzers. This data is combined across all trials in a + campaign. In addition to the plots, the source code coverage reports for each fuzzing + trial can be found here: Source code coverage reports. +

+{% for target in cov_data%} +

{{ target }}

+ {% for program in cov_data[target] %} + {% set program_data = cov_data[target][program] %} +

{{ program }}

+
+
{{ program_data['fig_percent'] }}
+
{{ program_data['fig_covered'] }}
+
+
+
{{ program_data['fig_percent_overtime'] }}
+
{{ program_data['fig_covered_overtime'] }}
+
+ {% endfor %} +{% endfor%} +
+{% endblock %} \ No newline at end of file diff --git a/tools/report_df/templates/main_template.md b/tools/report_df/templates/main_template.md index 73b13e734..5863c690a 100644 --- a/tools/report_df/templates/main_template.md +++ b/tools/report_df/templates/main_template.md @@ -32,6 +32,14 @@ +
+

Coverage Report

+

Detailed code coverage analysis across all fuzzing campaigns:

+ +
+ diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 000000000..819e8873c --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,16 @@ +# for benchd and report_df +pandas>=2.3.2 + +# for benchd +lifelines>=0.30.0 + +# for report_df +Jinja2>=3.1.6 +scipy>=1.16.2 +seaborn>=0.13.2 +scikit-posthocs>=0.11.4 + +# for gen_cov_html +plotly>=6.3.0 +numpy>=2.3.3 +kaleido>=1.1.0 \ No newline at end of file