Merge branch 'apache:main' into main

Author: martin-augment

.github/workflows/codeql.yml

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '16 4 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze Actions
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+      packages: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      with:
+        persist-credentials: false
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@c793b717bc78562f491db7b0e93a3a178b099162 # v4
+      with:
+        languages: actions
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@c793b717bc78562f491db7b0e93a3a178b099162 # v4
+      with:
+        category: "/language:actions"

.github/workflows/pr_build_linux.yml

@@ -69,7 +69,7 @@ jobs:
   build-native:
     needs: lint
     name: Build Native Library
-    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
     steps:
@@ -122,7 +122,7 @@ jobs:
   linux-test-rust:
     needs: lint
     name: ubuntu-latest/rust-test
-    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
     steps:
@@ -277,7 +277,7 @@ jobs:
               org.apache.spark.sql.CometToPrettyStringSuite
       fail-fast: false
     name: ${{ matrix.profile.name }}/${{ matrix.profile.scan_impl }} [${{ matrix.suite.name }}]
-    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
       env:
@@ -325,7 +325,7 @@ jobs:
   verify-benchmark-results-tpch:
     needs: build-native
     name: Verify TPC-H Results
-    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
     steps:
@@ -379,7 +379,7 @@ jobs:
   verify-benchmark-results-tpcds:
     needs: build-native
     name: Verify TPC-DS Results (${{ matrix.join }})
-    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
     strategy:

benchmarks/tpc/README.md

@@ -38,17 +38,21 @@ All benchmarks are run via `run.py`:
 python3 run.py --engine <engine> --benchmark <tpch|tpcds> [options]
 ```
 
-| Option         | Description                                              |
-| -------------- | -------------------------------------------------------- |
-| `--engine`     | Engine name (matches a TOML file in `engines/`)          |
-| `--benchmark`  | `tpch` or `tpcds`                                        |
-| `--iterations` | Number of iterations (default: 1)                        |
-| `--output`     | Output directory (default: `.`)                          |
-| `--query`      | Run a single query number                                |
-| `--no-restart` | Skip Spark master/worker restart                         |
-| `--dry-run`    | Print the spark-submit command without executing         |
-| `--jfr`        | Enable Java Flight Recorder profiling                    |
-| `--jfr-dir`    | Directory for JFR output files (default: `/results/jfr`) |
+| Option                    | Description                                                                     |
+| ------------------------- | ------------------------------------------------------------------------------- |
+| `--engine`                | Engine name (matches a TOML file in `engines/`)                                 |
+| `--benchmark`             | `tpch` or `tpcds`                                                               |
+| `--iterations`            | Number of iterations (default: 1)                                               |
+| `--output`                | Output directory (default: `.`)                                                 |
+| `--query`                 | Run a single query number                                                       |
+| `--no-restart`            | Skip Spark master/worker restart                                                |
+| `--dry-run`               | Print the spark-submit command without executing                                |
+| `--jfr`                   | Enable Java Flight Recorder profiling                                           |
+| `--jfr-dir`               | Directory for JFR output files (default: `/results/jfr`)                        |
+| `--async-profiler`        | Enable async-profiler (profiles Java + native code)                             |
+| `--async-profiler-dir`    | Directory for async-profiler output (default: `/results/async-profiler`)        |
+| `--async-profiler-event`  | Event type: `cpu`, `wall`, `alloc`, `lock`, etc. (default: `cpu`)               |
+| `--async-profiler-format` | Output format: `flamegraph`, `jfr`, `collapsed`, `text` (default: `flamegraph`) |
 
 Available engines: `spark`, `comet`, `comet-iceberg`, `gluten`
 
@@ -392,3 +396,88 @@ docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml \
 
 Open the `.jfr` files with [JDK Mission Control](https://jdk.java.net/jmc/),
 IntelliJ IDEA's profiler, or `jfr` CLI tool (`jfr summary driver.jfr`).
+
+## async-profiler Profiling
+
+Use the `--async-profiler` flag to capture profiles with
+[async-profiler](https://github.com/async-profiler/async-profiler). Unlike JFR,
+async-profiler can profile **both Java and native (Rust/C++) code** in the same
+flame graph, making it especially useful for profiling Comet workloads.
+
+### Prerequisites
+
+async-profiler must be installed on every node where the driver or executors run.
+Set `ASYNC_PROFILER_HOME` to the installation directory:
+
+```shell
+# Download and extract (Linux x64 example)
+wget https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz
+tar xzf async-profiler-3.0-linux-x64.tar.gz -C /opt/async-profiler --strip-components=1
+export ASYNC_PROFILER_HOME=/opt/async-profiler
+```
+
+On Linux, `perf_event_paranoid` must be set to allow profiling:
+
+```shell
+sudo sysctl kernel.perf_event_paranoid=1   # or 0 / -1 for full access
+sudo sysctl kernel.kptr_restrict=0          # optional: enable kernel symbols
+```
+
+### Basic usage
+
+```shell
+python3 run.py --engine comet --benchmark tpch --async-profiler
+```
+
+This produces HTML flame graphs in `/results/async-profiler/` by default
+(`driver.html` and `executor.html`).
+
+### Choosing events and output format
+
+```shell
+# Wall-clock profiling (includes time spent waiting/sleeping)
+python3 run.py --engine comet --benchmark tpch \
+    --async-profiler --async-profiler-event wall
+
+# Allocation profiling with JFR output
+python3 run.py --engine comet --benchmark tpch \
+    --async-profiler --async-profiler-event alloc --async-profiler-format jfr
+
+# Lock contention profiling
+python3 run.py --engine comet --benchmark tpch \
+    --async-profiler --async-profiler-event lock
+```
+
+| Event   | Description                                         |
+| ------- | --------------------------------------------------- |
+| `cpu`   | On-CPU time (default). Shows where CPU cycles go.   |
+| `wall`  | Wall-clock time. Includes threads that are blocked. |
+| `alloc` | Heap allocation profiling.                          |
+| `lock`  | Lock contention profiling.                          |
+
+| Format       | Extension | Description                              |
+| ------------ | --------- | ---------------------------------------- |
+| `flamegraph` | `.html`   | Interactive HTML flame graph (default).  |
+| `jfr`        | `.jfr`    | JFR format, viewable in JMC or IntelliJ. |
+| `collapsed`  | `.txt`    | Collapsed stacks for FlameGraph scripts. |
+| `text`       | `.txt`    | Flat text summary of hot methods.        |
+
+### Docker usage
+
+The Docker image includes async-profiler pre-installed at
+`/opt/async-profiler`. The `ASYNC_PROFILER_HOME` environment variable is
+already set in the compose files, so no extra configuration is needed:
+
+```shell
+docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml \
+    run --rm bench \
+    python3 /opt/benchmarks/run.py \
+    --engine comet --benchmark tpch --output /results --no-restart --async-profiler
+```
+
+Output files are collected in `$RESULTS_DIR/async-profiler/` on the host.
+
+**Note:** On Linux, the Docker container needs `--privileged` or
+`SYS_PTRACE` capability and `perf_event_paranoid <= 1` on the host for
+`cpu`/`wall` events. Allocation (`alloc`) and lock (`lock`) events work
+without special privileges.

benchmarks/tpc/infra/docker/Dockerfile

@@ -29,10 +29,23 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends \
        openjdk-8-jdk-headless \
        openjdk-17-jdk-headless \
-       python3 python3-pip procps \
+       python3 python3-pip procps wget \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Install async-profiler for profiling Java + native (Rust/C++) code.
+ARG ASYNC_PROFILER_VERSION=3.0
+RUN ARCH=$(uname -m) && \
+    if [ "$ARCH" = "x86_64" ]; then AP_ARCH="linux-x64"; \
+    elif [ "$ARCH" = "aarch64" ]; then AP_ARCH="linux-aarch64"; \
+    else echo "Unsupported architecture: $ARCH" && exit 1; fi && \
+    wget -q "https://github.com/async-profiler/async-profiler/releases/download/v${ASYNC_PROFILER_VERSION}/async-profiler-${ASYNC_PROFILER_VERSION}-${AP_ARCH}.tar.gz" \
+         -O /tmp/async-profiler.tar.gz && \
+    mkdir -p /opt/async-profiler && \
+    tar xzf /tmp/async-profiler.tar.gz -C /opt/async-profiler --strip-components=1 && \
+    rm /tmp/async-profiler.tar.gz
+ENV ASYNC_PROFILER_HOME=/opt/async-profiler
+
 # Default to Java 17 (override with JAVA_HOME at runtime for Gluten).
 # Detect architecture (amd64 or arm64) so the image works on both Linux and macOS.
 ARG TARGETARCH

benchmarks/tpc/infra/docker/docker-compose-laptop.yml

@@ -30,6 +30,7 @@
 #   ICEBERG_JAR        - Host path to Iceberg Spark runtime JAR
 #   BENCH_JAVA_HOME    - Java home inside container (default: /usr/lib/jvm/java-17-openjdk)
 #                        Set to /usr/lib/jvm/java-8-openjdk for Gluten
+#   ASYNC_PROFILER_HOME - async-profiler install path (default: /opt/async-profiler)
 
 x-volumes: &volumes
   - ${DATA_DIR:-/tmp/tpc-data}:/data:ro
@@ -95,5 +96,6 @@ services:
       - TPCH_DATA=/data
       - TPCDS_DATA=/data
       - SPARK_EVENT_LOG_DIR=/results/spark-events
+      - ASYNC_PROFILER_HOME=/opt/async-profiler
     mem_limit: 4g
     memswap_limit: 4g

benchmarks/tpc/infra/docker/docker-compose.yml

@@ -33,6 +33,7 @@
 #   BENCH_MEM_LIMIT    - Hard memory limit for the bench runner (default: 10g)
 #   BENCH_JAVA_HOME    - Java home inside container (default: /usr/lib/jvm/java-17-openjdk)
 #                        Set to /usr/lib/jvm/java-8-openjdk for Gluten
+#   ASYNC_PROFILER_HOME - async-profiler install path (default: /opt/async-profiler)
 
 x-volumes: &volumes
   - ${DATA_DIR:-/tmp/tpc-data}:/data:ro
@@ -109,6 +110,7 @@ services:
       - TPCH_DATA=/data
       - TPCDS_DATA=/data
       - SPARK_EVENT_LOG_DIR=/results/spark-events
+      - ASYNC_PROFILER_HOME=/opt/async-profiler
     mem_limit: ${BENCH_MEM_LIMIT:-10g}
     memswap_limit: ${BENCH_MEM_LIMIT:-10g}
 

benchmarks/tpc/run.py

@@ -279,6 +279,38 @@ def build_spark_submit_cmd(config, benchmark, args):
             existing = conf.get(spark_key, "")
             conf[spark_key] = f"{existing} {jfr_opts}".strip()
 
+    # async-profiler: attach as a Java agent via -agentpath
+    if args.async_profiler:
+        ap_home = os.environ.get("ASYNC_PROFILER_HOME", "")
+        if not ap_home:
+            print(
+                "Error: ASYNC_PROFILER_HOME is not set. "
+                "Set it to the async-profiler installation directory.",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        lib_ext = "dylib" if sys.platform == "darwin" else "so"
+        ap_lib = os.path.join(ap_home, "lib", f"libasyncProfiler.{lib_ext}")
+        ap_dir = args.async_profiler_dir
+        ap_event = args.async_profiler_event
+        ap_fmt = args.async_profiler_format
+        ext = {"flamegraph": "html", "jfr": "jfr", "collapsed": "txt", "text": "txt"}[ap_fmt]
+
+        driver_ap = (
+            f"-agentpath:{ap_lib}=start,event={ap_event},"
+            f"{ap_fmt},file={ap_dir}/driver.{ext}"
+        )
+        executor_ap = (
+            f"-agentpath:{ap_lib}=start,event={ap_event},"
+            f"{ap_fmt},file={ap_dir}/executor.{ext}"
+        )
+        for spark_key, ap_opts in [
+            ("spark.driver.extraJavaOptions", driver_ap),
+            ("spark.executor.extraJavaOptions", executor_ap),
+        ]:
+            existing = conf.get(spark_key, "")
+            conf[spark_key] = f"{existing} {ap_opts}".strip()
+
     for key, val in sorted(conf.items()):
         cmd += ["--conf", f"{key}={val}"]
 
@@ -385,6 +417,27 @@ def main():
         default="/results/jfr",
         help="Directory for JFR output files (default: /results/jfr)",
     )
+    parser.add_argument(
+        "--async-profiler",
+        action="store_true",
+        help="Enable async-profiler for driver and executors (profiles Java + native code)",
+    )
+    parser.add_argument(
+        "--async-profiler-dir",
+        default="/results/async-profiler",
+        help="Directory for async-profiler output files (default: /results/async-profiler)",
+    )
+    parser.add_argument(
+        "--async-profiler-event",
+        default="cpu",
+        help="async-profiler event type: cpu, wall, alloc, lock, etc. (default: cpu)",
+    )
+    parser.add_argument(
+        "--async-profiler-format",
+        default="flamegraph",
+        choices=["flamegraph", "jfr", "collapsed", "text"],
+        help="async-profiler output format (default: flamegraph)",
+    )
     args = parser.parse_args()
 
     config = load_engine_config(args.engine)
@@ -401,9 +454,12 @@ def main():
     if not args.no_restart and not args.dry_run:
         restart_spark()
 
-    # Create JFR output directory if profiling is enabled
-    if args.jfr:
-        os.makedirs(args.jfr_dir, exist_ok=True)
+    # Create profiling output directories (skip for dry-run)
+    if not args.dry_run:
+        if args.jfr:
+            os.makedirs(args.jfr_dir, exist_ok=True)
+        if args.async_profiler:
+            os.makedirs(args.async_profiler_dir, exist_ok=True)
 
     cmd = build_spark_submit_cmd(config, args.benchmark, args)
 

docs/source/contributor-guide/parquet_scans.md

@@ -49,10 +49,10 @@ The following features are not supported by either scan implementation, and Come
 
 The following shared limitation may produce incorrect results without falling back to Spark:
 
-- No support for datetime rebasing detection or the `spark.comet.exceptionOnDatetimeRebase` configuration. When
-  reading Parquet files containing dates or timestamps written before Spark 3.0 (which used a hybrid
-  Julian/Gregorian calendar), dates/timestamps will be read as if they were written using the Proleptic Gregorian
-  calendar. This may produce incorrect results for dates before October 15, 1582.
+- No support for datetime rebasing. When reading Parquet files containing dates or timestamps written before
+  Spark 3.0 (which used a hybrid Julian/Gregorian calendar), dates/timestamps will be read as if they were
+  written using the Proleptic Gregorian calendar. This may produce incorrect results for dates before
+  October 15, 1582.
 
 The `native_datafusion` scan has some additional limitations, mostly related to Parquet metadata. All of these
 cause Comet to fall back to Spark.

docs/source/index.md

@@ -86,7 +86,7 @@ To get started with Apache DataFusion Comet, follow the
 [DataFusion Slack and Discord channels](https://datafusion.apache.org/contributor-guide/communication.html) to connect
 with other users, ask questions, and share your experiences with Comet.
 
-Follow [Apache DataFusion Comet Overview](https://datafusion.apache.org/comet/user-guide/overview.html) to get more detailed information
+Follow [Apache DataFusion Comet Overview](https://datafusion.apache.org/comet/about/index.html) to get more detailed information
 
 ## Contributing