GPU kernel for sorted patches with chunk_offsets #28039
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # Concurrency control: | |
| # - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. | |
| # - Push to develop: runs queue sequentially, never cancelled. This allows us to have benchmarks | |
| # run on every commit for our benchmarks website. | |
| # - `workflow_dispatch`: groups by branch and queues if run on develop. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} | |
| on: | |
| push: | |
| branches: [develop] | |
| pull_request: { } | |
| workflow_dispatch: { } | |
| permissions: | |
| actions: read | |
| contents: read | |
| checks: write # audit-check creates checks | |
| issues: write # audit-check creates issues | |
| env: | |
| CARGO_TERM_COLOR: auto | |
| RUST_BACKTRACE: 1 | |
| NIGHTLY_TOOLCHAIN: nightly-2026-02-05 | |
| jobs: | |
| lint-toml: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: spiraldb/actions/.github/actions/lint-toml@0.18.5 | |
| validate-workflow-yaml: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Validate YAML file | |
| run: | | |
| # Lint the workflows and yamllint's configuration file. | |
| yamllint \ | |
| --strict \ | |
| -c .yamllint.yaml \ | |
| .github/ | |
| python-lint: | |
| name: "Python (lint)" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) | |
| - name: Python Lint - Format | |
| run: uvx ruff format --check . | |
| - name: Python Lint - Ruff | |
| run: uvx ruff check . | |
| # PyRight needs the project for type information, so use uv run | |
| - name: Python Lint - PyRight | |
| env: | |
| MATURIN_PEP517_ARGS: "--profile ci" | |
| run: | | |
| uv sync --all-packages | |
| uv run basedpyright vortex-python | |
| python-test: | |
| name: "Python (test)" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| env: | |
| RUST_LOG: "info,maturin=off,uv=debug" | |
| MATURIN_PEP517_ARGS: "--profile ci" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Pytest - Vortex | |
| run: | | |
| uv run --all-packages pytest --benchmark-disable -n auto test/ | |
| working-directory: vortex-python/ | |
| - name: Setup benchmark environment | |
| run: sudo bash scripts/setup-benchmark.sh | |
| - name: Pytest Benchmarks - Vortex | |
| run: | | |
| bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ | |
| working-directory: vortex-python/ | |
| - name: Doctest - PyVortex | |
| run: | | |
| uv run --all-packages make doctest | |
| working-directory: docs/ | |
| - name: Ensure docs build - PyVortex | |
| run: | | |
| uv run --all-packages make html | |
| working-directory: docs/ | |
| python-wheel-build: | |
| name: "Python (wheel build)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| save-if: ${{ github.ref_name == 'develop' }} | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| enable-sccache: "false" | |
| - uses: mlugg/setup-zig@v2.2.1 | |
| - name: Install uv | |
| uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 | |
| with: | |
| sync: false | |
| prune-cache: false | |
| - name: Ensure wheel and sdist can be built on Linux - PyVortex | |
| shell: bash | |
| run: | | |
| echo "Clearing wheel target directory" | |
| rm -rf ../target/wheels/ | |
| uv venv | |
| uv tool run maturin@1.10 build --interpreter python3.11 --zig | |
| uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist | |
| file_count=$(ls -1 ../target/wheels/ | wc -l) | |
| if [[ $file_count -ne 2 ]]; then | |
| echo "Unexpected number of files detected ${file_count}:" | |
| ls ../target/wheels/ | |
| exit 1 | |
| else | |
| echo "Generated two files" | |
| fi | |
| working-directory: vortex-python/ | |
| rust-docs: | |
| name: "Rust (docs)" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/tag=rust-docs', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Docs | |
| run: | | |
| RUSTDOCFLAGS="-D warnings" cargo doc --profile ci --no-deps | |
| # nextest doesn't support doc tests, so we run it here | |
| cargo test --profile ci --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast | |
| build-rust: | |
| name: "Rust build (${{matrix.config.name}})" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) | |
| || 'ubuntu-latest' }} | |
| env: | |
| # disable lints for build, they will be caught in Rust lint job. | |
| RUSTFLAGS: "-A warnings" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - name: "all-features" | |
| runner: amd64-large | |
| args: "--all-features --all-targets" | |
| - name: "default features" | |
| runner: amd64-large | |
| args: "--all-targets" | |
| - name: "with tokio dispatcher" | |
| runner: amd64-small | |
| # Only build the crates that have the tokio features, not re-building other crates with no-default-features | |
| args: "--no-default-features --features tokio --all-targets -p vortex -p vortex-io -p vortex-file -p vortex-layout" | |
| - name: "wasm32 with default features" | |
| runner: amd64-medium | |
| target: wasm32-unknown-unknown | |
| env: | |
| rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'" | |
| args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install wasm32 target | |
| if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }} | |
| run: rustup target add wasm32-unknown-unknown | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: "${{matrix.config.env.rustflags}} cargo hack build --profile ci --locked ${{matrix.config.args}} --ignore-private" | |
| - name: "Make sure no files changed after build" | |
| run: | | |
| git status --porcelain | |
| test -z "$(git status --porcelain)" | |
| check-min-deps: | |
| name: "Check build with minimal dependencies" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-min-deps', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - run: cargo minimal-versions check --direct --workspace --ignore-private | |
| rust-lint: | |
| name: "Rust (lint)" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-lint', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly for fmt | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt | |
| - name: Rust Lint - Format | |
| run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check | |
| - name: Rustc check | |
| run: RUSTFLAGS="-D warnings" cargo check --profile ci --locked --all-features --all-targets | |
| - name: Rustc check (release) | |
| run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release | |
| - name: Rust Lint - Clippy All Features | |
| run: cargo clippy --profile ci --locked --all-features --all-targets -- -D warnings | |
| - name: Rust Lint - Clippy Default Features | |
| run: cargo clippy --profile ci --locked --all-targets -- -D warnings | |
| cpp-lint: | |
| name: "C/C++ (lint)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: C/C++ Lint - clang-format | |
| run: | | |
| git ls-files vortex-cuda vortex-cxx vortex-duckdb vortex-ffi \ | |
| | grep -E '\.(cpp|hpp|cu|cuh|h)$' \ | |
| | grep -v 'kernels/src/bit_unpack_.*\.cu$' \ | |
| | xargs clang-format --dry-run --Werror --style=file | |
| rust-lint-no-default: | |
| name: "Rust (lint, no default)" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-lint-no-default', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Rust Lint - Clippy No Default Features | |
| shell: bash | |
| run: | | |
| cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings | |
| public-api: | |
| name: "Public API lock files" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=public-api', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly for public-api | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN | |
| - name: Regenerate public API lock files | |
| run: cargo +$NIGHTLY_TOOLCHAIN run --profile ci -p xtask -- public-api | |
| - name: Verify lock files are up to date | |
| run: | | |
| if ! git diff --quiet '**/public-api.lock'; then | |
| git diff -U0 '**/public-api.lock' | |
| echo "" | |
| echo "" | |
| echo "Changed files:" | |
| git diff --name-only '**/public-api.lock' | |
| echo "" | |
| echo "" | |
| echo "Public API lock files are out of date." | |
| echo "Run './scripts/public-api.sh' locally to" | |
| echo "regenerate them, then commit the changes." | |
| exit 1 | |
| fi | |
| rust-coverage: | |
| name: "Rust tests (coverage) (${{ matrix.suite }})" | |
| timeout-minutes: 40 | |
| permissions: | |
| id-token: write | |
| strategy: | |
| matrix: | |
| include: | |
| - suite: tests | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite) | |
| || 'ubuntu-latest' }} | |
| env: | |
| RUSTFLAGS: "-Cinstrument-coverage -A warnings" | |
| CARGO_INCREMENTAL: 0 # Disable incremental compilation to get accurate coverage | |
| LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" | |
| GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Rust Tests | |
| if: ${{ matrix.suite == 'tests' }} | |
| run: | | |
| cargo nextest run --locked --workspace --all-features --no-fail-fast | |
| - name: Generate coverage report | |
| run: | | |
| grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \ | |
| --threads $(nproc) \ | |
| --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \ | |
| --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \ | |
| --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \ | |
| --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \ | |
| --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \ | |
| --ignore benchmarks/* --ignore 'vortex-test/*' \ | |
| -o ${{ env.GRCOV_OUTPUT_FILE }} | |
| - name: Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| name: run-${{ matrix.suite }} | |
| files: ${{ env.GRCOV_OUTPUT_FILE }} | |
| disable_search: true | |
| flags: ${{ matrix.suite }} | |
| use_oidc: true | |
| rust-test-sanitizer: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # We don't run memory sanitizer as it provides many false positives | |
| # for std | |
| - sanitizer: asan | |
| sanitizer_flags: "-Zsanitizer=address,leak" | |
| - sanitizer: tsan | |
| sanitizer_flags: "-Zsanitizer=thread" | |
| name: "Rust tests (${{ matrix.sanitizer }})" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| env: | |
| ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" | |
| LSAN_OPTIONS: "report_objects=1" | |
| ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| MSAN_OPTIONS: "symbolize=1" | |
| MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt" | |
| TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| VORTEX_SKIP_SLOW_TESTS: "1" | |
| # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins | |
| # unset -Zsanitizer flag and we should allow that. | |
| RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install Rust nightly toolchain | |
| run: | | |
| rustup toolchain install $NIGHTLY_TOOLCHAIN | |
| rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview | |
| - name: Build tests with sanitizer | |
| run: | | |
| RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ | |
| cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \ | |
| --target x86_64-unknown-linux-gnu -Zbuild-std \ | |
| -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array | |
| - name: Run tests with sanitizer | |
| run: | | |
| RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ | |
| cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \ | |
| --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \ | |
| -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array | |
| # vortex-ffi requires --no-default-features as otherwise we pull in | |
| # Mimalloc which interferes with sanitizers | |
| # cargo nextest reports less sanitizer issues than cargo test | |
| # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in | |
| - name: Run vortex-ffi tests with sanitizer | |
| run: | | |
| RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ | |
| cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \ | |
| --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \ | |
| -p vortex-ffi -- --no-capture | |
| rust-ffi-test-sanitizer: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # We don't run memory sanitizer as it's clang-only and provides many | |
| # false positives for Catch2 | |
| - sanitizer: asan | |
| sanitizer_flags: "-Zsanitizer=address,leak" | |
| - sanitizer: tsan | |
| sanitizer_flags: "-Zsanitizer=thread" | |
| name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})" | |
| timeout-minutes: 40 | |
| env: | |
| ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" | |
| LSAN_OPTIONS: "report_objects=1" | |
| ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| MSAN_OPTIONS: "symbolize=1" | |
| MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| TSAN_OPTIONS: "symbolize=1" | |
| TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| VORTEX_SKIP_SLOW_TESTS: "1" | |
| # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins | |
| # unset -Zsanitizer flag and we should allow that. | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install rustfilt | |
| run: | | |
| cargo install rustfilt | |
| - name: Install Rust nightly toolchain | |
| run: | | |
| rustup toolchain install $NIGHTLY_TOOLCHAIN | |
| rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview | |
| - name: Build FFI library | |
| run: | | |
| # TODO(myrrc): remove --no-default-features | |
| RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ | |
| -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ | |
| ${{ matrix.sanitizer_flags }}" \ | |
| cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ | |
| --target x86_64-unknown-linux-gnu -Zbuild-std \ | |
| -p vortex-ffi | |
| - name: Build FFI library tests | |
| run: | | |
| cd vortex-ffi | |
| cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" | |
| cmake --build build -j | |
| - name: Run tests | |
| run: | | |
| set -o pipefail | |
| ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i- | |
| cuda-build-lint: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA build & lint" | |
| timeout-minutes: 40 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: >- | |
| cargo build --profile ci --locked --all-features --all-targets | |
| -p vortex-cuda -p vortex-cub -p vortex-nvcomp | |
| -p gpu-scan-cli -p vortex-test-e2e-cuda | |
| - name: Clippy CUDA crates | |
| run: | | |
| cargo clippy --profile ci --locked --all-features --all-targets \ | |
| -p vortex-cuda \ | |
| -p vortex-cub \ | |
| -p vortex-nvcomp \ | |
| -p gpu-scan-cli \ | |
| -p vortex-test-e2e-cuda \ | |
| -- -D warnings | |
| cuda-test: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA tests" | |
| timeout-minutes: 30 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - name: Display NVIDIA SMI details | |
| run: | | |
| nvidia-smi | |
| nvidia-smi -L | |
| nvidia-smi -q -d Memory | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install nextest | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: nextest | |
| - name: Rust Tests | |
| env: | |
| FLAT_LAYOUT_INLINE_ARRAY_NODE: true | |
| run: | | |
| cargo nextest run \ | |
| --cargo-profile ci \ | |
| --locked \ | |
| -p vortex-file \ | |
| -p vortex-cuda \ | |
| -p vortex-cub \ | |
| -p vortex-nvcomp \ | |
| -p vortex-test-e2e-cuda \ | |
| --all-features \ | |
| --no-fail-fast \ | |
| --target x86_64-unknown-linux-gnu \ | |
| --verbose | |
| cuda-test-sanitizer: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA tests (${{ matrix.sanitizer }})" | |
| timeout-minutes: 30 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - sanitizer: memcheck | |
| runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" | |
| # TODO(joe): try to re-enable racecheck, it is hanging in CI. | |
| # - sanitizer: racecheck | |
| # runner_flags: "--tool racecheck --error-exitcode 1" | |
| - sanitizer: synccheck | |
| runner_flags: "--tool synccheck --error-exitcode 1" | |
| - sanitizer: initcheck | |
| runner_flags: "--tool initcheck --error-exitcode 1" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - name: Display NVIDIA SMI details | |
| run: | | |
| nvidia-smi | |
| nvidia-smi -L | |
| nvidia-smi -q -d Memory | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build tests | |
| run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run | |
| - name: "CUDA - ${{ matrix.sanitizer }}" | |
| env: | |
| CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" | |
| run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu | |
| cuda-test-cudf: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA tests (cudf)" | |
| timeout-minutes: 30 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - name: Display NVIDIA SMI details | |
| run: | | |
| nvidia-smi | |
| nvidia-smi -L | |
| nvidia-smi -q -d Memory | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build cudf test library | |
| run: cargo build --profile ci --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu | |
| - name: Download and run cudf-test-harness | |
| run: | | |
| curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz | |
| cd cudf-test-harness-x86_64 | |
| compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/ci/libvortex_test_e2e_cuda.so | |
| rust-test-other: | |
| name: "Rust tests (${{ matrix.os }})" | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: windows-x64 | |
| runner: runs-on=${{ github.run_id }}/pool=windows-x64-pre | |
| fallback_runner: windows-latest | |
| - os: linux-arm64 | |
| runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=rust-test-linux-arm64 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && matrix.runner | |
| || matrix.fallback_runner }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - name: Setup (Windows) | |
| if: matrix.os == 'windows-x64' | |
| run: | | |
| echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Rust Tests (Windows) | |
| if: matrix.os == 'windows-x64' | |
| run: | | |
| cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast ` | |
| --exclude vortex-bench --exclude vortex-python --exclude vortex-duckdb ` | |
| --exclude vortex-fuzz --exclude vortex-cuda --exclude vortex-nvcomp ` | |
| --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude duckdb-bench ` | |
| --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench ` | |
| --exclude compress-bench --exclude xtask --exclude vortex-datafusion ` | |
| --exclude gpu-scan-cli --exclude vortex-sqllogictest | |
| - name: Rust Tests (Other) | |
| if: matrix.os != 'windows-x64' | |
| run: | | |
| cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest | |
| - uses: ./.github/actions/check-rebuild | |
| if: matrix.os != 'windows-x64' | |
| with: | |
| command: "cargo test --profile ci --locked --workspace --all-features --no-run --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest" | |
| - name: Alert incident.io | |
| if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop' | |
| uses: ./.github/actions/alert-incident-io | |
| with: | |
| api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} | |
| alert-title: "Rust tests (${{ matrix.os }}) failed on develop" | |
| deduplication-key: ci-rust-test-${{ matrix.os }}-failure | |
| build-java: | |
| name: "Java" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=java', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - run: ./gradlew test --parallel | |
| working-directory: ./java | |
| bench-codspeed: | |
| strategy: | |
| matrix: | |
| include: | |
| - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error" } | |
| - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" } | |
| - { shard: 3, name: "Main library", packages: "vortex" } | |
| - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" } | |
| - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" } | |
| - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" } | |
| - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" } | |
| - { shard: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" } | |
| name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - name: Setup benchmark environment | |
| run: sudo bash scripts/setup-benchmark.sh | |
| - uses: ./.github/actions/setup-prebuild | |
| - uses: ./.github/actions/system-info | |
| - name: Install Codspeed | |
| uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 | |
| with: | |
| tool: cargo-codspeed | |
| - name: Build benchmarks | |
| env: | |
| RUSTFLAGS: "-C target-feature=+avx2" | |
| run: cargo codspeed build ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench | |
| - name: Run benchmarks | |
| uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2 | |
| with: | |
| run: bash scripts/bench-taskset.sh cargo codspeed run | |
| token: ${{ secrets.CODSPEED_TOKEN }} | |
| mode: "simulation" | |
| license-check-and-audit-check: | |
| name: License Check and Audit Check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| strategy: | |
| matrix: | |
| checks: | |
| - advisories | |
| - bans licenses sources | |
| # Prevent sudden announcement of a new advisory from failing ci: | |
| continue-on-error: ${{ matrix.checks == 'advisories' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: EmbarkStudios/cargo-deny-action@v2 | |
| with: | |
| command: check ${{ matrix.checks }} | |
| cxx-test: | |
| name: "C++ build" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Build and run C++ unit tests | |
| run: | | |
| mkdir -p vortex-cxx/build | |
| cmake -S vortex-cxx -B vortex-cxx/build -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON | |
| cmake --build vortex-cxx/build --parallel $(nproc) | |
| ctest --test-dir vortex-cxx/build -j $(nproc) -V | |
| - name: Build and run the example in release mode | |
| run: | | |
| cmake -S vortex-cxx/examples -B vortex-cxx/examples/build -DCMAKE_BUILD_TYPE=Release | |
| cmake --build vortex-cxx/examples/build --parallel $(nproc) | |
| vortex-cxx/examples/build/hello-vortex vortex-cxx/examples/goldenfiles/example.vortex | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: "cargo build --profile ci --locked -p vortex-cxx --lib" | |
| sqllogic-test: | |
| name: "SQL logic tests" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Run sqllogictest tests | |
| run: | | |
| ./vortex-sqllogictest/slt/tpch/generate_data.sh | |
| cargo test --profile ci -p vortex-sqllogictest --test sqllogictests | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: "cargo test --profile ci -p vortex-sqllogictest --test sqllogictests --no-run" | |
| wasm-integration: | |
| name: "WASM integration smoke test" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| targets: "wasm32-wasip1" | |
| - name: Setup Wasmer | |
| shell: bash | |
| run: | | |
| curl https://get.wasmer.io -sSfL | sh | |
| echo "$HOME/.wasmer/bin" >> $GITHUB_PATH | |
| - run: cargo build --profile ci --target wasm32-wasip1 | |
| working-directory: ./wasm-test | |
| - run: wasmer run ./target/wasm32-wasip1/ci/wasm-test.wasm | |
| working-directory: ./wasm-test | |
| miri: | |
| name: "Rust tests (miri)" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| env: | |
| MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE | |
| RUSTFLAGS: "-A warnings" | |
| RUST_BACKTRACE: full | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly with miri | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri | |
| - name: Run Miri | |
| run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi | |
| generated-files: | |
| name: "Check generated source files are up to date" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=generated-files', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly for cbindgen macro expansion | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN | |
| - name: "regenerate all .fbs/.proto Rust code" | |
| run: | | |
| cargo run --profile ci -p xtask -- generate-fbs | |
| cargo run --profile ci -p xtask -- generate-proto | |
| - name: "regenerate FFI header file" | |
| run: | | |
| cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi | |
| - name: "Make sure no files changed after regenerating" | |
| run: | | |
| git status --porcelain | |
| test -z "$(git status --porcelain)" | |
| - name: "Checkout develop flatbuffers" | |
| working-directory: vortex-flatbuffers/ | |
| run: | | |
| cp -R flatbuffers flatbuffers.HEAD | |
| git fetch origin develop --depth 1 | |
| git checkout origin/develop -- flatbuffers | |
| - name: "Verify flatbuffer back-compat" | |
| working-directory: vortex-flatbuffers/ | |
| run: | | |
| find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{} | |
| ffi-c-test: | |
| name: "C API test build" | |
| timeout-minutes: 10 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: "regenerate FFI header file" | |
| run: | | |
| cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi | |
| - name: Build and run C++ unit tests | |
| run: | | |
| cd vortex-ffi | |
| mkdir build | |
| cmake -Bbuild -DRUST_BUILD_PROFILE=ci | |
| cmake --build build -j $(nproc) | |
| ctest --test-dir build -j $(nproc) | |
| check-java-publish-build: | |
| runs-on: ${{ matrix.target.runs-on }} | |
| container: | |
| image: "ubuntu:20.04" | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } | |
| - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/prepare-java-linux | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: "corretto" | |
| java-version: "17" | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| targets: ${{ matrix.target.target }} | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| enable-sccache: "false" | |
| - run: cargo build --profile ci --package vortex-jni | |
| compat-check: | |
| name: "Compat check" | |
| uses: ./.github/workflows/compat-validation.yml | |
| with: | |
| mode: last | |
| rust-publish-dry-run: | |
| name: "Rust publish dry-run" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install cargo-edit | |
| uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 | |
| with: | |
| tool: cargo-edit | |
| - name: Set Version | |
| run: | | |
| # This is mostly a dummy version, we don't actually publish anything but it cannot exist in crates.io | |
| cargo set-version --workspace 0.100000.0 | |
| - name: Test Release | |
| run: | | |
| cargo publish --dry-run --no-verify --allow-dirty --workspace |