fix: resolve DuckDB lock conflicts with guard, read-only connections,… #12
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 QuantCore Contributors | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: CI | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main, develop] | |
| schedule: | |
| # Nightly integration test run at 02:00 UTC | |
| - cron: "0 2 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHON_VERSION: "3.11" | |
| FORCE_COLOR: "1" | |
| jobs: | |
| # ========================================================================== | |
| # Linting and Type Checking | |
| # ========================================================================== | |
| lint: | |
| name: Lint & Type Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| - name: Set up Python | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: uv sync --all-packages | |
| - name: Run Ruff linter | |
| run: uv run ruff check packages/ tests/ | |
| - name: Run Ruff formatter check | |
| run: uv run ruff format --check packages/ tests/ | |
| - name: Run MyPy (critical path) | |
| # Type-check the trading hot path: execution, risk, and kill switch. | |
| # Broader codebase type coverage tracked in GitHub type-coverage label. | |
| run: > | |
| uv run mypy | |
| packages/quantcore/execution | |
| packages/quant_pod/execution | |
| --config-file pyproject.toml | |
| # ========================================================================== | |
| # Unit Tests (fast, in-memory DB, no external services) | |
| # ========================================================================== | |
| test: | |
| name: Test (Python ${{ matrix.python-version }}, ${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest] | |
| python-version: ["3.11", "3.12"] | |
| exclude: | |
| # Reduce matrix for faster CI — 3.11 on ubuntu is the primary target | |
| - os: macos-latest | |
| python-version: "3.11" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-uv-${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-${{ matrix.python-version }}- | |
| ${{ runner.os }}-uv- | |
| - name: Install dependencies | |
| run: uv sync --all-packages --group dev | |
| - name: Run unit tests with coverage | |
| run: | | |
| uv run pytest tests/ -v \ | |
| --cov=packages/quantcore \ | |
| --cov=packages/quant_pod \ | |
| --cov-report=xml \ | |
| --cov-report=term-missing \ | |
| --cov-fail-under=80 \ | |
| -m "not slow and not integration and not requires_api and not requires_gpu" | |
| - name: Upload coverage to Codecov | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./coverage.xml | |
| # Fail CI if Codecov upload fails — prevents silently missing coverage | |
| fail_ci_if_error: true | |
| verbose: true | |
| # ========================================================================== | |
| # Integration Tests (nightly — hit in-memory DB, no external services) | |
| # Run on schedule OR when triggered manually. Not required for PR merges. | |
| # ========================================================================== | |
| integration: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| - name: Set up Python | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: uv sync --all-packages | |
| - name: Run integration tests | |
| run: | | |
| uv run pytest tests/ -v \ | |
| --cov=packages/quant_pod \ | |
| --cov-report=term-missing \ | |
| -m "integration" | |
| # ========================================================================== | |
| # Security Scanning | |
| # ========================================================================== | |
| security: | |
| name: Security Scan | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| - name: Set up Python | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Install security tools | |
| run: uv tool install bandit | |
| - name: Run Bandit security scan | |
| # -lll = HIGH severity only. Medium B608 (SQL injection) is a known | |
| # false positive for DuckDB internal queries that use no user input. | |
| run: uvx bandit -r packages/ -lll --exclude packages/quantcore/rl/ | |
| # ========================================================================== | |
| # Docker Image Build + Trivy Scan | |
| # ========================================================================== | |
| docker: | |
| name: Docker Build & Scan | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build Docker image | |
| run: docker build -t quantpod:${{ github.sha }} . | |
| - name: Run Trivy vulnerability scan | |
| uses: aquasecurity/trivy-action@master | |
| with: | |
| image-ref: quantpod:${{ github.sha }} | |
| format: "table" | |
| exit-code: "1" | |
| ignore-unfixed: true | |
| severity: "CRITICAL,HIGH" | |
| # ========================================================================== | |
| # All required checks pass (gate for merges) | |
| # ========================================================================== | |
| all-checks: | |
| name: All Checks Pass | |
| if: always() | |
| needs: [lint, test, security] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check all required jobs passed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| # integration and docker are not required for PR merges | |
| jobs: ${{ toJSON(needs) }} |