feat(fetch): add content quality signals (word_count, redirect_chain,… #165
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_call: | |
| # Cancel in-progress runs for same PR/branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Check formatting | |
| run: cargo fmt --all -- --check | |
| - name: Clippy | |
| run: cargo clippy --workspace --all-targets -- -D warnings | |
| test: | |
| name: Test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Test | |
| run: cargo test --workspace | |
| - name: Test (network-dependent) | |
| run: cargo test --workspace -- --ignored | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build release | |
| run: cargo build --workspace --exclude fetchkit-python --release | |
| - name: Upload CLI artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fetchkit-cli-linux | |
| path: target/release/fetchkit | |
| doc: | |
| name: Documentation | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Check documentation | |
| run: cargo doc --workspace --no-deps | |
| env: | |
| RUSTDOCFLAGS: -D warnings | |
| examples: | |
| name: Examples | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run fetch_urls example | |
| run: cargo run -p fetchkit --example fetch_urls | |
| timeout-minutes: 2 | |
| - name: Run save_to_file example | |
| run: cargo run -p fetchkit --example save_to_file | |
| timeout-minutes: 2 | |
| python: | |
| name: Python Example | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Set up virtualenv and install maturin | |
| run: | | |
| python -m venv .venv | |
| . .venv/bin/activate | |
| pip install maturin | |
| - name: Build Python bindings | |
| run: | | |
| . .venv/bin/activate | |
| maturin develop -m crates/fetchkit-python/Cargo.toml | |
| - name: Run Python example | |
| run: | | |
| . .venv/bin/activate | |
| python examples/python_fetchkit.py | |
| timeout-minutes: 2 | |
| # Gate job for branch protection. | |
| check: | |
| name: Check | |
| if: always() | |
| needs: [lint, test, build, doc, examples, python] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Verify all jobs passed | |
| run: | | |
| if [[ "${{ needs.lint.result }}" != "success" ]] || \ | |
| [[ "${{ needs.test.result }}" != "success" ]] || \ | |
| [[ "${{ needs.build.result }}" != "success" ]] || \ | |
| [[ "${{ needs.doc.result }}" != "success" ]] || \ | |
| [[ "${{ needs.examples.result }}" != "success" ]] || \ | |
| [[ "${{ needs.python.result }}" != "success" ]]; then | |
| echo "One or more required jobs failed" | |
| exit 1 | |
| fi |