active ci tests + husky for pre-commit

.github/workflows/dbt_ci.yml

@@ -1,17 +1,17 @@
 name: dbt CI
 
-# NOTE: Pull request CI jobs are disabled by default. Uncomment to enable standard CI workflows
-# Check the README.md to make sure you understand the credit costs associated with running dbt.
-# on:
-#   pull_request:
-#     types: [opened, synchronize, reopened]
-#     paths:
-#       - 'models/**'
-#       - 'macros/**'
-#       - 'dbt_project.yml'
-#       - 'profiles.yml'
-#       - 'packages.yml'
-#       - '.github/workflows/dbt_ci.yml'
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - 'models/**'
+      - 'macros/**'
+      - 'tests/**'
+      - 'dbt_project.yml'
+      - 'profiles.yml'
+      - 'packages.yml'
+      - '.github/workflows/dbt_ci.yml'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -24,13 +24,21 @@ jobs:
 
     env:
       DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }}
-      DUNE_TEAM_NAME: ${{ vars.DUNE_TEAM_NAME || 'dune' }}  # Set as GitHub Variable or defaults to 'dune'
-      DEV_SCHEMA_SUFFIX: pr${{ github.event.pull_request.number }}
+      DUNE_TEAM_NAME: ${{ vars.DUNE_TEAM_NAME }}
+      DEV_SCHEMA_SUFFIX: ${{ github.event_name == 'pull_request' && format('pr{0}', github.event.pull_request.number) || 'manual' }}
 
     steps:
       - name: Check out code
         uses: actions/checkout@v4
 
+      - name: Validate required environment
+        run: |
+          if [ -z "${DUNE_TEAM_NAME}" ]; then
+            echo "❌ Missing required GitHub variable: DUNE_TEAM_NAME"
+            echo "Set it in Settings -> Secrets and variables -> Actions -> Variables."
+            exit 1
+          fi
+
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:

.github/workflows/dbt_deploy.yml

@@ -28,13 +28,24 @@ jobs:
 
     env:
       DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }}
-      DUNE_TEAM_NAME: ${{ vars.DUNE_TEAM_NAME || 'dune' }}  # Set as GitHub Variable or defaults to 'dune'
+      DUNE_TEAM_NAME: ${{ vars.DUNE_TEAM_NAME }}
       DBT_TARGET: prod  # All dbt commands will use the prod target from profiles.yml
+      # Workaround for API keys without permission to call dune._internal.alter_view_properties.
+      # Remove once key permissions are upgraded for view/table metadata property updates.
+      DUNE_SKIP_VIEW_PROPERTIES: ${{ vars.DUNE_SKIP_VIEW_PROPERTIES || 'true' }}
 
     steps:
       - name: Check out code
         uses: actions/checkout@v4
 
+      - name: Validate required environment
+        run: |
+          if [ -z "${DUNE_TEAM_NAME}" ]; then
+            echo "❌ Missing required GitHub variable: DUNE_TEAM_NAME"
+            echo "Set it in Settings -> Secrets and variables -> Actions -> Variables."
+            exit 1
+          fi
+
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:

.github/workflows/dbt_prod.yml

@@ -18,15 +18,26 @@ jobs:
 
     env:
       DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }}
-      DUNE_TEAM_NAME: ${{ vars.DUNE_TEAM_NAME || 'dune' }}  # Set as GitHub Variable or defaults to 'dune'
+      DUNE_TEAM_NAME: ${{ vars.DUNE_TEAM_NAME }}
       DBT_TARGET: prod  # All dbt commands will use the prod target from profiles.yml
+      # Workaround for API keys without permission to call dune._internal.alter_view_properties.
+      # Remove once key permissions are upgraded for view/table metadata property updates.
+      DUNE_SKIP_VIEW_PROPERTIES: ${{ vars.DUNE_SKIP_VIEW_PROPERTIES || 'true' }}
 
     steps:
       - name: Check out code
         uses: actions/checkout@v4
         with:
           ref: main
 
+      - name: Validate required environment
+        run: |
+          if [ -z "${DUNE_TEAM_NAME}" ]; then
+            echo "❌ Missing required GitHub variable: DUNE_TEAM_NAME"
+            echo "Set it in Settings -> Secrets and variables -> Actions -> Variables."
+            exit 1
+          fi
+
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:

.github/workflows/dbt_quality.yml

@@ -0,0 +1,42 @@
+name: dbt quality
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - 'models/**'
+      - 'macros/**'
+      - 'tests/**'
+      - 'dbt_project.yml'
+      - 'packages.yml'
+      - 'profiles.yml'
+      - 'pyproject.toml'
+      - 'uv.lock'
+      - '.github/workflows/dbt_quality.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  parse:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --locked
+
+      - name: Install dbt packages
+        run: uv run dbt deps
+
+      - name: Parse dbt project
+        run: uv run dbt parse --no-partial-parse

.gitignore

@@ -37,4 +37,8 @@ logs/
 .envrc
 
 # Compiled files
-*.pyc
+*.pyc
+
+# JavaScript tooling (husky/lint-staged)
+node_modules/
+.husky/_/

.husky/pre-commit

@@ -0,0 +1,2 @@
+#!/usr/bin/env sh
+npm exec lint-staged

README.md

@@ -111,6 +111,22 @@ uv run dbt test --select model_name        # Test specific model
 uv run dbt docs generate && uv run dbt docs serve # View documentation
 ```
 
+## Contributing
+
+Install local git hooks to run fast dbt checks before commits:
+
+```bash
+npm install
+npm run prepare
+```
+
+Manually run the same local validation used in pre-commit:
+
+```bash
+uv run dbt deps
+uv run dbt parse --no-partial-parse
+```
+
 ## Cursor AI Rules
 
 This repo includes **optional** Cursor AI guidelines in `.cursor/rules/`:

docs/cicd.md

@@ -11,16 +11,17 @@ GitHub provides and manages these runners - no infrastructure setup required on
 ## Pull Request Workflow (CI)
 
 **Trigger:** Every pull request
-**File:** `.github/workflows/dbt_run.yml`
+**File:** `.github/workflows/dbt_ci.yml`
 
 ### What It Does
 
-1. Enforces branch is up-to-date with main
+1. Runs on every pull request affecting dbt project paths (or manual dispatch)
 2. Installs dependencies and dbt packages
-3. Runs modified models with `--full-refresh`
-4. Tests modified models
-5. Runs modified incremental models (incremental run)
-6. Tests modified incremental models again
+3. Compiles project and compares against main manifest
+4. Runs modified models with `--full-refresh`
+5. Tests modified models
+6. Runs modified incremental models (incremental run)
+7. Tests modified incremental models again
 
 ### PR Schema Isolation
 
@@ -53,6 +54,14 @@ uv run dbt test --select modified_model
 
 ✅ **Fix failing tests** - Don't skip tests or disable checks
 
+### Cost Control Policy
+
+The Dune integration CI runs on matching PR changes, so monitor credit usage:
+
+- Keep the quality workflow (`dbt_quality.yml`) as a zero-credit static validation baseline
+- Use `state:modified` in CI (already configured) to limit heavy runs to changed models
+- Use path filters to avoid triggering on irrelevant repository changes
+
 ## Production Workflow
 
 **Trigger:** Manual (schedule disabled by default)
@@ -99,7 +108,20 @@ DUNE_API_KEY=your_api_key
 DUNE_TEAM_NAME=your_team_name
 ```
 
-Optional - defaults to `'dune'` if not set.
+This variable is required for deploy/prod/CI workflows.
+
+## Quality Workflow (No Dune Credits)
+
+**Trigger:** Every pull request affecting dbt project files  
+**File:** `.github/workflows/dbt_quality.yml`
+
+This workflow does not require `DUNE_API_KEY` and runs only static project checks:
+
+1. `uv sync --locked`
+2. `uv run dbt deps`
+3. `uv run dbt parse --no-partial-parse`
+
+Use this as the mandatory baseline check for every PR.
 
 ## Email Notifications
 
@@ -119,8 +141,19 @@ To receive failure alerts:
 
 Runs when:
 
-- PR opened, synchronized, or reopened
-- Changes to: `models/`, `macros/`, `dbt_project.yml`, `profiles.yml`, `packages.yml`, workflow file
+- PR opened, synchronized, reopened, or marked ready for review
+- Changes to: `models/`, `macros/`, `tests/`, `dbt_project.yml`, `profiles.yml`, `packages.yml`, workflow file
+
+## Branch Protection (Required Checks)
+
+To block merges unless CI passes:
+
+1. Go to GitHub → Repository Settings → Branches
+2. Edit the protection rule for `main`
+3. Enable **Require status checks to pass before merging**
+4. Mark these checks as required:
+   - `parse` (from `dbt quality`)
+   - `dbt-ci` (from `dbt CI`)
 
 ### Production Workflow
 
@@ -149,6 +182,17 @@ dbt test output → specific test name → error message
 
 Query the model in Dune to investigate.
 
+### Main Manifest Missing in PR CI
+
+PR Dune CI depends on `prod-manifest-latest` uploaded by deploy workflow.
+
+If CI fails with missing manifest:
+
+1. Go to Actions → `dbt deploy`
+2. Run workflow on `main`
+3. Wait for completion (artifact upload)
+4. Re-run PR check
+
 ### Connection Errors
 
 - Verify `DUNE_API_KEY` secret is set correctly

docs/testing.md

@@ -102,6 +102,7 @@ Use based on your data quality requirements:
 - `accepted_values` - Enum/categorical validation
 - `relationships` - Foreign key checks
 - Custom tests in `tests/` directory
+- dbt native `unit_tests` for deterministic model logic checks with mocked inputs
 
 Example:
 ```yaml
@@ -126,13 +127,49 @@ uv run dbt test
 # Test specific model
 uv run dbt test --select my_model
 
+# Test only singular tests in tests/
+uv run dbt test --select test_type:singular
+
+# Test only dbt unit tests
+uv run dbt test --select test_type:unit
+
 # Test specific model and downstream
 uv run dbt test --select my_model+
 
 # Run and test in sequence
 uv run dbt run --select my_model && uv run dbt test --select my_model
 ```
 
+## Singular Tests (Business Invariants)
+
+Store custom SQL tests in `tests/` when a rule is business-specific and not covered by generic tests.
+
+Current examples:
+
+- `tests/labels/test_labels_balancer_v2_pools_category.sql`
+- `tests/labels/test_labels_balancer_v3_pools_category.sql`
+
+These tests enforce that labels keep their expected category values.
+
+## Unit Tests (Model Logic with Mocked Inputs)
+
+Use dbt `unit_tests` for models that mostly combine or transform upstream model outputs deterministically.
+
+Current examples are defined in:
+
+- `models/_projects/balancer/labels/_schema.yml`
+  - `labels_balancer_v2_pools_unions_chain_outputs`
+  - `labels_balancer_v3_pools_unions_chain_outputs`
+
+These tests verify that cross-chain union models preserve rows from upstream chain models.
+
+## Local vs CI Test Matrix
+
+- **Pre-commit (Husky):** `dbt parse` only (fast and zero-credit)
+- **PR Quality CI (`dbt_quality.yml`):** `dbt parse` only (fast and zero-credit)
+- **PR Dune CI (`dbt_ci.yml`):** full integration runs/tests against Dune (credit-consuming, label-gated)
+- **Prod workflows (`dbt_deploy.yml`, `dbt_prod.yml`):** deployment and incremental health checks
+
 ## Why These Tests?
 
 **For incremental models:**