Default restatement van external models + QC

In `uv run pipeline` worden de 'raw.*'-modellen (uit dlt) nu standaard gerestate; dat zorgt ervoor dat stg/silver-lagen refreshen bij nieuwe runs

Verder enkele kleine quality-issues opgelost (o.a. Docker health-check)

Author: lukakoning

.github/copilot-instructions.md

@@ -22,11 +22,19 @@ uv run dev                                 # Full dev setup: Docker + Oracle + d
 uv run dev --dest mssql                    # Dev with MSSQL target
 uv run pipeline --dest postgres            # Production pipeline
 uv run pipeline --dest postgres --dry-run  # Preview commands
+uv run pipeline --dest postgres --no-restate-raw  # Skip restatement (model changes only)
 uv run sqlmesh -p sqlmesh plan --auto-apply  # Apply SQLMesh transformations
 uv run python scripts/validate_schema.py   # Validate silver vs DDL (runs in CI)
 uv run pytest                              # Run all tests
 ```
 
+### External Model Restatement
+
+By default, `uv run pipeline` includes `--restate-model raw.*` to ensure stg/silver models are refreshed when new data is loaded. This is required because raw tables are external models managed by dlt.
+
+- **Default behavior**: All raw.* models are restated, triggering cascading backfill of stg.* and silver.*
+- **Skip restatement**: Use `--no-restate-raw` to only apply model changes without refreshing data
+
 ## SQL Model Patterns
 
 ### Staging (`sqlmesh/models/stg/*.sql`)

Dockerfile

@@ -52,16 +52,15 @@ RUN uv sync --frozen --no-dev --no-install-project
 # Copy the rest of the application
 COPY . .
 
-# Set Python path to include the project root
-ENV PYTHONPATH=/app
-
 # Default environment variables (can be overridden)
 # Gateway is auto-detected from destination
 ENV GGM_DESTINATION=postgres
 
 # Healthcheck - verify Python and key modules are available
+# Use `-P` to avoid current working directory shadowing installed packages
+# (this repo contains top-level `dlt/` and `sqlmesh/` project folders).
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-    CMD uv run python -c "import dlt; import sqlmesh; print('OK')" || exit 1
+    CMD uv run python -P -c "from dlt.sources.sql_database import sql_database; from sqlmesh import Context; print('OK')" || exit 1
 
 # Default entrypoint runs the pipeline script
 ENTRYPOINT ["uv", "run", "python", "scripts/pipeline.py"]

dlt/pipeline.py

@@ -24,10 +24,12 @@
 import sys
 from pathlib import Path
 
-# Add parent directory to path to allow imports when running directly
-sys.path.insert(0, str(Path(__file__).parent.parent))
+# NOTE: Do NOT add the repo root to sys.path here.
+# This repository contains a top-level `dlt/` directory (this one) which would
+# shadow the third-party `dlt` package via Python namespace-package resolution.
+# Keeping imports isolated to the script directory avoids that class of failure.
 
-# Import dlt library (using full package name to avoid confusion)
+# Import dlt library
 import dlt as dlt_lib
 from dlt.sources.sql_database import sql_database
 

scripts/dev.py

@@ -132,7 +132,16 @@ def main() -> None:
     sqlmesh_env = {**os.environ}
     subprocess.run(
         _get_sqlmesh_command()
-        + ["-p", "sqlmesh", "--gateway", gateway, "plan", "--auto-apply"],
+        + [
+            "-p",
+            "sqlmesh",
+            "--gateway",
+            gateway,
+            "plan",
+            "--auto-apply",
+            "--restate-model",
+            "raw.*",
+        ],
         cwd=project_root,
         check=True,
         env=sqlmesh_env,

scripts/pipeline.py

@@ -149,17 +149,44 @@ def run_dlt(
     return result.returncode
 
 
-def run_sqlmesh(gateway: str, auto_apply: bool, dry_run: bool, verbose: bool) -> int:
-    """Run SQLMesh: raw -> stg -> silver."""
+def run_sqlmesh(
+    gateway: str,
+    auto_apply: bool,
+    restate_models: list[str] | None = None,
+    dry_run: bool = False,
+    verbose: bool = False,
+) -> int:
+    """Run SQLMesh: raw -> stg -> silver.
+
+    Args:
+        gateway: SQLMesh gateway to use
+        auto_apply: If True, auto-apply the plan without prompting
+        restate_models: List of model patterns to restate. Defaults to ["raw.*"]
+            to ensure stg/silver refresh on new data loads. Pass empty list to skip.
+        dry_run: If True, only show what would be executed
+        verbose: If True, show detailed output
+    """
+    # Default to restating raw.* to ensure stg/silver refresh on new data
+    if restate_models is None:
+        restate_models = ["raw.*"]
+
     print(f"\n{'=' * 60}")
     print(f"  SQLMesh: Transforming raw -> stg -> silver (gateway: {gateway})")
+    if restate_models:
+        print(f"  Restating: {', '.join(restate_models)}")
     print(f"{'=' * 60}\n")
 
     # Use sqlmesh CLI directly to avoid local 'sqlmesh/' directory shadowing the package
     cmd = _get_sqlmesh_command() + ["-p", "sqlmesh", "--gateway", gateway, "plan"]
     if auto_apply:
         cmd.append("--auto-apply")
 
+    # Add restate-model flags for each pattern
+    # This triggers cascading backfill for downstream models (stg, silver)
+    if restate_models:
+        for model in restate_models:
+            cmd.extend(["--restate-model", model])
+
     return run_command(cmd, dry_run=dry_run, verbose=verbose)
 
 
@@ -227,6 +254,12 @@ def main() -> int:
         action="store_true",
         help="Don't auto-apply SQLMesh plan (interactive mode)",
     )
+    parser.add_argument(
+        "--restate-raw",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Restate raw.* external models to refresh stg/silver (default: True)",
+    )
 
     # General options
     parser.add_argument(
@@ -253,6 +286,9 @@ def main() -> int:
     dataset = args.dataset or get_env("GGM_DATASET", DEFAULT_DATASET)
     dlt_backend = args.dlt_backend or get_env("GGM_DLT_BACKEND", "auto")
     auto_apply = not args.no_auto_apply
+    # Restate raw.* by default (unless --no-restate-raw is passed)
+    # This ensures stg/silver are refreshed when new data is loaded to raw
+    restate_raw = args.restate_raw
 
     # Print configuration
     print("\n" + "=" * 60)
@@ -265,6 +301,7 @@ def main() -> int:
     print(f"  dlt         : {'skip' if args.skip_dlt else 'run'}")
     print(f"  SQLMesh     : {'skip' if args.skip_sqlmesh else 'run'}")
     print(f"  Auto-apply  : {auto_apply}")
+    print(f"  Restate raw : {restate_raw}")
     print("=" * 60)
 
     if args.skip_dlt and args.skip_sqlmesh:
@@ -280,7 +317,13 @@ def main() -> int:
 
     # Run SQLMesh
     if not args.skip_sqlmesh:
-        rc = run_sqlmesh(gateway, auto_apply, args.dry_run, args.verbose)
+        # Determine which models to restate
+        # Restating external models (raw.*) triggers cascading backfill of stg/silver
+        # Pass empty list to explicitly disable restatement (None uses default of raw.*)
+        restate_models = ["raw.*"] if restate_raw else []
+        rc = run_sqlmesh(
+            gateway, auto_apply, restate_models, args.dry_run, args.verbose
+        )
         if rc != 0:
             print(f"\n[!] SQLMesh failed with exit code {rc}")
             return rc
@@ -289,9 +332,7 @@ def main() -> int:
     print("\n" + "=" * 60)
     print("  Pipeline complete!")
     print("=" * 60)
-    print(
-        f"\n  Explore with: uv run sqlmesh -p sqlmesh --gateway {gateway} ui"
-    )
+    print(f"\n  Explore with: uv run sqlmesh -p sqlmesh --gateway {gateway} ui")
     print(
         f'  Query data:   uv run sqlmesh -p sqlmesh --gateway {gateway} fetchdf "SELECT * FROM silver.client LIMIT 10"'
     )