diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..5594ebd3 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,206 @@ +# ============================================================ +# isnad-scan GitLab CI/CD Integration Template +# ============================================================ +# This is the ROOT template that demonstrates isnad-scan +# integrated into a full GitLab CI/CD pipeline. +# +# For use as a standalone component in your project, see: +# templates/isnad-scan.gitlab-ci.yml +# +# Documentation: docs/gitlab-ci-integration.md +# ============================================================ + +# ── Configurable Variables ─────────────────────────────────── +variables: + # isnad-scan version to use (use a pinned version in production) + ISNAD_SCAN_VERSION: "latest" + + # Target to scan — can be overridden per-job or via CI/CD settings + # Accepts: directory path, git URL, npm package name, or PyPI package name + ISNAD_SCAN_TARGET: "." + + # Output format: sarif | json | table + ISNAD_SCAN_OUTPUT_FORMAT: "sarif" + + # Path where the report will be written + ISNAD_SCAN_REPORT_PATH: "isnad-report" + + # Minimum trust score to pass (0–100). Jobs fail below this threshold. + ISNAD_SCAN_MIN_TRUST_SCORE: "70" + + # Severity level that triggers a pipeline failure: critical | high | medium | low + ISNAD_SCAN_FAIL_ON_SEVERITY: "high" + + # Space-separated list of rule IDs to skip + ISNAD_SCAN_SKIP_RULES: "" + + # Enable verbose logging: "true" | "false" + ISNAD_SCAN_VERBOSE: "false" + + # Docker image used for scanning jobs + ISNAD_SCAN_IMAGE: "python:3.11-slim" + +# ── Stages ─────────────────────────────────────────────────── +stages: + - scan # isnad security scanning + - report # upload / process results + - gate # quality gate — blocks merge if score too low + +# ── Hidden job: reusable scan setup ───────────────────────── +.isnad_scan_setup: &isnad_scan_setup + image: $ISNAD_SCAN_IMAGE + before_script: + - echo "[isnad] Installing isnad-scan v${ISNAD_SCAN_VERSION}..." + - pip install --quiet --upgrade pip + - | + if [ "$ISNAD_SCAN_VERSION" = "latest" ]; then + pip install --quiet isnad-scan + else + pip install --quiet "isnad-scan==${ISNAD_SCAN_VERSION}" + fi + - isnad-scan --version + - mkdir -p "$ISNAD_SCAN_REPORT_PATH" + +# ── Job: SARIF scan (feeds GitLab Security Dashboard) ──────── +isnad:scan:sarif: + <<: *isnad_scan_setup + stage: scan + script: + - | + echo "[isnad] Starting SARIF scan of target: ${ISNAD_SCAN_TARGET}" + + ARGS="--format sarif" + ARGS="$ARGS --output ${ISNAD_SCAN_REPORT_PATH}/gl-sast-report.sarif" + ARGS="$ARGS --min-trust ${ISNAD_SCAN_MIN_TRUST_SCORE}" + ARGS="$ARGS --fail-on ${ISNAD_SCAN_FAIL_ON_SEVERITY}" + + if [ -n "$ISNAD_SCAN_SKIP_RULES" ]; then + ARGS="$ARGS --skip-rules ${ISNAD_SCAN_SKIP_RULES}" + fi + + if [ "$ISNAD_SCAN_VERBOSE" = "true" ]; then + ARGS="$ARGS --verbose" + fi + + isnad-scan $ARGS "${ISNAD_SCAN_TARGET}" + artifacts: + # GitLab Security Dashboard ingests this artifact automatically + reports: + sast: $ISNAD_SCAN_REPORT_PATH/gl-sast-report.sarif + paths: + - $ISNAD_SCAN_REPORT_PATH/gl-sast-report.sarif + expire_in: 30 days + when: always # upload even on failure so the dashboard shows issues + rules: + # Run on every push and MR; skip if [skip-isnad] is in the commit message + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - if: '$CI_COMMIT_BRANCH' + when: always + - when: never + +# ── Job: JSON scan (machine-readable, for custom tooling) ───── +isnad:scan:json: + <<: *isnad_scan_setup + stage: scan + script: + - | + echo "[isnad] Starting JSON scan of target: ${ISNAD_SCAN_TARGET}" + + isnad-scan \ + --format json \ + --output "${ISNAD_SCAN_REPORT_PATH}/isnad-report.json" \ + --min-trust "${ISNAD_SCAN_MIN_TRUST_SCORE}" \ + --fail-on "${ISNAD_SCAN_FAIL_ON_SEVERITY}" \ + "${ISNAD_SCAN_TARGET}" + artifacts: + paths: + - $ISNAD_SCAN_REPORT_PATH/isnad-report.json + expire_in: 30 days + when: always + rules: + - if: '$ISNAD_ENABLE_JSON_REPORT == "true"' + when: always + - when: never + +# ── Job: Human-readable table output (for MR comments) ──────── +isnad:scan:summary: + <<: *isnad_scan_setup + stage: scan + script: + - | + echo "[isnad] Generating human-readable summary..." + + isnad-scan \ + --format table \ + --output "${ISNAD_SCAN_REPORT_PATH}/isnad-summary.txt" \ + --min-trust "${ISNAD_SCAN_MIN_TRUST_SCORE}" \ + "${ISNAD_SCAN_TARGET}" | tee "${ISNAD_SCAN_REPORT_PATH}/isnad-summary.txt" + artifacts: + paths: + - $ISNAD_SCAN_REPORT_PATH/isnad-summary.txt + expire_in: 7 days + when: always + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: never + +# ── Job: Upload results to isnad registry (optional) ───────── +isnad:report:upload: + image: $ISNAD_SCAN_IMAGE + stage: report + needs: + - job: isnad:scan:sarif + artifacts: true + script: + - pip install --quiet isnad-scan + - | + echo "[isnad] Uploading report to isnad registry..." + + isnad-scan publish \ + --report "${ISNAD_SCAN_REPORT_PATH}/gl-sast-report.sarif" \ + --project "${CI_PROJECT_PATH}" \ + --commit "${CI_COMMIT_SHA}" \ + --ref "${CI_COMMIT_REF_NAME}" \ + --token "${ISNAD_API_TOKEN}" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $ISNAD_API_TOKEN != ""' + when: on_success + - when: never + +# ── Job: Quality gate — block MR if trust score too low ─────── +isnad:gate: + image: $ISNAD_SCAN_IMAGE + stage: gate + needs: + - job: isnad:scan:json + artifacts: true + optional: true + - job: isnad:scan:sarif + artifacts: true + script: + - pip install --quiet isnad-scan jq + - | + echo "[isnad] Running quality gate (min trust score: ${ISNAD_SCAN_MIN_TRUST_SCORE})..." + + # Parse trust score from JSON report if available + if [ -f "${ISNAD_SCAN_REPORT_PATH}/isnad-report.json" ]; then + SCORE=$(jq -r '.summary.trustScore // 0' \ + "${ISNAD_SCAN_REPORT_PATH}/isnad-report.json") + echo "[isnad] Detected trust score: ${SCORE}" + + if [ "$(echo "$SCORE < $ISNAD_SCAN_MIN_TRUST_SCORE" | bc)" = "1" ]; then + echo "[isnad] ❌ GATE FAILED: Trust score ${SCORE} is below threshold ${ISNAD_SCAN_MIN_TRUST_SCORE}" + exit 1 + else + echo "[isnad] ✅ GATE PASSED: Trust score ${SCORE} >= ${ISNAD_SCAN_MIN_TRUST_SCORE}" + fi + else + echo "[isnad] No JSON report found — relying on SARIF exit code." + fi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $ISNAD_ENABLE_GATE == "true"' + when: on_success + - when: never + allow_failure: false diff --git a/.gitlab-ci.yml.example-override b/.gitlab-ci.yml.example-override new file mode 100644 index 00000000..73c7d16e --- /dev/null +++ b/.gitlab-ci.yml.example-override @@ -0,0 +1,17 @@ +# ============================================================ +# EXAMPLE — How to use the component as a CI/CD Catalog item +# ============================================================ +# This shows the GitLab CI Catalog `component` syntax (GitLab 16.0+). +# Use this approach when the template is published to the CI/CD Catalog. +# ============================================================ + +include: + - component: gitlab.com/counterspec/isnad/isnad-scan@main + inputs: + target: "." + min_trust_score: "80" + fail_on_severity: "high" + output_format: "sarif" + +stages: + - test diff --git "a/advanced\\.gitlab-ci.yml" "b/advanced\\.gitlab-ci.yml" new file mode 100644 index 00000000..6710a1af --- /dev/null +++ "b/advanced\\.gitlab-ci.yml" @@ -0,0 +1,140 @@ +# ============================================================ +# EXAMPLE — Advanced isnad-scan integration +# ============================================================ +# Demonstrates: +# • Scanning multiple targets (repo + external package) +# • Custom trust-score threshold +# • JSON report for downstream processing +# • Quality gate blocking merges +# • Scheduled nightly deep scan +# • Custom runner tags +# ============================================================ + +include: + - project: 'counterspec/isnad' + file: 'templates/isnad-scan.gitlab-ci.yml' + ref: main + +variables: + # Raise the trust bar for production-bound code + ISNAD_SCAN_MIN_TRUST_SCORE: "85" + +stages: + - build + - scan + - gate + - deploy + +# ── Override the default scan job with project-specific settings ── +isnad:scan: + stage: scan + variables: + ISNAD_TARGET: "." + ISNAD_FORMAT: "sarif" + ISNAD_MIN_TRUST: "85" + ISNAD_FAIL_ON: "high" + # Suppress known-false-positive rules + ISNAD_SKIP_RULES: "ISNAD-001,ISNAD-007" + rules: + - when: always + +# ── Scan a third-party PyPI package before adding it ──────────── +isnad:scan:dependency: + extends: isnad:scan + stage: scan + variables: + ISNAD_TARGET: "pypi:some-new-dependency" + ISNAD_FORMAT: "json" + ISNAD_MIN_TRUST: "90" + ISNAD_FAIL_ON: "medium" + ISNAD_REPORT_DIR: "isnad-reports/dependency" + artifacts: + paths: + - isnad-reports/dependency/ + expire_in: 7 days + when: always + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: never + +# ── Quality gate: block merge if trust score insufficient ──────── +isnad:gate: + image: python:3.11-slim + stage: gate + needs: + - job: isnad:scan + artifacts: true + script: + - pip install --quiet jq python-json-tool + - | + REPORT="isnad-reports/isnad-report.json" + if [ ! -f "$REPORT" ]; then + echo "[isnad:gate] No JSON report found — skipping numeric gate." + exit 0 + fi + + SCORE=$(python3 -c " + import json, sys + data = json.load(open('$REPORT')) + print(data.get('summary', {}).get('trustScore', 0)) + ") + + echo "[isnad:gate] Trust score: ${SCORE} (required: 85)" + + python3 -c " + score = float('${SCORE}') + threshold = 85.0 + if score < threshold: + print(f'❌ GATE FAILED — score {score} < {threshold}') + raise SystemExit(1) + print(f'✅ GATE PASSED — score {score} >= {threshold}') + " + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: on_success + - when: never + allow_failure: false + +# ── Nightly deep scan (schedule-triggered) ────────────────────── +isnad:nightly: + extends: isnad:scan + stage: scan + variables: + ISNAD_TARGET: "." + ISNAD_FORMAT: "json" + ISNAD_MIN_TRUST: "75" + ISNAD_FAIL_ON: "critical" + # Enable deep scan mode for nightly runs + ISNAD_EXTRA_ARGS: "--deep --include-dev-dependencies" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: always + - when: never + artifacts: + paths: + - isnad-reports/ + expire_in: 90 days + when: always + +# ── Example build / deploy stubs ──────────────────────────────── +build: + stage: build + image: node:20-slim + script: + - echo "Building project..." + rules: + - when: always + +deploy:production: + stage: deploy + image: alpine:latest + script: + - echo "Deploying to production..." + needs: + - build + - isnad:scan + - isnad:gate + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: manual diff --git "a/basic\\.gitlab-ci.yml" "b/basic\\.gitlab-ci.yml" new file mode 100644 index 00000000..1b414564 --- /dev/null +++ "b/basic\\.gitlab-ci.yml" @@ -0,0 +1,17 @@ +# ============================================================ +# EXAMPLE — Basic isnad-scan integration +# ============================================================ +# Scans the repository root on every push and MR. +# Findings appear in the GitLab Security Dashboard (SAST). +# ============================================================ + +include: + - project: 'counterspec/isnad' + file: 'templates/isnad-scan.gitlab-ci.yml' + ref: main + +stages: + - test + +# The include block above injects the `isnad:scan` job. +# No additional configuration is required for a basic setup. diff --git a/gitlab-ci-integration.md b/gitlab-ci-integration.md new file mode 100644 index 00000000..c1087783 --- /dev/null +++ b/gitlab-ci-integration.md @@ -0,0 +1,37 @@ +# isnad-scan — GitLab CI/CD Integration Guide + +## Table of Contents + +1. [Overview](#overview) +2. [Quick Start](#quick-start) +3. [How It Works](#how-it-works) +4. [Configuration Reference](#configuration-reference) +5. [GitLab Security Dashboard](#gitlab-security-dashboard) +6. [Quality Gate](#quality-gate) +7. [Scanning External Packages](#scanning-external-packages) +8. [Scheduled / Nightly Scans](#scheduled--nightly-scans) +9. [Advanced Configuration](#advanced-configuration) +10. [Troubleshooting](#troubleshooting) +11. [CI/CD Variable Reference](#cicd-variable-reference) + +--- + +## Overview + +`isnad-scan` is a proof-of-stake auditing tool for AI agent skills and +software packages. This guide explains how to integrate it into a GitLab +CI/CD pipeline so that: + +- Every merge request is scanned before merge. +- Security findings surface in the **GitLab Security Dashboard**. +- A configurable **quality gate** can block merges with low trust scores. +- Scheduled nightly scans catch newly-discovered issues. + +--- + +## Quick Start + +### 1. Include the component template + +Add the following to your project's `.gitlab-ci.yml`: + diff --git a/isnad-scan.gitlab-ci.yml b/isnad-scan.gitlab-ci.yml new file mode 100644 index 00000000..007dc372 --- /dev/null +++ b/isnad-scan.gitlab-ci.yml @@ -0,0 +1,153 @@ +# ============================================================ +# isnad-scan — Reusable GitLab CI Component +# ============================================================ +# Include this file in your project's .gitlab-ci.yml via the +# `include` keyword (local, remote, or GitLab CI Catalog). +# +# Minimal usage: +# +# include: +# - project: 'counterspec/isnad' +# file: 'templates/isnad-scan.gitlab-ci.yml' +# ref: main +# +# That single include adds the `isnad:scan` job to your pipeline +# and surfaces findings in the GitLab Security Dashboard. +# +# See docs/gitlab-ci-integration.md for full configuration. +# ============================================================ + +spec: + inputs: + # ── Scan target ───────────────────────────────────────── + target: + description: "Path, git URL, npm package, or PyPI package to scan." + default: "." + type: string + + # ── isnad-scan version ─────────────────────────────────── + version: + description: "isnad-scan package version. Pin this in production." + default: "latest" + type: string + + # ── Output format ──────────────────────────────────────── + output_format: + description: "Report format: sarif | json | table" + default: "sarif" + options: ["sarif", "json", "table"] + + # ── Thresholds ─────────────────────────────────────────── + min_trust_score: + description: "Minimum trust score (0–100) required to pass." + default: "70" + type: string + + fail_on_severity: + description: "Fail the pipeline when a finding of this severity or higher exists." + default: "high" + options: ["critical", "high", "medium", "low", "none"] + + # ── Rule exclusions ────────────────────────────────────── + skip_rules: + description: "Comma-separated rule IDs to skip." + default: "" + type: string + + # ── Stage placement ────────────────────────────────────── + stage: + description: "Pipeline stage to run the scan in." + default: "test" + type: string + + # ── Runner tags ────────────────────────────────────────── + runner_tags: + description: "JSON array of GitLab runner tags." + default: '[]' + type: string + +--- + +# ── Reusable scan job ───────────────────────────────────────── +isnad:scan: + stage: $[[ inputs.stage ]] + image: python:3.11-slim + tags: $[[ inputs.runner_tags ]] + + variables: + ISNAD_TARGET: "$[[ inputs.target ]]" + ISNAD_VERSION: "$[[ inputs.version ]]" + ISNAD_FORMAT: "$[[ inputs.output_format ]]" + ISNAD_MIN_TRUST: "$[[ inputs.min_trust_score ]]" + ISNAD_FAIL_ON: "$[[ inputs.fail_on_severity ]]" + ISNAD_SKIP_RULES: "$[[ inputs.skip_rules ]]" + ISNAD_REPORT_DIR: "isnad-reports" + + before_script: + - pip install --quiet --upgrade pip + - | + if [ "$ISNAD_VERSION" = "latest" ]; then + pip install --quiet isnad-scan + else + pip install --quiet "isnad-scan==${ISNAD_VERSION}" + fi + - isnad-scan --version + - mkdir -p "$ISNAD_REPORT_DIR" + + script: + - | + # ── Build argument list ────────────────────────────── + ARGS="--format ${ISNAD_FORMAT}" + ARGS="$ARGS --min-trust ${ISNAD_MIN_TRUST}" + + # Output file extension depends on format + case "$ISNAD_FORMAT" in + sarif) EXT="sarif" ;; + json) EXT="json" ;; + *) EXT="txt" ;; + esac + + REPORT_FILE="${ISNAD_REPORT_DIR}/isnad-report.${EXT}" + ARGS="$ARGS --output ${REPORT_FILE}" + + if [ "$ISNAD_FAIL_ON" != "none" ]; then + ARGS="$ARGS --fail-on ${ISNAD_FAIL_ON}" + fi + + if [ -n "$ISNAD_SKIP_RULES" ]; then + # Convert comma-separated to space-separated flags + for RULE in $(echo "$ISNAD_SKIP_RULES" | tr ',' ' '); do + ARGS="$ARGS --skip-rule ${RULE}" + done + fi + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " isnad-scan" + echo " Target : ${ISNAD_TARGET}" + echo " Format : ${ISNAD_FORMAT}" + echo " Min Trust: ${ISNAD_MIN_TRUST}" + echo " Fail On : ${ISNAD_FAIL_ON}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + isnad-scan $ARGS "${ISNAD_TARGET}" + + echo "" + echo "[isnad] Report written to: ${REPORT_FILE}" + + artifacts: + reports: + # SARIF files are automatically parsed by GitLab Security Dashboard + sast: isnad-reports/isnad-report.sarif + paths: + - isnad-reports/ + expire_in: 30 days + when: always # always upload so failures are visible in the dashboard + + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: always + - if: '$CI_COMMIT_TAG' + when: always + - when: never diff --git "a/npm-package\\.gitlab-ci.yml" "b/npm-package\\.gitlab-ci.yml" new file mode 100644 index 00000000..94824f2b --- /dev/null +++ "b/npm-package\\.gitlab-ci.yml" @@ -0,0 +1,42 @@ +# ============================================================ +# EXAMPLE — Scan an npm package before installing it +# ============================================================ +# Use isnad-scan to vet npm packages in your CI pipeline +# before they ever land in a node_modules folder. +# ============================================================ + +include: + - project: 'counterspec/isnad' + file: 'templates/isnad-scan.gitlab-ci.yml' + ref: main + +variables: + # Package to audit — set this in your CI/CD variables or + # override per-job as shown below. + NPM_PACKAGE_TO_AUDIT: "left-pad@1.3.0" + +stages: + - audit + - install + +isnad:audit:npm: + extends: isnad:scan + stage: audit + variables: + ISNAD_TARGET: "npm:${NPM_PACKAGE_TO_AUDIT}" + ISNAD_FORMAT: "sarif" + ISNAD_MIN_TRUST: "80" + ISNAD_FAIL_ON: "high" + rules: + - when: always + +install:dependencies: + stage: install + image: node:20-slim + needs: + - job: isnad:audit:npm + script: + - echo "isnad-scan passed — safe to install ${NPM_PACKAGE_TO_AUDIT}" + - npm install "${NPM_PACKAGE_TO_AUDIT}" + rules: + - when: on_success diff --git "a/python-package\\.gitlab-ci.yml" "b/python-package\\.gitlab-ci.yml" new file mode 100644 index 00000000..9d258eb9 --- /dev/null +++ "b/python-package\\.gitlab-ci.yml" @@ -0,0 +1,37 @@ +# ============================================================ +# EXAMPLE — Scan a PyPI package before adding to requirements +# ============================================================ + +include: + - project: 'counterspec/isnad' + file: 'templates/isnad-scan.gitlab-ci.yml' + ref: main + +variables: + PYPI_PACKAGE_TO_AUDIT: "requests==2.31.0" + +stages: + - audit + - test + +isnad:audit:pypi: + extends: isnad:scan + stage: audit + variables: + ISNAD_TARGET: "pypi:${PYPI_PACKAGE_TO_AUDIT}" + ISNAD_FORMAT: "sarif" + ISNAD_MIN_TRUST: "80" + ISNAD_FAIL_ON: "high" + rules: + - when: always + +run:tests: + stage: test + image: python:3.11-slim + needs: + - job: isnad:audit:pypi + script: + - pip install "${PYPI_PACKAGE_TO_AUDIT}" + - python -m pytest + rules: + - when: on_success diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..82805bd8 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,6 @@ +# Test dependencies for isnad-scan GitLab CI template tests +pytest>=7.4.0 +pyyaml>=6.0.1 + +# Optional — for CLI smoke tests +# isnad-scan>=1.0.0 diff --git a/test_gitlab_ci_template.py b/test_gitlab_ci_template.py new file mode 100644 index 00000000..2a6dc5ff --- /dev/null +++ b/test_gitlab_ci_template.py @@ -0,0 +1,436 @@ +""" +tests/test_gitlab_ci_template.py +================================= +Validates the isnad-scan GitLab CI/CD templates. + +Tests verify: + 1. Valid YAML syntax for all template files + 2. Required keys and structure + 3. Variable defaults and allowed values + 4. Artifact configuration (SARIF for Security Dashboard) + 5. Rules logic + 6. Example pipeline schemas + +Run with: + pip install pytest pyyaml + pytest tests/test_gitlab_ci_template.py -v +""" + +import os +import pathlib +import pytest +import yaml + +# ── Paths ───────────────────────────────────────────────────── +ROOT = pathlib.Path(__file__).parent.parent +TEMPLATES_DIR = ROOT / "templates" +EXAMPLES_DIR = ROOT / "examples" + +ROOT_CI = ROOT / ".gitlab-ci.yml" +COMPONENT = TEMPLATES_DIR / "isnad-scan.gitlab-ci.yml" +BASIC = EXAMPLES_DIR / "basic" / ".gitlab-ci.yml" +ADVANCED = EXAMPLES_DIR / "advanced" / ".gitlab-ci.yml" +NPM_EXAMPLE = EXAMPLES_DIR / "npm-package" / ".gitlab-ci.yml" +PYPI_EXAMPLE = EXAMPLES_DIR / "python-package" / ".gitlab-ci.yml" + +ALL_FILES = [ROOT_CI, COMPONENT, BASIC, ADVANCED, NPM_EXAMPLE, PYPI_EXAMPLE] + + +# ── Helpers ─────────────────────────────────────────────────── + +def load_yaml(path: pathlib.Path) -> dict: + """Load a YAML file; handle GitLab CI component spec front-matter.""" + content = path.read_text() + + # GitLab CI component templates use `spec:` followed by `---` separator. + # Strip the spec block so standard YAML parsers can read the job definitions. + if content.startswith("spec:"): + # Split on the `---` document separator + parts = content.split("\n---\n", maxsplit=1) + if len(parts) == 2: + content = parts[1] + + return yaml.safe_load(content) + + +# ── Fixture: all template + example paths ───────────────────── + +@pytest.fixture(params=ALL_FILES, ids=lambda p: p.name) +def ci_file(request): + return request.param + + +# ══════════════════════════════════════════════════════════════ +# 1. File existence +# ══════════════════════════════════════════════════════════════ + +class TestFileExistence: + def test_all_files_exist(self, ci_file): + assert ci_file.exists(), f"Expected file not found: {ci_file}" + + def test_all_files_non_empty(self, ci_file): + assert ci_file.stat().st_size > 0, f"File is empty: {ci_file}" + + +# ══════════════════════════════════════════════════════════════ +# 2. YAML validity +# ══════════════════════════════════════════════════════════════ + +class TestYamlValidity: + def test_valid_yaml(self, ci_file): + """All CI files must parse as valid YAML.""" + try: + data = load_yaml(ci_file) + assert data is not None, f"YAML parsed to None: {ci_file}" + except yaml.YAMLError as exc: + pytest.fail(f"YAML parse error in {ci_file}: {exc}") + + def test_yaml_is_dict(self, ci_file): + data = load_yaml(ci_file) + assert isinstance(data, dict), ( + f"Top-level YAML must be a mapping, got {type(data)} in {ci_file}" + ) + + +# ══════════════════════════════════════════════════════════════ +# 3. Root .gitlab-ci.yml — structure +# ══════════════════════════════════════════════════════════════ + +class TestRootCi: + @pytest.fixture(autouse=True) + def _load(self): + self.data = load_yaml(ROOT_CI) + + def test_has_stages(self): + assert "stages" in self.data, "Root CI must define `stages`" + + def test_stages_is_list(self): + assert isinstance(self.data["stages"], list) + + def test_required_stages_present(self): + stages = self.data["stages"] + for required in ("scan", "report"): + assert required in stages, f"Stage `{required}` missing from root CI" + + def test_has_variables(self): + assert "variables" in self.data, "Root CI should define top-level `variables`" + + def test_required_variables_defined(self): + variables = self.data["variables"] + required_vars = [ + "ISNAD_SCAN_VERSION", + "ISNAD_SCAN_TARGET", + "ISNAD_SCAN_OUTPUT_FORMAT", + "ISNAD_SCAN_REPORT_PATH", + "ISNAD_SCAN_MIN_TRUST_SCORE", + "ISNAD_SCAN_FAIL_ON_SEVERITY", + ] + for var in required_vars: + assert var in variables, f"Variable `{var}` missing from root CI" + + def test_default_output_format(self): + fmt = self.data["variables"]["ISNAD_SCAN_OUTPUT_FORMAT"] + assert fmt in ("sarif", "json", "table"), ( + f"Unexpected default output format: {fmt}" + ) + + def test_default_min_trust_score_is_numeric(self): + score = self.data["variables"]["ISNAD_SCAN_MIN_TRUST_SCORE"] + assert str(score).isdigit(), "ISNAD_SCAN_MIN_TRUST_SCORE must be numeric" + assert 0 <= int(score) <= 100, "Trust score must be in range 0–100" + + def test_sarif_scan_job_exists(self): + assert "isnad:scan:sarif" in self.data, ( + "Root CI must define `isnad:scan:sarif` job" + ) + + def test_sarif_job_has_artifacts(self): + job = self.data["isnad:scan:sarif"] + assert "artifacts" in job, "`isnad:scan:sarif` must define artifacts" + + def test_sarif_job_reports_sast(self): + job = self.data["isnad:scan:sarif"] + reports = job.get("artifacts", {}).get("reports", {}) + assert "sast" in reports, ( + "`isnad:scan:sarif` must declare `artifacts.reports.sast` " + "for GitLab Security Dashboard" + ) + + def test_sarif_artifact_when_always(self): + """Artifacts should upload even on failure so the dashboard sees issues.""" + job = self.data["isnad:scan:sarif"] + when = job.get("artifacts", {}).get("when", "on_success") + assert when == "always", ( + "`isnad:scan:sarif` artifacts.when should be `always`" + ) + + def test_json_scan_job_exists(self): + assert "isnad:scan:json" in self.data + + def test_summary_scan_job_exists(self): + assert "isnad:scan:summary" in self.data + + def test_report_upload_job_exists(self): + assert "isnad:report:upload" in self.data + + def test_gate_job_exists(self): + assert "isnad:gate" in self.data + + +# ══════════════════════════════════════════════════════════════ +# 4. Component template — spec and job structure +# ══════════════════════════════════════════════════════════════ + +class TestComponentTemplate: + @pytest.fixture(autouse=True) + def _load(self): + # Load full raw text for spec block checks + self.raw = COMPONENT.read_text() + # Load parsed job portion + self.data = load_yaml(COMPONENT) + + def test_has_spec_block(self): + assert self.raw.startswith("spec:"), ( + "Component template must begin with a `spec:` block" + ) + + def test_has_document_separator(self): + assert "\n---\n" in self.raw, ( + "Component template must use `---` to separate spec from jobs" + ) + + def test_isnad_scan_job_defined(self): + assert "isnad:scan" in self.data, ( + "Component must define an `isnad:scan` job" + ) + + def test_job_has_image(self): + job = self.data["isnad:scan"] + assert "image" in job, "`isnad:scan` must specify an `image`" + + def test_job_has_script(self): + job = self.data["isnad:scan"] + assert "script" in job, "`isnad:scan` must have a `script` section" + + def test_job_has_artifacts(self): + job = self.data["isnad:scan"] + assert "artifacts" in job + + def test_job_has_rules(self): + job = self.data["isnad:scan"] + assert "rules" in job, "`isnad:scan` must define `rules`" + + def test_spec_defines_target_input(self): + assert "inputs.target" in self.raw or '"target"' in self.raw or "target:" in self.raw, ( + "Component spec must expose a `target` input" + ) + + def test_spec_defines_min_trust_input(self): + assert "min_trust_score" in self.raw, ( + "Component spec must expose a `min_trust_score` input" + ) + + def test_spec_defines_output_format_input(self): + assert "output_format" in self.raw + + def test_spec_defines_fail_on_input(self): + assert "fail_on_severity" in self.raw + + +# ══════════════════════════════════════════════════════════════ +# 5. Example pipelines +# ══════════════════════════════════════════════════════════════ + +class TestBasicExample: + @pytest.fixture(autouse=True) + def _load(self): + self.data = load_yaml(BASIC) + + def test_has_include(self): + assert "include" in self.data, "Basic example must use `include`" + + def test_has_stages(self): + assert "stages" in self.data + + +class TestAdvancedExample: + @pytest.fixture(autouse=True) + def _load(self): + self.data = load_yaml(ADVANCED) + + def test_has_include(self): + assert "include" in self.data + + def test_has_gate_job(self): + assert "isnad:gate" in self.data, ( + "Advanced example must define a quality gate job" + ) + + def test_gate_job_allow_failure_false(self): + gate = self.data["isnad:gate"] + allow_failure = gate.get("allow_failure", True) + assert allow_failure is False, ( + "Quality gate must set `allow_failure: false`" + ) + + def test_has_nightly_job(self): + assert "isnad:nightly" in self.data, ( + "Advanced example should define a nightly scheduled scan" + ) + + def test_nightly_triggered_by_schedule(self): + nightly = self.data["isnad:nightly"] + rules = nightly.get("rules", []) + schedule_rule_found = any( + "schedule" in str(r) for r in rules + ) + assert schedule_rule_found, ( + "Nightly job must be triggered by `CI_PIPELINE_SOURCE == 'schedule'`" + ) + + def test_has_deploy_job(self): + assert "deploy:production" in self.data + + +class TestNpmExample: + @pytest.fixture(autouse=True) + def _load(self): + self.data = load_yaml(NPM_EXAMPLE) + + def test_has_include(self): + assert "include" in self.data + + def test_has_audit_job(self): + assert "isnad:audit:npm" in self.data + + def test_has_install_job(self): + assert "install:dependencies" in self.data + + def test_install_needs_audit(self): + install = self.data["install:dependencies"] + needs = install.get("needs", []) + needs_names = [ + (n["job"] if isinstance(n, dict) else n) for n in needs + ] + assert "isnad:audit:npm" in needs_names, ( + "install job must `needs` the audit job to enforce ordering" + ) + + +class TestPypiExample: + @pytest.fixture(autouse=True) + def _load(self): + self.data = load_yaml(PYPI_EXAMPLE) + + def test_has_include(self): + assert "include" in self.data + + def test_has_audit_job(self): + assert "isnad:audit:pypi" in self.data + + def test_has_test_job(self): + assert "run:tests" in self.data + + def test_tests_need_audit(self): + tests = self.data["run:tests"] + needs = tests.get("needs", []) + needs_names = [ + (n["job"] if isinstance(n, dict) else n) for n in needs + ] + assert "isnad:audit:pypi" in needs_names + + +# ══════════════════════════════════════════════════════════════ +# 6. Security / best-practice checks +# ══════════════════════════════════════════════════════════════ + +class TestSecurityBestPractices: + def test_no_hardcoded_tokens(self, ci_file): + """CI files must never hardcode secret values.""" + content = ci_file.read_text().lower() + suspicious_patterns = [ + "glpat-", # GitLab personal access token prefix + "isnad_secret=", + "api_key=", + "password=", + ] + for pattern in suspicious_patterns: + assert pattern not in content, ( + f"Possible hardcoded secret `{pattern}` found in {ci_file}" + ) + + def test_sarif_report_not_excluded_from_artifacts(self): + """The SARIF report must be included in artifacts for dashboard ingestion.""" + data = load_yaml(ROOT_CI) + sarif_job = data.get("isnad:scan:sarif", {}) + paths = sarif_job.get("artifacts", {}).get("paths", []) + # At least one path should reference the report directory or sarif file + has_report_path = any( + "isnad" in str(p).lower() or "report" in str(p).lower() + for p in paths + ) + assert has_report_path, ( + "SARIF artifact path not configured in `isnad:scan:sarif`" + ) + + def test_artifacts_expire_in_set(self): + """Artifacts should have an expiry to avoid storage bloat.""" + data = load_yaml(ROOT_CI) + for job_name, job in data.items(): + if not isinstance(job, dict): + continue + if "artifacts" not in job: + continue + assert "expire_in" in job["artifacts"], ( + f"Job `{job_name}` has artifacts but no `expire_in`" + ) + + +# ══════════════════════════════════════════════════════════════ +# 7. Smoke test — isnad-scan CLI (if installed) +# ══════════════════════════════════════════════════════════════ + +class TestIsnadScanCli: + """ + Optional smoke tests that run only when isnad-scan is installed. + These are skipped in environments without the package. + """ + + @pytest.fixture(autouse=True) + def _skip_if_not_installed(self): + pytest.importorskip( + "isnad_scan", + reason="isnad-scan not installed — skipping CLI smoke tests" + ) + + def test_cli_version(self): + import subprocess + result = subprocess.run( + ["isnad-scan", "--version"], + capture_output=True, text=True + ) + assert result.returncode == 0, ( + f"isnad-scan --version failed: {result.stderr}" + ) + + def test_cli_help(self): + import subprocess + result = subprocess.run( + ["isnad-scan", "--help"], + capture_output=True, text=True + ) + assert result.returncode == 0 + + def test_cli_scan_current_directory(self, tmp_path): + """Scan a safe empty directory — should exit 0.""" + import subprocess + result = subprocess.run( + ["isnad-scan", "--format", "json", + "--output", str(tmp_path / "report.json"), + str(tmp_path)], + capture_output=True, text=True + ) + # Exit code 0 = clean; exit code 2 = findings above threshold + assert result.returncode in (0, 2), ( + f"Unexpected exit code {result.returncode}: {result.stderr}" + )