diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index e882381..6ef677d 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -353,6 +353,10 @@ export default defineConfig({ text: "Pre-requisite Services", link: "/overlay/tutorials/custom-deployment", }, + { + text: "Using Secrets", + link: "/overlay/tutorials/using-secrets", + }, { text: "Running Tests Locally", link: "/overlay/tutorials/running-locally", @@ -376,12 +380,16 @@ export default defineConfig({ }, { text: "Reference", - collapsed: true, + collapsed: false, items: [ { text: "Environment Variables", link: "/overlay/reference/environment-variables", }, + { + text: "Local OCI Testing", + link: "/overlay/reference/local-oci-testing", + }, { text: "Package.json Scripts", link: "/overlay/reference/scripts", diff --git a/docs/guide/configuration/config-files.md b/docs/guide/configuration/config-files.md index 4326dcd..46d9cff 100644 --- a/docs/guide/configuration/config-files.md +++ b/docs/guide/configuration/config-files.md @@ -2,6 +2,8 @@ RHDH deployment uses YAML configuration files for app config, plugins, and secrets. +If you are working inside the overlay repository, see [Overlay Configuration Files](/overlay/test-structure/configuration-files) for overlay-specific behavior and CI integration details. + ## File Structure Create configuration files in `tests/config/`: @@ -137,6 +139,21 @@ When you deploy RHDH, configurations are merged: Later files override earlier files. You only need to specify what's different from defaults. +## Verify the Final Merged Config + +If you need to confirm the final merged configuration: + +1. Run a test once so RHDH is deployed. +2. Inspect the ConfigMaps in the test namespace: + +```bash +oc get configmap -n +oc get configmap app-config-rhdh -n -o yaml +oc get configmap dynamic-plugins -n -o yaml +``` + +This shows the exact content after defaults, auth config, and your overrides are merged. + ## Plugin Metadata Injection During deployment, the package automatically handles plugin configurations from metadata files. The behavior depends on whether your [`dynamic-plugins.yaml`](#dynamic-plugins-yaml) file exists: @@ -219,6 +236,31 @@ When `GIT_PR_NUMBER` is set (by OpenShift CI), local plugin paths are automatica This allows E2E tests to run against the actual OCI images built for the PR. +### Local Reproduction (Optional) + +If you want to reproduce OCI URL replacement locally, create the required files at the workspace root: + +**source.json** +```json +{ + "repo": "https://github.com/redhat-developer/rhdh-plugin-export-overlays", + "repo-ref": "abc123def4567890" +} +``` + +**plugins-list.yaml** +```yaml +plugins/tech-radar: +plugins/my-plugin: +``` + +Then set `GIT_PR_NUMBER` and run tests: + +```bash +export GIT_PR_NUMBER=1845 +yarn test +``` + ::: warning For PR builds, OCI URL generation is required. Deployment will fail if `source.json` or `plugins-list.yaml` doesn't exist, or if version fetching fails. ::: diff --git a/docs/guide/configuration/environment-variables.md b/docs/guide/configuration/environment-variables.md index 2b1e52f..c99225f 100644 --- a/docs/guide/configuration/environment-variables.md +++ b/docs/guide/configuration/environment-variables.md @@ -2,12 +2,14 @@ Complete reference of all environment variables used by the package. -## Required Variables +## Recommended Variables -| Variable | Description | Example | -|----------|-------------|---------| -| `RHDH_VERSION` | RHDH version to deploy | `"1.5"` | -| `INSTALLATION_METHOD` | Deployment method | `"helm"` or `"operator"` | +These are optional but commonly set to control deployment behavior: + +| Variable | Description | Example | Default | +|----------|-------------|---------|---------| +| `RHDH_VERSION` | RHDH version to deploy | `"1.5"` | `"next"` | +| `INSTALLATION_METHOD` | Deployment method | `"helm"` or `"operator"` | `"helm"` | ## Auto-Generated Variables diff --git a/docs/guide/core-concepts/playwright-config.md b/docs/guide/core-concepts/playwright-config.md index 34ec8e2..3126cb0 100644 --- a/docs/guide/core-concepts/playwright-config.md +++ b/docs/guide/core-concepts/playwright-config.md @@ -25,25 +25,28 @@ The `defineConfig` function extends your configuration with sensible defaults fo |---------|-------|-------------| | `testDir` | `./tests` | Test files location | | `timeout` | 90,000ms | Test timeout | -| `expect.timeout` | 30,000ms | Assertion timeout | +| `expect.timeout` | 10,000ms | Assertion timeout | | `retries` | 2 (CI), 0 (local) | Test retries | | `workers` | 50% of CPUs | Parallel workers | -| `fullyParallel` | `true` | Parallel test execution | +| `outputDir` | `node_modules/.cache/e2e-test-results` | Playwright artifacts | ### Reporter Settings | Setting | Value | |---------|-------| -| `reporter` | `[["list"], ["html"]]` | +| `reporter` | `[["list"], ["html"], ["json"]]` | ### Browser Settings | Setting | Value | |---------|-------| -| `viewport` | `{ width: 1920, height: 1080 }` | -| `video` | `"on"` | +| `ignoreHTTPSErrors` | `true` | | `trace` | `"retain-on-failure"` | | `screenshot` | `"only-on-failure"` | +| `viewport` | `{ width: 1920, height: 1080 }` | +| `video` | `"on"` | +| `actionTimeout` | 10,000ms | +| `navigationTimeout` | 50,000ms | ## Global Setup @@ -55,22 +58,12 @@ The base configuration includes a global setup that runs before all tests: ## Customizing Configuration -You can override any setting by passing it to `defineConfig`: +`defineConfig` only accepts `projects` overrides. To change other settings, use `baseConfig` with Playwright's `defineConfig`. ```typescript import { defineConfig } from "rhdh-e2e-test-utils/playwright-config"; export default defineConfig({ - // Override timeout - timeout: 120000, - - // Override retries - retries: 3, - - // Override workers - workers: 2, - - // Add projects projects: [ { name: "my-plugin", @@ -81,9 +74,6 @@ export default defineConfig({ testMatch: "**/another-*.spec.ts", }, ], - - // Add custom reporter - reporter: [["list"], ["html"], ["json", { outputFile: "results.json" }]], }); ``` @@ -179,12 +169,14 @@ GITHUB_TOKEN=ghp_xxxxx ## Example: Full Configuration ```typescript -import { defineConfig } from "rhdh-e2e-test-utils/playwright-config"; +import { baseConfig } from "rhdh-e2e-test-utils/playwright-config"; +import { defineConfig } from "@playwright/test"; import dotenv from "dotenv"; dotenv.config({ path: `${import.meta.dirname}/.env` }); export default defineConfig({ + ...baseConfig, // Test settings timeout: 120000, retries: process.env.CI ? 3 : 0, diff --git a/docs/guide/core-concepts/playwright-fixtures.md b/docs/guide/core-concepts/playwright-fixtures.md index 0fbd830..e24a67a 100644 --- a/docs/guide/core-concepts/playwright-fixtures.md +++ b/docs/guide/core-concepts/playwright-fixtures.md @@ -25,9 +25,9 @@ This import replaces the standard Playwright import and provides additional fixt The `rhdh` fixture is worker-scoped, meaning: -- One deployment is shared across all tests in a worker -- Deployment happens once per worker, not per test -- All tests in the same worker share the same RHDH instance +- One deployment object is shared across all tests in a worker +- You control *when* deployment happens (usually in `test.beforeAll`) +- All tests in the same worker can share the same RHDH instance ```typescript test.beforeAll(async ({ rhdh }) => { @@ -165,6 +165,47 @@ For local development: - Namespaces are preserved for debugging - Manual cleanup may be required +## Best Practices for Projects and Spec Files + +Each Playwright project name creates a **separate namespace**. The fixture creates one `RHDHDeployment` per worker, and you typically call `rhdh.deploy()` once in `beforeAll`. + +**Recommended for overlay workspaces:** + +- Use **one Playwright project** named after the workspace. +- Keep **one spec file** per workspace unless you have a strong reason to split. + +This keeps deployment cost low and avoids multiple namespaces unless required. + +## When You Need Multiple Projects or Spec Files + +If requirements differ (different auth, configs, or namespaces), you can: + +1. **Use multiple projects** with different names and config overrides. +2. **Manually manage deployments** using `RHDHDeployment` for advanced flows. + +Example using multiple projects: + +```typescript +export default defineConfig({ + projects: [ + { name: "workspace-default" }, + { name: "workspace-guest" }, + ], +}); +``` + +Example with manual deployment: + +```typescript +import { RHDHDeployment } from "rhdh-e2e-test-utils/rhdh"; + +test.beforeAll(async () => { + const rhdh = new RHDHDeployment("custom-namespace"); + await rhdh.configure({ auth: "guest" }); + await rhdh.deploy(); +}); +``` + ## Example: Complete Test Setup ```typescript diff --git a/docs/guide/deployment/rhdh-deployment.md b/docs/guide/deployment/rhdh-deployment.md index 61a5b2c..f0b7305 100644 --- a/docs/guide/deployment/rhdh-deployment.md +++ b/docs/guide/deployment/rhdh-deployment.md @@ -48,9 +48,9 @@ test("example", async ({ rhdh }) => { | Option | Type | Description | |--------|------|-------------| -| `version` | `string` | RHDH version (e.g., "1.5"). Defaults to `RHDH_VERSION` env var | +| `version` | `string` | RHDH version (e.g., "1.5"). Defaults to `RHDH_VERSION` or `"next"` | | `namespace` | `string` | Kubernetes namespace. Set via constructor | -| `method` | `"helm" \| "operator"` | Installation method. Defaults to `INSTALLATION_METHOD` env var | +| `method` | `"helm" \| "operator"` | Installation method. Defaults to `INSTALLATION_METHOD` or `"helm"` | | `auth` | `"guest" \| "keycloak"` | Authentication provider. Defaults to `"keycloak"` | | `appConfig` | `string` | Path to app-config YAML | | `secrets` | `string` | Path to secrets YAML | @@ -102,6 +102,26 @@ This method: 6. Waits for the deployment to be ready 7. Sets `RHDH_BASE_URL` environment variable +#### Base URL format + +The base URL prefix depends on the installation method: + +- Helm: `https://redhat-developer-hub-.` +- Operator: `https://backstage-developer-hub-.` + +#### Helm behavior + +Helm deployments perform a scale-down and restart after applying configs to avoid migration locks. + +#### Operator version constraints + +Operator deployments accept only: + +- Semantic versions like `1.5` +- `"next"` + +Any other value will throw an error during deployment. + ### `waitUntilReady(timeout?)` Wait for the RHDH deployment to be ready: diff --git a/docs/guide/installation.md b/docs/guide/installation.md index a4ed48b..6f68b95 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -41,74 +41,10 @@ import { defineConfig } from "rhdh-e2e-test-utils/playwright-config"; ## Project Setup -### 1. Create E2E Test Directory +For full project scaffolding (config files, folder structure, and first test), follow: -```bash -mkdir e2e-tests && cd e2e-tests -yarn init -y -``` - -### 2. Install Dependencies - -```bash -yarn add @playwright/test rhdh-e2e-test-utils typescript -``` - -### 3. Create Configuration Files - -Create the following files in your project: - -**playwright.config.ts** -```typescript -import { defineConfig } from "rhdh-e2e-test-utils/playwright-config"; - -export default defineConfig({ - projects: [ - { - name: "my-plugin", - }, - ], -}); -``` - -**tsconfig.json** -```json -{ - "extends": "rhdh-e2e-test-utils/tsconfig", - "include": ["tests/**/*.ts"] -} -``` - -**eslint.config.js** -```javascript -import { createEslintConfig } from "rhdh-e2e-test-utils/eslint"; - -export default createEslintConfig(import.meta.dirname); -``` - -### 4. Create Test Directory Structure - -```bash -mkdir -p tests/config tests/specs -``` - -Your project structure should look like: - -``` -e2e-tests/ -├── package.json -├── playwright.config.ts -├── tsconfig.json -├── eslint.config.js -├── .env # Environment variables -└── tests/ - ├── config/ - │ ├── app-config-rhdh.yaml # RHDH app configuration - │ ├── dynamic-plugins.yaml # Plugin configuration - │ └── rhdh-secrets.yaml # Secrets template - └── specs/ - └── my-plugin.spec.ts # Test files -``` +- [Quick Start](/guide/quick-start) - complete setup walkthrough +- [Your First Test](/tutorials/first-test) - step-by-step with explanations ## Next Steps diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md index 09e25dd..4a9b0af 100644 --- a/docs/guide/quick-start.md +++ b/docs/guide/quick-start.md @@ -38,7 +38,7 @@ export default defineConfig({ }); ``` -## Step 3: Create Configuration Files +## Step 3: Create Configuration Files (Optional) Create `tests/config/app-config-rhdh.yaml`: @@ -49,7 +49,7 @@ app: # Add plugin-specific configuration here ``` -Create `tests/config/dynamic-plugins.yaml`: +Create `tests/config/dynamic-plugins.yaml` only if you need to override plugin defaults or you do not have metadata files: ```yaml includes: @@ -62,7 +62,7 @@ plugins: disabled: false ``` -Create `tests/config/rhdh-secrets.yaml`: +Create `tests/config/rhdh-secrets.yaml` only if you need to pass secrets into RHDH config: ```yaml apiVersion: v1 @@ -83,6 +83,8 @@ For PR builds in CI (when `GIT_PR_NUMBER` is set), the package also automaticall See [Plugin Metadata Injection](/guide/configuration/config-files#plugin-metadata-injection) for details. ::: +All files in `tests/config/` are optional; only create what you need. For merge order and file behavior, see [Configuration Files](/guide/configuration/config-files). + ## Step 4: Create Environment File Create `.env`: diff --git a/docs/guide/requirements.md b/docs/guide/requirements.md index 36b6937..973803d 100644 --- a/docs/guide/requirements.md +++ b/docs/guide/requirements.md @@ -87,7 +87,7 @@ oc auth can-i create configmap | Component | Description | |-----------|-------------| -| RHDH Version | Set via `RHDH_VERSION` environment variable (e.g., "1.5") | +| RHDH Version | Set via `RHDH_VERSION` environment variable (default: `next`) | | Helm Chart | Default: `oci://quay.io/rhdh/chart` (customizable via `CHART_URL`) | ## Keycloak Requirements (Optional) @@ -124,7 +124,7 @@ Minimum cluster resources for running tests: | Variable | Description | Example | |----------|-------------|---------| -| `RHDH_VERSION` | RHDH version to deploy | `"1.5"` | +| `RHDH_VERSION` | RHDH version to deploy (default: `next`) | `"1.5"` | | `INSTALLATION_METHOD` | Deployment method | `"helm"` or `"operator"` | ### Optional diff --git a/docs/guide/utilities/environment-substitution.md b/docs/guide/utilities/environment-substitution.md index e470193..a5aa4b4 100644 --- a/docs/guide/utilities/environment-substitution.md +++ b/docs/guide/utilities/environment-substitution.md @@ -78,6 +78,8 @@ stringData: When deployed, environment variables are substituted automatically. +If you are writing tests in the overlay repository, see [Overlay Configuration Files](/overlay/test-structure/configuration-files) for how secrets and substitution are applied in CI. + ## Complete Example ```typescript diff --git a/docs/index.md b/docs/index.md index 4af2739..5b1ffd1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,26 +18,23 @@ hero: features: - icon: 🚀 - title: Automated RHDH Deployment - details: Deploy RHDH instances via Helm or the RHDH Operator with automatic namespace management and cleanup. + title: Deploy RHDH in Tests + details: Deploy via Helm or Operator with automatic namespace management and cleanup. + - icon: 🧪 + title: Playwright Fixtures & Config + details: Fixtures manage deployment lifecycle with rhdh, uiHelper, loginHelper, and baseURL. - icon: 🔑 - title: Keycloak Integration - details: Deploy and configure Keycloak for OIDC authentication testing with pre-configured realms, clients, and users. - - icon: 🧩 - title: Playwright Integration - details: Custom test fixtures that manage deployment lifecycle with rhdh, uiHelper, loginHelper, and baseURL fixtures. - - icon: 📦 - title: Helper Classes - details: UIhelper for Material-UI interactions, LoginHelper for multi-provider auth, APIHelper for GitHub and Backstage APIs. - - icon: 📄 - title: Page Objects - details: Pre-built page objects for CatalogPage, HomePage, CatalogImportPage, ExtensionsPage, and NotificationPage. + title: Keycloak OIDC Testing + details: Deploy and configure Keycloak for auth testing with pre-configured realms, clients, and users. - icon: ⚙️ - title: Configuration Tools - details: ESLint configuration factory with Playwright best practices and TypeScript base configuration. - - icon: 🔧 - title: Overlay Repository Support - details: Dedicated documentation for writing E2E tests in the rhdh-plugin-export-overlays repository. + title: Metadata Auto‑Config + details: Generate dynamic plugin configuration from metadata with sensible defaults. + - icon: 📦 + title: Helpers & Page Objects + details: UIhelper, LoginHelper, APIHelper, and page objects for common RHDH workflows. + - icon: 🧩 + title: Utilities & Tooling + details: Kubernetes helpers, YAML merging, env substitution, ESLint, and TS config. --- ## Quick Example @@ -70,6 +67,13 @@ test("verify catalog", async ({ uiHelper }) => { }); ``` +## Choose Your Path + +::: tip Overlay or Package? +If you're working in `rhdh-plugin-export-overlays`, start with [Overlay Testing](/overlay/). +Otherwise, start with the [Guide](/guide/). +::: + ## What is rhdh-e2e-test-utils? `rhdh-e2e-test-utils` is a comprehensive test utility package for Red Hat Developer Hub (RHDH) end-to-end testing. It provides a unified framework for: diff --git a/docs/overlay/getting-started.md b/docs/overlay/getting-started.md index a4d8247..b2896cb 100644 --- a/docs/overlay/getting-started.md +++ b/docs/overlay/getting-started.md @@ -26,7 +26,7 @@ export default defineConfig({ }); ``` -The package provides all Playwright settings, timeouts, and reporter configuration. You just specify your project name. +The package provides Playwright settings, timeouts, and reporter configuration. You just specify your project name. If your setup takes longer (e.g., external services or slow clusters), increase the timeout in `beforeAll` as shown in [Common Patterns](/overlay/reference/patterns#long-running-setup-and-deployments). ## The Test @@ -104,11 +104,7 @@ workspaces/my-plugin/ ## When You Need Configuration -Only create configuration files if your plugin needs: - -- **External service URLs** → Create `app-config-rhdh.yaml` -- **Secrets in RHDH configs** → Create `rhdh-secrets.yaml` -- **Custom plugin settings** → Override with `dynamic-plugins.yaml` +Only create configuration files if you need to override or extend defaults. If the defaults already include the config keys you need, you can often just provide secrets without adding new config files. See [Configuration Files](./test-structure/configuration-files) for details. diff --git a/docs/overlay/index.md b/docs/overlay/index.md index a5837b4..be56f93 100644 --- a/docs/overlay/index.md +++ b/docs/overlay/index.md @@ -55,6 +55,26 @@ When testing PRs, the `/publish` command builds OCI images that RHDH uses automa See [CI Pipeline](./tutorials/ci-pipeline#pr-oci-image-builds) for details. +## Visual Flows + +### Metadata → Config → Deploy + +```text +metadata/*.yaml + -> auto-generate plugin config + -> merge with defaults + auth config (if exists) + -> deploy RHDH +``` + +### Vault → Secrets → Config + +```text +Vault / .env + -> rhdh-secrets.yaml + -> app-config-rhdh.yaml + -> deploy RHDH +``` + ## Quick Start 1. Navigate to your plugin workspace in the overlay repository @@ -65,6 +85,13 @@ See [CI Pipeline](./tutorials/ci-pipeline#pr-oci-image-builds) for details. See [Getting Started](./getting-started) for a step-by-step walkthrough. +## Common Tasks + +- [Add tests to a new workspace](./tutorials/new-workspace) +- [Run tests locally](./tutorials/running-locally) +- [Run in CI / OpenShift pipeline](./tutorials/ci-pipeline) +- [Trigger PR OCI build (`/publish`)](./tutorials/ci-pipeline#pr-oci-image-builds) + ## Documentation Structure ### Test Structure diff --git a/docs/overlay/reference/local-oci-testing.md b/docs/overlay/reference/local-oci-testing.md new file mode 100644 index 0000000..174b19b --- /dev/null +++ b/docs/overlay/reference/local-oci-testing.md @@ -0,0 +1,43 @@ +# Local OCI Testing + +This page explains how to run overlay tests locally using OCI images built from a PR. + +## When to Use This + +Use local OCI testing when you want your local tests to use the **same OCI images** that CI uses for a PR build. + +## Requirements + +- You have a PR number with OCI images already built via `/publish` +- You have access to the workspace root files: + - `source.json` + - `plugins-list.yaml` + +## Steps + +1. Go to the workspace root (not the `e2e-tests` directory). +2. Ensure `source.json` and `plugins-list.yaml` exist. +3. Set `GIT_PR_NUMBER` and run tests. + +```bash +export GIT_PR_NUMBER=1845 +yarn test +``` + +## What Happens + +When `GIT_PR_NUMBER` is set, the framework: + +- Reads `source.json` and `plugins-list.yaml` +- Fetches plugin versions from the source repository +- Replaces local plugin paths with OCI references for that PR + +## Troubleshooting + +- Missing `source.json` or `plugins-list.yaml` will fail the deployment. +- If OCI URLs cannot be generated, check the PR build logs and ensure `/publish` completed successfully. + +## Related Pages + +- [Configuration Files](../test-structure/configuration-files) +- [CI Pipeline](../tutorials/ci-pipeline) diff --git a/docs/overlay/reference/patterns.md b/docs/overlay/reference/patterns.md index 7f850cf..81c74a5 100644 --- a/docs/overlay/reference/patterns.md +++ b/docs/overlay/reference/patterns.md @@ -24,6 +24,15 @@ test.describe("Plugin tests", () => { }); ``` +## Project and Spec Best Practices + +Each Playwright project name creates a **separate namespace**. To keep deployments fast and predictable: + +- Use **one project** named after the workspace. +- Keep **one spec file** per workspace unless your requirements differ. + +If you need different auth/configs or separate namespaces, use multiple projects or manage deployments directly with `RHDHDeployment`. + ### Guest Authentication Setup ```typescript @@ -208,6 +217,39 @@ await page.locator('text="Content"').waitFor({ state: "visible" }); await page.waitForSelector('[data-testid="loaded"]'); ``` +## Long-Running Setup and Deployments (Timeouts) + +If `beforeAll` performs slow setup (deployment, external services), increase the timeout explicitly. + +### Which Timeout Are You Increasing? + +- `test.setTimeout(...)` inside `beforeAll` increases the **timeout for that hook**. +- `test.setTimeout(...)` inside a test increases the **timeout for that test**. +- The Playwright config `timeout` is the **default per-test timeout**. + +### Increase `beforeAll` Timeout + +```typescript +test.beforeAll(async ({ rhdh }) => { + test.setTimeout(10 * 60 * 1000); // 10 minutes + await rhdh.configure({ auth: "keycloak" }); + await rhdh.deploy(); +}); +``` + +### Extend RHDH Readiness Wait + +```typescript +test.beforeAll(async ({ rhdh }) => { + test.setTimeout(10 * 60 * 1000); + await rhdh.configure({ auth: "keycloak" }); + await rhdh.deploy(); + await rhdh.waitUntilReady(600); // seconds +}); +``` + +Note: `rhdh.deploy()` already increases the test timeout (500s). If your setup does more work before deploy, set a higher timeout in `beforeAll`. + ## Error Handling Patterns ### Expect Error Message diff --git a/docs/overlay/reference/troubleshooting.md b/docs/overlay/reference/troubleshooting.md index b9310f9..0a1d3af 100644 --- a/docs/overlay/reference/troubleshooting.md +++ b/docs/overlay/reference/troubleshooting.md @@ -103,7 +103,14 @@ oc login --token= --server= ```bash oc logs -f deployment/rhdh -n ``` -- Increase timeout in config +- Increase timeout in setup: + ```typescript + test.beforeAll(async ({ rhdh }) => { + test.setTimeout(10 * 60 * 1000); + await rhdh.configure({ auth: "keycloak" }); + await rhdh.deploy(); + }); + ``` ### "ImagePullBackOff" @@ -210,6 +217,16 @@ oc login --token= --server= ## OpenShift CI Issues +### OCI and Metadata Errors + +**"source.json not found" or "plugins-list.yaml not found"** +- These files are required when `GIT_PR_NUMBER` is set. +- In CI they are generated automatically; for local runs, copy them from a CI job or create them manually. + +**"metadata directory not found" or "no valid metadata files"** +- Ensure `workspaces//metadata/` exists and contains valid Package CRD YAML files. +- If you intentionally want to skip metadata injection, set `RHDH_SKIP_PLUGIN_METADATA_INJECTION=true`. + ### "Tests pass locally but fail in CI" **Common causes:** diff --git a/docs/overlay/repository-structure.md b/docs/overlay/repository-structure.md index 82a6d77..a82a311 100644 --- a/docs/overlay/repository-structure.md +++ b/docs/overlay/repository-structure.md @@ -76,7 +76,7 @@ E2E tests are kept separate from plugin source code because: ## Configuration Files Location ::: tip All Configuration Files Are Optional -The `tests/config/` directory can be empty or omitted entirely. The package provides sensible defaults and auto-generates plugin configuration from metadata. Only create files when you need to override defaults. +The `tests/config/` directory can be empty or omitted entirely. The package provides sensible defaults and auto-generates plugin configuration from metadata. Only create files when you need to override or extend defaults. ::: Configuration files in `tests/config/` are merged with package defaults: diff --git a/docs/overlay/test-structure/configuration-files.md b/docs/overlay/test-structure/configuration-files.md index c996000..7479d3f 100644 --- a/docs/overlay/test-structure/configuration-files.md +++ b/docs/overlay/test-structure/configuration-files.md @@ -7,13 +7,12 @@ For using rhdh-e2e-test-utils in external projects, see the [Guide](/guide/). This page explains the YAML configuration files used in overlay E2E tests. +If you're looking for the general file formats and examples, see [Configuration Files (Guide)](/guide/configuration/config-files). This page focuses on overlay-specific behavior such as metadata auto-generation and OCI URL replacement. + ## All Configuration Files Are Optional ::: warning Key Concept -**All configuration files in `tests/config/` are optional.** The package provides sensible defaults for everything. Only create a configuration file when you need to: -- Override a default value -- Add plugin-specific configuration -- Configure external service URLs +**All configuration files in `tests/config/` are optional.** The package provides sensible defaults. Only create a configuration file when you need to **override** or **extend** the defaults (i.e., when the default config does not already cover your use case). ::: If your plugin works with the package defaults and metadata-based configuration, you may not need any configuration files at all. @@ -31,17 +30,15 @@ tests/config/ └── subscription.yaml # Operator subscription (optional, Operator only) ``` -**All of these files are optional.** Only create them when you need to override defaults or add additional configuration. +**All of these files are optional.** Only create them when you need to override or extend defaults. ## app-config-rhdh.yaml (Optional) The main RHDH configuration file. This file is merged with default configurations from `rhdh-e2e-test-utils`. **Only create this file when you need to:** -- Set plugin-specific configuration values (e.g., `techRadar.url`) -- Allow external hosts for backend reading -- Customize the RHDH instance title -- Override any default RHDH configuration +- Override a default value in the RHDH app config +- Add config keys that are not provided by the defaults ### Purpose @@ -105,7 +102,7 @@ techRadar: A Kubernetes Secret manifest that bridges environment variables to the RHDH deployment. -**Only create this file when you need to use environment variables in RHDH configuration files.** +**Only create this file when you need to pass environment variables into RHDH configuration files.** ### Environment Variable Substitution @@ -328,6 +325,8 @@ export GIT_PR_NUMBER=1845 yarn test ``` +For a full local workflow, see [Local OCI Testing](../reference/local-oci-testing). + ### What You Don't Need to Do The OCI URL replacement is **automatic**. You don't need to: diff --git a/docs/overlay/test-structure/directory-layout.md b/docs/overlay/test-structure/directory-layout.md index ed19881..ebccb4f 100644 --- a/docs/overlay/test-structure/directory-layout.md +++ b/docs/overlay/test-structure/directory-layout.md @@ -140,7 +140,7 @@ Optional file for environment variables. Can contain: Contains YAML configuration files that are merged with defaults when deploying RHDH. ::: tip All Files Are Optional -**All configuration files in this directory are optional.** The package provides sensible defaults. Only create files when you need to override defaults or add plugin-specific configuration. +**All configuration files in this directory are optional.** The package provides sensible defaults. Only create files when you need to override or extend defaults. ::: | File | Purpose | When to Create | diff --git a/docs/overlay/tutorials/ci-pipeline.md b/docs/overlay/tutorials/ci-pipeline.md index 3bed10e..4109f4b 100644 --- a/docs/overlay/tutorials/ci-pipeline.md +++ b/docs/overlay/tutorials/ci-pipeline.md @@ -19,6 +19,29 @@ The CI system automatically: - Runs the E2E tests - Reports results on the PR +## e2e-ocp-helm Job + +The `e2e-ocp-helm` job runs Helm-based E2E tests on an OpenShift cluster. + +### When It Runs + +- Automatically after `/publish` completes successfully (and the workspace contains `e2e-tests`) +- Manually via PR comment: + +``` +/test e2e-ocp-helm +``` + +### Where to Find the Playwright Report + +1. Open the **ci/prow/e2e-ocp-helm** check on the PR +2. Navigate to the Prow job page +3. Open: + +``` +artifacts/e2e-ocp-helm/redhat-developer-rhdh-plugin-export-overlays-ocp-helm/artifacts/playwright-report/index.html +``` + ## PR OCI Image Builds When testing a PR, the plugins need to be built into OCI images that RHDH can use. This is handled through the `/publish` command. @@ -54,6 +77,8 @@ export GIT_PR_NUMBER=1845 yarn test ``` +See [Local OCI Testing](/overlay/reference/local-oci-testing) for the full local workflow and required files. + ### What You Don't Need to Do The OCI URL replacement is **automatic**. You don't need to: @@ -76,140 +101,12 @@ In CI, these are generated automatically. For local testing with `GIT_PR_NUMBER` OCI URL generation is strict - deployment will fail if required files are missing or version fetching fails. This prevents builds from silently falling back to local paths. ::: -## Vault Secrets - -Secrets are managed through [HashiCorp Vault](https://vault.ci.openshift.org) and automatically exported as environment variables during CI execution. - -### Secret Naming Convention - -All secrets **must** start with the `VAULT_` prefix: - -``` -VAULT_MY_SECRET_NAME -VAULT_GITHUB_TOKEN -VAULT_API_KEY -``` - -This prefix differentiates secrets from Vault and ensures they are automatically exported during CI execution. - -### Global Secrets - -Global secrets are available to **all** workspace tests. Use these for common secrets shared across multiple plugins. - -**Vault Path:** [Global Secrets](https://vault.ci.openshift.org/ui/vault/secrets/kv/kv/selfservice%2Frhdh-plugin-export-overlays%2Fglobal/details) - -### Workspace-Specific Secrets - -Secrets that are only needed for a specific plugin workspace should be stored in the workspace-specific path. - -**Example for tech-radar workspace:** [Tech Radar Secrets](https://vault.ci.openshift.org/ui/vault/secrets/kv/kv/selfservice%2Frhdh-plugin-export-overlays%2Fworkspaces%2Ftech-radar/details) - -**Pattern:** -``` -selfservice/rhdh-plugin-export-overlays/workspaces/ -``` +## Secrets in CI -### Required Vault Annotations - -Each workspace-specific secret path must include these annotations for OpenShift CI to automatically import the secrets: - -```json -{ - "secretsync/target-name": "rhdh-plugin-export-overlays", - "secretsync/target-namespace": "test-credentials" -} -``` - -### Requesting Vault Access - -If you don't have access to the Vault, reach out in the team-rhdh channel to request access. - -## Using Vault Secrets - -There are two ways to use Vault secrets, depending on where you need them: - -### In Test Code (Direct Access) - -For use in test code (`*.spec.ts`), access secrets directly via `process.env`: - -```typescript -test.beforeAll(async ({ rhdh }) => { - // Direct access - no rhdh-secrets.yaml needed - const apiKey = process.env.VAULT_API_KEY; - - if (!apiKey) { - throw new Error("VAULT_API_KEY is not set"); - } - - await rhdh.configure({ auth: "keycloak" }); - await rhdh.deploy(); -}); -``` - -### In RHDH Configuration Files - -To use Vault secrets in `app-config-rhdh.yaml` or `dynamic-plugins.yaml`, you must first add them to `rhdh-secrets.yaml`: - -#### Step 1: Add to rhdh-secrets.yaml - -**tests/config/rhdh-secrets.yaml:** -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: rhdh-secrets -type: Opaque -stringData: - # Left side: name to use in app-config - # Right side: reference to Vault secret (with $) - EXTERNAL_HOST: $VAULT_EXTERNAL_HOST - MY_PLUGIN_API_KEY: $VAULT_MY_PLUGIN_API_KEY -``` - -#### Step 2: Use in app-config-rhdh.yaml - -**tests/config/app-config-rhdh.yaml:** -```yaml -backend: - reading: - allow: - - host: ${EXTERNAL_HOST} -myPlugin: - apiKey: ${MY_PLUGIN_API_KEY} -``` - -### Summary - -| Where you need it | How to access | -|-------------------|---------------| -| Test code (`*.spec.ts`) | `process.env.VAULT_*` directly | -| RHDH configs | Add to `rhdh-secrets.yaml` first | +Vault setup and usage details are documented here: [Using Secrets](/overlay/tutorials/using-secrets). See [Configuration Files - rhdh-secrets.yaml](/overlay/test-structure/configuration-files#rhdh-secrets-yaml-optional) for more details on the secrets flow. -## Adding a New Workspace to CI - -When adding E2E tests to a new workspace: - -1. **Create workspace-specific secret path in Vault:** - ``` - selfservice/rhdh-plugin-export-overlays/workspaces/ - ``` - -2. **Add required annotations:** - ```json - { - "secretsync/target-name": "rhdh-plugin-export-overlays", - "secretsync/target-namespace": "test-credentials" - } - ``` - -3. **Add secrets with `VAULT_` prefix:** - ``` - VAULT_YOUR_SECRET: - ``` - -4. **Reference secrets in your configuration files** ## CI Environment Variables diff --git a/docs/overlay/tutorials/running-locally.md b/docs/overlay/tutorials/running-locally.md index eb0e5d1..b9f34dc 100644 --- a/docs/overlay/tutorials/running-locally.md +++ b/docs/overlay/tutorials/running-locally.md @@ -153,7 +153,7 @@ SKIP_KEYCLOAK_DEPLOYMENT=false | Variable | Description | Default | |----------|-------------|---------| -| `RHDH_VERSION` | RHDH version to deploy | Latest | +| `RHDH_VERSION` | RHDH version to deploy | `next` (latest) | | `INSTALLATION_METHOD` | `helm` or `operator` | `helm` | | `SKIP_KEYCLOAK_DEPLOYMENT` | Skip Keycloak deployment entirely (for guest auth) | `false` | diff --git a/docs/overlay/tutorials/using-secrets.md b/docs/overlay/tutorials/using-secrets.md new file mode 100644 index 0000000..353780e --- /dev/null +++ b/docs/overlay/tutorials/using-secrets.md @@ -0,0 +1,130 @@ +# Using Secrets + +This page explains how to consume Vault secrets in overlay E2E tests. + +## Where Secrets Come From + +In OpenShift CI, Vault secrets are exported as environment variables with the `VAULT_` prefix. + +## Vault Setup (CI) + +### Secret Naming Convention + +All secrets must start with the `VAULT_` prefix (e.g., `VAULT_API_KEY`). + +### Global Secrets + +Global secrets are available to **all** workspace tests. Use these for shared values. + +**Vault Path:** [Global Secrets](https://vault.ci.openshift.org/ui/vault/secrets/kv/kv/selfservice%2Frhdh-plugin-export-overlays%2Fglobal/details) + +### Workspace-Specific Secrets + +Secrets for a specific workspace should be stored here: + +``` +selfservice/rhdh-plugin-export-overlays/workspaces/ +``` + +**Example (tech-radar):** [Tech Radar Secrets](https://vault.ci.openshift.org/ui/vault/secrets/kv/kv/selfservice%2Frhdh-plugin-export-overlays%2Fworkspaces%2Ftech-radar/details) + +### Required Vault Annotations + +Each workspace-specific secret path must include: + +```json +{ + "secretsync/target-name": "rhdh-plugin-export-overlays", + "secretsync/target-namespace": "test-credentials" +} +``` + +### Requesting Vault Access + +If you don't have access, request it in the team-rhdh channel. + +## Use in Test Code (Direct Access) + +For use in test code (`*.spec.ts`), access secrets directly via `process.env`: + +```typescript +test.beforeAll(async ({ rhdh }) => { + // Direct access - no rhdh-secrets.yaml needed + const apiKey = process.env.VAULT_API_KEY; + + if (!apiKey) { + throw new Error("VAULT_API_KEY is not set"); + } + + await rhdh.configure({ auth: "keycloak" }); + await rhdh.deploy(); +}); +``` + +## Use in RHDH Configuration Files + +To use Vault secrets in `app-config-rhdh.yaml` or `dynamic-plugins.yaml`, you must first add them to `rhdh-secrets.yaml`. + +### Step 1: Add to rhdh-secrets.yaml + +**tests/config/rhdh-secrets.yaml:** +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: rhdh-secrets +type: Opaque +stringData: + # Left side: name to use in app-config + # Right side: reference to Vault secret (with $) + EXTERNAL_HOST: $VAULT_EXTERNAL_HOST + MY_PLUGIN_API_KEY: $VAULT_MY_PLUGIN_API_KEY +``` + +### Step 2: Use in app-config-rhdh.yaml + +**tests/config/app-config-rhdh.yaml:** +```yaml +backend: + reading: + allow: + - host: ${EXTERNAL_HOST} +myPlugin: + apiKey: ${MY_PLUGIN_API_KEY} +``` + +## Summary + +| Where you need it | How to access | +|-------------------|---------------| +| Test code (`*.spec.ts`) | `process.env.VAULT_*` directly | +| RHDH configs | Add to `rhdh-secrets.yaml` first | + +## Related Pages + +- [CI Pipeline](/overlay/tutorials/ci-pipeline) - CI and Vault setup +- [Configuration Files](/overlay/test-structure/configuration-files) - YAML config flow + +## Adding a New Workspace to CI + +When adding E2E tests to a new workspace: + +1. **Create workspace-specific secret path in Vault:** + ``` + selfservice/rhdh-plugin-export-overlays/workspaces/ + ``` + +2. **Add required annotations:** + ```json + { + "secretsync/target-name": "rhdh-plugin-export-overlays", + "secretsync/target-namespace": "test-credentials" + } + ``` + +3. **Add secrets with `VAULT_` prefix:** + ``` + VAULT_YOUR_SECRET: + ``` + +4. **Reference secrets in your configuration files** diff --git a/docs/tutorials/first-test.md b/docs/tutorials/first-test.md index 1fd9427..6a1c3fb 100644 --- a/docs/tutorials/first-test.md +++ b/docs/tutorials/first-test.md @@ -13,84 +13,9 @@ For a faster setup, see the [Quick Start Guide](/guide/quick-start.md). This tut - OpenShift cluster access - `oc`, `kubectl`, `helm` installed -## Step 1: Create Project +Before continuing, complete the project scaffolding in [Quick Start](/guide/quick-start). This tutorial focuses on *why* each step matters and how to debug effectively. -```bash -mkdir my-plugin-e2e -cd my-plugin-e2e -yarn init -y -``` - -## Step 2: Install Dependencies - -```bash -yarn add @playwright/test rhdh-e2e-test-utils typescript dotenv -``` - -## Step 3: Create Configuration Files - -**playwright.config.ts:** -```typescript -import { defineConfig } from "rhdh-e2e-test-utils/playwright-config"; -import dotenv from "dotenv"; - -dotenv.config({ path: `${import.meta.dirname}/.env` }); - -export default defineConfig({ - projects: [ - { - name: "my-plugin", - }, - ], -}); -``` - -**tsconfig.json:** -```json -{ - "extends": "rhdh-e2e-test-utils/tsconfig", - "include": ["tests/**/*.ts", "playwright.config.ts"] -} -``` - -**.env:** -```bash -RHDH_VERSION="1.5" -INSTALLATION_METHOD="helm" -SKIP_KEYCLOAK_DEPLOYMENT=true -``` - -## Step 4: Create Test Directory - -```bash -mkdir -p tests/config tests/specs -``` - -## Step 5: Create Config Files - -**tests/config/app-config-rhdh.yaml:** -```yaml -app: - title: My First Test Instance -``` - -**tests/config/dynamic-plugins.yaml:** -```yaml -includes: - - dynamic-plugins.default.yaml -``` - -**tests/config/rhdh-secrets.yaml:** -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: rhdh-secrets -type: Opaque -stringData: {} -``` - -## Step 6: Write Your Test +## Step 1: Write Your Test **tests/specs/first-test.spec.ts:** ```typescript @@ -120,13 +45,13 @@ test.describe("My First Test Suite", () => { }); ``` -## Step 7: Login to OpenShift +## Step 2: Login to OpenShift ```bash oc login --server=https://api.your-cluster.com:6443 --token=your-token ``` -## Step 8: Run Tests +## Step 3: Run Tests ```bash yarn playwright test