diff --git a/.gitignore b/.gitignore index 490cee3..1ae6545 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,11 @@ # testing /coverage -/playwright-report/* -/test-results/* +/playwright/report/* +/playwright/test-results/* +/blob-report/ +/playwright/.cache/ +/playwright/.auth/ # next.js /.next/ diff --git a/package.json b/package.json index 81fc07e..26b10ad 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,11 @@ "test:run": "vitest run", "test:coverage": "vitest run --coverage", "test:e2e": "playwright test", - "test:e2e:ui": "playwright test --ui", - "test:e2e:headed": "playwright test --headed", - "playwright:install": "playwright install", - "clean": "rm -rf .next out build dist coverage test-results playwright-report", + "test:e2e-ui": "playwright test --ui", + "test:e2e-headed": "playwright test --headed", + "test:e2e-report": "playwright show-report playwright/report", + "test:e2e-install": "playwright install-deps && playwright install", + "clean": "rm -rf .next out build dist coverage playwright/test-results playwright/report && playwright clear-cache", "check-licenses": "tsx scripts/check-licenses.ts" }, "dependencies": { @@ -55,6 +56,7 @@ "zustand": "5.0.7" }, "devDependencies": { + "@axe-core/playwright": "4.11.0", "@biomejs/biome": "2.1.3", "@playwright/test": "1.54.1", "@tailwindcss/aspect-ratio": "0.4.2", @@ -70,6 +72,7 @@ "@vitejs/plugin-react": "4.3.4", "@vitest/coverage-v8": "3.2.4", "@vitest/ui": "3.2.4", + "axe-html-reporter": "2.2.11", "jsdom": "26.0.0", "license-checker": "25.0.1", "prisma": "7.1.0", diff --git a/playwright.config.ts b/playwright.config.ts index 1e87c2f..37ec5bd 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,21 +4,31 @@ import { defineConfig, devices } from '@playwright/test' * @see https://playwright.dev/docs/test-configuration */ export default defineConfig({ - testDir: './e2e', + /* Where to look and the name of file to match for test files */ + testDir: './playwright/tests', + testMatch: '*.spec.ts', + /* Where to store the test results */ + outputDir: './playwright/test-results/', + /* Run tests in files in parallel */ fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, + /* Retry on CI only */ retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [ - ['html'], - ['json', { outputFile: 'playwright-report/results.json' }], - ['junit', { outputFile: 'playwright-report/results.xml' }] + ['line'], + ['html', { outputFolder: 'playwright/report', open: 'never'}], + ['json', { outputFile: 'playwright/report/results.json' }] ], + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ @@ -41,15 +51,23 @@ export default defineConfig({ use: { ...devices['Desktop Chrome'] }, }, - { - name: 'firefox', - use: { ...devices['Desktop Firefox'] }, - }, + // Uncomment to test in Firefox - this currently hits issues on my Mac because permissions with HTTP and DNS maybe? + // { + // name: 'firefox', + // use: { ...devices['Desktop Firefox'] }, + // }, { name: 'webkit', use: { ...devices['Desktop Safari'] }, }, + /* Test against branded browsers. */ + // Currently Microsoft Edge for Linux is not available for arm64 systems + // Can uncomment when support is added + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, /* Test against mobile viewports. */ { @@ -61,18 +79,11 @@ export default defineConfig({ use: { ...devices['iPhone 12'] }, }, - /* Test against branded browsers. */ - // { - // name: 'Microsoft Edge', - // use: { ...devices['Desktop Edge'], channel: 'msedge' }, - // }, - // { - // name: 'Google Chrome', - // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, - // }, + ], /* Run your local dev server before starting the tests */ + /* Comment out if running with Docker container */ webServer: { command: 'npm run dev', url: 'http://localhost:3000', diff --git a/playwright/tests/accessibilityTest.spec.ts b/playwright/tests/accessibilityTest.spec.ts new file mode 100644 index 0000000..da83f75 --- /dev/null +++ b/playwright/tests/accessibilityTest.spec.ts @@ -0,0 +1,60 @@ +import { test, parseResults } from './customAxeBuilder'; +import { expect } from '@playwright/test'; +import { createHtmlReport } from 'axe-html-reporter'; + +// List of URLS to check for accessibility +// may have to work on this to get a way to log in first +const urlsToCheck = [ + { + url: '/', + name: 'Homepage', + }, +]; + +test.describe('Accessibility Testing', () => { + urlsToCheck.forEach(({ url, name }) => { + + test(`should raise no accessibility violations: ${url} ${name}`, async ({ page, makeAxeBuilder }, testInfo) => { + await page.goto(url); + + const accessibilityScanResults = await makeAxeBuilder() + .analyze(); + + // If there are incomplete tests get better error to write to report and console + if (accessibilityScanResults.incomplete.length > 0) { + + const failedColorContrast = parseResults(accessibilityScanResults.incomplete); + + if (failedColorContrast.length != accessibilityScanResults.incomplete.length) { + // if not all the incomplete tests were color-contrast failures, report them as violations + expect.soft(accessibilityScanResults.incomplete.length).toEqual(0); + } else { + // add failed colour contrast tests to violations for reporting + accessibilityScanResults.violations.push(...failedColorContrast); + accessibilityScanResults.incomplete = []; + } + + } + + const reportHTML = createHtmlReport({ + results: accessibilityScanResults, + options: { + // comment out this line if you want to have a report file created locally + doNotCreateReportFile:true, + // uncomment these lines if you want to have a report file created locally + reportFileName: `${name}.html`, + outputDir: `playwright/test-results/accessibility-reports/`, + }, + }); + + // Send test results to reporter to see more info + await testInfo.attach('accessibility-scan-results', { + body: reportHTML, + contentType: 'text/html' + }) + + expect.soft(accessibilityScanResults.violations.length).toEqual(0); + + }); + }); +}); diff --git a/playwright/tests/customAxeBuilder.ts b/playwright/tests/customAxeBuilder.ts new file mode 100644 index 0000000..c95d8b0 --- /dev/null +++ b/playwright/tests/customAxeBuilder.ts @@ -0,0 +1,72 @@ +import { test as base } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; +import type {Result} from 'axe-core'; + +type AxeFixture = { + makeAxeBuilder: () => AxeBuilder; +}; + +// Extend base test by providing "makeAxeBuilder" +// +// This new "test" can be used in multiple test files, and each of them will get +// a consistently configured AxeBuilder instance. +export const test = base.extend({ + makeAxeBuilder: async ({ page }, use) => { + const makeAxeBuilder = () => new AxeBuilder({ page }) + .withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'best-practice']); + + await use(makeAxeBuilder); + } +}); + + +/** + * Given data from Axe-Core testing, parse incomplete object to determine if a violation has occurred or not. + * See detailed information on report groups here: + * https://docs.deque.com/devtools-for-web/4/en/java-use-results#results-overview + * + */ +export const parseResults = (inapplicableData: Result[]) => { + + /** + * Incomplete group - tests which ran, but the results require further (manual) review to + * determine what category the results should ultimately fall into. A common test in this group + * is color contrast as there is no way to set what contrast level is acceptable. + */ + let failedColorContrast = []; + + if (inapplicableData.length > 0) { + // Process each incomplete item to check for failed color-contrast rules + for (const item of inapplicableData) { + if (item.id === 'color-contrast' && item.nodes) { + let failedNode = []; + // Review each node in the color-contrast rule + for (const node of item.nodes) { + if (node.any && Array.isArray(node.any)) { + for (const check of node.any) { + if (check.id === 'color-contrast' && check.data) { + const { fontSize, contrastRatio } = check.data; + + // Extract pixel size from fontSize (e.g., "12.0pt (16px)" -> 16) + const fontSizeMatch = fontSize.match(/\((\d+)px\)/); + const sizeInPx = fontSizeMatch ? parseInt(fontSizeMatch[1], 10) : null; + + if (sizeInPx !== null && contrastRatio !== null) { + if ((sizeInPx < 18 && contrastRatio < 4.5 ) || (sizeInPx >= 18 && contrastRatio < 3)) { + failedNode.push(node); + } + } + } + } + } + } + if (failedNode.length === item.nodes.length) { + // All nodes failed, add everything to violations. + failedColorContrast.push(item); + } + } + } + } + + return failedColorContrast; +} diff --git a/playwright/tests/example.spec.ts b/playwright/tests/example.spec.ts new file mode 100644 index 0000000..6dd4e8a --- /dev/null +++ b/playwright/tests/example.spec.ts @@ -0,0 +1,15 @@ +import { test, expect } from '@playwright/test'; + +/** + * Example test that doesn't use the Axe Builder + */ + +test.describe('homepage', () => { + test('has paragraph', async ({ page }) => { + await page.goto('/'); + + // Expect a title "to contain" a substring. + // Can test how failing tests are reported by changing 'h1' or 'TEST' here to any other string + await expect(page.locator('h1')).toContainText('TEST'); + }); +}); diff --git a/src/app/page.tsx b/src/app/page.tsx index 451b8db..334d8c5 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -4,12 +4,19 @@ import { ExampleModal } from "@/components/examples/ExampleModal" export default function Home() { return (
-

TEST

+

TEST

Some body content here
+

Passing Lines

+
This should pass
+
+ {/*

Failing lines

+
2 - Longer content that may fail accessibility checks
+
+
3
*/}
) } diff --git a/tsconfig.json b/tsconfig.json index fbd46e6..df0e6ca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -36,6 +36,7 @@ ".next/dev/types/**/*.ts" ], "exclude": [ - "node_modules" + "node_modules", + "playwright" ] } diff --git a/vitest.config.ts b/vitest.config.ts index f7f2f02..5fed134 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -16,7 +16,7 @@ export default defineConfig({ '.next', 'coverage', 'e2e/**/*', - 'playwright-tests/**/*' + 'playwright/**/*' ], coverage: { provider: 'v8',