Skip to content

fix(playwright): fix JSON parser to match real Playwright output format #193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
USerik wants to merge 1 commit into
base: master
Choose a base branch
from
+154 −43
Open

fix(playwright): fix JSON parser to match real Playwright output format #193

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 154 additions & 43 deletions src/playwright_cmd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,50 +12,61 @@ use crate::parser::{
/// Playwright JSON output structures (tool-specific format)
#[derive(Debug, Deserialize)]
struct PlaywrightJsonOutput {
#[serde(rename = "stats")]
stats: PlaywrightStats,
#[serde(rename = "suites")]
#[serde(default)]
suites: Vec<PlaywrightSuite>,
}

#[derive(Debug, Deserialize)]
struct PlaywrightStats {
#[serde(rename = "expected")]
expected: usize,
#[serde(rename = "unexpected")]
unexpected: usize,
#[serde(rename = "skipped")]
skipped: usize,
#[serde(rename = "duration", default)]
duration: u64,
/// Duration in milliseconds (float in real Playwright output)
#[serde(default)]
duration: f64,
}

/// File-level or describe-level suite
#[derive(Debug, Deserialize)]
struct PlaywrightSuite {
title: String,
#[serde(rename = "tests")]
tests: Vec<PlaywrightTest>,
#[serde(rename = "suites", default)]
/// Individual test specs (test functions)
#[serde(default)]
specs: Vec<PlaywrightSpec>,
/// Nested describe blocks
#[serde(default)]
suites: Vec<PlaywrightSuite>,
}

/// A single test function (may run in multiple browsers/projects)
#[derive(Debug, Deserialize)]
struct PlaywrightTest {
struct PlaywrightSpec {
title: String,
#[serde(rename = "status")]
/// Overall pass/fail status across all projects
ok: bool,
/// Per-project/browser executions
#[serde(default)]
tests: Vec<PlaywrightExecution>,
}

/// A test execution in a specific browser/project
#[derive(Debug, Deserialize)]
struct PlaywrightExecution {
/// "expected", "unexpected", "skipped", "flaky"
status: String,
#[serde(rename = "results")]
results: Vec<PlaywrightTestResult>,
#[serde(default)]
results: Vec<PlaywrightAttempt>,
}

/// A single attempt/result for a test execution
#[derive(Debug, Deserialize)]
struct PlaywrightTestResult {
#[serde(rename = "status")]
struct PlaywrightAttempt {
/// "passed", "failed", "timedOut", "interrupted"
status: String,
#[serde(rename = "error")]
error: Option<PlaywrightError>,
#[serde(rename = "duration", default)]
duration: u64,
/// Error details (array in Playwright >= v1.30)
#[serde(default)]
errors: Vec<PlaywrightError>,
}

#[derive(Debug, Deserialize)]
Expand All @@ -82,7 +93,7 @@ impl OutputParser for PlaywrightParser {
passed: json.stats.expected,
failed: json.stats.unexpected,
skipped: json.stats.skipped,
duration_ms: Some(json.stats.duration),
duration_ms: Some(json.stats.duration as u64),
failures,
};

Expand Down Expand Up @@ -111,27 +122,34 @@ fn collect_test_results(
failures: &mut Vec<TestFailure>,
) {
for suite in suites {
for test in &suite.tests {
for spec in &suite.specs {
*total += 1;

if test.status == "failed" || test.status == "timedOut" {
let error_msg = test
.results
.first()
.and_then(|r| r.error.as_ref())
if !spec.ok {
// Find the first failed execution and its error message
let error_msg = spec
.tests
.iter()
.find(|t| t.status == "unexpected")
.and_then(|t| {
t.results
.iter()
.find(|r| r.status == "failed" || r.status == "timedOut")
})
.and_then(|r| r.errors.first())
.map(|e| e.message.clone())
.unwrap_or_else(|| "Unknown error".to_string());
.unwrap_or_else(|| "Test failed".to_string());

failures.push(TestFailure {
test_name: test.title.clone(),
test_name: spec.title.clone(),
file_path: suite.title.clone(),
error_message: error_msg,
stack_trace: None,
});
}
}

// Recurse into nested suites
// Recurse into nested suites (describe blocks)
collect_test_results(&suite.suites, total, failures);
}
}
Expand Down Expand Up @@ -221,10 +239,16 @@ pub fn run(args: &[String], verbose: u8) -> Result<()> {

let mut cmd = package_manager_exec("playwright");

// Add JSON reporter for structured output
// The subcommand (e.g. "test") must come before reporter flags
if let Some(first) = args.first() {
cmd.arg(first);
}

// Add JSON reporter for structured output (must come after the subcommand)
cmd.arg("--reporter=json");

for arg in args {
// Add remaining user args
for arg in args.iter().skip(1) {
cmd.arg(arg);
}

Expand Down Expand Up @@ -286,38 +310,125 @@ mod tests {

#[test]
fn test_playwright_parser_json() {
// Real Playwright JSON structure: suites → specs, with float duration
let json = r#"{
"stats": {
"expected": 3,
"startTime": "2026-01-01T00:00:00.000Z",
"expected": 1,
"unexpected": 0,
"skipped": 0,
"duration": 7300
"flaky": 0,
"duration": 7300.5
},
"suites": [
{
"title": "auth/login.spec.ts",
"tests": [
"title": "auth",
"specs": [],
"suites": [
{
"title": "should login",
"status": "passed",
"results": [{"status": "passed", "duration": 2300}]
"title": "login.spec.ts",
"specs": [
{
"title": "should login",
"ok": true,
"tests": [
{
"status": "expected",
"results": [{"status": "passed", "errors": [], "duration": 2300}]
}
]
}
],
"suites": []
}
],
"suites": []
]
}
]
],
"errors": []
}"#;

let result = PlaywrightParser::parse(json);
assert_eq!(result.tier(), 1);
assert!(result.is_ok());

let data = result.unwrap();
assert_eq!(data.passed, 3);
assert_eq!(data.passed, 1);
assert_eq!(data.failed, 0);
assert_eq!(data.duration_ms, Some(7300));
}

#[test]
fn test_playwright_parser_json_float_duration() {
// Real Playwright output uses float duration (e.g. 3519.7039999999997)
let json = r#"{
"stats": {
"startTime": "2026-02-18T10:17:53.187Z",
"expected": 4,
"unexpected": 0,
"skipped": 0,
"flaky": 0,
"duration": 3519.7039999999997
},
"suites": [],
"errors": []
}"#;

let result = PlaywrightParser::parse(json);
assert_eq!(result.tier(), 1);
assert!(result.is_ok());

let data = result.unwrap();
assert_eq!(data.passed, 4);
assert_eq!(data.duration_ms, Some(3519));
}

#[test]
fn test_playwright_parser_json_with_failure() {
let json = r#"{
"stats": {
"expected": 0,
"unexpected": 1,
"skipped": 0,
"duration": 1500.0
},
"suites": [
{
"title": "my.spec.ts",
"specs": [
{
"title": "should work",
"ok": false,
"tests": [
{
"status": "unexpected",
"results": [
{
"status": "failed",
"errors": [{"message": "Expected true to be false"}],
"duration": 500
}
]
}
]
}
],
"suites": []
}
],
"errors": []
}"#;

let result = PlaywrightParser::parse(json);
assert_eq!(result.tier(), 1);
assert!(result.is_ok());

let data = result.unwrap();
assert_eq!(data.failed, 1);
assert_eq!(data.failures.len(), 1);
assert_eq!(data.failures[0].test_name, "should work");
assert_eq!(data.failures[0].error_message, "Expected true to be false");
}

#[test]
fn test_playwright_parser_regex_fallback() {
let text = "3 passed (7.3s)";
Expand Down