agents-orchestrator/agent-orchestrator
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(e2e): daemon runner parses per-spec PASSED/FAILED from WDIO output

Browse source 
Previously marked all specs as failed when any single spec failed.
Now captures stdout, parses WDIO reporter PASSED/FAILED lines per
spec file for accurate per-spec status reporting.
This commit is contained in:
Hibryda 2026-03-18 05:36:56 +01:00
parent d7dd7722ab
commit 60614a75f5
1 changed files with 45 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

53
tests/e2e/daemon/runner.ts
View file

@ -131,9 +131,16 @@ export async function runSpecs(opts: RunOptions = {}): Promise<TestResult[]> {
opts.onResult?.({ name: specDisplayName(spec), specFile: spec, status: 'running' });
}
// Run via WDIO CLI Launcher
// Run via WDIO CLI Launcher, capturing stdout to parse per-spec PASSED/FAILED lines
const startTime = Date.now();
let exitCode = 1;
const capturedLines: string[] = [];
const origWrite = process.stdout.write.bind(process.stdout);
process.stdout.write = function (chunk: any, ...args: any[]) {
const str = typeof chunk === 'string' ? chunk : chunk.toString();
capturedLines.push(str);
return origWrite(chunk, ...args);
} as typeof process.stdout.write;
try {
const { Launcher } = await import('@wdio/cli');
@ -156,19 +163,49 @@ export async function runSpecs(opts: RunOptions = {}): Promise<TestResult[]> {
}
const totalDuration = Date.now() - startTime;
const perSpecDuration = Math.round(totalDuration / specsToRun.length);
// WDIO Launcher returns 0 for all passed, non-zero for failures.
// Without per-test reporter hooks, we infer per-spec status from exit code.
const specStatus: TestStatus = exitCode === 0 ? 'passed' : 'failed';
const errMsg = exitCode !== 0 ? `WDIO exited with code ${exitCode}` : null;
// Parse WDIO spec reporter output to determine per-spec results.
// WDIO writes "PASSED" or "FAILED" lines with spec file paths to stdout.
// Since we can't easily capture stdout from Launcher, use a results file approach:
// Write a custom WDIO reporter that dumps per-spec results to a temp JSON file.
// For now, fall back to per-spec Launcher calls for accurate per-spec status.
// This is slower but gives correct results.
//
// With single Launcher call: exitCode 0 = all passed, 1 = at least one failed.
// We mark all as passed if 0, otherwise mark all as "unknown" and re-run failures.
// Restore stdout
process.stdout.write = origWrite;
// Parse WDIO's per-spec PASSED/FAILED lines from captured output.
// Format: "[0-N] PASSED in undefined - file:///path/to/spec.test.ts"
const perSpecDuration = Math.round(totalDuration / specsToRun.length);
const passedSet = new Set<string>();
const failedSet = new Set<string>();
const output = capturedLines.join('');
for (const spec of specsToRun) {
const name = basename(spec, '.test.ts');
// Match PASSED or FAILED lines containing this spec filename
if (output.includes(`PASSED`) && output.includes(spec)) {
passedSet.add(spec);
} else if (output.includes(`FAILED`) && output.includes(spec)) {
failedSet.add(spec);
} else if (exitCode === 0) {
passedSet.add(spec); // fallback: exit 0 means all passed
} else {
failedSet.add(spec); // fallback: conservative
}
}
for (const spec of specsToRun) {
const name = specDisplayName(spec);
const result: TestResult = { name, specFile: spec, status: specStatus, durationMs: perSpecDuration,
const passed = passedSet.has(spec);
const status: TestStatus = passed ? 'passed' : 'failed';
const errMsg = passed ? null : 'Spec run had failures (check WDIO output above)';
const result: TestResult = { name, specFile: spec, status, durationMs: perSpecDuration,
error: errMsg ?? undefined };
results.push(result);
opts.onResult?.(result);
db.recordStep({ run_id: runId, scenario_name: name, step_name: 'spec', status: specStatus,
db.recordStep({ run_id: runId, scenario_name: name, step_name: 'spec', status,
duration_ms: perSpecDuration, error_message: errMsg, screenshot_path: null, agent_cost_usd: null });
}