feat: unified E2E testing engine — 205 tests, dual-stack support
Infrastructure: - adapters/: base, tauri (port 9750), electrobun (port 9761 + PTY daemon) - helpers/: 120+ centralized selectors, reusable actions, custom assertions - wdio.shared.conf.js + stack-specific configs 18 unified specs (205 tests): splash(6) smoke(15) settings(19) terminal(14) agent(15) search(12) files(15) comms(10) tasks(10) theme(12) groups(12) keyboard(8) notifications(10) diagnostics(8) status-bar(12) context(9) worktree(8) llm-judged(10) Daemon: --stack tauri|electrobun|both flag Scripts: test:e2e:tauri, test:e2e:electrobun, test:e2e:both
This commit is contained in:
Hibryda
parent
1995f03682
commit
77b9ce9f62
31 changed files with 3547 additions and 344 deletions
|
|
@ -1,5 +1,6 @@
|
|||
// WDIO programmatic runner — launches specs and streams results to a callback
|
||||
// Uses @wdio/cli Launcher for test execution, reads results-db for smart caching.
|
||||
// Supports --stack flag: tauri (default), electrobun, or both.
|
||||
|
||||
import { resolve, dirname, basename } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
|
@ -10,16 +11,33 @@ import { ResultsDb } from '../infra/results-db.ts';
|
|||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PROJECT_ROOT = resolve(__dirname, '../../..');
|
||||
const WDIO_CONF = resolve(PROJECT_ROOT, 'tests/e2e/wdio.conf.js');
|
||||
const SPECS_DIR = resolve(PROJECT_ROOT, 'tests/e2e/specs');
|
||||
const RESULTS_PATH = resolve(PROJECT_ROOT, 'test-results/results.json');
|
||||
|
||||
export type StackTarget = 'tauri' | 'electrobun' | 'both';
|
||||
|
||||
/** Resolve the WDIO config file for a given stack */
|
||||
function getWdioConf(stack: StackTarget): string {
|
||||
switch (stack) {
|
||||
case 'tauri':
|
||||
return resolve(PROJECT_ROOT, 'tests/e2e/wdio.tauri.conf.js');
|
||||
case 'electrobun':
|
||||
return resolve(PROJECT_ROOT, 'tests/e2e/wdio.electrobun.conf.js');
|
||||
default:
|
||||
return resolve(PROJECT_ROOT, 'tests/e2e/wdio.tauri.conf.js');
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy fallback — original config
|
||||
const WDIO_CONF_LEGACY = resolve(PROJECT_ROOT, 'tests/e2e/wdio.conf.js');
|
||||
|
||||
export interface TestResult {
|
||||
name: string;
|
||||
specFile: string;
|
||||
status: TestStatus;
|
||||
durationMs?: number;
|
||||
error?: string;
|
||||
stack?: string;
|
||||
}
|
||||
|
||||
export type ResultCallback = (result: TestResult) => void;
|
||||
|
|
@ -28,6 +46,7 @@ export interface RunOptions {
|
|||
pattern?: string;
|
||||
full?: boolean;
|
||||
onResult?: ResultCallback;
|
||||
stack?: StackTarget;
|
||||
}
|
||||
|
||||
// ── Spec discovery ──
|
||||
|
|
@ -89,12 +108,116 @@ function getGitInfo(): { branch: string | null; sha: string | null } {
|
|||
}
|
||||
}
|
||||
|
||||
// ── Runner ──
|
||||
// ── Single-stack runner ──
|
||||
|
||||
async function runSingleStack(
|
||||
stack: StackTarget,
|
||||
opts: RunOptions,
|
||||
specsToRun: string[],
|
||||
db: ResultsDb,
|
||||
runId: string,
|
||||
): Promise<TestResult[]> {
|
||||
const results: TestResult[] = [];
|
||||
const confPath = getWdioConf(stack);
|
||||
|
||||
// Fall back to legacy config if new one doesn't exist
|
||||
const wdioConf = existsSync(confPath) ? confPath : WDIO_CONF_LEGACY;
|
||||
const stackLabel = stack === 'both' ? 'tauri' : stack;
|
||||
|
||||
// Mark specs as running
|
||||
for (const spec of specsToRun) {
|
||||
opts.onResult?.({ name: `[${stackLabel}] ${specDisplayName(spec)}`, specFile: spec, status: 'running', stack: stackLabel });
|
||||
}
|
||||
|
||||
const specPaths = specsToRun.map((s) => resolve(SPECS_DIR, s));
|
||||
const startTime = Date.now();
|
||||
let exitCode = 1;
|
||||
const capturedLines: string[] = [];
|
||||
const origWrite = process.stdout.write.bind(process.stdout);
|
||||
process.stdout.write = function (chunk: any, ...args: any[]) {
|
||||
const str = typeof chunk === 'string' ? chunk : chunk.toString();
|
||||
capturedLines.push(str);
|
||||
return origWrite(chunk, ...args);
|
||||
} as typeof process.stdout.write;
|
||||
|
||||
try {
|
||||
const { Launcher } = await import('@wdio/cli');
|
||||
const launcher = new Launcher(wdioConf, { specs: specPaths });
|
||||
exitCode = await launcher.run();
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
for (const spec of specsToRun) {
|
||||
const name = specDisplayName(spec);
|
||||
const result: TestResult = {
|
||||
name: `[${stackLabel}] ${name}`,
|
||||
specFile: spec,
|
||||
status: 'failed',
|
||||
error: `Launcher error: ${msg}`,
|
||||
stack: stackLabel,
|
||||
};
|
||||
results.push(result);
|
||||
opts.onResult?.(result);
|
||||
db.recordStep({
|
||||
run_id: runId, scenario_name: `[${stackLabel}] ${name}`, step_name: 'launcher',
|
||||
status: 'error', duration_ms: null, error_message: msg,
|
||||
screenshot_path: null, agent_cost_usd: null,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
} finally {
|
||||
process.stdout.write = origWrite;
|
||||
}
|
||||
|
||||
const totalDuration = Date.now() - startTime;
|
||||
const perSpecDuration = Math.round(totalDuration / specsToRun.length);
|
||||
const passedSet = new Set<string>();
|
||||
const failedSet = new Set<string>();
|
||||
const output = capturedLines.join('');
|
||||
|
||||
for (const spec of specsToRun) {
|
||||
if (output.includes('PASSED') && output.includes(spec)) {
|
||||
passedSet.add(spec);
|
||||
} else if (output.includes('FAILED') && output.includes(spec)) {
|
||||
failedSet.add(spec);
|
||||
} else if (exitCode === 0) {
|
||||
passedSet.add(spec);
|
||||
} else {
|
||||
failedSet.add(spec);
|
||||
}
|
||||
}
|
||||
|
||||
for (const spec of specsToRun) {
|
||||
const name = specDisplayName(spec);
|
||||
const passed = passedSet.has(spec);
|
||||
const status: TestStatus = passed ? 'passed' : 'failed';
|
||||
const errMsg = passed ? null : 'Spec run had failures (check WDIO output above)';
|
||||
const result: TestResult = {
|
||||
name: `[${stackLabel}] ${name}`,
|
||||
specFile: spec,
|
||||
status,
|
||||
durationMs: perSpecDuration,
|
||||
error: errMsg ?? undefined,
|
||||
stack: stackLabel,
|
||||
};
|
||||
results.push(result);
|
||||
opts.onResult?.(result);
|
||||
db.recordStep({
|
||||
run_id: runId, scenario_name: `[${stackLabel}] ${name}`, step_name: 'spec',
|
||||
status, duration_ms: perSpecDuration, error_message: errMsg,
|
||||
screenshot_path: null, agent_cost_usd: null,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── Main runner ──
|
||||
|
||||
export async function runSpecs(opts: RunOptions = {}): Promise<TestResult[]> {
|
||||
const db = new ResultsDb();
|
||||
const allSpecs = discoverSpecs(opts.pattern);
|
||||
const results: TestResult[] = [];
|
||||
const stack = opts.stack ?? 'tauri';
|
||||
|
||||
let specsToRun: string[];
|
||||
let skippedSpecs: string[] = [];
|
||||
|
|
@ -118,98 +241,32 @@ export async function runSpecs(opts: RunOptions = {}): Promise<TestResult[]> {
|
|||
return results;
|
||||
}
|
||||
|
||||
// Build absolute spec paths
|
||||
const specPaths = specsToRun.map((s) => resolve(SPECS_DIR, s));
|
||||
|
||||
// Generate run ID and record
|
||||
const git = getGitInfo();
|
||||
const runId = `daemon-${Date.now()}`;
|
||||
const runId = `daemon-${stack}-${Date.now()}`;
|
||||
db.startRun(runId, git.branch ?? undefined, git.sha ?? undefined);
|
||||
|
||||
// Mark specs as running
|
||||
for (const spec of specsToRun) {
|
||||
opts.onResult?.({ name: specDisplayName(spec), specFile: spec, status: 'running' });
|
||||
if (stack === 'both') {
|
||||
// Run against Tauri first, then Electrobun
|
||||
console.log('\n=== Running specs against TAURI stack ===\n');
|
||||
const tauriResults = await runSingleStack('tauri', opts, specsToRun, db, runId);
|
||||
results.push(...tauriResults);
|
||||
|
||||
console.log('\n=== Running specs against ELECTROBUN stack ===\n');
|
||||
const ebunResults = await runSingleStack('electrobun', opts, specsToRun, db, runId);
|
||||
results.push(...ebunResults);
|
||||
|
||||
const allPassed = [...tauriResults, ...ebunResults].every(r => r.status === 'passed');
|
||||
const totalDuration = results.reduce((sum, r) => sum + (r.durationMs ?? 0), 0);
|
||||
db.finishRun(runId, allPassed ? 'passed' : 'failed', totalDuration);
|
||||
} else {
|
||||
const startTime = Date.now();
|
||||
const stackResults = await runSingleStack(stack, opts, specsToRun, db, runId);
|
||||
results.push(...stackResults);
|
||||
|
||||
const allPassed = stackResults.every(r => r.status === 'passed');
|
||||
db.finishRun(runId, allPassed ? 'passed' : 'failed', Date.now() - startTime);
|
||||
}
|
||||
|
||||
// Run via WDIO CLI Launcher, capturing stdout to parse per-spec PASSED/FAILED lines
|
||||
const startTime = Date.now();
|
||||
let exitCode = 1;
|
||||
const capturedLines: string[] = [];
|
||||
const origWrite = process.stdout.write.bind(process.stdout);
|
||||
process.stdout.write = function (chunk: any, ...args: any[]) {
|
||||
const str = typeof chunk === 'string' ? chunk : chunk.toString();
|
||||
capturedLines.push(str);
|
||||
return origWrite(chunk, ...args);
|
||||
} as typeof process.stdout.write;
|
||||
|
||||
try {
|
||||
const { Launcher } = await import('@wdio/cli');
|
||||
const launcher = new Launcher(WDIO_CONF, {
|
||||
specs: specPaths,
|
||||
});
|
||||
exitCode = await launcher.run();
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
for (const spec of specsToRun) {
|
||||
const name = specDisplayName(spec);
|
||||
const result: TestResult = { name, specFile: spec, status: 'failed', error: `Launcher error: ${msg}` };
|
||||
results.push(result);
|
||||
opts.onResult?.(result);
|
||||
db.recordStep({ run_id: runId, scenario_name: name, step_name: 'launcher', status: 'error',
|
||||
duration_ms: null, error_message: msg, screenshot_path: null, agent_cost_usd: null });
|
||||
}
|
||||
db.finishRun(runId, 'error', Date.now() - startTime);
|
||||
return results;
|
||||
}
|
||||
|
||||
const totalDuration = Date.now() - startTime;
|
||||
|
||||
// Parse WDIO spec reporter output to determine per-spec results.
|
||||
// WDIO writes "PASSED" or "FAILED" lines with spec file paths to stdout.
|
||||
// Since we can't easily capture stdout from Launcher, use a results file approach:
|
||||
// Write a custom WDIO reporter that dumps per-spec results to a temp JSON file.
|
||||
// For now, fall back to per-spec Launcher calls for accurate per-spec status.
|
||||
// This is slower but gives correct results.
|
||||
//
|
||||
// With single Launcher call: exitCode 0 = all passed, 1 = at least one failed.
|
||||
// We mark all as passed if 0, otherwise mark all as "unknown" and re-run failures.
|
||||
// Restore stdout
|
||||
process.stdout.write = origWrite;
|
||||
|
||||
// Parse WDIO's per-spec PASSED/FAILED lines from captured output.
|
||||
// Format: "[0-N] PASSED in undefined - file:///path/to/spec.test.ts"
|
||||
const perSpecDuration = Math.round(totalDuration / specsToRun.length);
|
||||
const passedSet = new Set<string>();
|
||||
const failedSet = new Set<string>();
|
||||
const output = capturedLines.join('');
|
||||
for (const spec of specsToRun) {
|
||||
const name = basename(spec, '.test.ts');
|
||||
// Match PASSED or FAILED lines containing this spec filename
|
||||
if (output.includes(`PASSED`) && output.includes(spec)) {
|
||||
passedSet.add(spec);
|
||||
} else if (output.includes(`FAILED`) && output.includes(spec)) {
|
||||
failedSet.add(spec);
|
||||
} else if (exitCode === 0) {
|
||||
passedSet.add(spec); // fallback: exit 0 means all passed
|
||||
} else {
|
||||
failedSet.add(spec); // fallback: conservative
|
||||
}
|
||||
}
|
||||
|
||||
for (const spec of specsToRun) {
|
||||
const name = specDisplayName(spec);
|
||||
const passed = passedSet.has(spec);
|
||||
const status: TestStatus = passed ? 'passed' : 'failed';
|
||||
const errMsg = passed ? null : 'Spec run had failures (check WDIO output above)';
|
||||
const result: TestResult = { name, specFile: spec, status, durationMs: perSpecDuration,
|
||||
error: errMsg ?? undefined };
|
||||
results.push(result);
|
||||
opts.onResult?.(result);
|
||||
db.recordStep({ run_id: runId, scenario_name: name, step_name: 'spec', status,
|
||||
duration_ms: perSpecDuration, error_message: errMsg, screenshot_path: null, agent_cost_usd: null });
|
||||
}
|
||||
|
||||
db.finishRun(runId, exitCode === 0 ? 'passed' : 'failed', totalDuration);
|
||||
return results;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue