feat(e2e): add smart test caching and error toast catching
- results-db.ts: TestPassCache with consecutivePasses counter, recordTestResult(), shouldSkip(threshold=3), resetCache() - wdio.conf.js: afterTest hook catches unexpected .toast.error/.load-error elements, records results to smart cache. FULL_RESCAN=1 bypasses caching
This commit is contained in:
parent
0803dc3844
commit
46f51d7941
2 changed files with 120 additions and 3 deletions
|
|
@ -33,9 +33,16 @@ export interface TestStepRow {
|
||||||
created_at: string;
|
created_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TestPassCache {
|
||||||
|
testKey: string;
|
||||||
|
consecutivePasses: number;
|
||||||
|
lastPassedAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
interface ResultsStore {
|
interface ResultsStore {
|
||||||
runs: TestRunRow[];
|
runs: TestRunRow[];
|
||||||
steps: TestStepRow[];
|
steps: TestStepRow[];
|
||||||
|
passCache: TestPassCache[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ResultsDb {
|
export class ResultsDb {
|
||||||
|
|
@ -51,12 +58,13 @@ export class ResultsDb {
|
||||||
private load(): ResultsStore {
|
private load(): ResultsStore {
|
||||||
if (existsSync(this.filePath)) {
|
if (existsSync(this.filePath)) {
|
||||||
try {
|
try {
|
||||||
return JSON.parse(readFileSync(this.filePath, 'utf-8'));
|
const data = JSON.parse(readFileSync(this.filePath, 'utf-8'));
|
||||||
|
return { runs: data.runs ?? [], steps: data.steps ?? [], passCache: data.passCache ?? [] };
|
||||||
} catch {
|
} catch {
|
||||||
return { runs: [], steps: [] };
|
return { runs: [], steps: [], passCache: [] };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return { runs: [], steps: [] };
|
return { runs: [], steps: [], passCache: [] };
|
||||||
}
|
}
|
||||||
|
|
||||||
private save(): void {
|
private save(): void {
|
||||||
|
|
@ -110,4 +118,61 @@ export class ResultsDb {
|
||||||
getStepsForRun(runId: string): TestStepRow[] {
|
getStepsForRun(runId: string): TestStepRow[] {
|
||||||
return this.store.steps.filter(s => s.run_id === runId);
|
return this.store.steps.filter(s => s.run_id === runId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Pass Cache ──
|
||||||
|
|
||||||
|
private static makeTestKey(specFile: string, testTitle: string): string {
|
||||||
|
return `${specFile}::${testTitle}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
recordTestResult(specFile: string, testTitle: string, passed: boolean): void {
|
||||||
|
const key = ResultsDb.makeTestKey(specFile, testTitle);
|
||||||
|
const entry = this.store.passCache.find(e => e.testKey === key);
|
||||||
|
|
||||||
|
if (passed) {
|
||||||
|
if (entry) {
|
||||||
|
entry.consecutivePasses += 1;
|
||||||
|
entry.lastPassedAt = new Date().toISOString();
|
||||||
|
} else {
|
||||||
|
this.store.passCache.push({
|
||||||
|
testKey: key,
|
||||||
|
consecutivePasses: 1,
|
||||||
|
lastPassedAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (entry) {
|
||||||
|
entry.consecutivePasses = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.save();
|
||||||
|
}
|
||||||
|
|
||||||
|
shouldSkip(specFile: string, testTitle: string, threshold = 3): boolean {
|
||||||
|
const key = ResultsDb.makeTestKey(specFile, testTitle);
|
||||||
|
const entry = this.store.passCache.find(e => e.testKey === key);
|
||||||
|
return entry !== undefined && entry.consecutivePasses >= threshold;
|
||||||
|
}
|
||||||
|
|
||||||
|
resetCache(): void {
|
||||||
|
this.store.passCache = [];
|
||||||
|
this.save();
|
||||||
|
}
|
||||||
|
|
||||||
|
getCacheStats(threshold = 3): { total: number; skippable: number; threshold: number } {
|
||||||
|
const total = this.store.passCache.length;
|
||||||
|
const skippable = this.store.passCache.filter(e => e.consecutivePasses >= threshold).length;
|
||||||
|
return { total, skippable, threshold };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Lazy singleton for use in wdio hooks ──
|
||||||
|
|
||||||
|
let _singleton: ResultsDb | null = null;
|
||||||
|
|
||||||
|
export function getResultsDb(): ResultsDb {
|
||||||
|
if (!_singleton) {
|
||||||
|
_singleton = new ResultsDb();
|
||||||
|
}
|
||||||
|
return _singleton;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import { resolve, dirname } from 'node:path';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { rmSync, existsSync } from 'node:fs';
|
import { rmSync, existsSync } from 'node:fs';
|
||||||
import { createTestFixture } from './infra/fixtures.ts';
|
import { createTestFixture } from './infra/fixtures.ts';
|
||||||
|
import { getResultsDb } from './infra/results-db.ts';
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
const projectRoot = resolve(__dirname, '../..');
|
const projectRoot = resolve(__dirname, '../..');
|
||||||
|
|
@ -251,6 +252,57 @@ export const config = {
|
||||||
console.log('App identity verified: Agent Orchestrator connected.');
|
console.log('App identity verified: Agent Orchestrator connected.');
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Smart test caching: skip tests that have passed consecutively N times.
|
||||||
|
* Set FULL_RESCAN=1 to bypass caching and run all tests.
|
||||||
|
*/
|
||||||
|
beforeTest(test) {
|
||||||
|
// Reset expected errors for this test
|
||||||
|
browser.__expectedErrors = [];
|
||||||
|
|
||||||
|
if (process.env.FULL_RESCAN) return;
|
||||||
|
|
||||||
|
const db = getResultsDb();
|
||||||
|
const specFile = test.file?.replace(/.*specs\//, '') ?? '';
|
||||||
|
if (db.shouldSkip(specFile, test.title)) {
|
||||||
|
const stats = db.getCacheStats();
|
||||||
|
console.log(`⏭ Skipping (3+ consecutive passes): ${test.title} [${stats.skippable}/${stats.total} skippable]`);
|
||||||
|
this.skip();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* After each test: check for unexpected error toasts in the DOM,
|
||||||
|
* then record the result in the pass cache.
|
||||||
|
*/
|
||||||
|
async afterTest(test, _context, { passed }) {
|
||||||
|
// 1. Check for unexpected error toasts
|
||||||
|
let unexpected = [];
|
||||||
|
try {
|
||||||
|
const errors = await browser.execute(() => {
|
||||||
|
const toasts = [...document.querySelectorAll('.toast-error, .load-error')];
|
||||||
|
return toasts.map(t => t.textContent?.trim()).filter(Boolean);
|
||||||
|
});
|
||||||
|
|
||||||
|
const expected = browser.__expectedErrors || [];
|
||||||
|
unexpected = errors.filter(e => !expected.some(exp => e.includes(exp)));
|
||||||
|
|
||||||
|
if (unexpected.length > 0 && passed) {
|
||||||
|
throw new Error(
|
||||||
|
`Unexpected error toast(s) during "${test.title}": ${unexpected.join('; ')}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// Re-throw toast errors, swallow browser.execute failures (e.g., session closed)
|
||||||
|
if (e.message?.includes('Unexpected error toast')) throw e;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Record result in pass cache
|
||||||
|
const db = getResultsDb();
|
||||||
|
const specFile = test.file?.replace(/.*specs\//, '') ?? '';
|
||||||
|
db.recordTestResult(specFile, test.title, passed && unexpected.length === 0);
|
||||||
|
},
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Kill tauri-driver after the test run.
|
* Kill tauri-driver after the test run.
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue