agent-orchestrator/tests/e2e/daemon/runner.ts

// WDIO programmatic runner — launches specs and streams results to a callback
// Uses @wdio/cli Launcher for test execution, reads results-db for smart caching.

import { resolve, dirname, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
import { existsSync, readdirSync, writeFileSync } from 'node:fs';
import { execSync } from 'node:child_process';
import type { TestStatus } from './dashboard.ts';
import { ResultsDb } from '../infra/results-db.ts';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PROJECT_ROOT = resolve(__dirname, '../../..');
const WDIO_CONF = resolve(PROJECT_ROOT, 'tests/e2e/wdio.conf.js');
const SPECS_DIR = resolve(PROJECT_ROOT, 'tests/e2e/specs');
const RESULTS_PATH = resolve(PROJECT_ROOT, 'test-results/results.json');

export interface TestResult {
  name: string;
  specFile: string;
  status: TestStatus;
  durationMs?: number;
  error?: string;
}

export type ResultCallback = (result: TestResult) => void;

export interface RunOptions {
  pattern?: string;
  full?: boolean;
  onResult?: ResultCallback;
}

// ── Spec discovery ──

export function discoverSpecs(pattern?: string): string[] {
  const files = readdirSync(SPECS_DIR)
    .filter((f) => f.endsWith('.test.ts'))
    .sort();

  if (pattern) {
    const lp = pattern.toLowerCase();
    return files.filter((f) => f.toLowerCase().includes(lp));
  }
  return files;
}

export function specDisplayName(specFile: string): string {
  return basename(specFile, '.test.ts');
}

// ── Smart cache ──

function getPassedSpecs(db: ResultsDb): Set<string> {
  const passed = new Set<string>();
  for (const run of db.getRecentRuns(5)) {
    if (run.status !== 'passed' && run.status !== 'failed') continue;
    for (const step of db.getStepsForRun(run.run_id)) {
      if (step.status === 'passed') passed.add(step.scenario_name);
    }
  }
  return passed;
}

function filterByCache(specs: string[], db: ResultsDb): { run: string[]; skipped: string[] } {
  const cached = getPassedSpecs(db);
  const run: string[] = [];
  const skipped: string[] = [];
  for (const spec of specs) {
    (cached.has(specDisplayName(spec)) ? skipped : run).push(spec);
  }
  return { run, skipped };
}

// ── Git info ──

function getGitInfo(): { branch: string | null; sha: string | null } {
  try {
    const branch = execSync('git rev-parse --abbrev-ref HEAD', {
      cwd: PROJECT_ROOT,
      encoding: 'utf-8',
    }).trim();
    const sha = execSync('git rev-parse --short HEAD', {
      cwd: PROJECT_ROOT,
      encoding: 'utf-8',
    }).trim();
    return { branch, sha };
  } catch {
    return { branch: null, sha: null };
  }
}

// ── Runner ──

export async function runSpecs(opts: RunOptions = {}): Promise<TestResult[]> {
  const db = new ResultsDb();
  const allSpecs = discoverSpecs(opts.pattern);
  const results: TestResult[] = [];

  let specsToRun: string[];
  let skippedSpecs: string[] = [];

  if (opts.full) {
    specsToRun = allSpecs;
  } else {
    const filtered = filterByCache(allSpecs, db);
    specsToRun = filtered.run;
    skippedSpecs = filtered.skipped;
  }

  // Emit skipped specs immediately
  for (const spec of skippedSpecs) {
    const result: TestResult = { name: specDisplayName(spec), specFile: spec, status: 'skipped' };
    results.push(result);
    opts.onResult?.(result);
  }

  if (specsToRun.length === 0) {
    return results;
  }

  // Build absolute spec paths
  const specPaths = specsToRun.map((s) => resolve(SPECS_DIR, s));

  // Generate run ID and record
  const git = getGitInfo();
  const runId = `daemon-${Date.now()}`;
  db.startRun(runId, git.branch ?? undefined, git.sha ?? undefined);

  // Mark specs as running
  for (const spec of specsToRun) {
    opts.onResult?.({ name: specDisplayName(spec), specFile: spec, status: 'running' });
  }

  // Run via WDIO CLI Launcher, capturing stdout to parse per-spec PASSED/FAILED lines
  const startTime = Date.now();
  let exitCode = 1;
  const capturedLines: string[] = [];
  const origWrite = process.stdout.write.bind(process.stdout);
  process.stdout.write = function (chunk: any, ...args: any[]) {
    const str = typeof chunk === 'string' ? chunk : chunk.toString();
    capturedLines.push(str);
    return origWrite(chunk, ...args);
  } as typeof process.stdout.write;

  try {
    const { Launcher } = await import('@wdio/cli');
    const launcher = new Launcher(WDIO_CONF, {
      specs: specPaths,
    });
    exitCode = await launcher.run();
  } catch (err: unknown) {
    const msg = err instanceof Error ? err.message : String(err);
    for (const spec of specsToRun) {
      const name = specDisplayName(spec);
      const result: TestResult = { name, specFile: spec, status: 'failed', error: `Launcher error: ${msg}` };
      results.push(result);
      opts.onResult?.(result);
      db.recordStep({ run_id: runId, scenario_name: name, step_name: 'launcher', status: 'error',
        duration_ms: null, error_message: msg, screenshot_path: null, agent_cost_usd: null });
    }
    db.finishRun(runId, 'error', Date.now() - startTime);
    return results;
  }

  const totalDuration = Date.now() - startTime;

  // Parse WDIO spec reporter output to determine per-spec results.
  // WDIO writes "PASSED" or "FAILED" lines with spec file paths to stdout.
  // Since we can't easily capture stdout from Launcher, use a results file approach:
  // Write a custom WDIO reporter that dumps per-spec results to a temp JSON file.
  // For now, fall back to per-spec Launcher calls for accurate per-spec status.
  // This is slower but gives correct results.
  //
  // With single Launcher call: exitCode 0 = all passed, 1 = at least one failed.
  // We mark all as passed if 0, otherwise mark all as "unknown" and re-run failures.
  // Restore stdout
  process.stdout.write = origWrite;

  // Parse WDIO's per-spec PASSED/FAILED lines from captured output.
  // Format: "[0-N] PASSED in undefined - file:///path/to/spec.test.ts"
  const perSpecDuration = Math.round(totalDuration / specsToRun.length);
  const passedSet = new Set<string>();
  const failedSet = new Set<string>();
  const output = capturedLines.join('');
  for (const spec of specsToRun) {
    const name = basename(spec, '.test.ts');
    // Match PASSED or FAILED lines containing this spec filename
    if (output.includes(`PASSED`) && output.includes(spec)) {
      passedSet.add(spec);
    } else if (output.includes(`FAILED`) && output.includes(spec)) {
      failedSet.add(spec);
    } else if (exitCode === 0) {
      passedSet.add(spec); // fallback: exit 0 means all passed
    } else {
      failedSet.add(spec); // fallback: conservative
    }
  }

  for (const spec of specsToRun) {
    const name = specDisplayName(spec);
    const passed = passedSet.has(spec);
    const status: TestStatus = passed ? 'passed' : 'failed';
    const errMsg = passed ? null : 'Spec run had failures (check WDIO output above)';
    const result: TestResult = { name, specFile: spec, status, durationMs: perSpecDuration,
      error: errMsg ?? undefined };
    results.push(result);
    opts.onResult?.(result);
    db.recordStep({ run_id: runId, scenario_name: name, step_name: 'spec', status,
      duration_ms: perSpecDuration, error_message: errMsg, screenshot_path: null, agent_cost_usd: null });
  }

  db.finishRun(runId, exitCode === 0 ? 'passed' : 'failed', totalDuration);
  return results;
}

export function clearCache(): void {
  if (existsSync(RESULTS_PATH)) {
    writeFileSync(RESULTS_PATH, JSON.stringify({ runs: [], steps: [] }, null, 2));
  }
}