agent-orchestrator/tests/e2e/wdio.conf.js

import { spawn, execSync } from 'node:child_process';
import { createConnection } from 'node:net';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { rmSync, existsSync } from 'node:fs';
import { createTestFixture } from './infra/fixtures.ts';
import { getResultsDb } from './infra/results-db.ts';

const __dirname = dirname(fileURLToPath(import.meta.url));
const projectRoot = resolve(__dirname, '../..');

// Debug binary path (Cargo workspace target at repo root)
const tauriBinary = resolve(projectRoot, 'target/debug/agent-orchestrator');

let tauriDriver;

// ── Test Fixture ──
// IMPORTANT: Must be created at module top-level (synchronously) because the
// capabilities object below references fixtureDataDir/fixtureConfigDir at eval time.
// tauri:options.env may not reliably set process-level env vars, so we also
// inject into process.env for tauri-driver inheritance.
const fixture = createTestFixture('agor-e2e');

process.env.AGOR_TEST = '1';
process.env.AGOR_TEST_DATA_DIR = fixture.dataDir;
process.env.AGOR_TEST_CONFIG_DIR = fixture.configDir;

// ── Port ──
// Unique port for this project's tauri-driver (avoids conflict with other
// Tauri apps or WebDriver instances on the same machine).
// Range 9000-9999 per project convention. See CLAUDE.md port table.
const TAURI_DRIVER_PORT = 9750;

console.log(`Test fixture created at ${fixture.rootDir}`);

export const config = {
  // ── Runner ──
  runner: 'local',
  maxInstances: 1, // Tauri doesn't support parallel sessions

  // ── Connection (tauri-driver on dedicated port) ──
  hostname: 'localhost',
  port: TAURI_DRIVER_PORT,
  path: '/',

  // ── Specs ──
  // All specs run in a single Tauri app session — state persists between files.
  // Stateful describe blocks include reset-to-home-state in their before() hooks.
  specs: [
    resolve(projectRoot, 'tests/e2e/specs/smoke.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/workspace.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/settings.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/features.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/terminal-theme.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-a-structure.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-a-agent.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-a-navigation.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-b-grid.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-b-llm.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-c-ui.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-c-tabs.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-c-llm.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-d-settings.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-d-errors.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-e-agents.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-e-health.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-f-search.test.ts'),
    resolve(projectRoot, 'tests/e2e/specs/phase-f-llm.test.ts'),
  ],

  // ── Capabilities ──
  capabilities: [{
    // Disable BiDi negotiation — tauri-driver doesn't support webSocketUrl
    'wdio:enforceWebDriverClassic': true,
    'tauri:options': {
      application: tauriBinary,
      // Test isolation: fixture-created data/config dirs, disable watchers/telemetry
      env: fixture.env,
    },
  }],

  // ── Framework ──
  framework: 'mocha',
  mochaOpts: {
    ui: 'bdd',
    timeout: 180_000,
  },

  // ── Reporter ──
  reporters: ['spec'],

  // ── Logging ──
  logLevel: 'warn',

  // ── Timeouts ──
  waitforTimeout: 10_000,
  connectionRetryTimeout: 30_000,
  connectionRetryCount: 3,

  // ── Hooks ──

  /**
   * Build the debug binary before the test run.
   * Kills any stale tauri-driver on our port first.
   */
  onPrepare() {
    // Kill stale tauri-driver on our port to avoid connecting to wrong app
    try {
      const pids = execSync(`lsof -ti:${TAURI_DRIVER_PORT} 2>/dev/null`, { encoding: 'utf8' }).trim();
      if (pids) {
        console.log(`Killing stale process(es) on port ${TAURI_DRIVER_PORT}: ${pids}`);
        execSync(`kill ${pids} 2>/dev/null`);
      }
    } catch { /* no process on port — good */ }

    // CRITICAL: The debug binary has devUrl (localhost:9700) baked in.
    // If another app (e.g., BridgeCoach) is serving on that port, the Tauri
    // WebView loads the WRONG frontend. Fail fast if port 9700 is in use.
    const DEV_URL_PORT = 9710;
    try {
      const devPids = execSync(`lsof -ti:${DEV_URL_PORT} 2>/dev/null`, { encoding: 'utf8' }).trim();
      if (devPids) {
        throw new Error(
          `Port ${DEV_URL_PORT} (Tauri devUrl) is in use by another process (PIDs: ${devPids}). ` +
          `The debug binary will load that app instead of Agent Orchestrator frontend. ` +
          `Either stop the process on port ${DEV_URL_PORT}, or use a release build.`
        );
      }
    } catch (e) {
      if (e.message.includes('Port 9700')) throw e;
      // lsof returned non-zero = port is free, good
    }

    // Verify binary exists
    if (!existsSync(tauriBinary)) {
      if (process.env.SKIP_BUILD) {
        throw new Error(`Debug binary not found at ${tauriBinary}. Build first or unset SKIP_BUILD.`);
      }
    }

    if (process.env.SKIP_BUILD) {
      // Even with SKIP_BUILD, verify the frontend dist exists
      if (!existsSync(resolve(projectRoot, 'dist/index.html'))) {
        console.log('Frontend dist/ missing — building frontend only...');
        execSync('npm run build', { cwd: projectRoot, stdio: 'inherit' });
      }
      console.log('SKIP_BUILD set — using existing debug binary.');
      return Promise.resolve();
    }
    return new Promise((resolveHook, reject) => {
      // Build frontend first (Tauri --no-bundle skips beforeBuildCommand)
      console.log('Building frontend...');
      try {
        execSync('npm run build', { cwd: projectRoot, stdio: 'inherit' });
      } catch (e) {
        reject(new Error(`Frontend build failed: ${e.message}`));
        return;
      }
      console.log('Building Tauri debug binary...');
      const build = spawn('cargo', ['tauri', 'build', '--debug', '--no-bundle'], {
        cwd: projectRoot,
        stdio: 'inherit',
      });
      build.on('close', (code) => {
        if (code === 0) {
          console.log('Debug binary ready.');
          resolveHook();
        } else {
          reject(new Error(`Tauri build failed with exit code ${code}`));
        }
      });
      build.on('error', reject);
    });
  },

  /**
   * Spawn tauri-driver on a dedicated port before the session.
   * First checks that the port is free (fails fast if another instance is running).
   * Uses TCP probe to confirm readiness after spawn.
   */
  beforeSession() {
    return new Promise((res, reject) => {
      // Fail fast if port is already in use (another tauri-driver or stale process)
      const preCheck = createConnection({ port: TAURI_DRIVER_PORT, host: 'localhost' }, () => {
        preCheck.destroy();
        reject(new Error(
          `Port ${TAURI_DRIVER_PORT} already in use. Kill the existing process: ` +
          `lsof -ti:${TAURI_DRIVER_PORT} | xargs kill`
        ));
      });
      preCheck.on('error', () => {
        preCheck.destroy();
        // Port is free — spawn tauri-driver
        tauriDriver = spawn('tauri-driver', ['--port', String(TAURI_DRIVER_PORT)], {
          stdio: ['ignore', 'pipe', 'pipe'],
        });

        tauriDriver.on('error', (err) => {
          reject(new Error(
            `Failed to start tauri-driver: ${err.message}. ` +
            'Install it with: cargo install tauri-driver'
          ));
        });

        // TCP readiness probe — poll until port accepts connections
        const deadline = Date.now() + 10_000;
        function probe() {
          if (Date.now() > deadline) {
            reject(new Error(`tauri-driver did not become ready on port ${TAURI_DRIVER_PORT} within 10s`));
            return;
          }
          const sock = createConnection({ port: TAURI_DRIVER_PORT, host: 'localhost' }, () => {
            sock.destroy();
            console.log(`tauri-driver ready on port ${TAURI_DRIVER_PORT}`);
            res();
          });
          sock.on('error', () => {
            sock.destroy();
            setTimeout(probe, 200);
          });
        }
        setTimeout(probe, 300);
      });
    });
  },

  /**
   * Verify the connected app is Agent Orchestrator (not another Tauri/WebKit2GTK app).
   * Runs once after the WebDriver session is created, before any spec files.
   */
  async before() {
    // Wait for app to render, then check for a known element
    await browser.waitUntil(
      async () => {
        const title = await browser.getTitle();
        const hasKnownEl = await browser.execute(
          'return document.querySelector(\'[data-testid="status-bar"]\') !== null' +
          ' || document.querySelector(".project-grid") !== null' +
          ' || document.querySelector(".settings-panel") !== null'
        );
        return hasKnownEl || title.toLowerCase().includes('agor') || title.toLowerCase().includes('orchestrator');
      },
      {
        timeout: 15_000,
        interval: 500,
        timeoutMsg:
          'Connected app is NOT Agent Orchestrator — wrong app detected. ' +
          'Check for other Tauri/WebKit2GTK apps running on this machine. ' +
          'Kill them or ensure the correct binary is at: ' + tauriBinary,
      },
    );
    console.log('App identity verified: Agent Orchestrator connected.');
  },

  /**
   * Smart test caching: skip tests that have passed consecutively N times.
   * Set FULL_RESCAN=1 to bypass caching and run all tests.
   */
  beforeTest(test) {
    // Reset expected errors for this test
    browser.__expectedErrors = [];

    if (process.env.FULL_RESCAN) return;

    const db = getResultsDb();
    const specFile = test.file?.replace(/.*specs\//, '') ?? '';
    if (db.shouldSkip(specFile, test.title)) {
      const stats = db.getCacheStats();
      console.log(`⏭ Skipping (3+ consecutive passes): ${test.title} [${stats.skippable}/${stats.total} skippable]`);
      this.skip();
    }
  },

  /**
   * After each test: check for unexpected error toasts in the DOM,
   * then record the result in the pass cache.
   */
  async afterTest(test, _context, { passed }) {
    // 1. Check for unexpected error toasts
    let unexpected = [];
    try {
      const errors = await browser.execute(
        'return (function() {' +
        '  var toasts = Array.from(document.querySelectorAll(".toast-error, .load-error"));' +
        '  return toasts.map(function(t) { return (t.textContent || "").trim(); }).filter(Boolean);' +
        '})()'
      );

      const expected = browser.__expectedErrors || [];
      unexpected = errors.filter(e => !expected.some(exp => e.includes(exp)));

      if (unexpected.length > 0 && passed) {
        throw new Error(
          `Unexpected error toast(s) during "${test.title}": ${unexpected.join('; ')}`
        );
      }
    } catch (e) {
      // Re-throw toast errors, swallow browser.execute failures (e.g., session closed)
      if (e.message?.includes('Unexpected error toast')) throw e;
    }

    // 2. Record result in pass cache
    const db = getResultsDb();
    const specFile = test.file?.replace(/.*specs\//, '') ?? '';
    db.recordTestResult(specFile, test.title, passed && unexpected.length === 0);
  },

  /**
   * Kill tauri-driver after the test run.
   */
  afterSession() {
    if (tauriDriver) {
      tauriDriver.kill();
      tauriDriver = null;
    }
    // Clean up test fixture
    try {
      rmSync(fixture.rootDir, { recursive: true, force: true });
      console.log('Test fixture cleaned up.');
    } catch { /* best-effort cleanup */ }
  },

  // ── TypeScript (auto-compile via tsx) ──
  autoCompileOpts: {
    tsNodeOpts: {
      project: resolve(projectRoot, 'tests/e2e/tsconfig.json'),
    },
  },
};