test(quality): add concurrency and real browser gates

apps/api/src/application/instances/instance-service.test.ts

@@ -1,5 +1,9 @@
 import Database from 'better-sqlite3';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { afterEach, describe, expect, it } from 'vitest';
+import { openDatabase } from '../../infrastructure/database/database.js';
 import { migrateDatabase } from '../../infrastructure/database/migrations.js';
 import type { SecretStore } from '../../infrastructure/secrets/secret-store.js';
 import type { Instance } from '@multi-simadmin/contracts';
@@ -38,8 +42,11 @@ class MemorySecrets implements SecretStore {
   }
 }
 const dbs: Database.Database[] = [];
+const temporaryDirectories: string[] = [];
 afterEach(() => {
   for (const db of dbs.splice(0)) db.close();
+  for (const directory of temporaryDirectories.splice(0))
+    rmSync(directory, { recursive: true, force: true });
 });
 function fixture(ids = ['id-1', 'slot-1', 'ref-1', 'id-2', 'slot-2', 'ref-2']) {
   const db = new Database(':memory:');
@@ -56,6 +63,30 @@ function fixture(ids = ['id-1', 'slot-1', 'ref-1', 'id-2', 'slot-2', 'ref-2']) {
   });
   return { db, store, service };
 }
+function crossConnectionFixture() {
+  const directory = mkdtempSync(join(tmpdir(), 'instance-cas-'));
+  temporaryDirectories.push(directory);
+  const path = join(directory, 'state.sqlite');
+  const firstDb = openDatabase(path);
+  migrateDatabase(firstDb);
+  const secondDb = openDatabase(path);
+  dbs.push(firstDb, secondDb);
+  const store = new MemorySecrets();
+  const options = { store, now: () => new Date('2026-07-16T12:00:00.000Z') };
+  const firstIds = ['id-1', 'first-slot', 'first-reference'];
+  const secondIds = ['second-slot', 'second-reference'];
+  const first = new InstanceService({
+    ...options,
+    db: firstDb,
+    idFactory: () => firstIds.shift()!,
+  });
+  const second = new InstanceService({
+    ...options,
+    db: secondDb,
+    idFactory: () => secondIds.shift()!,
+  });
+  return { firstDb, secondDb, store, first, second };
+}
 const basic = { name: ' Alpha ', origin: 'http://192.168.1.10:8080/', tags: [' z ', 'a', 'a'] };
 const code = async (promise: Promise<unknown>) => {
   try {
@@ -222,6 +253,23 @@ describe('InstanceService', () => {
     });
   });
 
+  it('rejects stale update and delete CAS across managed connections', async () => {
+    const { firstDb, first, second } = crossConnectionFixture();
+    await first.create(basic);
+    await second.update('id-1', 1, { name: 'Committed elsewhere' });
+
+    await expect(first.update('id-1', 1, { name: 'Stale update' })).rejects.toMatchObject({
+      code: 'REVISION_CONFLICT',
+    });
+    await expect(first.delete('id-1', 1)).rejects.toMatchObject({ code: 'REVISION_CONFLICT' });
+    expect(
+      firstDb.prepare('SELECT name,config_revision FROM instances WHERE id=?').get('id-1'),
+    ).toEqual({
+      name: 'Committed elsewhere',
+      config_revision: 2,
+    });
+  });
+
   it('preserves, replaces and clears credentials without exposing them', async () => {
     const { db, store, service } = fixture();
     await service.create({ ...basic, password: { action: 'set', password: 'first-secret' } });

apps/api/src/application/status/status-snapshot-service.test.ts

@@ -6,6 +6,7 @@ import { UpstreamError } from '../../infrastructure/transport/upstream-error.js'
 import {
   StatusSnapshotError,
   StatusSnapshotService,
+  type HealthSnapshotInstance,
   type HealthTransportResponse,
 } from './status-snapshot-service.js';
 
@@ -17,7 +18,12 @@ afterEach(() => {
 const instant = (value: string) => () => new Date(value);
 
 function fixture(
-  options: { response?: HealthTransportResponse; failure?: unknown; now?: () => Date } = {},
+  options: {
+    response?: HealthTransportResponse;
+    failure?: unknown;
+    now?: () => Date;
+    idFactory?: () => string;
+  } = {},
 ) {
   const db = new Database(':memory:');
   db.pragma('foreign_keys=ON');
@@ -27,11 +33,10 @@ function fixture(
   db.prepare(
     'INSERT INTO instances (id,name,base_url,auth_mode,enabled,config_revision,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?)',
   ).run('instance-1', 'LAN', 'http://192.168.1.20:3000', 'none', 1, 1, created, created);
-  const instances = {
-    get: vi.fn(async (id: string) =>
-      id === 'instance-1' ? { id, origin: 'http://192.168.1.20:3000' } : undefined,
-    ),
-  };
+  const owners = new Map<string, HealthSnapshotInstance>([
+    ['instance-1', { id: 'instance-1', origin: 'http://192.168.1.20:3000', revision: 1 }],
+  ]);
+  const instances = { get: vi.fn(async (id: string) => owners.get(id)) };
   const transport = {
     get: vi.fn(async (url: string) => {
       expect(url).toBe('http://192.168.1.20:3000/api/health');
@@ -51,9 +56,9 @@ function fixture(
     ttlMs: 60_000,
     maxStaleMs: 300_000,
     now: options.now ?? instant('2026-07-17T12:00:00.000Z'),
-    idFactory: () => 'snapshot-id',
+    idFactory: options.idFactory ?? (() => 'snapshot-id'),
   });
-  return { db, instances, transport, service };
+  return { db, instances, owners, transport, service };
 }
 
 const row = (db: Database.Database) =>
@@ -257,6 +262,129 @@ describe('StatusSnapshotService health snapshots', () => {
     });
   });
 
+  it('lets the latest overlapping probe replace an equal-time payload', async () => {
+    const first = fixture();
+    const secondTransport = { get: vi.fn<() => Promise<HealthTransportResponse>>() };
+    const second = new StatusSnapshotService({
+      db: first.db,
+      instances: first.instances,
+      transport: secondTransport,
+      ttlMs: 60_000,
+      maxStaleMs: 300_000,
+      now: instant('2026-07-17T12:00:00.000Z'),
+      idFactory: () => 'second-snapshot-id',
+    });
+    let resolveOlder!: (value: HealthTransportResponse) => void;
+    let resolveLatest!: (value: HealthTransportResponse) => void;
+    first.transport.get.mockImplementationOnce(
+      async () => new Promise((resolve) => (resolveOlder = resolve)),
+    );
+    secondTransport.get.mockImplementationOnce(
+      async () => new Promise((resolve) => (resolveLatest = resolve)),
+    );
+
+    const older = first.service.refreshHealth('instance-1');
+    await vi.waitFor(() => expect(first.transport.get).toHaveBeenCalledTimes(1));
+    const latest = second.refreshHealth('instance-1');
+    await vi.waitFor(() => expect(secondTransport.get).toHaveBeenCalledTimes(1));
+    resolveOlder({ status: 200, body: '{"status":"older"}' });
+    await older;
+    expect(payload(first.db)).toMatchObject({ data: { status: 'older' } });
+    resolveLatest({ status: 200, body: '{"status":"latest"}' });
+    await latest;
+
+    expect(row(first.db)).toMatchObject({ observed_at: '2026-07-17T12:00:00.000Z' });
+    expect(payload(first.db)).toMatchObject({ data: { status: 'latest' } });
+  });
+
+  it.each([
+    [{ id: 'instance-1', origin: 'http://192.168.1.20:3000', revision: 0 }],
+    [
+      {
+        id: 'instance-1',
+        origin: 'http://192.168.1.20:3000',
+        revision: Number.MAX_SAFE_INTEGER + 1,
+      },
+    ],
+    [{ id: 'different-id', origin: 'http://192.168.1.20:3000', revision: 1 }],
+    [{ id: 'instance-1', origin: 'http://192.168.1.20:3000/', revision: 1 }],
+  ] as const)('fails closed for an invalid owner token %#', async (invalidOwner) => {
+    const { db, owners, service, transport } = fixture();
+    owners.set('instance-1', invalidOwner);
+
+    await expect(service.refreshHealth('instance-1')).rejects.toEqual(
+      expect.objectContaining<Partial<StatusSnapshotError>>({ code: 'INSTANCE_NOT_FOUND' }),
+    );
+    expect(transport.get).not.toHaveBeenCalled();
+    expect(row(db)).toBeUndefined();
+  });
+
+  it('does not persist a probe after its owner origin and revision change', async () => {
+    const { db, instances, owners, service, transport } = fixture();
+    let resolve!: (value: HealthTransportResponse) => void;
+    transport.get.mockImplementationOnce(
+      async () =>
+        new Promise((done) => {
+          resolve = done;
+        }),
+    );
+
+    const running = service.refreshHealth('instance-1');
+    await vi.waitFor(() => expect(transport.get).toHaveBeenCalledTimes(1));
+    owners.set('instance-1', {
+      id: 'instance-1',
+      origin: 'http://192.168.1.21:3000',
+      revision: 2,
+    });
+    resolve({ status: 200, body: '{"status":"old-owner"}' });
+
+    await expect(running).resolves.toMatchObject({
+      state: 'fresh',
+      payload: { status: 'old-owner' },
+    });
+    expect(instances.get).toHaveBeenCalledTimes(2);
+    expect(row(db)).toBeUndefined();
+  });
+
+  it('does not persist a probe after its owner is deleted', async () => {
+    const { db, instances, owners, service, transport } = fixture();
+    let resolve!: (value: HealthTransportResponse) => void;
+    transport.get.mockImplementationOnce(
+      async () =>
+        new Promise((done) => {
+          resolve = done;
+        }),
+    );
+
+    const running = service.refreshHealth('instance-1');
+    await vi.waitFor(() => expect(transport.get).toHaveBeenCalledTimes(1));
+    owners.delete('instance-1');
+    resolve({ status: 200, body: '{"status":"deleted-owner"}' });
+
+    await expect(running).resolves.toMatchObject({ payload: { status: 'deleted-owner' } });
+    expect(instances.get).toHaveBeenCalledTimes(2);
+    expect(row(db)).toBeUndefined();
+  });
+
+  it('atomically fences persistence when the database owner changes after the reread', async () => {
+    const holder: { db?: Database.Database } = {};
+    const result = fixture({
+      idFactory: () => {
+        holder
+          .db!.prepare('UPDATE instances SET base_url=?,config_revision=? WHERE id=?')
+          .run('http://192.168.1.21:3000', 2, 'instance-1');
+        return 'snapshot-id';
+      },
+    });
+    holder.db = result.db;
+
+    await expect(result.service.refreshHealth('instance-1')).resolves.toMatchObject({
+      payload: { status: 'ok' },
+    });
+    expect(result.instances.get).toHaveBeenCalledTimes(2);
+    expect(row(result.db)).toBeUndefined();
+  });
+
   it('does no network/write for missing instances and no write after in-flight deletion', async () => {
     const first = fixture();
     await expect(first.service.refreshHealth('missing')).rejects.toEqual(
@@ -274,6 +402,7 @@ describe('StatusSnapshotService health snapshots', () => {
     );
     const running = first.service.refreshHealth('instance-1');
     await vi.waitFor(() => expect(first.transport.get).toHaveBeenCalledTimes(1));
+    first.owners.delete('instance-1');
     first.db.prepare('DELETE FROM instances WHERE id=?').run('instance-1');
     resolve({ status: 200, body: '{"status":"ok"}' });
     await running;

apps/api/src/application/status/status-snapshot-service.ts

@@ -19,6 +19,7 @@ export type HealthSnapshotErrorCode =
 export interface HealthSnapshotInstance {
   readonly id: string;
   readonly origin: string;
+  readonly revision: number;
 }
 
 export interface HealthEnvelope {
@@ -157,7 +158,32 @@ function transportErrorCode(error: unknown): SnapshotErrorCode {
   return 'UPSTREAM_UNAVAILABLE';
 }
 
-const healthUrl = (origin: string): string => `${origin.replace(/\/$/, '')}/api/health`;
+function isValidOwner(
+  instanceId: string,
+  owner: HealthSnapshotInstance | undefined,
+): owner is HealthSnapshotInstance {
+  if (
+    !owner ||
+    owner.id !== instanceId ||
+    !Number.isSafeInteger(owner.revision) ||
+    owner.revision <= 0
+  ) {
+    return false;
+  }
+  try {
+    const parsed = new URL(owner.origin);
+    return (
+      (parsed.protocol === 'http:' || parsed.protocol === 'https:') &&
+      parsed.origin === owner.origin &&
+      parsed.username === '' &&
+      parsed.password === ''
+    );
+  } catch {
+    return false;
+  }
+}
+
+const healthUrl = (origin: string): string => `${origin}/api/health`;
 
 export class StatusSnapshotService {
   readonly #db: Database.Database;
@@ -188,7 +214,7 @@ export class StatusSnapshotService {
 
   async refreshHealth(instanceId: string): Promise<HealthSnapshot> {
     const instance = await this.#instances.get(instanceId);
-    if (!instance) throw new StatusSnapshotError('INSTANCE_NOT_FOUND');
+    if (!isValidOwner(instanceId, instance)) throw new StatusSnapshotError('INSTANCE_NOT_FOUND');
 
     const generation = ++this.#nextGeneration;
     this.#latestGeneration.set(instanceId, generation);
@@ -201,6 +227,12 @@ export class StatusSnapshotService {
     } catch (error) {
       transportError = error;
     }
+    let currentOwner: HealthSnapshotInstance | undefined;
+    try {
+      currentOwner = await this.#instances.get(instanceId);
+    } catch {
+      // Owner lookup failures fence persistence, but do not discard the computed probe result.
+    }
     const completed = this.#now();
     const observedAt = completed.toISOString();
     const previous = this.#previous(instanceId);
@@ -216,7 +248,13 @@ export class StatusSnapshotService {
     const { snapshot } = classified;
 
     if (this.#latestGeneration.get(instanceId) === generation) {
-      this.#persist(snapshot, classified.persisted);
+      if (
+        isValidOwner(instanceId, currentOwner) &&
+        currentOwner.origin === instance.origin &&
+        currentOwner.revision === instance.revision
+      ) {
+        this.#persist(instance, snapshot, classified.persisted);
+      }
       this.#latestGeneration.delete(instanceId);
     }
     return Object.freeze({ ...snapshot, payload: Object.freeze({ ...snapshot.payload }) });
@@ -354,18 +392,23 @@ export class StatusSnapshotService {
     };
   }
 
-  #persist(snapshot: HealthSnapshot, persisted: PersistedHealthEnvelope): void {
+  #persist(
+    owner: HealthSnapshotInstance,
+    snapshot: HealthSnapshot,
+    persisted: PersistedHealthEnvelope,
+  ): void {
     this.#db
       .prepare(
         `INSERT INTO status_snapshots
           (id,instance_id,category,state,payload_json,observed_at,expires_at,created_at)
-         SELECT ?,id,'health',?,?,?,?,? FROM instances WHERE id=?
+         SELECT ?,id,'health',?,?,?,?,? FROM instances
+         WHERE id=? AND base_url=? AND config_revision=?
          ON CONFLICT(instance_id,category) DO UPDATE SET
           state=excluded.state,
           payload_json=excluded.payload_json,
           observed_at=excluded.observed_at,
           expires_at=excluded.expires_at
-         WHERE excluded.observed_at > status_snapshots.observed_at`,
+         WHERE excluded.observed_at >= status_snapshots.observed_at`,
       )
       .run(
         this.#id(),
@@ -375,6 +418,8 @@ export class StatusSnapshotService {
         snapshot.expiresAt,
         snapshot.observedAt,
         snapshot.instanceId,
+        owner.origin,
+        owner.revision,
       );
   }
 }

package.json

@@ -12,6 +12,7 @@
     "test:legacy": "node --test test/*.test.js",
     "test:unit": "vitest run",
     "test:contract": "node --test packages/operation-registry/test/*.test.ts && vitest run packages/contracts/src/api-v1.contract.test.ts packages/contracts/src/openapi-validator.test.ts",
+    "test:e2e:browser": "corepack pnpm --filter @multi-simadmin/web build && node scripts/real-browser-e2e.mjs --clean-dist",
     "lint": "eslint apps packages/contracts test/phase-one-workspace.test.js eslint.config.js vitest.config.ts",
     "format:check": "prettier --check apps packages/contracts openapi test/phase-one-workspace.test.js eslint.config.js vitest.config.ts tsconfig.base.json pnpm-workspace.yaml package.json",
     "typecheck": "corepack pnpm --recursive --if-present run typecheck"

scripts/real-browser-e2e.mjs

@@ -0,0 +1,406 @@
+#!/usr/bin/env node
+
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { createServer as createHttpServer } from 'node:http';
+import { tmpdir } from 'node:os';
+import { extname, join } from 'node:path';
+import process from 'node:process';
+import { setTimeout as delay } from 'node:timers/promises';
+import { fileURLToPath } from 'node:url';
+
+const { WebSocket } = globalThis;
+const DIST = fileURLToPath(new URL('../apps/web/dist/', import.meta.url));
+const CLEAN_DIST = process.argv.includes('--clean-dist');
+const FORBIDDEN_PORT = 8788;
+
+const CHROME_CANDIDATES = [
+  process.env.CHROME_BIN,
+  '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+  '/Applications/Chromium.app/Contents/MacOS/Chromium',
+  '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
+].filter(Boolean);
+
+const EMPTY_PAGE = { items: [], page: { page: 1, pageSize: 25, total: 0 } };
+
+async function builtAssetHandler(request, response) {
+  const pathname = new URL(request.url ?? '/', 'http://e2e.local').pathname;
+  if (pathname === '/favicon.ico' || pathname === '/api/v1/events') {
+    response.statusCode = 204;
+    response.end();
+    return;
+  }
+  let body;
+  if (pathname === '/api/v1/instances') body = EMPTY_PAGE;
+  if (pathname === '/api/v1/jobs' || pathname === '/api/v1/audit') body = EMPTY_PAGE;
+  if (body !== undefined) {
+    response.statusCode = 200;
+    response.setHeader('content-type', 'application/json; charset=utf-8');
+    response.end(JSON.stringify(body));
+    return;
+  }
+  if (pathname === '/api' || pathname.startsWith('/api/')) {
+    response.statusCode = 404;
+    response.setHeader('content-type', 'application/json; charset=utf-8');
+    response.end(JSON.stringify({ error: 'Not Found', statusCode: 404 }));
+    return;
+  }
+  const relative = pathname.startsWith('/assets/') ? pathname.slice(1) : 'index.html';
+  try {
+    const asset = await readFile(join(DIST, relative));
+    response.statusCode = 200;
+    response.setHeader(
+      'content-type',
+      {
+        '.html': 'text/html; charset=utf-8',
+        '.js': 'text/javascript; charset=utf-8',
+        '.css': 'text/css; charset=utf-8',
+      }[extname(relative)] ?? 'application/octet-stream',
+    );
+    response.end(asset);
+  } catch {
+    response.statusCode = 500;
+    response.end('Built web assets are missing. Run the web build before this gate.');
+  }
+}
+
+async function listenOnSafeEphemeralPort(server) {
+  for (let attempt = 0; attempt < 10; attempt += 1) {
+    await new Promise((resolve, reject) => {
+      server.once('error', reject);
+      server.listen(0, '127.0.0.1', resolve);
+    });
+    const address = server.address();
+    assert(address && typeof address === 'object');
+    if (address.port !== FORBIDDEN_PORT) return address.port;
+    await new Promise((resolve, reject) =>
+      server.close((error) => (error ? reject(error) : resolve())),
+    );
+  }
+  throw new Error(
+    `OS repeatedly selected forbidden legacy port ${FORBIDDEN_PORT} for the E2E server.`,
+  );
+}
+
+async function executableChrome() {
+  const { access, constants } = await import('node:fs/promises');
+  for (const candidate of CHROME_CANDIDATES) {
+    try {
+      await access(candidate, constants.X_OK);
+      return candidate;
+    } catch {
+      // Continue through known local browser locations.
+    }
+  }
+  throw new Error(
+    `No executable Chrome/Chromium found. Set CHROME_BIN to run this real-browser gate.`,
+  );
+}
+
+async function waitForFile(pathname, child) {
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    if (child.exitCode !== null)
+      throw new Error(`Chrome exited before CDP became ready (${child.exitCode}).`);
+    try {
+      return await readFile(pathname, 'utf8');
+    } catch {
+      await delay(50);
+    }
+  }
+  throw new Error('Timed out waiting for Chrome DevToolsActivePort.');
+}
+
+async function waitForPageTarget(port) {
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    try {
+      const response = await fetch(`http://127.0.0.1:${port}/json/list`);
+      const targets = await response.json();
+      const page = targets.find((target) => target.type === 'page');
+      if (page?.webSocketDebuggerUrl) return page.webSocketDebuggerUrl;
+    } catch {
+      // Chrome may expose the port just before the target endpoint is ready.
+    }
+    await delay(50);
+  }
+  throw new Error('Timed out waiting for a Chrome page target.');
+}
+
+class CdpClient {
+  constructor(url) {
+    this.nextId = 1;
+    this.pending = new Map();
+    this.listeners = new Map();
+    this.socket = new WebSocket(url);
+    this.socket.addEventListener('message', (event) => {
+      const message = JSON.parse(event.data);
+      if (message.id) {
+        const pending = this.pending.get(message.id);
+        if (!pending) return;
+        this.pending.delete(message.id);
+        if (message.error) pending.reject(new Error(message.error.message));
+        else pending.resolve(message.result);
+        return;
+      }
+      for (const listener of this.listeners.get(message.method) ?? [])
+        listener(message.params ?? {});
+    });
+  }
+
+  async open() {
+    if (this.socket.readyState === WebSocket.OPEN) return;
+    await new Promise((resolve, reject) => {
+      this.socket.addEventListener('open', resolve, { once: true });
+      this.socket.addEventListener('error', () => reject(new Error('CDP WebSocket failed.')), {
+        once: true,
+      });
+    });
+  }
+
+  send(method, params = {}) {
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject });
+      this.socket.send(JSON.stringify({ id, method, params }));
+    });
+  }
+
+  once(method, timeout = 10_000) {
+    return new Promise((resolve, reject) => {
+      const listener = (params) => {
+        clearTimeout(timer);
+        this.listeners.set(
+          method,
+          (this.listeners.get(method) ?? []).filter((candidate) => candidate !== listener),
+        );
+        resolve(params);
+      };
+      this.listeners.set(method, [...(this.listeners.get(method) ?? []), listener]);
+      const timer = setTimeout(() => {
+        this.listeners.set(
+          method,
+          (this.listeners.get(method) ?? []).filter((candidate) => candidate !== listener),
+        );
+        reject(new Error(`Timed out waiting for CDP ${method}.`));
+      }, timeout);
+    });
+  }
+
+  on(method, listener) {
+    this.listeners.set(method, [...(this.listeners.get(method) ?? []), listener]);
+  }
+
+  async close() {
+    if (this.socket.readyState === WebSocket.CLOSED) return;
+    const closed = new Promise((resolve) =>
+      this.socket.addEventListener('close', resolve, { once: true }),
+    );
+    this.socket.close();
+    await Promise.race([closed, delay(2_000)]);
+  }
+}
+
+async function stopChrome(child) {
+  if (!child || child.exitCode !== null || child.signalCode !== null) return;
+  const exited = new Promise((resolve) => child.once('exit', resolve));
+  child.kill('SIGTERM');
+  if ((await Promise.race([exited.then(() => true), delay(5_000, false)])) === true) return;
+  child.kill('SIGKILL');
+  if ((await Promise.race([exited.then(() => true), delay(2_000, false)])) !== true) {
+    throw new Error('Chrome did not exit after SIGTERM and SIGKILL.');
+  }
+}
+
+async function evaluate(cdp, expression) {
+  const result = await cdp.send('Runtime.evaluate', {
+    expression,
+    awaitPromise: true,
+    returnByValue: true,
+  });
+  if (result.exceptionDetails)
+    throw new Error(result.exceptionDetails.text ?? 'Browser evaluation failed.');
+  return result.result.value;
+}
+
+async function eventually(cdp, expression, message) {
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    try {
+      const value = await evaluate(cdp, expression);
+      if (value) return value;
+    } catch {
+      // A full-page navigation briefly destroys the execution context.
+    }
+    await delay(50);
+  }
+  throw new Error(message);
+}
+
+async function press(cdp, key, code, windowsVirtualKeyCode, text) {
+  await cdp.send('Input.dispatchKeyEvent', {
+    type: 'keyDown',
+    key,
+    code,
+    windowsVirtualKeyCode,
+    text,
+  });
+  await cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', key, code, windowsVirtualKeyCode });
+}
+
+async function keyboardNavigate(cdp, label, expectedPath, expectedHeading) {
+  const focused = await evaluate(
+    cdp,
+    `(() => { const link = [...document.querySelectorAll('nav[aria-label="Global navigation"] a')].find((item) => item.textContent.trim() === ${JSON.stringify(label)}); if (!link) return false; link.focus(); return document.activeElement === link; })()`,
+  );
+  assert.equal(focused, true, `${label} navigation link must accept keyboard focus`);
+  await press(cdp, 'Enter', 'Enter', 13, '\r');
+  await eventually(
+    cdp,
+    `location.pathname === ${JSON.stringify(expectedPath)}`,
+    `${label} keyboard navigation failed`,
+  );
+  await eventually(
+    cdp,
+    `document.querySelector('h1')?.textContent.trim() === ${JSON.stringify(expectedHeading)}`,
+    `${label} heading did not render`,
+  );
+  assert.equal(
+    await evaluate(cdp, `document.querySelector('nav a[aria-current="page"]')?.textContent.trim()`),
+    label,
+  );
+}
+
+let http;
+let chrome;
+let cdp;
+let profile;
+
+try {
+  const chromeBinary = await executableChrome();
+  http = createHttpServer((request, response) => void builtAssetHandler(request, response));
+  const serverPort = await listenOnSafeEphemeralPort(http);
+  const origin = `http://127.0.0.1:${serverPort}`;
+
+  profile = await mkdtemp(join(tmpdir(), 'multi-simadmin-chrome-'));
+  chrome = spawn(
+    chromeBinary,
+    [
+      '--headless=new',
+      '--disable-gpu',
+      '--no-first-run',
+      '--no-default-browser-check',
+      '--remote-debugging-port=0',
+      `--user-data-dir=${profile}`,
+      'about:blank',
+    ],
+    { stdio: ['ignore', 'ignore', 'pipe'] },
+  );
+  let chromeStderr = '';
+  chrome.stderr.on('data', (chunk) => {
+    chromeStderr += chunk.toString();
+  });
+
+  const activePort = await waitForFile(join(profile, 'DevToolsActivePort'), chrome);
+  const port = Number(activePort.split(/\r?\n/u)[0]);
+  assert(Number.isInteger(port) && port > 0, 'Chrome did not publish a valid CDP port');
+  cdp = new CdpClient(await waitForPageTarget(port));
+  await cdp.open();
+
+  const failures = [];
+  cdp.on('Runtime.exceptionThrown', ({ exceptionDetails }) =>
+    failures.push(`uncaught exception: ${exceptionDetails?.text ?? 'unknown'}`),
+  );
+  cdp.on('Runtime.consoleAPICalled', ({ type, args }) => {
+    if (type === 'error' || type === 'assert')
+      failures.push(
+        `console.${type}: ${args?.map((arg) => arg.value ?? arg.description).join(' ')}`,
+      );
+  });
+  cdp.on('Log.entryAdded', ({ entry }) => {
+    if (entry?.level === 'error') failures.push(`browser log: ${entry.text}`);
+  });
+  await Promise.all([cdp.send('Page.enable'), cdp.send('Runtime.enable'), cdp.send('Log.enable')]);
+  assert.equal(
+    await evaluate(cdp, 'location.href'),
+    'about:blank',
+    'Chrome must attach at about:blank before app navigation',
+  );
+
+  const unknownApi = await fetch(`${origin}/api/v1/not-a-real-route`);
+  assert.equal(unknownApi.status, 404, 'Unknown API routes must not receive the SPA shell');
+  assert.match(unknownApi.headers.get('content-type') ?? '', /^application\/json\b/u);
+  assert.deepEqual(await unknownApi.json(), { error: 'Not Found', statusCode: 404 });
+
+  const loaded = cdp.once('Page.loadEventFired');
+  await cdp.send('Page.navigate', { url: `${origin}/fleet` });
+  await loaded;
+
+  await eventually(
+    cdp,
+    `document.querySelector('h1')?.textContent.trim() === 'Fleet'`,
+    'Fleet did not render',
+  );
+  await eventually(
+    cdp,
+    `document.body.textContent.includes('No instances are configured.')`,
+    'Fleet mock data did not load',
+  );
+
+  assert.equal(
+    await evaluate(
+      cdp,
+      `(() => { const input = document.querySelector('input[type="search"]'); input.focus(); return document.activeElement === input; })()`,
+    ),
+    true,
+    'Fleet search must accept focus',
+  );
+  for (const character of 'edge')
+    await press(
+      cdp,
+      character,
+      `Key${character.toUpperCase()}`,
+      character.toUpperCase().charCodeAt(0),
+      character,
+    );
+  assert.equal(await evaluate(cdp, `document.querySelector('input[type="search"]').value`), 'edge');
+
+  await keyboardNavigate(cdp, 'Jobs', '/jobs', 'Jobs');
+  await eventually(
+    cdp,
+    `document.body.textContent.includes('No jobs match the current query.')`,
+    'Jobs mock data did not load',
+  );
+  await keyboardNavigate(cdp, 'Audit', '/audit', 'Audit');
+  await eventually(
+    cdp,
+    `document.body.textContent.includes('No audit events match the current query.')`,
+    'Audit mock data did not load',
+  );
+  await keyboardNavigate(cdp, 'Settings', '/settings/instances', 'Instances');
+  await eventually(
+    cdp,
+    `document.body.textContent.includes('No instances are configured.')`,
+    'Settings mock data did not load',
+  );
+
+  assert.deepEqual(failures, [], `Browser failures detected:\n${failures.join('\n')}`);
+  console.log(`PASS real Chrome E2E (${chromeBinary})`);
+  console.log(`PASS isolated built-asset server on ${origin} (legacy port 8788 untouched)`);
+  console.log(
+    'PASS keyboard navigation and rendered API states: /fleet -> /jobs -> /audit -> /settings/instances',
+  );
+} catch (error) {
+  console.error(error instanceof Error ? error.stack : error);
+  process.exitCode = 1;
+} finally {
+  try {
+    await cdp?.close();
+    await stopChrome(chrome);
+  } finally {
+    if (http) {
+      http.closeAllConnections?.();
+      await new Promise((resolve) => http.close(resolve));
+    }
+    if (profile) await rm(profile, { recursive: true, force: true });
+    if (CLEAN_DIST) await rm(DIST, { recursive: true, force: true });
+  }
+}