feat(release): add reversible production cutover gates

apps/api/package.json

@@ -8,6 +8,7 @@
     "test": "vitest run --root ../.. apps/api/src",
     "typecheck": "tsc -p tsconfig.json",
     "canary": "tsx src/canary-cli.ts",
+    "gateway:production": "tsx src/production-gateway-cli.ts",
     "start:production": "tsx src/production-cli.ts",
     "migration": "tsx src/application/legacy-import/migration-command.ts"
   },

apps/api/src/canary-gateway.ts

@@ -18,6 +18,7 @@ export const CANARY_DEFAULT_UPSTREAM_HOST = '127.0.0.1';
 export const CANARY_DEFAULT_UPSTREAM_PORT = 8790;
 
 const RESERVED_PORTS = new Set([8788, 8790]);
+export const PRODUCTION_GATEWAY_PORT = 8788;
 const BASE_HOP_BY_HOP_HEADERS = new Set([
   'connection',
   'keep-alive',
@@ -62,6 +63,8 @@ export interface CanaryGatewayOptions {
   readonly port?: number;
   readonly upstreamHost?: string;
   readonly upstreamPort?: number;
+  /** Set only after the production runtime validates its explicit acknowledgement. */
+  readonly ownsProductionPort?: boolean;
 }
 
 export interface CanaryGateway {
@@ -201,7 +204,7 @@ export function createCanaryGateway(options: CanaryGatewayOptions): CanaryGatewa
     throw new Error('Canary upstream must use a loopback host');
   }
   if (!Number.isInteger(port) || port < 0 || port > 65_535) throw new Error('Invalid canary port');
-  if (RESERVED_PORTS.has(port))
+  if (RESERVED_PORTS.has(port) && !(port === PRODUCTION_GATEWAY_PORT && options.ownsProductionPort))
     throw new Error(`Port ${port} is reserved and cannot run the canary`);
   if (!options.distDir) throw new Error('distDir is required');
 

apps/api/src/canary-runtime.ts

@@ -10,8 +10,11 @@ export interface CanaryRuntimeEnvironment {
   readonly CANARY_PORT?: string;
   readonly CANARY_UPSTREAM_PORT?: string;
   readonly MULTI_SIMADMIN_GATEWAY_TOKEN?: string;
+  readonly MULTI_SIMADMIN_CUTOVER_ACK?: string;
 }
 
+export const PRODUCTION_CUTOVER_ACK = 'I ACKNOWLEDGE MULTI-SIMADMIN OWNS PORT 8788';
+
 function environmentPort(value: string | undefined, name: string, fallback: number): number {
   if (value === undefined || value === '') return fallback;
   const port = Number(value);
@@ -41,3 +44,15 @@ export function readCanaryRuntimeOptions(
     gatewayToken,
   });
 }
+
+export function readProductionGatewayRuntimeOptions(
+  environment: CanaryRuntimeEnvironment,
+): CanaryGatewayOptions {
+  if (environment.MULTI_SIMADMIN_CUTOVER_ACK !== PRODUCTION_CUTOVER_ACK) {
+    throw new Error(
+      `Production gateway requires MULTI_SIMADMIN_CUTOVER_ACK=${PRODUCTION_CUTOVER_ACK}`,
+    );
+  }
+  const options = readCanaryRuntimeOptions({ ...environment, CANARY_PORT: '8788' });
+  return Object.freeze({ ...options, ownsProductionPort: true });
+}

apps/api/src/cli-runtime.test.ts

@@ -18,7 +18,7 @@ interface RunningCommand {
 }
 
 function startPackageCommand(
-  script: 'canary' | 'start:production',
+  script: 'canary' | 'gateway:production' | 'start:production',
   environment: NodeJS.ProcessEnv,
 ) {
   const child = spawn('corepack', ['pnpm', '--filter', '@multi-simadmin/api', 'run', script], {
@@ -55,9 +55,13 @@ async function stopCommand(command: RunningCommand): Promise<void> {
   expect(result.code === 0 || result.code === 143 || result.signal === 'SIGTERM').toBe(true);
 }
 
-async function waitForOutput(command: RunningCommand, text: string): Promise<void> {
-  const deadline = Date.now() + 10_000;
-  while (!command.output().includes(text)) {
+async function waitForOutput(
+  command: RunningCommand,
+  pattern: string,
+  timeoutMs = 20_000,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (!command.output().includes(pattern)) {
     if (command.child.exitCode !== null || command.child.signalCode !== null) {
       throw new Error(`Runtime exited before becoming ready:\n${command.output()}`);
     }
@@ -103,6 +107,33 @@ afterEach(async () => {
 });
 
 describe.sequential('executable package runtimes', () => {
+  it('makes the exact production gateway executable attempt 8788 only with acknowledgement', async () => {
+    await expectPortFree(8788);
+    const fixture = await listenOn(8788);
+    cleanup.push(() => closeServer(fixture));
+    const directory = await mkdtemp(join(tmpdir(), 'multi-simadmin-production-gateway-cli-'));
+    cleanup.push(() => rm(directory, { recursive: true, force: true }));
+    await writeFile(join(directory, 'index.html'), '<main>production</main>');
+
+    const denied = startPackageCommand('gateway:production', {
+      CANARY_DIST_DIR: directory,
+      MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+    });
+    await waitForOutput(denied, 'Production gateway requires MULTI_SIMADMIN_CUTOVER_ACK=');
+    await new Promise<void>((resolveExit) => denied.child.once('exit', () => resolveExit()));
+    expect(denied.child.exitCode).not.toBe(0);
+
+    const acknowledged = startPackageCommand('gateway:production', {
+      CANARY_DIST_DIR: directory,
+      MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+      MULTI_SIMADMIN_CUTOVER_ACK: 'I ACKNOWLEDGE MULTI-SIMADMIN OWNS PORT 8788',
+    });
+    await waitForOutput(acknowledged, 'EADDRINUSE');
+    await new Promise<void>((resolveExit) => acknowledged.child.once('exit', () => resolveExit()));
+    expect(acknowledged.output()).toContain('127.0.0.1:8788');
+    expect(acknowledged.child.exitCode).not.toBe(0);
+  }, 20_000);
+
   it('starts the exact production command on 8790 and shuts down on SIGTERM', async () => {
     await expectPortFree(8790);
     const legacyListeners = await listenerPids(8788);
@@ -115,6 +146,7 @@ describe.sequential('executable package runtimes', () => {
       MULTI_SIMADMIN_DATA_ROOT: dataRoot,
       MULTI_SIMADMIN_DATABASE_PATH: join(dataRoot, 'synthetic.sqlite'),
       MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+      MULTI_SIMADMIN_WEB_DIST: directory,
       API_HOST: '127.0.0.1',
       API_PORT: '8790',
     });

apps/api/src/cutover-cli.ts

@@ -0,0 +1,23 @@
+import { readFile, stat } from 'node:fs/promises';
+import {
+  cutover,
+  nodeCutoverSystem,
+  parseCutoverPlan,
+  preflight,
+  rollback,
+} from './cutover-orchestrator.js';
+
+const [action, planPath] = process.argv.slice(2);
+if (!['preflight', 'cutover', 'rollback'].includes(action ?? '') || !planPath) {
+  throw new Error('Usage: pnpm cutover <preflight|cutover|rollback> <plan.json>');
+}
+const planInfo = await stat(planPath);
+if (!planInfo.isFile() || (planInfo.mode & 0o077) !== 0) {
+  throw new Error('Cutover plan must be a regular file inaccessible to group and other users');
+}
+const plan = parseCutoverPlan(JSON.parse(await readFile(planPath, 'utf8')) as unknown);
+const gatewayToken = process.env.MULTI_SIMADMIN_GATEWAY_TOKEN ?? '';
+if (action === 'preflight') await preflight(plan, nodeCutoverSystem, gatewayToken);
+else if (action === 'cutover') await cutover(plan, nodeCutoverSystem, gatewayToken);
+else await rollback(plan, nodeCutoverSystem);
+process.stdout.write(`${action} completed\n`);

apps/api/src/cutover-orchestrator.ts

@@ -0,0 +1,470 @@
+import { spawn, execFile } from 'node:child_process';
+import { constants } from 'node:fs';
+import {
+  access,
+  chmod,
+  lstat,
+  mkdir,
+  open,
+  readdir,
+  readFile,
+  readlink,
+  realpath,
+  rename,
+} from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { request } from 'node:http';
+import { isAbsolute, join, resolve } from 'node:path';
+import { GATEWAY_AUTH_HEADER } from './runtime-config.js';
+import { PRODUCTION_CUTOVER_ACK } from './canary-runtime.js';
+
+const STATE_VERSION = 1;
+const STATE_FILE = 'cutover-state.json';
+const GATEWAY_ARGV = [
+  resolve(process.cwd(), 'apps/api/node_modules/.bin/tsx'),
+  resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'),
+] as const;
+const DIGEST = /^[a-f0-9]{64}$/u;
+
+export interface CutoverPlan {
+  readonly assetDir: string;
+  readonly backupEvidence: string;
+  readonly backupEvidenceSha256: string;
+  readonly drillEvidence: string;
+  readonly drillEvidenceSha256: string;
+  readonly legacyPid: number;
+  readonly legacyCommand: string;
+  readonly legacyIdentity: string;
+  readonly legacyStart: readonly [string, ...string[]];
+  readonly stateDir: string;
+}
+
+interface CutoverState {
+  readonly version: 1;
+  readonly phase: 'prepared' | 'active';
+  readonly legacyPid: number;
+  readonly legacyCommand: string;
+  readonly legacyIdentity: string;
+  readonly legacyStart: readonly [string, ...string[]];
+  readonly gatewayPid?: number;
+  readonly gatewayCommand?: string;
+  readonly gatewayIdentity?: string;
+}
+
+export interface CutoverSystem {
+  command(pid: number): Promise<string>;
+  identity(pid: number): Promise<string>;
+  ready(port: 8788 | 8790, gatewayToken: string): Promise<boolean>;
+  ownsProductionPort(pid: number): Promise<boolean>;
+  signal(pid: number, signal: NodeJS.Signals): void;
+  waitGone(pid: number): Promise<void>;
+  start(argv: readonly [string, ...string[]], environment?: NodeJS.ProcessEnv): number;
+  delay(milliseconds: number): Promise<void>;
+}
+
+function runFile(file: string, args: readonly string[]): Promise<string> {
+  return new Promise((resolvePromise, reject) =>
+    execFile(file, [...args], (error, stdout) =>
+      error ? reject(error) : resolvePromise(stdout.trim()),
+    ),
+  );
+}
+
+function authenticatedReady(port: 8788 | 8790, gatewayToken: string): Promise<boolean> {
+  return new Promise((resolvePromise) => {
+    const req = request(
+      {
+        host: '127.0.0.1',
+        port,
+        path: '/readyz',
+        timeout: 2_000,
+        headers: { [GATEWAY_AUTH_HEADER]: gatewayToken },
+      },
+      (response) => {
+        response.resume();
+        resolvePromise(response.statusCode === 200);
+      },
+    );
+    req.once('error', () => resolvePromise(false));
+    req.once('timeout', () => {
+      req.destroy();
+      resolvePromise(false);
+    });
+    req.end();
+  });
+}
+
+export const nodeCutoverSystem: CutoverSystem = {
+  async command(pid) {
+    const proc = await readFile(`/proc/${pid}/cmdline`).catch(() => undefined);
+    if (proc) return proc.toString().split('\0').filter(Boolean).join(' ');
+    return await runFile('ps', ['-p', String(pid), '-o', 'command=']);
+  },
+  async identity(pid) {
+    if (process.platform === 'linux') {
+      const value = await readFile(`/proc/${pid}/stat`, 'utf8');
+      const fields = value.slice(value.lastIndexOf(')') + 2).split(/\s+/u);
+      return fields[19] ?? '';
+    }
+    return await runFile('ps', ['-p', String(pid), '-o', 'lstart=']);
+  },
+  ready: authenticatedReady,
+  async ownsProductionPort(pid) {
+    if (process.platform === 'linux') {
+      const descriptorDirectory = `/proc/${pid}/fd`;
+      const descriptors = await readdir(descriptorDirectory).catch(() => []);
+      const listeners = await Promise.all(
+        descriptors.map((descriptor) =>
+          readlink(join(descriptorDirectory, descriptor)).catch(() => ''),
+        ),
+      );
+      const socketInodes = new Set(
+        listeners
+          .map((line) => /^socket:\[(\d+)\]$/u.exec(line)?.[1])
+          .filter((inode): inode is string => inode !== undefined),
+      );
+      const tcp = await readFile('/proc/net/tcp', 'utf8').catch(() => '');
+      return tcp.split('\n').some((line) => {
+        const fields = line.trim().split(/\s+/u);
+        return (
+          fields[1]?.endsWith(':2254') && fields[3] === '0A' && socketInodes.has(fields[9] ?? '')
+        );
+      });
+    }
+    const output = await runFile('lsof', [
+      '-nP',
+      '-a',
+      '-p',
+      String(pid),
+      '-iTCP:8788',
+      '-sTCP:LISTEN',
+      '-t',
+    ]).catch(() => '');
+    return output.split(/\s+/u).includes(String(pid));
+  },
+  signal: (pid, signal) => process.kill(pid, signal),
+  async waitGone(pid) {
+    const deadline = Date.now() + 10_000;
+    while (Date.now() < deadline) {
+      try {
+        process.kill(pid, 0);
+      } catch {
+        return;
+      }
+      await new Promise((resolveWait) => setTimeout(resolveWait, 100));
+    }
+    throw new Error(`PID ${pid} did not exit after SIGTERM`);
+  },
+  start(argv, environment) {
+    const child = spawn(argv[0], argv.slice(1), {
+      detached: true,
+      stdio: 'ignore',
+      env: environment ?? process.env,
+    });
+    child.unref();
+    if (!child.pid) throw new Error('Failed to start command');
+    return child.pid;
+  },
+  delay: (milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)),
+};
+
+function record(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value))
+    throw new Error(`${label} must be an object`);
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  expected: readonly string[],
+  label: string,
+): void {
+  const actual = Object.keys(value).sort();
+  const wanted = [...expected].sort();
+  if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index]))
+    throw new Error(`${label} has an invalid schema`);
+}
+
+function safeString(value: unknown, label: string): string {
+  if (typeof value !== 'string' || value.length === 0 || value.includes('\0'))
+    throw new Error(`${label} must be a non-empty string`);
+  return value;
+}
+
+function argv(value: unknown, label: string): readonly [string, ...string[]] {
+  if (
+    !Array.isArray(value) ||
+    value.length === 0 ||
+    value.length > 64 ||
+    value.some((item) => typeof item !== 'string' || item.length === 0 || item.includes('\0'))
+  )
+    throw new Error(`${label} must be a non-empty string array`);
+  return value as [string, ...string[]];
+}
+
+export function parseCutoverPlan(value: unknown): CutoverPlan {
+  const input = record(value, 'Cutover plan');
+  exactKeys(
+    input,
+    [
+      'assetDir',
+      'backupEvidence',
+      'backupEvidenceSha256',
+      'drillEvidence',
+      'drillEvidenceSha256',
+      'legacyPid',
+      'legacyCommand',
+      'legacyIdentity',
+      'legacyStart',
+      'stateDir',
+    ],
+    'Cutover plan',
+  );
+  const plan: CutoverPlan = {
+    assetDir: safeString(input.assetDir, 'assetDir'),
+    backupEvidence: safeString(input.backupEvidence, 'backupEvidence'),
+    backupEvidenceSha256: safeString(input.backupEvidenceSha256, 'backupEvidenceSha256'),
+    drillEvidence: safeString(input.drillEvidence, 'drillEvidence'),
+    drillEvidenceSha256: safeString(input.drillEvidenceSha256, 'drillEvidenceSha256'),
+    legacyPid: input.legacyPid as number,
+    legacyCommand: safeString(input.legacyCommand, 'legacyCommand'),
+    legacyIdentity: safeString(input.legacyIdentity, 'legacyIdentity'),
+    legacyStart: argv(input.legacyStart, 'legacyStart'),
+    stateDir: safeString(input.stateDir, 'stateDir'),
+  };
+  if (
+    ![
+      plan.assetDir,
+      plan.backupEvidence,
+      plan.drillEvidence,
+      plan.stateDir,
+      plan.legacyStart[0],
+    ].every(isAbsolute)
+  )
+    throw new Error('Cutover paths and legacy executable must be absolute');
+  if (!Number.isSafeInteger(plan.legacyPid) || plan.legacyPid <= 1)
+    throw new Error('legacyPid must be a safe PID greater than 1');
+  if (!DIGEST.test(plan.backupEvidenceSha256) || !DIGEST.test(plan.drillEvidenceSha256))
+    throw new Error('Evidence digests must be lowercase SHA-256');
+  if (plan.legacyStart.join(' ') !== plan.legacyCommand)
+    throw new Error('legacyStart must exactly reproduce legacyCommand');
+  return Object.freeze(plan);
+}
+
+function requireGatewayToken(gatewayToken: string): void {
+  const bytes = Buffer.byteLength(gatewayToken);
+  if (bytes < 32 || bytes > 512)
+    throw new Error('Gateway token must contain between 32 and 512 bytes');
+}
+
+async function verifyEvidence(path: string, digest: string, label: string): Promise<void> {
+  const info = await lstat(path);
+  if (!info.isFile()) throw new Error(`${label} evidence must be a regular file`);
+  const content = await readFile(path);
+  const actual = createHash('sha256').update(content).digest('hex');
+  if (actual !== digest) throw new Error(`${label} evidence SHA-256 mismatch`);
+}
+
+async function durableState(plan: CutoverPlan, state: CutoverState): Promise<void> {
+  await mkdir(plan.stateDir, { recursive: true, mode: 0o700 });
+  await chmod(plan.stateDir, 0o700);
+  const target = join(plan.stateDir, STATE_FILE);
+  const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
+  const handle = await open(
+    temporary,
+    constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
+    0o600,
+  );
+  try {
+    await handle.writeFile(`${JSON.stringify(state)}\n`);
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+  await rename(temporary, target);
+  const directory = await open(plan.stateDir, constants.O_RDONLY);
+  try {
+    await directory.sync();
+  } finally {
+    await directory.close();
+  }
+}
+
+function preparedState(plan: CutoverPlan): CutoverState {
+  return {
+    version: STATE_VERSION,
+    phase: 'prepared',
+    legacyPid: plan.legacyPid,
+    legacyCommand: plan.legacyCommand,
+    legacyIdentity: plan.legacyIdentity,
+    legacyStart: plan.legacyStart,
+  };
+}
+
+async function readState(plan: CutoverPlan): Promise<CutoverState> {
+  const path = join(plan.stateDir, STATE_FILE);
+  const info = await lstat(path);
+  if (!info.isFile() || (info.mode & 0o777) !== 0o600)
+    throw new Error('Cutover state must be a mode-0600 regular file');
+  const input = record(JSON.parse(await readFile(path, 'utf8')) as unknown, 'Cutover state');
+  const base = ['version', 'phase', 'legacyPid', 'legacyCommand', 'legacyIdentity', 'legacyStart'];
+  exactKeys(
+    input,
+    input.phase === 'active' ? [...base, 'gatewayPid', 'gatewayCommand', 'gatewayIdentity'] : base,
+    'Cutover state',
+  );
+  const state: CutoverState = {
+    version: input.version as 1,
+    phase: input.phase as 'prepared' | 'active',
+    legacyPid: input.legacyPid as number,
+    legacyCommand: safeString(input.legacyCommand, 'state legacyCommand'),
+    legacyIdentity: safeString(input.legacyIdentity, 'state legacyIdentity'),
+    legacyStart: argv(input.legacyStart, 'state legacyStart'),
+    ...(input.phase === 'active'
+      ? {
+          gatewayPid: input.gatewayPid as number,
+          gatewayCommand: safeString(input.gatewayCommand, 'state gatewayCommand'),
+          gatewayIdentity: safeString(input.gatewayIdentity, 'state gatewayIdentity'),
+        }
+      : {}),
+  };
+  if (state.version !== STATE_VERSION || !['prepared', 'active'].includes(state.phase))
+    throw new Error('Unsupported cutover state');
+  if (
+    !Number.isSafeInteger(state.legacyPid) ||
+    state.legacyPid <= 1 ||
+    (state.gatewayPid !== undefined &&
+      (!Number.isSafeInteger(state.gatewayPid) || state.gatewayPid <= 1))
+  )
+    throw new Error('Cutover state contains an invalid PID');
+  if (
+    state.legacyPid !== plan.legacyPid ||
+    state.legacyCommand !== plan.legacyCommand ||
+    state.legacyIdentity !== plan.legacyIdentity ||
+    state.legacyStart.length !== plan.legacyStart.length ||
+    state.legacyStart.some((item, index) => item !== plan.legacyStart[index])
+  )
+    throw new Error('Cutover state does not match the reviewed plan');
+  return state;
+}
+
+export async function preflight(
+  plan: CutoverPlan,
+  system: CutoverSystem,
+  gatewayToken: string,
+): Promise<void> {
+  requireGatewayToken(gatewayToken);
+  if (!(await system.ready(8790, gatewayToken)))
+    throw new Error('Authenticated API readiness preflight failed on 127.0.0.1:8790/readyz');
+  const indexPath = join(await realpath(plan.assetDir), 'index.html');
+  const asset = await lstat(indexPath);
+  if (!asset.isFile()) throw new Error('assetDir/index.html must be a regular file');
+  await access(plan.legacyStart[0], constants.X_OK);
+  await verifyEvidence(plan.backupEvidence, plan.backupEvidenceSha256, 'Backup');
+  await verifyEvidence(plan.drillEvidence, plan.drillEvidenceSha256, 'Rollback drill');
+  const actual = await system.command(plan.legacyPid);
+  const identity = await system.identity(plan.legacyPid);
+  if (actual !== plan.legacyCommand || identity !== plan.legacyIdentity)
+    throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
+}
+
+async function restoreLegacy(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
+  return system.start(plan.legacyStart);
+}
+
+async function waitForGateway(
+  system: CutoverSystem,
+  gatewayPid: number,
+  gatewayToken: string,
+): Promise<boolean> {
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    if ((await system.ownsProductionPort(gatewayPid)) && (await system.ready(8788, gatewayToken)))
+      return true;
+    await system.delay(100);
+  }
+  return false;
+}
+
+export async function cutover(
+  plan: CutoverPlan,
+  system: CutoverSystem,
+  gatewayToken: string,
+): Promise<number> {
+  await preflight(plan, system, gatewayToken);
+  await durableState(plan, preparedState(plan));
+  let legacyStopped = false;
+  let gatewayPid: number | undefined;
+  let gatewayCommand: string | undefined;
+  let gatewayIdentity: string | undefined;
+  try {
+    if (
+      (await system.command(plan.legacyPid)) !== plan.legacyCommand ||
+      (await system.identity(plan.legacyPid)) !== plan.legacyIdentity
+    )
+      throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
+    system.signal(plan.legacyPid, 'SIGTERM');
+    await system.waitGone(plan.legacyPid);
+    legacyStopped = true;
+    gatewayPid = system.start(GATEWAY_ARGV, {
+      ...process.env,
+      CANARY_DIST_DIR: plan.assetDir,
+      MULTI_SIMADMIN_CUTOVER_ACK: PRODUCTION_CUTOVER_ACK,
+      MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+    });
+    gatewayCommand = await system.command(gatewayPid);
+    gatewayIdentity = await system.identity(gatewayPid);
+    if (!(await waitForGateway(system, gatewayPid, gatewayToken)))
+      throw new Error(
+        'Production gateway failed authenticated ownership verification on port 8788',
+      );
+    await durableState(plan, {
+      ...preparedState(plan),
+      phase: 'active',
+      gatewayPid,
+      gatewayCommand,
+      gatewayIdentity,
+    });
+    return gatewayPid;
+  } catch (error) {
+    if (gatewayPid !== undefined) {
+      try {
+        const actual = await system.command(gatewayPid);
+        const identity = await system.identity(gatewayPid);
+        if (actual === gatewayCommand && identity === gatewayIdentity) {
+          system.signal(gatewayPid, 'SIGTERM');
+          await system.waitGone(gatewayPid);
+        }
+      } catch {
+        // Continue the safety-critical legacy restart even if gateway cleanup races its exit.
+      }
+    }
+    if (legacyStopped) {
+      if (gatewayPid !== undefined && (await system.ownsProductionPort(gatewayPid)))
+        throw new Error('Automatic rollback refused while production gateway still owns port 8788');
+      await restoreLegacy(plan, system);
+    }
+    throw error;
+  }
+}
+
+export async function rollback(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
+  const state = await readState(plan);
+  if (
+    state.phase !== 'active' ||
+    state.gatewayPid === undefined ||
+    state.gatewayCommand === undefined ||
+    state.gatewayIdentity === undefined
+  )
+    throw new Error('Cutover state is not active; refusing rollback PID operations');
+  const actual = await system.command(state.gatewayPid);
+  const identity = await system.identity(state.gatewayPid);
+  if (
+    actual !== state.gatewayCommand ||
+    identity !== state.gatewayIdentity ||
+    !(await system.ownsProductionPort(state.gatewayPid))
+  )
+    throw new Error(`Gateway PID identity mismatch; refusing to signal PID ${state.gatewayPid}`);
+  system.signal(state.gatewayPid, 'SIGTERM');
+  await system.waitGone(state.gatewayPid);
+  return restoreLegacy(plan, system);
+}

apps/api/src/production-cli.ts

@@ -1,12 +1,52 @@
+import { lstat, readdir } from 'node:fs/promises';
+import { basename, dirname, join } from 'node:path';
+
 import { buildProductionControlPlane } from './production-control-plane.js';
-import { createRuntimeConfig } from './runtime-config.js';
+import { createRuntimeConfig, GATEWAY_AUTH_HEADER } from './runtime-config.js';
 import { installBoundedShutdown } from './shutdown.js';
 
+async function regularFile(path: string): Promise<boolean> {
+  try {
+    const value = await lstat(path);
+    return value.isFile() && !value.isSymbolicLink();
+  } catch {
+    return false;
+  }
+}
+
+async function recoveryMarkers(databasePath: string): Promise<number> {
+  const directory = dirname(databasePath);
+  const name = basename(databasePath);
+  const names = await readdir(directory).catch(() => []);
+  return names.filter(
+    (entry) =>
+      entry === `${name}.restore-in-progress` ||
+      (entry.startsWith(`${name}.`) &&
+        (entry.endsWith('.restore') || entry.endsWith('.quarantine'))),
+  ).length;
+}
+
+async function gatewayStaticReachable(token: string): Promise<boolean> {
+  try {
+    const response = await fetch('http://127.0.0.1:8789/', {
+      headers: { [GATEWAY_AUTH_HEADER]: token },
+      signal: AbortSignal.timeout(2_000),
+    });
+    await response.body?.cancel();
+    return response.status === 200;
+  } catch {
+    return false;
+  }
+}
+
 try {
   const config = createRuntimeConfig(process.env);
   const app = buildProductionControlPlane({
     databasePath: config.databasePath,
     gatewayToken: config.gatewayToken,
+    assetsCheck: () => regularFile(join(config.webDist, 'index.html')),
+    recoveryMarkersCheck: () => recoveryMarkers(config.databasePath),
+    upstreamGatewayCheck: () => gatewayStaticReachable(config.gatewayToken),
   });
   await app.listen({ host: config.api.host, port: config.api.port });
   installBoundedShutdown(process, app, {

apps/api/src/production-control-plane.ts

@@ -17,6 +17,9 @@ export interface ProductionControlPlaneOptions {
   readonly upstream?: SafeControlPlaneUpstream;
   readonly gatewayToken?: string;
   readonly keychainMetadataCheck?: () => boolean | Promise<boolean>;
+  readonly assetsCheck?: () => boolean | Promise<boolean>;
+  readonly recoveryMarkersCheck?: () => number | Promise<number>;
+  readonly upstreamGatewayCheck?: () => boolean | Promise<boolean>;
 }
 
 export function buildProductionControlPlane(
@@ -48,6 +51,13 @@ export function buildProductionControlPlane(
           db,
           gatewayToken: options.gatewayToken ?? '',
           keychainMetadataCheck: options.keychainMetadataCheck ?? defaultKeychainMetadataCheck,
+          ...(options.assetsCheck ? { assetsCheck: options.assetsCheck } : {}),
+          ...(options.recoveryMarkersCheck
+            ? { recoveryMarkersCheck: options.recoveryMarkersCheck }
+            : {}),
+          ...(options.upstreamGatewayCheck
+            ? { upstreamGatewayCheck: options.upstreamGatewayCheck }
+            : {}),
         }),
       },
     });

apps/api/src/production-gateway-cli.ts

@@ -0,0 +1,21 @@
+import { createCanaryGateway } from './canary-gateway.js';
+import { readProductionGatewayRuntimeOptions } from './canary-runtime.js';
+
+const gateway = createCanaryGateway(readProductionGatewayRuntimeOptions(process.env));
+let stopping = false;
+async function stop(signal: string): Promise<void> {
+  if (stopping) return;
+  stopping = true;
+  process.stdout.write(`Stopping production gateway (${signal})\n`);
+  await gateway.stop();
+}
+process.once('SIGINT', () => void stop('SIGINT'));
+process.once('SIGTERM', () => void stop('SIGTERM'));
+
+try {
+  await gateway.start();
+  process.stdout.write(`Production gateway listening at ${gateway.origin}\n`);
+} catch (error) {
+  process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+  process.exitCode = 1;
+}

apps/api/src/production-gateway.test.ts

@@ -0,0 +1,193 @@
+import { createHash } from 'node:crypto';
+import { chmod, lstat, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { createCanaryGateway } from './canary-gateway.js';
+import { PRODUCTION_CUTOVER_ACK, readProductionGatewayRuntimeOptions } from './canary-runtime.js';
+import {
+  cutover,
+  parseCutoverPlan,
+  preflight,
+  rollback,
+  type CutoverPlan,
+  type CutoverSystem,
+} from './cutover-orchestrator.js';
+
+const token = 'synthetic-production-gateway-token-32-bytes';
+const dirs: string[] = [];
+afterEach(() =>
+  Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))),
+);
+
+function environment(ack?: string) {
+  return ack === undefined
+    ? { MULTI_SIMADMIN_GATEWAY_TOKEN: token }
+    : { MULTI_SIMADMIN_GATEWAY_TOKEN: token, MULTI_SIMADMIN_CUTOVER_ACK: ack };
+}
+
+async function fixture(name = 'cutover'): Promise<CutoverPlan> {
+  const root = await mkdtemp(join(tmpdir(), `${name}-`));
+  dirs.push(root);
+  const assets = join(root, 'assets');
+  await mkdir(assets);
+  await writeFile(join(assets, 'index.html'), 'asset');
+  const backup = join(root, 'backup');
+  const drill = join(root, 'drill');
+  const backupContent = 'verified backup artifact\n';
+  const drillContent = 'successful rollback drill\n';
+  await writeFile(backup, backupContent);
+  await writeFile(drill, drillContent);
+  return {
+    assetDir: assets,
+    backupEvidence: backup,
+    backupEvidenceSha256: createHash('sha256').update(backupContent).digest('hex'),
+    drillEvidence: drill,
+    drillEvidenceSha256: createHash('sha256').update(drillContent).digest('hex'),
+    legacyPid: 41,
+    legacyCommand: '/usr/bin/true legacy',
+    legacyIdentity: 'legacy-start-1',
+    legacyStart: ['/usr/bin/true', 'legacy'],
+    stateDir: join(root, 'state'),
+  };
+}
+
+interface FakeOptions {
+  gatewayOwnsPort?: boolean;
+  gatewayReady?: boolean;
+}
+
+function fakeSystem(plan: CutoverPlan, options: FakeOptions = {}) {
+  const commands = new Map<number, string>([[plan.legacyPid, plan.legacyCommand]]);
+  const identities = new Map<number, string>([[plan.legacyPid, plan.legacyIdentity]]);
+  const signals: number[] = [];
+  const starts: Array<{ argv: string[]; environment?: NodeJS.ProcessEnv }> = [];
+  let nextPid = 42;
+  const system: CutoverSystem = {
+    ready: async (port, suppliedToken) =>
+      suppliedToken === token && (port === 8790 || options.gatewayReady !== false),
+    command: async (pid) => commands.get(pid) ?? '',
+    identity: async (pid) => identities.get(pid) ?? '',
+    ownsProductionPort: async (pid) => pid === 42 && options.gatewayOwnsPort !== false,
+    signal: (pid) => {
+      signals.push(pid);
+      commands.delete(pid);
+      identities.delete(pid);
+    },
+    waitGone: async () => {},
+    start: (argv, environment) => {
+      starts.push({ argv: [...argv], ...(environment === undefined ? {} : { environment }) });
+      const pid = nextPid++;
+      commands.set(pid, argv.join(' '));
+      identities.set(pid, `start-${pid}`);
+      return pid;
+    },
+    delay: async () => {},
+  };
+  return { system, signals, starts, commands, identities };
+}
+
+describe('production gateway ownership', () => {
+  it('requires the exact strong acknowledgement and ignores a supplied canary port', () => {
+    expect(() => readProductionGatewayRuntimeOptions(environment())).toThrow(/CUTOVER_ACK/);
+    expect(() => readProductionGatewayRuntimeOptions(environment('yes'))).toThrow(/CUTOVER_ACK/);
+    const options = readProductionGatewayRuntimeOptions({
+      ...environment(PRODUCTION_CUTOVER_ACK),
+      CANARY_PORT: '1234',
+    });
+    expect(options).toMatchObject({ port: 8788, ownsProductionPort: true });
+  });
+
+  it('keeps ordinary canary construction unable to claim 8788', () => {
+    expect(() => createCanaryGateway({ distDir: '/tmp', port: 8788 })).toThrow(/reserved/);
+  });
+});
+
+describe('cutover validation and reversible orchestration', () => {
+  it('rejects extra untrusted JSON fields and non-absolute executables', async () => {
+    const plan = await fixture('cutover-schema');
+    expect(() => parseCutoverPlan({ ...plan, command: 'sh -c evil' })).toThrow(/schema/);
+    expect(() => parseCutoverPlan({ ...plan, legacyStart: ['sh', '-c', 'evil'] })).toThrow(
+      /absolute/,
+    );
+  });
+
+  it('authenticates readiness and verifies evidence digests and regular assets', async () => {
+    const plan = await fixture('cutover-preflight');
+    const { system } = fakeSystem(plan);
+    await expect(preflight(plan, system, '')).rejects.toThrow(/Gateway token/);
+    await writeFile(plan.backupEvidence, 'tampered');
+    await expect(preflight(plan, system, token)).rejects.toThrow(/SHA-256 mismatch/);
+    await rm(plan.backupEvidence);
+    await mkdir(plan.backupEvidence);
+    await expect(preflight(plan, system, token)).rejects.toThrow(/regular file/);
+  });
+
+  it('durably records mode-0600 state, verifies 8788 ownership, and restores exact argv', async () => {
+    const plan = await fixture();
+    const { system, signals, starts } = fakeSystem(plan);
+    expect(await cutover(plan, system, token)).toBe(42);
+    expect(signals).toEqual([41]);
+    expect(starts[0]?.environment?.MULTI_SIMADMIN_GATEWAY_TOKEN).toBe(token);
+    const statePath = join(plan.stateDir, 'cutover-state.json');
+    expect((await lstat(statePath)).mode & 0o777).toBe(0o600);
+    expect(await readFile(statePath, 'utf8')).not.toContain(token);
+    expect(await rollback(plan, system)).toBe(43);
+    expect(signals).toEqual([41, 42]);
+    expect(starts[1]?.argv).toEqual(['/usr/bin/true', 'legacy']);
+  });
+
+  it('automatically restarts legacy if the gateway fails to own authenticated 8788', async () => {
+    const plan = await fixture('cutover-failure');
+    const { system, signals, starts } = fakeSystem(plan, { gatewayOwnsPort: false });
+    await expect(cutover(plan, system, token)).rejects.toThrow(/ownership verification/);
+    expect(signals).toEqual([41, 42]);
+    expect(starts.map((start) => start.argv)).toEqual([
+      [
+        resolve(process.cwd(), 'apps/api/node_modules/.bin/tsx'),
+        resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'),
+      ],
+      ['/usr/bin/true', 'legacy'],
+    ]);
+    const state = JSON.parse(await readFile(join(plan.stateDir, 'cutover-state.json'), 'utf8')) as {
+      phase: string;
+    };
+    expect(state.phase).toBe('prepared');
+  });
+
+  it('never signals mismatched legacy or gateway PID identities', async () => {
+    const plan = await fixture('cutover-mismatch');
+    const first = fakeSystem(plan);
+    first.commands.set(plan.legacyPid, 'foreign');
+    await expect(cutover(plan, first.system, token)).rejects.toThrow(/identity mismatch/);
+    expect(first.signals).toEqual([]);
+
+    const second = fakeSystem(plan);
+    await cutover(plan, second.system, token);
+    second.commands.set(42, 'foreign');
+    await expect(rollback(plan, second.system)).rejects.toThrow(/identity mismatch/);
+    expect(second.signals).toEqual([41]);
+  });
+
+  it('refuses rollback state with permissive permissions or invalid schema', async () => {
+    const plan = await fixture('cutover-state-validation');
+    const first = fakeSystem(plan);
+    await cutover(plan, first.system, token);
+    const statePath = join(plan.stateDir, 'cutover-state.json');
+    await chmod(statePath, 0o644);
+    await expect(rollback(plan, first.system)).rejects.toThrow(/mode-0600/);
+    expect(first.signals).toEqual([41]);
+
+    await chmod(statePath, 0o600);
+    const state = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, unknown>;
+    await writeFile(
+      statePath,
+      JSON.stringify({ ...state, injectedCommand: ['sh', '-c', 'evil'] }),
+      {
+        mode: 0o600,
+      },
+    );
+    await expect(rollback(plan, first.system)).rejects.toThrow(/schema/);
+    expect(first.signals).toEqual([41]);
+  });
+});

apps/api/src/production-readiness.release.test.ts

@@ -0,0 +1,67 @@
+import Database from 'better-sqlite3';
+import { describe, expect, it } from 'vitest';
+import { migrateDatabase } from './infrastructure/database/migrations.js';
+import { createProductionReadiness } from './production-readiness.js';
+
+function fixture() {
+  const db = new Database(':memory:');
+  migrateDatabase(db);
+  return db;
+}
+
+describe('production readiness release gates', () => {
+  it('fails closed on pending credentials, recovery markers, missing assets and configured gateway failure', async () => {
+    const db = fixture();
+    db.prepare(
+      "INSERT INTO app_settings(key,value_json,created_at,updated_at) VALUES('legacyImport.pendingCredentials','2','x','x')",
+    ).run();
+    const readiness = createProductionReadiness({
+      db,
+      gatewayToken: 'x'.repeat(32),
+      keychainMetadataCheck: async () => true,
+      assetsCheck: async () => false,
+      recoveryMarkersCheck: async () => 1,
+      upstreamGatewayCheck: async () => false,
+    });
+    await expect(readiness()).resolves.toEqual({
+      ready: false,
+      checks: {
+        database: 'ready',
+        migrations: 'ready',
+        gatewayAuth: 'ready',
+        keychain: 'ready',
+        pendingCredentials: 'pending:2',
+        recoveryMarkers: 'present:1',
+        assets: 'unavailable',
+        upstreamGateway: 'unavailable',
+      },
+    });
+    db.close();
+  });
+
+  it('requires explicit release checks and reports ready only when every gate passes', async () => {
+    const db = fixture();
+    const missing = await createProductionReadiness({
+      db,
+      gatewayToken: 'x'.repeat(32),
+      keychainMetadataCheck: async () => true,
+    })();
+    expect(missing.ready).toBe(false);
+    expect(missing.checks).toMatchObject({
+      assets: 'unconfigured',
+      recoveryMarkers: 'unconfigured',
+      upstreamGateway: 'unconfigured',
+      pendingCredentials: 'ready',
+    });
+    const ready = await createProductionReadiness({
+      db,
+      gatewayToken: 'x'.repeat(32),
+      keychainMetadataCheck: async () => true,
+      assetsCheck: async () => true,
+      recoveryMarkersCheck: async () => 0,
+      upstreamGatewayCheck: async () => true,
+    })();
+    expect(ready.ready).toBe(true);
+    db.close();
+  });
+});

apps/api/src/production-readiness.ts

@@ -8,6 +8,10 @@ export interface ProductionReadinessOptions {
   readonly gatewayToken: string;
   /** Metadata/availability probe only. This callback must never retrieve a secret. */
   readonly keychainMetadataCheck: () => boolean | Promise<boolean>;
+  /** Release gates are deliberately required: omitting one fails readiness closed. */
+  readonly assetsCheck?: () => boolean | Promise<boolean>;
+  readonly recoveryMarkersCheck?: () => number | Promise<number>;
+  readonly upstreamGatewayCheck?: () => boolean | Promise<boolean>;
 }
 interface ExecutableMetadata {
   isFile(): boolean;
@@ -57,6 +61,57 @@ export function createProductionReadiness(
       checks.keychain = 'unavailable';
     }
     if (checks.keychain !== 'ready') ready = false;
+
+    try {
+      const row = options.db
+        .prepare(
+          "SELECT value_json FROM app_settings WHERE key = 'legacyImport.pendingCredentials'",
+        )
+        .get() as { value_json: string } | undefined;
+      const value: unknown = row === undefined ? 0 : JSON.parse(row.value_json);
+      checks.pendingCredentials =
+        Number.isSafeInteger(value) && (value as number) >= 0
+          ? value === 0
+            ? 'ready'
+            : `pending:${String(value)}`
+          : 'unavailable';
+    } catch {
+      checks.pendingCredentials = 'unavailable';
+    }
+    if (checks.pendingCredentials !== 'ready') ready = false;
+
+    if (options.recoveryMarkersCheck === undefined) {
+      checks.recoveryMarkers = 'unconfigured';
+    } else {
+      try {
+        const count = await options.recoveryMarkersCheck();
+        checks.recoveryMarkers =
+          Number.isSafeInteger(count) && count >= 0
+            ? count === 0
+              ? 'ready'
+              : `present:${String(count)}`
+            : 'unavailable';
+      } catch {
+        checks.recoveryMarkers = 'unavailable';
+      }
+    }
+    if (checks.recoveryMarkers !== 'ready') ready = false;
+
+    for (const [name, check] of [
+      ['assets', options.assetsCheck],
+      ['upstreamGateway', options.upstreamGatewayCheck],
+    ] as const) {
+      if (check === undefined) {
+        checks[name] = 'unconfigured';
+      } else {
+        try {
+          checks[name] = (await check()) ? 'ready' : 'unavailable';
+        } catch {
+          checks[name] = 'unavailable';
+        }
+      }
+      if (checks[name] !== 'ready') ready = false;
+    }
     return { ready, checks };
   };
 }

apps/api/src/release-evidence.test.ts

@@ -0,0 +1,128 @@
+import Database from 'better-sqlite3';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { migrateDatabase } from './infrastructure/database/migrations.js';
+import {
+  appendReleaseEvidence,
+  compareShadowSnapshot,
+  sha256File,
+  verifyReleaseEvidenceLog,
+} from './release-evidence.js';
+
+const cleanup: Array<() => Promise<void>> = [];
+afterEach(async () => Promise.all(cleanup.splice(0).map((fn) => fn())));
+
+async function directory(): Promise<string> {
+  const path = await mkdtemp(join(tmpdir(), 'release-evidence-'));
+  cleanup.push(() => rm(path, { recursive: true, force: true }));
+  return path;
+}
+
+describe('durable redacted release evidence', () => {
+  it('atomically appends hash-chained, digest-bound evidence without origins or secrets', async () => {
+    const root = await directory();
+    const log = join(root, 'evidence.jsonl');
+    const record = await appendReleaseEvidence(log, {
+      releaseSha: 'a'.repeat(40),
+      webAssetDigest: 'b'.repeat(64),
+      databaseBackupDigest: 'c'.repeat(64),
+      rollbackDrill: {
+        performedAt: '2026-07-18T00:00:00.000Z',
+        result: 'passed',
+        recordDigest: 'd'.repeat(64),
+      },
+      pendingCredentialCount: 0,
+      readiness: { ready: true, checks: { assets: 'ready' } },
+      shadow: {
+        inputDigest: 'e'.repeat(64),
+        matches: true,
+        legacy: { instances: { total: 2, enabled: 2 }, statuses: { fresh: 2 } },
+        current: { instances: { total: 2, enabled: 2 }, statuses: { fresh: 2 } },
+        differences: [],
+      },
+      recordedAt: '2026-07-18T01:00:00.000Z',
+    });
+    expect(record.sequence).toBe(1);
+    expect(record.recordDigest).toMatch(/^[a-f0-9]{64}$/);
+    expect(await verifyReleaseEvidenceLog(log)).toEqual({
+      valid: true,
+      records: 1,
+      lastDigest: record.recordDigest,
+    });
+    const persisted = await readFile(log, 'utf8');
+    expect(persisted).not.toContain('https://');
+    expect(persisted).not.toContain('token');
+
+    await expect(
+      appendReleaseEvidence(log, {
+        releaseSha: 'a'.repeat(40),
+        webAssetDigest: 'b'.repeat(64),
+        databaseBackupDigest: 'c'.repeat(64),
+        rollbackDrill: {
+          performedAt: '2026-07-18T00:00:00.000Z',
+          result: 'passed',
+          recordDigest: 'd'.repeat(64),
+        },
+        pendingCredentialCount: 0,
+        readiness: { ready: true, checks: { upstream: 'https://secret.example' } },
+        shadow: {
+          inputDigest: 'e'.repeat(64),
+          matches: true,
+          legacy: { instances: { total: 0, enabled: 0 }, statuses: {} },
+          current: { instances: { total: 0, enabled: 0 }, statuses: {} },
+          differences: [],
+        },
+      }),
+    ).rejects.toThrow(/unsafe evidence/i);
+  });
+
+  it('detects evidence tampering', async () => {
+    const root = await directory();
+    const log = join(root, 'evidence.jsonl');
+    await writeFile(log, '{"sequence":1,"recordDigest":"bad"}\n');
+    expect((await verifyReleaseEvidenceLog(log)).valid).toBe(false);
+  });
+});
+
+describe('read-only digest-bound shadow reconciliation', () => {
+  it('compares only safe counts/status projections and does not mutate the DB', async () => {
+    const root = await directory();
+    const snapshot = join(root, 'legacy.json');
+    await writeFile(
+      snapshot,
+      JSON.stringify({
+        instances: [
+          { name: 'one', url: 'https://secret.invalid', enabled: true, status: 'fresh' },
+          { name: 'two', password: 'never-log', enabled: false, status: 'stale' },
+        ],
+      }),
+    );
+    const db = new Database(':memory:');
+    migrateDatabase(db);
+    const now = '2026-07-18T00:00:00.000Z';
+    db.prepare(
+      'INSERT INTO instances(id,name,base_url,enabled,created_at,updated_at) VALUES(?,?,?,?,?,?)',
+    ).run('1', 'one', 'https://new.invalid', 1, now, now);
+    db.prepare(
+      'INSERT INTO instances(id,name,base_url,enabled,created_at,updated_at) VALUES(?,?,?,?,?,?)',
+    ).run('2', 'two', 'https://new2.invalid', 0, now, now);
+    db.prepare(
+      'INSERT INTO status_snapshots(id,instance_id,category,state,payload_json,observed_at,created_at) VALUES(?,?,?,?,?,?,?)',
+    ).run('s1', '1', 'health', 'fresh', '{}', now, now);
+    db.prepare(
+      'INSERT INTO status_snapshots(id,instance_id,category,state,payload_json,observed_at,created_at) VALUES(?,?,?,?,?,?,?)',
+    ).run('s2', '2', 'health', 'stale', '{}', now, now);
+    const before = db.serialize();
+    const result = await compareShadowSnapshot(snapshot, db);
+    expect(result).toMatchObject({
+      inputDigest: await sha256File(snapshot),
+      matches: true,
+      legacy: { instances: { total: 2, enabled: 1 }, statuses: { fresh: 1, stale: 1 } },
+    });
+    expect(JSON.stringify(result)).not.toContain('invalid');
+    expect(db.serialize()).toEqual(before);
+    db.close();
+  });
+});

apps/api/src/release-evidence.ts

@@ -0,0 +1,328 @@
+import type Database from 'better-sqlite3';
+import { createHash } from 'node:crypto';
+import { open, readFile, rename, rm, stat } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+
+const DIGEST = /^[a-f0-9]{64}$/u;
+const SHA = /^[a-f0-9]{40}$/u;
+const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/u;
+const SAFE_VALUE = /^[a-zA-Z0-9 _.:+-]{1,128}$/u;
+
+export interface SafeProjection {
+  readonly instances: { readonly total: number; readonly enabled: number };
+  readonly statuses: Readonly<Record<string, number>>;
+}
+export interface ShadowComparison {
+  readonly inputDigest: string;
+  readonly matches: boolean;
+  readonly legacy: SafeProjection;
+  readonly current: SafeProjection;
+  readonly differences: readonly string[];
+}
+export interface ReleaseEvidenceInput {
+  readonly releaseSha: string;
+  readonly webAssetDigest: string;
+  readonly databaseBackupDigest: string;
+  readonly rollbackDrill: {
+    readonly performedAt: string;
+    readonly result: 'passed';
+    readonly recordDigest: string;
+  };
+  readonly pendingCredentialCount: number;
+  readonly readiness: {
+    readonly ready: boolean;
+    readonly checks: Readonly<Record<string, string>>;
+  };
+  readonly shadow: ShadowComparison;
+  readonly recordedAt?: string;
+}
+export interface ReleaseEvidenceRecord extends ReleaseEvidenceInput {
+  readonly sequence: number;
+  readonly previousDigest: string | null;
+  readonly recordedAt: string;
+  readonly recordDigest: string;
+}
+
+function canonical(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  if (value !== null && typeof value === 'object') {
+    const record = value as Record<string, unknown>;
+    return `{${Object.keys(record)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`)
+      .join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+function digest(value: unknown): string {
+  return createHash('sha256').update(canonical(value), 'utf8').digest('hex');
+}
+function safeCount(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) >= 0;
+}
+function exactKeys(value: Record<string, unknown>, expected: readonly string[]): void {
+  const actual = Object.keys(value).sort();
+  const wanted = [...expected].sort();
+  if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index]))
+    throw new Error('Unsafe evidence schema');
+}
+function validateProjection(value: SafeProjection): void {
+  exactKeys(value as unknown as Record<string, unknown>, ['instances', 'statuses']);
+  exactKeys(value.instances as unknown as Record<string, unknown>, ['enabled', 'total']);
+  if (!safeCount(value.instances.total) || !safeCount(value.instances.enabled))
+    throw new Error('Unsafe evidence projection count');
+  if (value.instances.enabled > value.instances.total)
+    throw new Error('Unsafe evidence projection');
+  for (const [key, count] of Object.entries(value.statuses)) {
+    if (!/^[a-z0-9_-]{1,64}$/u.test(key) || !safeCount(count))
+      throw new Error('Unsafe evidence status projection');
+  }
+}
+function validateInput(input: ReleaseEvidenceInput): void {
+  exactKeys(input as unknown as Record<string, unknown>, [
+    'databaseBackupDigest',
+    'pendingCredentialCount',
+    'readiness',
+    'releaseSha',
+    'rollbackDrill',
+    'shadow',
+    'webAssetDigest',
+    ...(input.recordedAt === undefined ? [] : ['recordedAt']),
+  ]);
+  exactKeys(input.rollbackDrill as unknown as Record<string, unknown>, [
+    'performedAt',
+    'recordDigest',
+    'result',
+  ]);
+  exactKeys(input.readiness as unknown as Record<string, unknown>, ['checks', 'ready']);
+  exactKeys(input.shadow as unknown as Record<string, unknown>, [
+    'current',
+    'differences',
+    'inputDigest',
+    'legacy',
+    'matches',
+  ]);
+  if (
+    !SHA.test(input.releaseSha) ||
+    !DIGEST.test(input.webAssetDigest) ||
+    !DIGEST.test(input.databaseBackupDigest)
+  )
+    throw new Error('Unsafe evidence digest');
+  if (
+    !ISO.test(input.rollbackDrill.performedAt) ||
+    input.rollbackDrill.result !== 'passed' ||
+    !DIGEST.test(input.rollbackDrill.recordDigest)
+  )
+    throw new Error('Unsafe evidence rollback drill');
+  if (!safeCount(input.pendingCredentialCount)) throw new Error('Unsafe evidence credential count');
+  for (const [key, value] of Object.entries(input.readiness.checks)) {
+    if (!/^[a-zA-Z][a-zA-Z0-9]{0,63}$/u.test(key) || !SAFE_VALUE.test(value))
+      throw new Error('Unsafe evidence readiness value');
+  }
+  if (!DIGEST.test(input.shadow.inputDigest)) throw new Error('Unsafe evidence shadow digest');
+  validateProjection(input.shadow.legacy);
+  validateProjection(input.shadow.current);
+  if (input.shadow.differences.some((value) => !SAFE_VALUE.test(value)))
+    throw new Error('Unsafe evidence difference');
+  if (input.recordedAt !== undefined && !ISO.test(input.recordedAt))
+    throw new Error('Unsafe evidence timestamp');
+}
+
+export async function sha256File(path: string): Promise<string> {
+  const metadata = await stat(path);
+  if (!metadata.isFile()) throw new Error('Digest input must be a regular file');
+  return createHash('sha256')
+    .update(await readFile(path))
+    .digest('hex');
+}
+
+function parseRecords(text: string): ReleaseEvidenceRecord[] {
+  if (text === '') return [];
+  const lines = text.split('\n');
+  if (lines.at(-1) !== '') throw new Error('Evidence log is not newline terminated');
+  lines.pop();
+  return lines.map((line) => JSON.parse(line) as ReleaseEvidenceRecord);
+}
+function validateRecord(
+  record: ReleaseEvidenceRecord,
+  index: number,
+  previous: string | null,
+): boolean {
+  const { recordDigest, ...unsigned } = record;
+  try {
+    exactKeys(record as unknown as Record<string, unknown>, [
+      'databaseBackupDigest',
+      'pendingCredentialCount',
+      'previousDigest',
+      'readiness',
+      'recordDigest',
+      'recordedAt',
+      'releaseSha',
+      'rollbackDrill',
+      'sequence',
+      'shadow',
+      'webAssetDigest',
+    ]);
+    const evidenceInput = {
+      releaseSha: record.releaseSha,
+      webAssetDigest: record.webAssetDigest,
+      databaseBackupDigest: record.databaseBackupDigest,
+      rollbackDrill: record.rollbackDrill,
+      pendingCredentialCount: record.pendingCredentialCount,
+      readiness: record.readiness,
+      shadow: record.shadow,
+      recordedAt: record.recordedAt,
+    };
+    validateInput(evidenceInput);
+  } catch {
+    return false;
+  }
+  return (
+    record.sequence === index + 1 &&
+    record.previousDigest === previous &&
+    DIGEST.test(recordDigest) &&
+    digest(unsigned) === recordDigest
+  );
+}
+export async function verifyReleaseEvidenceLog(path: string): Promise<{
+  readonly valid: boolean;
+  readonly records: number;
+  readonly lastDigest: string | null;
+}> {
+  try {
+    const records = parseRecords(await readFile(path, 'utf8'));
+    let previous: string | null = null;
+    for (const [index, record] of records.entries()) {
+      if (!validateRecord(record, index, previous))
+        return { valid: false, records: index, lastDigest: previous };
+      previous = record.recordDigest;
+    }
+    return { valid: true, records: records.length, lastDigest: previous };
+  } catch {
+    return { valid: false, records: 0, lastDigest: null };
+  }
+}
+
+export async function appendReleaseEvidence(
+  path: string,
+  input: ReleaseEvidenceInput,
+): Promise<ReleaseEvidenceRecord> {
+  validateInput(input);
+  const absolute = resolve(path);
+  const parent = dirname(absolute);
+  const parentInfo = await stat(parent);
+  if (!parentInfo.isDirectory() || parentInfo.isSymbolicLink())
+    throw new Error('Unsafe evidence directory');
+  const lockPath = `${absolute}.lock`;
+  let lock;
+  try {
+    lock = await open(lockPath, 'wx', 0o600);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'EEXIST')
+      throw new Error('Release evidence log is locked by another writer');
+    throw error;
+  }
+  try {
+    let existing = '';
+    try {
+      const info = await stat(absolute);
+      if (!info.isFile() || info.isSymbolicLink() || (info.mode & 0o077) !== 0)
+        throw new Error('Unsafe evidence log');
+      existing = await readFile(absolute, 'utf8');
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+    }
+    const records = parseRecords(existing);
+    let previous: string | null = null;
+    for (const [index, record] of records.entries()) {
+      if (!validateRecord(record, index, previous)) throw new Error('Unsafe evidence log chain');
+      previous = record.recordDigest;
+    }
+    const unsigned = {
+      ...input,
+      recordedAt: input.recordedAt ?? new Date().toISOString(),
+      sequence: records.length + 1,
+      previousDigest: previous,
+    };
+    const record: ReleaseEvidenceRecord = { ...unsigned, recordDigest: digest(unsigned) };
+    const temporary = `${absolute}.${process.pid}.${Date.now()}.tmp`;
+    const handle = await open(temporary, 'wx', 0o600);
+    try {
+      await handle.writeFile(`${existing}${canonical(record)}\n`, 'utf8');
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await rename(temporary, absolute);
+    const directory = await open(parent, 'r');
+    try {
+      await directory.sync();
+    } finally {
+      await directory.close();
+    }
+    return Object.freeze(record);
+  } finally {
+    await lock.close();
+    await rm(lockPath, { force: true });
+  }
+}
+
+function projectionFromLegacy(value: unknown): SafeProjection {
+  if (value === null || typeof value !== 'object' || Array.isArray(value))
+    throw new Error('Invalid shadow snapshot');
+  const instances = (value as { instances?: unknown }).instances;
+  if (!Array.isArray(instances)) throw new Error('Invalid shadow snapshot');
+  const statuses: Record<string, number> = {};
+  let enabled = 0;
+  for (const item of instances) {
+    if (item === null || typeof item !== 'object' || Array.isArray(item))
+      throw new Error('Invalid shadow snapshot');
+    const record = item as Record<string, unknown>;
+    if (
+      typeof record.enabled !== 'boolean' ||
+      typeof record.status !== 'string' ||
+      !/^[a-z0-9_-]{1,64}$/u.test(record.status)
+    )
+      throw new Error('Invalid shadow snapshot');
+    if (record.enabled) enabled += 1;
+    statuses[record.status] = (statuses[record.status] ?? 0) + 1;
+  }
+  return { instances: { total: instances.length, enabled }, statuses };
+}
+function projectionFromDatabase(db: Database.Database): SafeProjection {
+  const instances = db
+    .prepare('SELECT COUNT(*) AS total, COALESCE(SUM(enabled), 0) AS enabled FROM instances')
+    .get() as { total: number; enabled: number };
+  const rows = db
+    .prepare('SELECT state, COUNT(*) AS count FROM status_snapshots GROUP BY state')
+    .all() as Array<{ state: string; count: number }>;
+  const statuses: Record<string, number> = {};
+  for (const row of rows) {
+    if (!/^[a-z0-9_-]{1,64}$/u.test(row.state) || !safeCount(row.count))
+      throw new Error('Unsafe database status');
+    statuses[row.state] = row.count;
+  }
+  return { instances: { total: instances.total, enabled: instances.enabled }, statuses };
+}
+export async function compareShadowSnapshot(
+  path: string,
+  db: Database.Database,
+): Promise<ShadowComparison> {
+  const inputDigest = await sha256File(path);
+  const legacy = projectionFromLegacy(JSON.parse(await readFile(path, 'utf8')) as unknown);
+  const current = projectionFromDatabase(db);
+  const differences: string[] = [];
+  if (legacy.instances.total !== current.instances.total)
+    differences.push('instance total differs');
+  if (legacy.instances.enabled !== current.instances.enabled)
+    differences.push('enabled instance count differs');
+  if (canonical(legacy.statuses) !== canonical(current.statuses))
+    differences.push('status counts differ');
+  return Object.freeze({
+    inputDigest,
+    matches: differences.length === 0,
+    legacy,
+    current,
+    differences,
+  });
+}

apps/api/src/runtime-config.ts

@@ -8,12 +8,14 @@ export interface RuntimeEnvironment {
   readonly MULTI_SIMADMIN_GATEWAY_TOKEN?: string;
   readonly API_HOST?: string;
   readonly API_PORT?: string;
+  readonly MULTI_SIMADMIN_WEB_DIST?: string;
 }
 export interface RuntimeConfig {
   readonly api: { readonly host: '127.0.0.1'; readonly port: 8790 };
   readonly dataRoot: string;
   readonly databasePath: string;
   readonly gatewayToken: string;
+  readonly webDist: string;
 }
 function within(root: string, candidate: string): boolean {
   const child = relative(root, candidate);
@@ -62,10 +64,15 @@ export function createRuntimeConfig(environment: RuntimeEnvironment): RuntimeCon
     Buffer.byteLength(gatewayToken) > 512
   )
     throw new Error('Gateway token must contain at least 32 bytes');
+  const webDistInput = environment.MULTI_SIMADMIN_WEB_DIST;
+  if (!webDistInput || !isAbsolute(webDistInput))
+    throw new Error('An explicit absolute web distribution path is required');
+  const webDist = resolve(webDistInput);
   return Object.freeze({
     api: Object.freeze({ host: '127.0.0.1' as const, port: 8790 as const }),
     dataRoot,
     databasePath,
     gatewayToken,
+    webDist,
   });
 }

apps/api/src/runtime-foundation.test.ts

@@ -28,6 +28,7 @@ describe('production runtime foundation', () => {
       MULTI_SIMADMIN_DATA_ROOT: root,
       MULTI_SIMADMIN_DATABASE_PATH: join(root, 'db.sqlite'),
       MULTI_SIMADMIN_GATEWAY_TOKEN: token,
+      MULTI_SIMADMIN_WEB_DIST: root,
       API_HOST: '127.0.0.1',
       API_PORT: '8790',
     });
@@ -36,6 +37,7 @@ describe('production runtime foundation', () => {
       dataRoot: root,
       databasePath: join(root, 'db.sqlite'),
       gatewayToken: token,
+      webDist: root,
     });
     expect(Object.isFrozen(config)).toBe(true);
     expect(Object.isFrozen(config.api)).toBe(true);
@@ -44,6 +46,7 @@ describe('production runtime foundation', () => {
         MULTI_SIMADMIN_DATA_ROOT: root,
         MULTI_SIMADMIN_DATABASE_PATH: '/tmp/escape.sqlite',
         MULTI_SIMADMIN_GATEWAY_TOKEN: token,
+        MULTI_SIMADMIN_WEB_DIST: root,
       }),
     ).toThrow(/data root/i);
     expect(() =>
@@ -70,6 +73,7 @@ describe('production runtime foundation', () => {
         MULTI_SIMADMIN_DATA_ROOT: root,
         MULTI_SIMADMIN_DATABASE_PATH: databasePath,
         MULTI_SIMADMIN_GATEWAY_TOKEN: token,
+        MULTI_SIMADMIN_WEB_DIST: root,
       }),
     ).toThrow(/data root|symlink/i);
     await expect(lstat(join(outside, 'db.sqlite'))).rejects.toMatchObject({ code: 'ENOENT' });
@@ -150,10 +154,26 @@ describe('production runtime foundation', () => {
     const db = openDatabase(join(directory, 'db.sqlite'));
     migrateDatabase(db);
     const keychainMetadataCheck = vi.fn(async () => true);
-    const readiness = createProductionReadiness({ db, gatewayToken: token, keychainMetadataCheck });
+    const readiness = createProductionReadiness({
+      db,
+      gatewayToken: token,
+      keychainMetadataCheck,
+      assetsCheck: async () => true,
+      recoveryMarkersCheck: async () => 0,
+      upstreamGatewayCheck: async () => true,
+    });
     expect(await readiness()).toEqual({
       ready: true,
-      checks: { database: 'ready', migrations: 'ready', gatewayAuth: 'ready', keychain: 'ready' },
+      checks: {
+        database: 'ready',
+        migrations: 'ready',
+        gatewayAuth: 'ready',
+        keychain: 'ready',
+        pendingCredentials: 'ready',
+        assets: 'ready',
+        recoveryMarkers: 'ready',
+        upstreamGateway: 'ready',
+      },
     });
     expect(keychainMetadataCheck).toHaveBeenCalledOnce();
     db.close();

docs/production-gateway-cutover.md

@@ -0,0 +1,45 @@
+# Production gateway cutover and rollback
+
+The browser gateway normally runs as a canary on loopback port 8789. Port 8788 remains reserved: neither `pnpm canary` nor `CANARY_PORT=8788 pnpm canary` can claim it. The only owner is the separate `pnpm gateway:production` command, and it exits before listening unless:
+
+```text
+MULTI_SIMADMIN_CUTOVER_ACK=I ACKNOWLEDGE MULTI-SIMADMIN OWNS PORT 8788
+```
+
+Do not place that acknowledgement in a persistent dotenv file. Supply it only through the reviewed cutover operation.
+
+## Plan and evidence
+
+Create a mode-0600 plan outside the repository. It names evidence, not secrets:
+
+```json
+{
+  "assetDir": "/absolute/release/apps/web/dist",
+  "backupEvidence": "/absolute/evidence/backup-verified.txt",
+  "backupEvidenceSha256": "<lowercase SHA-256 of backup evidence>",
+  "drillEvidence": "/absolute/evidence/rollback-drill-passed.txt",
+  "drillEvidenceSha256": "<lowercase SHA-256 of drill evidence>",
+  "legacyPid": 1234,
+  "legacyCommand": "/usr/local/bin/node /absolute/legacy/server.js",
+  "legacyIdentity": "<process start identity from ps -o lstart=>",
+  "legacyStart": ["/usr/local/bin/node", "/absolute/legacy/server.js"],
+  "stateDir": "/absolute/operator-state"
+}
+```
+
+`legacyCommand` must exactly equal `ps -p PID -o command=` and `legacyStart` must reproduce that command exactly. The executable and all paths must be absolute. The evidence files must be regular files whose contents match the reviewed SHA-256 values. `assetDir/index.html` must be a regular file. The plan must not contain secrets or extra command fields.
+
+## Procedure
+
+1. Build/release assets and start the API on loopback 8790 by the established production procedure.
+2. Verify backup restoration and perform a rollback drill; write the two evidence files.
+3. Record the legacy listener PID and its exact command. Never reuse a stale plan after a restart.
+4. Export `MULTI_SIMADMIN_GATEWAY_TOKEN` only in the operator process environment. Run `corepack pnpm cutover preflight /absolute/plan.json`. This performs authenticated API `/readyz` on 8790, asset and evidence digest checks, and exact PID command ownership. The token is neither logged nor persisted.
+5. In the approved window run `corepack pnpm cutover cutover /absolute/plan.json` once. Before SIGTERM it durably fsyncs mode-0600 prepared state. It waits for the verified legacy PID to exit, starts the production gateway, verifies that exact PID owns 8788 and passes authenticated readiness, then durably records active state. If gateway verification fails, it automatically stops the attempted gateway and restarts the exact legacy argv.
+6. Verify `http://127.0.0.1:8788/healthz`, the UI, API reads, and an approved low-risk operation. Retain the state file.
+
+## Rollback
+
+Run `corepack pnpm cutover rollback /absolute/plan.json`. Rollback first validates the strict state schema, mode-0600 permissions, agreement with the reviewed plan, exact recorded PID command, and 8788 ownership. It refuses to signal any other process. It sends SIGTERM, waits, and restores the exact saved legacy argv without shell interpretation.
+
+If any identity check fails, stop and investigate manually. **Never kill the PID, edit the state to fit a process, use SIGKILL, or start a second listener.** The orchestrator deliberately does not access configuration or keychain material and does not modify the legacy server, public tree, or contracts.

package.json

@@ -8,6 +8,8 @@
   "scripts": {
     "start": "node server/index.js",
     "dev": "node --watch server/index.js",
+    "gateway:production": "corepack pnpm --filter @multi-simadmin/api run gateway:production",
+    "cutover": "tsx apps/api/src/cutover-cli.ts",
     "test": "node --test test/*.test.js packages/operation-registry/test/*.test.ts packages/test-fixtures/test/*.test.ts && vitest run",
     "test:legacy": "node --test test/*.test.js",
     "test:unit": "vitest run",