feat(release): add reversible production cutover gates

apps/api/src/canary-gateway.ts

@@ -18,6 +18,7 @@ export const CANARY_DEFAULT_UPSTREAM_HOST = '127.0.0.1';
 export const CANARY_DEFAULT_UPSTREAM_PORT = 8790;
 
 const RESERVED_PORTS = new Set([8788, 8790]);
+export const PRODUCTION_GATEWAY_PORT = 8788;
 const BASE_HOP_BY_HOP_HEADERS = new Set([
   'connection',
   'keep-alive',
@@ -62,6 +63,8 @@ export interface CanaryGatewayOptions {
   readonly port?: number;
   readonly upstreamHost?: string;
   readonly upstreamPort?: number;
+  /** Set only after the production runtime validates its explicit acknowledgement. */
+  readonly ownsProductionPort?: boolean;
 }
 
 export interface CanaryGateway {
@@ -201,7 +204,7 @@ export function createCanaryGateway(options: CanaryGatewayOptions): CanaryGatewa
     throw new Error('Canary upstream must use a loopback host');
   }
   if (!Number.isInteger(port) || port < 0 || port > 65_535) throw new Error('Invalid canary port');
-  if (RESERVED_PORTS.has(port))
+  if (RESERVED_PORTS.has(port) && !(port === PRODUCTION_GATEWAY_PORT && options.ownsProductionPort))
     throw new Error(`Port ${port} is reserved and cannot run the canary`);
   if (!options.distDir) throw new Error('distDir is required');
 

apps/api/src/canary-runtime.ts

@@ -10,8 +10,11 @@ export interface CanaryRuntimeEnvironment {
   readonly CANARY_PORT?: string;
   readonly CANARY_UPSTREAM_PORT?: string;
   readonly MULTI_SIMADMIN_GATEWAY_TOKEN?: string;
+  readonly MULTI_SIMADMIN_CUTOVER_ACK?: string;
 }
 
+export const PRODUCTION_CUTOVER_ACK = 'I ACKNOWLEDGE MULTI-SIMADMIN OWNS PORT 8788';
+
 function environmentPort(value: string | undefined, name: string, fallback: number): number {
   if (value === undefined || value === '') return fallback;
   const port = Number(value);
@@ -41,3 +44,15 @@ export function readCanaryRuntimeOptions(
     gatewayToken,
   });
 }
+
+export function readProductionGatewayRuntimeOptions(
+  environment: CanaryRuntimeEnvironment,
+): CanaryGatewayOptions {
+  if (environment.MULTI_SIMADMIN_CUTOVER_ACK !== PRODUCTION_CUTOVER_ACK) {
+    throw new Error(
+      `Production gateway requires MULTI_SIMADMIN_CUTOVER_ACK=${PRODUCTION_CUTOVER_ACK}`,
+    );
+  }
+  const options = readCanaryRuntimeOptions({ ...environment, CANARY_PORT: '8788' });
+  return Object.freeze({ ...options, ownsProductionPort: true });
+}

apps/api/src/cli-runtime.test.ts

@@ -18,7 +18,7 @@ interface RunningCommand {
 }
 
 function startPackageCommand(
-  script: 'canary' | 'start:production',
+  script: 'canary' | 'gateway:production' | 'start:production',
   environment: NodeJS.ProcessEnv,
 ) {
   const child = spawn('corepack', ['pnpm', '--filter', '@multi-simadmin/api', 'run', script], {
@@ -55,9 +55,13 @@ async function stopCommand(command: RunningCommand): Promise<void> {
   expect(result.code === 0 || result.code === 143 || result.signal === 'SIGTERM').toBe(true);
 }
 
-async function waitForOutput(command: RunningCommand, text: string): Promise<void> {
-  const deadline = Date.now() + 10_000;
-  while (!command.output().includes(text)) {
+async function waitForOutput(
+  command: RunningCommand,
+  pattern: string,
+  timeoutMs = 20_000,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (!command.output().includes(pattern)) {
     if (command.child.exitCode !== null || command.child.signalCode !== null) {
       throw new Error(`Runtime exited before becoming ready:\n${command.output()}`);
     }
@@ -103,6 +107,33 @@ afterEach(async () => {
 });
 
 describe.sequential('executable package runtimes', () => {
+  it('makes the exact production gateway executable attempt 8788 only with acknowledgement', async () => {
+    await expectPortFree(8788);
+    const fixture = await listenOn(8788);
+    cleanup.push(() => closeServer(fixture));
+    const directory = await mkdtemp(join(tmpdir(), 'multi-simadmin-production-gateway-cli-'));
+    cleanup.push(() => rm(directory, { recursive: true, force: true }));
+    await writeFile(join(directory, 'index.html'), '<main>production</main>');
+
+    const denied = startPackageCommand('gateway:production', {
+      CANARY_DIST_DIR: directory,
+      MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+    });
+    await waitForOutput(denied, 'Production gateway requires MULTI_SIMADMIN_CUTOVER_ACK=');
+    await new Promise<void>((resolveExit) => denied.child.once('exit', () => resolveExit()));
+    expect(denied.child.exitCode).not.toBe(0);
+
+    const acknowledged = startPackageCommand('gateway:production', {
+      CANARY_DIST_DIR: directory,
+      MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+      MULTI_SIMADMIN_CUTOVER_ACK: 'I ACKNOWLEDGE MULTI-SIMADMIN OWNS PORT 8788',
+    });
+    await waitForOutput(acknowledged, 'EADDRINUSE');
+    await new Promise<void>((resolveExit) => acknowledged.child.once('exit', () => resolveExit()));
+    expect(acknowledged.output()).toContain('127.0.0.1:8788');
+    expect(acknowledged.child.exitCode).not.toBe(0);
+  }, 20_000);
+
   it('starts the exact production command on 8790 and shuts down on SIGTERM', async () => {
     await expectPortFree(8790);
     const legacyListeners = await listenerPids(8788);
@@ -115,6 +146,7 @@ describe.sequential('executable package runtimes', () => {
       MULTI_SIMADMIN_DATA_ROOT: dataRoot,
       MULTI_SIMADMIN_DATABASE_PATH: join(dataRoot, 'synthetic.sqlite'),
       MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+      MULTI_SIMADMIN_WEB_DIST: directory,
       API_HOST: '127.0.0.1',
       API_PORT: '8790',
     });

apps/api/src/cutover-cli.ts

@@ -0,0 +1,23 @@
+import { readFile, stat } from 'node:fs/promises';
+import {
+  cutover,
+  nodeCutoverSystem,
+  parseCutoverPlan,
+  preflight,
+  rollback,
+} from './cutover-orchestrator.js';
+
+const [action, planPath] = process.argv.slice(2);
+if (!['preflight', 'cutover', 'rollback'].includes(action ?? '') || !planPath) {
+  throw new Error('Usage: pnpm cutover <preflight|cutover|rollback> <plan.json>');
+}
+const planInfo = await stat(planPath);
+if (!planInfo.isFile() || (planInfo.mode & 0o077) !== 0) {
+  throw new Error('Cutover plan must be a regular file inaccessible to group and other users');
+}
+const plan = parseCutoverPlan(JSON.parse(await readFile(planPath, 'utf8')) as unknown);
+const gatewayToken = process.env.MULTI_SIMADMIN_GATEWAY_TOKEN ?? '';
+if (action === 'preflight') await preflight(plan, nodeCutoverSystem, gatewayToken);
+else if (action === 'cutover') await cutover(plan, nodeCutoverSystem, gatewayToken);
+else await rollback(plan, nodeCutoverSystem);
+process.stdout.write(`${action} completed\n`);

apps/api/src/cutover-orchestrator.ts

@@ -0,0 +1,470 @@
+import { spawn, execFile } from 'node:child_process';
+import { constants } from 'node:fs';
+import {
+  access,
+  chmod,
+  lstat,
+  mkdir,
+  open,
+  readdir,
+  readFile,
+  readlink,
+  realpath,
+  rename,
+} from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { request } from 'node:http';
+import { isAbsolute, join, resolve } from 'node:path';
+import { GATEWAY_AUTH_HEADER } from './runtime-config.js';
+import { PRODUCTION_CUTOVER_ACK } from './canary-runtime.js';
+
+const STATE_VERSION = 1;
+const STATE_FILE = 'cutover-state.json';
+const GATEWAY_ARGV = [
+  resolve(process.cwd(), 'apps/api/node_modules/.bin/tsx'),
+  resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'),
+] as const;
+const DIGEST = /^[a-f0-9]{64}$/u;
+
+export interface CutoverPlan {
+  readonly assetDir: string;
+  readonly backupEvidence: string;
+  readonly backupEvidenceSha256: string;
+  readonly drillEvidence: string;
+  readonly drillEvidenceSha256: string;
+  readonly legacyPid: number;
+  readonly legacyCommand: string;
+  readonly legacyIdentity: string;
+  readonly legacyStart: readonly [string, ...string[]];
+  readonly stateDir: string;
+}
+
+interface CutoverState {
+  readonly version: 1;
+  readonly phase: 'prepared' | 'active';
+  readonly legacyPid: number;
+  readonly legacyCommand: string;
+  readonly legacyIdentity: string;
+  readonly legacyStart: readonly [string, ...string[]];
+  readonly gatewayPid?: number;
+  readonly gatewayCommand?: string;
+  readonly gatewayIdentity?: string;
+}
+
+export interface CutoverSystem {
+  command(pid: number): Promise<string>;
+  identity(pid: number): Promise<string>;
+  ready(port: 8788 | 8790, gatewayToken: string): Promise<boolean>;
+  ownsProductionPort(pid: number): Promise<boolean>;
+  signal(pid: number, signal: NodeJS.Signals): void;
+  waitGone(pid: number): Promise<void>;
+  start(argv: readonly [string, ...string[]], environment?: NodeJS.ProcessEnv): number;
+  delay(milliseconds: number): Promise<void>;
+}
+
+function runFile(file: string, args: readonly string[]): Promise<string> {
+  return new Promise((resolvePromise, reject) =>
+    execFile(file, [...args], (error, stdout) =>
+      error ? reject(error) : resolvePromise(stdout.trim()),
+    ),
+  );
+}
+
+function authenticatedReady(port: 8788 | 8790, gatewayToken: string): Promise<boolean> {
+  return new Promise((resolvePromise) => {
+    const req = request(
+      {
+        host: '127.0.0.1',
+        port,
+        path: '/readyz',
+        timeout: 2_000,
+        headers: { [GATEWAY_AUTH_HEADER]: gatewayToken },
+      },
+      (response) => {
+        response.resume();
+        resolvePromise(response.statusCode === 200);
+      },
+    );
+    req.once('error', () => resolvePromise(false));
+    req.once('timeout', () => {
+      req.destroy();
+      resolvePromise(false);
+    });
+    req.end();
+  });
+}
+
+export const nodeCutoverSystem: CutoverSystem = {
+  async command(pid) {
+    const proc = await readFile(`/proc/${pid}/cmdline`).catch(() => undefined);
+    if (proc) return proc.toString().split('\0').filter(Boolean).join(' ');
+    return await runFile('ps', ['-p', String(pid), '-o', 'command=']);
+  },
+  async identity(pid) {
+    if (process.platform === 'linux') {
+      const value = await readFile(`/proc/${pid}/stat`, 'utf8');
+      const fields = value.slice(value.lastIndexOf(')') + 2).split(/\s+/u);
+      return fields[19] ?? '';
+    }
+    return await runFile('ps', ['-p', String(pid), '-o', 'lstart=']);
+  },
+  ready: authenticatedReady,
+  async ownsProductionPort(pid) {
+    if (process.platform === 'linux') {
+      const descriptorDirectory = `/proc/${pid}/fd`;
+      const descriptors = await readdir(descriptorDirectory).catch(() => []);
+      const listeners = await Promise.all(
+        descriptors.map((descriptor) =>
+          readlink(join(descriptorDirectory, descriptor)).catch(() => ''),
+        ),
+      );
+      const socketInodes = new Set(
+        listeners
+          .map((line) => /^socket:\[(\d+)\]$/u.exec(line)?.[1])
+          .filter((inode): inode is string => inode !== undefined),
+      );
+      const tcp = await readFile('/proc/net/tcp', 'utf8').catch(() => '');
+      return tcp.split('\n').some((line) => {
+        const fields = line.trim().split(/\s+/u);
+        return (
+          fields[1]?.endsWith(':2254') && fields[3] === '0A' && socketInodes.has(fields[9] ?? '')
+        );
+      });
+    }
+    const output = await runFile('lsof', [
+      '-nP',
+      '-a',
+      '-p',
+      String(pid),
+      '-iTCP:8788',
+      '-sTCP:LISTEN',
+      '-t',
+    ]).catch(() => '');
+    return output.split(/\s+/u).includes(String(pid));
+  },
+  signal: (pid, signal) => process.kill(pid, signal),
+  async waitGone(pid) {
+    const deadline = Date.now() + 10_000;
+    while (Date.now() < deadline) {
+      try {
+        process.kill(pid, 0);
+      } catch {
+        return;
+      }
+      await new Promise((resolveWait) => setTimeout(resolveWait, 100));
+    }
+    throw new Error(`PID ${pid} did not exit after SIGTERM`);
+  },
+  start(argv, environment) {
+    const child = spawn(argv[0], argv.slice(1), {
+      detached: true,
+      stdio: 'ignore',
+      env: environment ?? process.env,
+    });
+    child.unref();
+    if (!child.pid) throw new Error('Failed to start command');
+    return child.pid;
+  },
+  delay: (milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)),
+};
+
+function record(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value))
+    throw new Error(`${label} must be an object`);
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  expected: readonly string[],
+  label: string,
+): void {
+  const actual = Object.keys(value).sort();
+  const wanted = [...expected].sort();
+  if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index]))
+    throw new Error(`${label} has an invalid schema`);
+}
+
+function safeString(value: unknown, label: string): string {
+  if (typeof value !== 'string' || value.length === 0 || value.includes('\0'))
+    throw new Error(`${label} must be a non-empty string`);
+  return value;
+}
+
+function argv(value: unknown, label: string): readonly [string, ...string[]] {
+  if (
+    !Array.isArray(value) ||
+    value.length === 0 ||
+    value.length > 64 ||
+    value.some((item) => typeof item !== 'string' || item.length === 0 || item.includes('\0'))
+  )
+    throw new Error(`${label} must be a non-empty string array`);
+  return value as [string, ...string[]];
+}
+
+export function parseCutoverPlan(value: unknown): CutoverPlan {
+  const input = record(value, 'Cutover plan');
+  exactKeys(
+    input,
+    [
+      'assetDir',
+      'backupEvidence',
+      'backupEvidenceSha256',
+      'drillEvidence',
+      'drillEvidenceSha256',
+      'legacyPid',
+      'legacyCommand',
+      'legacyIdentity',
+      'legacyStart',
+      'stateDir',
+    ],
+    'Cutover plan',
+  );
+  const plan: CutoverPlan = {
+    assetDir: safeString(input.assetDir, 'assetDir'),
+    backupEvidence: safeString(input.backupEvidence, 'backupEvidence'),
+    backupEvidenceSha256: safeString(input.backupEvidenceSha256, 'backupEvidenceSha256'),
+    drillEvidence: safeString(input.drillEvidence, 'drillEvidence'),
+    drillEvidenceSha256: safeString(input.drillEvidenceSha256, 'drillEvidenceSha256'),
+    legacyPid: input.legacyPid as number,
+    legacyCommand: safeString(input.legacyCommand, 'legacyCommand'),
+    legacyIdentity: safeString(input.legacyIdentity, 'legacyIdentity'),
+    legacyStart: argv(input.legacyStart, 'legacyStart'),
+    stateDir: safeString(input.stateDir, 'stateDir'),
+  };
+  if (
+    ![
+      plan.assetDir,
+      plan.backupEvidence,
+      plan.drillEvidence,
+      plan.stateDir,
+      plan.legacyStart[0],
+    ].every(isAbsolute)
+  )
+    throw new Error('Cutover paths and legacy executable must be absolute');
+  if (!Number.isSafeInteger(plan.legacyPid) || plan.legacyPid <= 1)
+    throw new Error('legacyPid must be a safe PID greater than 1');
+  if (!DIGEST.test(plan.backupEvidenceSha256) || !DIGEST.test(plan.drillEvidenceSha256))
+    throw new Error('Evidence digests must be lowercase SHA-256');
+  if (plan.legacyStart.join(' ') !== plan.legacyCommand)
+    throw new Error('legacyStart must exactly reproduce legacyCommand');
+  return Object.freeze(plan);
+}
+
+function requireGatewayToken(gatewayToken: string): void {
+  const bytes = Buffer.byteLength(gatewayToken);
+  if (bytes < 32 || bytes > 512)
+    throw new Error('Gateway token must contain between 32 and 512 bytes');
+}
+
+async function verifyEvidence(path: string, digest: string, label: string): Promise<void> {
+  const info = await lstat(path);
+  if (!info.isFile()) throw new Error(`${label} evidence must be a regular file`);
+  const content = await readFile(path);
+  const actual = createHash('sha256').update(content).digest('hex');
+  if (actual !== digest) throw new Error(`${label} evidence SHA-256 mismatch`);
+}
+
+async function durableState(plan: CutoverPlan, state: CutoverState): Promise<void> {
+  await mkdir(plan.stateDir, { recursive: true, mode: 0o700 });
+  await chmod(plan.stateDir, 0o700);
+  const target = join(plan.stateDir, STATE_FILE);
+  const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
+  const handle = await open(
+    temporary,
+    constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
+    0o600,
+  );
+  try {
+    await handle.writeFile(`${JSON.stringify(state)}\n`);
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+  await rename(temporary, target);
+  const directory = await open(plan.stateDir, constants.O_RDONLY);
+  try {
+    await directory.sync();
+  } finally {
+    await directory.close();
+  }
+}
+
+function preparedState(plan: CutoverPlan): CutoverState {
+  return {
+    version: STATE_VERSION,
+    phase: 'prepared',
+    legacyPid: plan.legacyPid,
+    legacyCommand: plan.legacyCommand,
+    legacyIdentity: plan.legacyIdentity,
+    legacyStart: plan.legacyStart,
+  };
+}
+
+async function readState(plan: CutoverPlan): Promise<CutoverState> {
+  const path = join(plan.stateDir, STATE_FILE);
+  const info = await lstat(path);
+  if (!info.isFile() || (info.mode & 0o777) !== 0o600)
+    throw new Error('Cutover state must be a mode-0600 regular file');
+  const input = record(JSON.parse(await readFile(path, 'utf8')) as unknown, 'Cutover state');
+  const base = ['version', 'phase', 'legacyPid', 'legacyCommand', 'legacyIdentity', 'legacyStart'];
+  exactKeys(
+    input,
+    input.phase === 'active' ? [...base, 'gatewayPid', 'gatewayCommand', 'gatewayIdentity'] : base,
+    'Cutover state',
+  );
+  const state: CutoverState = {
+    version: input.version as 1,
+    phase: input.phase as 'prepared' | 'active',
+    legacyPid: input.legacyPid as number,
+    legacyCommand: safeString(input.legacyCommand, 'state legacyCommand'),
+    legacyIdentity: safeString(input.legacyIdentity, 'state legacyIdentity'),
+    legacyStart: argv(input.legacyStart, 'state legacyStart'),
+    ...(input.phase === 'active'
+      ? {
+          gatewayPid: input.gatewayPid as number,
+          gatewayCommand: safeString(input.gatewayCommand, 'state gatewayCommand'),
+          gatewayIdentity: safeString(input.gatewayIdentity, 'state gatewayIdentity'),
+        }
+      : {}),
+  };
+  if (state.version !== STATE_VERSION || !['prepared', 'active'].includes(state.phase))
+    throw new Error('Unsupported cutover state');
+  if (
+    !Number.isSafeInteger(state.legacyPid) ||
+    state.legacyPid <= 1 ||
+    (state.gatewayPid !== undefined &&
+      (!Number.isSafeInteger(state.gatewayPid) || state.gatewayPid <= 1))
+  )
+    throw new Error('Cutover state contains an invalid PID');
+  if (
+    state.legacyPid !== plan.legacyPid ||
+    state.legacyCommand !== plan.legacyCommand ||
+    state.legacyIdentity !== plan.legacyIdentity ||
+    state.legacyStart.length !== plan.legacyStart.length ||
+    state.legacyStart.some((item, index) => item !== plan.legacyStart[index])
+  )
+    throw new Error('Cutover state does not match the reviewed plan');
+  return state;
+}
+
+export async function preflight(
+  plan: CutoverPlan,
+  system: CutoverSystem,
+  gatewayToken: string,
+): Promise<void> {
+  requireGatewayToken(gatewayToken);
+  if (!(await system.ready(8790, gatewayToken)))
+    throw new Error('Authenticated API readiness preflight failed on 127.0.0.1:8790/readyz');
+  const indexPath = join(await realpath(plan.assetDir), 'index.html');
+  const asset = await lstat(indexPath);
+  if (!asset.isFile()) throw new Error('assetDir/index.html must be a regular file');
+  await access(plan.legacyStart[0], constants.X_OK);
+  await verifyEvidence(plan.backupEvidence, plan.backupEvidenceSha256, 'Backup');
+  await verifyEvidence(plan.drillEvidence, plan.drillEvidenceSha256, 'Rollback drill');
+  const actual = await system.command(plan.legacyPid);
+  const identity = await system.identity(plan.legacyPid);
+  if (actual !== plan.legacyCommand || identity !== plan.legacyIdentity)
+    throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
+}
+
+async function restoreLegacy(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
+  return system.start(plan.legacyStart);
+}
+
+async function waitForGateway(
+  system: CutoverSystem,
+  gatewayPid: number,
+  gatewayToken: string,
+): Promise<boolean> {
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    if ((await system.ownsProductionPort(gatewayPid)) && (await system.ready(8788, gatewayToken)))
+      return true;
+    await system.delay(100);
+  }
+  return false;
+}
+
+export async function cutover(
+  plan: CutoverPlan,
+  system: CutoverSystem,
+  gatewayToken: string,
+): Promise<number> {
+  await preflight(plan, system, gatewayToken);
+  await durableState(plan, preparedState(plan));
+  let legacyStopped = false;
+  let gatewayPid: number | undefined;
+  let gatewayCommand: string | undefined;
+  let gatewayIdentity: string | undefined;
+  try {
+    if (
+      (await system.command(plan.legacyPid)) !== plan.legacyCommand ||
+      (await system.identity(plan.legacyPid)) !== plan.legacyIdentity
+    )
+      throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
+    system.signal(plan.legacyPid, 'SIGTERM');
+    await system.waitGone(plan.legacyPid);
+    legacyStopped = true;
+    gatewayPid = system.start(GATEWAY_ARGV, {
+      ...process.env,
+      CANARY_DIST_DIR: plan.assetDir,
+      MULTI_SIMADMIN_CUTOVER_ACK: PRODUCTION_CUTOVER_ACK,
+      MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
+    });
+    gatewayCommand = await system.command(gatewayPid);
+    gatewayIdentity = await system.identity(gatewayPid);
+    if (!(await waitForGateway(system, gatewayPid, gatewayToken)))
+      throw new Error(
+        'Production gateway failed authenticated ownership verification on port 8788',
+      );
+    await durableState(plan, {
+      ...preparedState(plan),
+      phase: 'active',
+      gatewayPid,
+      gatewayCommand,
+      gatewayIdentity,
+    });
+    return gatewayPid;
+  } catch (error) {
+    if (gatewayPid !== undefined) {
+      try {
+        const actual = await system.command(gatewayPid);
+        const identity = await system.identity(gatewayPid);
+        if (actual === gatewayCommand && identity === gatewayIdentity) {
+          system.signal(gatewayPid, 'SIGTERM');
+          await system.waitGone(gatewayPid);
+        }
+      } catch {
+        // Continue the safety-critical legacy restart even if gateway cleanup races its exit.
+      }
+    }
+    if (legacyStopped) {
+      if (gatewayPid !== undefined && (await system.ownsProductionPort(gatewayPid)))
+        throw new Error('Automatic rollback refused while production gateway still owns port 8788');
+      await restoreLegacy(plan, system);
+    }
+    throw error;
+  }
+}
+
+export async function rollback(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
+  const state = await readState(plan);
+  if (
+    state.phase !== 'active' ||
+    state.gatewayPid === undefined ||
+    state.gatewayCommand === undefined ||
+    state.gatewayIdentity === undefined
+  )
+    throw new Error('Cutover state is not active; refusing rollback PID operations');
+  const actual = await system.command(state.gatewayPid);
+  const identity = await system.identity(state.gatewayPid);
+  if (
+    actual !== state.gatewayCommand ||
+    identity !== state.gatewayIdentity ||
+    !(await system.ownsProductionPort(state.gatewayPid))
+  )
+    throw new Error(`Gateway PID identity mismatch; refusing to signal PID ${state.gatewayPid}`);
+  system.signal(state.gatewayPid, 'SIGTERM');
+  await system.waitGone(state.gatewayPid);
+  return restoreLegacy(plan, system);
+}

apps/api/src/production-cli.ts

@@ -1,12 +1,52 @@
+import { lstat, readdir } from 'node:fs/promises';
+import { basename, dirname, join } from 'node:path';
+
 import { buildProductionControlPlane } from './production-control-plane.js';
-import { createRuntimeConfig } from './runtime-config.js';
+import { createRuntimeConfig, GATEWAY_AUTH_HEADER } from './runtime-config.js';
 import { installBoundedShutdown } from './shutdown.js';
 
+async function regularFile(path: string): Promise<boolean> {
+  try {
+    const value = await lstat(path);
+    return value.isFile() && !value.isSymbolicLink();
+  } catch {
+    return false;
+  }
+}
+
+async function recoveryMarkers(databasePath: string): Promise<number> {
+  const directory = dirname(databasePath);
+  const name = basename(databasePath);
+  const names = await readdir(directory).catch(() => []);
+  return names.filter(
+    (entry) =>
+      entry === `${name}.restore-in-progress` ||
+      (entry.startsWith(`${name}.`) &&
+        (entry.endsWith('.restore') || entry.endsWith('.quarantine'))),
+  ).length;
+}
+
+async function gatewayStaticReachable(token: string): Promise<boolean> {
+  try {
+    const response = await fetch('http://127.0.0.1:8789/', {
+      headers: { [GATEWAY_AUTH_HEADER]: token },
+      signal: AbortSignal.timeout(2_000),
+    });
+    await response.body?.cancel();
+    return response.status === 200;
+  } catch {
+    return false;
+  }
+}
+
 try {
   const config = createRuntimeConfig(process.env);
   const app = buildProductionControlPlane({
     databasePath: config.databasePath,
     gatewayToken: config.gatewayToken,
+    assetsCheck: () => regularFile(join(config.webDist, 'index.html')),
+    recoveryMarkersCheck: () => recoveryMarkers(config.databasePath),
+    upstreamGatewayCheck: () => gatewayStaticReachable(config.gatewayToken),
   });
   await app.listen({ host: config.api.host, port: config.api.port });
   installBoundedShutdown(process, app, {

apps/api/src/production-control-plane.ts

@@ -17,6 +17,9 @@ export interface ProductionControlPlaneOptions {
   readonly upstream?: SafeControlPlaneUpstream;
   readonly gatewayToken?: string;
   readonly keychainMetadataCheck?: () => boolean | Promise<boolean>;
+  readonly assetsCheck?: () => boolean | Promise<boolean>;
+  readonly recoveryMarkersCheck?: () => number | Promise<number>;
+  readonly upstreamGatewayCheck?: () => boolean | Promise<boolean>;
 }
 
 export function buildProductionControlPlane(
@@ -48,6 +51,13 @@ export function buildProductionControlPlane(
           db,
           gatewayToken: options.gatewayToken ?? '',
           keychainMetadataCheck: options.keychainMetadataCheck ?? defaultKeychainMetadataCheck,
+          ...(options.assetsCheck ? { assetsCheck: options.assetsCheck } : {}),
+          ...(options.recoveryMarkersCheck
+            ? { recoveryMarkersCheck: options.recoveryMarkersCheck }
+            : {}),
+          ...(options.upstreamGatewayCheck
+            ? { upstreamGatewayCheck: options.upstreamGatewayCheck }
+            : {}),
         }),
       },
     });

apps/api/src/production-gateway-cli.ts

@@ -0,0 +1,21 @@
+import { createCanaryGateway } from './canary-gateway.js';
+import { readProductionGatewayRuntimeOptions } from './canary-runtime.js';
+
+const gateway = createCanaryGateway(readProductionGatewayRuntimeOptions(process.env));
+let stopping = false;
+async function stop(signal: string): Promise<void> {
+  if (stopping) return;
+  stopping = true;
+  process.stdout.write(`Stopping production gateway (${signal})\n`);
+  await gateway.stop();
+}
+process.once('SIGINT', () => void stop('SIGINT'));
+process.once('SIGTERM', () => void stop('SIGTERM'));
+
+try {
+  await gateway.start();
+  process.stdout.write(`Production gateway listening at ${gateway.origin}\n`);
+} catch (error) {
+  process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+  process.exitCode = 1;
+}

apps/api/src/production-gateway.test.ts

@@ -0,0 +1,193 @@
+import { createHash } from 'node:crypto';
+import { chmod, lstat, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { createCanaryGateway } from './canary-gateway.js';
+import { PRODUCTION_CUTOVER_ACK, readProductionGatewayRuntimeOptions } from './canary-runtime.js';
+import {
+  cutover,
+  parseCutoverPlan,
+  preflight,
+  rollback,
+  type CutoverPlan,
+  type CutoverSystem,
+} from './cutover-orchestrator.js';
+
+const token = 'synthetic-production-gateway-token-32-bytes';
+const dirs: string[] = [];
+afterEach(() =>
+  Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))),
+);
+
+function environment(ack?: string) {
+  return ack === undefined
+    ? { MULTI_SIMADMIN_GATEWAY_TOKEN: token }
+    : { MULTI_SIMADMIN_GATEWAY_TOKEN: token, MULTI_SIMADMIN_CUTOVER_ACK: ack };
+}
+
+async function fixture(name = 'cutover'): Promise<CutoverPlan> {
+  const root = await mkdtemp(join(tmpdir(), `${name}-`));
+  dirs.push(root);
+  const assets = join(root, 'assets');
+  await mkdir(assets);
+  await writeFile(join(assets, 'index.html'), 'asset');
+  const backup = join(root, 'backup');
+  const drill = join(root, 'drill');
+  const backupContent = 'verified backup artifact\n';
+  const drillContent = 'successful rollback drill\n';
+  await writeFile(backup, backupContent);
+  await writeFile(drill, drillContent);
+  return {
+    assetDir: assets,
+    backupEvidence: backup,
+    backupEvidenceSha256: createHash('sha256').update(backupContent).digest('hex'),
+    drillEvidence: drill,
+    drillEvidenceSha256: createHash('sha256').update(drillContent).digest('hex'),
+    legacyPid: 41,
+    legacyCommand: '/usr/bin/true legacy',
+    legacyIdentity: 'legacy-start-1',
+    legacyStart: ['/usr/bin/true', 'legacy'],
+    stateDir: join(root, 'state'),
+  };
+}
+
+interface FakeOptions {
+  gatewayOwnsPort?: boolean;
+  gatewayReady?: boolean;
+}
+
+function fakeSystem(plan: CutoverPlan, options: FakeOptions = {}) {
+  const commands = new Map<number, string>([[plan.legacyPid, plan.legacyCommand]]);
+  const identities = new Map<number, string>([[plan.legacyPid, plan.legacyIdentity]]);
+  const signals: number[] = [];
+  const starts: Array<{ argv: string[]; environment?: NodeJS.ProcessEnv }> = [];
+  let nextPid = 42;
+  const system: CutoverSystem = {
+    ready: async (port, suppliedToken) =>
+      suppliedToken === token && (port === 8790 || options.gatewayReady !== false),
+    command: async (pid) => commands.get(pid) ?? '',
+    identity: async (pid) => identities.get(pid) ?? '',
+    ownsProductionPort: async (pid) => pid === 42 && options.gatewayOwnsPort !== false,
+    signal: (pid) => {
+      signals.push(pid);
+      commands.delete(pid);
+      identities.delete(pid);
+    },
+    waitGone: async () => {},
+    start: (argv, environment) => {
+      starts.push({ argv: [...argv], ...(environment === undefined ? {} : { environment }) });
+      const pid = nextPid++;
+      commands.set(pid, argv.join(' '));
+      identities.set(pid, `start-${pid}`);
+      return pid;
+    },
+    delay: async () => {},
+  };
+  return { system, signals, starts, commands, identities };
+}
+
+describe('production gateway ownership', () => {
+  it('requires the exact strong acknowledgement and ignores a supplied canary port', () => {
+    expect(() => readProductionGatewayRuntimeOptions(environment())).toThrow(/CUTOVER_ACK/);
+    expect(() => readProductionGatewayRuntimeOptions(environment('yes'))).toThrow(/CUTOVER_ACK/);
+    const options = readProductionGatewayRuntimeOptions({
+      ...environment(PRODUCTION_CUTOVER_ACK),
+      CANARY_PORT: '1234',
+    });
+    expect(options).toMatchObject({ port: 8788, ownsProductionPort: true });
+  });
+
+  it('keeps ordinary canary construction unable to claim 8788', () => {
+    expect(() => createCanaryGateway({ distDir: '/tmp', port: 8788 })).toThrow(/reserved/);
+  });
+});
+
+describe('cutover validation and reversible orchestration', () => {
+  it('rejects extra untrusted JSON fields and non-absolute executables', async () => {
+    const plan = await fixture('cutover-schema');
+    expect(() => parseCutoverPlan({ ...plan, command: 'sh -c evil' })).toThrow(/schema/);
+    expect(() => parseCutoverPlan({ ...plan, legacyStart: ['sh', '-c', 'evil'] })).toThrow(
+      /absolute/,
+    );
+  });
+
+  it('authenticates readiness and verifies evidence digests and regular assets', async () => {
+    const plan = await fixture('cutover-preflight');
+    const { system } = fakeSystem(plan);
+    await expect(preflight(plan, system, '')).rejects.toThrow(/Gateway token/);
+    await writeFile(plan.backupEvidence, 'tampered');
+    await expect(preflight(plan, system, token)).rejects.toThrow(/SHA-256 mismatch/);
+    await rm(plan.backupEvidence);
+    await mkdir(plan.backupEvidence);
+    await expect(preflight(plan, system, token)).rejects.toThrow(/regular file/);
+  });
+
+  it('durably records mode-0600 state, verifies 8788 ownership, and restores exact argv', async () => {
+    const plan = await fixture();
+    const { system, signals, starts } = fakeSystem(plan);
+    expect(await cutover(plan, system, token)).toBe(42);
+    expect(signals).toEqual([41]);
+    expect(starts[0]?.environment?.MULTI_SIMADMIN_GATEWAY_TOKEN).toBe(token);
+    const statePath = join(plan.stateDir, 'cutover-state.json');
+    expect((await lstat(statePath)).mode & 0o777).toBe(0o600);
+    expect(await readFile(statePath, 'utf8')).not.toContain(token);
+    expect(await rollback(plan, system)).toBe(43);
+    expect(signals).toEqual([41, 42]);
+    expect(starts[1]?.argv).toEqual(['/usr/bin/true', 'legacy']);
+  });
+
+  it('automatically restarts legacy if the gateway fails to own authenticated 8788', async () => {
+    const plan = await fixture('cutover-failure');
+    const { system, signals, starts } = fakeSystem(plan, { gatewayOwnsPort: false });
+    await expect(cutover(plan, system, token)).rejects.toThrow(/ownership verification/);
+    expect(signals).toEqual([41, 42]);
+    expect(starts.map((start) => start.argv)).toEqual([
+      [
+        resolve(process.cwd(), 'apps/api/node_modules/.bin/tsx'),
+        resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'),
+      ],
+      ['/usr/bin/true', 'legacy'],
+    ]);
+    const state = JSON.parse(await readFile(join(plan.stateDir, 'cutover-state.json'), 'utf8')) as {
+      phase: string;
+    };
+    expect(state.phase).toBe('prepared');
+  });
+
+  it('never signals mismatched legacy or gateway PID identities', async () => {
+    const plan = await fixture('cutover-mismatch');
+    const first = fakeSystem(plan);
+    first.commands.set(plan.legacyPid, 'foreign');
+    await expect(cutover(plan, first.system, token)).rejects.toThrow(/identity mismatch/);
+    expect(first.signals).toEqual([]);
+
+    const second = fakeSystem(plan);
+    await cutover(plan, second.system, token);
+    second.commands.set(42, 'foreign');
+    await expect(rollback(plan, second.system)).rejects.toThrow(/identity mismatch/);
+    expect(second.signals).toEqual([41]);
+  });
+
+  it('refuses rollback state with permissive permissions or invalid schema', async () => {
+    const plan = await fixture('cutover-state-validation');
+    const first = fakeSystem(plan);
+    await cutover(plan, first.system, token);
+    const statePath = join(plan.stateDir, 'cutover-state.json');
+    await chmod(statePath, 0o644);
+    await expect(rollback(plan, first.system)).rejects.toThrow(/mode-0600/);
+    expect(first.signals).toEqual([41]);
+
+    await chmod(statePath, 0o600);
+    const state = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, unknown>;
+    await writeFile(
+      statePath,
+      JSON.stringify({ ...state, injectedCommand: ['sh', '-c', 'evil'] }),
+      {
+        mode: 0o600,
+      },
+    );
+    await expect(rollback(plan, first.system)).rejects.toThrow(/schema/);
+    expect(first.signals).toEqual([41]);
+  });
+});

apps/api/src/production-readiness.release.test.ts

@@ -0,0 +1,67 @@
+import Database from 'better-sqlite3';
+import { describe, expect, it } from 'vitest';
+import { migrateDatabase } from './infrastructure/database/migrations.js';
+import { createProductionReadiness } from './production-readiness.js';
+
+function fixture() {
+  const db = new Database(':memory:');
+  migrateDatabase(db);
+  return db;
+}
+
+describe('production readiness release gates', () => {
+  it('fails closed on pending credentials, recovery markers, missing assets and configured gateway failure', async () => {
+    const db = fixture();
+    db.prepare(
+      "INSERT INTO app_settings(key,value_json,created_at,updated_at) VALUES('legacyImport.pendingCredentials','2','x','x')",
+    ).run();
+    const readiness = createProductionReadiness({
+      db,
+      gatewayToken: 'x'.repeat(32),
+      keychainMetadataCheck: async () => true,
+      assetsCheck: async () => false,
+      recoveryMarkersCheck: async () => 1,
+      upstreamGatewayCheck: async () => false,
+    });
+    await expect(readiness()).resolves.toEqual({
+      ready: false,
+      checks: {
+        database: 'ready',
+        migrations: 'ready',
+        gatewayAuth: 'ready',
+        keychain: 'ready',
+        pendingCredentials: 'pending:2',
+        recoveryMarkers: 'present:1',
+        assets: 'unavailable',
+        upstreamGateway: 'unavailable',
+      },
+    });
+    db.close();
+  });
+
+  it('requires explicit release checks and reports ready only when every gate passes', async () => {
+    const db = fixture();
+    const missing = await createProductionReadiness({
+      db,
+      gatewayToken: 'x'.repeat(32),
+      keychainMetadataCheck: async () => true,
+    })();
+    expect(missing.ready).toBe(false);
+    expect(missing.checks).toMatchObject({
+      assets: 'unconfigured',
+      recoveryMarkers: 'unconfigured',
+      upstreamGateway: 'unconfigured',
+      pendingCredentials: 'ready',
+    });
+    const ready = await createProductionReadiness({
+      db,
+      gatewayToken: 'x'.repeat(32),
+      keychainMetadataCheck: async () => true,
+      assetsCheck: async () => true,
+      recoveryMarkersCheck: async () => 0,
+      upstreamGatewayCheck: async () => true,
+    })();
+    expect(ready.ready).toBe(true);
+    db.close();
+  });
+});

apps/api/src/production-readiness.ts

@@ -8,6 +8,10 @@ export interface ProductionReadinessOptions {
   readonly gatewayToken: string;
   /** Metadata/availability probe only. This callback must never retrieve a secret. */
   readonly keychainMetadataCheck: () => boolean | Promise<boolean>;
+  /** Release gates are deliberately required: omitting one fails readiness closed. */
+  readonly assetsCheck?: () => boolean | Promise<boolean>;
+  readonly recoveryMarkersCheck?: () => number | Promise<number>;
+  readonly upstreamGatewayCheck?: () => boolean | Promise<boolean>;
 }
 interface ExecutableMetadata {
   isFile(): boolean;
@@ -57,6 +61,57 @@ export function createProductionReadiness(
       checks.keychain = 'unavailable';
     }
     if (checks.keychain !== 'ready') ready = false;
+
+    try {
+      const row = options.db
+        .prepare(
+          "SELECT value_json FROM app_settings WHERE key = 'legacyImport.pendingCredentials'",
+        )
+        .get() as { value_json: string } | undefined;
+      const value: unknown = row === undefined ? 0 : JSON.parse(row.value_json);
+      checks.pendingCredentials =
+        Number.isSafeInteger(value) && (value as number) >= 0
+          ? value === 0
+            ? 'ready'
+            : `pending:${String(value)}`
+          : 'unavailable';
+    } catch {
+      checks.pendingCredentials = 'unavailable';
+    }
+    if (checks.pendingCredentials !== 'ready') ready = false;
+
+    if (options.recoveryMarkersCheck === undefined) {
+      checks.recoveryMarkers = 'unconfigured';
+    } else {
+      try {
+        const count = await options.recoveryMarkersCheck();
+        checks.recoveryMarkers =
+          Number.isSafeInteger(count) && count >= 0
+            ? count === 0
+              ? 'ready'
+              : `present:${String(count)}`
+            : 'unavailable';
+      } catch {
+        checks.recoveryMarkers = 'unavailable';
+      }
+    }
+    if (checks.recoveryMarkers !== 'ready') ready = false;
+
+    for (const [name, check] of [
+      ['assets', options.assetsCheck],
+      ['upstreamGateway', options.upstreamGatewayCheck],
+    ] as const) {
+      if (check === undefined) {
+        checks[name] = 'unconfigured';
+      } else {
+        try {
+          checks[name] = (await check()) ? 'ready' : 'unavailable';
+        } catch {
+          checks[name] = 'unavailable';
+        }
+      }
+      if (checks[name] !== 'ready') ready = false;
+    }
     return { ready, checks };
   };
 }

apps/api/src/release-evidence.test.ts

@@ -0,0 +1,128 @@
+import Database from 'better-sqlite3';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { migrateDatabase } from './infrastructure/database/migrations.js';
+import {
+  appendReleaseEvidence,
+  compareShadowSnapshot,
+  sha256File,
+  verifyReleaseEvidenceLog,
+} from './release-evidence.js';
+
+const cleanup: Array<() => Promise<void>> = [];
+afterEach(async () => Promise.all(cleanup.splice(0).map((fn) => fn())));
+
+async function directory(): Promise<string> {
+  const path = await mkdtemp(join(tmpdir(), 'release-evidence-'));
+  cleanup.push(() => rm(path, { recursive: true, force: true }));
+  return path;
+}
+
+describe('durable redacted release evidence', () => {
+  it('atomically appends hash-chained, digest-bound evidence without origins or secrets', async () => {
+    const root = await directory();
+    const log = join(root, 'evidence.jsonl');
+    const record = await appendReleaseEvidence(log, {
+      releaseSha: 'a'.repeat(40),
+      webAssetDigest: 'b'.repeat(64),
+      databaseBackupDigest: 'c'.repeat(64),
+      rollbackDrill: {
+        performedAt: '2026-07-18T00:00:00.000Z',
+        result: 'passed',
+        recordDigest: 'd'.repeat(64),
+      },
+      pendingCredentialCount: 0,
+      readiness: { ready: true, checks: { assets: 'ready' } },
+      shadow: {
+        inputDigest: 'e'.repeat(64),
+        matches: true,
+        legacy: { instances: { total: 2, enabled: 2 }, statuses: { fresh: 2 } },
+        current: { instances: { total: 2, enabled: 2 }, statuses: { fresh: 2 } },
+        differences: [],
+      },
+      recordedAt: '2026-07-18T01:00:00.000Z',
+    });
+    expect(record.sequence).toBe(1);
+    expect(record.recordDigest).toMatch(/^[a-f0-9]{64}$/);
+    expect(await verifyReleaseEvidenceLog(log)).toEqual({
+      valid: true,
+      records: 1,
+      lastDigest: record.recordDigest,
+    });
+    const persisted = await readFile(log, 'utf8');
+    expect(persisted).not.toContain('https://');
+    expect(persisted).not.toContain('token');
+
+    await expect(
+      appendReleaseEvidence(log, {
+        releaseSha: 'a'.repeat(40),
+        webAssetDigest: 'b'.repeat(64),
+        databaseBackupDigest: 'c'.repeat(64),
+        rollbackDrill: {
+          performedAt: '2026-07-18T00:00:00.000Z',
+          result: 'passed',
+          recordDigest: 'd'.repeat(64),
+        },
+        pendingCredentialCount: 0,
+        readiness: { ready: true, checks: { upstream: 'https://secret.example' } },
+        shadow: {
+          inputDigest: 'e'.repeat(64),
+          matches: true,
+          legacy: { instances: { total: 0, enabled: 0 }, statuses: {} },
+          current: { instances: { total: 0, enabled: 0 }, statuses: {} },
+          differences: [],
+        },
+      }),
+    ).rejects.toThrow(/unsafe evidence/i);
+  });
+
+  it('detects evidence tampering', async () => {
+    const root = await directory();
+    const log = join(root, 'evidence.jsonl');
+    await writeFile(log, '{"sequence":1,"recordDigest":"bad"}\n');
+    expect((await verifyReleaseEvidenceLog(log)).valid).toBe(false);
+  });
+});
+
+describe('read-only digest-bound shadow reconciliation', () => {
+  it('compares only safe counts/status projections and does not mutate the DB', async () => {
+    const root = await directory();
+    const snapshot = join(root, 'legacy.json');
+    await writeFile(
+      snapshot,
+      JSON.stringify({
+        instances: [
+          { name: 'one', url: 'https://secret.invalid', enabled: true, status: 'fresh' },
+          { name: 'two', password: 'never-log', enabled: false, status: 'stale' },
+        ],
+      }),
+    );
+    const db = new Database(':memory:');
+    migrateDatabase(db);
+    const now = '2026-07-18T00:00:00.000Z';
+    db.prepare(
+      'INSERT INTO instances(id,name,base_url,enabled,created_at,updated_at) VALUES(?,?,?,?,?,?)',
+    ).run('1', 'one', 'https://new.invalid', 1, now, now);
+    db.prepare(
+      'INSERT INTO instances(id,name,base_url,enabled,created_at,updated_at) VALUES(?,?,?,?,?,?)',
+    ).run('2', 'two', 'https://new2.invalid', 0, now, now);
+    db.prepare(
+      'INSERT INTO status_snapshots(id,instance_id,category,state,payload_json,observed_at,created_at) VALUES(?,?,?,?,?,?,?)',
+    ).run('s1', '1', 'health', 'fresh', '{}', now, now);
+    db.prepare(
+      'INSERT INTO status_snapshots(id,instance_id,category,state,payload_json,observed_at,created_at) VALUES(?,?,?,?,?,?,?)',
+    ).run('s2', '2', 'health', 'stale', '{}', now, now);
+    const before = db.serialize();
+    const result = await compareShadowSnapshot(snapshot, db);
+    expect(result).toMatchObject({
+      inputDigest: await sha256File(snapshot),
+      matches: true,
+      legacy: { instances: { total: 2, enabled: 1 }, statuses: { fresh: 1, stale: 1 } },
+    });
+    expect(JSON.stringify(result)).not.toContain('invalid');
+    expect(db.serialize()).toEqual(before);
+    db.close();
+  });
+});

apps/api/src/release-evidence.ts

@@ -0,0 +1,328 @@
+import type Database from 'better-sqlite3';
+import { createHash } from 'node:crypto';
+import { open, readFile, rename, rm, stat } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+
+const DIGEST = /^[a-f0-9]{64}$/u;
+const SHA = /^[a-f0-9]{40}$/u;
+const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/u;
+const SAFE_VALUE = /^[a-zA-Z0-9 _.:+-]{1,128}$/u;
+
+export interface SafeProjection {
+  readonly instances: { readonly total: number; readonly enabled: number };
+  readonly statuses: Readonly<Record<string, number>>;
+}
+export interface ShadowComparison {
+  readonly inputDigest: string;
+  readonly matches: boolean;
+  readonly legacy: SafeProjection;
+  readonly current: SafeProjection;
+  readonly differences: readonly string[];
+}
+export interface ReleaseEvidenceInput {
+  readonly releaseSha: string;
+  readonly webAssetDigest: string;
+  readonly databaseBackupDigest: string;
+  readonly rollbackDrill: {
+    readonly performedAt: string;
+    readonly result: 'passed';
+    readonly recordDigest: string;
+  };
+  readonly pendingCredentialCount: number;
+  readonly readiness: {
+    readonly ready: boolean;
+    readonly checks: Readonly<Record<string, string>>;
+  };
+  readonly shadow: ShadowComparison;
+  readonly recordedAt?: string;
+}
+export interface ReleaseEvidenceRecord extends ReleaseEvidenceInput {
+  readonly sequence: number;
+  readonly previousDigest: string | null;
+  readonly recordedAt: string;
+  readonly recordDigest: string;
+}
+
+function canonical(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  if (value !== null && typeof value === 'object') {
+    const record = value as Record<string, unknown>;
+    return `{${Object.keys(record)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`)
+      .join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+function digest(value: unknown): string {
+  return createHash('sha256').update(canonical(value), 'utf8').digest('hex');
+}
+function safeCount(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) >= 0;
+}
+function exactKeys(value: Record<string, unknown>, expected: readonly string[]): void {
+  const actual = Object.keys(value).sort();
+  const wanted = [...expected].sort();
+  if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index]))
+    throw new Error('Unsafe evidence schema');
+}
+function validateProjection(value: SafeProjection): void {
+  exactKeys(value as unknown as Record<string, unknown>, ['instances', 'statuses']);
+  exactKeys(value.instances as unknown as Record<string, unknown>, ['enabled', 'total']);
+  if (!safeCount(value.instances.total) || !safeCount(value.instances.enabled))
+    throw new Error('Unsafe evidence projection count');
+  if (value.instances.enabled > value.instances.total)
+    throw new Error('Unsafe evidence projection');
+  for (const [key, count] of Object.entries(value.statuses)) {
+    if (!/^[a-z0-9_-]{1,64}$/u.test(key) || !safeCount(count))
+      throw new Error('Unsafe evidence status projection');
+  }
+}
+function validateInput(input: ReleaseEvidenceInput): void {
+  exactKeys(input as unknown as Record<string, unknown>, [
+    'databaseBackupDigest',
+    'pendingCredentialCount',
+    'readiness',
+    'releaseSha',
+    'rollbackDrill',
+    'shadow',
+    'webAssetDigest',
+    ...(input.recordedAt === undefined ? [] : ['recordedAt']),
+  ]);
+  exactKeys(input.rollbackDrill as unknown as Record<string, unknown>, [
+    'performedAt',
+    'recordDigest',
+    'result',
+  ]);
+  exactKeys(input.readiness as unknown as Record<string, unknown>, ['checks', 'ready']);
+  exactKeys(input.shadow as unknown as Record<string, unknown>, [
+    'current',
+    'differences',
+    'inputDigest',
+    'legacy',
+    'matches',
+  ]);
+  if (
+    !SHA.test(input.releaseSha) ||
+    !DIGEST.test(input.webAssetDigest) ||
+    !DIGEST.test(input.databaseBackupDigest)
+  )
+    throw new Error('Unsafe evidence digest');
+  if (
+    !ISO.test(input.rollbackDrill.performedAt) ||
+    input.rollbackDrill.result !== 'passed' ||
+    !DIGEST.test(input.rollbackDrill.recordDigest)
+  )
+    throw new Error('Unsafe evidence rollback drill');
+  if (!safeCount(input.pendingCredentialCount)) throw new Error('Unsafe evidence credential count');
+  for (const [key, value] of Object.entries(input.readiness.checks)) {
+    if (!/^[a-zA-Z][a-zA-Z0-9]{0,63}$/u.test(key) || !SAFE_VALUE.test(value))
+      throw new Error('Unsafe evidence readiness value');
+  }
+  if (!DIGEST.test(input.shadow.inputDigest)) throw new Error('Unsafe evidence shadow digest');
+  validateProjection(input.shadow.legacy);
+  validateProjection(input.shadow.current);
+  if (input.shadow.differences.some((value) => !SAFE_VALUE.test(value)))
+    throw new Error('Unsafe evidence difference');
+  if (input.recordedAt !== undefined && !ISO.test(input.recordedAt))
+    throw new Error('Unsafe evidence timestamp');
+}
+
+export async function sha256File(path: string): Promise<string> {
+  const metadata = await stat(path);
+  if (!metadata.isFile()) throw new Error('Digest input must be a regular file');
+  return createHash('sha256')
+    .update(await readFile(path))
+    .digest('hex');
+}
+
+function parseRecords(text: string): ReleaseEvidenceRecord[] {
+  if (text === '') return [];
+  const lines = text.split('\n');
+  if (lines.at(-1) !== '') throw new Error('Evidence log is not newline terminated');
+  lines.pop();
+  return lines.map((line) => JSON.parse(line) as ReleaseEvidenceRecord);
+}
+function validateRecord(
+  record: ReleaseEvidenceRecord,
+  index: number,
+  previous: string | null,
+): boolean {
+  const { recordDigest, ...unsigned } = record;
+  try {
+    exactKeys(record as unknown as Record<string, unknown>, [
+      'databaseBackupDigest',
+      'pendingCredentialCount',
+      'previousDigest',
+      'readiness',
+      'recordDigest',
+      'recordedAt',
+      'releaseSha',
+      'rollbackDrill',
+      'sequence',
+      'shadow',
+      'webAssetDigest',
+    ]);
+    const evidenceInput = {
+      releaseSha: record.releaseSha,
+      webAssetDigest: record.webAssetDigest,
+      databaseBackupDigest: record.databaseBackupDigest,
+      rollbackDrill: record.rollbackDrill,
+      pendingCredentialCount: record.pendingCredentialCount,
+      readiness: record.readiness,
+      shadow: record.shadow,
+      recordedAt: record.recordedAt,
+    };
+    validateInput(evidenceInput);
+  } catch {
+    return false;
+  }
+  return (
+    record.sequence === index + 1 &&
+    record.previousDigest === previous &&
+    DIGEST.test(recordDigest) &&
+    digest(unsigned) === recordDigest
+  );
+}
+export async function verifyReleaseEvidenceLog(path: string): Promise<{
+  readonly valid: boolean;
+  readonly records: number;
+  readonly lastDigest: string | null;
+}> {
+  try {
+    const records = parseRecords(await readFile(path, 'utf8'));
+    let previous: string | null = null;
+    for (const [index, record] of records.entries()) {
+      if (!validateRecord(record, index, previous))
+        return { valid: false, records: index, lastDigest: previous };
+      previous = record.recordDigest;
+    }
+    return { valid: true, records: records.length, lastDigest: previous };
+  } catch {
+    return { valid: false, records: 0, lastDigest: null };
+  }
+}
+
+export async function appendReleaseEvidence(
+  path: string,
+  input: ReleaseEvidenceInput,
+): Promise<ReleaseEvidenceRecord> {
+  validateInput(input);
+  const absolute = resolve(path);
+  const parent = dirname(absolute);
+  const parentInfo = await stat(parent);
+  if (!parentInfo.isDirectory() || parentInfo.isSymbolicLink())
+    throw new Error('Unsafe evidence directory');
+  const lockPath = `${absolute}.lock`;
+  let lock;
+  try {
+    lock = await open(lockPath, 'wx', 0o600);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'EEXIST')
+      throw new Error('Release evidence log is locked by another writer');
+    throw error;
+  }
+  try {
+    let existing = '';
+    try {
+      const info = await stat(absolute);
+      if (!info.isFile() || info.isSymbolicLink() || (info.mode & 0o077) !== 0)
+        throw new Error('Unsafe evidence log');
+      existing = await readFile(absolute, 'utf8');
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+    }
+    const records = parseRecords(existing);
+    let previous: string | null = null;
+    for (const [index, record] of records.entries()) {
+      if (!validateRecord(record, index, previous)) throw new Error('Unsafe evidence log chain');
+      previous = record.recordDigest;
+    }
+    const unsigned = {
+      ...input,
+      recordedAt: input.recordedAt ?? new Date().toISOString(),
+      sequence: records.length + 1,
+      previousDigest: previous,
+    };
+    const record: ReleaseEvidenceRecord = { ...unsigned, recordDigest: digest(unsigned) };
+    const temporary = `${absolute}.${process.pid}.${Date.now()}.tmp`;
+    const handle = await open(temporary, 'wx', 0o600);
+    try {
+      await handle.writeFile(`${existing}${canonical(record)}\n`, 'utf8');
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await rename(temporary, absolute);
+    const directory = await open(parent, 'r');
+    try {
+      await directory.sync();
+    } finally {
+      await directory.close();
+    }
+    return Object.freeze(record);
+  } finally {
+    await lock.close();
+    await rm(lockPath, { force: true });
+  }
+}
+
+function projectionFromLegacy(value: unknown): SafeProjection {
+  if (value === null || typeof value !== 'object' || Array.isArray(value))
+    throw new Error('Invalid shadow snapshot');
+  const instances = (value as { instances?: unknown }).instances;
+  if (!Array.isArray(instances)) throw new Error('Invalid shadow snapshot');
+  const statuses: Record<string, number> = {};
+  let enabled = 0;
+  for (const item of instances) {
+    if (item === null || typeof item !== 'object' || Array.isArray(item))
+      throw new Error('Invalid shadow snapshot');
+    const record = item as Record<string, unknown>;
+    if (
+      typeof record.enabled !== 'boolean' ||
+      typeof record.status !== 'string' ||
+      !/^[a-z0-9_-]{1,64}$/u.test(record.status)
+    )
+      throw new Error('Invalid shadow snapshot');
+    if (record.enabled) enabled += 1;
+    statuses[record.status] = (statuses[record.status] ?? 0) + 1;
+  }
+  return { instances: { total: instances.length, enabled }, statuses };
+}
+function projectionFromDatabase(db: Database.Database): SafeProjection {
+  const instances = db
+    .prepare('SELECT COUNT(*) AS total, COALESCE(SUM(enabled), 0) AS enabled FROM instances')
+    .get() as { total: number; enabled: number };
+  const rows = db
+    .prepare('SELECT state, COUNT(*) AS count FROM status_snapshots GROUP BY state')
+    .all() as Array<{ state: string; count: number }>;
+  const statuses: Record<string, number> = {};
+  for (const row of rows) {
+    if (!/^[a-z0-9_-]{1,64}$/u.test(row.state) || !safeCount(row.count))
+      throw new Error('Unsafe database status');
+    statuses[row.state] = row.count;
+  }
+  return { instances: { total: instances.total, enabled: instances.enabled }, statuses };
+}
+export async function compareShadowSnapshot(
+  path: string,
+  db: Database.Database,
+): Promise<ShadowComparison> {
+  const inputDigest = await sha256File(path);
+  const legacy = projectionFromLegacy(JSON.parse(await readFile(path, 'utf8')) as unknown);
+  const current = projectionFromDatabase(db);
+  const differences: string[] = [];
+  if (legacy.instances.total !== current.instances.total)
+    differences.push('instance total differs');
+  if (legacy.instances.enabled !== current.instances.enabled)
+    differences.push('enabled instance count differs');
+  if (canonical(legacy.statuses) !== canonical(current.statuses))
+    differences.push('status counts differ');
+  return Object.freeze({
+    inputDigest,
+    matches: differences.length === 0,
+    legacy,
+    current,
+    differences,
+  });
+}

apps/api/src/runtime-config.ts

@@ -8,12 +8,14 @@ export interface RuntimeEnvironment {
   readonly MULTI_SIMADMIN_GATEWAY_TOKEN?: string;
   readonly API_HOST?: string;
   readonly API_PORT?: string;
+  readonly MULTI_SIMADMIN_WEB_DIST?: string;
 }
 export interface RuntimeConfig {
   readonly api: { readonly host: '127.0.0.1'; readonly port: 8790 };
   readonly dataRoot: string;
   readonly databasePath: string;
   readonly gatewayToken: string;
+  readonly webDist: string;
 }
 function within(root: string, candidate: string): boolean {
   const child = relative(root, candidate);
@@ -62,10 +64,15 @@ export function createRuntimeConfig(environment: RuntimeEnvironment): RuntimeCon
     Buffer.byteLength(gatewayToken) > 512
   )
     throw new Error('Gateway token must contain at least 32 bytes');
+  const webDistInput = environment.MULTI_SIMADMIN_WEB_DIST;
+  if (!webDistInput || !isAbsolute(webDistInput))
+    throw new Error('An explicit absolute web distribution path is required');
+  const webDist = resolve(webDistInput);
   return Object.freeze({
     api: Object.freeze({ host: '127.0.0.1' as const, port: 8790 as const }),
     dataRoot,
     databasePath,
     gatewayToken,
+    webDist,
   });
 }

apps/api/src/runtime-foundation.test.ts

@@ -28,6 +28,7 @@ describe('production runtime foundation', () => {
       MULTI_SIMADMIN_DATA_ROOT: root,
       MULTI_SIMADMIN_DATABASE_PATH: join(root, 'db.sqlite'),
       MULTI_SIMADMIN_GATEWAY_TOKEN: token,
+      MULTI_SIMADMIN_WEB_DIST: root,
       API_HOST: '127.0.0.1',
       API_PORT: '8790',
     });
@@ -36,6 +37,7 @@ describe('production runtime foundation', () => {
       dataRoot: root,
       databasePath: join(root, 'db.sqlite'),
       gatewayToken: token,
+      webDist: root,
     });
     expect(Object.isFrozen(config)).toBe(true);
     expect(Object.isFrozen(config.api)).toBe(true);
@@ -44,6 +46,7 @@ describe('production runtime foundation', () => {
         MULTI_SIMADMIN_DATA_ROOT: root,
         MULTI_SIMADMIN_DATABASE_PATH: '/tmp/escape.sqlite',
         MULTI_SIMADMIN_GATEWAY_TOKEN: token,
+        MULTI_SIMADMIN_WEB_DIST: root,
       }),
     ).toThrow(/data root/i);
     expect(() =>
@@ -70,6 +73,7 @@ describe('production runtime foundation', () => {
         MULTI_SIMADMIN_DATA_ROOT: root,
         MULTI_SIMADMIN_DATABASE_PATH: databasePath,
         MULTI_SIMADMIN_GATEWAY_TOKEN: token,
+        MULTI_SIMADMIN_WEB_DIST: root,
       }),
     ).toThrow(/data root|symlink/i);
     await expect(lstat(join(outside, 'db.sqlite'))).rejects.toMatchObject({ code: 'ENOENT' });
@@ -150,10 +154,26 @@ describe('production runtime foundation', () => {
     const db = openDatabase(join(directory, 'db.sqlite'));
     migrateDatabase(db);
     const keychainMetadataCheck = vi.fn(async () => true);
-    const readiness = createProductionReadiness({ db, gatewayToken: token, keychainMetadataCheck });
+    const readiness = createProductionReadiness({
+      db,
+      gatewayToken: token,
+      keychainMetadataCheck,
+      assetsCheck: async () => true,
+      recoveryMarkersCheck: async () => 0,
+      upstreamGatewayCheck: async () => true,
+    });
     expect(await readiness()).toEqual({
       ready: true,
-      checks: { database: 'ready', migrations: 'ready', gatewayAuth: 'ready', keychain: 'ready' },
+      checks: {
+        database: 'ready',
+        migrations: 'ready',
+        gatewayAuth: 'ready',
+        keychain: 'ready',
+        pendingCredentials: 'ready',
+        assets: 'ready',
+        recoveryMarkers: 'ready',
+        upstreamGateway: 'ready',
+      },
     });
     expect(keychainMetadataCheck).toHaveBeenCalledOnce();
     db.close();