import { execFile, spawn } from 'node:child_process'; import { constants } from 'node:fs'; import { access, chmod, lstat, mkdir, open, readdir, readFile, readlink, realpath, rename, } from 'node:fs/promises'; import { createHash } from 'node:crypto'; import { request } from 'node:http'; import { isAbsolute, join, resolve } from 'node:path'; import { GATEWAY_AUTH_HEADER } from './runtime-config.js'; import { PRODUCTION_CUTOVER_ACK } from './canary-runtime.js'; const STATE_VERSION = 1; const STATE_FILE = 'cutover-state.json'; const GATEWAY_ARGV = [ process.execPath, '--import', 'tsx', resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'), ] as const; const DIGEST = /^[a-f0-9]{64}$/u; export interface CutoverPlan { readonly assetDir: string; readonly backupEvidence: string; readonly backupEvidenceSha256: string; readonly drillEvidence: string; readonly drillEvidenceSha256: string; readonly legacyPid: number; readonly legacyCommand: string; readonly legacyIdentity: string; readonly legacyStart: readonly [string, ...string[]]; readonly stateDir: string; } interface CutoverState { readonly version: 1; readonly phase: 'prepared' | 'active'; readonly legacyPid: number; readonly legacyCommand: string; readonly legacyIdentity: string; readonly legacyStart: readonly [string, ...string[]]; readonly gatewayPid?: number; readonly gatewayCommand?: string; readonly gatewayIdentity?: string; } export interface CutoverSystem { command(pid: number): Promise; identity(pid: number): Promise; ready(port: 8788 | 8790, gatewayToken: string): Promise; ownsProductionPort(pid: number): Promise; signal(pid: number, signal: NodeJS.Signals): void; waitGone(pid: number): Promise; start(argv: readonly [string, ...string[]], environment?: NodeJS.ProcessEnv): number; delay(milliseconds: number): Promise; } function runFile(file: string, args: readonly string[]): Promise { return new Promise((resolvePromise, reject) => execFile(file, [...args], (error, stdout) => error ? reject(error) : resolvePromise(stdout.trim()), ), ); } function authenticatedReady(port: 8788 | 8790, gatewayToken: string): Promise { return new Promise((resolvePromise) => { const path = port === 8790 ? '/healthz' : '/readyz'; const req = request( { host: '127.0.0.1', port, path, timeout: 2_000, headers: { [GATEWAY_AUTH_HEADER]: gatewayToken }, }, (response) => { response.resume(); resolvePromise(response.statusCode === 200); }, ); req.once('error', () => resolvePromise(false)); req.once('timeout', () => { req.destroy(); resolvePromise(false); }); req.end(); }); } export const nodeCutoverSystem: CutoverSystem = { async command(pid) { const proc = await readFile(`/proc/${pid}/cmdline`).catch(() => undefined); if (proc) return proc.toString().split('\0').filter(Boolean).join(' '); return await runFile('ps', ['-p', String(pid), '-o', 'command=']); }, async identity(pid) { if (process.platform === 'linux') { const value = await readFile(`/proc/${pid}/stat`, 'utf8'); const fields = value.slice(value.lastIndexOf(')') + 2).split(/\s+/u); return fields[19] ?? ''; } return await runFile('ps', ['-p', String(pid), '-o', 'lstart=']); }, ready: authenticatedReady, async ownsProductionPort(pid) { if (process.platform === 'linux') { const descriptorDirectory = `/proc/${pid}/fd`; const descriptors = await readdir(descriptorDirectory).catch(() => []); const listeners = await Promise.all( descriptors.map((descriptor) => readlink(join(descriptorDirectory, descriptor)).catch(() => ''), ), ); const socketInodes = new Set( listeners .map((line) => /^socket:\[(\d+)\]$/u.exec(line)?.[1]) .filter((inode): inode is string => inode !== undefined), ); const tcp = await readFile('/proc/net/tcp', 'utf8').catch(() => ''); return tcp.split('\n').some((line) => { const fields = line.trim().split(/\s+/u); return ( fields[1]?.endsWith(':2254') && fields[3] === '0A' && socketInodes.has(fields[9] ?? '') ); }); } const output = await runFile('lsof', [ '-nP', '-a', '-p', String(pid), '-iTCP:8788', '-sTCP:LISTEN', '-t', ]).catch(() => ''); return output.split(/\s+/u).includes(String(pid)); }, signal: (pid, signal) => process.kill(pid, signal), async waitGone(pid) { const deadline = Date.now() + 10_000; while (Date.now() < deadline) { try { process.kill(pid, 0); } catch { return; } await new Promise((resolveWait) => setTimeout(resolveWait, 100)); } throw new Error(`PID ${pid} did not exit after SIGTERM`); }, start(argv, environment) { const child = spawn(argv[0], argv.slice(1), { detached: true, stdio: 'ignore', env: environment ?? process.env, }); child.unref(); if (!child.pid) throw new Error('Failed to start command'); return child.pid; }, delay: (milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)), }; function record(value: unknown, label: string): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`${label} must be an object`); return value as Record; } function exactKeys( value: Record, expected: readonly string[], label: string, ): void { const actual = Object.keys(value).sort(); const wanted = [...expected].sort(); if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) throw new Error(`${label} has an invalid schema`); } function commandMatchesArgv(command: string, commandArgv: readonly string[]): boolean { return commandArgv.join(' ') === command; } function safeString(value: unknown, label: string): string { if (typeof value !== 'string' || value.length === 0 || value.includes('\0')) throw new Error(`${label} must be a non-empty string`); return value; } function argv(value: unknown, label: string): readonly [string, ...string[]] { if ( !Array.isArray(value) || value.length === 0 || value.length > 64 || value.some((item) => typeof item !== 'string' || item.length === 0 || item.includes('\0')) ) throw new Error(`${label} must be a non-empty string array`); return value as [string, ...string[]]; } export function parseCutoverPlan(value: unknown): CutoverPlan { const input = record(value, 'Cutover plan'); exactKeys( input, [ 'assetDir', 'backupEvidence', 'backupEvidenceSha256', 'drillEvidence', 'drillEvidenceSha256', 'legacyPid', 'legacyCommand', 'legacyIdentity', 'legacyStart', 'stateDir', ], 'Cutover plan', ); const plan: CutoverPlan = { assetDir: safeString(input.assetDir, 'assetDir'), backupEvidence: safeString(input.backupEvidence, 'backupEvidence'), backupEvidenceSha256: safeString(input.backupEvidenceSha256, 'backupEvidenceSha256'), drillEvidence: safeString(input.drillEvidence, 'drillEvidence'), drillEvidenceSha256: safeString(input.drillEvidenceSha256, 'drillEvidenceSha256'), legacyPid: input.legacyPid as number, legacyCommand: safeString(input.legacyCommand, 'legacyCommand'), legacyIdentity: safeString(input.legacyIdentity, 'legacyIdentity'), legacyStart: argv(input.legacyStart, 'legacyStart'), stateDir: safeString(input.stateDir, 'stateDir'), }; if ( ![ plan.assetDir, plan.backupEvidence, plan.drillEvidence, plan.stateDir, plan.legacyStart[0], ].every(isAbsolute) ) throw new Error('Cutover paths and legacy executable must be absolute'); if (!Number.isSafeInteger(plan.legacyPid) || plan.legacyPid <= 1) throw new Error('legacyPid must be a safe PID greater than 1'); if (!DIGEST.test(plan.backupEvidenceSha256) || !DIGEST.test(plan.drillEvidenceSha256)) throw new Error('Evidence digests must be lowercase SHA-256'); if (!commandMatchesArgv(plan.legacyCommand, plan.legacyStart)) throw new Error('legacyStart must exactly reproduce legacyCommand'); return Object.freeze(plan); } function requireGatewayToken(gatewayToken: string): void { const bytes = Buffer.byteLength(gatewayToken); if (bytes < 32 || bytes > 512) throw new Error('Gateway token must contain between 32 and 512 bytes'); } async function verifyEvidence(path: string, digest: string, label: string): Promise { const info = await lstat(path); if (!info.isFile()) throw new Error(`${label} evidence must be a regular file`); const content = await readFile(path); const actual = createHash('sha256').update(content).digest('hex'); if (actual !== digest) throw new Error(`${label} evidence SHA-256 mismatch`); } async function durableState(plan: CutoverPlan, state: CutoverState): Promise { await mkdir(plan.stateDir, { recursive: true, mode: 0o700 }); await chmod(plan.stateDir, 0o700); const target = join(plan.stateDir, STATE_FILE); const temporary = `${target}.${process.pid}.${Date.now()}.tmp`; const handle = await open( temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600, ); try { await handle.writeFile(`${JSON.stringify(state)}\n`); await handle.sync(); } finally { await handle.close(); } await rename(temporary, target); if (process.platform !== 'win32') { // directory fsync only exists on POSIX const directory = await open(plan.stateDir, constants.O_RDONLY); try { await directory.sync(); } finally { await directory.close(); } } } function preparedState(plan: CutoverPlan): CutoverState { return { version: STATE_VERSION, phase: 'prepared', legacyPid: plan.legacyPid, legacyCommand: plan.legacyCommand, legacyIdentity: plan.legacyIdentity, legacyStart: plan.legacyStart, }; } async function readState(plan: CutoverPlan): Promise { const path = join(plan.stateDir, STATE_FILE); const info = await lstat(path); // Windows keeps no POSIX mode bits (chmod is a no-op); ACLs govern access there. if (!info.isFile() || (process.platform !== 'win32' && (info.mode & 0o777) !== 0o600)) throw new Error('Cutover state must be a mode-0600 regular file'); const input = record(JSON.parse(await readFile(path, 'utf8')) as unknown, 'Cutover state'); const base = ['version', 'phase', 'legacyPid', 'legacyCommand', 'legacyIdentity', 'legacyStart']; exactKeys( input, input.phase === 'active' ? [...base, 'gatewayPid', 'gatewayCommand', 'gatewayIdentity'] : base, 'Cutover state', ); const state: CutoverState = { version: input.version as 1, phase: input.phase as 'prepared' | 'active', legacyPid: input.legacyPid as number, legacyCommand: safeString(input.legacyCommand, 'state legacyCommand'), legacyIdentity: safeString(input.legacyIdentity, 'state legacyIdentity'), legacyStart: argv(input.legacyStart, 'state legacyStart'), ...(input.phase === 'active' ? { gatewayPid: input.gatewayPid as number, gatewayCommand: safeString(input.gatewayCommand, 'state gatewayCommand'), gatewayIdentity: safeString(input.gatewayIdentity, 'state gatewayIdentity'), } : {}), }; if (state.version !== STATE_VERSION || !['prepared', 'active'].includes(state.phase)) throw new Error('Unsupported cutover state'); if ( !Number.isSafeInteger(state.legacyPid) || state.legacyPid <= 1 || (state.gatewayPid !== undefined && (!Number.isSafeInteger(state.gatewayPid) || state.gatewayPid <= 1)) ) throw new Error('Cutover state contains an invalid PID'); if ( state.legacyPid !== plan.legacyPid || state.legacyCommand !== plan.legacyCommand || state.legacyIdentity !== plan.legacyIdentity || state.legacyStart.length !== plan.legacyStart.length || state.legacyStart.some((item, index) => item !== plan.legacyStart[index]) ) throw new Error('Cutover state does not match the reviewed plan'); return state; } export async function preflight( plan: CutoverPlan, system: CutoverSystem, gatewayToken: string, ): Promise { requireGatewayToken(gatewayToken); if (!(await system.ready(8790, gatewayToken))) throw new Error('Authenticated API health preflight failed on 127.0.0.1:8790/healthz'); const indexPath = join(await realpath(plan.assetDir), 'index.html'); const asset = await lstat(indexPath); if (!asset.isFile()) throw new Error('assetDir/index.html must be a regular file'); await access(plan.legacyStart[0], constants.X_OK); await verifyEvidence(plan.backupEvidence, plan.backupEvidenceSha256, 'Backup'); await verifyEvidence(plan.drillEvidence, plan.drillEvidenceSha256, 'Rollback drill'); const actual = await system.command(plan.legacyPid); const identity = await system.identity(plan.legacyPid); if (actual !== plan.legacyCommand || identity !== plan.legacyIdentity) throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`); } async function restoreLegacy(plan: CutoverPlan, system: CutoverSystem): Promise { return system.start(plan.legacyStart); } async function waitForGateway( system: CutoverSystem, gatewayPid: number, gatewayToken: string, ): Promise { for (let attempt = 0; attempt < 100; attempt += 1) { if ((await system.ownsProductionPort(gatewayPid)) && (await system.ready(8788, gatewayToken))) return true; await system.delay(100); } return false; } export async function cutover( plan: CutoverPlan, system: CutoverSystem, gatewayToken: string, ): Promise { await preflight(plan, system, gatewayToken); await durableState(plan, preparedState(plan)); let legacyStopped = false; let gatewayPid: number | undefined; let gatewayCommand: string | undefined; let gatewayIdentity: string | undefined; try { if ( (await system.command(plan.legacyPid)) !== plan.legacyCommand || (await system.identity(plan.legacyPid)) !== plan.legacyIdentity ) throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`); system.signal(plan.legacyPid, 'SIGTERM'); await system.waitGone(plan.legacyPid); legacyStopped = true; gatewayPid = system.start(GATEWAY_ARGV, { ...process.env, CANARY_DIST_DIR: plan.assetDir, MULTI_SIMADMIN_CUTOVER_ACK: PRODUCTION_CUTOVER_ACK, MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken, ...(process.env.MULTI_SIMADMIN_GATEWAY_HOST ? { MULTI_SIMADMIN_GATEWAY_HOST: process.env.MULTI_SIMADMIN_GATEWAY_HOST } : {}), ...(process.env.MULTI_SIMADMIN_GATEWAY_ALLOWED_HOSTS ? { MULTI_SIMADMIN_GATEWAY_ALLOWED_HOSTS: process.env.MULTI_SIMADMIN_GATEWAY_ALLOWED_HOSTS, } : {}), }); if (!(await waitForGateway(system, gatewayPid, gatewayToken))) throw new Error( 'Production gateway failed authenticated ownership verification on port 8788', ); gatewayCommand = await system.command(gatewayPid); gatewayIdentity = await system.identity(gatewayPid); await durableState(plan, { ...preparedState(plan), phase: 'active', gatewayPid, gatewayCommand, gatewayIdentity, }); return gatewayPid; } catch (error) { if (gatewayPid !== undefined) { try { const actual = await system.command(gatewayPid); const identity = await system.identity(gatewayPid); if ( (gatewayCommand === undefined || actual === gatewayCommand) && (gatewayIdentity === undefined || identity === gatewayIdentity) && actual !== '' && identity !== '' ) { system.signal(gatewayPid, 'SIGTERM'); await system.waitGone(gatewayPid); } } catch { // Continue the safety-critical legacy restart even if gateway cleanup races its exit. } } if (legacyStopped) { if (gatewayPid !== undefined && (await system.ownsProductionPort(gatewayPid))) throw new Error('Automatic rollback refused while production gateway still owns port 8788'); await restoreLegacy(plan, system); } throw error; } } export async function rollback(plan: CutoverPlan, system: CutoverSystem): Promise { const state = await readState(plan); if ( state.phase !== 'active' || state.gatewayPid === undefined || state.gatewayCommand === undefined || state.gatewayIdentity === undefined ) throw new Error('Cutover state is not active; refusing rollback PID operations'); const actual = await system.command(state.gatewayPid); const identity = await system.identity(state.gatewayPid); if ( actual !== state.gatewayCommand || identity !== state.gatewayIdentity || !(await system.ownsProductionPort(state.gatewayPid)) ) throw new Error(`Gateway PID identity mismatch; refusing to signal PID ${state.gatewayPid}`); system.signal(state.gatewayPid, 'SIGTERM'); await system.waitGone(state.gatewayPid); const legacyPid = await restoreLegacy(plan, system); await durableState(plan, { version: STATE_VERSION, phase: 'prepared', legacyPid, legacyCommand: plan.legacyCommand, legacyIdentity: await system.identity(legacyPid), legacyStart: plan.legacyStart, }); return legacyPid; }