multi-simadmin/apps/api/src/cutover-orchestrator.ts

import { execFile, spawn } from 'node:child_process';
import { constants } from 'node:fs';
import {
  access,
  chmod,
  lstat,
  mkdir,
  open,
  readdir,
  readFile,
  readlink,
  realpath,
  rename,
} from 'node:fs/promises';
import { createHash } from 'node:crypto';
import { request } from 'node:http';
import { isAbsolute, join, resolve } from 'node:path';
import { GATEWAY_AUTH_HEADER } from './runtime-config.js';
import { PRODUCTION_CUTOVER_ACK } from './canary-runtime.js';

const STATE_VERSION = 1;
const STATE_FILE = 'cutover-state.json';
const GATEWAY_ARGV = [
  process.execPath,
  '--import',
  'tsx',
  resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'),
] as const;
const DIGEST = /^[a-f0-9]{64}$/u;

export interface CutoverPlan {
  readonly assetDir: string;
  readonly backupEvidence: string;
  readonly backupEvidenceSha256: string;
  readonly drillEvidence: string;
  readonly drillEvidenceSha256: string;
  readonly legacyPid: number;
  readonly legacyCommand: string;
  readonly legacyIdentity: string;
  readonly legacyStart: readonly [string, ...string[]];
  readonly stateDir: string;
}

interface CutoverState {
  readonly version: 1;
  readonly phase: 'prepared' | 'active';
  readonly legacyPid: number;
  readonly legacyCommand: string;
  readonly legacyIdentity: string;
  readonly legacyStart: readonly [string, ...string[]];
  readonly gatewayPid?: number;
  readonly gatewayCommand?: string;
  readonly gatewayIdentity?: string;
}

export interface CutoverSystem {
  command(pid: number): Promise<string>;
  identity(pid: number): Promise<string>;
  ready(port: 8788 | 8790, gatewayToken: string): Promise<boolean>;
  ownsProductionPort(pid: number): Promise<boolean>;
  signal(pid: number, signal: NodeJS.Signals): void;
  waitGone(pid: number): Promise<void>;
  start(argv: readonly [string, ...string[]], environment?: NodeJS.ProcessEnv): number;
  delay(milliseconds: number): Promise<void>;
}

function runFile(file: string, args: readonly string[]): Promise<string> {
  return new Promise((resolvePromise, reject) =>
    execFile(file, [...args], (error, stdout) =>
      error ? reject(error) : resolvePromise(stdout.trim()),
    ),
  );
}

function authenticatedReady(port: 8788 | 8790, gatewayToken: string): Promise<boolean> {
  return new Promise((resolvePromise) => {
    const path = port === 8790 ? '/healthz' : '/readyz';
    const req = request(
      {
        host: '127.0.0.1',
        port,
        path,
        timeout: 2_000,
        headers: { [GATEWAY_AUTH_HEADER]: gatewayToken },
      },
      (response) => {
        response.resume();
        resolvePromise(response.statusCode === 200);
      },
    );
    req.once('error', () => resolvePromise(false));
    req.once('timeout', () => {
      req.destroy();
      resolvePromise(false);
    });
    req.end();
  });
}

export const nodeCutoverSystem: CutoverSystem = {
  async command(pid) {
    const proc = await readFile(`/proc/${pid}/cmdline`).catch(() => undefined);
    if (proc) return proc.toString().split('\0').filter(Boolean).join(' ');
    return await runFile('ps', ['-p', String(pid), '-o', 'command=']);
  },
  async identity(pid) {
    if (process.platform === 'linux') {
      const value = await readFile(`/proc/${pid}/stat`, 'utf8');
      const fields = value.slice(value.lastIndexOf(')') + 2).split(/\s+/u);
      return fields[19] ?? '';
    }
    return await runFile('ps', ['-p', String(pid), '-o', 'lstart=']);
  },
  ready: authenticatedReady,
  async ownsProductionPort(pid) {
    if (process.platform === 'linux') {
      const descriptorDirectory = `/proc/${pid}/fd`;
      const descriptors = await readdir(descriptorDirectory).catch(() => []);
      const listeners = await Promise.all(
        descriptors.map((descriptor) =>
          readlink(join(descriptorDirectory, descriptor)).catch(() => ''),
        ),
      );
      const socketInodes = new Set(
        listeners
          .map((line) => /^socket:\[(\d+)\]$/u.exec(line)?.[1])
          .filter((inode): inode is string => inode !== undefined),
      );
      const tcp = await readFile('/proc/net/tcp', 'utf8').catch(() => '');
      return tcp.split('\n').some((line) => {
        const fields = line.trim().split(/\s+/u);
        return (
          fields[1]?.endsWith(':2254') && fields[3] === '0A' && socketInodes.has(fields[9] ?? '')
        );
      });
    }
    const output = await runFile('lsof', [
      '-nP',
      '-a',
      '-p',
      String(pid),
      '-iTCP:8788',
      '-sTCP:LISTEN',
      '-t',
    ]).catch(() => '');
    return output.split(/\s+/u).includes(String(pid));
  },
  signal: (pid, signal) => process.kill(pid, signal),
  async waitGone(pid) {
    const deadline = Date.now() + 10_000;
    while (Date.now() < deadline) {
      try {
        process.kill(pid, 0);
      } catch {
        return;
      }
      await new Promise((resolveWait) => setTimeout(resolveWait, 100));
    }
    throw new Error(`PID ${pid} did not exit after SIGTERM`);
  },
  start(argv, environment) {
    const child = spawn(argv[0], argv.slice(1), {
      detached: true,
      stdio: 'ignore',
      env: environment ?? process.env,
    });
    child.unref();
    if (!child.pid) throw new Error('Failed to start command');
    return child.pid;
  },
  delay: (milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)),
};

function record(value: unknown, label: string): Record<string, unknown> {
  if (!value || typeof value !== 'object' || Array.isArray(value))
    throw new Error(`${label} must be an object`);
  return value as Record<string, unknown>;
}

function exactKeys(
  value: Record<string, unknown>,
  expected: readonly string[],
  label: string,
): void {
  const actual = Object.keys(value).sort();
  const wanted = [...expected].sort();
  if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index]))
    throw new Error(`${label} has an invalid schema`);
}

function commandMatchesArgv(command: string, commandArgv: readonly string[]): boolean {
  return commandArgv.join(' ') === command;
}

function safeString(value: unknown, label: string): string {
  if (typeof value !== 'string' || value.length === 0 || value.includes('\0'))
    throw new Error(`${label} must be a non-empty string`);
  return value;
}

function argv(value: unknown, label: string): readonly [string, ...string[]] {
  if (
    !Array.isArray(value) ||
    value.length === 0 ||
    value.length > 64 ||
    value.some((item) => typeof item !== 'string' || item.length === 0 || item.includes('\0'))
  )
    throw new Error(`${label} must be a non-empty string array`);
  return value as [string, ...string[]];
}

export function parseCutoverPlan(value: unknown): CutoverPlan {
  const input = record(value, 'Cutover plan');
  exactKeys(
    input,
    [
      'assetDir',
      'backupEvidence',
      'backupEvidenceSha256',
      'drillEvidence',
      'drillEvidenceSha256',
      'legacyPid',
      'legacyCommand',
      'legacyIdentity',
      'legacyStart',
      'stateDir',
    ],
    'Cutover plan',
  );
  const plan: CutoverPlan = {
    assetDir: safeString(input.assetDir, 'assetDir'),
    backupEvidence: safeString(input.backupEvidence, 'backupEvidence'),
    backupEvidenceSha256: safeString(input.backupEvidenceSha256, 'backupEvidenceSha256'),
    drillEvidence: safeString(input.drillEvidence, 'drillEvidence'),
    drillEvidenceSha256: safeString(input.drillEvidenceSha256, 'drillEvidenceSha256'),
    legacyPid: input.legacyPid as number,
    legacyCommand: safeString(input.legacyCommand, 'legacyCommand'),
    legacyIdentity: safeString(input.legacyIdentity, 'legacyIdentity'),
    legacyStart: argv(input.legacyStart, 'legacyStart'),
    stateDir: safeString(input.stateDir, 'stateDir'),
  };
  if (
    ![
      plan.assetDir,
      plan.backupEvidence,
      plan.drillEvidence,
      plan.stateDir,
      plan.legacyStart[0],
    ].every(isAbsolute)
  )
    throw new Error('Cutover paths and legacy executable must be absolute');
  if (!Number.isSafeInteger(plan.legacyPid) || plan.legacyPid <= 1)
    throw new Error('legacyPid must be a safe PID greater than 1');
  if (!DIGEST.test(plan.backupEvidenceSha256) || !DIGEST.test(plan.drillEvidenceSha256))
    throw new Error('Evidence digests must be lowercase SHA-256');
  if (!commandMatchesArgv(plan.legacyCommand, plan.legacyStart))
    throw new Error('legacyStart must exactly reproduce legacyCommand');
  return Object.freeze(plan);
}

function requireGatewayToken(gatewayToken: string): void {
  const bytes = Buffer.byteLength(gatewayToken);
  if (bytes < 32 || bytes > 512)
    throw new Error('Gateway token must contain between 32 and 512 bytes');
}

async function verifyEvidence(path: string, digest: string, label: string): Promise<void> {
  const info = await lstat(path);
  if (!info.isFile()) throw new Error(`${label} evidence must be a regular file`);
  const content = await readFile(path);
  const actual = createHash('sha256').update(content).digest('hex');
  if (actual !== digest) throw new Error(`${label} evidence SHA-256 mismatch`);
}

async function durableState(plan: CutoverPlan, state: CutoverState): Promise<void> {
  await mkdir(plan.stateDir, { recursive: true, mode: 0o700 });
  await chmod(plan.stateDir, 0o700);
  const target = join(plan.stateDir, STATE_FILE);
  const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
  const handle = await open(
    temporary,
    constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
    0o600,
  );
  try {
    await handle.writeFile(`${JSON.stringify(state)}\n`);
    await handle.sync();
  } finally {
    await handle.close();
  }
  await rename(temporary, target);
  if (process.platform !== 'win32') {
    // directory fsync only exists on POSIX
    const directory = await open(plan.stateDir, constants.O_RDONLY);
    try {
      await directory.sync();
    } finally {
      await directory.close();
    }
  }
}

function preparedState(plan: CutoverPlan): CutoverState {
  return {
    version: STATE_VERSION,
    phase: 'prepared',
    legacyPid: plan.legacyPid,
    legacyCommand: plan.legacyCommand,
    legacyIdentity: plan.legacyIdentity,
    legacyStart: plan.legacyStart,
  };
}

async function readState(plan: CutoverPlan): Promise<CutoverState> {
  const path = join(plan.stateDir, STATE_FILE);
  const info = await lstat(path);
  // Windows keeps no POSIX mode bits (chmod is a no-op); ACLs govern access there.
  if (!info.isFile() || (process.platform !== 'win32' && (info.mode & 0o777) !== 0o600))
    throw new Error('Cutover state must be a mode-0600 regular file');
  const input = record(JSON.parse(await readFile(path, 'utf8')) as unknown, 'Cutover state');
  const base = ['version', 'phase', 'legacyPid', 'legacyCommand', 'legacyIdentity', 'legacyStart'];
  exactKeys(
    input,
    input.phase === 'active' ? [...base, 'gatewayPid', 'gatewayCommand', 'gatewayIdentity'] : base,
    'Cutover state',
  );
  const state: CutoverState = {
    version: input.version as 1,
    phase: input.phase as 'prepared' | 'active',
    legacyPid: input.legacyPid as number,
    legacyCommand: safeString(input.legacyCommand, 'state legacyCommand'),
    legacyIdentity: safeString(input.legacyIdentity, 'state legacyIdentity'),
    legacyStart: argv(input.legacyStart, 'state legacyStart'),
    ...(input.phase === 'active'
      ? {
          gatewayPid: input.gatewayPid as number,
          gatewayCommand: safeString(input.gatewayCommand, 'state gatewayCommand'),
          gatewayIdentity: safeString(input.gatewayIdentity, 'state gatewayIdentity'),
        }
      : {}),
  };
  if (state.version !== STATE_VERSION || !['prepared', 'active'].includes(state.phase))
    throw new Error('Unsupported cutover state');
  if (
    !Number.isSafeInteger(state.legacyPid) ||
    state.legacyPid <= 1 ||
    (state.gatewayPid !== undefined &&
      (!Number.isSafeInteger(state.gatewayPid) || state.gatewayPid <= 1))
  )
    throw new Error('Cutover state contains an invalid PID');
  if (
    state.legacyPid !== plan.legacyPid ||
    state.legacyCommand !== plan.legacyCommand ||
    state.legacyIdentity !== plan.legacyIdentity ||
    state.legacyStart.length !== plan.legacyStart.length ||
    state.legacyStart.some((item, index) => item !== plan.legacyStart[index])
  )
    throw new Error('Cutover state does not match the reviewed plan');
  return state;
}

export async function preflight(
  plan: CutoverPlan,
  system: CutoverSystem,
  gatewayToken: string,
): Promise<void> {
  requireGatewayToken(gatewayToken);
  if (!(await system.ready(8790, gatewayToken)))
    throw new Error('Authenticated API health preflight failed on 127.0.0.1:8790/healthz');
  const indexPath = join(await realpath(plan.assetDir), 'index.html');
  const asset = await lstat(indexPath);
  if (!asset.isFile()) throw new Error('assetDir/index.html must be a regular file');
  await access(plan.legacyStart[0], constants.X_OK);
  await verifyEvidence(plan.backupEvidence, plan.backupEvidenceSha256, 'Backup');
  await verifyEvidence(plan.drillEvidence, plan.drillEvidenceSha256, 'Rollback drill');
  const actual = await system.command(plan.legacyPid);
  const identity = await system.identity(plan.legacyPid);
  if (actual !== plan.legacyCommand || identity !== plan.legacyIdentity)
    throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
}

async function restoreLegacy(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
  return system.start(plan.legacyStart);
}

async function waitForGateway(
  system: CutoverSystem,
  gatewayPid: number,
  gatewayToken: string,
): Promise<boolean> {
  for (let attempt = 0; attempt < 100; attempt += 1) {
    if ((await system.ownsProductionPort(gatewayPid)) && (await system.ready(8788, gatewayToken)))
      return true;
    await system.delay(100);
  }
  return false;
}

export async function cutover(
  plan: CutoverPlan,
  system: CutoverSystem,
  gatewayToken: string,
): Promise<number> {
  await preflight(plan, system, gatewayToken);
  await durableState(plan, preparedState(plan));
  let legacyStopped = false;
  let gatewayPid: number | undefined;
  let gatewayCommand: string | undefined;
  let gatewayIdentity: string | undefined;
  try {
    if (
      (await system.command(plan.legacyPid)) !== plan.legacyCommand ||
      (await system.identity(plan.legacyPid)) !== plan.legacyIdentity
    )
      throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
    system.signal(plan.legacyPid, 'SIGTERM');
    await system.waitGone(plan.legacyPid);
    legacyStopped = true;
    gatewayPid = system.start(GATEWAY_ARGV, {
      ...process.env,
      CANARY_DIST_DIR: plan.assetDir,
      MULTI_SIMADMIN_CUTOVER_ACK: PRODUCTION_CUTOVER_ACK,
      MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
      ...(process.env.MULTI_SIMADMIN_GATEWAY_HOST
        ? { MULTI_SIMADMIN_GATEWAY_HOST: process.env.MULTI_SIMADMIN_GATEWAY_HOST }
        : {}),
      ...(process.env.MULTI_SIMADMIN_GATEWAY_ALLOWED_HOSTS
        ? {
            MULTI_SIMADMIN_GATEWAY_ALLOWED_HOSTS: process.env.MULTI_SIMADMIN_GATEWAY_ALLOWED_HOSTS,
          }
        : {}),
    });
    if (!(await waitForGateway(system, gatewayPid, gatewayToken)))
      throw new Error(
        'Production gateway failed authenticated ownership verification on port 8788',
      );
    gatewayCommand = await system.command(gatewayPid);
    gatewayIdentity = await system.identity(gatewayPid);
    await durableState(plan, {
      ...preparedState(plan),
      phase: 'active',
      gatewayPid,
      gatewayCommand,
      gatewayIdentity,
    });
    return gatewayPid;
  } catch (error) {
    if (gatewayPid !== undefined) {
      try {
        const actual = await system.command(gatewayPid);
        const identity = await system.identity(gatewayPid);
        if (
          (gatewayCommand === undefined || actual === gatewayCommand) &&
          (gatewayIdentity === undefined || identity === gatewayIdentity) &&
          actual !== '' &&
          identity !== ''
        ) {
          system.signal(gatewayPid, 'SIGTERM');
          await system.waitGone(gatewayPid);
        }
      } catch {
        // Continue the safety-critical legacy restart even if gateway cleanup races its exit.
      }
    }
    if (legacyStopped) {
      if (gatewayPid !== undefined && (await system.ownsProductionPort(gatewayPid)))
        throw new Error('Automatic rollback refused while production gateway still owns port 8788');
      await restoreLegacy(plan, system);
    }
    throw error;
  }
}

export async function rollback(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
  const state = await readState(plan);
  if (
    state.phase !== 'active' ||
    state.gatewayPid === undefined ||
    state.gatewayCommand === undefined ||
    state.gatewayIdentity === undefined
  )
    throw new Error('Cutover state is not active; refusing rollback PID operations');
  const actual = await system.command(state.gatewayPid);
  const identity = await system.identity(state.gatewayPid);
  if (
    actual !== state.gatewayCommand ||
    identity !== state.gatewayIdentity ||
    !(await system.ownsProductionPort(state.gatewayPid))
  )
    throw new Error(`Gateway PID identity mismatch; refusing to signal PID ${state.gatewayPid}`);
  system.signal(state.gatewayPid, 'SIGTERM');
  await system.waitGone(state.gatewayPid);
  const legacyPid = await restoreLegacy(plan, system);
  await durableState(plan, {
    version: STATE_VERSION,
    phase: 'prepared',
    legacyPid,
    legacyCommand: plan.legacyCommand,
    legacyIdentity: await system.identity(legacyPid),
    legacyStart: plan.legacyStart,
  });
  return legacyPid;
}