feat(release): add reversible production cutover gates
This commit is contained in:
chick
@@ -0,0 +1,470 @@
|
||||
import { spawn, execFile } from 'node:child_process';
|
||||
import { constants } from 'node:fs';
|
||||
import {
|
||||
access,
|
||||
chmod,
|
||||
lstat,
|
||||
mkdir,
|
||||
open,
|
||||
readdir,
|
||||
readFile,
|
||||
readlink,
|
||||
realpath,
|
||||
rename,
|
||||
} from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { request } from 'node:http';
|
||||
import { isAbsolute, join, resolve } from 'node:path';
|
||||
import { GATEWAY_AUTH_HEADER } from './runtime-config.js';
|
||||
import { PRODUCTION_CUTOVER_ACK } from './canary-runtime.js';
|
||||
|
||||
const STATE_VERSION = 1;
|
||||
const STATE_FILE = 'cutover-state.json';
|
||||
const GATEWAY_ARGV = [
|
||||
resolve(process.cwd(), 'apps/api/node_modules/.bin/tsx'),
|
||||
resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'),
|
||||
] as const;
|
||||
const DIGEST = /^[a-f0-9]{64}$/u;
|
||||
|
||||
export interface CutoverPlan {
|
||||
readonly assetDir: string;
|
||||
readonly backupEvidence: string;
|
||||
readonly backupEvidenceSha256: string;
|
||||
readonly drillEvidence: string;
|
||||
readonly drillEvidenceSha256: string;
|
||||
readonly legacyPid: number;
|
||||
readonly legacyCommand: string;
|
||||
readonly legacyIdentity: string;
|
||||
readonly legacyStart: readonly [string, ...string[]];
|
||||
readonly stateDir: string;
|
||||
}
|
||||
|
||||
interface CutoverState {
|
||||
readonly version: 1;
|
||||
readonly phase: 'prepared' | 'active';
|
||||
readonly legacyPid: number;
|
||||
readonly legacyCommand: string;
|
||||
readonly legacyIdentity: string;
|
||||
readonly legacyStart: readonly [string, ...string[]];
|
||||
readonly gatewayPid?: number;
|
||||
readonly gatewayCommand?: string;
|
||||
readonly gatewayIdentity?: string;
|
||||
}
|
||||
|
||||
export interface CutoverSystem {
|
||||
command(pid: number): Promise<string>;
|
||||
identity(pid: number): Promise<string>;
|
||||
ready(port: 8788 | 8790, gatewayToken: string): Promise<boolean>;
|
||||
ownsProductionPort(pid: number): Promise<boolean>;
|
||||
signal(pid: number, signal: NodeJS.Signals): void;
|
||||
waitGone(pid: number): Promise<void>;
|
||||
start(argv: readonly [string, ...string[]], environment?: NodeJS.ProcessEnv): number;
|
||||
delay(milliseconds: number): Promise<void>;
|
||||
}
|
||||
|
||||
function runFile(file: string, args: readonly string[]): Promise<string> {
|
||||
return new Promise((resolvePromise, reject) =>
|
||||
execFile(file, [...args], (error, stdout) =>
|
||||
error ? reject(error) : resolvePromise(stdout.trim()),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function authenticatedReady(port: 8788 | 8790, gatewayToken: string): Promise<boolean> {
|
||||
return new Promise((resolvePromise) => {
|
||||
const req = request(
|
||||
{
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
path: '/readyz',
|
||||
timeout: 2_000,
|
||||
headers: { [GATEWAY_AUTH_HEADER]: gatewayToken },
|
||||
},
|
||||
(response) => {
|
||||
response.resume();
|
||||
resolvePromise(response.statusCode === 200);
|
||||
},
|
||||
);
|
||||
req.once('error', () => resolvePromise(false));
|
||||
req.once('timeout', () => {
|
||||
req.destroy();
|
||||
resolvePromise(false);
|
||||
});
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
export const nodeCutoverSystem: CutoverSystem = {
|
||||
async command(pid) {
|
||||
const proc = await readFile(`/proc/${pid}/cmdline`).catch(() => undefined);
|
||||
if (proc) return proc.toString().split('\0').filter(Boolean).join(' ');
|
||||
return await runFile('ps', ['-p', String(pid), '-o', 'command=']);
|
||||
},
|
||||
async identity(pid) {
|
||||
if (process.platform === 'linux') {
|
||||
const value = await readFile(`/proc/${pid}/stat`, 'utf8');
|
||||
const fields = value.slice(value.lastIndexOf(')') + 2).split(/\s+/u);
|
||||
return fields[19] ?? '';
|
||||
}
|
||||
return await runFile('ps', ['-p', String(pid), '-o', 'lstart=']);
|
||||
},
|
||||
ready: authenticatedReady,
|
||||
async ownsProductionPort(pid) {
|
||||
if (process.platform === 'linux') {
|
||||
const descriptorDirectory = `/proc/${pid}/fd`;
|
||||
const descriptors = await readdir(descriptorDirectory).catch(() => []);
|
||||
const listeners = await Promise.all(
|
||||
descriptors.map((descriptor) =>
|
||||
readlink(join(descriptorDirectory, descriptor)).catch(() => ''),
|
||||
),
|
||||
);
|
||||
const socketInodes = new Set(
|
||||
listeners
|
||||
.map((line) => /^socket:\[(\d+)\]$/u.exec(line)?.[1])
|
||||
.filter((inode): inode is string => inode !== undefined),
|
||||
);
|
||||
const tcp = await readFile('/proc/net/tcp', 'utf8').catch(() => '');
|
||||
return tcp.split('\n').some((line) => {
|
||||
const fields = line.trim().split(/\s+/u);
|
||||
return (
|
||||
fields[1]?.endsWith(':2254') && fields[3] === '0A' && socketInodes.has(fields[9] ?? '')
|
||||
);
|
||||
});
|
||||
}
|
||||
const output = await runFile('lsof', [
|
||||
'-nP',
|
||||
'-a',
|
||||
'-p',
|
||||
String(pid),
|
||||
'-iTCP:8788',
|
||||
'-sTCP:LISTEN',
|
||||
'-t',
|
||||
]).catch(() => '');
|
||||
return output.split(/\s+/u).includes(String(pid));
|
||||
},
|
||||
signal: (pid, signal) => process.kill(pid, signal),
|
||||
async waitGone(pid) {
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 100));
|
||||
}
|
||||
throw new Error(`PID ${pid} did not exit after SIGTERM`);
|
||||
},
|
||||
start(argv, environment) {
|
||||
const child = spawn(argv[0], argv.slice(1), {
|
||||
detached: true,
|
||||
stdio: 'ignore',
|
||||
env: environment ?? process.env,
|
||||
});
|
||||
child.unref();
|
||||
if (!child.pid) throw new Error('Failed to start command');
|
||||
return child.pid;
|
||||
},
|
||||
delay: (milliseconds) => new Promise((resolveWait) => setTimeout(resolveWait, milliseconds)),
|
||||
};
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value))
|
||||
throw new Error(`${label} must be an object`);
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: Record<string, unknown>,
|
||||
expected: readonly string[],
|
||||
label: string,
|
||||
): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const wanted = [...expected].sort();
|
||||
if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index]))
|
||||
throw new Error(`${label} has an invalid schema`);
|
||||
}
|
||||
|
||||
function safeString(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || value.length === 0 || value.includes('\0'))
|
||||
throw new Error(`${label} must be a non-empty string`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function argv(value: unknown, label: string): readonly [string, ...string[]] {
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
value.length === 0 ||
|
||||
value.length > 64 ||
|
||||
value.some((item) => typeof item !== 'string' || item.length === 0 || item.includes('\0'))
|
||||
)
|
||||
throw new Error(`${label} must be a non-empty string array`);
|
||||
return value as [string, ...string[]];
|
||||
}
|
||||
|
||||
export function parseCutoverPlan(value: unknown): CutoverPlan {
|
||||
const input = record(value, 'Cutover plan');
|
||||
exactKeys(
|
||||
input,
|
||||
[
|
||||
'assetDir',
|
||||
'backupEvidence',
|
||||
'backupEvidenceSha256',
|
||||
'drillEvidence',
|
||||
'drillEvidenceSha256',
|
||||
'legacyPid',
|
||||
'legacyCommand',
|
||||
'legacyIdentity',
|
||||
'legacyStart',
|
||||
'stateDir',
|
||||
],
|
||||
'Cutover plan',
|
||||
);
|
||||
const plan: CutoverPlan = {
|
||||
assetDir: safeString(input.assetDir, 'assetDir'),
|
||||
backupEvidence: safeString(input.backupEvidence, 'backupEvidence'),
|
||||
backupEvidenceSha256: safeString(input.backupEvidenceSha256, 'backupEvidenceSha256'),
|
||||
drillEvidence: safeString(input.drillEvidence, 'drillEvidence'),
|
||||
drillEvidenceSha256: safeString(input.drillEvidenceSha256, 'drillEvidenceSha256'),
|
||||
legacyPid: input.legacyPid as number,
|
||||
legacyCommand: safeString(input.legacyCommand, 'legacyCommand'),
|
||||
legacyIdentity: safeString(input.legacyIdentity, 'legacyIdentity'),
|
||||
legacyStart: argv(input.legacyStart, 'legacyStart'),
|
||||
stateDir: safeString(input.stateDir, 'stateDir'),
|
||||
};
|
||||
if (
|
||||
![
|
||||
plan.assetDir,
|
||||
plan.backupEvidence,
|
||||
plan.drillEvidence,
|
||||
plan.stateDir,
|
||||
plan.legacyStart[0],
|
||||
].every(isAbsolute)
|
||||
)
|
||||
throw new Error('Cutover paths and legacy executable must be absolute');
|
||||
if (!Number.isSafeInteger(plan.legacyPid) || plan.legacyPid <= 1)
|
||||
throw new Error('legacyPid must be a safe PID greater than 1');
|
||||
if (!DIGEST.test(plan.backupEvidenceSha256) || !DIGEST.test(plan.drillEvidenceSha256))
|
||||
throw new Error('Evidence digests must be lowercase SHA-256');
|
||||
if (plan.legacyStart.join(' ') !== plan.legacyCommand)
|
||||
throw new Error('legacyStart must exactly reproduce legacyCommand');
|
||||
return Object.freeze(plan);
|
||||
}
|
||||
|
||||
function requireGatewayToken(gatewayToken: string): void {
|
||||
const bytes = Buffer.byteLength(gatewayToken);
|
||||
if (bytes < 32 || bytes > 512)
|
||||
throw new Error('Gateway token must contain between 32 and 512 bytes');
|
||||
}
|
||||
|
||||
async function verifyEvidence(path: string, digest: string, label: string): Promise<void> {
|
||||
const info = await lstat(path);
|
||||
if (!info.isFile()) throw new Error(`${label} evidence must be a regular file`);
|
||||
const content = await readFile(path);
|
||||
const actual = createHash('sha256').update(content).digest('hex');
|
||||
if (actual !== digest) throw new Error(`${label} evidence SHA-256 mismatch`);
|
||||
}
|
||||
|
||||
async function durableState(plan: CutoverPlan, state: CutoverState): Promise<void> {
|
||||
await mkdir(plan.stateDir, { recursive: true, mode: 0o700 });
|
||||
await chmod(plan.stateDir, 0o700);
|
||||
const target = join(plan.stateDir, STATE_FILE);
|
||||
const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
|
||||
const handle = await open(
|
||||
temporary,
|
||||
constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
|
||||
0o600,
|
||||
);
|
||||
try {
|
||||
await handle.writeFile(`${JSON.stringify(state)}\n`);
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
await rename(temporary, target);
|
||||
const directory = await open(plan.stateDir, constants.O_RDONLY);
|
||||
try {
|
||||
await directory.sync();
|
||||
} finally {
|
||||
await directory.close();
|
||||
}
|
||||
}
|
||||
|
||||
function preparedState(plan: CutoverPlan): CutoverState {
|
||||
return {
|
||||
version: STATE_VERSION,
|
||||
phase: 'prepared',
|
||||
legacyPid: plan.legacyPid,
|
||||
legacyCommand: plan.legacyCommand,
|
||||
legacyIdentity: plan.legacyIdentity,
|
||||
legacyStart: plan.legacyStart,
|
||||
};
|
||||
}
|
||||
|
||||
async function readState(plan: CutoverPlan): Promise<CutoverState> {
|
||||
const path = join(plan.stateDir, STATE_FILE);
|
||||
const info = await lstat(path);
|
||||
if (!info.isFile() || (info.mode & 0o777) !== 0o600)
|
||||
throw new Error('Cutover state must be a mode-0600 regular file');
|
||||
const input = record(JSON.parse(await readFile(path, 'utf8')) as unknown, 'Cutover state');
|
||||
const base = ['version', 'phase', 'legacyPid', 'legacyCommand', 'legacyIdentity', 'legacyStart'];
|
||||
exactKeys(
|
||||
input,
|
||||
input.phase === 'active' ? [...base, 'gatewayPid', 'gatewayCommand', 'gatewayIdentity'] : base,
|
||||
'Cutover state',
|
||||
);
|
||||
const state: CutoverState = {
|
||||
version: input.version as 1,
|
||||
phase: input.phase as 'prepared' | 'active',
|
||||
legacyPid: input.legacyPid as number,
|
||||
legacyCommand: safeString(input.legacyCommand, 'state legacyCommand'),
|
||||
legacyIdentity: safeString(input.legacyIdentity, 'state legacyIdentity'),
|
||||
legacyStart: argv(input.legacyStart, 'state legacyStart'),
|
||||
...(input.phase === 'active'
|
||||
? {
|
||||
gatewayPid: input.gatewayPid as number,
|
||||
gatewayCommand: safeString(input.gatewayCommand, 'state gatewayCommand'),
|
||||
gatewayIdentity: safeString(input.gatewayIdentity, 'state gatewayIdentity'),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
if (state.version !== STATE_VERSION || !['prepared', 'active'].includes(state.phase))
|
||||
throw new Error('Unsupported cutover state');
|
||||
if (
|
||||
!Number.isSafeInteger(state.legacyPid) ||
|
||||
state.legacyPid <= 1 ||
|
||||
(state.gatewayPid !== undefined &&
|
||||
(!Number.isSafeInteger(state.gatewayPid) || state.gatewayPid <= 1))
|
||||
)
|
||||
throw new Error('Cutover state contains an invalid PID');
|
||||
if (
|
||||
state.legacyPid !== plan.legacyPid ||
|
||||
state.legacyCommand !== plan.legacyCommand ||
|
||||
state.legacyIdentity !== plan.legacyIdentity ||
|
||||
state.legacyStart.length !== plan.legacyStart.length ||
|
||||
state.legacyStart.some((item, index) => item !== plan.legacyStart[index])
|
||||
)
|
||||
throw new Error('Cutover state does not match the reviewed plan');
|
||||
return state;
|
||||
}
|
||||
|
||||
export async function preflight(
|
||||
plan: CutoverPlan,
|
||||
system: CutoverSystem,
|
||||
gatewayToken: string,
|
||||
): Promise<void> {
|
||||
requireGatewayToken(gatewayToken);
|
||||
if (!(await system.ready(8790, gatewayToken)))
|
||||
throw new Error('Authenticated API readiness preflight failed on 127.0.0.1:8790/readyz');
|
||||
const indexPath = join(await realpath(plan.assetDir), 'index.html');
|
||||
const asset = await lstat(indexPath);
|
||||
if (!asset.isFile()) throw new Error('assetDir/index.html must be a regular file');
|
||||
await access(plan.legacyStart[0], constants.X_OK);
|
||||
await verifyEvidence(plan.backupEvidence, plan.backupEvidenceSha256, 'Backup');
|
||||
await verifyEvidence(plan.drillEvidence, plan.drillEvidenceSha256, 'Rollback drill');
|
||||
const actual = await system.command(plan.legacyPid);
|
||||
const identity = await system.identity(plan.legacyPid);
|
||||
if (actual !== plan.legacyCommand || identity !== plan.legacyIdentity)
|
||||
throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
|
||||
}
|
||||
|
||||
async function restoreLegacy(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
|
||||
return system.start(plan.legacyStart);
|
||||
}
|
||||
|
||||
async function waitForGateway(
|
||||
system: CutoverSystem,
|
||||
gatewayPid: number,
|
||||
gatewayToken: string,
|
||||
): Promise<boolean> {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
if ((await system.ownsProductionPort(gatewayPid)) && (await system.ready(8788, gatewayToken)))
|
||||
return true;
|
||||
await system.delay(100);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function cutover(
|
||||
plan: CutoverPlan,
|
||||
system: CutoverSystem,
|
||||
gatewayToken: string,
|
||||
): Promise<number> {
|
||||
await preflight(plan, system, gatewayToken);
|
||||
await durableState(plan, preparedState(plan));
|
||||
let legacyStopped = false;
|
||||
let gatewayPid: number | undefined;
|
||||
let gatewayCommand: string | undefined;
|
||||
let gatewayIdentity: string | undefined;
|
||||
try {
|
||||
if (
|
||||
(await system.command(plan.legacyPid)) !== plan.legacyCommand ||
|
||||
(await system.identity(plan.legacyPid)) !== plan.legacyIdentity
|
||||
)
|
||||
throw new Error(`Legacy PID identity mismatch; refusing to signal PID ${plan.legacyPid}`);
|
||||
system.signal(plan.legacyPid, 'SIGTERM');
|
||||
await system.waitGone(plan.legacyPid);
|
||||
legacyStopped = true;
|
||||
gatewayPid = system.start(GATEWAY_ARGV, {
|
||||
...process.env,
|
||||
CANARY_DIST_DIR: plan.assetDir,
|
||||
MULTI_SIMADMIN_CUTOVER_ACK: PRODUCTION_CUTOVER_ACK,
|
||||
MULTI_SIMADMIN_GATEWAY_TOKEN: gatewayToken,
|
||||
});
|
||||
gatewayCommand = await system.command(gatewayPid);
|
||||
gatewayIdentity = await system.identity(gatewayPid);
|
||||
if (!(await waitForGateway(system, gatewayPid, gatewayToken)))
|
||||
throw new Error(
|
||||
'Production gateway failed authenticated ownership verification on port 8788',
|
||||
);
|
||||
await durableState(plan, {
|
||||
...preparedState(plan),
|
||||
phase: 'active',
|
||||
gatewayPid,
|
||||
gatewayCommand,
|
||||
gatewayIdentity,
|
||||
});
|
||||
return gatewayPid;
|
||||
} catch (error) {
|
||||
if (gatewayPid !== undefined) {
|
||||
try {
|
||||
const actual = await system.command(gatewayPid);
|
||||
const identity = await system.identity(gatewayPid);
|
||||
if (actual === gatewayCommand && identity === gatewayIdentity) {
|
||||
system.signal(gatewayPid, 'SIGTERM');
|
||||
await system.waitGone(gatewayPid);
|
||||
}
|
||||
} catch {
|
||||
// Continue the safety-critical legacy restart even if gateway cleanup races its exit.
|
||||
}
|
||||
}
|
||||
if (legacyStopped) {
|
||||
if (gatewayPid !== undefined && (await system.ownsProductionPort(gatewayPid)))
|
||||
throw new Error('Automatic rollback refused while production gateway still owns port 8788');
|
||||
await restoreLegacy(plan, system);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function rollback(plan: CutoverPlan, system: CutoverSystem): Promise<number> {
|
||||
const state = await readState(plan);
|
||||
if (
|
||||
state.phase !== 'active' ||
|
||||
state.gatewayPid === undefined ||
|
||||
state.gatewayCommand === undefined ||
|
||||
state.gatewayIdentity === undefined
|
||||
)
|
||||
throw new Error('Cutover state is not active; refusing rollback PID operations');
|
||||
const actual = await system.command(state.gatewayPid);
|
||||
const identity = await system.identity(state.gatewayPid);
|
||||
if (
|
||||
actual !== state.gatewayCommand ||
|
||||
identity !== state.gatewayIdentity ||
|
||||
!(await system.ownsProductionPort(state.gatewayPid))
|
||||
)
|
||||
throw new Error(`Gateway PID identity mismatch; refusing to signal PID ${state.gatewayPid}`);
|
||||
system.signal(state.gatewayPid, 'SIGTERM');
|
||||
await system.waitGone(state.gatewayPid);
|
||||
return restoreLegacy(plan, system);
|
||||
}
|
||||
Reference in New Issue
Block a user