feat(release): add reversible production cutover gates
This commit is contained in:
chick
@@ -0,0 +1,193 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { chmod, lstat, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { createCanaryGateway } from './canary-gateway.js';
|
||||
import { PRODUCTION_CUTOVER_ACK, readProductionGatewayRuntimeOptions } from './canary-runtime.js';
|
||||
import {
|
||||
cutover,
|
||||
parseCutoverPlan,
|
||||
preflight,
|
||||
rollback,
|
||||
type CutoverPlan,
|
||||
type CutoverSystem,
|
||||
} from './cutover-orchestrator.js';
|
||||
|
||||
const token = 'synthetic-production-gateway-token-32-bytes';
|
||||
const dirs: string[] = [];
|
||||
afterEach(() =>
|
||||
Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))),
|
||||
);
|
||||
|
||||
function environment(ack?: string) {
|
||||
return ack === undefined
|
||||
? { MULTI_SIMADMIN_GATEWAY_TOKEN: token }
|
||||
: { MULTI_SIMADMIN_GATEWAY_TOKEN: token, MULTI_SIMADMIN_CUTOVER_ACK: ack };
|
||||
}
|
||||
|
||||
async function fixture(name = 'cutover'): Promise<CutoverPlan> {
|
||||
const root = await mkdtemp(join(tmpdir(), `${name}-`));
|
||||
dirs.push(root);
|
||||
const assets = join(root, 'assets');
|
||||
await mkdir(assets);
|
||||
await writeFile(join(assets, 'index.html'), 'asset');
|
||||
const backup = join(root, 'backup');
|
||||
const drill = join(root, 'drill');
|
||||
const backupContent = 'verified backup artifact\n';
|
||||
const drillContent = 'successful rollback drill\n';
|
||||
await writeFile(backup, backupContent);
|
||||
await writeFile(drill, drillContent);
|
||||
return {
|
||||
assetDir: assets,
|
||||
backupEvidence: backup,
|
||||
backupEvidenceSha256: createHash('sha256').update(backupContent).digest('hex'),
|
||||
drillEvidence: drill,
|
||||
drillEvidenceSha256: createHash('sha256').update(drillContent).digest('hex'),
|
||||
legacyPid: 41,
|
||||
legacyCommand: '/usr/bin/true legacy',
|
||||
legacyIdentity: 'legacy-start-1',
|
||||
legacyStart: ['/usr/bin/true', 'legacy'],
|
||||
stateDir: join(root, 'state'),
|
||||
};
|
||||
}
|
||||
|
||||
interface FakeOptions {
|
||||
gatewayOwnsPort?: boolean;
|
||||
gatewayReady?: boolean;
|
||||
}
|
||||
|
||||
function fakeSystem(plan: CutoverPlan, options: FakeOptions = {}) {
|
||||
const commands = new Map<number, string>([[plan.legacyPid, plan.legacyCommand]]);
|
||||
const identities = new Map<number, string>([[plan.legacyPid, plan.legacyIdentity]]);
|
||||
const signals: number[] = [];
|
||||
const starts: Array<{ argv: string[]; environment?: NodeJS.ProcessEnv }> = [];
|
||||
let nextPid = 42;
|
||||
const system: CutoverSystem = {
|
||||
ready: async (port, suppliedToken) =>
|
||||
suppliedToken === token && (port === 8790 || options.gatewayReady !== false),
|
||||
command: async (pid) => commands.get(pid) ?? '',
|
||||
identity: async (pid) => identities.get(pid) ?? '',
|
||||
ownsProductionPort: async (pid) => pid === 42 && options.gatewayOwnsPort !== false,
|
||||
signal: (pid) => {
|
||||
signals.push(pid);
|
||||
commands.delete(pid);
|
||||
identities.delete(pid);
|
||||
},
|
||||
waitGone: async () => {},
|
||||
start: (argv, environment) => {
|
||||
starts.push({ argv: [...argv], ...(environment === undefined ? {} : { environment }) });
|
||||
const pid = nextPid++;
|
||||
commands.set(pid, argv.join(' '));
|
||||
identities.set(pid, `start-${pid}`);
|
||||
return pid;
|
||||
},
|
||||
delay: async () => {},
|
||||
};
|
||||
return { system, signals, starts, commands, identities };
|
||||
}
|
||||
|
||||
describe('production gateway ownership', () => {
|
||||
it('requires the exact strong acknowledgement and ignores a supplied canary port', () => {
|
||||
expect(() => readProductionGatewayRuntimeOptions(environment())).toThrow(/CUTOVER_ACK/);
|
||||
expect(() => readProductionGatewayRuntimeOptions(environment('yes'))).toThrow(/CUTOVER_ACK/);
|
||||
const options = readProductionGatewayRuntimeOptions({
|
||||
...environment(PRODUCTION_CUTOVER_ACK),
|
||||
CANARY_PORT: '1234',
|
||||
});
|
||||
expect(options).toMatchObject({ port: 8788, ownsProductionPort: true });
|
||||
});
|
||||
|
||||
it('keeps ordinary canary construction unable to claim 8788', () => {
|
||||
expect(() => createCanaryGateway({ distDir: '/tmp', port: 8788 })).toThrow(/reserved/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cutover validation and reversible orchestration', () => {
|
||||
it('rejects extra untrusted JSON fields and non-absolute executables', async () => {
|
||||
const plan = await fixture('cutover-schema');
|
||||
expect(() => parseCutoverPlan({ ...plan, command: 'sh -c evil' })).toThrow(/schema/);
|
||||
expect(() => parseCutoverPlan({ ...plan, legacyStart: ['sh', '-c', 'evil'] })).toThrow(
|
||||
/absolute/,
|
||||
);
|
||||
});
|
||||
|
||||
it('authenticates readiness and verifies evidence digests and regular assets', async () => {
|
||||
const plan = await fixture('cutover-preflight');
|
||||
const { system } = fakeSystem(plan);
|
||||
await expect(preflight(plan, system, '')).rejects.toThrow(/Gateway token/);
|
||||
await writeFile(plan.backupEvidence, 'tampered');
|
||||
await expect(preflight(plan, system, token)).rejects.toThrow(/SHA-256 mismatch/);
|
||||
await rm(plan.backupEvidence);
|
||||
await mkdir(plan.backupEvidence);
|
||||
await expect(preflight(plan, system, token)).rejects.toThrow(/regular file/);
|
||||
});
|
||||
|
||||
it('durably records mode-0600 state, verifies 8788 ownership, and restores exact argv', async () => {
|
||||
const plan = await fixture();
|
||||
const { system, signals, starts } = fakeSystem(plan);
|
||||
expect(await cutover(plan, system, token)).toBe(42);
|
||||
expect(signals).toEqual([41]);
|
||||
expect(starts[0]?.environment?.MULTI_SIMADMIN_GATEWAY_TOKEN).toBe(token);
|
||||
const statePath = join(plan.stateDir, 'cutover-state.json');
|
||||
expect((await lstat(statePath)).mode & 0o777).toBe(0o600);
|
||||
expect(await readFile(statePath, 'utf8')).not.toContain(token);
|
||||
expect(await rollback(plan, system)).toBe(43);
|
||||
expect(signals).toEqual([41, 42]);
|
||||
expect(starts[1]?.argv).toEqual(['/usr/bin/true', 'legacy']);
|
||||
});
|
||||
|
||||
it('automatically restarts legacy if the gateway fails to own authenticated 8788', async () => {
|
||||
const plan = await fixture('cutover-failure');
|
||||
const { system, signals, starts } = fakeSystem(plan, { gatewayOwnsPort: false });
|
||||
await expect(cutover(plan, system, token)).rejects.toThrow(/ownership verification/);
|
||||
expect(signals).toEqual([41, 42]);
|
||||
expect(starts.map((start) => start.argv)).toEqual([
|
||||
[
|
||||
resolve(process.cwd(), 'apps/api/node_modules/.bin/tsx'),
|
||||
resolve(process.cwd(), 'apps/api/src/production-gateway-cli.ts'),
|
||||
],
|
||||
['/usr/bin/true', 'legacy'],
|
||||
]);
|
||||
const state = JSON.parse(await readFile(join(plan.stateDir, 'cutover-state.json'), 'utf8')) as {
|
||||
phase: string;
|
||||
};
|
||||
expect(state.phase).toBe('prepared');
|
||||
});
|
||||
|
||||
it('never signals mismatched legacy or gateway PID identities', async () => {
|
||||
const plan = await fixture('cutover-mismatch');
|
||||
const first = fakeSystem(plan);
|
||||
first.commands.set(plan.legacyPid, 'foreign');
|
||||
await expect(cutover(plan, first.system, token)).rejects.toThrow(/identity mismatch/);
|
||||
expect(first.signals).toEqual([]);
|
||||
|
||||
const second = fakeSystem(plan);
|
||||
await cutover(plan, second.system, token);
|
||||
second.commands.set(42, 'foreign');
|
||||
await expect(rollback(plan, second.system)).rejects.toThrow(/identity mismatch/);
|
||||
expect(second.signals).toEqual([41]);
|
||||
});
|
||||
|
||||
it('refuses rollback state with permissive permissions or invalid schema', async () => {
|
||||
const plan = await fixture('cutover-state-validation');
|
||||
const first = fakeSystem(plan);
|
||||
await cutover(plan, first.system, token);
|
||||
const statePath = join(plan.stateDir, 'cutover-state.json');
|
||||
await chmod(statePath, 0o644);
|
||||
await expect(rollback(plan, first.system)).rejects.toThrow(/mode-0600/);
|
||||
expect(first.signals).toEqual([41]);
|
||||
|
||||
await chmod(statePath, 0o600);
|
||||
const state = JSON.parse(await readFile(statePath, 'utf8')) as Record<string, unknown>;
|
||||
await writeFile(
|
||||
statePath,
|
||||
JSON.stringify({ ...state, injectedCommand: ['sh', '-c', 'evil'] }),
|
||||
{
|
||||
mode: 0o600,
|
||||
},
|
||||
);
|
||||
await expect(rollback(plan, first.system)).rejects.toThrow(/schema/);
|
||||
expect(first.signals).toEqual([41]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user