feat(release): add reversible production cutover gates
This commit is contained in:
chick
@@ -0,0 +1,328 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { open, readFile, rename, rm, stat } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
|
||||
const DIGEST = /^[a-f0-9]{64}$/u;
|
||||
const SHA = /^[a-f0-9]{40}$/u;
|
||||
const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/u;
|
||||
const SAFE_VALUE = /^[a-zA-Z0-9 _.:+-]{1,128}$/u;
|
||||
|
||||
export interface SafeProjection {
|
||||
readonly instances: { readonly total: number; readonly enabled: number };
|
||||
readonly statuses: Readonly<Record<string, number>>;
|
||||
}
|
||||
export interface ShadowComparison {
|
||||
readonly inputDigest: string;
|
||||
readonly matches: boolean;
|
||||
readonly legacy: SafeProjection;
|
||||
readonly current: SafeProjection;
|
||||
readonly differences: readonly string[];
|
||||
}
|
||||
export interface ReleaseEvidenceInput {
|
||||
readonly releaseSha: string;
|
||||
readonly webAssetDigest: string;
|
||||
readonly databaseBackupDigest: string;
|
||||
readonly rollbackDrill: {
|
||||
readonly performedAt: string;
|
||||
readonly result: 'passed';
|
||||
readonly recordDigest: string;
|
||||
};
|
||||
readonly pendingCredentialCount: number;
|
||||
readonly readiness: {
|
||||
readonly ready: boolean;
|
||||
readonly checks: Readonly<Record<string, string>>;
|
||||
};
|
||||
readonly shadow: ShadowComparison;
|
||||
readonly recordedAt?: string;
|
||||
}
|
||||
export interface ReleaseEvidenceRecord extends ReleaseEvidenceInput {
|
||||
readonly sequence: number;
|
||||
readonly previousDigest: string | null;
|
||||
readonly recordedAt: string;
|
||||
readonly recordDigest: string;
|
||||
}
|
||||
|
||||
function canonical(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const record = value as Record<string, unknown>;
|
||||
return `{${Object.keys(record)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`)
|
||||
.join(',')}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
function digest(value: unknown): string {
|
||||
return createHash('sha256').update(canonical(value), 'utf8').digest('hex');
|
||||
}
|
||||
function safeCount(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0;
|
||||
}
|
||||
function exactKeys(value: Record<string, unknown>, expected: readonly string[]): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const wanted = [...expected].sort();
|
||||
if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index]))
|
||||
throw new Error('Unsafe evidence schema');
|
||||
}
|
||||
function validateProjection(value: SafeProjection): void {
|
||||
exactKeys(value as unknown as Record<string, unknown>, ['instances', 'statuses']);
|
||||
exactKeys(value.instances as unknown as Record<string, unknown>, ['enabled', 'total']);
|
||||
if (!safeCount(value.instances.total) || !safeCount(value.instances.enabled))
|
||||
throw new Error('Unsafe evidence projection count');
|
||||
if (value.instances.enabled > value.instances.total)
|
||||
throw new Error('Unsafe evidence projection');
|
||||
for (const [key, count] of Object.entries(value.statuses)) {
|
||||
if (!/^[a-z0-9_-]{1,64}$/u.test(key) || !safeCount(count))
|
||||
throw new Error('Unsafe evidence status projection');
|
||||
}
|
||||
}
|
||||
function validateInput(input: ReleaseEvidenceInput): void {
|
||||
exactKeys(input as unknown as Record<string, unknown>, [
|
||||
'databaseBackupDigest',
|
||||
'pendingCredentialCount',
|
||||
'readiness',
|
||||
'releaseSha',
|
||||
'rollbackDrill',
|
||||
'shadow',
|
||||
'webAssetDigest',
|
||||
...(input.recordedAt === undefined ? [] : ['recordedAt']),
|
||||
]);
|
||||
exactKeys(input.rollbackDrill as unknown as Record<string, unknown>, [
|
||||
'performedAt',
|
||||
'recordDigest',
|
||||
'result',
|
||||
]);
|
||||
exactKeys(input.readiness as unknown as Record<string, unknown>, ['checks', 'ready']);
|
||||
exactKeys(input.shadow as unknown as Record<string, unknown>, [
|
||||
'current',
|
||||
'differences',
|
||||
'inputDigest',
|
||||
'legacy',
|
||||
'matches',
|
||||
]);
|
||||
if (
|
||||
!SHA.test(input.releaseSha) ||
|
||||
!DIGEST.test(input.webAssetDigest) ||
|
||||
!DIGEST.test(input.databaseBackupDigest)
|
||||
)
|
||||
throw new Error('Unsafe evidence digest');
|
||||
if (
|
||||
!ISO.test(input.rollbackDrill.performedAt) ||
|
||||
input.rollbackDrill.result !== 'passed' ||
|
||||
!DIGEST.test(input.rollbackDrill.recordDigest)
|
||||
)
|
||||
throw new Error('Unsafe evidence rollback drill');
|
||||
if (!safeCount(input.pendingCredentialCount)) throw new Error('Unsafe evidence credential count');
|
||||
for (const [key, value] of Object.entries(input.readiness.checks)) {
|
||||
if (!/^[a-zA-Z][a-zA-Z0-9]{0,63}$/u.test(key) || !SAFE_VALUE.test(value))
|
||||
throw new Error('Unsafe evidence readiness value');
|
||||
}
|
||||
if (!DIGEST.test(input.shadow.inputDigest)) throw new Error('Unsafe evidence shadow digest');
|
||||
validateProjection(input.shadow.legacy);
|
||||
validateProjection(input.shadow.current);
|
||||
if (input.shadow.differences.some((value) => !SAFE_VALUE.test(value)))
|
||||
throw new Error('Unsafe evidence difference');
|
||||
if (input.recordedAt !== undefined && !ISO.test(input.recordedAt))
|
||||
throw new Error('Unsafe evidence timestamp');
|
||||
}
|
||||
|
||||
export async function sha256File(path: string): Promise<string> {
|
||||
const metadata = await stat(path);
|
||||
if (!metadata.isFile()) throw new Error('Digest input must be a regular file');
|
||||
return createHash('sha256')
|
||||
.update(await readFile(path))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function parseRecords(text: string): ReleaseEvidenceRecord[] {
|
||||
if (text === '') return [];
|
||||
const lines = text.split('\n');
|
||||
if (lines.at(-1) !== '') throw new Error('Evidence log is not newline terminated');
|
||||
lines.pop();
|
||||
return lines.map((line) => JSON.parse(line) as ReleaseEvidenceRecord);
|
||||
}
|
||||
function validateRecord(
|
||||
record: ReleaseEvidenceRecord,
|
||||
index: number,
|
||||
previous: string | null,
|
||||
): boolean {
|
||||
const { recordDigest, ...unsigned } = record;
|
||||
try {
|
||||
exactKeys(record as unknown as Record<string, unknown>, [
|
||||
'databaseBackupDigest',
|
||||
'pendingCredentialCount',
|
||||
'previousDigest',
|
||||
'readiness',
|
||||
'recordDigest',
|
||||
'recordedAt',
|
||||
'releaseSha',
|
||||
'rollbackDrill',
|
||||
'sequence',
|
||||
'shadow',
|
||||
'webAssetDigest',
|
||||
]);
|
||||
const evidenceInput = {
|
||||
releaseSha: record.releaseSha,
|
||||
webAssetDigest: record.webAssetDigest,
|
||||
databaseBackupDigest: record.databaseBackupDigest,
|
||||
rollbackDrill: record.rollbackDrill,
|
||||
pendingCredentialCount: record.pendingCredentialCount,
|
||||
readiness: record.readiness,
|
||||
shadow: record.shadow,
|
||||
recordedAt: record.recordedAt,
|
||||
};
|
||||
validateInput(evidenceInput);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
record.sequence === index + 1 &&
|
||||
record.previousDigest === previous &&
|
||||
DIGEST.test(recordDigest) &&
|
||||
digest(unsigned) === recordDigest
|
||||
);
|
||||
}
|
||||
export async function verifyReleaseEvidenceLog(path: string): Promise<{
|
||||
readonly valid: boolean;
|
||||
readonly records: number;
|
||||
readonly lastDigest: string | null;
|
||||
}> {
|
||||
try {
|
||||
const records = parseRecords(await readFile(path, 'utf8'));
|
||||
let previous: string | null = null;
|
||||
for (const [index, record] of records.entries()) {
|
||||
if (!validateRecord(record, index, previous))
|
||||
return { valid: false, records: index, lastDigest: previous };
|
||||
previous = record.recordDigest;
|
||||
}
|
||||
return { valid: true, records: records.length, lastDigest: previous };
|
||||
} catch {
|
||||
return { valid: false, records: 0, lastDigest: null };
|
||||
}
|
||||
}
|
||||
|
||||
export async function appendReleaseEvidence(
|
||||
path: string,
|
||||
input: ReleaseEvidenceInput,
|
||||
): Promise<ReleaseEvidenceRecord> {
|
||||
validateInput(input);
|
||||
const absolute = resolve(path);
|
||||
const parent = dirname(absolute);
|
||||
const parentInfo = await stat(parent);
|
||||
if (!parentInfo.isDirectory() || parentInfo.isSymbolicLink())
|
||||
throw new Error('Unsafe evidence directory');
|
||||
const lockPath = `${absolute}.lock`;
|
||||
let lock;
|
||||
try {
|
||||
lock = await open(lockPath, 'wx', 0o600);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'EEXIST')
|
||||
throw new Error('Release evidence log is locked by another writer');
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
let existing = '';
|
||||
try {
|
||||
const info = await stat(absolute);
|
||||
if (!info.isFile() || info.isSymbolicLink() || (info.mode & 0o077) !== 0)
|
||||
throw new Error('Unsafe evidence log');
|
||||
existing = await readFile(absolute, 'utf8');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
const records = parseRecords(existing);
|
||||
let previous: string | null = null;
|
||||
for (const [index, record] of records.entries()) {
|
||||
if (!validateRecord(record, index, previous)) throw new Error('Unsafe evidence log chain');
|
||||
previous = record.recordDigest;
|
||||
}
|
||||
const unsigned = {
|
||||
...input,
|
||||
recordedAt: input.recordedAt ?? new Date().toISOString(),
|
||||
sequence: records.length + 1,
|
||||
previousDigest: previous,
|
||||
};
|
||||
const record: ReleaseEvidenceRecord = { ...unsigned, recordDigest: digest(unsigned) };
|
||||
const temporary = `${absolute}.${process.pid}.${Date.now()}.tmp`;
|
||||
const handle = await open(temporary, 'wx', 0o600);
|
||||
try {
|
||||
await handle.writeFile(`${existing}${canonical(record)}\n`, 'utf8');
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
await rename(temporary, absolute);
|
||||
const directory = await open(parent, 'r');
|
||||
try {
|
||||
await directory.sync();
|
||||
} finally {
|
||||
await directory.close();
|
||||
}
|
||||
return Object.freeze(record);
|
||||
} finally {
|
||||
await lock.close();
|
||||
await rm(lockPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function projectionFromLegacy(value: unknown): SafeProjection {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value))
|
||||
throw new Error('Invalid shadow snapshot');
|
||||
const instances = (value as { instances?: unknown }).instances;
|
||||
if (!Array.isArray(instances)) throw new Error('Invalid shadow snapshot');
|
||||
const statuses: Record<string, number> = {};
|
||||
let enabled = 0;
|
||||
for (const item of instances) {
|
||||
if (item === null || typeof item !== 'object' || Array.isArray(item))
|
||||
throw new Error('Invalid shadow snapshot');
|
||||
const record = item as Record<string, unknown>;
|
||||
if (
|
||||
typeof record.enabled !== 'boolean' ||
|
||||
typeof record.status !== 'string' ||
|
||||
!/^[a-z0-9_-]{1,64}$/u.test(record.status)
|
||||
)
|
||||
throw new Error('Invalid shadow snapshot');
|
||||
if (record.enabled) enabled += 1;
|
||||
statuses[record.status] = (statuses[record.status] ?? 0) + 1;
|
||||
}
|
||||
return { instances: { total: instances.length, enabled }, statuses };
|
||||
}
|
||||
function projectionFromDatabase(db: Database.Database): SafeProjection {
|
||||
const instances = db
|
||||
.prepare('SELECT COUNT(*) AS total, COALESCE(SUM(enabled), 0) AS enabled FROM instances')
|
||||
.get() as { total: number; enabled: number };
|
||||
const rows = db
|
||||
.prepare('SELECT state, COUNT(*) AS count FROM status_snapshots GROUP BY state')
|
||||
.all() as Array<{ state: string; count: number }>;
|
||||
const statuses: Record<string, number> = {};
|
||||
for (const row of rows) {
|
||||
if (!/^[a-z0-9_-]{1,64}$/u.test(row.state) || !safeCount(row.count))
|
||||
throw new Error('Unsafe database status');
|
||||
statuses[row.state] = row.count;
|
||||
}
|
||||
return { instances: { total: instances.total, enabled: instances.enabled }, statuses };
|
||||
}
|
||||
export async function compareShadowSnapshot(
|
||||
path: string,
|
||||
db: Database.Database,
|
||||
): Promise<ShadowComparison> {
|
||||
const inputDigest = await sha256File(path);
|
||||
const legacy = projectionFromLegacy(JSON.parse(await readFile(path, 'utf8')) as unknown);
|
||||
const current = projectionFromDatabase(db);
|
||||
const differences: string[] = [];
|
||||
if (legacy.instances.total !== current.instances.total)
|
||||
differences.push('instance total differs');
|
||||
if (legacy.instances.enabled !== current.instances.enabled)
|
||||
differences.push('enabled instance count differs');
|
||||
if (canonical(legacy.statuses) !== canonical(current.statuses))
|
||||
differences.push('status counts differ');
|
||||
return Object.freeze({
|
||||
inputDigest,
|
||||
matches: differences.length === 0,
|
||||
legacy,
|
||||
current,
|
||||
differences,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user