feat(migration): add verified import and rollback workflow

This commit is contained in:
chick
2026-07-18 14:13:11 +08:00
parent 293c015f89
commit ae27b522ab
12 changed files with 902 additions and 44 deletions
+2 -1
View File
@@ -8,7 +8,8 @@
"test": "vitest run --root ../.. apps/api/src",
"typecheck": "tsc -p tsconfig.json",
"canary": "node --experimental-strip-types src/canary-cli.ts",
"start:production": "node --experimental-strip-types src/production-cli.ts"
"start:production": "node --experimental-strip-types src/production-cli.ts",
"migration": "node --experimental-strip-types src/application/legacy-import/migration-command.ts"
},
"dependencies": {
"@multi-simadmin/contracts": "workspace:*",
@@ -103,6 +103,7 @@ describe('legacy import', () => {
description: 'primary',
tags: ['prod', 'blue'],
capabilities: ['status.read', 'sim.restart'],
authMode: 'password',
credentialPresent: true,
status: 'planned',
});
@@ -142,6 +143,15 @@ describe('legacy import', () => {
expect(scan(error)).not.toContain(password);
});
it('never includes a malformed source excerpt in an error', async () => {
const path = await legacyFile({});
const marker = 'malformed-secret-never-echo';
await writeFile(path, `{"instances":[],"password":"${marker}" trailing`);
const error = await rejection(() => previewLegacyImport(path));
expect(error.message).toBe('Legacy config is not valid JSON');
expect(scan(error)).not.toContain(marker);
});
it('rejects symlinks and non-regular files', async () => {
const path = await legacyFile(valid);
const link = `${path}.link`;
@@ -243,6 +253,23 @@ describe('legacy import', () => {
await expect(confirmLegacyImport(fresh, tampered, db)).rejects.toThrow(/digest/i);
});
it('binds status and auth mode into the execution plan digest', async () => {
const path = await legacyFile(valid);
const db = database();
const preview = await previewLegacyImport(path, db);
const instance = preview.instances[0]!;
const statusTampered = {
...preview,
instances: [{ ...instance, status: 'unchanged' as const }, ...preview.instances.slice(1)],
} as LegacyImportPreview;
await expect(confirmLegacyImport(path, statusTampered, db)).rejects.toThrow(/digest/i);
const authTampered = {
...preview,
instances: [{ ...instance, authMode: 'none' as const }, ...preview.instances.slice(1)],
} as LegacyImportPreview;
await expect(confirmLegacyImport(path, authTampered, db)).rejects.toThrow(/digest/i);
});
it('reports conflicts and confirmation rejects the entire transaction', async () => {
const path = await legacyFile(valid);
const db = database();
@@ -14,6 +14,7 @@ export interface LegacyImportInstancePreview {
readonly tags: readonly string[];
readonly capabilities: readonly string[];
readonly credentialPresent: boolean;
readonly authMode: 'none' | 'password';
readonly status: LegacyImportStatus;
}
@@ -41,9 +42,9 @@ export interface LegacyImportResult {
readonly planDigest: string;
}
interface NormalizedInstance extends Omit<LegacyImportInstancePreview, 'status'> {
readonly authMode: 'none' | 'password';
}
export type LegacyImportTransactionHook = (result: LegacyImportResult) => void;
interface NormalizedInstance extends Omit<LegacyImportInstancePreview, 'status'> {}
interface SourceRead {
readonly bytes: Buffer;
@@ -142,7 +143,7 @@ function parseJson(bytes: Buffer): unknown {
});
} catch (error) {
if (error instanceof LegacyImportError) throw error;
throw new LegacyImportError(`Legacy config is not valid JSON: ${(error as Error).message}`);
throw new LegacyImportError('Legacy config is not valid JSON');
}
}
@@ -277,7 +278,10 @@ function statusFor(
: 'conflict';
}
function digestPlan(source: SourceRead, instances: readonly LegacyImportInstancePreview[]): string {
function digestPlan(
source: Pick<SourceRead, 'digest' | 'identity'>,
instances: readonly LegacyImportInstancePreview[],
): string {
const normalizedPlan = instances.map((instance) => ({
id: instance.id,
name: instance.name,
@@ -286,6 +290,8 @@ function digestPlan(source: SourceRead, instances: readonly LegacyImportInstance
tags: instance.tags,
capabilities: instance.capabilities,
credentialPresent: instance.credentialPresent,
authMode: instance.authMode,
status: instance.status,
}));
return sha256(
JSON.stringify({
@@ -302,9 +308,9 @@ export async function previewLegacyImport(
): Promise<LegacyImportPreview> {
const source = await securelyRead(path);
const normalized = normalize(parseJson(source.bytes));
const instances = normalized.map(({ authMode: _authMode, ...item }) => ({
const instances = normalized.map((item) => ({
...item,
status: statusFor(database, { ...item, authMode: _authMode }),
status: statusFor(database, item),
}));
const counts = {
total: instances.length,
@@ -329,7 +335,10 @@ function validateConfirmation(preview: LegacyImportPreview, current: LegacyImpor
preview.source?.identity?.inode !== current.source.identity.inode
)
throw new LegacyImportError('Legacy config source changed since preview');
if (preview.planDigest !== current.planDigest)
if (
preview.planDigest !== digestPlan(preview.source, preview.instances) ||
preview.planDigest !== current.planDigest
)
throw new LegacyImportError('Legacy import plan digest mismatch');
if (current.counts.conflict > 0)
throw new LegacyImportError('Legacy import plan contains database conflicts');
@@ -339,6 +348,7 @@ export async function confirmLegacyImport(
path: string,
preview: LegacyImportPreview,
database: Database.Database,
transactionHook?: LegacyImportTransactionHook,
): Promise<LegacyImportResult> {
const current = await previewLegacyImport(path, database);
validateConfirmation(preview, current);
@@ -356,6 +366,16 @@ export async function confirmLegacyImport(
current.instances.filter(({ status }) => status === 'planned').map(({ id }) => id),
);
const result = deepFreeze({
imported: current.counts.planned,
unchanged: current.counts.unchanged,
secretPending: normalized
.filter(({ credentialPresent }) => credentialPresent)
.map(({ id }) => id),
sourceDigest: current.source.digest,
planDigest: current.planDigest,
});
database.transaction(() => {
const insertInstance = database.prepare(
'INSERT INTO instances (id,name,base_url,auth_mode,enabled,config_revision,created_at,updated_at) VALUES (?,?,?,?,1,1,?,?)',
@@ -379,15 +399,8 @@ export async function confirmLegacyImport(
for (const tag of item.tags) insertTag.run(item.id, tag, now);
insertMetadata.run(metadataKey(item.id), metadata(item), now, now);
}
transactionHook?.(result);
})();
return deepFreeze({
imported: current.counts.planned,
unchanged: current.counts.unchanged,
secretPending: normalized
.filter(({ credentialPresent }) => credentialPresent)
.map(({ id }) => id),
sourceDigest: current.source.digest,
planDigest: current.planDigest,
});
return result;
}
@@ -0,0 +1,144 @@
import Database from 'better-sqlite3';
import { access, mkdtemp, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, it, vi } from 'vitest';
import { runMigrationCli } from './migration-cli.js';
async function fixture() {
const root = await mkdtemp(join(tmpdir(), 'phase9-cli-'));
const source = join(root, 'legacy.json');
const database = join(root, 'app.sqlite');
await writeFile(
source,
JSON.stringify({
instances: [
{
id: 'alpha',
url: 'https://203.0.113.8',
auth: { mode: 'password', password: 'never-print-this-secret' },
},
],
}),
{ mode: 0o600 },
);
return { root, source, database };
}
describe('operator migration CLI', () => {
it('does not create or migrate a missing target during preview', async () => {
const paths = await fixture();
await runMigrationCli(['preview', '--source', paths.source, '--database', paths.database], {
stdout: vi.fn(),
});
await expect(access(paths.database)).rejects.toThrow();
});
it('previews redacted JSON and requires digest-bound explicit confirmation', async () => {
const paths = await fixture();
const output: string[] = [];
expect(
await runMigrationCli(['preview', '--source', paths.source, '--database', paths.database], {
stdout: (line) => output.push(line),
}),
).toBe(0);
const preview = JSON.parse(output.join('')) as { planDigest: string };
expect(output.join('')).not.toContain('never-print-this-secret');
await expect(
runMigrationCli(
[
'confirm',
'--source',
paths.source,
'--database',
paths.database,
'--plan-digest',
'0'.repeat(64),
],
{ stdout: vi.fn() },
),
).rejects.toThrow(/digest/i);
output.length = 0;
expect(
await runMigrationCli(
[
'confirm',
'--source',
paths.source,
'--database',
paths.database,
'--plan-digest',
preview.planDigest,
],
{ stdout: (line) => output.push(line) },
),
).toBe(0);
expect(output.join('')).not.toContain('never-print-this-secret');
const db = new Database(paths.database);
expect(db.prepare('SELECT id FROM instances').all()).toEqual([{ id: 'alpha' }]);
const release = db
.prepare("SELECT value_json FROM app_settings WHERE key LIKE 'release.legacy-import.%'")
.get() as { value_json: string };
expect(JSON.parse(release.value_json)).toMatchObject({
kind: 'legacy-import',
planDigest: preview.planDigest,
imported: 1,
reconciliation: { status: 'pending-secret-activation', pendingSecretCount: 1 },
});
db.close();
});
it('is idempotent and records a reconciled second release', async () => {
const paths = await fixture();
const collect = async (args: string[]) => {
let text = '';
await runMigrationCli(args, { stdout: (line) => (text += line) });
return JSON.parse(text) as { planDigest: string };
};
const preview = await collect([
'preview',
'--source',
paths.source,
'--database',
paths.database,
]);
await collect([
'confirm',
'--source',
paths.source,
'--database',
paths.database,
'--plan-digest',
preview.planDigest,
]);
const second = await collect([
'preview',
'--source',
paths.source,
'--database',
paths.database,
]);
const result = await collect([
'confirm',
'--source',
paths.source,
'--database',
paths.database,
'--plan-digest',
second.planDigest,
]);
expect(result).toMatchObject({ imported: 0, unchanged: 1 });
const db = new Database(paths.database, { readonly: true });
const releases = db
.prepare(
"SELECT value_json FROM app_settings WHERE key LIKE 'release.legacy-import.%' ORDER BY key",
)
.all() as { value_json: string }[];
expect(JSON.parse(releases.at(-1)!.value_json)).toMatchObject({
reconciliation: { status: 'pending-secret-activation', pendingSecretCount: 1 },
});
db.close();
});
});
@@ -0,0 +1,82 @@
import Database from 'better-sqlite3';
import { randomUUID } from 'node:crypto';
import { access } from 'node:fs/promises';
import { resolve } from 'node:path';
import { migrateDatabase } from '../../infrastructure/database/migrations.js';
import { openDatabase } from '../../infrastructure/database/database.js';
import { confirmLegacyImport, previewLegacyImport } from './legacy-import.js';
export interface MigrationCliIo {
readonly stdout: (text: string) => void;
}
function option(args: readonly string[], name: string): string {
const index = args.indexOf(name);
const value = index < 0 ? undefined : args[index + 1];
if (!value || value.startsWith('--')) throw new Error(`Required option missing: ${name}`);
return value;
}
function print(io: MigrationCliIo, value: unknown): void {
io.stdout(`${JSON.stringify(value, null, 2)}\n`);
}
export async function runMigrationCli(
args: readonly string[],
io: MigrationCliIo = { stdout: (text) => process.stdout.write(text) },
): Promise<number> {
const command = args[0];
if (command !== 'preview' && command !== 'confirm')
throw new Error(
'Usage: migration-cli preview|confirm --source PATH --database PATH [--plan-digest SHA256]',
);
const sourcePath = resolve(option(args, '--source'));
const databasePath = resolve(option(args, '--database'));
if (command === 'preview') {
let database: Database.Database | undefined;
try {
await access(databasePath);
database = new Database(databasePath, { readonly: true, fileMustExist: true });
print(io, await previewLegacyImport(sourcePath, database));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
print(io, await previewLegacyImport(sourcePath));
} finally {
database?.close();
}
return 0;
}
const database = openDatabase(databasePath);
try {
migrateDatabase(database);
const preview = await previewLegacyImport(sourcePath, database);
const confirmedDigest = option(args, '--plan-digest');
if (!/^[a-f0-9]{64}$/.test(confirmedDigest) || confirmedDigest !== preview.planDigest)
throw new Error('Legacy import plan digest mismatch; run preview again');
const result = await confirmLegacyImport(sourcePath, preview, database, (confirmed) => {
const now = new Date().toISOString();
const release = {
kind: 'legacy-import',
sourceDigest: confirmed.sourceDigest,
planDigest: confirmed.planDigest,
imported: confirmed.imported,
unchanged: confirmed.unchanged,
reconciliation: {
status: confirmed.secretPending.length > 0 ? 'pending-secret-activation' : 'reconciled',
pendingSecretCount: confirmed.secretPending.length,
},
recordedAt: now,
};
database
.prepare('INSERT INTO app_settings (key,value_json,created_at,updated_at) VALUES (?,?,?,?)')
.run(`release.legacy-import.${now}.${randomUUID()}`, JSON.stringify(release), now, now);
});
print(io, result);
return 0;
} finally {
database.close();
}
}
@@ -0,0 +1,10 @@
#!/usr/bin/env node
import { runMigrationCli } from './migration-cli.js';
try {
process.exitCode = await runMigrationCli(process.argv.slice(2));
} catch (error) {
// Import errors are deliberately content-free; never print source JSON or credentials.
process.stderr.write(`${error instanceof Error ? error.message : 'Migration failed'}\n`);
process.exitCode = 1;
}
+9 -1
View File
@@ -16,7 +16,13 @@ export {
createCanaryGateway,
} from './canary-gateway.js';
export type { CanaryGateway, CanaryGatewayOptions } from './canary-gateway.js';
export { backupDatabase, restoreDatabase } from './infrastructure/database/backup.js';
export {
activateRestoreCandidate,
backupDatabase,
prepareRestoreCandidate,
runRollbackDrill,
} from './infrastructure/database/backup.js';
export type { DatabaseArtifact, RollbackDrillRecord } from './infrastructure/database/backup.js';
export { openDatabase } from './infrastructure/database/database.js';
export {
LegacyImportError,
@@ -29,6 +35,8 @@ export type {
LegacyImportResult,
LegacyImportStatus,
} from './application/legacy-import/legacy-import.js';
export { runMigrationCli } from './application/legacy-import/migration-cli.js';
export type { MigrationCliIo } from './application/legacy-import/migration-cli.js';
export {
ActivationError,
activatePendingSecret,
@@ -0,0 +1,181 @@
import { execFile } from 'node:child_process';
import Database from 'better-sqlite3';
import { chmod, mkdtemp, open, readFile, rename, stat, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { promisify } from 'node:util';
import { afterEach, describe, expect, it } from 'vitest';
import {
activateRestoreCandidate,
backupDatabase,
prepareRestoreCandidate,
runRollbackDrill,
} from './backup.js';
import { openDatabase, type SqliteDatabase } from './database.js';
import { migrateDatabase } from './migrations.js';
const databases: SqliteDatabase[] = [];
const execFileAsync = promisify(execFile);
afterEach(() => {
for (const database of databases.splice(0)) if (database.open) database.close();
});
async function fixture() {
const root = await mkdtemp(join(tmpdir(), 'phase9-backup-'));
const livePath = join(root, 'live.sqlite');
const backupPath = join(root, 'snapshots', 'before.sqlite');
const database = openDatabase(livePath);
databases.push(database);
migrateDatabase(database);
database
.prepare('INSERT INTO app_settings (key,value_json,created_at,updated_at) VALUES (?,?,?,?)')
.run('fixture', '"before"', '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z');
return { root, livePath, backupPath, database };
}
describe('verified backup, restore, and rollback foundation', () => {
it('creates a private WAL-safe snapshot containing committed WAL rows', async () => {
const { database, backupPath } = await fixture();
database
.prepare('INSERT INTO app_settings (key,value_json,created_at,updated_at) VALUES (?,?,?,?)')
.run('wal', '"in-wal"', '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z');
const snapshot = await backupDatabase(database, backupPath);
const copy = new Database(backupPath, { readonly: true });
expect(copy.prepare('SELECT value_json FROM app_settings ORDER BY rowid').all()).toEqual([
{ value_json: '"before"' },
{ value_json: '"in-wal"' },
]);
copy.close();
expect(snapshot.sha256).toMatch(/^[a-f0-9]{64}$/);
expect((await stat(backupPath)).mode & 0o777).toBe(0o600);
});
it('verifies a separate candidate and digest-binds activation', async () => {
const { livePath, backupPath, database } = await fixture();
const snapshot = await backupDatabase(database, backupPath);
database.exec('UPDATE app_settings SET value_json = \'"after"\'');
database.close();
const candidate = await prepareRestoreCandidate(backupPath, `${livePath}.candidate`);
expect(candidate.sha256).toBe(snapshot.sha256);
await writeFile(backupPath, 'not sqlite');
await expect(activateRestoreCandidate(livePath, candidate, '0'.repeat(64))).rejects.toThrow(
/digest/i,
);
expect(new Database(livePath).prepare('SELECT value_json FROM app_settings').get()).toEqual({
value_json: '"after"',
});
await activateRestoreCandidate(livePath, candidate, candidate.sha256);
expect(new Database(livePath).prepare('SELECT value_json FROM app_settings').get()).toEqual({
value_json: '"before"',
});
});
it('rejects empty SQLite and application databases with missing migrations or FK violations', async () => {
const root = await mkdtemp(join(tmpdir(), 'phase9-invalid-'));
const empty = join(root, 'empty.sqlite');
new Database(empty).close();
await expect(prepareRestoreCandidate(empty, join(root, 'candidate'))).rejects.toThrow(
/schema|migration|application/i,
);
const { backupPath, database } = await fixture();
database.pragma('foreign_keys = OFF');
database.exec("INSERT INTO instance_tags VALUES ('missing','bad','now')");
await expect(backupDatabase(database, backupPath)).rejects.toThrow(/foreign key/i);
});
it('rejects an otherwise valid database with any extra sqlite_master object', async () => {
const { root, backupPath, database } = await fixture();
database.exec('CREATE VIEW unexpected_view AS SELECT 1 AS value');
await expect(backupDatabase(database, backupPath)).rejects.toThrow(/schema|unexpected|view/i);
database.exec(
'DROP VIEW unexpected_view; CREATE TRIGGER unexpected_trigger AFTER INSERT ON app_settings BEGIN SELECT 1; END',
);
await expect(backupDatabase(database, join(root, 'trigger.sqlite'))).rejects.toThrow(
/schema|unexpected|trigger/i,
);
});
it('binds activation to the exact staged inode and rejects a staged-path swap', async () => {
const { livePath, backupPath, database } = await fixture();
await backupDatabase(database, backupPath);
database.close();
const candidate = await prepareRestoreCandidate(backupPath, `${livePath}.candidate`);
const before = await readFile(livePath);
await expect(
activateRestoreCandidate(livePath, candidate, candidate.sha256, {
beforeActivation: async (stagedPath) => {
await rename(stagedPath, `${stagedPath}.original`);
await writeFile(stagedPath, await readFile(candidate.path), { mode: 0o400 });
},
}),
).rejects.toThrow(/staged|identity|changed/i);
expect(await readFile(livePath)).toEqual(before);
});
it('rejects in-place staged-byte mutation even when metadata is preserved', async () => {
const { livePath, backupPath, database } = await fixture();
await backupDatabase(database, backupPath);
database.close();
const candidate = await prepareRestoreCandidate(backupPath, `${livePath}.candidate`);
const before = await readFile(livePath);
await expect(
activateRestoreCandidate(livePath, candidate, candidate.sha256, {
beforeActivation: async (stagedPath) => {
const original = await stat(stagedPath);
const originalTimes = await stat(stagedPath, { bigint: true });
await chmod(stagedPath, 0o600);
const handle = await open(stagedPath, 'r+');
try {
const byte = Buffer.alloc(1);
await handle.read(byte, 0, 1, 0);
byte[0] = byte[0]! ^ 0xff;
await handle.write(byte, 0, 1, 0);
await handle.sync();
} finally {
await handle.close();
}
await chmod(stagedPath, original.mode & 0o777);
await execFileAsync('python3', [
'-c',
'import os,sys; os.utime(sys.argv[1], ns=(int(sys.argv[2]), int(sys.argv[3])))',
stagedPath,
originalTimes.atimeNs.toString(),
originalTimes.mtimeNs.toString(),
]);
const restored = await stat(stagedPath);
expect(restored.dev).toBe(original.dev);
expect(restored.ino).toBe(original.ino);
expect(restored.size).toBe(original.size);
expect(restored.mode).toBe(original.mode);
expect(restored.mtimeMs).toBe(original.mtimeMs);
},
}),
).rejects.toThrow(/staged|digest|changed/i);
expect(await readFile(livePath)).toEqual(before);
});
it('runs a deterministic non-activating rollback drill with a reconciliation record', async () => {
const { root, livePath, backupPath, database } = await fixture();
const snapshot = await backupDatabase(database, backupPath);
database.close();
const before = await readFile(livePath);
const first = await runRollbackDrill(backupPath, join(root, 'drill-a'));
const second = await runRollbackDrill(backupPath, join(root, 'drill-b'));
expect(first).toMatchObject({ snapshotSha256: snapshot.sha256, integrity: 'ok' });
expect(second.schemaDigest).toBe(first.schemaDigest);
expect(second.rowCounts).toEqual(first.rowCounts);
expect(second.migrationsDigest).toBe(first.migrationsDigest);
expect(first.reconciliationDigest).toMatch(/^[a-f0-9]{64}$/);
expect(await readFile(livePath)).toEqual(before);
expect(JSON.stringify(first)).not.toContain('before');
});
});
+347 -23
View File
@@ -1,8 +1,36 @@
import Database from 'better-sqlite3';
import { chmod, copyFile, lstat, mkdir, open, rename, rm, stat } from 'node:fs/promises';
import { randomUUID } from 'node:crypto';
import { createHash, randomUUID } from 'node:crypto';
import type { FileHandle } from 'node:fs/promises';
import {
chmod,
copyFile,
lstat,
mkdir,
open,
readFile,
rename,
rm,
stat,
writeFile,
} from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import { beginRestore, getManagedDatabaseIdentity, type SqliteDatabase } from './database.js';
import { MIGRATIONS, migrateDatabase } from './migrations.js';
export interface DatabaseArtifact {
readonly path: string;
readonly sha256: string;
}
export interface RollbackDrillRecord {
readonly kind: 'rollback-drill';
readonly snapshotSha256: string;
readonly integrity: 'ok';
readonly schemaDigest: string;
readonly migrationsDigest: string;
readonly rowCounts: Readonly<Record<string, number>>;
readonly reconciliationDigest: string;
}
async function assertRegularFile(path: string, label: string): Promise<void> {
const details = await lstat(path);
@@ -34,14 +62,102 @@ async function assertValidDatabase(path: string): Promise<void> {
try {
database = new Database(path, { fileMustExist: true, readonly: true });
const integrity = database.pragma('integrity_check', { simple: true });
if (integrity !== 'ok') throw new Error(`Backup integrity check failed: ${String(integrity)}`);
if (integrity !== 'ok')
throw new Error(`Database integrity check failed: ${String(integrity)}`);
assertExpectedSchema(database);
const migrations = database
.prepare('SELECT id,name,checksum FROM schema_migrations ORDER BY id')
.all() as { id: number; name: string; checksum: string }[];
if (migrations.length !== MIGRATIONS.length)
throw new Error('Application migration set incomplete');
for (const [index, migration] of MIGRATIONS.entries()) {
const actual = migrations[index];
const checksum = createHash('sha256')
.update(JSON.stringify(migration.statements))
.digest('hex');
if (
!actual ||
actual.id !== migration.id ||
actual.name !== migration.name ||
actual.checksum !== checksum
)
throw new Error(`Application migration ${migration.id} mismatch`);
}
const violations = database.pragma('foreign_key_check') as unknown[];
if (violations.length) throw new Error('Application database has foreign key violations');
} catch (error) {
throw new Error('Backup is not a valid SQLite database', { cause: error });
throw new Error(
`Artifact is not a valid application SQLite database: ${(error as Error).message}`,
{
cause: error,
},
);
} finally {
database?.close();
}
}
interface SchemaObject {
readonly type: string;
readonly name: string;
readonly tableName: string;
readonly sql: string | null;
}
function readSchema(database: Database.Database): SchemaObject[] {
return database
.prepare(
`SELECT type,name,tbl_name AS tableName,sql
FROM sqlite_master
WHERE name NOT LIKE 'sqlite_%'
ORDER BY type,name`,
)
.all()
.map((object) => {
const value = object as SchemaObject;
return {
...value,
sql: value.sql?.trim().replaceAll(/\s+/g, ' ') ?? null,
};
}) as SchemaObject[];
}
let expectedSchema: readonly SchemaObject[] | undefined;
function getExpectedSchema(): readonly SchemaObject[] {
if (expectedSchema) return expectedSchema;
const reference = new Database(':memory:');
try {
migrateDatabase(reference);
expectedSchema = Object.freeze(readSchema(reference).map((object) => Object.freeze(object)));
return expectedSchema;
} finally {
reference.close();
}
}
function assertExpectedSchema(database: Database.Database): void {
const actual = readSchema(database);
const expected = getExpectedSchema();
if (JSON.stringify(actual) !== JSON.stringify(expected))
throw new Error('Application schema object set mismatch');
}
async function digestFile(path: string): Promise<string> {
return createHash('sha256')
.update(await readFile(path))
.digest('hex');
}
async function digestFileHandle(handle: FileHandle): Promise<string> {
const details = await handle.stat();
const bytes = Buffer.alloc(details.size);
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
if (bytesRead !== bytes.length)
throw new Error('Staged restore candidate could not be read fully');
return createHash('sha256').update(bytes).digest('hex');
}
async function syncFile(path: string): Promise<void> {
const handle = await open(path, 'r');
try {
@@ -60,21 +176,24 @@ async function syncDirectory(path: string): Promise<void> {
}
}
export async function backupDatabase(database: SqliteDatabase, backupPath: string): Promise<void> {
export async function backupDatabase(
database: SqliteDatabase,
backupPath: string,
): Promise<DatabaseArtifact> {
const source = getManagedDatabaseIdentity(database);
const directory = dirname(backupPath);
await mkdir(directory, { recursive: true });
try {
const destination = await stat(backupPath);
if (`${destination.dev}:${destination.ino}` === source.deviceInode) {
if (`${destination.dev}:${destination.ino}` === source.deviceInode)
throw new Error('Backup destination aliases the live database identity');
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
const temporaryPath = `${backupPath}.${randomUUID()}.tmp`;
try {
await reservePrivateFile(temporaryPath);
// SQLite's online backup API includes committed WAL pages in one consistent snapshot.
await database.backup(temporaryPath);
await chmod(temporaryPath, 0o600);
await assertValidDatabase(temporaryPath);
@@ -82,35 +201,240 @@ export async function backupDatabase(database: SqliteDatabase, backupPath: strin
await rename(temporaryPath, backupPath);
await chmod(backupPath, 0o600);
await syncDirectory(directory);
return Object.freeze({ path: backupPath, sha256: await digestFile(backupPath) });
} finally {
await rm(temporaryPath, { force: true });
}
}
export async function restoreDatabase(databasePath: string, backupPath: string): Promise<void> {
const endRestore = beginRestore(databasePath);
const directory = dirname(databasePath);
const temporaryPath = `${databasePath}.${randomUUID()}.restore`;
export async function prepareRestoreCandidate(
backupPath: string,
candidatePath: string,
): Promise<DatabaseArtifact> {
await assertDistinctFiles(candidatePath, backupPath);
await assertValidDatabase(backupPath);
await mkdir(dirname(candidatePath), { recursive: true });
const temporaryPath = `${candidatePath}.${randomUUID()}.tmp`;
try {
await assertDistinctFiles(databasePath, backupPath);
await assertValidDatabase(backupPath);
await mkdir(directory, { recursive: true });
await reservePrivateFile(temporaryPath);
await copyFile(backupPath, temporaryPath);
await chmod(temporaryPath, 0o600);
await assertValidDatabase(temporaryPath);
await syncFile(temporaryPath);
// Atomic replacement comes first: any failure before this point leaves live and sidecars intact.
await rename(temporaryPath, databasePath);
await chmod(databasePath, 0o600);
await syncDirectory(directory);
// No managed connection can exist under the exclusive gate; stale sidecars are now safe to remove.
await rm(`${databasePath}-wal`, { force: true });
await rm(`${databasePath}-shm`, { force: true });
await syncDirectory(directory);
await rename(temporaryPath, candidatePath);
await syncDirectory(dirname(candidatePath));
return Object.freeze({ path: candidatePath, sha256: await digestFile(candidatePath) });
} finally {
await rm(temporaryPath, { force: true });
}
}
export async function activateRestoreCandidate(
databasePath: string,
candidate: DatabaseArtifact,
confirmedDigest: string,
hooks: { readonly beforeActivation?: (stagedPath: string) => Promise<void> } = {},
): Promise<void> {
if (confirmedDigest !== candidate.sha256) throw new Error('Restore candidate digest mismatch');
const endRestore = beginRestore(databasePath);
const directory = dirname(databasePath);
const stagedPath = `${databasePath}.${randomUUID()}.restore`;
try {
await assertDistinctFiles(databasePath, candidate.path);
await mkdir(directory, { recursive: true });
await reservePrivateFile(stagedPath);
const before = await stat(candidate.path);
await copyFile(candidate.path, stagedPath);
const after = await stat(candidate.path);
if (
before.dev !== after.dev ||
before.ino !== after.ino ||
before.size !== after.size ||
before.mtimeMs !== after.mtimeMs
)
throw new Error('Restore candidate changed while being staged');
// Deny path-based writers after population, then hold the exact inode open through
// validation and the final path identity check.
await chmod(stagedPath, 0o400);
const stagedHandle = await open(stagedPath, 'r');
try {
const stagedIdentity = await stagedHandle.stat();
const stagedDigest = await digestFileHandle(stagedHandle);
if (stagedDigest !== candidate.sha256) throw new Error('Restore candidate digest mismatch');
await assertValidDatabase(stagedPath);
if ((await digestFileHandle(stagedHandle)) !== stagedDigest)
throw new Error('Staged restore candidate changed during validation');
await stagedHandle.sync();
await activateStagedDatabase(
databasePath,
stagedPath,
stagedHandle,
stagedIdentity,
confirmedDigest,
hooks.beforeActivation,
);
} finally {
await stagedHandle.close();
}
} finally {
await rm(stagedPath, { force: true });
endRestore();
}
}
/** Internal compatibility helper. Public callers must use prepare + digest-confirmed activation. */
export async function restoreDatabase(databasePath: string, backupPath: string): Promise<void> {
await assertDistinctFiles(databasePath, backupPath);
const candidatePath = `${databasePath}.${randomUUID()}.candidate`;
try {
const candidate = await prepareRestoreCandidate(backupPath, candidatePath);
await activateRestoreCandidate(databasePath, candidate, candidate.sha256);
} finally {
await rm(candidatePath, { force: true });
}
}
interface RestoreMarker {
readonly version: 1;
readonly quarantinedSidecars: readonly string[];
}
async function readPreviousRestoreMarker(markerPath: string): Promise<RestoreMarker | undefined> {
try {
await assertRegularFile(markerPath, 'Restore marker');
const value = JSON.parse(await readFile(markerPath, 'utf8')) as Partial<RestoreMarker>;
if (value.version !== 1 || !Array.isArray(value.quarantinedSidecars))
throw new Error('Restore marker is invalid; manual recovery is required');
return value as RestoreMarker;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
throw error;
}
}
async function writeRestoreMarker(markerPath: string, marker: RestoreMarker): Promise<void> {
const temporary = `${markerPath}.${randomUUID()}.tmp`;
try {
await writeFile(temporary, JSON.stringify(marker), { flag: 'wx', mode: 0o600 });
await syncFile(temporary);
await rename(temporary, markerPath);
await syncDirectory(dirname(markerPath));
} finally {
await rm(temporary, { force: true });
}
}
async function activateStagedDatabase(
databasePath: string,
stagedPath: string,
stagedHandle: FileHandle,
stagedIdentity: Awaited<ReturnType<FileHandle['stat']>>,
confirmedDigest: string,
beforeActivation?: (stagedPath: string) => Promise<void>,
): Promise<void> {
const directory = dirname(databasePath);
const markerPath = `${databasePath}.restore-in-progress`;
const previous = await readPreviousRestoreMarker(markerPath);
const transactionId = randomUUID();
const sidecars = [`${databasePath}-wal`, `${databasePath}-shm`];
const quarantined = sidecars.map((path) => `${path}.${transactionId}.quarantine`);
const allQuarantined = [...(previous?.quarantinedSidecars ?? []), ...quarantined];
await writeRestoreMarker(markerPath, { version: 1, quarantinedSidecars: allQuarantined });
try {
// Make old WAL state unreachable before replacing the main file. The durable marker
// forces managed opens to fail closed at every crash boundary.
for (const [index, sidecar] of sidecars.entries()) {
try {
await rename(sidecar, quarantined[index]!);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
}
await syncDirectory(directory);
await beforeActivation?.(stagedPath);
if ((await digestFileHandle(stagedHandle)) !== confirmedDigest)
throw new Error('Staged restore candidate digest changed before activation');
const [held, path] = await Promise.all([stagedHandle.stat(), lstat(stagedPath)]);
if (
!path.isFile() ||
path.isSymbolicLink() ||
(path.mode & 0o222) !== 0 ||
held.dev !== stagedIdentity.dev ||
held.ino !== stagedIdentity.ino ||
held.size !== stagedIdentity.size ||
held.mtimeMs !== stagedIdentity.mtimeMs ||
path.dev !== stagedIdentity.dev ||
path.ino !== stagedIdentity.ino ||
path.size !== stagedIdentity.size ||
path.mtimeMs !== stagedIdentity.mtimeMs
)
throw new Error('Staged restore candidate identity or bytes changed before activation');
await rename(stagedPath, databasePath);
await chmod(databasePath, 0o600);
await syncDirectory(directory);
for (const path of allQuarantined) await rm(path, { force: true });
await rm(markerPath);
await syncDirectory(directory);
} catch (error) {
throw new Error(
`Restore activation did not complete; ${markerPath} was retained to force recovery: ${(error as Error).message}`,
{ cause: error },
);
}
}
export async function runRollbackDrill(
backupPath: string,
candidatePath: string,
): Promise<RollbackDrillRecord> {
const candidate = await prepareRestoreCandidate(backupPath, candidatePath);
let database: Database.Database | undefined;
try {
database = new Database(candidate.path, { readonly: true, fileMustExist: true });
const integrity = database.pragma('integrity_check', { simple: true });
if (integrity !== 'ok') throw new Error('Rollback drill integrity check failed');
const schema = database
.prepare(
"SELECT type,name,tbl_name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
)
.all();
const tables = database
.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
)
.all() as { name: string }[];
const rowCounts: Record<string, number> = {};
for (const { name } of tables) {
const quoted = `"${name.replaceAll('"', '""')}"`;
rowCounts[name] = (
database.prepare(`SELECT COUNT(*) AS count FROM ${quoted}`).get() as { count: number }
).count;
}
const schemaDigest = createHash('sha256').update(JSON.stringify(schema)).digest('hex');
const migrations = database
.prepare('SELECT id,name,checksum FROM schema_migrations ORDER BY id')
.all();
const migrationsDigest = createHash('sha256').update(JSON.stringify(migrations)).digest('hex');
const reconciliationDigest = createHash('sha256')
.update(
JSON.stringify({
snapshotSha256: candidate.sha256,
schemaDigest,
migrationsDigest,
rowCounts,
}),
)
.digest('hex');
return Object.freeze({
kind: 'rollback-drill',
snapshotSha256: candidate.sha256,
integrity: 'ok',
schemaDigest,
migrationsDigest,
rowCounts: Object.freeze(rowCounts),
reconciliationDigest,
});
} finally {
database?.close();
}
}
@@ -277,6 +277,24 @@ describe('database migrations', () => {
});
describe('database backup and restore', () => {
it('honours a canonical restore marker when opening through a symlinked parent alias', async () => {
const directory = await temporaryDirectory();
const realDirectory = join(directory, 'real');
const aliasDirectory = join(directory, 'alias');
await mkdir(realDirectory);
const databasePath = join(realDirectory, 'app.sqlite');
const database = openDatabase(databasePath);
migrateDatabase(database);
database.close();
await writeFile(
`${databasePath}.restore-in-progress`,
JSON.stringify({ version: 1, quarantinedSidecars: [] }),
);
await symlink(realDirectory, aliasDirectory);
expect(() => openDatabase(join(aliasDirectory, 'app.sqlite'))).toThrow(/restore|recovery/i);
});
it('refuses restore while a managed live connection is open', async () => {
const directory = await temporaryDirectory();
const databasePath = join(directory, 'app.sqlite');
@@ -1,5 +1,5 @@
import Database from 'better-sqlite3';
import { mkdirSync, realpathSync, statSync } from 'node:fs';
import { existsSync, mkdirSync, realpathSync, statSync } from 'node:fs';
import { basename, dirname, join, resolve } from 'node:path';
export type SqliteDatabase = Database.Database;
@@ -77,7 +77,11 @@ export function beginRestore(path: string): () => void {
export function openDatabase(path: string): SqliteDatabase {
const absolute = resolve(path);
mkdirSync(dirname(absolute), { recursive: true });
const preOpenPath = canonicalPath(absolute);
const preOpenPath = existsSync(absolute)
? identityForExistingPath(absolute).canonicalPath
: canonicalPath(absolute);
if (existsSync(`${preOpenPath}.restore-in-progress`))
throw new Error('Database restore recovery is required before opening this database');
if (restorePaths.has(preOpenPath))
throw new Error('Cannot open database while restore is in progress');