import type Database from 'better-sqlite3'; import type { EventEnvelope } from '../events/event-journal.js'; import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; import type { Job, Preparation, PrepareOperationRequest } from '@multi-simadmin/contracts'; import { InstanceService, InstanceServiceError } from '../instances/instance-service.js'; export const DELETE_INSTANCE_PARAMETER_SCHEMA_ID = 'deleteInstance.parameters.v1' as const; const OPERATION_ID = 'deleteInstance'; const ACTOR = 'loopback-control-plane'; /** Job item code for "the device still holds its central binding", surfaced to the console. */ export const BINDING_REFUSAL_CODE = 'UNBIND_FAILED' as const; const TTL_MS = 5 * 60 * 1000; const PARAMETERS_DIGEST = createHash('sha256') .update(JSON.stringify({ parameterSchemaId: DELETE_INSTANCE_PARAMETER_SCHEMA_ID, fields: [] })) .digest('hex'); export type DeleteInstanceOperationErrorCode = | 'VALIDATION_FAILED' | 'NOT_FOUND' | 'REVISION_CONFLICT' | 'CONFIRMATION_INVALID'; export class DeleteInstanceOperationError extends Error { constructor( readonly code: DeleteInstanceOperationErrorCode, message: string, ) { super(message); this.name = 'DeleteInstanceOperationError'; } } interface PreparationRow { operation_id: string; risk_level: string; status: string; target_instance_id: string; target_revision: number; target_origin: string; method: string; path: string; canonical_query: string; body_digest: string; content_type: string; parameter_schema_id: string; parameters_digest: string; nonce: string; token_digest: string; expires_at: string; } interface ExecuteInput { readonly instanceId: string; readonly revision: number; readonly preparationId: string; readonly confirmationToken: string; readonly actor: string; readonly requestId: string; } /** * Answer to "release the central binding on this device before we forget it". * `skipped` means the node never answered a heartbeat, so there is nobody to notify and the * console deletes its own record; the device re-binds if it ever comes back. `failed` keeps the * record, because a device that refused or timed out must not be silently orphaned. */ export type BindingRelease = | { readonly status: 'released' } | { readonly status: 'skipped' } | { readonly status: 'failed' }; interface Options { readonly db: Database.Database; readonly instances: InstanceService; /** Optional so a control plane without a device transport can still delete. */ readonly releaseBinding?: (instanceId: string, requestId: string) => Promise; readonly now?: () => Date; readonly idFactory?: () => string; readonly tokenFactory?: () => string; /** Receives job/instance envelopes so the SSE journal can invalidate fleet views. */ readonly emit?: (envelope: EventEnvelope) => void; } const digest = (value: string): string => createHash('sha256').update(value, 'utf8').digest('hex'); const equalDigest = (left: string, right: string): boolean => { const a = Buffer.from(left, 'hex'); const b = Buffer.from(right, 'hex'); return a.length === 32 && b.length === 32 && timingSafeEqual(a, b); }; const validRevision = (value: unknown): value is number => typeof value === 'number' && Number.isSafeInteger(value) && value > 0; export class DeleteInstanceOperation { private readonly db: Database.Database; private readonly instances: InstanceService; private readonly releaseBinding: Options['releaseBinding']; private readonly clock: () => Date; private readonly id: () => string; private readonly token: () => string; private readonly emit: Options['emit']; constructor(options: Options) { this.db = options.db; this.instances = options.instances; this.releaseBinding = options.releaseBinding; this.clock = options.now ?? (() => new Date()); this.id = options.idFactory ?? randomUUID; this.token = options.tokenFactory ?? (() => randomBytes(32).toString('base64url')); this.emit = options.emit; } async prepare(input: PrepareOperationRequest, requestId = this.id()): Promise { if ( !input || typeof input !== 'object' || Array.isArray(input) || Object.keys(input).some((key) => !['operationId', 'targets', 'parameters'].includes(key)) || input.operationId !== OPERATION_ID || !Array.isArray(input.targets) || input.targets.length !== 1 || !input.targets[0] || typeof input.targets[0] !== 'object' || Array.isArray(input.targets[0]) || Object.keys(input.targets[0]).some((key) => !['instanceId', 'revision'].includes(key)) || typeof input.targets[0].instanceId !== 'string' || input.targets[0].instanceId.length === 0 || !validRevision(input.targets[0].revision) || !input.parameters || typeof input.parameters !== 'object' || Array.isArray(input.parameters) || Object.keys(input.parameters).some((key) => !['parameterSchemaId', 'fields'].includes(key)) || input.parameters.parameterSchemaId !== DELETE_INSTANCE_PARAMETER_SCHEMA_ID || !Array.isArray(input.parameters.fields) || input.parameters.fields.length !== 0 ) { throw new DeleteInstanceOperationError('VALIDATION_FAILED', 'Invalid delete preparation'); } const target = input.targets[0]!; const current = await this.instances.get(target.instanceId); if (!current) throw new DeleteInstanceOperationError('NOT_FOUND', 'Instance was not found'); if (current.revision !== target.revision) throw new DeleteInstanceOperationError( 'REVISION_CONFLICT', 'Instance revision does not match', ); const token = this.token(); let tokenBytes: Buffer; try { tokenBytes = Buffer.from(token, 'base64url'); } catch { tokenBytes = Buffer.alloc(0); } if (tokenBytes.length < 32 || token.length < 20 || !/^[A-Za-z0-9_-]+$/.test(token)) throw new Error('Confirmation token factory returned an unsafe token'); const id = this.id(); const created = this.clock(); const now = created.toISOString(); const expiresAt = new Date(created.getTime() + TTL_MS).toISOString(); this.db .prepare( `INSERT INTO operation_preparations (id,operation_id,risk_level,status,target_instance_id,target_revision,target_origin,method,path, canonical_query,body_digest,content_type,parameter_schema_id,parameters_digest,nonce,token_digest, requested_by,request_id,expires_at,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, ) .run( id, OPERATION_ID, 'R3', 'prepared', target.instanceId, target.revision, current.origin, 'DELETE', `/api/v1/instances/${target.instanceId}`, '', digest(''), '', DELETE_INSTANCE_PARAMETER_SCHEMA_ID, PARAMETERS_DIGEST, `delete-${id}`, digest(token), ACTOR, requestId, expiresAt, now, now, ); return { id, status: 'prepared', operationId: OPERATION_ID, risk: 'R3', expiresAt, confirmationToken: token, confirmationPrompt: `Delete instance ${target.instanceId}? This action cannot be undone.`, targetCount: 1, }; } async executeDelete(input: ExecuteInput): Promise { if (!validRevision(input.revision)) throw new DeleteInstanceOperationError('VALIDATION_FAILED', 'Invalid revision'); const now = this.clock().toISOString(); const ids = { job: this.id(), item: this.id(), attempt: this.id() }; let accepted = false; let targetError: 'NOT_FOUND' | 'REVISION_CONFLICT' | undefined; this.db.transaction(() => { const row = this.db .prepare( `SELECT operation_id,risk_level,status,target_instance_id,target_revision,target_origin, method,path,canonical_query,body_digest,content_type,parameter_schema_id, parameters_digest,nonce,token_digest,expires_at FROM operation_preparations WHERE id=?`, ) .get(input.preparationId) as PreparationRow | undefined; if (!row || row.status !== 'prepared') throw new DeleteInstanceOperationError( 'CONFIRMATION_INVALID', 'Confirmation could not be accepted', ); const confirmationValid = row.expires_at > now && equalDigest(row.token_digest, digest(input.confirmationToken)); if (!confirmationValid) return; const expectedPath = `/api/v1/instances/${input.instanceId}`; const bindingValid = row.operation_id === OPERATION_ID && row.risk_level === 'R3' && row.target_instance_id === input.instanceId && row.target_revision === input.revision && row.method === 'DELETE' && row.path === expectedPath && row.canonical_query === '' && row.body_digest === digest('') && row.content_type === '' && row.parameter_schema_id === DELETE_INSTANCE_PARAMETER_SCHEMA_ID && row.parameters_digest === PARAMETERS_DIGEST && row.nonce === `delete-${input.preparationId}`; if (!bindingValid) { this.db .prepare( "UPDATE operation_preparations SET status='invalidated',consumed_at=?,updated_at=? WHERE id=? AND status='prepared'", ) .run(now, now, input.preparationId); return; } const current = this.db .prepare('SELECT base_url,config_revision FROM instances WHERE id=?') .get(input.instanceId) as { base_url: string; config_revision: number } | undefined; if (current && current.base_url !== row.target_origin) { this.db .prepare( "UPDATE operation_preparations SET status='invalidated',consumed_at=?,updated_at=? WHERE id=? AND status='prepared'", ) .run(now, now, input.preparationId); return; } // A correctly authenticated confirmation is one-shot, including stale/missing target results. this.db .prepare( "UPDATE operation_preparations SET status='consumed',consumed_at=?,updated_at=? WHERE id=? AND status='prepared'", ) .run(now, now, input.preparationId); if (!current) { targetError = 'NOT_FOUND'; return; } if (current.config_revision !== input.revision) { targetError = 'REVISION_CONFLICT'; return; } this.db .prepare( `INSERT INTO jobs (id,parent_job_id,root_job_id,retry_of_job_id,operation_id,risk_level,status,requested_by,request_id,parameters_digest,created_at,started_at,updated_at) VALUES (?,NULL,?,NULL,?,'R3','running',?,?,?,?,?,?)`, ) .run( ids.job, ids.job, OPERATION_ID, input.actor, input.requestId, PARAMETERS_DIGEST, now, now, now, ); this.db .prepare( `INSERT INTO job_attempts (id,job_id,status,started_at,created_at) VALUES (?,?,'running',?,?)`, ) .run(ids.attempt, ids.job, now, now); this.db .prepare( `INSERT INTO job_items (id,job_id,instance_id,attempt_number,status,created_at,started_at,updated_at) VALUES (?,?,?,1,'running',?,?,?)`, ) .run(ids.item, ids.job, input.instanceId, now, now, now); accepted = true; })(); if (targetError) throw new DeleteInstanceOperationError( targetError, targetError === 'NOT_FOUND' ? 'Instance was not found' : 'Instance revision does not match', ); if (!accepted) throw new DeleteInstanceOperationError( 'CONFIRMATION_INVALID', 'Confirmation could not be accepted', ); // An online device is told to release its central binding first, and a refusal keeps the // record so the operator can retry instead of losing sight of a device that is still // reporting to a hub. Only a node that cannot be reached at all is forgotten locally. const release = await this.releaseBindingPort(input.instanceId, input.requestId); if (release?.status === 'failed') { this.closeAsFailed(ids, input.requestId, BINDING_REFUSAL_CODE); this.#emitJob(ids.job, input.requestId); return this.job(ids.job); } try { await this.instances.delete(input.instanceId, input.revision, { allowJobHistory: true }); this.#emit({ kind: 'instance', id: this.id(), occurredAt: this.clock().toISOString(), requestId: input.requestId, instanceId: input.instanceId, }); this.#emitJob(ids.job, input.requestId); const finished = this.clock().toISOString(); this.db.transaction(() => { this.db .prepare( `UPDATE job_items SET status='succeeded',result_code='INSTANCE_DELETED',finished_at=?,updated_at=? WHERE id=? AND status='running'`, ) .run(finished, finished, ids.item); this.db .prepare( "UPDATE job_attempts SET status='succeeded',finished_at=? WHERE id=? AND status='running'", ) .run(finished, ids.attempt); this.db .prepare( "UPDATE jobs SET status='succeeded',finished_at=?,updated_at=? WHERE id=? AND status='running'", ) .run(finished, finished, ids.job); })(); } catch (error) { const finished = this.clock().toISOString(); const state = error instanceof InstanceServiceError ? 'failed' : 'unknown-result'; const code = error instanceof InstanceServiceError ? error.code : 'DELETE_RESULT_UNKNOWN'; this.db.transaction(() => { this.db .prepare( `UPDATE job_items SET status=?,result_code=?,finished_at=?,updated_at=? WHERE id=? AND status='running'`, ) .run(state, code, finished, finished, ids.item); this.db .prepare('UPDATE job_attempts SET status=?,finished_at=? WHERE id=?') .run(state, finished, ids.attempt); this.db .prepare('UPDATE jobs SET status=?,finished_at=?,updated_at=? WHERE id=?') .run(state, finished, finished, ids.job); })(); } return this.job(ids.job); } /** A transport that throws while the node was reachable counts as a refusal, never a pass. */ private async releaseBindingPort( instanceId: string, requestId: string, ): Promise { if (!this.releaseBinding) return undefined; try { return await this.releaseBinding(instanceId, requestId); } catch { return { status: 'failed' }; } } /** Closes the job without touching the instance row, so the device stays listed and retryable. */ #emit(envelope: EventEnvelope): void { if (!this.emit) return; try { this.emit(envelope); } catch { // A journal failure must never fail the operation itself. } } #emitJob(jobId: string, requestId: string): void { this.#emit({ kind: 'job', id: this.id(), occurredAt: this.clock().toISOString(), requestId, jobId, }); } private closeAsFailed( ids: { job: string; item: string; attempt: string }, requestId: string, resultCode: string, ): void { const finished = this.clock().toISOString(); this.db.transaction(() => { this.db .prepare( `UPDATE job_items SET status='failed',result_code=?,error_json=?,finished_at=?,updated_at=? WHERE id=? AND status='running'`, ) .run( resultCode, JSON.stringify({ status: 502, code: resultCode, requestId }), finished, finished, ids.item, ); this.db .prepare( "UPDATE job_attempts SET status='failed',finished_at=? WHERE id=? AND status='running'", ) .run(finished, ids.attempt); this.db .prepare( "UPDATE jobs SET status='failed',finished_at=?,updated_at=? WHERE id=? AND status='running'", ) .run(finished, finished, ids.job); })(); } reconcileInterruptedJobs(): number { const finished = this.clock().toISOString(); return this.db.transaction(() => { const jobs = this.db .prepare("SELECT id FROM jobs WHERE operation_id=? AND status='running'") .all(OPERATION_ID) as Array<{ id: string }>; for (const { id } of jobs) { this.db .prepare( "UPDATE job_items SET status='unknown-result',result_code='INTERRUPTED',finished_at=?,updated_at=? WHERE job_id=? AND status='running'", ) .run(finished, finished, id); this.db .prepare( "UPDATE job_attempts SET status='unknown-result',finished_at=? WHERE job_id=? AND status='running'", ) .run(finished, id); this.db .prepare( "UPDATE jobs SET status='unknown-result',finished_at=?,updated_at=? WHERE id=? AND status='running'", ) .run(finished, finished, id); } return jobs.length; })(); } private job(id: string): Job { const row = this.db .prepare( 'SELECT operation_id,status,root_job_id,retry_of_job_id,created_at,request_id FROM jobs WHERE id=?', ) .get(id) as { operation_id: string; status: Job['status']; root_job_id: string; retry_of_job_id: string | null; created_at: string; request_id: string; }; const items = this.db .prepare( 'SELECT id,instance_id,status,result_code FROM job_items WHERE job_id=? ORDER BY created_at,id', ) .all(id) as Array<{ id: string; instance_id: string; status: 'succeeded' | 'failed' | 'unknown-result'; result_code: string | null; }>; const attempts = this.db .prepare( 'SELECT id,status,started_at,finished_at FROM job_attempts WHERE job_id=? ORDER BY started_at,id', ) .all(id) as Array<{ id: string; status: 'succeeded' | 'failed' | 'unknown-result'; started_at: string; finished_at: string | null; }>; return { id, operationId: row.operation_id, status: row.status, ...(row.retry_of_job_id ? { retryOfJobId: row.retry_of_job_id } : {}), rootJobId: row.root_job_id, items: items.map((item) => ({ id: item.id, targetId: item.instance_id, state: item.status, // The console reads this code straight out of the execute response to explain why the // node it just confirmed is still in the list. ...(item.result_code === BINDING_REFUSAL_CODE ? { error: { type: 'about:blank', title: 'Job item failed', status: 502, detail: 'The job item did not complete successfully.', code: BINDING_REFUSAL_CODE, requestId: row.request_id, }, } : {}), })), attempts: attempts.map((attempt) => ({ id: attempt.id, state: attempt.status, startedAt: attempt.started_at, ...(attempt.finished_at ? { finishedAt: attempt.finished_at } : {}), })), createdAt: row.created_at, }; } }