371 lines
13 KiB
TypeScript
371 lines
13 KiB
TypeScript
import type Database from 'better-sqlite3';
|
|
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
|
|
import type { Job, Preparation, PrepareOperationRequest } from '@multi-simadmin/contracts';
|
|
import { InstanceService, InstanceServiceError } from '../instances/instance-service.js';
|
|
|
|
export const DELETE_INSTANCE_PARAMETER_SCHEMA_ID = 'deleteInstance.parameters.v1' as const;
|
|
const OPERATION_ID = 'deleteInstance';
|
|
const ACTOR = 'loopback-control-plane';
|
|
const TTL_MS = 5 * 60 * 1000;
|
|
const PARAMETERS_DIGEST = createHash('sha256')
|
|
.update(JSON.stringify({ parameterSchemaId: DELETE_INSTANCE_PARAMETER_SCHEMA_ID, fields: [] }))
|
|
.digest('hex');
|
|
|
|
export type DeleteInstanceOperationErrorCode =
|
|
| 'VALIDATION_FAILED'
|
|
| 'NOT_FOUND'
|
|
| 'REVISION_CONFLICT'
|
|
| 'CONFIRMATION_INVALID';
|
|
|
|
export class DeleteInstanceOperationError extends Error {
|
|
constructor(
|
|
readonly code: DeleteInstanceOperationErrorCode,
|
|
message: string,
|
|
) {
|
|
super(message);
|
|
this.name = 'DeleteInstanceOperationError';
|
|
}
|
|
}
|
|
|
|
interface PreparationRow {
|
|
operation_id: string;
|
|
status: string;
|
|
target_instance_id: string;
|
|
target_revision: number;
|
|
parameter_schema_id: string;
|
|
parameters_digest: string;
|
|
token_digest: string;
|
|
expires_at: string;
|
|
}
|
|
interface ExecuteInput {
|
|
readonly instanceId: string;
|
|
readonly revision: number;
|
|
readonly preparationId: string;
|
|
readonly confirmationToken: string;
|
|
readonly actor: string;
|
|
readonly requestId: string;
|
|
}
|
|
interface Options {
|
|
readonly db: Database.Database;
|
|
readonly instances: InstanceService;
|
|
readonly now?: () => Date;
|
|
readonly idFactory?: () => string;
|
|
readonly tokenFactory?: () => string;
|
|
}
|
|
|
|
const digest = (value: string): string => createHash('sha256').update(value, 'utf8').digest('hex');
|
|
const equalDigest = (left: string, right: string): boolean => {
|
|
const a = Buffer.from(left, 'hex');
|
|
const b = Buffer.from(right, 'hex');
|
|
return a.length === 32 && b.length === 32 && timingSafeEqual(a, b);
|
|
};
|
|
const validRevision = (value: unknown): value is number =>
|
|
typeof value === 'number' && Number.isSafeInteger(value) && value > 0;
|
|
|
|
export class DeleteInstanceOperation {
|
|
private readonly db: Database.Database;
|
|
private readonly instances: InstanceService;
|
|
private readonly clock: () => Date;
|
|
private readonly id: () => string;
|
|
private readonly token: () => string;
|
|
|
|
constructor(options: Options) {
|
|
this.db = options.db;
|
|
this.instances = options.instances;
|
|
this.clock = options.now ?? (() => new Date());
|
|
this.id = options.idFactory ?? randomUUID;
|
|
this.token = options.tokenFactory ?? (() => randomBytes(32).toString('base64url'));
|
|
}
|
|
|
|
async prepare(input: PrepareOperationRequest, requestId = this.id()): Promise<Preparation> {
|
|
if (
|
|
!input ||
|
|
typeof input !== 'object' ||
|
|
Array.isArray(input) ||
|
|
Object.keys(input).some((key) => !['operationId', 'targets', 'parameters'].includes(key)) ||
|
|
input.operationId !== OPERATION_ID ||
|
|
!Array.isArray(input.targets) ||
|
|
input.targets.length !== 1 ||
|
|
!input.targets[0] ||
|
|
typeof input.targets[0] !== 'object' ||
|
|
Array.isArray(input.targets[0]) ||
|
|
Object.keys(input.targets[0]).some((key) => !['instanceId', 'revision'].includes(key)) ||
|
|
typeof input.targets[0].instanceId !== 'string' ||
|
|
input.targets[0].instanceId.length === 0 ||
|
|
!validRevision(input.targets[0].revision) ||
|
|
!input.parameters ||
|
|
typeof input.parameters !== 'object' ||
|
|
Array.isArray(input.parameters) ||
|
|
Object.keys(input.parameters).some((key) => !['parameterSchemaId', 'fields'].includes(key)) ||
|
|
input.parameters.parameterSchemaId !== DELETE_INSTANCE_PARAMETER_SCHEMA_ID ||
|
|
!Array.isArray(input.parameters.fields) ||
|
|
input.parameters.fields.length !== 0
|
|
) {
|
|
throw new DeleteInstanceOperationError('VALIDATION_FAILED', 'Invalid delete preparation');
|
|
}
|
|
const target = input.targets[0]!;
|
|
const current = await this.instances.get(target.instanceId);
|
|
if (!current) throw new DeleteInstanceOperationError('NOT_FOUND', 'Instance was not found');
|
|
if (current.revision !== target.revision)
|
|
throw new DeleteInstanceOperationError(
|
|
'REVISION_CONFLICT',
|
|
'Instance revision does not match',
|
|
);
|
|
|
|
const token = this.token();
|
|
let tokenBytes: Buffer;
|
|
try {
|
|
tokenBytes = Buffer.from(token, 'base64url');
|
|
} catch {
|
|
tokenBytes = Buffer.alloc(0);
|
|
}
|
|
if (tokenBytes.length < 32 || token.length < 20 || !/^[A-Za-z0-9_-]+$/.test(token))
|
|
throw new Error('Confirmation token factory returned an unsafe token');
|
|
const id = this.id();
|
|
const created = this.clock();
|
|
const now = created.toISOString();
|
|
const expiresAt = new Date(created.getTime() + TTL_MS).toISOString();
|
|
this.db
|
|
.prepare(
|
|
`INSERT INTO operation_preparations
|
|
(id,operation_id,risk_level,status,target_instance_id,target_revision,parameter_schema_id,parameters_digest,token_digest,requested_by,request_id,expires_at,created_at,updated_at)
|
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
)
|
|
.run(
|
|
id,
|
|
OPERATION_ID,
|
|
'R3',
|
|
'prepared',
|
|
target.instanceId,
|
|
target.revision,
|
|
DELETE_INSTANCE_PARAMETER_SCHEMA_ID,
|
|
PARAMETERS_DIGEST,
|
|
digest(token),
|
|
ACTOR,
|
|
requestId,
|
|
expiresAt,
|
|
now,
|
|
now,
|
|
);
|
|
return {
|
|
id,
|
|
status: 'prepared',
|
|
operationId: OPERATION_ID,
|
|
risk: 'R3',
|
|
expiresAt,
|
|
confirmationToken: token,
|
|
confirmationPrompt: `Delete instance ${target.instanceId}? This action cannot be undone.`,
|
|
targetCount: 1,
|
|
};
|
|
}
|
|
|
|
async executeDelete(input: ExecuteInput): Promise<Job> {
|
|
if (!validRevision(input.revision))
|
|
throw new DeleteInstanceOperationError('VALIDATION_FAILED', 'Invalid revision');
|
|
const now = this.clock().toISOString();
|
|
const ids = { job: this.id(), item: this.id(), attempt: this.id() };
|
|
let accepted = false;
|
|
let targetError: 'NOT_FOUND' | 'REVISION_CONFLICT' | undefined;
|
|
this.db.transaction(() => {
|
|
const row = this.db
|
|
.prepare(
|
|
`SELECT operation_id,status,target_instance_id,target_revision,parameter_schema_id,
|
|
parameters_digest,token_digest,expires_at
|
|
FROM operation_preparations WHERE id=?`,
|
|
)
|
|
.get(input.preparationId) as PreparationRow | undefined;
|
|
if (!row || row.status !== 'prepared')
|
|
throw new DeleteInstanceOperationError(
|
|
'CONFIRMATION_INVALID',
|
|
'Confirmation could not be accepted',
|
|
);
|
|
|
|
const secretValid =
|
|
row.expires_at > now &&
|
|
equalDigest(row.token_digest, digest(input.confirmationToken)) &&
|
|
row.operation_id === OPERATION_ID &&
|
|
row.target_instance_id === input.instanceId &&
|
|
row.target_revision === input.revision &&
|
|
row.parameter_schema_id === DELETE_INSTANCE_PARAMETER_SCHEMA_ID &&
|
|
row.parameters_digest === PARAMETERS_DIGEST;
|
|
if (!secretValid) return;
|
|
|
|
// A correctly authenticated confirmation is one-shot, including stale/missing target results.
|
|
this.db
|
|
.prepare(
|
|
"UPDATE operation_preparations SET status='consumed',consumed_at=?,updated_at=? WHERE id=? AND status='prepared'",
|
|
)
|
|
.run(now, now, input.preparationId);
|
|
const current = this.db
|
|
.prepare('SELECT config_revision FROM instances WHERE id=?')
|
|
.get(input.instanceId) as { config_revision: number } | undefined;
|
|
if (!current) {
|
|
targetError = 'NOT_FOUND';
|
|
return;
|
|
}
|
|
if (current.config_revision !== input.revision) {
|
|
targetError = 'REVISION_CONFLICT';
|
|
return;
|
|
}
|
|
|
|
this.db
|
|
.prepare(
|
|
`INSERT INTO jobs
|
|
(id,parent_job_id,root_job_id,retry_of_job_id,operation_id,risk_level,status,requested_by,request_id,parameters_digest,created_at,started_at,updated_at)
|
|
VALUES (?,NULL,?,NULL,?,'R3','running',?,?,?,?,?,?)`,
|
|
)
|
|
.run(
|
|
ids.job,
|
|
ids.job,
|
|
OPERATION_ID,
|
|
input.actor,
|
|
input.requestId,
|
|
PARAMETERS_DIGEST,
|
|
now,
|
|
now,
|
|
now,
|
|
);
|
|
this.db
|
|
.prepare(
|
|
`INSERT INTO job_attempts (id,job_id,status,started_at,created_at) VALUES (?,?,'running',?,?)`,
|
|
)
|
|
.run(ids.attempt, ids.job, now, now);
|
|
this.db
|
|
.prepare(
|
|
`INSERT INTO job_items
|
|
(id,job_id,instance_id,attempt_number,status,created_at,started_at,updated_at)
|
|
VALUES (?,?,?,1,'running',?,?,?)`,
|
|
)
|
|
.run(ids.item, ids.job, input.instanceId, now, now, now);
|
|
accepted = true;
|
|
})();
|
|
if (targetError)
|
|
throw new DeleteInstanceOperationError(
|
|
targetError,
|
|
targetError === 'NOT_FOUND' ? 'Instance was not found' : 'Instance revision does not match',
|
|
);
|
|
if (!accepted)
|
|
throw new DeleteInstanceOperationError(
|
|
'CONFIRMATION_INVALID',
|
|
'Confirmation could not be accepted',
|
|
);
|
|
|
|
try {
|
|
await this.instances.delete(input.instanceId, input.revision, { allowJobHistory: true });
|
|
const finished = this.clock().toISOString();
|
|
this.db.transaction(() => {
|
|
this.db
|
|
.prepare(
|
|
`UPDATE job_items
|
|
SET status='succeeded',result_code='INSTANCE_DELETED',finished_at=?,updated_at=?
|
|
WHERE id=? AND status='running'`,
|
|
)
|
|
.run(finished, finished, ids.item);
|
|
this.db
|
|
.prepare(
|
|
"UPDATE job_attempts SET status='succeeded',finished_at=? WHERE id=? AND status='running'",
|
|
)
|
|
.run(finished, ids.attempt);
|
|
this.db
|
|
.prepare(
|
|
"UPDATE jobs SET status='succeeded',finished_at=?,updated_at=? WHERE id=? AND status='running'",
|
|
)
|
|
.run(finished, finished, ids.job);
|
|
})();
|
|
} catch (error) {
|
|
const finished = this.clock().toISOString();
|
|
const state = error instanceof InstanceServiceError ? 'failed' : 'unknown-result';
|
|
const code = error instanceof InstanceServiceError ? error.code : 'DELETE_RESULT_UNKNOWN';
|
|
this.db.transaction(() => {
|
|
this.db
|
|
.prepare(
|
|
`UPDATE job_items
|
|
SET status=?,result_code=?,finished_at=?,updated_at=?
|
|
WHERE id=? AND status='running'`,
|
|
)
|
|
.run(state, code, finished, finished, ids.item);
|
|
this.db
|
|
.prepare('UPDATE job_attempts SET status=?,finished_at=? WHERE id=?')
|
|
.run(state, finished, ids.attempt);
|
|
this.db
|
|
.prepare('UPDATE jobs SET status=?,finished_at=?,updated_at=? WHERE id=?')
|
|
.run(state, finished, finished, ids.job);
|
|
})();
|
|
}
|
|
return this.job(ids.job);
|
|
}
|
|
|
|
reconcileInterruptedJobs(): number {
|
|
const finished = this.clock().toISOString();
|
|
return this.db.transaction(() => {
|
|
const jobs = this.db
|
|
.prepare("SELECT id FROM jobs WHERE operation_id=? AND status='running'")
|
|
.all(OPERATION_ID) as Array<{ id: string }>;
|
|
for (const { id } of jobs) {
|
|
this.db
|
|
.prepare(
|
|
"UPDATE job_items SET status='unknown-result',result_code='INTERRUPTED',finished_at=?,updated_at=? WHERE job_id=? AND status='running'",
|
|
)
|
|
.run(finished, finished, id);
|
|
this.db
|
|
.prepare(
|
|
"UPDATE job_attempts SET status='unknown-result',finished_at=? WHERE job_id=? AND status='running'",
|
|
)
|
|
.run(finished, id);
|
|
this.db
|
|
.prepare(
|
|
"UPDATE jobs SET status='unknown-result',finished_at=?,updated_at=? WHERE id=? AND status='running'",
|
|
)
|
|
.run(finished, finished, id);
|
|
}
|
|
return jobs.length;
|
|
})();
|
|
}
|
|
|
|
private job(id: string): Job {
|
|
const row = this.db
|
|
.prepare(
|
|
'SELECT operation_id,status,root_job_id,retry_of_job_id,created_at FROM jobs WHERE id=?',
|
|
)
|
|
.get(id) as {
|
|
operation_id: string;
|
|
status: Job['status'];
|
|
root_job_id: string;
|
|
retry_of_job_id: string | null;
|
|
created_at: string;
|
|
};
|
|
const items = this.db
|
|
.prepare('SELECT id,instance_id,status FROM job_items WHERE job_id=? ORDER BY created_at,id')
|
|
.all(id) as Array<{
|
|
id: string;
|
|
instance_id: string;
|
|
status: 'succeeded' | 'failed' | 'unknown-result';
|
|
}>;
|
|
const attempts = this.db
|
|
.prepare(
|
|
'SELECT id,status,started_at,finished_at FROM job_attempts WHERE job_id=? ORDER BY started_at,id',
|
|
)
|
|
.all(id) as Array<{
|
|
id: string;
|
|
status: 'succeeded' | 'failed' | 'unknown-result';
|
|
started_at: string;
|
|
finished_at: string | null;
|
|
}>;
|
|
return {
|
|
id,
|
|
operationId: row.operation_id,
|
|
status: row.status,
|
|
...(row.retry_of_job_id ? { retryOfJobId: row.retry_of_job_id } : {}),
|
|
rootJobId: row.root_job_id,
|
|
items: items.map((item) => ({ id: item.id, targetId: item.instance_id, state: item.status })),
|
|
attempts: attempts.map((attempt) => ({
|
|
id: attempt.id,
|
|
state: attempt.status,
|
|
startedAt: attempt.started_at,
|
|
...(attempt.finished_at ? { finishedAt: attempt.finished_at } : {}),
|
|
})),
|
|
createdAt: row.created_at,
|
|
};
|
|
}
|
|
}
|