multi-simadmin/apps/api/src/application/operations/delete-instance-operation.ts

import type Database from 'better-sqlite3';
import type { EventEnvelope } from '../events/event-journal.js';
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
import type { Job, Preparation, PrepareOperationRequest } from '@multi-simadmin/contracts';
import { InstanceService, InstanceServiceError } from '../instances/instance-service.js';

export const DELETE_INSTANCE_PARAMETER_SCHEMA_ID = 'deleteInstance.parameters.v1' as const;
const OPERATION_ID = 'deleteInstance';
const ACTOR = 'loopback-control-plane';
/** Job item code for "the device still holds its central binding", surfaced to the console. */
export const BINDING_REFUSAL_CODE = 'UNBIND_FAILED' as const;
const TTL_MS = 5 * 60 * 1000;
const PARAMETERS_DIGEST = createHash('sha256')
  .update(JSON.stringify({ parameterSchemaId: DELETE_INSTANCE_PARAMETER_SCHEMA_ID, fields: [] }))
  .digest('hex');

export type DeleteInstanceOperationErrorCode =
  | 'VALIDATION_FAILED'
  | 'NOT_FOUND'
  | 'REVISION_CONFLICT'
  | 'CONFIRMATION_INVALID';

export class DeleteInstanceOperationError extends Error {
  constructor(
    readonly code: DeleteInstanceOperationErrorCode,
    message: string,
  ) {
    super(message);
    this.name = 'DeleteInstanceOperationError';
  }
}

interface PreparationRow {
  operation_id: string;
  risk_level: string;
  status: string;
  target_instance_id: string;
  target_revision: number;
  target_origin: string;
  method: string;
  path: string;
  canonical_query: string;
  body_digest: string;
  content_type: string;
  parameter_schema_id: string;
  parameters_digest: string;
  nonce: string;
  token_digest: string;
  expires_at: string;
}
interface ExecuteInput {
  readonly instanceId: string;
  readonly revision: number;
  readonly preparationId: string;
  readonly confirmationToken: string;
  readonly actor: string;
  readonly requestId: string;
}

/**
 * Answer to "release the central binding on this device before we forget it".
 * `skipped` means the node never answered a heartbeat, so there is nobody to notify and the
 * console deletes its own record; the device re-binds if it ever comes back. `failed` keeps the
 * record, because a device that refused or timed out must not be silently orphaned.
 */
export type BindingRelease =
  | { readonly status: 'released' }
  | { readonly status: 'skipped' }
  | { readonly status: 'failed' };

interface Options {
  readonly db: Database.Database;
  readonly instances: InstanceService;
  /** Optional so a control plane without a device transport can still delete. */
  readonly releaseBinding?: (instanceId: string, requestId: string) => Promise<BindingRelease>;
  readonly now?: () => Date;
  readonly idFactory?: () => string;
  readonly tokenFactory?: () => string;
  /** Receives job/instance envelopes so the SSE journal can invalidate fleet views. */
  readonly emit?: (envelope: EventEnvelope) => void;
}

const digest = (value: string): string => createHash('sha256').update(value, 'utf8').digest('hex');
const equalDigest = (left: string, right: string): boolean => {
  const a = Buffer.from(left, 'hex');
  const b = Buffer.from(right, 'hex');
  return a.length === 32 && b.length === 32 && timingSafeEqual(a, b);
};
const validRevision = (value: unknown): value is number =>
  typeof value === 'number' && Number.isSafeInteger(value) && value > 0;

export class DeleteInstanceOperation {
  private readonly db: Database.Database;
  private readonly instances: InstanceService;
  private readonly releaseBinding: Options['releaseBinding'];
  private readonly clock: () => Date;
  private readonly id: () => string;
  private readonly token: () => string;
  private readonly emit: Options['emit'];

  constructor(options: Options) {
    this.db = options.db;
    this.instances = options.instances;
    this.releaseBinding = options.releaseBinding;
    this.clock = options.now ?? (() => new Date());
    this.id = options.idFactory ?? randomUUID;
    this.token = options.tokenFactory ?? (() => randomBytes(32).toString('base64url'));
    this.emit = options.emit;
  }

  async prepare(input: PrepareOperationRequest, requestId = this.id()): Promise<Preparation> {
    if (
      !input ||
      typeof input !== 'object' ||
      Array.isArray(input) ||
      Object.keys(input).some((key) => !['operationId', 'targets', 'parameters'].includes(key)) ||
      input.operationId !== OPERATION_ID ||
      !Array.isArray(input.targets) ||
      input.targets.length !== 1 ||
      !input.targets[0] ||
      typeof input.targets[0] !== 'object' ||
      Array.isArray(input.targets[0]) ||
      Object.keys(input.targets[0]).some((key) => !['instanceId', 'revision'].includes(key)) ||
      typeof input.targets[0].instanceId !== 'string' ||
      input.targets[0].instanceId.length === 0 ||
      !validRevision(input.targets[0].revision) ||
      !input.parameters ||
      typeof input.parameters !== 'object' ||
      Array.isArray(input.parameters) ||
      Object.keys(input.parameters).some((key) => !['parameterSchemaId', 'fields'].includes(key)) ||
      input.parameters.parameterSchemaId !== DELETE_INSTANCE_PARAMETER_SCHEMA_ID ||
      !Array.isArray(input.parameters.fields) ||
      input.parameters.fields.length !== 0
    ) {
      throw new DeleteInstanceOperationError('VALIDATION_FAILED', 'Invalid delete preparation');
    }
    const target = input.targets[0]!;
    const current = await this.instances.get(target.instanceId);
    if (!current) throw new DeleteInstanceOperationError('NOT_FOUND', 'Instance was not found');
    if (current.revision !== target.revision)
      throw new DeleteInstanceOperationError(
        'REVISION_CONFLICT',
        'Instance revision does not match',
      );

    const token = this.token();
    let tokenBytes: Buffer;
    try {
      tokenBytes = Buffer.from(token, 'base64url');
    } catch {
      tokenBytes = Buffer.alloc(0);
    }
    if (tokenBytes.length < 32 || token.length < 20 || !/^[A-Za-z0-9_-]+$/.test(token))
      throw new Error('Confirmation token factory returned an unsafe token');
    const id = this.id();
    const created = this.clock();
    const now = created.toISOString();
    const expiresAt = new Date(created.getTime() + TTL_MS).toISOString();
    this.db
      .prepare(
        `INSERT INTO operation_preparations
         (id,operation_id,risk_level,status,target_instance_id,target_revision,target_origin,method,path,
          canonical_query,body_digest,content_type,parameter_schema_id,parameters_digest,nonce,token_digest,
          requested_by,request_id,expires_at,created_at,updated_at)
         VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
      )
      .run(
        id,
        OPERATION_ID,
        'R3',
        'prepared',
        target.instanceId,
        target.revision,
        current.origin,
        'DELETE',
        `/api/v1/instances/${target.instanceId}`,
        '',
        digest(''),
        '',
        DELETE_INSTANCE_PARAMETER_SCHEMA_ID,
        PARAMETERS_DIGEST,
        `delete-${id}`,
        digest(token),
        ACTOR,
        requestId,
        expiresAt,
        now,
        now,
      );
    return {
      id,
      status: 'prepared',
      operationId: OPERATION_ID,
      risk: 'R3',
      expiresAt,
      confirmationToken: token,
      confirmationPrompt: `Delete instance ${target.instanceId}? This action cannot be undone.`,
      targetCount: 1,
    };
  }

  async executeDelete(input: ExecuteInput): Promise<Job> {
    if (!validRevision(input.revision))
      throw new DeleteInstanceOperationError('VALIDATION_FAILED', 'Invalid revision');
    const now = this.clock().toISOString();
    const ids = { job: this.id(), item: this.id(), attempt: this.id() };
    let accepted = false;
    let targetError: 'NOT_FOUND' | 'REVISION_CONFLICT' | undefined;
    this.db.transaction(() => {
      const row = this.db
        .prepare(
          `SELECT operation_id,risk_level,status,target_instance_id,target_revision,target_origin,
                  method,path,canonical_query,body_digest,content_type,parameter_schema_id,
                  parameters_digest,nonce,token_digest,expires_at
           FROM operation_preparations WHERE id=?`,
        )
        .get(input.preparationId) as PreparationRow | undefined;
      if (!row || row.status !== 'prepared')
        throw new DeleteInstanceOperationError(
          'CONFIRMATION_INVALID',
          'Confirmation could not be accepted',
        );

      const confirmationValid =
        row.expires_at > now && equalDigest(row.token_digest, digest(input.confirmationToken));
      if (!confirmationValid) return;

      const expectedPath = `/api/v1/instances/${input.instanceId}`;
      const bindingValid =
        row.operation_id === OPERATION_ID &&
        row.risk_level === 'R3' &&
        row.target_instance_id === input.instanceId &&
        row.target_revision === input.revision &&
        row.method === 'DELETE' &&
        row.path === expectedPath &&
        row.canonical_query === '' &&
        row.body_digest === digest('') &&
        row.content_type === '' &&
        row.parameter_schema_id === DELETE_INSTANCE_PARAMETER_SCHEMA_ID &&
        row.parameters_digest === PARAMETERS_DIGEST &&
        row.nonce === `delete-${input.preparationId}`;
      if (!bindingValid) {
        this.db
          .prepare(
            "UPDATE operation_preparations SET status='invalidated',consumed_at=?,updated_at=? WHERE id=? AND status='prepared'",
          )
          .run(now, now, input.preparationId);
        return;
      }

      const current = this.db
        .prepare('SELECT base_url,config_revision FROM instances WHERE id=?')
        .get(input.instanceId) as { base_url: string; config_revision: number } | undefined;
      if (current && current.base_url !== row.target_origin) {
        this.db
          .prepare(
            "UPDATE operation_preparations SET status='invalidated',consumed_at=?,updated_at=? WHERE id=? AND status='prepared'",
          )
          .run(now, now, input.preparationId);
        return;
      }

      // A correctly authenticated confirmation is one-shot, including stale/missing target results.
      this.db
        .prepare(
          "UPDATE operation_preparations SET status='consumed',consumed_at=?,updated_at=? WHERE id=? AND status='prepared'",
        )
        .run(now, now, input.preparationId);
      if (!current) {
        targetError = 'NOT_FOUND';
        return;
      }
      if (current.config_revision !== input.revision) {
        targetError = 'REVISION_CONFLICT';
        return;
      }

      this.db
        .prepare(
          `INSERT INTO jobs
         (id,parent_job_id,root_job_id,retry_of_job_id,operation_id,risk_level,status,requested_by,request_id,parameters_digest,created_at,started_at,updated_at)
         VALUES (?,NULL,?,NULL,?,'R3','running',?,?,?,?,?,?)`,
        )
        .run(
          ids.job,
          ids.job,
          OPERATION_ID,
          input.actor,
          input.requestId,
          PARAMETERS_DIGEST,
          now,
          now,
          now,
        );
      this.db
        .prepare(
          `INSERT INTO job_attempts (id,job_id,status,started_at,created_at) VALUES (?,?,'running',?,?)`,
        )
        .run(ids.attempt, ids.job, now, now);
      this.db
        .prepare(
          `INSERT INTO job_items
         (id,job_id,instance_id,attempt_number,status,created_at,started_at,updated_at)
         VALUES (?,?,?,1,'running',?,?,?)`,
        )
        .run(ids.item, ids.job, input.instanceId, now, now, now);
      accepted = true;
    })();
    if (targetError)
      throw new DeleteInstanceOperationError(
        targetError,
        targetError === 'NOT_FOUND' ? 'Instance was not found' : 'Instance revision does not match',
      );
    if (!accepted)
      throw new DeleteInstanceOperationError(
        'CONFIRMATION_INVALID',
        'Confirmation could not be accepted',
      );

    // An online device is told to release its central binding first, and a refusal keeps the
    // record so the operator can retry instead of losing sight of a device that is still
    // reporting to a hub. Only a node that cannot be reached at all is forgotten locally.
    const release = await this.releaseBindingPort(input.instanceId, input.requestId);
    if (release?.status === 'failed') {
      this.closeAsFailed(ids, input.requestId, BINDING_REFUSAL_CODE);
      this.#emitJob(ids.job, input.requestId);
      return this.job(ids.job);
    }
    try {
      await this.instances.delete(input.instanceId, input.revision, { allowJobHistory: true });
      this.#emit({
        kind: 'instance',
        id: this.id(),
        occurredAt: this.clock().toISOString(),
        requestId: input.requestId,
        instanceId: input.instanceId,
      });
      this.#emitJob(ids.job, input.requestId);
      const finished = this.clock().toISOString();
      this.db.transaction(() => {
        this.db
          .prepare(
            `UPDATE job_items
           SET status='succeeded',result_code='INSTANCE_DELETED',finished_at=?,updated_at=?
           WHERE id=? AND status='running'`,
          )
          .run(finished, finished, ids.item);
        this.db
          .prepare(
            "UPDATE job_attempts SET status='succeeded',finished_at=? WHERE id=? AND status='running'",
          )
          .run(finished, ids.attempt);
        this.db
          .prepare(
            "UPDATE jobs SET status='succeeded',finished_at=?,updated_at=? WHERE id=? AND status='running'",
          )
          .run(finished, finished, ids.job);
      })();
    } catch (error) {
      const finished = this.clock().toISOString();
      const state = error instanceof InstanceServiceError ? 'failed' : 'unknown-result';
      const code = error instanceof InstanceServiceError ? error.code : 'DELETE_RESULT_UNKNOWN';
      this.db.transaction(() => {
        this.db
          .prepare(
            `UPDATE job_items
           SET status=?,result_code=?,finished_at=?,updated_at=?
           WHERE id=? AND status='running'`,
          )
          .run(state, code, finished, finished, ids.item);
        this.db
          .prepare('UPDATE job_attempts SET status=?,finished_at=? WHERE id=?')
          .run(state, finished, ids.attempt);
        this.db
          .prepare('UPDATE jobs SET status=?,finished_at=?,updated_at=? WHERE id=?')
          .run(state, finished, finished, ids.job);
      })();
    }
    return this.job(ids.job);
  }

  /** A transport that throws while the node was reachable counts as a refusal, never a pass. */
  private async releaseBindingPort(
    instanceId: string,
    requestId: string,
  ): Promise<BindingRelease | undefined> {
    if (!this.releaseBinding) return undefined;
    try {
      return await this.releaseBinding(instanceId, requestId);
    } catch {
      return { status: 'failed' };
    }
  }

  /** Closes the job without touching the instance row, so the device stays listed and retryable. */
  #emit(envelope: EventEnvelope): void {
    if (!this.emit) return;
    try {
      this.emit(envelope);
    } catch {
      // A journal failure must never fail the operation itself.
    }
  }

  #emitJob(jobId: string, requestId: string): void {
    this.#emit({
      kind: 'job',
      id: this.id(),
      occurredAt: this.clock().toISOString(),
      requestId,
      jobId,
    });
  }

  private closeAsFailed(
    ids: { job: string; item: string; attempt: string },
    requestId: string,
    resultCode: string,
  ): void {
    const finished = this.clock().toISOString();
    this.db.transaction(() => {
      this.db
        .prepare(
          `UPDATE job_items
           SET status='failed',result_code=?,error_json=?,finished_at=?,updated_at=?
           WHERE id=? AND status='running'`,
        )
        .run(
          resultCode,
          JSON.stringify({ status: 502, code: resultCode, requestId }),
          finished,
          finished,
          ids.item,
        );
      this.db
        .prepare(
          "UPDATE job_attempts SET status='failed',finished_at=? WHERE id=? AND status='running'",
        )
        .run(finished, ids.attempt);
      this.db
        .prepare(
          "UPDATE jobs SET status='failed',finished_at=?,updated_at=? WHERE id=? AND status='running'",
        )
        .run(finished, finished, ids.job);
    })();
  }

  reconcileInterruptedJobs(): number {
    const finished = this.clock().toISOString();
    return this.db.transaction(() => {
      const jobs = this.db
        .prepare("SELECT id FROM jobs WHERE operation_id=? AND status='running'")
        .all(OPERATION_ID) as Array<{ id: string }>;
      for (const { id } of jobs) {
        this.db
          .prepare(
            "UPDATE job_items SET status='unknown-result',result_code='INTERRUPTED',finished_at=?,updated_at=? WHERE job_id=? AND status='running'",
          )
          .run(finished, finished, id);
        this.db
          .prepare(
            "UPDATE job_attempts SET status='unknown-result',finished_at=? WHERE job_id=? AND status='running'",
          )
          .run(finished, id);
        this.db
          .prepare(
            "UPDATE jobs SET status='unknown-result',finished_at=?,updated_at=? WHERE id=? AND status='running'",
          )
          .run(finished, finished, id);
      }
      return jobs.length;
    })();
  }

  private job(id: string): Job {
    const row = this.db
      .prepare(
        'SELECT operation_id,status,root_job_id,retry_of_job_id,created_at,request_id FROM jobs WHERE id=?',
      )
      .get(id) as {
      operation_id: string;
      status: Job['status'];
      root_job_id: string;
      retry_of_job_id: string | null;
      created_at: string;
      request_id: string;
    };
    const items = this.db
      .prepare(
        'SELECT id,instance_id,status,result_code FROM job_items WHERE job_id=? ORDER BY created_at,id',
      )
      .all(id) as Array<{
      id: string;
      instance_id: string;
      status: 'succeeded' | 'failed' | 'unknown-result';
      result_code: string | null;
    }>;
    const attempts = this.db
      .prepare(
        'SELECT id,status,started_at,finished_at FROM job_attempts WHERE job_id=? ORDER BY started_at,id',
      )
      .all(id) as Array<{
      id: string;
      status: 'succeeded' | 'failed' | 'unknown-result';
      started_at: string;
      finished_at: string | null;
    }>;
    return {
      id,
      operationId: row.operation_id,
      status: row.status,
      ...(row.retry_of_job_id ? { retryOfJobId: row.retry_of_job_id } : {}),
      rootJobId: row.root_job_id,
      items: items.map((item) => ({
        id: item.id,
        targetId: item.instance_id,
        state: item.status,
        // The console reads this code straight out of the execute response to explain why the
        // node it just confirmed is still in the list.
        ...(item.result_code === BINDING_REFUSAL_CODE
          ? {
              error: {
                type: 'about:blank',
                title: 'Job item failed',
                status: 502,
                detail: 'The job item did not complete successfully.',
                code: BINDING_REFUSAL_CODE,
                requestId: row.request_id,
              },
            }
          : {}),
      })),
      attempts: attempts.map((attempt) => ({
        id: attempt.id,
        state: attempt.status,
        startedAt: attempt.started_at,
        ...(attempt.finished_at ? { finishedAt: attempt.finished_at } : {}),
      })),
      createdAt: row.created_at,
    };
  }
}