fix(instances): release the hub binding before forgetting a node

Deletion now follows the published contract: prepare returns the one-time
token and the console spends it on DELETE /api/v1/instances/:id with
If-Match, where the generic execute endpoint had been rejecting it as an
operation the safe transport cannot replay.

An online device is told to release its own binding first. A refusal keeps
the record and reports UNBIND_FAILED on the job item so the console can say
why the node is still there; a node that never answers a heartbeat is
forgotten locally.

apps/api/src/application/operations/delete-instance-operation.ts

@@ -6,6 +6,8 @@ import { InstanceService, InstanceServiceError } from '../instances/instance-ser
 export const DELETE_INSTANCE_PARAMETER_SCHEMA_ID = 'deleteInstance.parameters.v1' as const;
 const OPERATION_ID = 'deleteInstance';
 const ACTOR = 'loopback-control-plane';
+/** Job item code for "the device still holds its central binding", surfaced to the console. */
+export const BINDING_REFUSAL_CODE = 'UNBIND_FAILED' as const;
 const TTL_MS = 5 * 60 * 1000;
 const PARAMETERS_DIGEST = createHash('sha256')
   .update(JSON.stringify({ parameterSchemaId: DELETE_INSTANCE_PARAMETER_SCHEMA_ID, fields: [] }))
@@ -53,9 +55,23 @@ interface ExecuteInput {
   readonly actor: string;
   readonly requestId: string;
 }
+
+/**
+ * Answer to "release the central binding on this device before we forget it".
+ * `skipped` means the node never answered a heartbeat, so there is nobody to notify and the
+ * console deletes its own record; the device re-binds if it ever comes back. `failed` keeps the
+ * record, because a device that refused or timed out must not be silently orphaned.
+ */
+export type BindingRelease =
+  | { readonly status: 'released' }
+  | { readonly status: 'skipped' }
+  | { readonly status: 'failed' };
+
 interface Options {
   readonly db: Database.Database;
   readonly instances: InstanceService;
+  /** Optional so a control plane without a device transport can still delete. */
+  readonly releaseBinding?: (instanceId: string, requestId: string) => Promise<BindingRelease>;
   readonly now?: () => Date;
   readonly idFactory?: () => string;
   readonly tokenFactory?: () => string;
@@ -73,6 +89,7 @@ const validRevision = (value: unknown): value is number =>
 export class DeleteInstanceOperation {
   private readonly db: Database.Database;
   private readonly instances: InstanceService;
+  private readonly releaseBinding: Options['releaseBinding'];
   private readonly clock: () => Date;
   private readonly id: () => string;
   private readonly token: () => string;
@@ -80,6 +97,7 @@ export class DeleteInstanceOperation {
   constructor(options: Options) {
     this.db = options.db;
     this.instances = options.instances;
+    this.releaseBinding = options.releaseBinding;
     this.clock = options.now ?? (() => new Date());
     this.id = options.idFactory ?? randomUUID;
     this.token = options.tokenFactory ?? (() => randomBytes(32).toString('base64url'));
@@ -294,6 +312,14 @@ export class DeleteInstanceOperation {
         'Confirmation could not be accepted',
       );
 
+    // An online device is told to release its central binding first, and a refusal keeps the
+    // record so the operator can retry instead of losing sight of a device that is still
+    // reporting to a hub. Only a node that cannot be reached at all is forgotten locally.
+    const release = await this.releaseBindingPort(input.instanceId, input.requestId);
+    if (release?.status === 'failed') {
+      this.closeAsFailed(ids, input.requestId, BINDING_REFUSAL_CODE);
+      return this.job(ids.job);
+    }
     try {
       await this.instances.delete(input.instanceId, input.revision, { allowJobHistory: true });
       const finished = this.clock().toISOString();
@@ -339,6 +365,53 @@ export class DeleteInstanceOperation {
     return this.job(ids.job);
   }
 
+  /** A transport that throws while the node was reachable counts as a refusal, never a pass. */
+  private async releaseBindingPort(
+    instanceId: string,
+    requestId: string,
+  ): Promise<BindingRelease | undefined> {
+    if (!this.releaseBinding) return undefined;
+    try {
+      return await this.releaseBinding(instanceId, requestId);
+    } catch {
+      return { status: 'failed' };
+    }
+  }
+
+  /** Closes the job without touching the instance row, so the device stays listed and retryable. */
+  private closeAsFailed(
+    ids: { job: string; item: string; attempt: string },
+    requestId: string,
+    resultCode: string,
+  ): void {
+    const finished = this.clock().toISOString();
+    this.db.transaction(() => {
+      this.db
+        .prepare(
+          `UPDATE job_items
+           SET status='failed',result_code=?,error_json=?,finished_at=?,updated_at=?
+           WHERE id=? AND status='running'`,
+        )
+        .run(
+          resultCode,
+          JSON.stringify({ status: 502, code: resultCode, requestId }),
+          finished,
+          finished,
+          ids.item,
+        );
+      this.db
+        .prepare(
+          "UPDATE job_attempts SET status='failed',finished_at=? WHERE id=? AND status='running'",
+        )
+        .run(finished, ids.attempt);
+      this.db
+        .prepare(
+          "UPDATE jobs SET status='failed',finished_at=?,updated_at=? WHERE id=? AND status='running'",
+        )
+        .run(finished, finished, ids.job);
+    })();
+  }
+
   reconcileInterruptedJobs(): number {
     const finished = this.clock().toISOString();
     return this.db.transaction(() => {
@@ -369,7 +442,7 @@ export class DeleteInstanceOperation {
   private job(id: string): Job {
     const row = this.db
       .prepare(
-        'SELECT operation_id,status,root_job_id,retry_of_job_id,created_at FROM jobs WHERE id=?',
+        'SELECT operation_id,status,root_job_id,retry_of_job_id,created_at,request_id FROM jobs WHERE id=?',
       )
       .get(id) as {
       operation_id: string;
@@ -377,13 +450,17 @@ export class DeleteInstanceOperation {
       root_job_id: string;
       retry_of_job_id: string | null;
       created_at: string;
+      request_id: string;
     };
     const items = this.db
-      .prepare('SELECT id,instance_id,status FROM job_items WHERE job_id=? ORDER BY created_at,id')
+      .prepare(
+        'SELECT id,instance_id,status,result_code FROM job_items WHERE job_id=? ORDER BY created_at,id',
+      )
       .all(id) as Array<{
       id: string;
       instance_id: string;
       status: 'succeeded' | 'failed' | 'unknown-result';
+      result_code: string | null;
     }>;
     const attempts = this.db
       .prepare(
@@ -401,7 +478,25 @@ export class DeleteInstanceOperation {
       status: row.status,
       ...(row.retry_of_job_id ? { retryOfJobId: row.retry_of_job_id } : {}),
       rootJobId: row.root_job_id,
-      items: items.map((item) => ({ id: item.id, targetId: item.instance_id, state: item.status })),
+      items: items.map((item) => ({
+        id: item.id,
+        targetId: item.instance_id,
+        state: item.status,
+        // The console reads this code straight out of the execute response to explain why the
+        // node it just confirmed is still in the list.
+        ...(item.result_code === BINDING_REFUSAL_CODE
+          ? {
+              error: {
+                type: 'about:blank',
+                title: 'Job item failed',
+                status: 502,
+                detail: 'The job item did not complete successfully.',
+                code: BINDING_REFUSAL_CODE,
+                requestId: row.request_id,
+              },
+            }
+          : {}),
+      })),
       attempts: attempts.map((attempt) => ({
         id: attempt.id,
         state: attempt.status,