multi-simadmin/apps/api/src/control-plane.test.ts

import Database from 'better-sqlite3';
import { afterEach, describe, expect, it } from 'vitest';
import { buildControlPlaneApp } from './control-plane.js';
import { ScheduledTaskRepository } from './application/automation/scheduled-task-repository.js';
import { migrateDatabase } from './infrastructure/database/migrations.js';
import { UpstreamError } from './infrastructure/transport/upstream-error.js';
import { SafeInstanceTransport } from './infrastructure/transport/safe-instance-transport.js';
import { createSafeControlPlaneUpstream } from './infrastructure/transport/safe-control-plane-upstream.js';
import type { SecretStore } from './infrastructure/secrets/secret-store.js';

const dbs: Database.Database[] = [];
afterEach(async () => {
  for (const db of dbs.splice(0)) db.close();
});
class Store implements SecretStore {
  readonly provider = 'macos-keychain';
  private readonly values = new Map<string, string>();
  async set(key: { instanceId: string; purpose: string; slot?: string }, value: string) {
    const account = Buffer.from(
      JSON.stringify(
        key.slot === undefined
          ? [key.instanceId, key.purpose]
          : [key.instanceId, key.purpose, key.slot],
      ),
      'utf8',
    ).toString('base64url');
    const reference = `keychain://multi-simadmin/${account}`;
    this.values.set(reference, value);
    return reference;
  }
  async get(reference: string) {
    return this.values.get(reference);
  }
  async delete(reference: string) {
    return this.values.delete(reference);
  }
}
describe('buildControlPlaneApp', () => {
  it('reconciles interrupted scheduled runs before starting the scheduler', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const repository = new ScheduledTaskRepository(db);
    const task = {
      name: 'Interrupted restart',
      operationType: 'restart-service' as const,
      cronExpression: '0 9 * * *',
      timezone: 'Asia/Shanghai' as const,
      targetSelector: { mode: 'fixed' as const, instanceIds: ['instance-a'] },
      misfirePolicy: 'skip' as const,
      overlapPolicy: 'skip' as const,
      retryPolicy: { maxRetries: 0, intervalSeconds: 60 },
      enabled: false,
    };
    repository.create({
      id: 'task-1',
      task,
      createdBy: 'operator',
      now: '2026-07-30T00:00:00.000Z',
    });
    repository.claimOccurrence({
      id: 'run-1',
      scheduledTaskId: 'task-1',
      scheduleVersion: 1,
      dueAt: '2026-07-30T01:00:00.000Z',
      claimedAt: '2026-07-30T01:00:00.000Z',
      triggerSource: 'scheduled',
      targetSnapshot: [],
      taskSnapshot: task,
    });
    repository.startRun('run-1', '2026-07-30T01:00:01.000Z');

    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async () => ({ status: 200, headers: {}, body: '' }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
      now: () => new Date('2026-07-30T02:00:00.000Z'),
    });

    expect(repository.getRun('run-1')).toMatchObject({
      outcome: 'needs-attention',
      finishedAt: '2026-07-30T02:00:00.000Z',
    });
    await app.close();
  });

  it('registers the durable event route fail-closed when no authentication dependency is supplied', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async () => ({ status: 200, headers: {}, body: '' }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });

    const response = await app.inject({ method: 'GET', url: '/api/v1/events' });

    expect(response.statusCode).toBe(401);
    expect(response.headers['content-type']).toContain('application/problem+json');
    expect(response.json()).toMatchObject({ status: 401, code: 'UNAUTHORIZED' });
    const jobs = await app.inject({ method: 'GET', url: '/api/v1/jobs' });
    expect(jobs.statusCode).toBe(200);
    expect(jobs.json()).toEqual({ items: [], page: { page: 1, pageSize: 25, total: 0 } });
    const audit = await app.inject({ method: 'GET', url: '/api/v1/audit' });
    expect(audit.statusCode).toBe(200);
    expect(audit.json()).toEqual({ items: [], page: { page: 1, pageSize: 25, total: 0 } });
    await app.close();
  });

  it('serves native system maintenance from the control plane composition', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async () => ({ status: 200, headers: {}, body: '' }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });

    const response = await app.inject({ method: 'GET', url: '/api/v1/system/maintenance' });

    expect(response.statusCode).toBe(200);
    expect(response.json()).toMatchObject({
      runtime: { version: '0.1.0', platform: process.platform, arch: process.arch },
      storage: { components: expect.any(Array) },
      retention: { auditEvents: { enabled: true, days: 180, maximumCount: 50_000 } },
    });
    await app.close();
  });

  it('serves central notification configuration from the control plane composition', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async () => ({ status: 200, headers: {}, body: '' }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });

    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/notifications/channels',
      payload: {
        name: 'Bark',
        type: 'bark',
        config: { server_url: 'https://api.day.app', device_key: 'opaque-device-key' },
      },
    });

    expect(created.statusCode).toBe(201);
    expect(created.body).not.toContain('opaque-device-key');
    const rules = await app.inject('/api/v1/notifications/rules');
    expect(rules.statusCode).toBe(200);
    expect(rules.json()).toEqual({ items: [], page: { page: 1, pageSize: 50, total: 0 } });
    await app.close();
  });

  it('passes an explicit event-stream authenticator to durable cursor replay', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async () => ({ status: 200, headers: {}, body: '' }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
      authenticateEventStream: (request) => request.headers.authorization === 'Bearer allowed',
    });

    const response = await app.inject({
      method: 'GET',
      url: '/api/v1/events',
      headers: {
        authorization: 'Bearer allowed',
        'last-event-id': 'unknown-cursor',
      },
    });

    expect(response.statusCode).toBe(409);
    expect(response.headers['content-type']).toContain('application/problem+json');
    expect(response.json()).toMatchObject({
      status: 409,
      code: 'EVENT_POSITION_UNAVAILABLE',
    });
    await app.close();
  });

  it('assembles instance CRUD, connection checks, login and logout without listening or reading external config', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: createSafeControlPlaneUpstream(
        new SafeInstanceTransport({
          resolve: async () => [{ address: '192.168.1.10', family: 4 }],
          request: async (request) => ({
            status: request.url.endsWith('/api/health') ? 401 : 200,
            headers: { 'set-cookie': 'simadmin_session=opaque' },
            body: '',
          }),
        }),
      ),
      now: () => new Date('2026-07-16T12:00:00.000Z'),
    });
    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: { name: 'A', origin: 'http://192.168.1.10:8080' },
    });
    expect(created.statusCode).toBe(201);
    const tested = await app.inject({
      method: 'POST',
      url: '/api/v1/instances/' + created.json().id + '/test-connection',
    });
    expect(tested.json()).toMatchObject({ authenticated: false });

    const insecureLogin = await app.inject({
      method: 'POST',
      url: '/api/v1/instances/' + created.json().id + '/login',
      payload: { password: 'never-leak-this' },
    });
    // Literal private HTTP origins are allowed for LAN instances; secrets still never leak.
    expect(insecureLogin.statusCode).toBe(200);
    expect(insecureLogin.json()).toMatchObject({
      authenticated: true,
      instanceId: created.json().id,
    });
    expect(insecureLogin.body).not.toContain('never-leak-this');
    expect(insecureLogin.body).not.toContain('simadmin_session');
    await app.close();
  });

  it('automatically logs in with saved credentials before aggregate messages and resource reads', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const calls: Array<{ url: string; cookie?: string }> = [];
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async (request) => {
          calls.push({
            url: request.url,
            ...(request.headers.cookie ? { cookie: request.headers.cookie } : {}),
          });
          if (request.url.endsWith('/api/auth/login'))
            return {
              status: 200,
              headers: { 'set-cookie': 'simadmin_session=aggregate-session' },
              body: '{"status":"success"}',
            };
          if (request.url.includes('/api/sms/list'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({
                status: 'success',
                data: {
                  messages: [
                    {
                      id: 7,
                      direction: 'incoming',
                      phone_number: '10086',
                      content: 'hi',
                      timestamp: '2026-07-29 12:00:00',
                      status: 'received',
                    },
                  ],
                },
              }),
            };
          if (request.url.endsWith('/api/health'))
            return {
              status: 200,
              headers: {},
              body: '{"status":"ok","version":"2.3.4","platform":"linux"}',
            };
          return { status: 200, headers: {}, body: '{"status":"success","data":{}}' };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });
    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: {
        name: 'Protected',
        origin: 'http://192.168.1.20:3000',
        password: { action: 'set', password: '[REDACTED]' },
      },
    });
    const id = created.json().id as string;
    const messages = await app.inject({
      method: 'GET',
      url: `/api/v1/instances/${id}/messages?limit=1&offset=0`,
    });
    expect(messages.statusCode).toBe(200);
    expect(messages.json().messages).toEqual([
      expect.objectContaining({ id: '7', phoneNumber: '10086', content: 'hi' }),
    ]);
    const resources = await app.inject({ method: 'GET', url: `/api/v1/instances/${id}/resources` });
    expect(resources.statusCode).toBe(200);
    expect(resources.json()).toMatchObject({ version: '2.3.4', platform: 'linux' });
    expect(calls.filter((call) => call.url.endsWith('/api/auth/login'))).toHaveLength(1);
    for (const call of calls.filter((item) => !item.url.endsWith('/api/auth/login')))
      expect(call.cookie).toBe('simadmin_session=aggregate-session');
    await app.close();
  });

  it('keeps the hub namespace closed while serving a native fleet overview aggregate', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async () => ({ status: 200, headers: {}, body: '{"status":"success"}' }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });

    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: { name: 'Alpha', origin: 'http://192.168.1.10:8080' },
    });
    expect(created.statusCode).toBe(201);
    const instance = created.json();

    const hubResponse = await app.inject({ method: 'GET', url: '/api/v1/hub/overview' });
    expect(hubResponse.statusCode).toBe(404);
    const overview = await app.inject({ method: 'GET', url: '/api/v1/fleet/overview' });
    expect(overview.statusCode).toBe(200);
    expect(overview.json()).toEqual({
      items: [
        {
          ...instance,
          connection: {
            probed: false,
            reachable: false,
            authenticated: false,
            checkedAt: null,
          },
          resources: {},
          identity: { tracked: false, status: 'confirmed', reasons: [] },
        },
      ],
    });

    // A live heartbeat snapshot turns the same row into a reported online device.
    db.prepare(
      `INSERT INTO status_snapshots (id,instance_id,category,state,payload_json,observed_at,expires_at,created_at)
       VALUES ('snap-1',?, 'connection','fresh',?,?,?,?)`,
    ).run(
      instance.id,
      JSON.stringify({ reachable: true, authenticated: true }),
      '2026-07-16T12:00:00.000Z',
      '2099-01-01T00:00:00.000Z',
      '2026-07-16T12:00:00.000Z',
    );
    const probed = await app.inject({ method: 'GET', url: '/api/v1/fleet/overview' });
    expect(probed.json().items[0].connection).toEqual({
      probed: true,
      reachable: true,
      authenticated: true,
      checkedAt: '2026-07-16T12:00:00.000Z',
    });

    await app.close();
  });

  it('serves cross-node message management from the Fleet domain', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async (request) => {
          if (request.url.endsWith('/api/auth/login'))
            return {
              status: 200,
              headers: { 'set-cookie': 'simadmin_session=fleet-msg-session' },
              body: '{"status":"success"}',
            };
          if (request.url.includes('/api/sms/list'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({
                status: 'success',
                data: {
                  messages: [
                    {
                      id: 2,
                      direction: 'incoming',
                      phone_number: '10086',
                      content: 'Alpha balance updated',
                      timestamp: '2026-07-17T01:02:00.000Z',
                      status: 'received',
                      transport: 'gsm',
                    },
                    {
                      id: 1,
                      direction: 'outgoing',
                      phone_number: '13900139000',
                      content: 'Hello Fleet',
                      timestamp: '2026-07-17T00:59:00.000Z',
                      status: 'delivered',
                      transport: 'modem',
                    },
                  ],
                },
              }),
            };
          if (request.url.endsWith('/api/sms/send'))
            return { status: 200, headers: {}, body: '{"status":"success"}' };
          if (request.url.endsWith('/api/sms/batch-delete'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({
                status: 'success',
                data: { deleted: 2 },
              }),
            };
          return { status: 200, headers: {}, body: '{"status":"success","data":{}}' };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
      now: () => new Date('2026-07-16T12:00:00.000Z'),
    });
    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: {
        name: 'Message Node',
        origin: 'http://192.168.1.40:8080',
        password: { action: 'set', password: '[REDACTED]' },
      },
    });
    expect(created.statusCode).toBe(201);
    const instanceId = created.json().id as string;

    const list = await app.inject({
      method: 'GET',
      url: '/api/v1/fleet/messages?limit=10&offset=0&search=Alpha',
    });

    expect(list.statusCode).toBe(200);
    expect(list.json()).toEqual({
      messages: [
        expect.objectContaining({
          instanceId,
          instanceName: 'Message Node',
          phoneNumber: '10086',
          content: 'Alpha balance updated',
        }),
      ],
      devices: [{ id: instanceId, name: 'Message Node', availability: 'online' }],
      total: 1,
    });

    const conversations = await app.inject({
      method: 'GET',
      url: '/api/v1/fleet/messages/conversations?limit=10&offset=0&direction=incoming',
    });
    expect(conversations.statusCode).toBe(200);
    expect(conversations.json()).toEqual({
      conversations: [
        expect.objectContaining({
          instanceId,
          instanceName: 'Message Node',
          phoneNumber: '10086',
          messageCount: 1,
          incomingCount: 1,
          lastMessage: expect.objectContaining({ content: 'Alpha balance updated' }),
        }),
      ],
      total: 1,
      // Direction counters describe the whole filtered archive, not just the returned page.
      stats: { incoming: 1, outgoing: 1, total: 2 },
    });

    const badDirection = await app.inject({
      method: 'GET',
      url: '/api/v1/fleet/messages/conversations?direction=both',
    });
    expect(badDirection.statusCode).toBe(400);
    // The plain message list has no direction concept and must keep rejecting the parameter.
    expect(
      (
        await app.inject({
          method: 'GET',
          url: '/api/v1/fleet/messages?limit=10&offset=0&direction=incoming',
        })
      ).statusCode,
    ).toBe(400);

    const sent = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/send',
      payload: {
        instanceId,
        phoneNumber: '13900139000',
        content: 'Verified via Fleet',
      },
    });
    expect(sent.statusCode).toBe(200);
    expect(sent.json()).toEqual({ sent: true, queued: false, instanceId });

    const deleted = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/delete',
      payload: {
        items: [
          { instanceId, id: 2 },
          { instanceId, id: 1 },
        ],
      },
    });
    expect(deleted.statusCode).toBe(200);
    expect(deleted.json()).toEqual({
      requested: 2,
      deleted: 2,
      failed: 0,
      failures: [],
    });

    const missing = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/send',
      payload: {
        instanceId: 'missing-instance',
        phoneNumber: '13900139000',
        content: 'No target',
      },
    });
    expect(missing.statusCode).toBe(404);
    expect(missing.headers['content-type']).toContain('application/problem+json');
    expect(missing.json()).toMatchObject({ status: 404, code: 'NOT_FOUND' });
    await app.close();
  });

  it('queues fleet sends for offline nodes and drains them from the outbox routes', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const delivered: string[] = [];
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async (request) => {
          if (request.url.endsWith('/api/auth/login'))
            return {
              status: 200,
              headers: { 'set-cookie': 'simadmin_session=fleet-outbox-session' },
              body: '{\"status\":\"success\"}',
            };
          if (request.url.endsWith('/api/sms/send')) {
            delivered.push(request.body ?? '');
            return { status: 200, headers: {}, body: '{\"status\":\"success\"}' };
          }
          return { status: 200, headers: {}, body: '{\"status\":\"success\",\"data\":{}}' };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
      now: () => new Date('2026-07-16T12:00:00.000Z'),
    });

    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: {
        name: 'Sleeping Node',
        origin: 'http://192.168.1.41:8080',
        password: { action: 'set', password: '[REDACTED]' },
      },
    });
    expect(created.statusCode).toBe(201);
    const instanceId = created.json().id as string;

    // The heartbeat journal says this node dropped off the LAN before the fixed clock time.
    db.prepare(
      `INSERT INTO status_snapshots (id,instance_id,category,state,payload_json,observed_at,expires_at,created_at)
       VALUES ('snap-outbox',?, 'connection','fresh',?,?,?,?)`,
    ).run(
      instanceId,
      JSON.stringify({ reachable: true, authenticated: true }),
      '2026-07-16T11:58:00.000Z',
      '2026-07-16T11:59:00.000Z',
      '2026-07-16T11:58:00.000Z',
    );

    const queued = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/send',
      payload: { instanceId, phoneNumber: '13900139000', content: 'Wait for the node' },
    });
    expect(queued.statusCode).toBe(200);
    expect(queued.json()).toEqual({
      sent: false,
      queued: true,
      instanceId,
      queueId: expect.any(String),
    });
    expect(delivered).toHaveLength(0);
    const queueId = queued.json().queueId as string;

    const page = await app.inject({
      method: 'GET',
      url: '/api/v1/fleet/messages/outbox?status=open&limit=10&offset=0',
    });
    expect(page.statusCode).toBe(200);
    expect(page.json()).toEqual({
      items: [
        expect.objectContaining({
          id: queueId,
          instanceId,
          instanceName: 'Sleeping Node',
          phoneNumber: '13900139000',
          content: 'Wait for the node',
          status: 'queued',
          attempts: 0,
        }),
      ],
      total: 1,
      summary: { queued: 1, sending: 0, failed: 0, sent: 0 },
    });

    // Still offline, so a manual drain defers the item instead of burning an attempt.
    const deferred = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/outbox/flush',
    });
    expect(deferred.statusCode).toBe(200);
    expect(deferred.json()).toEqual({
      attempted: 1,
      delivered: 0,
      deferred: 1,
      failed: 0,
      remaining: 1,
    });
    expect(delivered).toHaveLength(0);

    const cancelled = await app.inject({
      method: 'POST',
      url: `/api/v1/fleet/messages/outbox/${queueId}/cancel`,
    });
    expect(cancelled.statusCode).toBe(200);
    expect(cancelled.json()).toMatchObject({ id: queueId, status: 'cancelled' });

    const retried = await app.inject({
      method: 'POST',
      url: `/api/v1/fleet/messages/outbox/${queueId}/retry`,
    });
    expect(retried.statusCode).toBe(200);
    expect(retried.json()).toMatchObject({ id: queueId, status: 'queued', attempts: 1 });

    // The node comes back: the snapshot is refreshed and the queue drains for real.
    db.prepare(`UPDATE status_snapshots SET expires_at=? WHERE id='snap-outbox'`).run(
      '2099-01-01T00:00:00.000Z',
    );
    const flushed = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/outbox/flush',
    });
    expect(flushed.statusCode).toBe(200);
    expect(flushed.json()).toEqual({
      attempted: 1,
      delivered: 1,
      deferred: 0,
      failed: 0,
      remaining: 0,
    });
    expect(delivered).toHaveLength(1);

    // A reachable node keeps the old fast path: no queue row at all.
    const direct = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/send',
      payload: { instanceId, phoneNumber: '13900139000', content: 'Delivered now' },
    });
    expect(direct.json()).toEqual({ sent: true, queued: false, instanceId });
    expect(delivered).toHaveLength(2);

    const dropped = await app.inject({
      method: 'DELETE',
      url: `/api/v1/fleet/messages/outbox/${queueId}`,
    });
    expect(dropped.statusCode).toBe(200);
    expect(dropped.json()).toEqual({ removed: true, queueId });
    expect(
      (await app.inject({ method: 'GET', url: '/api/v1/fleet/messages/outbox?status=all' })).json(),
    ).toMatchObject({ total: 0 });

    const badId = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/outbox/not%20an%20id/cancel',
    });
    expect(badId.statusCode).toBe(400);
    const missing = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/messages/outbox/q-999/retry',
    });
    expect(missing.statusCode).toBe(404);
    expect(missing.json()).toMatchObject({ status: 404, code: 'NOT_FOUND' });
    await app.close();
  });

  it('aggregates notification channels, rules, logs and queue in the Fleet center', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async (request) => {
          if (request.url.endsWith('/api/auth/login'))
            return {
              status: 200,
              headers: { 'set-cookie': 'simadmin_session=fleet-notification-session' },
              body: '{"status":"success"}',
            };
          if (request.url.endsWith('/api/notifications/config'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({
                status: 'success',
                data: {
                  channels: [
                    { type: 'webhook', enabled: true },
                    { type: 'bark', enabled: false },
                    { type: 'wecom_robot', enabled: true },
                  ],
                  rules: [{ enabled: true }, { enabled: false }],
                },
              }),
            };
          if (request.url.includes('/api/notifications/logs'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({
                status: 'success',
                data: {
                  total: 5,
                  logs: [
                    {
                      id: 'log-1',
                      event_type: 'balance',
                      status: 'success',
                      rule_name: 'Balance Alert',
                      channel_name: 'webhook',
                      created_at: '2026-07-17T01:02:00.000Z',
                    },
                    {
                      id: 'log-2',
                      event_type: 'low-signal',
                      status: 'failed',
                      rule_name: 'Signal Alert',
                      channel_name: 'bark',
                      created_at: '2026-07-17T00:59:00.000Z',
                    },
                    {
                      id: 'log-3',
                      event_type: 'quiet',
                      status: 'quiet_hours',
                      created_at: '2026-07-17T00:58:00.000Z',
                    },
                    {
                      id: 'log-4',
                      event_type: 'unknown',
                      status: 'unmatched',
                      created_at: '2026-07-17T00:57:00.000Z',
                    },
                    {
                      id: 'log-5',
                      event_type: 'no-channel',
                      status: 'no_available_channel',
                      created_at: '2026-07-17T00:56:00.000Z',
                    },
                  ],
                },
              }),
            };
          if (
            request.url.includes('/api/notifications/queue') &&
            (request.method === 'POST' || request.method === 'DELETE')
          )
            return {
              status: 200,
              headers: {},
              body: '{"status":"success"}',
            };
          if (request.url.includes('/api/notifications/queue'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({
                status: 'success',
                data: {
                  total: 3,
                  items: [
                    {
                      id: 'q-1',
                      status: 'pending',
                      event_type: 'balance',
                      rule_name: 'Balance Alert',
                      channel_name: 'webhook',
                      created_at: '2026-07-17T01:03:00.000Z',
                    },
                    {
                      id: 'q-2',
                      status: 'scheduled',
                      event_type: 'report',
                      created_at: '2026-07-17T01:04:00.000Z',
                    },
                    {
                      id: 'q-3',
                      status: 'retrying',
                      event_type: 'restart',
                      created_at: '2026-07-17T01:05:00.000Z',
                    },
                  ],
                },
              }),
            };
          return { status: 200, headers: {}, body: '{"status":"success","data":{}}' };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
      now: () => new Date('2026-07-16T12:00:00.000Z'),
    });
    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: {
        name: 'Notification Node',
        origin: 'http://192.168.1.50:8080',
        password: { action: 'set', password: '[REDACTED]' },
      },
    });
    expect(created.statusCode).toBe(201);
    const instanceId = created.json().id as string;

    const response = await app.inject({
      method: 'GET',
      url: '/api/v1/fleet/notifications',
    });

    expect(response.statusCode).toBe(200);
    expect(response.json()).toMatchObject({
      observedAt: expect.any(String),
      deviceCount: 1,
      readyCount: 1,
      unavailableCount: 0,
      devices: [{ id: instanceId, name: 'Notification Node', state: 'ready' }],
      config: {
        channelCount: 3,
        channelEnabled: 2,
        ruleCount: 2,
        ruleEnabled: 1,
        channelTypes: [
          { type: 'webhook', total: 1, enabled: 1 },
          { type: 'bark', total: 1, enabled: 0 },
          { type: 'wecom_robot', total: 1, enabled: 1 },
        ],
      },
      logs: {
        total: 5,
        success: 1,
        failed: 1,
        quietHours: 1,
        unmatched: 1,
        noAvailableChannel: 1,
        other: 0,
      },
      queue: {
        total: 3,
        pending: 1,
        scheduled: 1,
        retrying: 1,
        sending: 0,
        failed: 0,
      },
    });
    expect(response.json().logs.recent[0]).toMatchObject({
      id: 'log-1',
      eventType: 'balance',
      status: 'success',
      ruleName: 'Balance Alert',
      channelName: 'webhook',
    });
    expect(response.json().queue.recent[0]).toMatchObject({
      id: 'q-1',
      eventType: 'balance',
      status: 'pending',
    });

    const requested = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/notifications/queue/retry-all',
    });
    expect(requested.statusCode).toBe(200);
    expect(requested.json()).toEqual({
      requested: 1,
      succeeded: 1,
      failed: 0,
      skipped: 0,
      failures: [],
    });

    const retried = await app.inject({
      method: 'POST',
      url: `/api/v1/fleet/notifications/queue/${instanceId}/items/q-1/retry`,
    });
    expect(retried.statusCode).toBe(200);
    expect(retried.json()).toEqual({ retried: true });

    const deleted = await app.inject({
      method: 'DELETE',
      url: `/api/v1/fleet/notifications/queue/${instanceId}/items/q-1`,
    });
    expect(deleted.statusCode).toBe(200);
    expect(deleted.json()).toEqual({ deleted: true });

    const missing = await app.inject({
      method: 'POST',
      url: '/api/v1/fleet/notifications/queue/missing-instance/items/q-1/retry',
    });
    expect(missing.statusCode).toBe(404);
    expect(missing.json()).toMatchObject({ status: 404, code: 'NOT_FOUND' });

    const invalidQueueId = await app.inject({
      method: 'POST',
      url: `/api/v1/fleet/notifications/queue/${instanceId}/items/bad%20id/retry`,
    });
    expect(invalidQueueId.statusCode).toBe(400);
    expect(invalidQueueId.json()).toMatchObject({ status: 400, code: 'VALIDATION_FAILED' });
    await app.close();
  });

  it('attaches an origin-bound session cookie when executing restart operations', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const calls: unknown[] = [];
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async (request) => {
          calls.push({ kind: 'auth', request });
          if (request.url.endsWith('/api/auth/login'))
            return {
              status: 200,
              headers: { 'set-cookie': 'simadmin_session=opaque-token' },
              body: '',
            };
          return { status: 200, headers: {}, body: '' };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async (origin, cookie) => {
          calls.push({ kind: 'restart', origin, cookie });
          return { status: 200 };
        },
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async (origin, delaySeconds, cookie) => {
          calls.push({ kind: 'reboot', origin, delaySeconds, cookie });
          return { status: 200 };
        },
      },
      now: () => new Date('2026-07-16T12:00:00.000Z'),
    });
    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: {
        name: 'Restart Target',
        origin: 'http://192.168.1.10:8080',
        password: { action: 'set', password: '[REDACTED]' },
      },
    });
    expect(created.statusCode).toBe(201);
    const instanceId = created.json().id as string;
    const revision = created.json().revision as number;
    const prepared = await app.inject({
      method: 'POST',
      url: '/api/v1/operations/prepare',
      payload: {
        operationId: 'postServiceRestart',
        targets: [{ instanceId, revision }],
        parameters: {
          parameterSchemaId: 'simadmin.58e2204.postServiceRestart.parameters.v1',
          fields: [],
        },
      },
    });
    expect(prepared.statusCode).toBe(200);
    const executed = await app.inject({
      method: 'POST',
      url: '/api/v1/operations/execute',
      payload: {
        preparationId: prepared.json().id,
        confirmationToken: prepared.json().confirmationToken,
      },
    });
    expect(executed.statusCode).toBe(202);
    expect(executed.json()).toMatchObject({
      operationId: 'postServiceRestart',
      status: 'succeeded',
    });
    expect(calls).toContainEqual({
      kind: 'restart',
      origin: 'http://192.168.1.10:8080',
      cookie: 'simadmin_session=opaque-token',
    });
    // Auth request body uses the redacted marker; the actual secret must never appear.
    expect(JSON.stringify(calls)).toContain('"[REDACTED]"');
    expect(JSON.stringify(calls)).not.toMatch(/never-leak|password":"[^[]/);
    await app.close();
  });

  it('maps typed upstream failures to redacted gateway Problem Details', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => {
          throw new UpstreamError('UPSTREAM_UNAVAILABLE', {
            cause: new Error('ECONNREFUSED 192.168.1.99 cookie=simadmin_session=secret'),
          });
        },
        request: async () => ({ status: 200, headers: {}, body: '' }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });
    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: { name: 'B', origin: 'http://192.168.1.99' },
    });
    const response = await app.inject({
      method: 'POST',
      url: '/api/v1/instances/' + created.json().id + '/test-connection',
    });
    expect(response.statusCode).toBe(502);
    expect(response.headers['content-type']).toContain('application/problem+json');
    expect(response.json()).toMatchObject({
      title: 'Bad Gateway',
      status: 502,
      code: 'UPSTREAM_UNAVAILABLE',
    });
    expect(response.body).not.toContain('ECONNREFUSED');
    expect(response.body).not.toContain('192.168.1.99');
    expect(response.body).not.toContain('simadmin_session');
    await app.close();
  });

  it('keeps a failed SMS node visible instead of pretending it has no messages', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    let smsListCall = 0;
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async (request) => {
          if (request.url.endsWith('/api/auth/login'))
            return {
              status: 200,
              headers: { 'set-cookie': 'simadmin_session=fleet-availability-session' },
              body: '{"status":"success"}',
            };
          if (request.url.includes('/api/sms/list') && ++smsListCall === 1)
            return { status: 502, headers: {}, body: '{"status":"error"}' };
          if (request.url.includes('/api/sms/list'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({
                status: 'success',
                data: {
                  messages: [
                    {
                      id: 1,
                      direction: 'incoming',
                      phone_number: '10086',
                      content: 'Healthy node',
                      timestamp: '2026-07-17T01:00:00.000Z',
                      status: 'received',
                    },
                  ],
                },
              }),
            };
          return { status: 200, headers: {}, body: '{"status":"success","data":{}}' };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });

    for (const [index, name] of ['Healthy', 'Broken'].entries()) {
      const created = await app.inject({
        method: 'POST',
        url: '/api/v1/instances',
        payload: {
          name,
          origin: `http://192.168.1.${40 + index}:8080`,
          password: { action: 'set', password: '[REDACTED]' },
        },
      });
      expect(created.statusCode).toBe(201);
    }

    const response = await app.inject({ method: 'GET', url: '/api/v1/fleet/messages' });

    expect(response.statusCode).toBe(200);
    const body = response.json();
    expect(body.devices).toEqual(
      expect.arrayContaining([
        expect.objectContaining({ name: 'Healthy', availability: 'online' }),
        expect.objectContaining({ name: 'Broken', availability: 'unavailable' }),
      ]),
    );
    expect(body.messages).toEqual([expect.objectContaining({ content: 'Healthy node' })]);
    expect(body.total).toBe(1);
    await app.close();
  });

  it('keeps a node that refuses to release its hub binding and drops one that is unreachable', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const unbinds: string[] = [];
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async (url) => {
          if (url.includes('192.168.1.21')) throw new Error('ECONNREFUSED');
          return { status: 200, headers: {}, body: '' };
        },
        request: async ({ url, method }) => {
          if (method === 'POST' && url.endsWith('/api/hub/unbind')) {
            unbinds.push(url);
            return { status: 409, headers: {}, body: '{"status":"error","message":"busy"}' };
          }
          return { status: 200, headers: {}, body: '{"status":"success"}' };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });
    async function remove(instanceId: string): Promise<Record<string, unknown>> {
      const prepared = await app.inject({
        method: 'POST',
        url: '/api/v1/operations/prepare',
        payload: {
          operationId: 'deleteInstance',
          targets: [{ instanceId, revision: 1 }],
          parameters: { parameterSchemaId: 'deleteInstance.parameters.v1', fields: [] },
        },
      });
      expect(prepared.statusCode).toBe(200);
      const executed = await app.inject({
        method: 'DELETE',
        url: `/api/v1/instances/${encodeURIComponent(instanceId)}`,
        headers: {
          'if-match': '"rev-1"',
          'x-preparation-id': prepared.json().id as string,
          'x-confirmation-token': prepared.json().confirmationToken as string,
        },
      });
      expect(executed.statusCode).toBe(202);
      return executed.json() as Record<string, unknown>;
    }

    const created: string[] = [];
    for (const [name, host] of [
      ['Refusing', '192.168.1.20'],
      ['Unreachable', '192.168.1.21'],
    ]) {
      const response = await app.inject({
        method: 'POST',
        url: '/api/v1/instances',
        payload: { name, origin: `http://${host}:8080` },
      });
      expect(response.statusCode).toBe(201);
      created.push(response.json().id as string);
    }
    const [refused, gone] = created;

    expect(await remove(refused!)).toMatchObject({
      status: 'failed',
      items: [{ state: 'failed', error: { code: 'UNBIND_FAILED' } }],
    });
    expect(await remove(gone!)).toMatchObject({ status: 'succeeded' });

    const listed = await app.inject('/api/v1/instances');
    expect((listed.json().items as Array<{ id: string }>).map((item) => item.id)).toEqual([
      refused,
    ]);
    expect(unbinds).toEqual([`http://192.168.1.20:8080/api/hub/unbind`]);
    await app.close();
  });

  it('holds controls for two nodes that claim the same device identity', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const imei = '860000000000007';
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async ({ url, method }) => {
          if (url.endsWith('/api/device'))
            return {
              status: 200,
              headers: {},
              body: JSON.stringify({ imei, model: 'RM500Q', manufacturer: 'Quectel' }),
            };
          return {
            status: method === 'POST' ? 200 : 404,
            headers: {},
            body: '{"status":"success"}',
          };
        },
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });
    const created: string[] = [];
    for (const host of ['192.168.1.30', '192.168.1.31']) {
      const response = await app.inject({
        method: 'POST',
        url: '/api/v1/instances',
        payload: { name: `节点 ${host}`, origin: `http://${host}:8080` },
      });
      created.push(response.json().id as string);
    }
    const [first, second] = created;

    // Reading the overview is enough: the device reports its own hardware and the guard notices
    // that two records now answer with one IMEI.
    for (const id of created) {
      const snapshot = await app.inject(`/api/v1/instances/${id}/modules/overview`);
      expect(snapshot.statusCode).toBe(200);
    }

    const listed = await app.inject('/api/v1/fleet/identities');
    expect(listed.json()).toMatchObject({
      summary: { tracked: 2, pending: 2 },
      items: [
        { instanceId: first, reasons: ['imei_claimed'] },
        { instanceId: second, reasons: ['imei_claimed'] },
      ],
    });

    const blocked = await app.inject({
      method: 'POST',
      url: `/api/v1/instances/${first}/device-actions/call.waiting`,
      payload: { confirm: true, params: { value: 'enabled' } },
    });
    expect(blocked.statusCode).toBe(409);
    expect(blocked.json()).toMatchObject({ code: 'IDENTITY_UNCONFIRMED' });

    const overview = await app.inject('/api/v1/fleet/overview');
    expect(overview.json().items[0].identity).toEqual({
      tracked: true,
      status: 'pending',
      reasons: ['imei_claimed'],
    });

    // The operator vouches for one node; the other keeps waiting.
    const confirmed = await app.inject({
      method: 'POST',
      url: `/api/v1/fleet/identities/${first}/confirm`,
    });
    expect(confirmed.json()).toMatchObject({
      identity: { instanceId: first, status: 'confirmed', reasons: [] },
    });

    const allowed = await app.inject({
      method: 'POST',
      url: `/api/v1/instances/${first}/device-actions/call.waiting`,
      payload: { confirm: true, params: { value: 'enabled' } },
    });
    expect(allowed.statusCode).toBe(200);
    await app.close();
  });

  it('scrapes the fused fleet through the metrics endpoint', async () => {
    const db = new Database(':memory:');
    db.pragma('foreign_keys=ON');
    migrateDatabase(db);
    dbs.push(db);
    const app = buildControlPlaneApp({
      db,
      store: new Store(),
      runtimeVersion: '9.9.9',
      upstream: {
        get: async () => ({ status: 200, headers: {}, body: '' }),
        request: async ({ method }) => ({
          status: method === 'POST' ? 200 : 404,
          headers: {},
          body: '{"status":"success"}',
        }),
        postNetworkRegisterAuto: async () => ({ status: 200 }),
        postServiceRestart: async () => ({ status: 200 }),
        postBasebandRestart: async () => ({ status: 200 }),
        postSystemReboot: async () => ({ status: 200 }),
      },
    });
    const created = await app.inject({
      method: 'POST',
      url: '/api/v1/instances',
      payload: { name: '节点 A', origin: 'http://192.168.1.40:8080' },
    });
    expect(created.statusCode).toBe(201);

    const response = await app.inject({ method: 'GET', url: '/api/v1/metrics' });
    expect(response.statusCode).toBe(200);
    expect(response.headers['content-type']).toContain('text/plain');
    expect(response.body).toContain('multi_simadmin_build_info{version="9.9.9"} 1');
    expect(response.body).toContain('multi_simadmin_nodes_total 1');
    expect(response.body).toContain('multi_simadmin_notification_queue{status="pending"} 0');
    await app.close();
  });
});