feat(api): add device identity guard and Prometheus metrics

- Persist device identity observations and expose fleet identity summaries. - Block device control actions when identity verification fails. - Add metrics collection and a Prometheus scrape endpoint.
2026-09-07 00:44:51 +08:00
parent 8406f14469
commit 961eb928c4
14 changed files with 1619 additions and 4 deletions
@@ -7,6 +7,8 @@ import {

 export interface DeviceActionRoutesOptions {
  readonly actions: DeviceActionService;
+  /** The identity guard, so the console can grey controls out before the operator tries one. */
+  readonly identities?: { isBlocked(instanceId: string): boolean };
 }

 function problem(
@@ -21,7 +23,7 @@ function problem(
    .type('application/problem+json')
    .send({
      type: 'about:blank',
-      title: status === 404 ? 'Not Found' : 'Bad Request',
+      title: titleFor(status),
      status,
      code,
      detail,
@@ -35,6 +37,8 @@ const statusFor = (code: DeviceActionError['code']): number => {
      return 404;
    case 'SESSION_INVALID':
      return 401;
+    case 'IDENTITY_UNCONFIRMED':
+      return 409;
    case 'UPSTREAM_FAILED':
    case 'NOT_DISPATCHED':
      return 502;
@@ -43,6 +47,9 @@ const statusFor = (code: DeviceActionError['code']): number => {
  }
 };

+const titleFor = (status: number): string =>
+  status === 404 ? 'Not Found' : status === 409 ? 'Conflict' : 'Bad Request';
+
 export function registerDeviceActionRoutes(
  app: FastifyInstance,
  options: DeviceActionRoutesOptions,
@@ -51,7 +58,10 @@ export function registerDeviceActionRoutes(
    const instanceId = (request.params as { instanceId?: unknown }).instanceId;
    if (typeof instanceId !== 'string' || instanceId.length === 0)
      return problem(request, reply, 400, 'VALIDATION_FAILED', 'Instance id is required');
-    return { actions: options.actions.list() };
+    return {
+      actions: options.actions.list(),
+      identityBlocked: options.identities?.isBlocked(instanceId) === true,
+    };
  });

  app.post(
@@ -98,7 +108,9 @@ export function registerDeviceActionRoutes(
            reply,
            statusFor(error.code),
            error.code,
-            'The device action could not be completed.',
+            error.code === 'IDENTITY_UNCONFIRMED'
+              ? '设备身份待确认，请先核对 IMEI 与硬件信息后解除阻止。'
+              : 'The device action could not be completed.',
          );
        throw error;
      }
@@ -20,6 +20,11 @@ import {
  NotificationServiceError,
  type InstanceNotificationService,
 } from '../../application/notifications/instance-notification-service.js';
+import type {
+  DeviceIdentity,
+  DeviceIdentityService,
+  IdentityReason,
+} from '../../application/identity/device-identity-service.js';
 export interface FleetMessageDevice {
  readonly id: string;
  readonly name: string;
@@ -68,6 +73,14 @@ export interface FleetRoutesOptions {
  readonly connections?: ConnectionProbe;
  /** Offline send queue; absent means sends fail fast instead of waiting for the device. */
  readonly outbox?: SmsOutboxService;
+  /** Identity guard; absent means the fleet list has no opinion about hardware drift. */
+  readonly identities?: DeviceIdentityService;
+}
+
+export interface FleetIdentitySummary {
+  readonly tracked: boolean;
+  readonly status: 'confirmed' | 'pending';
+  readonly reasons: readonly IdentityReason[];
 }

 /** One queued send, with the node label the queue table itself does not store. */
@@ -184,6 +197,11 @@ function connectionSummary(state: ConnectionState | undefined): FleetConnectionS
  };
 }

+function identitySummary(identity: DeviceIdentity | undefined): FleetIdentitySummary {
+  if (!identity) return { tracked: false, status: 'confirmed', reasons: [] };
+  return { tracked: true, status: identity.status, reasons: identity.reasons };
+}
+
 function record(value: unknown): Record<string, unknown> | undefined {
  if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
  return value as Record<string, unknown>;
@@ -357,6 +375,7 @@ export function registerFleetRoutes(app: FastifyInstance, options: FleetRoutesOp
      ...instance,
      connection: connectionSummary(reachability?.get(instance.id)),
      resources: await options.resources.get(instance.id),
+      identity: identitySummary(options.identities?.get(instance.id)),
    }));
    return { items };
  });
@@ -0,0 +1,173 @@
+import Database from 'better-sqlite3';
+import Fastify from 'fastify';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { DeviceIdentityService } from '../../application/identity/device-identity-service.js';
+import type { InstanceModuleService } from '../../application/instances/instance-module-service.js';
+import type { InstanceService } from '../../application/instances/instance-service.js';
+import { migrateDatabase } from '../../infrastructure/database/migrations.js';
+import { registerIdentityRoutes } from './identity-routes.js';
+
+let subject: { app: ReturnType<typeof Fastify>; db: Database.Database } | undefined;
+
+const nodes: Record<string, string> = {
+  'node-a': '机房 A',
+  'node-b': '机房 B',
+};
+
+function fixture(read?: (instanceId: string) => Promise<void>) {
+  const db = new Database(':memory:');
+  db.pragma('foreign_keys=ON');
+  migrateDatabase(db);
+  // Identity rows hang off instances, so the fixture nodes have to exist for real.
+  const seed = db.prepare(
+    `INSERT INTO instances (id,name,base_url,auth_mode,enabled,config_revision,created_at,updated_at)
+     VALUES (?,?,'http://' || ? || ':8080','password',1,1,'2026-09-05T02:00:00.000Z','2026-09-05T02:00:00.000Z')`,
+  );
+  for (const [id, name] of Object.entries(nodes)) seed.run(id, name, id);
+  const identities = new DeviceIdentityService({ db });
+  const app = Fastify();
+  registerIdentityRoutes(app, {
+    identities,
+    instances: {
+      list: async () => ({
+        items: Object.entries(nodes).map(([id, name]) => ({ id, name })),
+      }),
+    } as unknown as InstanceService,
+    ...(read
+      ? {
+          modules: {
+            read: async (instanceId: string) => {
+              await read(instanceId);
+            },
+          } as unknown as InstanceModuleService,
+        }
+      : {}),
+  });
+  subject = { app, db };
+  return { app, identities, db };
+}
+
+afterEach(async () => {
+  await subject?.app.close();
+  subject?.db.close();
+  subject = undefined;
+});
+
+const report = { imei: '860000000000001', model: 'RM500Q', manufacturer: 'Quectel' };
+
+describe('identity routes', () => {
+  it('serves the guard view with node labels for every claimant', async () => {
+    const { app, identities } = fixture();
+    identities.observe('node-a', report);
+    identities.observe('node-b', report);
+
+    const listed = await app.inject({ method: 'GET', url: '/api/v1/fleet/identities' });
+
+    expect(listed.statusCode).toBe(200);
+    const body = listed.json() as {
+      summary: { tracked: number; pending: number };
+      items: Array<Record<string, unknown>>;
+    };
+    expect(body.summary).toEqual({ tracked: 2, pending: 2 });
+    expect(body.items).toHaveLength(2);
+    expect(body.items[0]).toMatchObject({
+      instanceId: 'node-a',
+      instanceName: '机房 A',
+      status: 'pending',
+      reasons: ['imei_claimed'],
+      conflicts: [{ reason: 'imei_claimed', instanceIds: ['node-b'], instanceNames: ['机房 B'] }],
+      imei: '860000000000001',
+    });
+  });
+
+  it('puts held nodes first no matter which one reported last', async () => {
+    const { app, identities } = fixture();
+    // node-b registers first, so an insertion-order listing would lead with the wrong record.
+    identities.observe('node-b', report);
+    identities.observe('node-a', report);
+    await app.inject({ method: 'POST', url: '/api/v1/fleet/identities/node-b/confirm' });
+
+    const listed = await app.inject({ method: 'GET', url: '/api/v1/fleet/identities' });
+    const body = listed.json() as { items: Array<{ instanceId: string; status: string }> };
+
+    expect(body.items.map((item) => [item.instanceId, item.status])).toEqual([
+      ['node-a', 'pending'],
+      ['node-b', 'confirmed'],
+    ]);
+  });
+
+  it('clears the guard for the node the operator vouches for', async () => {
+    const { app, identities } = fixture();
+    identities.observe('node-a', report);
+    identities.observe('node-b', report);
+
+    const confirmed = await app.inject({
+      method: 'POST',
+      url: '/api/v1/fleet/identities/node-a/confirm',
+    });
+
+    expect(confirmed.statusCode).toBe(200);
+    expect(confirmed.json()).toMatchObject({
+      identity: { instanceId: 'node-a', status: 'confirmed', reasons: [] },
+    });
+    expect(identities.isBlocked('node-a')).toBe(false);
+    expect(identities.isBlocked('node-b')).toBe(true);
+  });
+
+  it('answers 404 when there is nothing to confirm', async () => {
+    const { app } = fixture();
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/v1/fleet/identities/ghost/confirm',
+    });
+    expect(response.statusCode).toBe(404);
+    expect(response.json()).toMatchObject({ code: 'NOT_FOUND', status: 404 });
+  });
+
+  it('refreshes by reading the device once', async () => {
+    const observed = vi.fn();
+    const { app, identities } = fixture(async (instanceId) => {
+      observed(instanceId);
+      identities.observe(instanceId, report);
+    });
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/v1/fleet/identities/node-a/refresh',
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(observed).toHaveBeenCalledTimes(1);
+    expect(observed).toHaveBeenCalledWith('node-a');
+    expect(response.json()).toMatchObject({
+      observed: true,
+      identity: { instanceId: 'node-a', status: 'confirmed', model: 'RM500Q' },
+    });
+  });
+
+  it('says so when a device answers without any identity at all', async () => {
+    const { app } = fixture(async () => {});
+
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/v1/fleet/identities/node-a/refresh',
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.json()).toEqual({ observed: false });
+  });
+
+  it('reports a device that could not be reached for a refresh', async () => {
+    const { app } = fixture(async () => {
+      throw new Error('ECONNREFUSED');
+    });
+
+    const response = await app.inject({
+      method: 'POST',
+      url: '/api/v1/fleet/identities/node-a/refresh',
+    });
+
+    expect(response.statusCode).toBe(502);
+    expect(response.json()).toMatchObject({ code: 'UPSTREAM_FAILED' });
+  });
+});
@@ -0,0 +1,175 @@
+import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify';
+
+import {
+  DeviceIdentityError,
+  type DeviceIdentity,
+  type DeviceIdentityService,
+} from '../../application/identity/device-identity-service.js';
+import type { InstanceService } from '../../application/instances/instance-service.js';
+import type { InstanceModuleService } from '../../application/instances/instance-module-service.js';
+
+export interface IdentityRoutesOptions {
+  readonly identities: DeviceIdentityService;
+  readonly instances: InstanceService;
+  /** Optional so a control plane without device reads can still show what it already knows. */
+  readonly modules?: InstanceModuleService;
+}
+
+interface IdentityView {
+  readonly instanceId: string;
+  readonly instanceName: string;
+  readonly status: DeviceIdentity['status'];
+  readonly reasons: DeviceIdentity['reasons'];
+  readonly conflicts: ReadonlyArray<{
+    readonly reason: string;
+    readonly instanceIds: readonly string[];
+    readonly instanceNames: readonly string[];
+  }>;
+  readonly imei: string;
+  readonly manufacturer: string;
+  readonly model: string;
+  readonly revision: string;
+  readonly agent: string;
+  readonly origin: string;
+  readonly fingerprint: string;
+  readonly confirmedFingerprint: string;
+  readonly changes: DeviceIdentity['changes'];
+  readonly observedAt: string;
+  readonly confirmedAt: string;
+}
+
+const PAGE_SIZE = 100;
+const MAX_PAGES = 100;
+
+/**
+ * Pending records lead, then the node label the operator actually reads. Instance ids are random
+ * uuids, so sorting by them alone would shuffle the panel between two opens.
+ */
+function compareViews(a: IdentityView, b: IdentityView): number {
+  if (a.status !== b.status) return a.status === 'pending' ? -1 : 1;
+  const byName = (a.instanceName || a.instanceId).localeCompare(
+    b.instanceName || b.instanceId,
+    'zh-Hans-CN',
+    { numeric: true },
+  );
+  return byName !== 0 ? byName : a.instanceId.localeCompare(b.instanceId);
+}
+
+async function namesById(instances: InstanceService): Promise<ReadonlyMap<string, string>> {
+  const names = new Map<string, string>();
+  for (let page = 1; page <= MAX_PAGES; page += 1) {
+    const current = await instances.list({ page, pageSize: PAGE_SIZE });
+    for (const instance of current.items) names.set(instance.id, instance.name);
+    if (current.items.length < PAGE_SIZE) break;
+  }
+  return names;
+}
+
+function view(
+  identity: DeviceIdentity,
+  names: ReadonlyMap<string, string>,
+  fallback: string,
+): IdentityView {
+  return {
+    instanceId: identity.instanceId,
+    instanceName: names.get(identity.instanceId) ?? fallback,
+    status: identity.status,
+    reasons: identity.reasons,
+    conflicts: identity.conflicts.map((conflict) => ({
+      reason: conflict.reason,
+      instanceIds: conflict.instanceIds,
+      instanceNames: conflict.instanceIds.map((id) => names.get(id) ?? id),
+    })),
+    imei: identity.imei,
+    manufacturer: identity.manufacturer,
+    model: identity.model,
+    revision: identity.revision,
+    agent: identity.agent,
+    origin: identity.origin,
+    fingerprint: identity.fingerprint,
+    confirmedFingerprint: identity.confirmedFingerprint,
+    changes: identity.changes,
+    observedAt: identity.observedAt,
+    confirmedAt: identity.confirmedAt,
+  };
+}
+
+function problem(
+  request: FastifyRequest,
+  reply: FastifyReply,
+  status: number,
+  code: string,
+  detail: string,
+): FastifyReply {
+  return reply
+    .code(status)
+    .type('application/problem+json')
+    .send({
+      type: 'about:blank',
+      title: status === 404 ? 'Not Found' : 'Bad Request',
+      status,
+      code,
+      detail,
+      requestId: request.id,
+    });
+}
+
+function instanceIdOf(request: FastifyRequest): string {
+  const params = request.params as { instanceId?: unknown };
+  return typeof params.instanceId === 'string' ? params.instanceId : '';
+}
+
+/**
+ * The identity guard's console surface: what each node claims to be, which claims disagree, and
+ * the one action that clears a disagreement once the operator has looked at the evidence.
+ */
+export function registerIdentityRoutes(app: FastifyInstance, options: IdentityRoutesOptions): void {
+  app.get('/api/v1/fleet/identities', async () => {
+    const names = await namesById(options.instances);
+    const items = options.identities
+      .list()
+      .map((identity) => view(identity, names, ''))
+      .sort(compareViews);
+    const summary = options.identities.summarize();
+    return {
+      items,
+      summary: { tracked: summary.tracked, pending: summary.pending.length },
+    };
+  });
+
+  app.post('/api/v1/fleet/identities/:instanceId/confirm', async (request, reply) => {
+    const instanceId = instanceIdOf(request);
+    if (!instanceId)
+      return problem(request, reply, 400, 'VALIDATION_FAILED', 'Instance id is required');
+    try {
+      const identity = options.identities.confirm(instanceId);
+      const names = await namesById(options.instances);
+      return { identity: view(identity, names, instanceId) };
+    } catch (error) {
+      if (error instanceof DeviceIdentityError)
+        return problem(request, reply, 404, error.code, '该节点还没有可确认的身份记录。');
+      throw error;
+    }
+  });
+
+  // A refresh is one overview read: the device answers with its own hardware report and the
+  // guard re-decides, which is also how a node first earns an identity row.
+  app.post('/api/v1/fleet/identities/:instanceId/refresh', async (request, reply) => {
+    const instanceId = instanceIdOf(request);
+    if (!instanceId)
+      return problem(request, reply, 400, 'VALIDATION_FAILED', 'Instance id is required');
+    if (!options.modules)
+      return problem(request, reply, 400, 'VALIDATION_FAILED', 'Device reads are not available');
+    try {
+      await options.modules.read(instanceId, 'overview');
+    } catch {
+      return problem(request, reply, 502, 'UPSTREAM_FAILED', '设备未能返回身份信息，请稍后重试。');
+    }
+    const identity = options.identities.get(instanceId);
+    const names = await namesById(options.instances);
+    return {
+      observed: identity !== undefined,
+      ...(identity ? { identity: view(identity, names, instanceId) } : {}),
+    };
+  });
+}
@@ -0,0 +1,64 @@
+import Database from 'better-sqlite3';
+import Fastify from 'fastify';
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { ConsoleAuthService } from '../../application/auth/console-auth-service.js';
+import { DeviceIdentityService } from '../../application/identity/device-identity-service.js';
+import { MetricsService } from '../../application/observability/metrics-service.js';
+import { migrateDatabase } from '../../infrastructure/database/migrations.js';
+import { registerConsoleAuth } from './console-auth-routes.js';
+import { PROMETHEUS_CONTENT_TYPE, registerMetricsRoutes } from './metrics-routes.js';
+
+let subject: { app: ReturnType<typeof Fastify>; db: Database.Database } | undefined;
+
+function fixture() {
+  const db = new Database(':memory:');
+  db.pragma('foreign_keys=ON');
+  migrateDatabase(db);
+  db.prepare(
+    `INSERT INTO instances (id,name,base_url,auth_mode,enabled,config_revision,created_at,updated_at)
+     VALUES ('node-a','机房 A','http://10.0.0.9:8080','password',1,1,'2026-09-05T02:00:00.000Z','2026-09-05T02:00:00.000Z')`,
+  ).run();
+  const identities = new DeviceIdentityService({ db });
+  const metrics = new MetricsService({
+    db,
+    version: '0.1.0',
+    sources: { connections: { summarize: () => [] }, identities },
+  });
+  const app = Fastify();
+  // The real guard, so the test proves the exemption rather than asserting a predicate.
+  registerConsoleAuth(app, new ConsoleAuthService({ db }));
+  registerMetricsRoutes(app, { metrics });
+  subject = { app, db };
+  return { app, db, identities };
+}
+
+afterEach(async () => {
+  await subject?.app.close();
+  subject?.db.close();
+  subject = undefined;
+});
+
+describe('metrics routes', () => {
+  it('serves the exposition in the format Prometheus expects', async () => {
+    const { app } = fixture();
+    const response = await app.inject({ method: 'GET', url: '/api/v1/metrics' });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.headers['content-type']).toContain(PROMETHEUS_CONTENT_TYPE);
+    expect(response.body).toContain('multi_simadmin_nodes_total 1');
+    expect(response.body.endsWith('\n')).toBe(true);
+  });
+
+  it('stays open to a scraper while the console itself demands a session', async () => {
+    const { app, db } = fixture();
+    const auth = new ConsoleAuthService({ db });
+    await auth.updateSettings({ enabled: true, newPassword: 'passw0rd-1' });
+
+    const guarded = await app.inject({ method: 'GET', url: '/api/v1/instances' });
+    expect(guarded.statusCode).toBe(401);
+
+    const scraped = await app.inject({ method: 'GET', url: '/api/v1/metrics' });
+    expect(scraped.statusCode).toBe(200);
+  });
+});
@@ -0,0 +1,21 @@
+import type { FastifyInstance } from 'fastify';
+
+import type { MetricsService } from '../../application/observability/metrics-service.js';
+
+/** The content type Prometheus expects; anything else makes the scrape fail silently. */
+export const PROMETHEUS_CONTENT_TYPE = 'text/plain; version=0.0.4; charset=utf-8';
+
+export interface MetricsRoutesOptions {
+  readonly metrics: MetricsService;
+}
+
+/**
+ * Monitoring surface, modelled on the Hub's `/metrics`: no browser session, no cookies, just
+ * text. It is deliberately read-only and built from rows the console already writes, so a
+ * scraper polling every few seconds cannot wake a device.
+ */
+export function registerMetricsRoutes(app: FastifyInstance, options: MetricsRoutesOptions): void {
+  app.get('/api/v1/metrics', async (_request, reply) => {
+    return reply.type(PROMETHEUS_CONTENT_TYPE).send(options.metrics.render());
+  });
+}