multi-simadmin/apps/api/src/application/observability/metrics-service.ts

import type Database from 'better-sqlite3';

import type { ConnectionSummary } from '../system/connection-log-service.js';
import type { DeviceIdentity } from '../identity/device-identity-service.js';

/**
 * Prometheus exposition for the fused control plane.
 *
 * The scrape must never reach a device: every value here comes from rows the console already
 * writes (instances, probes, queues, jobs, identity records). A monitoring system polling every
 * 15 seconds should not be able to wake a modem.
 */

const PREFIX = 'multi_simadmin';

export interface MetricsSource {
  readonly connections: { summarize(): readonly ConnectionSummary[] };
  readonly identities: { list(): readonly DeviceIdentity[] };
}

export interface MetricsOptions {
  readonly db: Database.Database;
  readonly version: string;
  readonly sources: MetricsSource;
  readonly now?: () => Date;
}

interface CountRow {
  readonly status: string;
  readonly count: number;
}

/** Label values are operator- or device-controlled, so they are escaped the Prometheus way. */
function label(value: string): string {
  return value.replace(/\\/gu, '\\\\').replace(/"/gu, '\\"').replace(/\n/gu, '\\n');
}

const number = (value: unknown): number => {
  const parsed = typeof value === 'number' ? value : Number(value);
  return Number.isFinite(parsed) ? parsed : 0;
};

function sample(name: string, help: string, type: 'gauge' | 'counter', lines: readonly string[]) {
  return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...lines];
}

export class MetricsService {
  readonly #db: Database.Database;
  readonly #version: string;
  readonly #sources: MetricsSource;
  readonly #clock: () => Date;

  constructor(options: MetricsOptions) {
    this.#db = options.db;
    this.#version = options.version;
    this.#sources = options.sources;
    this.#clock = options.now ?? (() => new Date());
  }

  render(): string {
    const nodes = this.#db
      .prepare('SELECT id,name,enabled FROM instances ORDER BY name, id')
      .all() as Array<{
      id: string;
      name: string;
      enabled: number;
    }>;
    const ids = new Set(nodes.map((node) => node.id));
    const lines: string[] = [];

    lines.push(
      ...sample(`${PREFIX}_build_info`, 'Build identity of the control plane.', 'gauge', [
        `${PREFIX}_build_info{version="${label(this.#version)}"} 1`,
      ]),
      ...sample(`${PREFIX}_scrape_timestamp_seconds`, 'Wall-clock time of this scrape.', 'gauge', [
        `${PREFIX}_scrape_timestamp_seconds ${Math.floor(this.#clock().getTime() / 1000)}`,
      ]),
    );

    // Operator-assigned names can leak hostnames/locations to unauthenticated
    // scrapers; the series intentionally stays opaque to node ids.
    const info = nodes.map((node) => `${PREFIX}_node_info{node="${label(node.id)}"} 1`);
    lines.push(
      ...sample(`${PREFIX}_node_info`, 'Registered nodes and their addresses.', 'gauge', info),
      ...sample(`${PREFIX}_nodes_total`, 'Number of registered nodes.', 'gauge', [
        `${PREFIX}_nodes_total ${nodes.length}`,
      ]),
      ...sample(
        `${PREFIX}_node_enabled`,
        'Whether the node is enabled (1) or paused (0).',
        'gauge',
        nodes.map(
          (node) => `${PREFIX}_node_enabled{node="${label(node.id)}"} ${node.enabled ? 1 : 0}`,
        ),
      ),
    );

    const probes = this.#sources.connections.summarize();
    lines.push(
      ...sample(
        `${PREFIX}_node_probe_total`,
        'Health probes recorded in the retained window, by outcome.',
        'counter',
        probes.flatMap((probe) => [
          `${PREFIX}_node_probe_total{node="${label(probe.instanceId)}",outcome="success"} ${probe.success}`,
          `${PREFIX}_node_probe_total{node="${label(probe.instanceId)}",outcome="failed"} ${probe.failed}`,
        ]),
      ),
      ...sample(
        `${PREFIX}_node_availability_ratio`,
        'Share of successful probes in the retained window.',
        'gauge',
        probes.map(
          (probe) =>
            `${PREFIX}_node_availability_ratio{node="${label(probe.instanceId)}"} ${(probe.availabilityPercent / 100).toFixed(3)}`,
        ),
      ),
      ...sample(
        `${PREFIX}_node_probe_duration_ms`,
        'Average probe latency in the retained window.',
        'gauge',
        probes.map(
          (probe) =>
            `${PREFIX}_node_probe_duration_ms{node="${label(probe.instanceId)}"} ${number(probe.averageDurationMs)}`,
        ),
      ),
    );

    const identities = this.#sources.identities.list();
    const pending = identities.filter((identity) => identity.status === 'pending');
    lines.push(
      ...sample(
        `${PREFIX}_node_identity_tracked`,
        'Whether the node has ever reported its hardware identity.',
        'gauge',
        identities.map(
          (identity) => `${PREFIX}_node_identity_tracked{node="${label(identity.instanceId)}"} 1`,
        ),
      ),
      ...sample(
        `${PREFIX}_node_identity_pending`,
        '1 while the identity guard holds the node and control actions are refused.',
        'gauge',
        identities.map(
          (identity) =>
            `${PREFIX}_node_identity_pending{node="${label(identity.instanceId)}"} ${identity.status === 'pending' ? 1 : 0}`,
        ),
      ),
      ...sample(
        `${PREFIX}_identity_pending_total`,
        'Nodes waiting for an operator to confirm the device behind them.',
        'gauge',
        [`${PREFIX}_identity_pending_total ${pending.length}`],
      ),
    );

    lines.push(
      ...this.#statusMetric(
        `${PREFIX}_sms_outbox`,
        'Queued cross-node SMS sends, by state.',
        'SELECT status, COUNT(*) AS count FROM sms_outbox GROUP BY status',
        ['queued', 'sending', 'sent', 'failed', 'cancelled'],
      ),
      ...this.#statusMetric(
        `${PREFIX}_notification_queue`,
        'Central notification queue, by state.',
        'SELECT status, COUNT(*) AS count FROM notification_queue GROUP BY status',
        ['pending', 'sending', 'succeeded', 'failed', 'cancelled'],
      ),
      ...this.#statusMetric(
        `${PREFIX}_jobs`,
        'Fleet operations, by state.',
        'SELECT status, COUNT(*) AS count FROM jobs GROUP BY status',
        [
          'queued',
          'running',
          'cancelling',
          'succeeded',
          'partially-succeeded',
          'failed',
          'cancelled',
          'unknown-result',
        ],
      ),
      ...this.#statusMetric(
        `${PREFIX}_scheduled_tasks`,
        'Automation tasks, by state.',
        `SELECT CASE WHEN enabled=1 THEN 'armed' ELSE 'paused' END AS status,
                COUNT(*) AS count FROM scheduled_tasks GROUP BY enabled`,
        ['armed', 'paused'],
      ),
    );

    const groups = this.#count('SELECT COUNT(*) AS count FROM device_groups');
    const tags = this.#count('SELECT COUNT(*) AS count FROM tag_registry');
    const messages = this.#count('SELECT COUNT(*) AS count FROM sms_messages');
    const probed = new Set(probes.map((probe) => probe.instanceId));
    const unprobed = [...ids].filter((id) => !probed.has(id)).length;
    lines.push(
      ...sample(`${PREFIX}_device_groups`, 'Node groups.', 'gauge', [
        `${PREFIX}_device_groups ${groups}`,
      ]),
      ...sample(`${PREFIX}_device_tags`, 'Labels in the tag registry.', 'gauge', [
        `${PREFIX}_device_tags ${tags}`,
      ]),
      ...sample(
        `${PREFIX}_sms_messages`,
        'SMS rows synchronised into the central message centre.',
        'gauge',
        [`${PREFIX}_sms_messages ${messages}`],
      ),
      ...sample(
        `${PREFIX}_node_unprobed`,
        'Registered nodes without a health probe in the retained window.',
        'gauge',
        [`${PREFIX}_node_unprobed ${unprobed}`],
      ),
    );

    return `${lines.join('\n')}\n`;
  }

  #count(sql: string): number {
    const row = this.#db.prepare(sql).get() as { count?: number } | undefined;
    return number(row?.count ?? 0);
  }

  /** Every state is emitted, including the empty ones, so a query never sees a series vanish. */
  #statusMetric(
    name: string,
    help: string,
    sql: string,
    states: readonly string[],
  ): readonly string[] {
    const rows = this.#db.prepare(sql).all() as readonly CountRow[];
    const counts = new Map(rows.map((row) => [String(row.status), number(row.count)]));
    return sample(
      name,
      help,
      'gauge',
      states.map((state) => `${name}{status="${label(state)}"} ${counts.get(state) ?? 0}`),
    );
  }
}