import type Database from 'better-sqlite3'; import type { ConnectionSummary } from '../system/connection-log-service.js'; import type { DeviceIdentity } from '../identity/device-identity-service.js'; /** * Prometheus exposition for the fused control plane. * * The scrape must never reach a device: every value here comes from rows the console already * writes (instances, probes, queues, jobs, identity records). A monitoring system polling every * 15 seconds should not be able to wake a modem. */ const PREFIX = 'multi_simadmin'; export interface MetricsSource { readonly connections: { summarize(): readonly ConnectionSummary[] }; readonly identities: { list(): readonly DeviceIdentity[] }; } export interface MetricsOptions { readonly db: Database.Database; readonly version: string; readonly sources: MetricsSource; readonly now?: () => Date; } interface CountRow { readonly status: string; readonly count: number; } /** Label values are operator- or device-controlled, so they are escaped the Prometheus way. */ function label(value: string): string { return value.replace(/\\/gu, '\\\\').replace(/"/gu, '\\"').replace(/\n/gu, '\\n'); } const number = (value: unknown): number => { const parsed = typeof value === 'number' ? value : Number(value); return Number.isFinite(parsed) ? parsed : 0; }; function sample(name: string, help: string, type: 'gauge' | 'counter', lines: readonly string[]) { return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...lines]; } export class MetricsService { readonly #db: Database.Database; readonly #version: string; readonly #sources: MetricsSource; readonly #clock: () => Date; constructor(options: MetricsOptions) { this.#db = options.db; this.#version = options.version; this.#sources = options.sources; this.#clock = options.now ?? (() => new Date()); } render(): string { const nodes = this.#db .prepare('SELECT id,name,enabled FROM instances ORDER BY name, id') .all() as Array<{ id: string; name: string; enabled: number; }>; const ids = new Set(nodes.map((node) => node.id)); const lines: string[] = []; lines.push( ...sample(`${PREFIX}_build_info`, 'Build identity of the control plane.', 'gauge', [ `${PREFIX}_build_info{version="${label(this.#version)}"} 1`, ]), ...sample(`${PREFIX}_scrape_timestamp_seconds`, 'Wall-clock time of this scrape.', 'gauge', [ `${PREFIX}_scrape_timestamp_seconds ${Math.floor(this.#clock().getTime() / 1000)}`, ]), ); // Operator-assigned names can leak hostnames/locations to unauthenticated // scrapers; the series intentionally stays opaque to node ids. const info = nodes.map((node) => `${PREFIX}_node_info{node="${label(node.id)}"} 1`); lines.push( ...sample(`${PREFIX}_node_info`, 'Registered nodes and their addresses.', 'gauge', info), ...sample(`${PREFIX}_nodes_total`, 'Number of registered nodes.', 'gauge', [ `${PREFIX}_nodes_total ${nodes.length}`, ]), ...sample( `${PREFIX}_node_enabled`, 'Whether the node is enabled (1) or paused (0).', 'gauge', nodes.map( (node) => `${PREFIX}_node_enabled{node="${label(node.id)}"} ${node.enabled ? 1 : 0}`, ), ), ); const probes = this.#sources.connections.summarize(); lines.push( ...sample( `${PREFIX}_node_probe_total`, 'Health probes recorded in the retained window, by outcome.', 'counter', probes.flatMap((probe) => [ `${PREFIX}_node_probe_total{node="${label(probe.instanceId)}",outcome="success"} ${probe.success}`, `${PREFIX}_node_probe_total{node="${label(probe.instanceId)}",outcome="failed"} ${probe.failed}`, ]), ), ...sample( `${PREFIX}_node_availability_ratio`, 'Share of successful probes in the retained window.', 'gauge', probes.map( (probe) => `${PREFIX}_node_availability_ratio{node="${label(probe.instanceId)}"} ${(probe.availabilityPercent / 100).toFixed(3)}`, ), ), ...sample( `${PREFIX}_node_probe_duration_ms`, 'Average probe latency in the retained window.', 'gauge', probes.map( (probe) => `${PREFIX}_node_probe_duration_ms{node="${label(probe.instanceId)}"} ${number(probe.averageDurationMs)}`, ), ), ); const identities = this.#sources.identities.list(); const pending = identities.filter((identity) => identity.status === 'pending'); lines.push( ...sample( `${PREFIX}_node_identity_tracked`, 'Whether the node has ever reported its hardware identity.', 'gauge', identities.map( (identity) => `${PREFIX}_node_identity_tracked{node="${label(identity.instanceId)}"} 1`, ), ), ...sample( `${PREFIX}_node_identity_pending`, '1 while the identity guard holds the node and control actions are refused.', 'gauge', identities.map( (identity) => `${PREFIX}_node_identity_pending{node="${label(identity.instanceId)}"} ${identity.status === 'pending' ? 1 : 0}`, ), ), ...sample( `${PREFIX}_identity_pending_total`, 'Nodes waiting for an operator to confirm the device behind them.', 'gauge', [`${PREFIX}_identity_pending_total ${pending.length}`], ), ); lines.push( ...this.#statusMetric( `${PREFIX}_sms_outbox`, 'Queued cross-node SMS sends, by state.', 'SELECT status, COUNT(*) AS count FROM sms_outbox GROUP BY status', ['queued', 'sending', 'sent', 'failed', 'cancelled'], ), ...this.#statusMetric( `${PREFIX}_notification_queue`, 'Central notification queue, by state.', 'SELECT status, COUNT(*) AS count FROM notification_queue GROUP BY status', ['pending', 'sending', 'succeeded', 'failed', 'cancelled'], ), ...this.#statusMetric( `${PREFIX}_jobs`, 'Fleet operations, by state.', 'SELECT status, COUNT(*) AS count FROM jobs GROUP BY status', [ 'queued', 'running', 'cancelling', 'succeeded', 'partially-succeeded', 'failed', 'cancelled', 'unknown-result', ], ), ...this.#statusMetric( `${PREFIX}_scheduled_tasks`, 'Automation tasks, by state.', `SELECT CASE WHEN enabled=1 THEN 'armed' ELSE 'paused' END AS status, COUNT(*) AS count FROM scheduled_tasks GROUP BY enabled`, ['armed', 'paused'], ), ); const groups = this.#count('SELECT COUNT(*) AS count FROM device_groups'); const tags = this.#count('SELECT COUNT(*) AS count FROM tag_registry'); const messages = this.#count('SELECT COUNT(*) AS count FROM sms_messages'); const probed = new Set(probes.map((probe) => probe.instanceId)); const unprobed = [...ids].filter((id) => !probed.has(id)).length; lines.push( ...sample(`${PREFIX}_device_groups`, 'Node groups.', 'gauge', [ `${PREFIX}_device_groups ${groups}`, ]), ...sample(`${PREFIX}_device_tags`, 'Labels in the tag registry.', 'gauge', [ `${PREFIX}_device_tags ${tags}`, ]), ...sample( `${PREFIX}_sms_messages`, 'SMS rows synchronised into the central message centre.', 'gauge', [`${PREFIX}_sms_messages ${messages}`], ), ...sample( `${PREFIX}_node_unprobed`, 'Registered nodes without a health probe in the retained window.', 'gauge', [`${PREFIX}_node_unprobed ${unprobed}`], ), ); return `${lines.join('\n')}\n`; } #count(sql: string): number { const row = this.#db.prepare(sql).get() as { count?: number } | undefined; return number(row?.count ?? 0); } /** Every state is emitted, including the empty ones, so a query never sees a series vanish. */ #statusMetric( name: string, help: string, sql: string, states: readonly string[], ): readonly string[] { const rows = this.#db.prepare(sql).all() as readonly CountRow[]; const counts = new Map(rows.map((row) => [String(row.status), number(row.count)])); return sample( name, help, 'gauge', states.map((state) => `${name}{status="${label(state)}"} ${counts.get(state) ?? 0}`), ); } }