feat(api): add device identity guard and Prometheus metrics
- Persist device identity observations and expose fleet identity summaries. - Block device control actions when identity verification fails. - Add metrics collection and a Prometheus scrape endpoint.
This commit is contained in:
chick
@@ -0,0 +1,244 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
|
||||
import type { ConnectionSummary } from '../system/connection-log-service.js';
|
||||
import type { DeviceIdentity } from '../identity/device-identity-service.js';
|
||||
|
||||
/**
|
||||
* Prometheus exposition for the fused control plane.
|
||||
*
|
||||
* The scrape must never reach a device: every value here comes from rows the console already
|
||||
* writes (instances, probes, queues, jobs, identity records). A monitoring system polling every
|
||||
* 15 seconds should not be able to wake a modem.
|
||||
*/
|
||||
|
||||
const PREFIX = 'multi_simadmin';
|
||||
|
||||
export interface MetricsSource {
|
||||
readonly connections: { summarize(): readonly ConnectionSummary[] };
|
||||
readonly identities: { list(): readonly DeviceIdentity[] };
|
||||
}
|
||||
|
||||
export interface MetricsOptions {
|
||||
readonly db: Database.Database;
|
||||
readonly version: string;
|
||||
readonly sources: MetricsSource;
|
||||
readonly now?: () => Date;
|
||||
}
|
||||
|
||||
interface CountRow {
|
||||
readonly status: string;
|
||||
readonly count: number;
|
||||
}
|
||||
|
||||
/** Label values are operator- or device-controlled, so they are escaped the Prometheus way. */
|
||||
function label(value: string): string {
|
||||
return value.replace(/\\/gu, '\\\\').replace(/"/gu, '\\"').replace(/\n/gu, '\\n');
|
||||
}
|
||||
|
||||
const number = (value: unknown): number => {
|
||||
const parsed = typeof value === 'number' ? value : Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : 0;
|
||||
};
|
||||
|
||||
function sample(name: string, help: string, type: 'gauge' | 'counter', lines: readonly string[]) {
|
||||
return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...lines];
|
||||
}
|
||||
|
||||
export class MetricsService {
|
||||
readonly #db: Database.Database;
|
||||
readonly #version: string;
|
||||
readonly #sources: MetricsSource;
|
||||
readonly #clock: () => Date;
|
||||
|
||||
constructor(options: MetricsOptions) {
|
||||
this.#db = options.db;
|
||||
this.#version = options.version;
|
||||
this.#sources = options.sources;
|
||||
this.#clock = options.now ?? (() => new Date());
|
||||
}
|
||||
|
||||
render(): string {
|
||||
const nodes = this.#db
|
||||
.prepare('SELECT id,name,enabled FROM instances ORDER BY name, id')
|
||||
.all() as Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
enabled: number;
|
||||
}>;
|
||||
const ids = new Set(nodes.map((node) => node.id));
|
||||
const lines: string[] = [];
|
||||
|
||||
lines.push(
|
||||
...sample(`${PREFIX}_build_info`, 'Build identity of the control plane.', 'gauge', [
|
||||
`${PREFIX}_build_info{version="${label(this.#version)}"} 1`,
|
||||
]),
|
||||
...sample(`${PREFIX}_scrape_timestamp_seconds`, 'Wall-clock time of this scrape.', 'gauge', [
|
||||
`${PREFIX}_scrape_timestamp_seconds ${Math.floor(this.#clock().getTime() / 1000)}`,
|
||||
]),
|
||||
);
|
||||
|
||||
const info = nodes.map(
|
||||
(node) => `${PREFIX}_node_info{node="${label(node.id)}",name="${label(node.name)}"} 1`,
|
||||
);
|
||||
lines.push(
|
||||
...sample(`${PREFIX}_node_info`, 'Registered nodes and their addresses.', 'gauge', info),
|
||||
...sample(`${PREFIX}_nodes_total`, 'Number of registered nodes.', 'gauge', [
|
||||
`${PREFIX}_nodes_total ${nodes.length}`,
|
||||
]),
|
||||
...sample(
|
||||
`${PREFIX}_node_enabled`,
|
||||
'Whether the node is enabled (1) or paused (0).',
|
||||
'gauge',
|
||||
nodes.map(
|
||||
(node) => `${PREFIX}_node_enabled{node="${label(node.id)}"} ${node.enabled ? 1 : 0}`,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const probes = this.#sources.connections.summarize();
|
||||
lines.push(
|
||||
...sample(
|
||||
`${PREFIX}_node_probe_total`,
|
||||
'Health probes recorded in the retained window, by outcome.',
|
||||
'counter',
|
||||
probes.flatMap((probe) => [
|
||||
`${PREFIX}_node_probe_total{node="${label(probe.instanceId)}",outcome="success"} ${probe.success}`,
|
||||
`${PREFIX}_node_probe_total{node="${label(probe.instanceId)}",outcome="failed"} ${probe.failed}`,
|
||||
]),
|
||||
),
|
||||
...sample(
|
||||
`${PREFIX}_node_availability_ratio`,
|
||||
'Share of successful probes in the retained window.',
|
||||
'gauge',
|
||||
probes.map(
|
||||
(probe) =>
|
||||
`${PREFIX}_node_availability_ratio{node="${label(probe.instanceId)}"} ${(probe.availabilityPercent / 100).toFixed(3)}`,
|
||||
),
|
||||
),
|
||||
...sample(
|
||||
`${PREFIX}_node_probe_duration_ms`,
|
||||
'Average probe latency in the retained window.',
|
||||
'gauge',
|
||||
probes.map(
|
||||
(probe) =>
|
||||
`${PREFIX}_node_probe_duration_ms{node="${label(probe.instanceId)}"} ${number(probe.averageDurationMs)}`,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const identities = this.#sources.identities.list();
|
||||
const pending = identities.filter((identity) => identity.status === 'pending');
|
||||
lines.push(
|
||||
...sample(
|
||||
`${PREFIX}_node_identity_tracked`,
|
||||
'Whether the node has ever reported its hardware identity.',
|
||||
'gauge',
|
||||
identities.map(
|
||||
(identity) => `${PREFIX}_node_identity_tracked{node="${label(identity.instanceId)}"} 1`,
|
||||
),
|
||||
),
|
||||
...sample(
|
||||
`${PREFIX}_node_identity_pending`,
|
||||
'1 while the identity guard holds the node and control actions are refused.',
|
||||
'gauge',
|
||||
identities.map(
|
||||
(identity) =>
|
||||
`${PREFIX}_node_identity_pending{node="${label(identity.instanceId)}"} ${identity.status === 'pending' ? 1 : 0}`,
|
||||
),
|
||||
),
|
||||
...sample(
|
||||
`${PREFIX}_identity_pending_total`,
|
||||
'Nodes waiting for an operator to confirm the device behind them.',
|
||||
'gauge',
|
||||
[`${PREFIX}_identity_pending_total ${pending.length}`],
|
||||
),
|
||||
);
|
||||
|
||||
lines.push(
|
||||
...this.#statusMetric(
|
||||
`${PREFIX}_sms_outbox`,
|
||||
'Queued cross-node SMS sends, by state.',
|
||||
'SELECT status, COUNT(*) AS count FROM sms_outbox GROUP BY status',
|
||||
['queued', 'sending', 'sent', 'failed', 'cancelled'],
|
||||
),
|
||||
...this.#statusMetric(
|
||||
`${PREFIX}_notification_queue`,
|
||||
'Central notification queue, by state.',
|
||||
'SELECT status, COUNT(*) AS count FROM notification_queue GROUP BY status',
|
||||
['pending', 'sending', 'succeeded', 'failed', 'cancelled'],
|
||||
),
|
||||
...this.#statusMetric(
|
||||
`${PREFIX}_jobs`,
|
||||
'Fleet operations, by state.',
|
||||
'SELECT status, COUNT(*) AS count FROM jobs GROUP BY status',
|
||||
[
|
||||
'queued',
|
||||
'running',
|
||||
'cancelling',
|
||||
'succeeded',
|
||||
'partially-succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'unknown-result',
|
||||
],
|
||||
),
|
||||
...this.#statusMetric(
|
||||
`${PREFIX}_scheduled_tasks`,
|
||||
'Automation tasks, by state.',
|
||||
`SELECT CASE WHEN enabled=1 THEN 'armed' ELSE 'paused' END AS status,
|
||||
COUNT(*) AS count FROM scheduled_tasks GROUP BY enabled`,
|
||||
['armed', 'paused'],
|
||||
),
|
||||
);
|
||||
|
||||
const groups = this.#count('SELECT COUNT(*) AS count FROM device_groups');
|
||||
const tags = this.#count('SELECT COUNT(*) AS count FROM tag_registry');
|
||||
const messages = this.#count('SELECT COUNT(*) AS count FROM sms_messages');
|
||||
const probed = new Set(probes.map((probe) => probe.instanceId));
|
||||
const unprobed = [...ids].filter((id) => !probed.has(id)).length;
|
||||
lines.push(
|
||||
...sample(`${PREFIX}_device_groups`, 'Node groups.', 'gauge', [
|
||||
`${PREFIX}_device_groups ${groups}`,
|
||||
]),
|
||||
...sample(`${PREFIX}_device_tags`, 'Labels in the tag registry.', 'gauge', [
|
||||
`${PREFIX}_device_tags ${tags}`,
|
||||
]),
|
||||
...sample(
|
||||
`${PREFIX}_sms_messages`,
|
||||
'SMS rows synchronised into the central message centre.',
|
||||
'gauge',
|
||||
[`${PREFIX}_sms_messages ${messages}`],
|
||||
),
|
||||
...sample(
|
||||
`${PREFIX}_node_unprobed`,
|
||||
'Registered nodes without a health probe in the retained window.',
|
||||
'gauge',
|
||||
[`${PREFIX}_node_unprobed ${unprobed}`],
|
||||
),
|
||||
);
|
||||
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
#count(sql: string): number {
|
||||
const row = this.#db.prepare(sql).get() as { count?: number } | undefined;
|
||||
return number(row?.count ?? 0);
|
||||
}
|
||||
|
||||
/** Every state is emitted, including the empty ones, so a query never sees a series vanish. */
|
||||
#statusMetric(
|
||||
name: string,
|
||||
help: string,
|
||||
sql: string,
|
||||
states: readonly string[],
|
||||
): readonly string[] {
|
||||
const rows = this.#db.prepare(sql).all() as readonly CountRow[];
|
||||
const counts = new Map(rows.map((row) => [String(row.status), number(row.count)]));
|
||||
return sample(
|
||||
name,
|
||||
help,
|
||||
'gauge',
|
||||
states.map((state) => `${name}{status="${label(state)}"} ${counts.get(state) ?? 0}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user