feat(api): complete phase 2.4 control plane
This commit is contained in:
@@ -64,9 +64,12 @@ describe('database migrations', () => {
|
||||
'capabilities',
|
||||
'instance_tags',
|
||||
'instances',
|
||||
'job_attempts',
|
||||
'job_items',
|
||||
'jobs',
|
||||
'operation_preparations',
|
||||
'schema_migrations',
|
||||
'secret_cleanup_tasks',
|
||||
'secret_references',
|
||||
'status_snapshots',
|
||||
]);
|
||||
@@ -91,7 +94,18 @@ describe('database migrations', () => {
|
||||
expect(foreignKeys).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ from: 'job_id', on_delete: 'CASCADE', table: 'jobs' }),
|
||||
expect.objectContaining({ from: 'instance_id', on_delete: 'RESTRICT', table: 'instances' }),
|
||||
]),
|
||||
);
|
||||
expect(foreignKeys).not.toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ from: 'instance_id', table: 'instances' }),
|
||||
]),
|
||||
);
|
||||
const jobColumns = database.pragma('table_info(jobs)') as Array<{ name: string }>;
|
||||
expect(jobColumns).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ name: 'root_job_id' }),
|
||||
expect.objectContaining({ name: 'retry_of_job_id' }),
|
||||
]),
|
||||
);
|
||||
const indexes = database
|
||||
@@ -107,6 +121,103 @@ describe('database migrations', () => {
|
||||
'idx_status_snapshots_instance_observed_at',
|
||||
]),
|
||||
);
|
||||
const now = '2026-07-16T00:00:00.000Z';
|
||||
database
|
||||
.prepare(
|
||||
'INSERT INTO instances (id,name,base_url,auth_mode,enabled,config_revision,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?)',
|
||||
)
|
||||
.run('cleanup-owner', 'Cleanup owner', 'http://10.0.0.9', 'password', 1, 1, now, now);
|
||||
database
|
||||
.prepare(
|
||||
'INSERT INTO secret_references (id,instance_id,purpose,provider,external_reference,created_at,updated_at) VALUES (?,?,?,?,?,?,?)',
|
||||
)
|
||||
.run(
|
||||
'ref-one',
|
||||
'cleanup-owner',
|
||||
'instance-password',
|
||||
'macos-keychain',
|
||||
'opaque-one',
|
||||
now,
|
||||
now,
|
||||
);
|
||||
expect(() =>
|
||||
database
|
||||
.prepare(
|
||||
'INSERT INTO secret_references (id,instance_id,purpose,provider,external_reference,created_at,updated_at) VALUES (?,?,?,?,?,?,?)',
|
||||
)
|
||||
.run(
|
||||
'ref-two',
|
||||
'cleanup-owner',
|
||||
'instance-password',
|
||||
'macos-keychain',
|
||||
'opaque-two',
|
||||
now,
|
||||
now,
|
||||
),
|
||||
).toThrow(/unique/i);
|
||||
database
|
||||
.prepare(
|
||||
'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
|
||||
)
|
||||
.run('opaque-one', 'cleanup-owner', 'instance-password', 'macos-keychain', now, now);
|
||||
database.prepare('DELETE FROM instances WHERE id=?').run('cleanup-owner');
|
||||
expect(database.prepare('SELECT reference FROM secret_cleanup_tasks').all()).toEqual([
|
||||
{ reference: 'opaque-one' },
|
||||
]);
|
||||
database.close();
|
||||
});
|
||||
|
||||
it('upgrades duplicate v1 secret references by retaining one canonical row and queuing the rest', async () => {
|
||||
const directory = await temporaryDirectory();
|
||||
const database = openDatabase(join(directory, 'app.sqlite'));
|
||||
migrateDatabase(database, [MIGRATIONS[0]!]);
|
||||
const older = '2026-07-15T00:00:00.000Z';
|
||||
const newer = '2026-07-16T00:00:00.000Z';
|
||||
database
|
||||
.prepare(
|
||||
'INSERT INTO instances (id,name,base_url,auth_mode,enabled,config_revision,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?)',
|
||||
)
|
||||
.run('owner', 'Owner', 'http://10.0.0.8', 'password', 1, 1, older, newer);
|
||||
const reference = (slot: string) =>
|
||||
`keychain://multi-simadmin/${Buffer.from(JSON.stringify(['owner', 'instance-password', slot]), 'utf8').toString('base64url')}`;
|
||||
const insert = database.prepare(
|
||||
'INSERT INTO secret_references (id,instance_id,purpose,provider,external_reference,created_at,updated_at) VALUES (?,?,?,?,?,?,?)',
|
||||
);
|
||||
insert.run(
|
||||
'old-ref',
|
||||
'owner',
|
||||
'instance-password',
|
||||
'macos-keychain',
|
||||
reference('old-slot'),
|
||||
older,
|
||||
older,
|
||||
);
|
||||
insert.run(
|
||||
'new-ref',
|
||||
'owner',
|
||||
'instance-password',
|
||||
'macos-keychain',
|
||||
reference('new-slot'),
|
||||
newer,
|
||||
newer,
|
||||
);
|
||||
|
||||
migrateDatabase(database);
|
||||
|
||||
expect(database.prepare('SELECT id FROM secret_references').all()).toEqual([{ id: 'new-ref' }]);
|
||||
expect(
|
||||
database
|
||||
.prepare('SELECT reference,instance_id,purpose,provider FROM secret_cleanup_tasks')
|
||||
.all(),
|
||||
).toEqual([
|
||||
{
|
||||
reference: reference('old-slot'),
|
||||
instance_id: 'owner',
|
||||
purpose: 'instance-password',
|
||||
provider: 'macos-keychain',
|
||||
},
|
||||
]);
|
||||
expect(() => migrateDatabase(database)).not.toThrow();
|
||||
database.close();
|
||||
});
|
||||
|
||||
|
||||
@@ -113,6 +113,121 @@ export const MIGRATIONS: readonly Migration[] = [
|
||||
'CREATE INDEX idx_job_items_job_status ON job_items(job_id, status)',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
name: 'secret-cleanup-task-outbox',
|
||||
statements: [
|
||||
`CREATE TABLE secret_cleanup_tasks (
|
||||
reference TEXT PRIMARY KEY,
|
||||
instance_id TEXT NOT NULL,
|
||||
purpose TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL DEFAULT 1 CHECK (generation > 0),
|
||||
queued_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)`,
|
||||
`INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,generation,queued_at,updated_at)
|
||||
SELECT external_reference,instance_id,purpose,provider,1,updated_at,updated_at
|
||||
FROM (
|
||||
SELECT external_reference,instance_id,purpose,provider,updated_at,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY instance_id,purpose
|
||||
ORDER BY updated_at DESC,created_at DESC,id DESC
|
||||
) AS row_number
|
||||
FROM secret_references
|
||||
WHERE instance_id IS NOT NULL
|
||||
) ranked
|
||||
WHERE row_number > 1`,
|
||||
`DELETE FROM secret_references
|
||||
WHERE id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id, ROW_NUMBER() OVER (
|
||||
PARTITION BY instance_id,purpose
|
||||
ORDER BY updated_at DESC,created_at DESC,id DESC
|
||||
) AS row_number
|
||||
FROM secret_references
|
||||
WHERE instance_id IS NOT NULL
|
||||
) ranked
|
||||
WHERE row_number > 1
|
||||
)`,
|
||||
'CREATE UNIQUE INDEX uq_secret_references_instance_purpose ON secret_references(instance_id, purpose) WHERE instance_id IS NOT NULL',
|
||||
'CREATE INDEX idx_secret_cleanup_tasks_queued_at ON secret_cleanup_tasks(queued_at)',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
name: 'current-status-snapshot-lifecycle',
|
||||
statements: [
|
||||
`DELETE FROM status_snapshots
|
||||
WHERE id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id, ROW_NUMBER() OVER (
|
||||
PARTITION BY instance_id, category
|
||||
ORDER BY observed_at DESC, created_at DESC, id DESC
|
||||
) AS row_number
|
||||
FROM status_snapshots
|
||||
) ranked
|
||||
WHERE row_number > 1
|
||||
)`,
|
||||
'CREATE UNIQUE INDEX uq_status_snapshots_instance_category ON status_snapshots(instance_id, category)',
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
name: 'r3-preparations-and-durable-job-lineage',
|
||||
statements: [
|
||||
'ALTER TABLE jobs ADD COLUMN root_job_id TEXT REFERENCES jobs(id) ON DELETE RESTRICT',
|
||||
'ALTER TABLE jobs ADD COLUMN retry_of_job_id TEXT REFERENCES jobs(id) ON DELETE RESTRICT',
|
||||
'UPDATE jobs SET root_job_id=id WHERE root_job_id IS NULL',
|
||||
`CREATE TABLE job_items_v4 (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
|
||||
instance_id TEXT NOT NULL,
|
||||
attempt_number INTEGER NOT NULL DEFAULT 1 CHECK (attempt_number > 0),
|
||||
status TEXT NOT NULL,
|
||||
result_code TEXT,
|
||||
error_json TEXT,
|
||||
source_job_item_id TEXT REFERENCES job_items_v4(id) ON DELETE RESTRICT,
|
||||
created_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
updated_at TEXT NOT NULL,
|
||||
UNIQUE (job_id, instance_id, attempt_number)
|
||||
)`,
|
||||
`INSERT INTO job_items_v4 (id,job_id,instance_id,attempt_number,status,result_code,created_at,started_at,finished_at,updated_at)
|
||||
SELECT id,job_id,instance_id,attempt_number,status,result_code,created_at,started_at,finished_at,updated_at FROM job_items`,
|
||||
'DROP TABLE job_items',
|
||||
'ALTER TABLE job_items_v4 RENAME TO job_items',
|
||||
'CREATE INDEX idx_job_items_job_status ON job_items(job_id, status)',
|
||||
`CREATE TABLE job_attempts (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL CHECK (status IN ('running','succeeded','failed','cancelled','unknown-result')),
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
)`,
|
||||
'CREATE INDEX idx_job_attempts_job_started_at ON job_attempts(job_id, started_at)',
|
||||
`CREATE TABLE operation_preparations (
|
||||
id TEXT PRIMARY KEY,
|
||||
operation_id TEXT NOT NULL CHECK (operation_id = 'deleteInstance'),
|
||||
risk_level TEXT NOT NULL CHECK (risk_level = 'R3'),
|
||||
status TEXT NOT NULL CHECK (status IN ('prepared','consumed','expired','invalidated')),
|
||||
target_instance_id TEXT NOT NULL,
|
||||
target_revision INTEGER NOT NULL CHECK (target_revision > 0),
|
||||
parameter_schema_id TEXT NOT NULL,
|
||||
parameters_digest TEXT NOT NULL,
|
||||
token_digest TEXT NOT NULL CHECK (length(token_digest) = 64),
|
||||
requested_by TEXT NOT NULL,
|
||||
request_id TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
consumed_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
)`,
|
||||
'CREATE INDEX idx_operation_preparations_status_expires_at ON operation_preparations(status, expires_at)',
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const createMigrationsTable = `CREATE TABLE schema_migrations (
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
sqliteTable,
|
||||
text,
|
||||
unique,
|
||||
uniqueIndex,
|
||||
} from 'drizzle-orm/sqlite-core';
|
||||
|
||||
const timestamps = {
|
||||
@@ -88,6 +89,7 @@ export const statusSnapshots = sqliteTable(
|
||||
sql`${table.state} in ('fresh', 'stale', 'expired', 'unknown')`,
|
||||
),
|
||||
index('idx_status_snapshots_instance_observed_at').on(table.instanceId, desc(table.observedAt)),
|
||||
uniqueIndex('uq_status_snapshots_instance_category').on(table.instanceId, table.category),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -98,6 +100,12 @@ export const jobs = sqliteTable(
|
||||
parentJobId: text('parent_job_id').references((): AnySQLiteColumn => jobs.id, {
|
||||
onDelete: 'restrict',
|
||||
}),
|
||||
rootJobId: text('root_job_id').references((): AnySQLiteColumn => jobs.id, {
|
||||
onDelete: 'restrict',
|
||||
}),
|
||||
retryOfJobId: text('retry_of_job_id').references((): AnySQLiteColumn => jobs.id, {
|
||||
onDelete: 'restrict',
|
||||
}),
|
||||
operationId: text('operation_id').notNull(),
|
||||
riskLevel: text('risk_level').notNull(),
|
||||
status: text('status').notNull(),
|
||||
@@ -122,12 +130,14 @@ export const jobItems = sqliteTable(
|
||||
jobId: text('job_id')
|
||||
.notNull()
|
||||
.references(() => jobs.id, { onDelete: 'cascade' }),
|
||||
instanceId: text('instance_id')
|
||||
.notNull()
|
||||
.references(() => instances.id, { onDelete: 'restrict' }),
|
||||
instanceId: text('instance_id').notNull(),
|
||||
attemptNumber: integer('attempt_number').notNull().default(1),
|
||||
status: text('status').notNull(),
|
||||
resultCode: text('result_code'),
|
||||
errorJson: text('error_json'),
|
||||
sourceJobItemId: text('source_job_item_id').references((): AnySQLiteColumn => jobItems.id, {
|
||||
onDelete: 'restrict',
|
||||
}),
|
||||
createdAt: text('created_at').notNull(),
|
||||
startedAt: text('started_at'),
|
||||
finishedAt: text('finished_at'),
|
||||
@@ -144,6 +154,44 @@ export const jobItems = sqliteTable(
|
||||
],
|
||||
);
|
||||
|
||||
export const jobAttempts = sqliteTable(
|
||||
'job_attempts',
|
||||
{
|
||||
id: text('id').primaryKey(),
|
||||
jobId: text('job_id')
|
||||
.notNull()
|
||||
.references(() => jobs.id, { onDelete: 'cascade' }),
|
||||
status: text('status').notNull(),
|
||||
startedAt: text('started_at').notNull(),
|
||||
finishedAt: text('finished_at'),
|
||||
createdAt: text('created_at').notNull(),
|
||||
},
|
||||
(table) => [index('idx_job_attempts_job_started_at').on(table.jobId, table.startedAt)],
|
||||
);
|
||||
|
||||
export const operationPreparations = sqliteTable(
|
||||
'operation_preparations',
|
||||
{
|
||||
id: text('id').primaryKey(),
|
||||
operationId: text('operation_id').notNull(),
|
||||
riskLevel: text('risk_level').notNull(),
|
||||
status: text('status').notNull(),
|
||||
targetInstanceId: text('target_instance_id').notNull(),
|
||||
targetRevision: integer('target_revision').notNull(),
|
||||
parameterSchemaId: text('parameter_schema_id').notNull(),
|
||||
parametersDigest: text('parameters_digest').notNull(),
|
||||
tokenDigest: text('token_digest').notNull(),
|
||||
requestedBy: text('requested_by').notNull(),
|
||||
requestId: text('request_id').notNull(),
|
||||
expiresAt: text('expires_at').notNull(),
|
||||
consumedAt: text('consumed_at'),
|
||||
...timestamps,
|
||||
},
|
||||
(table) => [
|
||||
index('idx_operation_preparations_status_expires_at').on(table.status, table.expiresAt),
|
||||
],
|
||||
);
|
||||
|
||||
export const auditEvents = sqliteTable(
|
||||
'audit_events',
|
||||
{
|
||||
@@ -188,5 +236,22 @@ export const secretReferences = sqliteTable(
|
||||
table.provider,
|
||||
table.externalReference,
|
||||
),
|
||||
uniqueIndex('uq_secret_references_instance_purpose')
|
||||
.on(table.instanceId, table.purpose)
|
||||
.where(sql`${table.instanceId} is not null`),
|
||||
],
|
||||
);
|
||||
|
||||
export const secretCleanupTasks = sqliteTable(
|
||||
'secret_cleanup_tasks',
|
||||
{
|
||||
reference: text('reference').primaryKey(),
|
||||
instanceId: text('instance_id').notNull(),
|
||||
purpose: text('purpose').notNull(),
|
||||
provider: text('provider').notNull(),
|
||||
generation: integer('generation').notNull().default(1),
|
||||
queuedAt: text('queued_at').notNull(),
|
||||
updatedAt: text('updated_at').notNull(),
|
||||
},
|
||||
(table) => [index('idx_secret_cleanup_tasks_queued_at').on(table.queuedAt)],
|
||||
);
|
||||
|
||||
@@ -61,6 +61,11 @@ describe('MacOSKeychainSecretStore', () => {
|
||||
);
|
||||
|
||||
expect(reference).toMatch(/^keychain:\/\/multi-simadmin\//);
|
||||
expect(parseKeychainReference(reference)).toMatchObject({
|
||||
service: 'multi-simadmin',
|
||||
instanceId: 'instance.one',
|
||||
purpose: 'instance-password',
|
||||
});
|
||||
expect(reference).not.toContain(secret);
|
||||
expect(runner.calls).toHaveLength(1);
|
||||
expect(runner.calls[0]).toEqual({
|
||||
|
||||
@@ -96,6 +96,9 @@ export class SpawnCommandRunner implements CommandRunner {
|
||||
export interface ParsedKeychainReference {
|
||||
readonly service: typeof SERVICE;
|
||||
readonly account: string;
|
||||
readonly instanceId: string;
|
||||
readonly purpose: string;
|
||||
readonly slot?: string;
|
||||
}
|
||||
|
||||
function invalid(code: 'INVALID_KEY' | 'INVALID_REFERENCE', message: string): never {
|
||||
@@ -161,7 +164,13 @@ export function parseKeychainReference(reference: string): ParsedKeychainReferen
|
||||
const key = { instanceId, purpose, ...(slot === undefined ? {} : { slot }) } as SecretKey;
|
||||
validateKey(key);
|
||||
if (accountFor(key) !== account) invalid('INVALID_REFERENCE', 'Keychain reference is invalid');
|
||||
return { service: SERVICE, account };
|
||||
return {
|
||||
service: SERVICE,
|
||||
account,
|
||||
instanceId: key.instanceId,
|
||||
purpose: key.purpose,
|
||||
...(key.slot === undefined ? {} : { slot: key.slot }),
|
||||
};
|
||||
}
|
||||
|
||||
export class MacOSKeychainSecretStore implements SecretStore {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { PinnedHttpRequester, type PinnedDispatchOptions } from './pinned-http-requester.js';
|
||||
|
||||
describe('PinnedHttpRequester', () => {
|
||||
it('pins Node lookup to the validated address and sets bounded request options', async () => {
|
||||
let captured: PinnedDispatchOptions | undefined;
|
||||
const requester = new PinnedHttpRequester({
|
||||
dispatch: async (options) => {
|
||||
captured = options;
|
||||
return { status: 200, headers: {}, body: '' };
|
||||
},
|
||||
});
|
||||
await requester.request(
|
||||
{ url: 'http://device.lan:8080/api/health', method: 'GET', headers: {} },
|
||||
[{ address: '192.168.1.20', family: 4 }],
|
||||
);
|
||||
if (!captured) throw new Error('dispatch was not called');
|
||||
const dispatchOptions = captured;
|
||||
expect(dispatchOptions.followRedirects).toBe(false);
|
||||
expect(dispatchOptions.timeoutMs).toBe(5000);
|
||||
expect(dispatchOptions.maxBodyBytes).toBe(65536);
|
||||
await expect(
|
||||
new Promise((resolve, reject) =>
|
||||
dispatchOptions.lookup('device.lan', { all: false }, (error, address, family) => {
|
||||
if (error) reject(error);
|
||||
else if (Array.isArray(address))
|
||||
reject(new Error('unexpected all-address lookup result'));
|
||||
else resolve({ a: address, f: family });
|
||||
}),
|
||||
),
|
||||
).resolves.toEqual({ a: '192.168.1.20', f: 4 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import * as http from 'node:http';
|
||||
import * as https from 'node:https';
|
||||
import type { LookupFunction } from 'node:net';
|
||||
import type {
|
||||
PinnedRequest,
|
||||
ResolvedAddress,
|
||||
TransportResponse,
|
||||
} from './safe-instance-transport.js';
|
||||
import { asUpstreamError, UpstreamError } from './upstream-error.js';
|
||||
|
||||
export interface PinnedDispatchOptions {
|
||||
readonly url: URL;
|
||||
readonly method: 'GET' | 'POST';
|
||||
readonly headers: Readonly<Record<string, string>>;
|
||||
readonly body?: string;
|
||||
readonly lookup: LookupFunction;
|
||||
readonly followRedirects: false;
|
||||
readonly timeoutMs: number;
|
||||
readonly maxBodyBytes: number;
|
||||
}
|
||||
export interface PinnedHttpRequesterOptions {
|
||||
readonly dispatch?: (options: PinnedDispatchOptions) => Promise<TransportResponse>;
|
||||
readonly timeoutMs?: number;
|
||||
readonly maxBodyBytes?: number;
|
||||
}
|
||||
const DEFAULT_TIMEOUT_MS = 5_000;
|
||||
const DEFAULT_MAX_BODY_BYTES = 65_536;
|
||||
const lookupFor = (addresses: readonly ResolvedAddress[]): LookupFunction => {
|
||||
const first = addresses[0];
|
||||
if (!first) throw new Error('No validated address');
|
||||
return (_hostname, _options, callback) => callback(null, first.address, first.family);
|
||||
};
|
||||
export class PinnedHttpRequester {
|
||||
private readonly dispatch: (options: PinnedDispatchOptions) => Promise<TransportResponse>;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly maxBodyBytes: number;
|
||||
constructor(options: PinnedHttpRequesterOptions = {}) {
|
||||
this.dispatch = options.dispatch ?? dispatchNative;
|
||||
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
this.maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES;
|
||||
}
|
||||
async request(
|
||||
request: PinnedRequest,
|
||||
addresses: readonly ResolvedAddress[],
|
||||
): Promise<TransportResponse> {
|
||||
try {
|
||||
return await this.dispatch({
|
||||
url: new URL(request.url),
|
||||
method: request.method,
|
||||
headers: request.headers,
|
||||
...(request.body === undefined ? {} : { body: request.body }),
|
||||
lookup: lookupFor(addresses),
|
||||
followRedirects: false,
|
||||
timeoutMs: this.timeoutMs,
|
||||
maxBodyBytes: this.maxBodyBytes,
|
||||
});
|
||||
} catch (error) {
|
||||
throw asUpstreamError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
const dispatchNative = (options: PinnedDispatchOptions): Promise<TransportResponse> =>
|
||||
new Promise((resolve, reject) => {
|
||||
const client = options.url.protocol === 'https:' ? https : http;
|
||||
const request = client.request({
|
||||
protocol: options.url.protocol,
|
||||
hostname: options.url.hostname,
|
||||
port: options.url.port || undefined,
|
||||
path: `${options.url.pathname}${options.url.search}`,
|
||||
method: options.method,
|
||||
headers: options.headers,
|
||||
lookup: options.lookup,
|
||||
agent: false,
|
||||
timeout: options.timeoutMs,
|
||||
maxHeaderSize: 16_384,
|
||||
...(options.url.protocol === 'https:' ? { servername: options.url.hostname } : {}),
|
||||
});
|
||||
let settled = false;
|
||||
const fail = (error: Error): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
request.destroy(error);
|
||||
reject(error);
|
||||
};
|
||||
request.once('timeout', () => fail(new UpstreamError('UPSTREAM_TIMEOUT')));
|
||||
request.once('error', fail);
|
||||
request.once('response', (response) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let bytes = 0;
|
||||
response.on('data', (chunk: Buffer) => {
|
||||
bytes += chunk.length;
|
||||
if (bytes > options.maxBodyBytes) fail(new UpstreamError('UPSTREAM_RESPONSE_TOO_LARGE'));
|
||||
else chunks.push(chunk);
|
||||
});
|
||||
response.once('error', fail);
|
||||
response.once('end', () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
const headers: Record<string, string | undefined> = {};
|
||||
for (const [key, value] of Object.entries(response.headers))
|
||||
headers[key] = Array.isArray(value) ? value.join(', ') : value;
|
||||
resolve({
|
||||
status: response.statusCode ?? 0,
|
||||
headers,
|
||||
body: Buffer.concat(chunks).toString('utf8'),
|
||||
});
|
||||
});
|
||||
});
|
||||
request.end(options.body);
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createProductionUpstream } from './production-upstream.js';
|
||||
|
||||
describe('createProductionUpstream', () => {
|
||||
it('passes every resolved address from its resolver to the pinned requester', async () => {
|
||||
let received: unknown;
|
||||
const upstream = createProductionUpstream({
|
||||
resolve: async () => [{ address: '192.168.1.20', family: 4 }],
|
||||
request: async (request, addresses) => {
|
||||
received = { request, addresses };
|
||||
return { status: 200, headers: {}, body: '' };
|
||||
},
|
||||
});
|
||||
await upstream.get('http://router.lan:8080/api/health');
|
||||
expect(received).toMatchObject({
|
||||
request: { method: 'GET', url: 'http://router.lan:8080/api/health' },
|
||||
addresses: [{ address: '192.168.1.20', family: 4 }],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { lookup as nodeLookup } from 'node:dns/promises';
|
||||
import {
|
||||
createSafeControlPlaneUpstream,
|
||||
type SafeControlPlaneUpstream,
|
||||
} from './safe-control-plane-upstream.js';
|
||||
import { PinnedHttpRequester } from './pinned-http-requester.js';
|
||||
import {
|
||||
SafeInstanceTransport,
|
||||
type PinnedRequest,
|
||||
type ResolvedAddress,
|
||||
type TransportResponse,
|
||||
} from './safe-instance-transport.js';
|
||||
|
||||
export interface ProductionUpstreamOptions {
|
||||
readonly resolve?: (hostname: string) => Promise<readonly ResolvedAddress[]>;
|
||||
readonly request?: (
|
||||
request: PinnedRequest,
|
||||
addresses: readonly ResolvedAddress[],
|
||||
) => Promise<TransportResponse>;
|
||||
}
|
||||
export function createProductionUpstream(
|
||||
options: ProductionUpstreamOptions = {},
|
||||
): SafeControlPlaneUpstream {
|
||||
const requester = options.request ? { request: options.request } : new PinnedHttpRequester();
|
||||
const transport = new SafeInstanceTransport({
|
||||
resolve:
|
||||
options.resolve ??
|
||||
((hostname) =>
|
||||
nodeLookup(hostname, { all: true, verbatim: true }) as Promise<ResolvedAddress[]>),
|
||||
request: (request, addresses) => requester.request(request, addresses),
|
||||
});
|
||||
return createSafeControlPlaneUpstream(transport);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createSafeControlPlaneUpstream } from './safe-control-plane-upstream.js';
|
||||
|
||||
describe('createSafeControlPlaneUpstream', () => {
|
||||
it('routes both health GET and login POST through the same safe transport', async () => {
|
||||
const calls: unknown[] = [];
|
||||
const upstream = createSafeControlPlaneUpstream({
|
||||
get: async (url) => {
|
||||
calls.push({ method: 'GET', url });
|
||||
return { status: 401, headers: {}, body: '' };
|
||||
},
|
||||
post: async (url, headers, body) => {
|
||||
calls.push({ method: 'POST', url, headers, body });
|
||||
return { status: 200, headers: {}, body: '' };
|
||||
},
|
||||
});
|
||||
await upstream.get('http://192.168.1.20/api/health');
|
||||
await upstream.request({
|
||||
url: 'https://192.168.1.20/api/auth/login',
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
secret: '[REDACTED]',
|
||||
body: '[REDACTED]',
|
||||
});
|
||||
expect(calls).toEqual([
|
||||
{ method: 'GET', url: 'http://192.168.1.20/api/health' },
|
||||
{
|
||||
method: 'POST',
|
||||
url: 'https://192.168.1.20/api/auth/login',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: '{"password":"[REDACTED]"}',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { ConnectionTransport } from '../../application/connections/connection-probe.js';
|
||||
import type { UpstreamSessionClientOptions } from '../../application/connections/upstream-session-client.js';
|
||||
import { SafeUpstreamGateway, type SafeUpstreamTransport } from './safe-upstream-gateway.js';
|
||||
|
||||
export interface SafeControlPlaneUpstream extends ConnectionTransport {
|
||||
request: UpstreamSessionClientOptions['request'];
|
||||
}
|
||||
export function createSafeControlPlaneUpstream(
|
||||
transport: SafeUpstreamTransport,
|
||||
): SafeControlPlaneUpstream {
|
||||
const gateway = new SafeUpstreamGateway({ transport });
|
||||
return {
|
||||
get: (url) => transport.get(url),
|
||||
request: (request) => gateway.request(request),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { SafeInstanceTransport, TransportError } from './safe-instance-transport.js';
|
||||
|
||||
const allowed = new SafeInstanceTransport({
|
||||
resolve: async () => [{ address: '192.168.1.20', family: 4 }],
|
||||
request: async (request) => ({
|
||||
status: 200,
|
||||
headers: {},
|
||||
body: `ok:${request.url}`,
|
||||
}),
|
||||
});
|
||||
|
||||
describe('SafeInstanceTransport', () => {
|
||||
it('resolves a hostname immediately before requesting its health endpoint', async () => {
|
||||
await expect(allowed.get('http://router.lan:3000/api/health')).resolves.toMatchObject({
|
||||
status: 200,
|
||||
body: 'ok:http://router.lan:3000/api/health',
|
||||
});
|
||||
});
|
||||
|
||||
it('supports HTTPS origins using the same validated address pinning path', async () => {
|
||||
await expect(allowed.get('https://router.lan:3443/api/health')).resolves.toMatchObject({
|
||||
status: 200,
|
||||
body: 'ok:https://router.lan:3443/api/health',
|
||||
});
|
||||
});
|
||||
|
||||
it('pins IPv6 ULA literals without attempting DNS resolution', async () => {
|
||||
let resolved = false;
|
||||
let addresses: readonly { address: string; family: 4 | 6 }[] | undefined;
|
||||
const transport = new SafeInstanceTransport({
|
||||
resolve: async () => {
|
||||
resolved = true;
|
||||
return [];
|
||||
},
|
||||
request: async (_request, received) => {
|
||||
addresses = received;
|
||||
return { status: 200, headers: {}, body: '' };
|
||||
},
|
||||
});
|
||||
await transport.get('http://[fd00::1]:8080/api/health');
|
||||
expect(resolved).toBe(false);
|
||||
expect(addresses).toEqual([{ address: 'fd00::1', family: 6 }]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: 'loopback', address: '127.0.0.1' },
|
||||
{ name: 'link-local', address: '169.254.169.254' },
|
||||
{ name: 'unspecified', address: '0.0.0.0' },
|
||||
{ name: 'multicast', address: '224.0.0.1' },
|
||||
{ name: 'public', address: '8.8.8.8' },
|
||||
])('rejects a DNS $name result before network I/O', async ({ address }) => {
|
||||
let requested = false;
|
||||
const transport = new SafeInstanceTransport({
|
||||
resolve: async () => [{ address, family: 4 }],
|
||||
request: async () => {
|
||||
requested = true;
|
||||
return { status: 200, headers: {}, body: '' };
|
||||
},
|
||||
});
|
||||
await expect(transport.get('http://device.lan/api/health')).rejects.toMatchObject({
|
||||
code: 'UNSAFE_RESOLUTION',
|
||||
} satisfies Partial<TransportError>);
|
||||
expect(requested).toBe(false);
|
||||
});
|
||||
|
||||
it('passes only the validated resolution set to POST requesters for pinning', async () => {
|
||||
let received:
|
||||
| { url: string; method: string; headers: Readonly<Record<string, string>>; body?: string }
|
||||
| undefined;
|
||||
let addresses: readonly { address: string; family: 4 | 6 }[] | undefined;
|
||||
const transport = new SafeInstanceTransport({
|
||||
resolve: async () => [{ address: '192.168.1.20', family: 4 }],
|
||||
request: async (request, resolved) => {
|
||||
received = request;
|
||||
addresses = resolved;
|
||||
return { status: 200, headers: {}, body: '' };
|
||||
},
|
||||
});
|
||||
await transport.post(
|
||||
'http://device.lan/api/auth/login',
|
||||
{ 'content-type': 'application/json' },
|
||||
'{"password":"[REDACTED]"}',
|
||||
);
|
||||
expect(received).toEqual({
|
||||
url: 'http://device.lan/api/auth/login',
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: '{"password":"[REDACTED]"}',
|
||||
});
|
||||
expect(addresses).toEqual([{ address: '192.168.1.20', family: 4 }]);
|
||||
});
|
||||
|
||||
it('passes only the validated resolution set to the requester for pinning', async () => {
|
||||
let received: readonly { address: string; family: 4 | 6 }[] | undefined;
|
||||
const transport = new SafeInstanceTransport({
|
||||
resolve: async () => [{ address: '192.168.1.20', family: 4 }],
|
||||
request: async (request, addresses) => {
|
||||
expect(request.method).toBe('GET');
|
||||
received = addresses;
|
||||
return { status: 200, headers: {}, body: '' };
|
||||
},
|
||||
});
|
||||
await transport.get('http://device.lan/api/health');
|
||||
expect(received).toEqual([{ address: '192.168.1.20', family: 4 }]);
|
||||
});
|
||||
|
||||
it('rejects mixed safe and unsafe DNS answers to prevent rebinding races', async () => {
|
||||
const transport = new SafeInstanceTransport({
|
||||
resolve: async () => [
|
||||
{ address: '192.168.1.20', family: 4 },
|
||||
{ address: '127.0.0.1', family: 4 },
|
||||
],
|
||||
request: async () => ({ status: 200, headers: {}, body: '' }),
|
||||
});
|
||||
await expect(transport.get('http://device.lan/api/health')).rejects.toMatchObject({
|
||||
code: 'UNSAFE_RESOLUTION',
|
||||
} satisfies Partial<TransportError>);
|
||||
});
|
||||
|
||||
it('does not follow redirects across trust boundaries', async () => {
|
||||
let calls = 0;
|
||||
const transport = new SafeInstanceTransport({
|
||||
resolve: async () => [{ address: '192.168.1.20', family: 4 }],
|
||||
request: async () => {
|
||||
calls += 1;
|
||||
return { status: 302, headers: { location: 'http://127.0.0.1/private' }, body: '' };
|
||||
},
|
||||
});
|
||||
await expect(transport.get('http://device.lan/api/health')).rejects.toMatchObject({
|
||||
code: 'REDIRECT_REJECTED',
|
||||
} satisfies Partial<TransportError>);
|
||||
expect(calls).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { isIP } from 'node:net';
|
||||
import { asUpstreamError, UpstreamError } from './upstream-error.js';
|
||||
|
||||
export interface ResolvedAddress {
|
||||
readonly address: string;
|
||||
readonly family: 4 | 6;
|
||||
}
|
||||
export interface TransportResponse {
|
||||
readonly status: number;
|
||||
readonly headers: Readonly<Record<string, string | undefined>>;
|
||||
readonly body: string;
|
||||
}
|
||||
export interface PinnedRequest {
|
||||
readonly url: string;
|
||||
readonly method: 'GET' | 'POST';
|
||||
readonly headers: Readonly<Record<string, string>>;
|
||||
readonly body?: string;
|
||||
}
|
||||
export interface SafeTransportOptions {
|
||||
readonly resolve: (hostname: string) => Promise<readonly ResolvedAddress[]>;
|
||||
readonly request: (
|
||||
request: PinnedRequest,
|
||||
addresses: readonly ResolvedAddress[],
|
||||
) => Promise<TransportResponse>;
|
||||
}
|
||||
export class TransportError extends UpstreamError {
|
||||
constructor(override readonly code: 'UNSAFE_ORIGIN' | 'UNSAFE_RESOLUTION' | 'REDIRECT_REJECTED') {
|
||||
super(code);
|
||||
this.name = 'TransportError';
|
||||
}
|
||||
}
|
||||
const isPrivateV4 = (address: string): boolean => {
|
||||
const parts = address.split('.').map(Number);
|
||||
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
|
||||
return false;
|
||||
const [a, b, c] = parts;
|
||||
if (a === undefined || b === undefined || c === undefined) return false;
|
||||
return (
|
||||
a === 10 ||
|
||||
(a === 172 && b >= 16 && b <= 31) ||
|
||||
(a === 192 && b === 168) ||
|
||||
(a === 100 && b >= 64 && b <= 127) ||
|
||||
(a === 192 && b === 0 && c === 0)
|
||||
);
|
||||
};
|
||||
const isAllowedAddress = ({ address, family }: ResolvedAddress): boolean => {
|
||||
if (family === 4) return isPrivateV4(address);
|
||||
const normalized = address.toLowerCase();
|
||||
return normalized.startsWith('fc') || normalized.startsWith('fd');
|
||||
};
|
||||
const origin = (raw: string): URL => {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(raw);
|
||||
} catch {
|
||||
throw new TransportError('UNSAFE_ORIGIN');
|
||||
}
|
||||
if (
|
||||
(parsed.protocol !== 'http:' && parsed.protocol !== 'https:') ||
|
||||
parsed.username ||
|
||||
parsed.password ||
|
||||
parsed.search ||
|
||||
parsed.hash
|
||||
)
|
||||
throw new TransportError('UNSAFE_ORIGIN');
|
||||
if (!parsed.pathname.startsWith('/')) throw new TransportError('UNSAFE_ORIGIN');
|
||||
return parsed;
|
||||
};
|
||||
export class SafeInstanceTransport {
|
||||
constructor(private readonly options: SafeTransportOptions) {}
|
||||
async get(raw: string): Promise<TransportResponse> {
|
||||
return this.send({ url: raw, method: 'GET', headers: {} });
|
||||
}
|
||||
async post(
|
||||
raw: string,
|
||||
headers: Readonly<Record<string, string>>,
|
||||
body: string,
|
||||
): Promise<TransportResponse> {
|
||||
return this.send({ url: raw, method: 'POST', headers, body });
|
||||
}
|
||||
private async send(request: PinnedRequest): Promise<TransportResponse> {
|
||||
const parsed = origin(request.url);
|
||||
const dialHost = parsed.hostname.replace(/^\[|\]$/g, '');
|
||||
const literalFamily = isIP(dialHost);
|
||||
let addresses: readonly ResolvedAddress[];
|
||||
try {
|
||||
addresses = literalFamily
|
||||
? [{ address: dialHost, family: literalFamily as 4 | 6 }]
|
||||
: await this.options.resolve(dialHost);
|
||||
} catch (error) {
|
||||
throw asUpstreamError(error);
|
||||
}
|
||||
if (addresses.length === 0 || addresses.some((entry) => !isAllowedAddress(entry)))
|
||||
throw new TransportError('UNSAFE_RESOLUTION');
|
||||
let response: TransportResponse;
|
||||
try {
|
||||
response = await this.options.request({ ...request, url: parsed.toString() }, addresses);
|
||||
} catch (error) {
|
||||
throw asUpstreamError(error);
|
||||
}
|
||||
if (response.status >= 300 && response.status < 400)
|
||||
throw new TransportError('REDIRECT_REJECTED');
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { SafeUpstreamGateway } from './safe-upstream-gateway.js';
|
||||
|
||||
describe('SafeUpstreamGateway', () => {
|
||||
it('sends login password only as JSON through the pinned POST transport', async () => {
|
||||
const calls: unknown[] = [];
|
||||
const gateway = new SafeUpstreamGateway({
|
||||
transport: {
|
||||
get: async () => ({ status: 200, headers: {}, body: '' }),
|
||||
post: async (url, headers, body) => {
|
||||
calls.push({ url, headers, body });
|
||||
return { status: 200, headers: { 'set-cookie': 'simadmin_session=opaque' }, body: '' };
|
||||
},
|
||||
},
|
||||
});
|
||||
await gateway.request({
|
||||
url: 'https://192.168.1.20:8080/api/auth/login',
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
secret: '[REDACTED]',
|
||||
body: '[REDACTED]',
|
||||
});
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
url: 'https://192.168.1.20:8080/api/auth/login',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: '{"password":"[REDACTED]"}',
|
||||
},
|
||||
]);
|
||||
});
|
||||
it('rejects HTTP login and logout so credentials and cookies are never sent in cleartext', async () => {
|
||||
const gateway = new SafeUpstreamGateway({
|
||||
transport: {
|
||||
get: async () => ({ status: 200, headers: {}, body: '' }),
|
||||
post: async () => ({ status: 200, headers: {}, body: '' }),
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
gateway.request({
|
||||
url: 'http://192.168.1.20:8080/api/auth/login',
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
secret: '[REDACTED]',
|
||||
body: '[REDACTED]',
|
||||
}),
|
||||
).rejects.toThrow('UPSTREAM_INSECURE_AUTH');
|
||||
await expect(
|
||||
gateway.request({
|
||||
url: 'http://192.168.1.20:8080/api/auth/logout',
|
||||
method: 'POST',
|
||||
headers: { cookie: 'simadmin_session=opaque' },
|
||||
}),
|
||||
).rejects.toThrow('UPSTREAM_INSECURE_AUTH');
|
||||
});
|
||||
|
||||
it('does not allow a supplied redacted body marker to become a network request body', async () => {
|
||||
const gateway = new SafeUpstreamGateway({
|
||||
transport: {
|
||||
get: async () => ({ status: 200, headers: {}, body: '' }),
|
||||
post: async () => ({ status: 200, headers: {}, body: '' }),
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
gateway.request({
|
||||
url: 'https://192.168.1.20:8080/api/auth/login',
|
||||
method: 'POST',
|
||||
headers: {},
|
||||
body: '[REDACTED]',
|
||||
}),
|
||||
).rejects.toThrow('UPSTREAM_REQUEST_INVALID');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import type {
|
||||
UpstreamRequest,
|
||||
UpstreamResponse,
|
||||
} from '../../application/connections/upstream-session-client.js';
|
||||
import type { TransportResponse } from './safe-instance-transport.js';
|
||||
import { UpstreamError } from './upstream-error.js';
|
||||
|
||||
export interface SafeUpstreamTransport {
|
||||
get(url: string): Promise<TransportResponse>;
|
||||
post(
|
||||
url: string,
|
||||
headers: Readonly<Record<string, string>>,
|
||||
body: string,
|
||||
): Promise<TransportResponse>;
|
||||
}
|
||||
export class SafeUpstreamGateway {
|
||||
constructor(private readonly options: { readonly transport: SafeUpstreamTransport }) {}
|
||||
async request(request: UpstreamRequest): Promise<UpstreamResponse> {
|
||||
const url = new URL(request.url);
|
||||
if (url.protocol !== 'https:') throw new UpstreamError('UPSTREAM_INSECURE_AUTH');
|
||||
if (request.method !== 'POST') throw new UpstreamError('UPSTREAM_REQUEST_INVALID');
|
||||
if (request.url.endsWith('/api/auth/login')) {
|
||||
if (typeof request.secret !== 'string' || request.body !== '[REDACTED]')
|
||||
throw new UpstreamError('UPSTREAM_REQUEST_INVALID');
|
||||
return this.options.transport.post(
|
||||
request.url,
|
||||
request.headers,
|
||||
JSON.stringify({ password: request.secret }),
|
||||
);
|
||||
}
|
||||
if (request.url.endsWith('/api/auth/logout')) {
|
||||
if (request.secret !== undefined || request.body !== undefined)
|
||||
throw new UpstreamError('UPSTREAM_REQUEST_INVALID');
|
||||
return this.options.transport.post(request.url, request.headers, '');
|
||||
}
|
||||
throw new UpstreamError('UPSTREAM_REQUEST_INVALID');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
export type UpstreamErrorCode =
|
||||
| 'UPSTREAM_UNAVAILABLE'
|
||||
| 'UPSTREAM_TIMEOUT'
|
||||
| 'UPSTREAM_RESPONSE_TOO_LARGE'
|
||||
| 'UPSTREAM_INSECURE_AUTH'
|
||||
| 'UPSTREAM_REQUEST_INVALID'
|
||||
| 'UNSAFE_ORIGIN'
|
||||
| 'UNSAFE_RESOLUTION'
|
||||
| 'REDIRECT_REJECTED';
|
||||
|
||||
export class UpstreamError extends Error {
|
||||
constructor(
|
||||
readonly code: UpstreamErrorCode,
|
||||
options?: { readonly cause?: unknown },
|
||||
) {
|
||||
super(code, options);
|
||||
this.name = 'UpstreamError';
|
||||
}
|
||||
}
|
||||
|
||||
export const asUpstreamError = (error: unknown): UpstreamError =>
|
||||
error instanceof UpstreamError
|
||||
? error
|
||||
: new UpstreamError('UPSTREAM_UNAVAILABLE', { cause: error });
|
||||
Reference in New Issue
Block a user