feat(api): complete phase 2.4 control plane

This commit is contained in:
chick
2026-07-17 00:37:11 +08:00
parent c2702b5fc6
commit 7b91dbbad1
40 changed files with 4526 additions and 74 deletions
@@ -0,0 +1,545 @@
import Database from 'better-sqlite3';
import { afterEach, describe, expect, it } from 'vitest';
import { migrateDatabase } from '../../infrastructure/database/migrations.js';
import type { SecretStore } from '../../infrastructure/secrets/secret-store.js';
import type { Instance } from '@multi-simadmin/contracts';
import { InstanceService, InstanceServiceError } from './instance-service.js';
class MemorySecrets implements SecretStore {
readonly values = new Map<string, string>();
readonly deletes: string[] = [];
readonly sets: Array<{
key: { instanceId: string; purpose: string; slot?: string };
value: string;
}> = [];
onDelete?: (reference: string) => void;
overrideReference?: string;
failSet = false;
failDelete = false;
async set(key: { instanceId: string; purpose: string; slot?: string }, value: string) {
if (this.failSet) throw new Error(`keychain leaked ${value}`);
const account = Buffer.from(
JSON.stringify([key.instanceId, key.purpose, key.slot]),
'utf8',
).toString('base64url');
const ref = this.overrideReference ?? `keychain://multi-simadmin/${account}`;
this.sets.push({ key, value });
this.values.set(ref, value);
return ref;
}
async get(reference: string) {
return this.values.get(reference);
}
async delete(reference: string) {
this.deletes.push(reference);
this.onDelete?.(reference);
if (this.failDelete) throw new Error(`cannot delete ${this.values.get(reference)}`);
return this.values.delete(reference);
}
}
const dbs: Database.Database[] = [];
afterEach(() => {
for (const db of dbs.splice(0)) db.close();
});
function fixture(ids = ['id-1', 'slot-1', 'ref-1', 'id-2', 'slot-2', 'ref-2']) {
const db = new Database(':memory:');
db.pragma('foreign_keys=ON');
migrateDatabase(db);
dbs.push(db);
const store = new MemorySecrets();
let index = 0;
const service = new InstanceService({
db,
store,
idFactory: () => ids[index++]!,
now: () => new Date('2026-07-16T12:00:00.000Z'),
});
return { db, store, service };
}
const basic = { name: ' Alpha ', origin: 'http://192.168.1.10:8080/', tags: [' z ', 'a', 'a'] };
const code = async (promise: Promise<unknown>) => {
try {
await promise;
} catch (e) {
return (e as InstanceServiceError).code;
}
throw new Error('expected rejection');
};
describe('InstanceService', () => {
it('starts RED until the repository module exists', () =>
expect(InstanceService).toBeTypeOf('function'));
it('creates and gets normalized instances without reading or persisting a secret', async () => {
const { db, store, service } = fixture();
const created = await service.create(basic);
expect(created).toEqual({
id: 'id-1',
name: 'Alpha',
origin: 'http://192.168.1.10:8080',
tags: ['a', 'z'],
revision: 1,
capabilityStatus: 'unknown',
freshness: 'unknown',
credentialConfigured: false,
});
expect(await service.get('id-1')).toEqual(created);
expect(store.sets).toHaveLength(0);
expect(
JSON.stringify(
db.prepare('select * from instances join instance_tags on id=instance_id').all(),
),
).not.toContain('password');
});
it('stores a password under a unique slot and SQLite contains only its opaque reference', async () => {
const { db, store, service } = fixture();
const secret = 'super-secret-2.4A';
const result = await service.create({
...basic,
password: { action: 'set', password: secret },
});
expect(result.credentialConfigured).toBe(true);
expect(store.sets[0]).toMatchObject({
key: { instanceId: 'id-1', purpose: 'instance-password', slot: 'slot-1' },
value: secret,
});
const allText = JSON.stringify(
db
.prepare('select id,instance_id,purpose,provider,external_reference from secret_references')
.all(),
);
expect(allText).toContain('macos-keychain');
expect(allText).not.toContain(secret);
expect(JSON.stringify(result)).not.toContain(secret);
expect(store.get).toBeTypeOf('function');
});
it('rejects a secret-store reference not bound to the requested instance and slot', async () => {
const { db, store, service } = fixture();
store.overrideReference = `keychain://multi-simadmin/${Buffer.from(
JSON.stringify(['other-instance', 'instance-password', 'other-slot']),
'utf8',
).toString('base64url')}`;
expect(
await code(service.create({ ...basic, password: { action: 'set', password: 'secret' } })),
).toBe('SECRET_STORE_FAILED');
expect(db.prepare('SELECT * FROM secret_references').all()).toEqual([]);
expect(store.deletes).toEqual([store.overrideReference]);
expect(store.values.size).toBe(0);
});
it('fails closed when an invalid returned secret reference cannot be removed', async () => {
const { db, store, service } = fixture();
store.overrideReference = `keychain://multi-simadmin/${Buffer.from(
JSON.stringify(['other-instance', 'instance-password', 'other-slot']),
'utf8',
).toString('base64url')}`;
store.failDelete = true;
expect(
await code(service.create({ ...basic, password: { action: 'set', password: 'secret' } })),
).toBe('COMPENSATION_PERSISTENCE_FAILED');
expect(db.prepare('SELECT * FROM secret_references').all()).toEqual([]);
expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
});
it('rejects duplicate normalized origins and duplicate generated ids with stable errors', async () => {
const x = fixture(['same', 'same', 'same']);
await x.service.create(basic);
expect(await code(x.service.create({ ...basic, name: 'B' }))).toBe('DUPLICATE_ORIGIN');
expect(await code(x.service.create({ ...basic, origin: 'http://10.1.2.3' }))).toBe(
'DUPLICATE_ID',
);
});
it('filters, sorts with id tie-breaks, and paginates aggregate output', async () => {
const { db, service } = fixture(['b', 'a']);
await service.create({ name: 'Same', origin: 'http://10.0.0.2', tags: ['blue'] });
await service.create({
name: 'Same',
origin: 'http://10.0.0.3',
tags: ['red'],
password: { action: 'preserve' },
});
db.prepare('insert into capabilities values (?,?,?,?,?,?,?,?)').run(
'b',
'x',
'supported',
null,
null,
'2026-07-16T11:00:00Z',
'2026-07-16T11:00:00Z',
'2026-07-16T11:00:00Z',
);
db.prepare('insert into capabilities values (?,?,?,?,?,?,?,?)').run(
'b',
'y',
'degraded',
null,
null,
'2026-07-16T11:00:00Z',
'2026-07-16T11:00:00Z',
'2026-07-16T11:00:00Z',
);
db.prepare('insert into status_snapshots values (?,?,?,?,?,?,?,?)').run(
's',
'b',
'health',
'fresh',
'{}',
'2026-07-16T11:00:00Z',
'2026-07-16T13:00:00Z',
'2026-07-16T11:00:00Z',
);
expect(
(
await service.list({
search: 'b',
tag: 'blue',
capabilityStatus: 'degraded',
freshness: 'fresh',
})
).items.map((x: Instance) => x.id),
).toEqual(['b']);
expect((await service.list({ sort: 'name', page: 1, pageSize: 1 })).items[0]?.id).toBe('a');
expect((await service.list({ sort: 'name', page: 2, pageSize: 1 })).page).toEqual({
page: 2,
pageSize: 1,
total: 2,
});
});
it('updates with revision CAS, normalized fields, and one revision increment', async () => {
const { service } = fixture();
await service.create(basic);
expect(await code(service.update('id-1', 9, { name: 'No' }))).toBe('REVISION_CONFLICT');
const result = await service.update('id-1', 1, { name: ' Beta ', tags: ['x'] });
expect(result).toMatchObject({
name: 'Beta',
tags: ['x'],
revision: 2,
credentialConfigured: false,
});
});
it('preserves, replaces and clears credentials without exposing them', async () => {
const { db, store, service } = fixture();
await service.create({ ...basic, password: { action: 'set', password: 'first-secret' } });
await service.update('id-1', 1, { password: { action: 'preserve' } });
expect(store.sets).toHaveLength(1);
const replaced = await service.update('id-1', 2, {
password: { action: 'set', password: 'second-secret' },
});
expect(replaced.credentialConfigured).toBe(true);
expect(store.deletes).toEqual([
expect.stringMatching(/^keychain:\/\/multi-simadmin\/[A-Za-z0-9_-]+$/),
]);
const cleared = await service.update('id-1', 3, { password: { action: 'clear' } });
expect(cleared.credentialConfigured).toBe(false);
expect(db.prepare('select * from secret_references').all()).toHaveLength(0);
});
it('compensates a newly stored secret when the DB transaction fails', async () => {
const { db, store, service } = fixture();
await service.create(basic);
const occupiedReference = `keychain://multi-simadmin/${Buffer.from(
JSON.stringify(['id-1', 'instance-password', 'slot-1']),
'utf8',
).toString('base64url')}`;
db.prepare('insert into secret_references values (?,?,?,?,?,?,?)').run(
'occupied',
null,
'other',
'macos-keychain',
occupiedReference,
'now',
'now',
);
const error = await code(
service.update('id-1', 1, { password: { action: 'set', password: 'super-secret' } }),
);
expect(error).toBe('DATABASE_FAILED');
expect(store.values.size).toBe(0);
});
it('surfaces an explicit fail-closed error when compensation cleanup cannot be persisted', async () => {
const { db, store, service } = fixture();
await service.create(basic);
const occupiedReference = `keychain://multi-simadmin/${Buffer.from(
JSON.stringify(['id-1', 'instance-password', 'slot-1']),
'utf8',
).toString('base64url')}`;
db.prepare('insert into secret_references values (?,?,?,?,?,?,?)').run(
'occupied',
null,
'other',
'macos-keychain',
occupiedReference,
'now',
'now',
);
db.exec(
`CREATE TRIGGER fail_cleanup_outbox BEFORE INSERT ON secret_cleanup_tasks BEGIN SELECT RAISE(ABORT, 'outbox unavailable'); END`,
);
store.failDelete = true;
expect(
await code(
service.update('id-1', 1, { password: { action: 'set', password: 'super-secret' } }),
),
).toBe('COMPENSATION_PERSISTENCE_FAILED');
expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
});
it('deletes with CAS, cascades dependent state, then cleans the secret', async () => {
const { db, store, service } = fixture();
await service.create({ ...basic, password: { action: 'set', password: 'gone' } });
db.prepare('insert into instance_tags values (?,?,?) on conflict do nothing').run(
'id-1',
'extra',
'now',
);
expect(await code(service.delete('id-1', 2))).toBe('REVISION_CONFLICT');
await service.delete('id-1', 1);
expect(db.prepare('select * from instances').all()).toHaveLength(0);
expect(db.prepare('select * from secret_references').all()).toHaveLength(0);
expect(store.values.size).toBe(0);
});
it.each([
'http://localhost',
'http://127.0.0.1',
'http://169.254.169.254',
'http://0.0.0.0',
'http://224.0.0.1',
'https://8.8.8.8',
'ftp://10.0.0.1',
'http://u:p@10.0.0.1',
'http://10.0.0.1/x?y=1',
'http://10.0.0.1/#x',
])('rejects unsafe origin %s', async (origin) => {
const { service } = fixture();
expect(await code(service.create({ name: 'x', origin }))).toBe('VALIDATION_FAILED');
});
it('validates bounded names/tags/password and accepts private IPv4 plus IPv6 ULA', async () => {
const { service } = fixture(['one', 'two']);
await service.create({ name: 'x', origin: 'https://172.16.0.1', tags: [' ok '] });
await service.create({ name: 'y', origin: 'http://[fd00::1]:80' });
for (const input of [
{ name: ' ', origin: 'http://10.0.0.1' },
{ name: 'x', origin: 'http://10.0.0.2', tags: [' '] },
{ name: 'x', origin: 'http://10.0.0.3', password: { action: 'set', password: '' } as const },
])
expect(await code(service.create(input))).toBe('VALIDATION_FAILED');
});
it('returns redacted stable errors for secret-store failures and keeps committed cleanup durable', async () => {
const { db, store, service } = fixture();
store.failSet = true;
const setErr = await service
.create({ ...basic, password: { action: 'set', password: 'never-print-me' } })
.then(
() => {
throw new Error('expected set rejection');
},
(e) => e as InstanceServiceError,
);
expect(setErr.code).toBe('SECRET_STORE_FAILED');
expect(JSON.stringify(setErr)).not.toContain('never-print-me');
store.failSet = false;
const created = await service.create({
...basic,
origin: 'http://10.0.0.2',
password: { action: 'set', password: 'cleanup-secret' },
});
store.failDelete = true;
await expect(service.delete(created.id, 1)).resolves.toBeUndefined();
expect(await service.get(created.id)).toBeUndefined();
const pending = db
.prepare('SELECT reference,instance_id,purpose,provider FROM secret_cleanup_tasks')
.all();
expect(pending).toEqual([
expect.objectContaining({
instance_id: created.id,
purpose: 'instance-password',
provider: 'macos-keychain',
reference: expect.stringContaining('keychain://multi-simadmin/'),
}),
]);
store.failDelete = false;
expect(await service.retryPendingSecretCleanup()).toEqual({
attempted: 1,
cleaned: 1,
remaining: 0,
});
expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
});
it('persists cleanup work for replacement and clear until explicit retry succeeds', async () => {
const { db, store, service } = fixture();
await service.create({ ...basic, password: { action: 'set', password: 'first-secret' } });
store.failDelete = true;
const replacement = await service.update('id-1', 1, {
password: { action: 'set', password: 'second-secret' },
});
expect(replacement).toMatchObject({ revision: 2, credentialConfigured: true });
expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toHaveLength(1);
store.failDelete = false;
expect(await service.retryPendingSecretCleanup()).toMatchObject({ remaining: 0 });
const updated = await service.get('id-1');
store.failDelete = true;
const cleared = await service.update('id-1', updated!.revision, {
password: { action: 'clear' },
});
expect(cleared).toMatchObject({ credentialConfigured: false });
expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toHaveLength(1);
});
it('rejects forged cleanup references and treats already absent credentials as cleaned', async () => {
const { db, store, service } = fixture();
const now = '2026-07-16T12:00:00.000Z';
const encoded = (parts: string[]) =>
`keychain://multi-simadmin/${Buffer.from(JSON.stringify(parts), 'utf8').toString('base64url')}`;
const forged = encoded(['other-instance', 'instance-password', 'slot']);
db.prepare(
'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
).run(forged, 'id-1', 'instance-password', 'macos-keychain', now, now);
expect(await service.retryPendingSecretCleanup()).toEqual({
attempted: 1,
cleaned: 0,
remaining: 1,
});
expect(store.deletes).toEqual([]);
db.prepare('DELETE FROM secret_cleanup_tasks').run();
const absent = encoded(['id-1', 'instance-password', 'slot']);
db.prepare(
'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
).run(absent, 'id-1', 'instance-password', 'macos-keychain', now, now);
expect(await service.retryPendingSecretCleanup()).toEqual({
attempted: 1,
cleaned: 1,
remaining: 0,
});
expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
});
it('isolates malformed cleanup rows while processing valid work', async () => {
const { db, store, service } = fixture();
const now = '2026-07-16T12:00:00.000Z';
const valid = `keychain://multi-simadmin/${Buffer.from(
JSON.stringify(['id-1', 'instance-password', 'slot']),
'utf8',
).toString('base64url')}`;
for (const [reference, instanceId] of [
['not-a-reference', 'id-1'],
[valid, 'id-1'],
])
db.prepare(
'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
).run(reference, instanceId, 'instance-password', 'macos-keychain', now, now);
expect(await service.retryPendingSecretCleanup()).toEqual({
attempted: 2,
cleaned: 1,
remaining: 1,
});
expect(store.deletes).toEqual([valid]);
});
it('never deletes a cleanup reference that is active again', async () => {
const { db, store, service } = fixture();
await service.create({ ...basic, password: { action: 'set', password: 'active' } });
const active = db.prepare('SELECT external_reference FROM secret_references').get() as {
external_reference: string;
};
const now = '2026-07-16T12:00:00.000Z';
db.prepare(
'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
).run(active.external_reference, 'id-1', 'instance-password', 'macos-keychain', now, now);
expect(await service.retryPendingSecretCleanup()).toEqual({
attempted: 1,
cleaned: 0,
remaining: 1,
});
expect(store.deletes).toEqual([]);
expect(store.values.has(active.external_reference)).toBe(true);
});
it('does not ACK a requeued cleanup task with an obsolete generation', async () => {
const { db, store, service } = fixture();
await service.create({ ...basic, password: { action: 'set', password: 'first' } });
store.failDelete = true;
await service.update('id-1', 1, { password: { action: 'set', password: 'second' } });
store.failDelete = false;
const reference = db.prepare('SELECT reference FROM secret_cleanup_tasks').get() as {
reference: string;
};
store.onDelete = () => {
db.prepare('UPDATE secret_cleanup_tasks SET generation=generation+1 WHERE reference=?').run(
reference.reference,
);
delete store.onDelete;
};
const result = await service.retryPendingSecretCleanup();
expect(result).toMatchObject({ attempted: 1, cleaned: 0, remaining: 1 });
expect(db.prepare('SELECT generation FROM secret_cleanup_tasks').get()).toEqual({
generation: 2,
});
});
it('aggregates freshness across categories by the worst observed state', async () => {
const { db, service } = fixture();
const created = await service.create(basic);
const now = '2026-07-16T12:00:00.000Z';
db.prepare(
'INSERT INTO status_snapshots (id,instance_id,category,state,payload_json,observed_at,expires_at,created_at) VALUES (?,?,?,?,?,?,?,?)',
).run('snapshot-fresh', created.id, 'cellular', 'fresh', '{}', now, null, now);
db.prepare(
'INSERT INTO status_snapshots (id,instance_id,category,state,payload_json,observed_at,expires_at,created_at) VALUES (?,?,?,?,?,?,?,?)',
).run('snapshot-stale', created.id, 'network', 'stale', '{}', now, null, now);
expect((await service.get(created.id))?.freshness).toBe('stale');
});
it('rejects control separators in tags rather than corrupting tag round-trips', async () => {
const { service } = fixture();
expect(
await code(service.create({ ...basic, tags: ['safe', `bad${String.fromCharCode(31)}tag`] })),
).toBe('VALIDATION_FAILED');
});
it('orders names and tags by stable code points instead of host locale rules', async () => {
const { service } = fixture(['z-id', 'slot-1', 'ref-1', 'a-id', 'slot-2', 'ref-2']);
await service.create({ name: 'a', origin: 'http://10.0.0.1', tags: ['a', 'Z'] });
await service.create({ name: 'Z', origin: 'http://10.0.0.2', tags: [] });
expect((await service.get('z-id'))?.tags).toEqual(['Z', 'a']);
expect((await service.list({ sort: 'name' })).items.map(({ id }) => id)).toEqual([
'slot-1',
'z-id',
]);
});
it('rejects page and revision values that would overflow subsequent calculations', async () => {
const { service } = fixture();
await service.create(basic);
expect(await code(service.list({ page: Number.MAX_SAFE_INTEGER, pageSize: 100 }))).toBe(
'VALIDATION_FAILED',
);
expect(await code(service.update('id-1', Number.MAX_SAFE_INTEGER, { name: 'x' }))).toBe(
'VALIDATION_FAILED',
);
});
it('reports a stable history conflict when deleting an instance referenced by a job item', async () => {
const { db, service } = fixture();
const created = await service.create(basic);
const now = '2026-07-16T12:00:00.000Z';
db.prepare(
'INSERT INTO jobs (id,operation_id,risk_level,status,requested_by,request_id,parameters_digest,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?)',
).run('job-1', 'system.health', 'R0', 'completed', 'test', 'request-1', 'digest', now, now);
db.prepare(
'INSERT INTO job_items (id,job_id,instance_id,attempt_number,status,created_at,updated_at) VALUES (?,?,?,?,?,?,?)',
).run('job-item-1', 'job-1', created.id, 1, 'completed', now, now);
expect(await code(service.delete(created.id, 1))).toBe('HAS_JOB_HISTORY');
expect(await service.get(created.id)).toBeDefined();
});
});