multi-simadmin/apps/api/src/application/instances/instance-service.test.ts

import Database from 'better-sqlite3';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { openDatabase } from '../../infrastructure/database/database.js';
import { migrateDatabase } from '../../infrastructure/database/migrations.js';
import type { SecretStore } from '../../infrastructure/secrets/secret-store.js';
import type { Instance } from '@multi-simadmin/contracts';
import { InstanceService, InstanceServiceError } from './instance-service.js';

class MemorySecrets implements SecretStore {
  readonly values = new Map<string, string>();
  readonly deletes: string[] = [];
  readonly sets: Array<{
    key: { instanceId: string; purpose: string; slot?: string };
    value: string;
  }> = [];
  onDelete?: (reference: string) => void;
  overrideReference?: string;
  failSet = false;
  failDelete = false;
  async set(key: { instanceId: string; purpose: string; slot?: string }, value: string) {
    if (this.failSet) throw new Error(`keychain leaked ${value}`);
    const account = Buffer.from(
      JSON.stringify([key.instanceId, key.purpose, key.slot]),
      'utf8',
    ).toString('base64url');
    const ref = this.overrideReference ?? `keychain://multi-simadmin/${account}`;
    this.sets.push({ key, value });
    this.values.set(ref, value);
    return ref;
  }
  async get(reference: string) {
    return this.values.get(reference);
  }
  async delete(reference: string) {
    this.deletes.push(reference);
    this.onDelete?.(reference);
    if (this.failDelete) throw new Error(`cannot delete ${this.values.get(reference)}`);
    return this.values.delete(reference);
  }
}
const dbs: Database.Database[] = [];
const temporaryDirectories: string[] = [];
afterEach(() => {
  for (const db of dbs.splice(0)) db.close();
  for (const directory of temporaryDirectories.splice(0))
    rmSync(directory, { recursive: true, force: true });
});
function fixture(ids = ['id-1', 'slot-1', 'ref-1', 'id-2', 'slot-2', 'ref-2']) {
  const db = new Database(':memory:');
  db.pragma('foreign_keys=ON');
  migrateDatabase(db);
  dbs.push(db);
  const store = new MemorySecrets();
  let index = 0;
  const service = new InstanceService({
    db,
    store,
    idFactory: () => ids[index++]!,
    now: () => new Date('2026-07-16T12:00:00.000Z'),
  });
  return { db, store, service };
}
function crossConnectionFixture() {
  const directory = mkdtempSync(join(tmpdir(), 'instance-cas-'));
  temporaryDirectories.push(directory);
  const path = join(directory, 'state.sqlite');
  const firstDb = openDatabase(path);
  migrateDatabase(firstDb);
  const secondDb = openDatabase(path);
  dbs.push(firstDb, secondDb);
  const store = new MemorySecrets();
  const options = { store, now: () => new Date('2026-07-16T12:00:00.000Z') };
  const firstIds = ['id-1', 'first-slot', 'first-reference'];
  const secondIds = ['second-slot', 'second-reference'];
  const first = new InstanceService({
    ...options,
    db: firstDb,
    idFactory: () => firstIds.shift()!,
  });
  const second = new InstanceService({
    ...options,
    db: secondDb,
    idFactory: () => secondIds.shift()!,
  });
  return { firstDb, secondDb, store, first, second };
}
const basic = { name: ' Alpha ', origin: 'http://192.168.1.10:8080/', tags: [' z ', 'a', 'a'] };
const code = async (promise: Promise<unknown>) => {
  try {
    await promise;
  } catch (e) {
    return (e as InstanceServiceError).code;
  }
  throw new Error('expected rejection');
};

describe('InstanceService', () => {
  it('starts RED until the repository module exists', () =>
    expect(InstanceService).toBeTypeOf('function'));

  it('creates and gets normalized instances without reading or persisting a secret', async () => {
    const { db, store, service } = fixture();
    const created = await service.create(basic);
    expect(created).toEqual({
      id: 'id-1',
      name: 'Alpha',
      origin: 'http://192.168.1.10:8080',
      tags: ['a', 'z'],
      groupId: null,
      revision: 1,
      capabilityStatus: 'unknown',
      freshness: 'unknown',
      credentialConfigured: false,
    });
    expect(await service.get('id-1')).toEqual(created);
    expect(store.sets).toHaveLength(0);
    expect(
      JSON.stringify(
        db.prepare('select * from instances join instance_tags on id=instance_id').all(),
      ),
    ).not.toContain('password');
  });

  it('stores a password under a unique slot and SQLite contains only its opaque reference', async () => {
    const { db, store, service } = fixture();
    const secret = 'super-secret-2.4A';
    const result = await service.create({
      ...basic,
      password: { action: 'set', password: secret },
    });
    expect(result.credentialConfigured).toBe(true);
    expect(store.sets[0]).toMatchObject({
      key: { instanceId: 'id-1', purpose: 'instance-password', slot: 'slot-1' },
      value: secret,
    });
    const allText = JSON.stringify(
      db
        .prepare('select id,instance_id,purpose,provider,external_reference from secret_references')
        .all(),
    );
    expect(allText).toContain('macos-keychain');
    expect(allText).not.toContain(secret);
    expect(JSON.stringify(result)).not.toContain(secret);
    expect(store.get).toBeTypeOf('function');
  });

  it('rejects a secret-store reference not bound to the requested instance and slot', async () => {
    const { db, store, service } = fixture();
    store.overrideReference = `keychain://multi-simadmin/${Buffer.from(
      JSON.stringify(['other-instance', 'instance-password', 'other-slot']),
      'utf8',
    ).toString('base64url')}`;
    expect(
      await code(service.create({ ...basic, password: { action: 'set', password: 'secret' } })),
    ).toBe('SECRET_STORE_FAILED');
    expect(db.prepare('SELECT * FROM secret_references').all()).toEqual([]);
    expect(store.deletes).toEqual([store.overrideReference]);
    expect(store.values.size).toBe(0);
  });

  it('fails closed when an invalid returned secret reference cannot be removed', async () => {
    const { db, store, service } = fixture();
    store.overrideReference = `keychain://multi-simadmin/${Buffer.from(
      JSON.stringify(['other-instance', 'instance-password', 'other-slot']),
      'utf8',
    ).toString('base64url')}`;
    store.failDelete = true;
    expect(
      await code(service.create({ ...basic, password: { action: 'set', password: 'secret' } })),
    ).toBe('COMPENSATION_PERSISTENCE_FAILED');
    expect(db.prepare('SELECT * FROM secret_references').all()).toEqual([]);
    expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
  });

  it('rejects duplicate normalized origins and duplicate generated ids with stable errors', async () => {
    const x = fixture(['same', 'same', 'same']);
    await x.service.create(basic);
    expect(await code(x.service.create({ ...basic, name: 'B' }))).toBe('DUPLICATE_ORIGIN');
    expect(await code(x.service.create({ ...basic, origin: 'http://10.1.2.3' }))).toBe(
      'DUPLICATE_ID',
    );
  });

  it('filters, sorts with id tie-breaks, and paginates aggregate output', async () => {
    const { db, service } = fixture(['b', 'a']);
    await service.create({ name: 'Same', origin: 'http://10.0.0.2', tags: ['blue'] });
    await service.create({
      name: 'Same',
      origin: 'http://10.0.0.3',
      tags: ['red'],
      password: { action: 'preserve' },
    });
    db.prepare('insert into capabilities values (?,?,?,?,?,?,?,?)').run(
      'b',
      'x',
      'supported',
      null,
      null,
      '2026-07-16T11:00:00Z',
      '2026-07-16T11:00:00Z',
      '2026-07-16T11:00:00Z',
    );
    db.prepare('insert into capabilities values (?,?,?,?,?,?,?,?)').run(
      'b',
      'y',
      'degraded',
      null,
      null,
      '2026-07-16T11:00:00Z',
      '2026-07-16T11:00:00Z',
      '2026-07-16T11:00:00Z',
    );
    db.prepare('insert into status_snapshots values (?,?,?,?,?,?,?,?)').run(
      's',
      'b',
      'health',
      'fresh',
      '{}',
      '2026-07-16T11:00:00Z',
      '2026-07-16T13:00:00Z',
      '2026-07-16T11:00:00Z',
    );
    expect(
      (
        await service.list({
          search: 'b',
          tag: 'blue',
          capabilityStatus: 'degraded',
          freshness: 'fresh',
        })
      ).items.map((x: Instance) => x.id),
    ).toEqual(['b']);
    expect((await service.list({ sort: 'name', page: 1, pageSize: 1 })).items[0]?.id).toBe('a');
    expect((await service.list({ sort: 'name', page: 2, pageSize: 1 })).page).toEqual({
      page: 2,
      pageSize: 1,
      total: 2,
    });
  });

  it('updates with revision CAS, normalized fields, and one revision increment', async () => {
    const { service } = fixture();
    await service.create(basic);
    expect(await code(service.update('id-1', 9, { name: 'No' }))).toBe('REVISION_CONFLICT');
    const result = await service.update('id-1', 1, { name: ' Beta ', tags: ['x'] });
    expect(result).toMatchObject({
      name: 'Beta',
      tags: ['x'],
      revision: 2,
      credentialConfigured: false,
    });
  });

  it('rejects stale update and delete CAS across managed connections', async () => {
    const { firstDb, first, second } = crossConnectionFixture();
    await first.create(basic);
    await second.update('id-1', 1, { name: 'Committed elsewhere' });

    await expect(first.update('id-1', 1, { name: 'Stale update' })).rejects.toMatchObject({
      code: 'REVISION_CONFLICT',
    });
    await expect(first.delete('id-1', 1)).rejects.toMatchObject({ code: 'REVISION_CONFLICT' });
    expect(
      firstDb.prepare('SELECT name,config_revision FROM instances WHERE id=?').get('id-1'),
    ).toEqual({
      name: 'Committed elsewhere',
      config_revision: 2,
    });
  });

  it('preserves, replaces and clears credentials without exposing them', async () => {
    const { db, store, service } = fixture();
    await service.create({ ...basic, password: { action: 'set', password: 'first-secret' } });
    await service.update('id-1', 1, { password: { action: 'preserve' } });
    expect(store.sets).toHaveLength(1);
    const replaced = await service.update('id-1', 2, {
      password: { action: 'set', password: 'second-secret' },
    });
    expect(replaced.credentialConfigured).toBe(true);
    expect(store.deletes).toEqual([
      expect.stringMatching(/^keychain:\/\/multi-simadmin\/[A-Za-z0-9_-]+$/),
    ]);
    const cleared = await service.update('id-1', 3, { password: { action: 'clear' } });
    expect(cleared.credentialConfigured).toBe(false);
    expect(db.prepare('select * from secret_references').all()).toHaveLength(0);
  });

  it('compensates a newly stored secret when the DB transaction fails', async () => {
    const { db, store, service } = fixture();
    await service.create(basic);
    const occupiedReference = `keychain://multi-simadmin/${Buffer.from(
      JSON.stringify(['id-1', 'instance-password', 'slot-1']),
      'utf8',
    ).toString('base64url')}`;
    db.prepare('insert into secret_references values (?,?,?,?,?,?,?)').run(
      'occupied',
      null,
      'other',
      'macos-keychain',
      occupiedReference,
      'now',
      'now',
    );
    const error = await code(
      service.update('id-1', 1, { password: { action: 'set', password: 'super-secret' } }),
    );
    expect(error).toBe('DATABASE_FAILED');
    expect(store.values.size).toBe(0);
  });

  it('surfaces an explicit fail-closed error when compensation cleanup cannot be persisted', async () => {
    const { db, store, service } = fixture();
    await service.create(basic);
    const occupiedReference = `keychain://multi-simadmin/${Buffer.from(
      JSON.stringify(['id-1', 'instance-password', 'slot-1']),
      'utf8',
    ).toString('base64url')}`;
    db.prepare('insert into secret_references values (?,?,?,?,?,?,?)').run(
      'occupied',
      null,
      'other',
      'macos-keychain',
      occupiedReference,
      'now',
      'now',
    );
    db.exec(
      `CREATE TRIGGER fail_cleanup_outbox BEFORE INSERT ON secret_cleanup_tasks BEGIN SELECT RAISE(ABORT, 'outbox unavailable'); END`,
    );
    store.failDelete = true;
    expect(
      await code(
        service.update('id-1', 1, { password: { action: 'set', password: 'super-secret' } }),
      ),
    ).toBe('COMPENSATION_PERSISTENCE_FAILED');
    expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
  });

  it('deletes with CAS, cascades dependent state, then cleans the secret', async () => {
    const { db, store, service } = fixture();
    await service.create({ ...basic, password: { action: 'set', password: 'gone' } });
    db.prepare('insert into instance_tags values (?,?,?) on conflict do nothing').run(
      'id-1',
      'extra',
      'now',
    );
    expect(await code(service.delete('id-1', 2))).toBe('REVISION_CONFLICT');
    await service.delete('id-1', 1);
    expect(db.prepare('select * from instances').all()).toHaveLength(0);
    expect(db.prepare('select * from secret_references').all()).toHaveLength(0);
    expect(store.values.size).toBe(0);
  });

  it.each([
    'http://localhost',
    'http://127.0.0.1',
    'http://169.254.169.254',
    'http://0.0.0.0',
    'http://224.0.0.1',
    'https://8.8.8.8',
    'ftp://10.0.0.1',
    'http://u:p@10.0.0.1',
    'http://10.0.0.1/x?y=1',
    'http://10.0.0.1/#x',
  ])('rejects unsafe origin %s', async (origin) => {
    const { service } = fixture();
    expect(await code(service.create({ name: 'x', origin }))).toBe('VALIDATION_FAILED');
  });

  it('validates bounded names/tags/password and accepts private IPv4 plus IPv6 ULA', async () => {
    const { service } = fixture(['one', 'two']);
    await service.create({ name: 'x', origin: 'https://172.16.0.1', tags: [' ok '] });
    await service.create({ name: 'y', origin: 'http://[fd00::1]:80' });
    for (const input of [
      { name: ' ', origin: 'http://10.0.0.1' },
      { name: 'x', origin: 'http://10.0.0.2', tags: [' '] },
      { name: 'x', origin: 'http://10.0.0.3', password: { action: 'set', password: '' } as const },
    ])
      expect(await code(service.create(input))).toBe('VALIDATION_FAILED');
  });

  it('returns redacted stable errors for secret-store failures and keeps committed cleanup durable', async () => {
    const { db, store, service } = fixture();
    store.failSet = true;
    const setErr = await service
      .create({ ...basic, password: { action: 'set', password: 'never-print-me' } })
      .then(
        () => {
          throw new Error('expected set rejection');
        },
        (e) => e as InstanceServiceError,
      );
    expect(setErr.code).toBe('SECRET_STORE_FAILED');
    expect(JSON.stringify(setErr)).not.toContain('never-print-me');
    store.failSet = false;
    const created = await service.create({
      ...basic,
      origin: 'http://10.0.0.2',
      password: { action: 'set', password: 'cleanup-secret' },
    });
    store.failDelete = true;
    await expect(service.delete(created.id, 1)).resolves.toBeUndefined();
    expect(await service.get(created.id)).toBeUndefined();
    const pending = db
      .prepare('SELECT reference,instance_id,purpose,provider FROM secret_cleanup_tasks')
      .all();
    expect(pending).toEqual([
      expect.objectContaining({
        instance_id: created.id,
        purpose: 'instance-password',
        provider: 'macos-keychain',
        reference: expect.stringContaining('keychain://multi-simadmin/'),
      }),
    ]);
    store.failDelete = false;
    expect(await service.retryPendingSecretCleanup()).toEqual({
      attempted: 1,
      cleaned: 1,
      remaining: 0,
    });
    expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
  });

  it('persists cleanup work for replacement and clear until explicit retry succeeds', async () => {
    const { db, store, service } = fixture();
    await service.create({ ...basic, password: { action: 'set', password: 'first-secret' } });
    store.failDelete = true;
    const replacement = await service.update('id-1', 1, {
      password: { action: 'set', password: 'second-secret' },
    });
    expect(replacement).toMatchObject({ revision: 2, credentialConfigured: true });
    expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toHaveLength(1);
    store.failDelete = false;
    expect(await service.retryPendingSecretCleanup()).toMatchObject({ remaining: 0 });
    const updated = await service.get('id-1');
    store.failDelete = true;
    const cleared = await service.update('id-1', updated!.revision, {
      password: { action: 'clear' },
    });
    expect(cleared).toMatchObject({ credentialConfigured: false });
    expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toHaveLength(1);
  });

  it('rejects forged cleanup references and treats already absent credentials as cleaned', async () => {
    const { db, store, service } = fixture();
    const now = '2026-07-16T12:00:00.000Z';
    const encoded = (parts: string[]) =>
      `keychain://multi-simadmin/${Buffer.from(JSON.stringify(parts), 'utf8').toString('base64url')}`;
    const forged = encoded(['other-instance', 'instance-password', 'slot']);
    db.prepare(
      'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
    ).run(forged, 'id-1', 'instance-password', 'macos-keychain', now, now);
    expect(await service.retryPendingSecretCleanup()).toEqual({
      attempted: 1,
      cleaned: 0,
      remaining: 1,
    });
    expect(store.deletes).toEqual([]);
    db.prepare('DELETE FROM secret_cleanup_tasks').run();

    const absent = encoded(['id-1', 'instance-password', 'slot']);
    db.prepare(
      'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
    ).run(absent, 'id-1', 'instance-password', 'macos-keychain', now, now);
    expect(await service.retryPendingSecretCleanup()).toEqual({
      attempted: 1,
      cleaned: 1,
      remaining: 0,
    });
    expect(db.prepare('SELECT * FROM secret_cleanup_tasks').all()).toEqual([]);
  });

  it('isolates malformed cleanup rows while processing valid work', async () => {
    const { db, store, service } = fixture();
    const now = '2026-07-16T12:00:00.000Z';
    const valid = `keychain://multi-simadmin/${Buffer.from(
      JSON.stringify(['id-1', 'instance-password', 'slot']),
      'utf8',
    ).toString('base64url')}`;
    for (const [reference, instanceId] of [
      ['not-a-reference', 'id-1'],
      [valid, 'id-1'],
    ])
      db.prepare(
        'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
      ).run(reference, instanceId, 'instance-password', 'macos-keychain', now, now);
    expect(await service.retryPendingSecretCleanup()).toEqual({
      attempted: 2,
      cleaned: 1,
      remaining: 1,
    });
    expect(store.deletes).toEqual([valid]);
  });

  it('never deletes a cleanup reference that is active again', async () => {
    const { db, store, service } = fixture();
    await service.create({ ...basic, password: { action: 'set', password: 'active' } });
    const active = db.prepare('SELECT external_reference FROM secret_references').get() as {
      external_reference: string;
    };
    const now = '2026-07-16T12:00:00.000Z';
    db.prepare(
      'INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,queued_at,updated_at) VALUES (?,?,?,?,?,?)',
    ).run(active.external_reference, 'id-1', 'instance-password', 'macos-keychain', now, now);
    expect(await service.retryPendingSecretCleanup()).toEqual({
      attempted: 1,
      cleaned: 0,
      remaining: 1,
    });
    expect(store.deletes).toEqual([]);
    expect(store.values.has(active.external_reference)).toBe(true);
  });

  it('does not ACK a requeued cleanup task with an obsolete generation', async () => {
    const { db, store, service } = fixture();
    await service.create({ ...basic, password: { action: 'set', password: 'first' } });
    store.failDelete = true;
    await service.update('id-1', 1, { password: { action: 'set', password: 'second' } });
    store.failDelete = false;
    const reference = db.prepare('SELECT reference FROM secret_cleanup_tasks').get() as {
      reference: string;
    };
    store.onDelete = () => {
      db.prepare('UPDATE secret_cleanup_tasks SET generation=generation+1 WHERE reference=?').run(
        reference.reference,
      );
      delete store.onDelete;
    };
    const result = await service.retryPendingSecretCleanup();
    expect(result).toMatchObject({ attempted: 1, cleaned: 0, remaining: 1 });
    expect(db.prepare('SELECT generation FROM secret_cleanup_tasks').get()).toEqual({
      generation: 2,
    });
  });

  it('aggregates freshness across categories by the worst observed state', async () => {
    const { db, service } = fixture();
    const created = await service.create(basic);
    const now = '2026-07-16T12:00:00.000Z';
    db.prepare(
      'INSERT INTO status_snapshots (id,instance_id,category,state,payload_json,observed_at,expires_at,created_at) VALUES (?,?,?,?,?,?,?,?)',
    ).run('snapshot-fresh', created.id, 'cellular', 'fresh', '{}', now, null, now);
    db.prepare(
      'INSERT INTO status_snapshots (id,instance_id,category,state,payload_json,observed_at,expires_at,created_at) VALUES (?,?,?,?,?,?,?,?)',
    ).run('snapshot-stale', created.id, 'network', 'stale', '{}', now, null, now);
    expect((await service.get(created.id))?.freshness).toBe('stale');
  });

  it('rejects control separators in tags rather than corrupting tag round-trips', async () => {
    const { service } = fixture();
    expect(
      await code(service.create({ ...basic, tags: ['safe', `bad${String.fromCharCode(31)}tag`] })),
    ).toBe('VALIDATION_FAILED');
  });

  it('orders names and tags by stable code points instead of host locale rules', async () => {
    const { service } = fixture(['z-id', 'slot-1', 'ref-1', 'a-id', 'slot-2', 'ref-2']);
    await service.create({ name: 'a', origin: 'http://10.0.0.1', tags: ['a', 'Z'] });
    await service.create({ name: 'Z', origin: 'http://10.0.0.2', tags: [] });
    expect((await service.get('z-id'))?.tags).toEqual(['Z', 'a']);
    expect((await service.list({ sort: 'name' })).items.map(({ id }) => id)).toEqual([
      'slot-1',
      'z-id',
    ]);
  });

  it('rejects page and revision values that would overflow subsequent calculations', async () => {
    const { service } = fixture();
    await service.create(basic);
    expect(await code(service.list({ page: Number.MAX_SAFE_INTEGER, pageSize: 100 }))).toBe(
      'VALIDATION_FAILED',
    );
    expect(await code(service.update('id-1', Number.MAX_SAFE_INTEGER, { name: 'x' }))).toBe(
      'VALIDATION_FAILED',
    );
  });

  it('reports a stable history conflict when deleting an instance referenced by a job item', async () => {
    const { db, service } = fixture();
    const created = await service.create(basic);
    const now = '2026-07-16T12:00:00.000Z';
    db.prepare(
      'INSERT INTO jobs (id,operation_id,risk_level,status,requested_by,request_id,parameters_digest,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?)',
    ).run('job-1', 'system.health', 'R0', 'completed', 'test', 'request-1', 'digest', now, now);
    db.prepare(
      'INSERT INTO job_items (id,job_id,instance_id,attempt_number,status,created_at,updated_at) VALUES (?,?,?,?,?,?,?)',
    ).run('job-item-1', 'job-1', created.id, 1, 'completed', now, now);
    expect(await code(service.delete(created.id, 1))).toBe('HAS_JOB_HISTORY');
    expect(await service.get(created.id)).toBeDefined();
  });
});