feat(api): complete phase 2.4 control plane

2026-07-17 00:37:11 +08:00
parent c2702b5fc6
commit 7b91dbbad1
40 changed files with 4526 additions and 74 deletions
@@ -0,0 +1,602 @@
+import type Database from 'better-sqlite3';
+import { randomUUID } from 'node:crypto';
+import type {
+  CapabilityStatus,
+  Instance,
+  InstanceInput,
+  InstancePage,
+  InstancePageQuery,
+  InstancePatch,
+  SnapshotFreshness,
+} from '@multi-simadmin/contracts';
+import type { SecretStore } from '../../infrastructure/secrets/secret-store.js';
+import { parseKeychainReference } from '../../infrastructure/secrets/keychain-secret-store.js';
+
+const PURPOSE = 'instance-password';
+const PROVIDER = 'macos-keychain';
+const MAX_TAGS = 50;
+const MAX_TAG_LENGTH = 100;
+
+export type InstanceServiceErrorCode =
+  | 'VALIDATION_FAILED'
+  | 'NOT_FOUND'
+  | 'REVISION_CONFLICT'
+  | 'HAS_JOB_HISTORY'
+  | 'DUPLICATE_ID'
+  | 'DUPLICATE_ORIGIN'
+  | 'SECRET_STORE_FAILED'
+  | 'DATABASE_FAILED'
+  | 'SECRET_CLEANUP_FAILED'
+  | 'COMPENSATION_FAILED'
+  | 'COMPENSATION_PERSISTENCE_FAILED';
+
+export class InstanceServiceError extends Error {
+  constructor(
+    readonly code: InstanceServiceErrorCode,
+    message: string,
+  ) {
+    super(message);
+    this.name = 'InstanceServiceError';
+  }
+}
+
+export interface InstanceServiceOptions {
+  readonly db: Database.Database;
+  readonly store: SecretStore;
+  readonly idFactory?: () => string;
+  readonly now?: () => Date;
+}
+
+interface InstanceRow {
+  id: string;
+  name: string;
+  base_url: string;
+  config_revision: number;
+  updated_at: string;
+}
+interface SecretRow {
+  id: string;
+  external_reference: string;
+}
+interface CleanupEntry {
+  readonly instanceId: string;
+  readonly reference: string;
+  readonly purpose: string;
+  readonly provider: string;
+  readonly generation: number;
+  readonly queuedAt: string;
+}
+interface AggregateRow extends InstanceRow {
+  tags: string | null;
+  credential: number;
+}
+
+function validation(message: string): never {
+  throw new InstanceServiceError('VALIDATION_FAILED', message);
+}
+function normalizeName(value: unknown): string {
+  if (typeof value !== 'string') return validation('Name must be a string');
+  const result = value.trim();
+  if (!result || result.length > 200) return validation('Name must contain 1 to 200 characters');
+  return result;
+}
+function normalizeTags(value: readonly string[] | undefined): string[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value) || value.length > MAX_TAGS) return validation('Tags are invalid');
+  const tags = value.map((tag) => {
+    if (typeof tag !== 'string') return validation('Tag must be a string');
+    const normalized = tag.trim();
+    if (!normalized || normalized.length > MAX_TAG_LENGTH || /[\x00-\x1F\x7F]/.test(normalized))
+      return validation('Tag is invalid');
+    return normalized;
+  });
+  const unique = [...new Set(tags)].sort(codePointCompare);
+  if (unique.length > MAX_TAGS) return validation('Too many tags');
+  return unique;
+}
+function privateIpv4(host: string): boolean {
+  const parts = host.split('.');
+  if (parts.length !== 4 || parts.some((part) => !/^\d{1,3}$/.test(part))) return false;
+  const octets = parts.map(Number);
+  if (octets.some((part) => part > 255)) return false;
+  const [a, b] = octets as [number, number, number, number];
+  return a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168);
+}
+function privateIpv6(host: string): boolean {
+  const normalized = host.toLowerCase().replace(/^\[|\]$/g, '');
+  return /^(fc|fd)[0-9a-f]{2}:/.test(normalized);
+}
+function normalizeOrigin(value: unknown): string {
+  if (typeof value !== 'string') return validation('Origin must be a URL');
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    return validation('Origin must be a valid URL');
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:')
+    return validation('Origin must use HTTP or HTTPS');
+  if (url.username || url.password || url.search || url.hash)
+    return validation('Origin must not contain credentials, query, or fragment');
+  if (url.pathname !== '/' && url.pathname !== '')
+    return validation('Origin must not contain a path');
+  if (!privateIpv4(url.hostname) && !privateIpv6(url.hostname))
+    return validation('Origin must target a private LAN address');
+  return url.origin;
+}
+function validatePassword(value: InstanceInput['password'] | InstancePatch['password']): void {
+  if (value === undefined || value.action === 'preserve' || value.action === 'clear') return;
+  if (value.action !== 'set' || typeof value.password !== 'string' || value.password.length === 0)
+    validation('Password update is invalid');
+}
+
+const capabilityRank: Record<CapabilityStatus, number> = {
+  degraded: 0,
+  'auth-required': 1,
+  unsupported: 2,
+  supported: 3,
+  unknown: 4,
+};
+const freshnessRank: Record<SnapshotFreshness, number> = {
+  expired: 0,
+  stale: 1,
+  fresh: 2,
+  unknown: 3,
+};
+
+export class InstanceService {
+  private readonly db: Database.Database;
+  private readonly store: SecretStore;
+  private readonly id: () => string;
+  private readonly clock: () => Date;
+
+  constructor(options: InstanceServiceOptions) {
+    this.db = options.db;
+    this.store = options.store;
+    this.id = options.idFactory ?? randomUUID;
+    this.clock = options.now ?? (() => new Date());
+  }
+
+  async create(input: InstanceInput): Promise<Instance> {
+    const name = normalizeName(input.name);
+    const origin = normalizeOrigin(input.origin);
+    const tags = normalizeTags(input.tags);
+    validatePassword(input.password);
+    const instanceId = this.id();
+    let newSecret: { id: string; external: string } | undefined;
+    if (input.password?.action === 'set')
+      newSecret = await this.storeSecret(instanceId, input.password.password);
+    const now = this.clock().toISOString();
+    try {
+      this.db.transaction(() => {
+        this.db
+          .prepare(
+            'INSERT INTO instances (id,name,base_url,auth_mode,enabled,config_revision,created_at,updated_at) VALUES (?,?,?, ?,1,1,?,?)',
+          )
+          .run(instanceId, name, origin, newSecret ? 'password' : 'none', now, now);
+        this.replaceTags(instanceId, tags, now);
+        if (newSecret) this.insertReference(newSecret.id, instanceId, newSecret.external, now);
+      })();
+    } catch (error) {
+      if (newSecret) await this.compensate(instanceId, newSecret.external);
+      if (isUnique(error, 'base_url'))
+        throw new InstanceServiceError('DUPLICATE_ORIGIN', 'Instance origin already exists');
+      if (isUnique(error, 'instances.id'))
+        throw new InstanceServiceError('DUPLICATE_ID', 'Instance id already exists');
+      throw new InstanceServiceError('DATABASE_FAILED', 'Could not create instance');
+    }
+    return (await this.get(instanceId))!;
+  }
+
+  async get(instanceId: string): Promise<Instance | undefined> {
+    const rows = this.loadRows('WHERE i.id = ?', [instanceId]);
+    return rows[0] ? this.toInstance(rows[0]) : undefined;
+  }
+
+  async list(query: InstancePageQuery = {}): Promise<InstancePage> {
+    const pageSize = integerInRange(query.pageSize ?? 25, 1, 100, 'pageSize');
+    const page = integerInRange(
+      query.page ?? 1,
+      1,
+      Math.floor(Number.MAX_SAFE_INTEGER / pageSize) + 1,
+      'page',
+    );
+    if (query.direction !== undefined && query.direction !== 'asc' && query.direction !== 'desc')
+      validation('Direction is invalid');
+    const validSort = ['name', 'status', 'freshness', 'updatedAt'];
+    if (query.sort !== undefined && !validSort.includes(query.sort)) validation('Sort is invalid');
+    let values = this.loadRows('', []).map((row) => ({
+      value: this.toInstance(row),
+      updatedAt: row.updated_at,
+    }));
+    const search = query.search?.trim().toLowerCase();
+    if (search)
+      values = values.filter(
+        ({ value }) =>
+          value.name.toLowerCase().includes(search) || value.id.toLowerCase().includes(search),
+      );
+    if (query.tag !== undefined)
+      values = values.filter(({ value }) => value.tags.includes(query.tag!.trim()));
+    if (query.credentialConfigured !== undefined)
+      values = values.filter(
+        ({ value }) => value.credentialConfigured === query.credentialConfigured,
+      );
+    if (query.capabilityStatus !== undefined)
+      values = values.filter(({ value }) => value.capabilityStatus === query.capabilityStatus);
+    if (query.freshness !== undefined)
+      values = values.filter(({ value }) => value.freshness === query.freshness);
+    const direction = query.direction === 'desc' ? -1 : 1;
+    const sort = query.sort ?? 'name';
+    values.sort((left, right) => {
+      let result: number;
+      if (sort === 'status')
+        result =
+          capabilityRank[left.value.capabilityStatus]! -
+          capabilityRank[right.value.capabilityStatus]!;
+      else if (sort === 'freshness')
+        result = freshnessRank[left.value.freshness]! - freshnessRank[right.value.freshness]!;
+      else if (sort === 'updatedAt') result = codePointCompare(left.updatedAt, right.updatedAt);
+      else result = codePointCompare(left.value.name, right.value.name);
+      return result === 0 ? codePointCompare(left.value.id, right.value.id) : result * direction;
+    });
+    const total = values.length;
+    const start = (page - 1) * pageSize;
+    return {
+      items: values.slice(start, start + pageSize).map(({ value }) => value),
+      page: { page, pageSize, total },
+    };
+  }
+
+  async update(instanceId: string, revision: number, patch: InstancePatch): Promise<Instance> {
+    integerInRange(revision, 1, Number.MAX_SAFE_INTEGER - 1, 'revision');
+    validatePassword(patch.password);
+    const current = this.row(instanceId);
+    if (!current) throw new InstanceServiceError('NOT_FOUND', 'Instance was not found');
+    if (current.config_revision !== revision)
+      throw new InstanceServiceError('REVISION_CONFLICT', 'Instance revision does not match');
+    const name = patch.name === undefined ? current.name : normalizeName(patch.name);
+    const origin = patch.origin === undefined ? current.base_url : normalizeOrigin(patch.origin);
+    const tags = patch.tags === undefined ? undefined : normalizeTags(patch.tags);
+    let committedOldSecret: SecretRow | undefined;
+    let newSecret: { id: string; external: string } | undefined;
+    if (patch.password?.action === 'set')
+      newSecret = await this.storeSecret(instanceId, patch.password.password);
+    const now = this.clock().toISOString();
+    try {
+      this.db.transaction(() => {
+        const transactionCurrent = this.row(instanceId);
+        if (!transactionCurrent)
+          throw new InstanceServiceError('NOT_FOUND', 'Instance was not found');
+        if (transactionCurrent.config_revision !== revision)
+          throw new InstanceServiceError('REVISION_CONFLICT', 'Instance revision does not match');
+        const transactionOldSecret = this.secret(instanceId);
+        const changed = this.db
+          .prepare(
+            'UPDATE instances SET name=?,base_url=?,auth_mode=?,config_revision=config_revision+1,updated_at=? WHERE id=? AND config_revision=?',
+          )
+          .run(
+            name,
+            origin,
+            newSecret || (transactionOldSecret && patch.password?.action !== 'clear')
+              ? 'password'
+              : 'none',
+            now,
+            instanceId,
+            revision,
+          );
+        if (changed.changes !== 1)
+          throw new InstanceServiceError('REVISION_CONFLICT', 'Instance revision does not match');
+        if (tags) this.replaceTags(instanceId, tags, now);
+        if (patch.password?.action === 'clear' || newSecret)
+          this.db
+            .prepare('DELETE FROM secret_references WHERE instance_id=? AND purpose=?')
+            .run(instanceId, PURPOSE);
+        if (newSecret) this.insertReference(newSecret.id, instanceId, newSecret.external, now);
+        if (transactionOldSecret && (patch.password?.action === 'clear' || newSecret)) {
+          this.queueCleanup(instanceId, transactionOldSecret.external_reference, now);
+          committedOldSecret = transactionOldSecret;
+        }
+      })();
+    } catch (error) {
+      if (newSecret) await this.compensate(instanceId, newSecret.external);
+      if (error instanceof InstanceServiceError) throw error;
+      if (isUnique(error, 'base_url'))
+        throw new InstanceServiceError('DUPLICATE_ORIGIN', 'Instance origin already exists');
+      throw new InstanceServiceError('DATABASE_FAILED', 'Could not update instance');
+    }
+    if (committedOldSecret)
+      await this.tryCleanup(instanceId, committedOldSecret.external_reference);
+    return (await this.get(instanceId))!;
+  }
+
+  async delete(
+    instanceId: string,
+    revision: number,
+    options?: { allowJobHistory?: boolean },
+  ): Promise<void> {
+    integerInRange(revision, 1, Number.MAX_SAFE_INTEGER - 1, 'revision');
+    const allowJobHistory = options?.allowJobHistory === true;
+    let committedOldSecret: SecretRow | undefined;
+    try {
+      this.db.transaction(() => {
+        const transactionCurrent = this.row(instanceId);
+        if (!transactionCurrent)
+          throw new InstanceServiceError('NOT_FOUND', 'Instance was not found');
+        if (transactionCurrent.config_revision !== revision)
+          throw new InstanceServiceError('REVISION_CONFLICT', 'Instance revision does not match');
+        if (
+          !allowJobHistory &&
+          this.db.prepare('SELECT 1 FROM job_items WHERE instance_id=? LIMIT 1').get(instanceId)
+        )
+          throw new InstanceServiceError('HAS_JOB_HISTORY', 'Instance has immutable job history');
+        const transactionOldSecret = this.secret(instanceId);
+        if (transactionOldSecret) {
+          this.queueCleanup(
+            instanceId,
+            transactionOldSecret.external_reference,
+            this.clock().toISOString(),
+          );
+          committedOldSecret = transactionOldSecret;
+        }
+        const deleted = this.db
+          .prepare('DELETE FROM instances WHERE id=? AND config_revision=?')
+          .run(instanceId, revision);
+        if (deleted.changes !== 1)
+          throw new InstanceServiceError('REVISION_CONFLICT', 'Instance revision does not match');
+      })();
+    } catch (error) {
+      if (error instanceof InstanceServiceError) throw error;
+      if (
+        !allowJobHistory &&
+        this.db.prepare('SELECT 1 FROM job_items WHERE instance_id=? LIMIT 1').get(instanceId)
+      )
+        throw new InstanceServiceError('HAS_JOB_HISTORY', 'Instance has immutable job history');
+      throw new InstanceServiceError('DATABASE_FAILED', 'Could not delete instance');
+    }
+    if (committedOldSecret)
+      await this.tryCleanup(instanceId, committedOldSecret.external_reference);
+  }
+
+  async retryPendingSecretCleanup(): Promise<{
+    attempted: number;
+    cleaned: number;
+    remaining: number;
+  }> {
+    const entries = this.cleanupEntries();
+    let cleaned = 0;
+    for (const entry of entries) {
+      try {
+        this.validateCleanupEntry(entry);
+        if (this.isActiveReference(entry.reference)) continue;
+        await this.store.delete(entry.reference);
+        if (this.removeCleanup(entry.reference, entry.generation)) cleaned += 1;
+      } catch {
+        // Retain the opaque reference for a later explicit retry.
+      }
+    }
+    return { attempted: entries.length, cleaned, remaining: this.cleanupEntries().length };
+  }
+
+  private row(id: string): InstanceRow | undefined {
+    return this.db
+      .prepare('SELECT id,name,base_url,config_revision,updated_at FROM instances WHERE id=?')
+      .get(id) as InstanceRow | undefined;
+  }
+  private secret(id: string): SecretRow | undefined {
+    return this.db
+      .prepare(
+        'SELECT id,external_reference FROM secret_references WHERE instance_id=? AND purpose=?',
+      )
+      .get(id, PURPOSE) as SecretRow | undefined;
+  }
+  private replaceTags(id: string, tags: readonly string[], now: string): void {
+    this.db.prepare('DELETE FROM instance_tags WHERE instance_id=?').run(id);
+    const insert = this.db.prepare(
+      'INSERT INTO instance_tags (instance_id,tag,created_at) VALUES (?,?,?)',
+    );
+    for (const tag of tags) insert.run(id, tag, now);
+  }
+  private insertReference(id: string, instanceId: string, external: string, now: string): void {
+    this.db
+      .prepare(
+        'INSERT INTO secret_references (id,instance_id,purpose,provider,external_reference,created_at,updated_at) VALUES (?,?,?,?,?,?,?)',
+      )
+      .run(id, instanceId, PURPOSE, PROVIDER, external, now, now);
+  }
+  private async storeSecret(
+    instanceId: string,
+    password: string,
+  ): Promise<{ id: string; external: string }> {
+    const slot = this.id();
+    let external: string;
+    try {
+      external = await this.store.set({ instanceId, purpose: PURPOSE, slot }, password);
+    } catch {
+      throw new InstanceServiceError('SECRET_STORE_FAILED', 'Could not store instance credential');
+    }
+    try {
+      const parsed = parseKeychainReference(external);
+      if (parsed.instanceId !== instanceId || parsed.purpose !== PURPOSE || parsed.slot !== slot)
+        throw new Error('secret store returned a reference with an invalid binding');
+    } catch {
+      try {
+        await this.store.delete(external);
+      } catch {
+        throw new InstanceServiceError(
+          'COMPENSATION_PERSISTENCE_FAILED',
+          'Credential binding was invalid and cleanup could not be confirmed',
+        );
+      }
+      throw new InstanceServiceError('SECRET_STORE_FAILED', 'Could not store instance credential');
+    }
+    return { id: this.id(), external };
+  }
+  private async compensate(instanceId: string, reference: string): Promise<void> {
+    try {
+      this.validateCleanupReference(instanceId, reference);
+      await this.store.delete(reference);
+    } catch {
+      try {
+        this.queueCleanup(instanceId, reference, this.clock().toISOString());
+      } catch {
+        throw new InstanceServiceError(
+          'COMPENSATION_PERSISTENCE_FAILED',
+          'Database operation failed and credential cleanup could not be persisted',
+        );
+      }
+      throw new InstanceServiceError(
+        'COMPENSATION_FAILED',
+        'Database operation failed and new credential cleanup failed',
+      );
+    }
+  }
+  private async tryCleanup(instanceId: string, reference: string): Promise<void> {
+    const generation = this.cleanupGeneration(reference);
+    try {
+      this.validateCleanupReference(instanceId, reference);
+      await this.store.delete(reference);
+      if (generation !== undefined) this.removeCleanup(reference, generation);
+    } catch {
+      // The durable outbox is authoritative after the business transaction commits.
+    }
+  }
+  private cleanupEntries(): CleanupEntry[] {
+    return this.db
+      .prepare(
+        'SELECT reference,instance_id,purpose,provider,generation,queued_at FROM secret_cleanup_tasks ORDER BY queued_at ASC,reference ASC',
+      )
+      .all()
+      .map((row) => {
+        const task = row as {
+          reference: string;
+          instance_id: string;
+          purpose: string;
+          provider: string;
+          generation: number;
+          queued_at: string;
+        };
+        return {
+          reference: task.reference,
+          instanceId: task.instance_id,
+          purpose: task.purpose,
+          provider: task.provider,
+          generation: task.generation,
+          queuedAt: task.queued_at,
+        };
+      });
+  }
+  private queueCleanup(instanceId: string, reference: string, now: string): void {
+    this.validateCleanupReference(instanceId, reference);
+    const queued = this.db
+      .prepare(
+        `INSERT INTO secret_cleanup_tasks (reference,instance_id,purpose,provider,generation,queued_at,updated_at) VALUES (?,?,?,?,1,?,?)
+        ON CONFLICT(reference) DO UPDATE SET
+          generation=secret_cleanup_tasks.generation+1,
+          updated_at=excluded.updated_at
+        WHERE secret_cleanup_tasks.instance_id=excluded.instance_id
+          AND secret_cleanup_tasks.purpose=excluded.purpose
+          AND secret_cleanup_tasks.provider=excluded.provider`,
+      )
+      .run(reference, instanceId, PURPOSE, PROVIDER, now, now);
+    if (queued.changes !== 1)
+      throw new InstanceServiceError('DATABASE_FAILED', 'Cleanup task could not be persisted');
+  }
+  private cleanupGeneration(reference: string): number | undefined {
+    const row = this.db
+      .prepare('SELECT generation FROM secret_cleanup_tasks WHERE reference=?')
+      .get(reference) as { generation: number } | undefined;
+    return row?.generation;
+  }
+  private removeCleanup(reference: string, generation: number): boolean {
+    return (
+      this.db
+        .prepare('DELETE FROM secret_cleanup_tasks WHERE reference=? AND generation=?')
+        .run(reference, generation).changes === 1
+    );
+  }
+  private isActiveReference(reference: string): boolean {
+    return Boolean(
+      this.db
+        .prepare('SELECT 1 FROM secret_references WHERE external_reference=? LIMIT 1')
+        .get(reference),
+    );
+  }
+  private validateCleanupEntry(entry: CleanupEntry): void {
+    if (entry.purpose !== PURPOSE || entry.provider !== PROVIDER)
+      throw new InstanceServiceError('DATABASE_FAILED', 'Cleanup task is invalid');
+    this.validateCleanupReference(entry.instanceId, entry.reference);
+  }
+  private validateCleanupReference(instanceId: string, reference: string): void {
+    try {
+      const parsed = parseKeychainReference(reference);
+      if (
+        parsed.instanceId !== instanceId ||
+        parsed.purpose !== PURPOSE ||
+        parsed.slot === undefined
+      )
+        throw new Error('mismatch');
+    } catch {
+      throw new InstanceServiceError('DATABASE_FAILED', 'Cleanup task is invalid');
+    }
+  }
+  private loadRows(where: string, parameters: unknown[]): AggregateRow[] {
+    return this.db
+      .prepare(
+        `SELECT i.id,i.name,i.base_url,i.config_revision,i.updated_at,
+      group_concat(t.tag, char(31)) tags,
+      CASE WHEN EXISTS(SELECT 1 FROM secret_references r WHERE r.instance_id=i.id AND r.purpose=?) THEN 1 ELSE 0 END credential
+      FROM instances i LEFT JOIN instance_tags t ON t.instance_id=i.id ${where} GROUP BY i.id`,
+      )
+      .all(PURPOSE, ...parameters) as AggregateRow[];
+  }
+  private toInstance(row: AggregateRow): Instance {
+    const capabilityRows = this.db
+      .prepare('SELECT state FROM capabilities WHERE instance_id=?')
+      .all(row.id) as { state: CapabilityStatus }[];
+    const capabilityStatus =
+      capabilityRows.length === 0
+        ? 'unknown'
+        : capabilityRows
+            .map((x) => x.state)
+            .sort((a, b) => capabilityRank[a]! - capabilityRank[b]!)[0]!;
+    const snapshots = this.db
+      .prepare('SELECT state,expires_at FROM status_snapshots WHERE instance_id=?')
+      .all(row.id) as { state: SnapshotFreshness; expires_at: string | null }[];
+    const freshness: SnapshotFreshness =
+      snapshots.length === 0
+        ? 'unknown'
+        : snapshots
+            .map((snapshot) =>
+              snapshot.expires_at && snapshot.expires_at <= this.clock().toISOString()
+                ? 'expired'
+                : snapshot.state,
+            )
+            .sort((left, right) => freshnessRank[left]! - freshnessRank[right]!)[0]!;
+    return {
+      id: row.id,
+      name: row.name,
+      origin: row.base_url,
+      tags: row.tags ? row.tags.split(String.fromCharCode(31)).sort(codePointCompare) : [],
+      revision: row.config_revision,
+      capabilityStatus,
+      freshness,
+      credentialConfigured: row.credential === 1,
+    };
+  }
+}
+
+function integerInRange(value: number, minimum: number, maximum: number, name: string): number {
+  if (!Number.isSafeInteger(value) || value < minimum || value > maximum)
+    validation(`${name} is invalid`);
+  return value;
+}
+function codePointCompare(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+function isUnique(error: unknown, fragment: string): boolean {
+  return (
+    error instanceof Error &&
+    error.message.includes('UNIQUE constraint failed') &&
+    error.message.includes(fragment)
+  );
+}