import { createHash, randomUUID } from 'node:crypto'; import type { AiCompletionClient } from '../ai/litellm-client'; import { LiteLlmClientError } from '../ai/litellm-client'; import type { AssessorRepository } from '../../src/persistence/assessor-repository'; import { RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors'; import { REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, REGULATORY_RETRIEVAL_VERSION, groundRegulatoryMappingResult, regulatoryMappingResultValidationErrors, type RegulatoryMappingCandidate, type RegulatoryMappingCreateRequest, type RegulatoryMappingDecision, type RegulatoryMappingDecisionCreateRequest, type RegulatoryMappingEvidence, type RegulatoryMappingResponse, type RegulatoryMappingResult, type RegulatoryMappingRun } from '../../src/domain/regulatory-mapping'; import type { RegulatoryRetrievalCandidate } from '../../src/domain/regulatory-rag'; import type { UnifiedResultProjection } from '../../src/domain/unified-result'; import { privacySafeFinding } from '../../src/domain/privacy-safe-finding'; import type { RegulatoryFindingRetrievalService } from './regulatory-finding-retrieval'; import type { RegulatoryChunkIndex } from './qdrant-regulatory-index'; import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query'; const POLL_AFTER_MS = 1_500; const MAXIMUM_PROMPT_CHARACTERS = 48_000; export interface RegulatoryMappingServiceOptions { findingRetrieval?: RegulatoryFindingRetrievalService; index?: RegulatoryChunkIndex; client?: AiCompletionClient; corpusId?: string; corpusVersion?: string; embeddingModel?: string; disabledReason?: string; now?: () => Date; } export class RegulatoryMappingService { private readonly activeTasks = new Set>(); private readonly now: () => Date; constructor(private readonly repository: AssessorRepository, private readonly options: RegulatoryMappingServiceOptions = {}) { this.now = options.now ?? (() => new Date()); } get generationEnabled(): boolean { return Boolean(this.options.findingRetrieval && this.options.index && this.options.client && this.options.corpusId && this.options.corpusVersion && this.options.embeddingModel); } async recoverInterruptedRuns(): Promise { const completedAt = this.now().toISOString(); for (const run of (await this.repository.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) { await this.repository.saveRegulatoryMappingRun({ ...run, status: 'failed', updatedAt: completedAt, completedAt, errorCode: 'regulatory_mapping_interrupted' }); } } async get(projectId: string, resultProjectionId: string): Promise { const finding = await this.finding(projectId, resultProjectionId); const supported = finding.sourceKind === 'scanner-finding' && Boolean(buildRegulatoryFindingQuery(finding)); const availability = supported ? await this.availability() : { enabled: false }; const enabled = availability.enabled && supported; const ruleKey = supported ? this.ruleKey(finding) : undefined; const run = enabled && ruleKey ? await this.repository.getRegulatoryMappingRun(this.mappingId(ruleKey)) : null; const decisions = ruleKey ? await this.repository.listRegulatoryMappingDecisions(ruleKey) : []; return { version: 1, projectId, resultProjectionId, generationEnabled: enabled, state: run?.status ?? (enabled ? 'empty' : 'unsupported'), ...(!supported ? { disabledReason: finding.sourceKind !== 'scanner-finding' ? 'Only canonical scanner findings can be mapped in this version.' : 'This finding has insufficient public context for regulatory mapping.' } : !availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}), ...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}), ...(run ? { run: withoutEvidenceText(run) } : {}), decisions }; } async start(projectId: string, request: RegulatoryMappingCreateRequest): Promise { if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503); const finding = await this.finding(projectId, request.resultProjectionId); if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422); const ruleKey = this.ruleKey(finding); const id = this.mappingId(ruleKey); const existing = await this.repository.getRegulatoryMappingRun(id); if (existing && ['queued', 'running'].includes(existing.status)) return this.get(projectId, request.resultProjectionId); let retrieval; try { retrieval = await this.options.findingRetrieval!.retrieve(finding, 32, { corpusId: this.options.corpusId, corpusVersion: this.options.corpusVersion, sourceStandard: 'ISO/IEC 27002', retrievalRole: 'semantic' }); } catch { throw new RegulatoryMappingServiceError('regulatory_retrieval_unavailable', 'Regulatory retrieval is currently unavailable.', 503); } if (retrieval.status !== 'candidates' || !retrieval.preparedQuery) { throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422); } const candidates = await this.groupCandidates(retrieval.candidates); if (!candidates.length) throw new RegulatoryMappingServiceError('regulatory_corpus_empty', 'The active regulatory corpus returned no eligible controls.', 503); const timestamp = this.now().toISOString(); const queryHash = createHash('sha256').update(JSON.stringify(retrieval.preparedQuery), 'utf8').digest('hex'); const proposed: RegulatoryMappingRun = { version: 1, id, ruleKey, sourceTool: finding.sourceTool ?? 'unknown', ...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}), status: 'queued', queryHash, preparedQuery: retrieval.preparedQuery, corpusId: this.options.corpusId!, corpusVersion: this.options.corpusVersion!, sourceHashes: [...new Set(candidates.flatMap(candidate => candidate.evidence.map(evidence => evidence.sourceSha256)))].sort(), embeddingModel: this.options.embeddingModel!, retrievalVersion: REGULATORY_RETRIEVAL_VERSION, promptVersion: REGULATORY_MAPPING_PROMPT_VERSION, outputSchemaVersion: REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION, model: this.options.client!.model, createdAt: timestamp, updatedAt: timestamp, candidates }; await this.repository.saveRegulatoryMappingRun(proposed); const task = this.process(proposed).finally(() => this.activeTasks.delete(task)); this.activeTasks.add(task); return this.get(projectId, request.resultProjectionId); } async decide(projectId: string, request: RegulatoryMappingDecisionCreateRequest): Promise { if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503); const finding = await this.finding(projectId, request.resultProjectionId); if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422); const ruleKey = this.ruleKey(finding); const comment = request.comment?.trim(); const decision: RegulatoryMappingDecision = { version: 1, id: `regulatory-decision-${randomUUID()}`, ruleKey, sourceTool: finding.sourceTool ?? 'unknown', ...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}), mappingId: this.mappingId(ruleKey), controlId: request.controlId, decision: request.decision, ...(comment ? { comment } : {}), createdAt: this.now().toISOString() }; await this.repository.createRegulatoryMappingDecision(decision, request.expectedLatestDecisionId); return this.get(projectId, request.resultProjectionId); } // The global sharing key: the rule identity `tool:ruleId`, shared across every tenant. // Rules without an id fall back to a stable hash of the non-personal check title so // identical findings still share, while never keying on tenant or occurrence. private ruleKey(finding: UnifiedResultProjection): string { const tool = finding.sourceTool?.trim() || 'unknown'; const ruleId = finding.sourceRuleId?.trim(); if (ruleId) return `${tool}:${ruleId}`.slice(0, 200); const hash = createHash('sha256').update(privacySafeFinding(finding).title.trim(), 'utf8').digest('hex').slice(0, 24); return `${tool}#title:${hash}`.slice(0, 200); } // Deterministic id per (ruleKey, pipeline versions) so one row exists per rule per // version tuple: regeneration reuses it, and a version bump yields a new mapping. private mappingId(ruleKey: string): string { const tuple = [ruleKey, this.options.corpusVersion, REGULATORY_RETRIEVAL_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, this.options.client!.model].join(''); return `regulatory-mapping-${createHash('sha256').update(tuple, 'utf8').digest('hex').slice(0, 40)}`; } async close(): Promise { await Promise.allSettled([...this.activeTasks]); } private async availability(): Promise<{ enabled: boolean; reason?: string }> { if (!this.generationEnabled) return { enabled: false, reason: this.options.disabledReason ?? 'Regulatory mapping is not configured.' }; try { const sample = await this.options.index!.list({ corpusId: this.options.corpusId, corpusVersion: this.options.corpusVersion, sourceStandard: 'ISO/IEC 27002', retrievalRole: 'semantic' }, 1); return sample.length ? { enabled: true } : { enabled: false, reason: 'The active regulatory corpus is not indexed.' }; } catch { return { enabled: false, reason: 'Qdrant or the active regulatory corpus is unavailable.' }; } } private async finding(projectId: string, resultProjectionId: string) { if (!resultProjectionId?.trim()) throw new RepositoryValidationError('A resultProjectionId is required.'); const finding = await this.repository.getUnifiedResultById(resultProjectionId); if (!finding || finding.projectId !== projectId) throw new RepositoryNotFoundError('The selected finding does not exist in this project.'); return finding; } private async groupCandidates(hits: RegulatoryRetrievalCandidate[]): Promise { const byControl = new Map(); for (const hit of hits) { if (!hit.chunk.annexAControlId || !hit.chunk.sectionKind || !hit.chunk.sourcePageStart || !hit.chunk.sourcePageEnd || !hit.chunk.sourceSha256) continue; const values = byControl.get(hit.chunk.controlId) ?? []; values.push(hit); byControl.set(hit.chunk.controlId, values); } const selected = [...byControl.entries()] .map(([controlId, values]) => ({ controlId, values: values.sort((left, right) => right.score - left.score), score: Math.max(...values.map(value => value.score)) })) .sort((left, right) => right.score - left.score || left.controlId.localeCompare(right.controlId, 'en', { numeric: true })) .slice(0, 8); return Promise.all(selected.map(async selection => { const semantic = selection.values.slice(0, 2).map(value => evidence(value.chunk)); const references = await this.options.index!.list({ corpusId: this.options.corpusId, corpusVersion: this.options.corpusVersion, controlId: selection.controlId, sourceStandard: 'ISO/IEC 27001', retrievalRole: 'reference' }, 1); if (references.length !== 1 || !references[0].sourceSha256) { throw new RegulatoryMappingServiceError('regulatory_reference_missing', `The active corpus is missing the ISO 27001 reference for ${selection.controlId}.`, 503); } const first = selection.values[0].chunk; return { controlId: selection.controlId, annexAControlId: first.annexAControlId!, title: first.title, score: selection.score, evidence: [...semantic, ...references.map(evidence)] }; })); } private async process(queued: RegulatoryMappingRun): Promise { const startedAt = this.now().toISOString(); const run = await this.repository.saveRegulatoryMappingRun({ ...queued, status: 'running', startedAt, updatedAt: startedAt }); try { const prompt = mappingPrompt(run); const completion = await this.options.client!.complete(SYSTEM_PROMPT, prompt); const allowed = new Map(run.candidates.map(candidate => [ candidate.controlId, new Set(candidate.evidence.map(item => item.chunkId)) ])); const grounded = groundRegulatoryMappingResult(completion.value, allowed); const errors = regulatoryMappingResultValidationErrors(grounded, allowed); if (errors.length) throw new RegulatoryMappingServiceError('regulatory_mapping_invalid_output', errors.join(' '), 502); const completedAt = this.now().toISOString(); await this.repository.saveRegulatoryMappingRun({ ...run, status: 'succeeded', result: grounded as RegulatoryMappingResult, ...(completion.usage ? { tokenUsage: completion.usage } : {}), updatedAt: completedAt, completedAt }); } catch (error) { const completedAt = this.now().toISOString(); const failure = safeFailure(error); const detail = safeFailureDetail(error); try { await this.repository.saveRegulatoryMappingRun({ ...run, status: 'failed', updatedAt: completedAt, completedAt, errorCode: failure, ...(detail ? { errorDetail: detail } : {}) }); } catch { /* Preserve the original safe failure code. */ } } } } export class RegulatoryMappingServiceError extends Error { constructor(readonly code: string, message: string, readonly status: number) { super(message); this.name = 'RegulatoryMappingServiceError'; } } // Response projection: the persisted run keeps the full ISO clause text (the model reads it), // but the copyrighted text is never shipped to the browser — only the citation fields are. function withoutEvidenceText(run: RegulatoryMappingRun): RegulatoryMappingRun { return { ...run, candidates: run.candidates.map(candidate => ({ ...candidate, evidence: candidate.evidence.map(({ text, ...reference }) => reference) })) }; } function evidence(chunk: RegulatoryRetrievalCandidate['chunk']): RegulatoryMappingEvidence { return { chunkId: chunk.chunkId, textSha256: chunk.textSha256, sourceSha256: chunk.sourceSha256!, sourceStandard: chunk.sourceStandard!, sectionKind: chunk.sectionKind!, pageStart: chunk.sourcePageStart!, pageEnd: chunk.sourcePageEnd!, text: chunk.text }; } function mappingPrompt(run: RegulatoryMappingRun): string { // The Annex A id is always "A." + controlId, so it is redundant for the model and // sending it invites a reply that uses the Annex A form; expose only the controlId it must return. const candidates = run.candidates.map(({ annexAControlId, ...candidate }) => candidate); const payload = JSON.stringify({ finding: run.preparedQuery, candidates }); if (payload.length > MAXIMUM_PROMPT_CHARACTERS) throw new RegulatoryMappingServiceError('regulatory_mapping_prompt_too_large', 'The bounded regulatory prompt exceeded its limit.', 422); return payload; } const SYSTEM_PROMPT = `Map one public security finding query to the supplied ISO control candidates. A mapping means the control is topically relevant to the finding, never proof of compliance. Map a candidate when it governs the same security subject as the finding (for example password or authentication management, logging, or access restriction), even when the control text does not state the finding's specific value, threshold, parameter, or configuration; ISO controls are principle-level and rarely restate such specifics. Judge relevance from the control's subject and its supplied evidence, not from whether the exact requirement text appears. Return JSON only with schemaVersion 1, outcome mapped|no_relevant_control|insufficient_context, optional bounded reason, and mappings (maximum five). Each mapping requires controlId, relationship primary|supporting, confidence low|medium|high, a concise rationale under 600 characters, and one to three evidenceChunkIds copied only from that control's supplied evidence. Use no_relevant_control only when no candidate governs the finding's subject, and insufficient_context only when the finding query itself lacks enough meaning to judge. Do not invent controls or evidence.`; function safeFailure(error: unknown): string { if (error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError) return error.code; return 'regulatory_mapping_failed'; } // Our own service and client error messages are developer-authored and structural // (field paths, ISO control numbers) — never finding, tenant, or report content. // Clamp the length and strip any markup-like sequence before it is stored or shown. function safeFailureDetail(error: unknown): string | undefined { if (!(error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError)) return undefined; const detail = error.message.replace(/<\/?[a-z][^>]*>/gi, ' ').replace(/\s+/g, ' ').trim().slice(0, 600); return detail || undefined; }