578 lines
30 KiB
TypeScript
578 lines
30 KiB
TypeScript
import { createHash, randomUUID } from 'node:crypto';
|
||
import type { AiCompletionClient } from '../ai/litellm-client';
|
||
import { LiteLlmClientError } from '../ai/litellm-client';
|
||
import type { AssessorRepository } from '../../src/persistence/assessor-repository';
|
||
import { RepositoryConflictError, RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors';
|
||
import {
|
||
REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION,
|
||
REGULATORY_MAPPING_PROMPT_VERSION,
|
||
REGULATORY_RETRIEVAL_VERSION,
|
||
emptyRegulatoryMappingBatchCounts,
|
||
groundRegulatoryCandidateAssessmentResult,
|
||
normalizeRegulatoryMappingBatchScope,
|
||
regulatoryCandidateAssessmentValidationErrors,
|
||
regulatoryMappingResultValidationErrors,
|
||
selectRegulatoryMappingsFromAssessments,
|
||
type RegulatoryCandidateAssessmentResult,
|
||
type RegulatoryMappingBatchCreateRequest,
|
||
type RegulatoryMappingBatchResponse,
|
||
type RegulatoryMappingBatchRun,
|
||
type RegulatoryMappingBatchScope,
|
||
type RegulatoryMappingCandidate,
|
||
type RegulatoryMappingCreateRequest,
|
||
type RegulatoryMappingDecision,
|
||
type RegulatoryMappingDecisionCreateRequest,
|
||
type RegulatoryMappingEvidence,
|
||
type RegulatoryMappingResponse,
|
||
type RegulatoryMappingResult,
|
||
type RegulatoryMappingRun
|
||
} from '../../src/domain/regulatory-mapping';
|
||
import type { RegulatoryRetrievalCandidate } from '../../src/domain/regulatory-rag';
|
||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||
import { privacySafeFinding } from '../../src/domain/privacy-safe-finding';
|
||
import type { RegulatoryFindingRetrievalService } from './regulatory-finding-retrieval';
|
||
import type { RegulatoryChunkIndex } from './qdrant-regulatory-index';
|
||
import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query';
|
||
import { canonicalJson } from '../../src/shared/canonical-json';
|
||
|
||
const POLL_AFTER_MS = 1_500;
|
||
const MAXIMUM_PROMPT_CHARACTERS = 48_000;
|
||
const BATCH_PAGE_SIZE = 500;
|
||
const DEFAULT_MAXIMUM_CONCURRENT_RUNS = 2;
|
||
const MAPPING_CONTROL_SHORTLIST_LIMIT = 12;
|
||
|
||
export interface RegulatoryMappingServiceOptions {
|
||
findingRetrieval?: RegulatoryFindingRetrievalService;
|
||
index?: RegulatoryChunkIndex;
|
||
client?: AiCompletionClient;
|
||
corpusId?: string;
|
||
corpusVersion?: string;
|
||
embeddingModel?: string;
|
||
disabledReason?: string;
|
||
now?: () => Date;
|
||
maximumConcurrentRuns?: number;
|
||
}
|
||
|
||
export class RegulatoryMappingService {
|
||
private readonly activeTasks = new Set<Promise<void>>();
|
||
private readonly now: () => Date;
|
||
private readonly maximumConcurrentRuns: number;
|
||
private readonly workQueue: Array<{
|
||
task: () => Promise<'succeeded' | 'failed'>;
|
||
resolve: (status: 'succeeded' | 'failed') => void;
|
||
}> = [];
|
||
private activeWorkerCount = 0;
|
||
private activationPending = false;
|
||
|
||
constructor(private readonly repository: AssessorRepository, private readonly options: RegulatoryMappingServiceOptions = {}) {
|
||
this.now = options.now ?? (() => new Date());
|
||
this.maximumConcurrentRuns = options.maximumConcurrentRuns ?? DEFAULT_MAXIMUM_CONCURRENT_RUNS;
|
||
if (!Number.isInteger(this.maximumConcurrentRuns) || this.maximumConcurrentRuns < 1 || this.maximumConcurrentRuns > 8) {
|
||
throw new Error('Regulatory mapping concurrency must be an integer between 1 and 8.');
|
||
}
|
||
}
|
||
|
||
get generationEnabled(): boolean {
|
||
return !this.activationPending && Boolean(this.options.findingRetrieval && this.options.index && this.options.client
|
||
&& this.options.corpusId && this.options.corpusVersion && this.options.embeddingModel);
|
||
}
|
||
|
||
async activateCorpus(corpusId: string, corpusVersion: string): Promise<void> {
|
||
this.activationPending = true;
|
||
try {
|
||
await this.close();
|
||
this.options.corpusId = corpusId;
|
||
this.options.corpusVersion = corpusVersion;
|
||
} finally {
|
||
this.activationPending = false;
|
||
}
|
||
}
|
||
|
||
async recoverInterruptedRuns(): Promise<void> {
|
||
const completedAt = this.now().toISOString();
|
||
for (const run of (await this.repository.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) {
|
||
await this.repository.saveRegulatoryMappingRun({
|
||
...run,
|
||
status: 'failed',
|
||
updatedAt: completedAt,
|
||
completedAt,
|
||
errorCode: 'regulatory_mapping_interrupted'
|
||
});
|
||
}
|
||
for (const project of await this.repository.listProjects()) {
|
||
for (const run of (await this.repository.listRegulatoryMappingBatchRuns(project.id)).filter(item => ['queued', 'running'].includes(item.status))) {
|
||
await this.repository.updateRegulatoryMappingBatchRun({
|
||
...run,
|
||
status: 'failed',
|
||
updatedAt: completedAt,
|
||
completedAt,
|
||
errorCode: 'regulatory_mapping_batch_interrupted',
|
||
errorDetail: 'Batch processing was interrupted before completion. Start a new batch to retry remaining rules.'
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
async get(projectId: string, resultProjectionId: string): Promise<RegulatoryMappingResponse> {
|
||
const finding = await this.finding(projectId, resultProjectionId);
|
||
const supported = finding.sourceKind === 'scanner-finding' && Boolean(buildRegulatoryFindingQuery(finding));
|
||
const availability = supported ? await this.availability() : { enabled: false };
|
||
const enabled = availability.enabled && supported;
|
||
const ruleKey = supported ? this.ruleKey(finding) : undefined;
|
||
const run = ruleKey && this.options.client ? await this.repository.getRegulatoryMappingRun(this.mappingId(ruleKey)) : null;
|
||
const decisions = ruleKey ? await this.repository.listRegulatoryMappingDecisions(ruleKey) : [];
|
||
return {
|
||
version: 1,
|
||
projectId,
|
||
resultProjectionId,
|
||
generationEnabled: enabled,
|
||
state: run?.status ?? (enabled ? 'empty' : 'unsupported'),
|
||
...(!supported ? { disabledReason: finding.sourceKind !== 'scanner-finding'
|
||
? 'Only canonical scanner findings can be mapped in this version.'
|
||
: 'This finding has insufficient public context for regulatory mapping.' }
|
||
: !availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}),
|
||
...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}),
|
||
...(run ? { run: withoutEvidenceText(run) } : {}),
|
||
decisions
|
||
};
|
||
}
|
||
|
||
async start(projectId: string, request: RegulatoryMappingCreateRequest): Promise<RegulatoryMappingResponse> {
|
||
if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503);
|
||
const finding = await this.finding(projectId, request.resultProjectionId);
|
||
if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422);
|
||
if (!buildRegulatoryFindingQuery(finding)) throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422);
|
||
const claimed = await this.claim(finding, true);
|
||
if (claimed.claimed) this.track(this.enqueue(() => this.process(claimed.run, finding)).then(() => undefined));
|
||
return this.get(projectId, request.resultProjectionId);
|
||
}
|
||
|
||
async getBatch(projectId: string, requestedScope: RegulatoryMappingBatchScope): Promise<RegulatoryMappingBatchResponse> {
|
||
const scope = await this.validateBatchScope(projectId, requestedScope);
|
||
const scopeKey = this.batchScopeKey(projectId, scope);
|
||
const run = (await this.repository.listRegulatoryMappingBatchRuns(projectId, scopeKey))[0];
|
||
const availability = await this.availability();
|
||
return {
|
||
version: 1,
|
||
projectId,
|
||
scope,
|
||
generationEnabled: availability.enabled,
|
||
state: run?.status ?? 'empty',
|
||
...(!availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}),
|
||
...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}),
|
||
...(run ? { run } : {})
|
||
};
|
||
}
|
||
|
||
async startBatch(projectId: string, request: RegulatoryMappingBatchCreateRequest): Promise<RegulatoryMappingBatchResponse> {
|
||
if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503);
|
||
if (typeof request.idempotencyKey !== 'string' || !/^[A-Za-z0-9._:-]{8,128}$/.test(request.idempotencyKey)) {
|
||
throw new RepositoryValidationError('idempotencyKey must contain 8 to 128 safe characters.');
|
||
}
|
||
const scope = await this.validateBatchScope(projectId, request);
|
||
const timestamp = this.now().toISOString();
|
||
const candidate: RegulatoryMappingBatchRun = {
|
||
version: 1,
|
||
id: `regulatory-mapping-batch-${randomUUID()}`,
|
||
projectId,
|
||
scopeKey: this.batchScopeKey(projectId, scope),
|
||
filterHash: sha256(canonicalJson(scope.filters)),
|
||
assessmentIds: scope.assessmentIds,
|
||
scanRunIds: scope.scanRunIds,
|
||
filters: scope.filters,
|
||
idempotencyKey: request.idempotencyKey,
|
||
status: 'queued',
|
||
counts: emptyRegulatoryMappingBatchCounts(),
|
||
createdAt: timestamp,
|
||
updatedAt: timestamp
|
||
};
|
||
const run = await this.repository.createRegulatoryMappingBatchRun(candidate);
|
||
if (run.id === candidate.id) this.track(this.processBatch(run));
|
||
return this.getBatch(projectId, {
|
||
assessmentIds: run.assessmentIds,
|
||
scanRunIds: run.scanRunIds,
|
||
filters: run.filters
|
||
});
|
||
}
|
||
|
||
async decide(projectId: string, request: RegulatoryMappingDecisionCreateRequest): Promise<RegulatoryMappingResponse> {
|
||
if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503);
|
||
const finding = await this.finding(projectId, request.resultProjectionId);
|
||
if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422);
|
||
const ruleKey = this.ruleKey(finding);
|
||
const comment = request.comment?.trim();
|
||
const decision: RegulatoryMappingDecision = {
|
||
version: 1,
|
||
id: `regulatory-decision-${randomUUID()}`,
|
||
ruleKey,
|
||
sourceTool: finding.sourceTool ?? 'unknown',
|
||
...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}),
|
||
mappingId: this.mappingId(ruleKey),
|
||
controlId: request.controlId,
|
||
decision: request.decision,
|
||
...(comment ? { comment } : {}),
|
||
createdAt: this.now().toISOString()
|
||
};
|
||
await this.repository.createRegulatoryMappingDecision(decision, request.expectedLatestDecisionId);
|
||
return this.get(projectId, request.resultProjectionId);
|
||
}
|
||
|
||
// The global sharing key: the rule identity `tool:ruleId`, shared across every tenant.
|
||
// Rules without an id fall back to a stable hash of the non-personal check title so
|
||
// identical findings still share, while never keying on tenant or occurrence.
|
||
private ruleKey(finding: UnifiedResultProjection): string {
|
||
const tool = finding.sourceTool?.trim() || 'unknown';
|
||
const ruleId = finding.sourceRuleId?.trim();
|
||
if (ruleId) return `${tool}:${ruleId}`.slice(0, 200);
|
||
const hash = createHash('sha256').update(privacySafeFinding(finding).title.trim(), 'utf8').digest('hex').slice(0, 24);
|
||
return `${tool}#title:${hash}`.slice(0, 200);
|
||
}
|
||
|
||
// Deterministic id per (ruleKey, pipeline versions) so one row exists per rule per
|
||
// version tuple: regeneration reuses it, and a version bump yields a new mapping.
|
||
private mappingId(ruleKey: string): string {
|
||
const tuple = [ruleKey, this.options.corpusVersion, REGULATORY_RETRIEVAL_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, this.options.client!.model].join(' ');
|
||
return `regulatory-mapping-${createHash('sha256').update(tuple, 'utf8').digest('hex').slice(0, 40)}`;
|
||
}
|
||
|
||
async close(): Promise<void> {
|
||
while (this.activeTasks.size) await Promise.allSettled([...this.activeTasks]);
|
||
}
|
||
|
||
private track(task: Promise<void>): void {
|
||
this.activeTasks.add(task);
|
||
void task.finally(() => this.activeTasks.delete(task));
|
||
}
|
||
|
||
private enqueue(task: () => Promise<'succeeded' | 'failed'>): Promise<'succeeded' | 'failed'> {
|
||
const promise = new Promise<'succeeded' | 'failed'>(resolve => this.workQueue.push({ task, resolve }));
|
||
this.drainQueue();
|
||
return promise;
|
||
}
|
||
|
||
private drainQueue(): void {
|
||
while (this.activeWorkerCount < this.maximumConcurrentRuns && this.workQueue.length) {
|
||
const work = this.workQueue.shift()!;
|
||
this.activeWorkerCount += 1;
|
||
void work.task().then(work.resolve, () => work.resolve('failed')).finally(() => {
|
||
this.activeWorkerCount -= 1;
|
||
this.drainQueue();
|
||
});
|
||
}
|
||
}
|
||
|
||
private queuedRun(finding: UnifiedResultProjection): RegulatoryMappingRun {
|
||
const preparedQuery = buildRegulatoryFindingQuery(finding);
|
||
if (!preparedQuery) throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422);
|
||
const timestamp = this.now().toISOString();
|
||
const ruleKey = this.ruleKey(finding);
|
||
return {
|
||
version: 1,
|
||
id: this.mappingId(ruleKey),
|
||
ruleKey,
|
||
sourceTool: finding.sourceTool ?? 'unknown',
|
||
...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}),
|
||
status: 'queued',
|
||
queryHash: sha256(JSON.stringify(preparedQuery)),
|
||
preparedQuery,
|
||
corpusId: this.options.corpusId!,
|
||
corpusVersion: this.options.corpusVersion!,
|
||
sourceHashes: [],
|
||
embeddingModel: this.options.embeddingModel!,
|
||
retrievalVersion: REGULATORY_RETRIEVAL_VERSION,
|
||
promptVersion: REGULATORY_MAPPING_PROMPT_VERSION,
|
||
outputSchemaVersion: REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION,
|
||
model: this.options.client!.model,
|
||
createdAt: timestamp,
|
||
updatedAt: timestamp,
|
||
candidates: []
|
||
};
|
||
}
|
||
|
||
private async claim(finding: UnifiedResultProjection, forceRegeneration: boolean) {
|
||
return this.repository.claimRegulatoryMappingRun(this.queuedRun(finding), forceRegeneration);
|
||
}
|
||
|
||
private async validateBatchScope(projectId: string, requested: RegulatoryMappingBatchScope): Promise<RegulatoryMappingBatchScope> {
|
||
if (!await this.repository.getProject(projectId)) throw new RepositoryNotFoundError('The selected project does not exist.');
|
||
const scope = normalizeRegulatoryMappingBatchScope(requested);
|
||
if (!scope.assessmentIds.length && !scope.scanRunIds.length) throw new RepositoryValidationError('A selected timeslot scope is required.');
|
||
const assessments = await this.repository.listAssessments(projectId);
|
||
const allowedAssessmentIds = new Set(assessments.map(item => item.id));
|
||
if (scope.assessmentIds.some(id => !allowedAssessmentIds.has(id))) throw new RepositoryValidationError('The batch scope contains an assessment outside this project.');
|
||
const scanRuns = await this.repository.listProjectScanRuns(projectId);
|
||
const allowedRuns = new Map(scanRuns.filter(item => !item.supersededByScanRunId).map(item => [item.id, item]));
|
||
if (scope.scanRunIds.some(id => !allowedRuns.has(id))) throw new RepositoryValidationError('The batch scope contains an unavailable scan run.');
|
||
if (scope.assessmentIds.length && scope.scanRunIds.some(id => !scope.assessmentIds.includes(allowedRuns.get(id)!.assessmentId))) {
|
||
throw new RepositoryValidationError('The assessment and scan-run scope do not describe the same timeslot.');
|
||
}
|
||
return scope;
|
||
}
|
||
|
||
private batchScopeKey(projectId: string, scope: RegulatoryMappingBatchScope): string {
|
||
return sha256(canonicalJson({ projectId, ...scope }));
|
||
}
|
||
|
||
private async queryBatchResults(projectId: string, scope: RegulatoryMappingBatchScope): Promise<UnifiedResultProjection[]> {
|
||
const query = {
|
||
...scope.filters,
|
||
audience: 'result-db' as const,
|
||
projectId,
|
||
assessmentIds: scope.assessmentIds,
|
||
scanRunIds: scope.scanRunIds
|
||
};
|
||
const items: UnifiedResultProjection[] = [];
|
||
let expectedTotal: number | undefined;
|
||
for (let offset = 0; ; offset += BATCH_PAGE_SIZE) {
|
||
const page = await this.repository.queryUnifiedResults({ ...query, offset, limit: BATCH_PAGE_SIZE });
|
||
expectedTotal ??= page.total;
|
||
if (page.total !== expectedTotal) throw new RepositoryConflictError('The filtered result set changed while the ISO mapping batch was being prepared.');
|
||
items.push(...page.items);
|
||
if (items.length >= page.total || page.items.length === 0) break;
|
||
}
|
||
if (items.length !== expectedTotal) throw new RepositoryConflictError('The complete filtered result set could not be read consistently.');
|
||
return items;
|
||
}
|
||
|
||
private async processBatch(queued: RegulatoryMappingBatchRun): Promise<void> {
|
||
const startedAt = this.now().toISOString();
|
||
let run = await this.repository.updateRegulatoryMappingBatchRun({ ...queued, status: 'running', startedAt, updatedAt: startedAt });
|
||
try {
|
||
const results = (await this.queryBatchResults(run.projectId, run)).slice()
|
||
.sort((left, right) => left.resultProjectionId.localeCompare(right.resultProjectionId));
|
||
const representatives = new Map<string, UnifiedResultProjection>();
|
||
const counts = { ...emptyRegulatoryMappingBatchCounts(), filteredResultCount: results.length };
|
||
for (const finding of results) {
|
||
if (finding.sourceKind !== 'scanner-finding') { counts.excludedNonScannerCount += 1; continue; }
|
||
if (!buildRegulatoryFindingQuery(finding)) { counts.unsupportedFindingCount += 1; continue; }
|
||
counts.eligibleFindingCount += 1;
|
||
const ruleKey = this.ruleKey(finding);
|
||
if (!representatives.has(ruleKey)) representatives.set(ruleKey, finding);
|
||
}
|
||
counts.distinctRuleCount = representatives.size;
|
||
const scheduled: Array<Promise<'succeeded' | 'failed'>> = [];
|
||
for (const finding of representatives.values()) {
|
||
const claimed = await this.claim(finding, false);
|
||
if (!claimed.claimed) {
|
||
if (claimed.run.status === 'succeeded') counts.reusedSucceededRuleCount += 1;
|
||
else counts.reusedActiveRuleCount += 1;
|
||
continue;
|
||
}
|
||
counts.scheduledRuleCount += 1;
|
||
const completion = this.enqueue(() => this.process(claimed.run, finding));
|
||
this.track(completion.then(() => undefined));
|
||
scheduled.push(completion);
|
||
}
|
||
run = await this.repository.updateRegulatoryMappingBatchRun({ ...run, counts, updatedAt: this.now().toISOString() });
|
||
let progress = Promise.resolve();
|
||
const completions = scheduled.map(completion => completion.then(status => {
|
||
progress = progress.then(async () => {
|
||
if (status === 'succeeded') counts.succeededRuleCount += 1;
|
||
else counts.failedRuleCount += 1;
|
||
run = await this.repository.updateRegulatoryMappingBatchRun({ ...run, counts: { ...counts }, updatedAt: this.now().toISOString() });
|
||
});
|
||
return progress;
|
||
}));
|
||
await Promise.all(completions);
|
||
await progress;
|
||
const completedAt = this.now().toISOString();
|
||
await this.repository.updateRegulatoryMappingBatchRun({ ...run, status: 'completed', counts, updatedAt: completedAt, completedAt });
|
||
} catch (error) {
|
||
const completedAt = this.now().toISOString();
|
||
try {
|
||
await this.repository.updateRegulatoryMappingBatchRun({
|
||
...run,
|
||
status: 'failed',
|
||
updatedAt: completedAt,
|
||
completedAt,
|
||
errorCode: error instanceof RepositoryConflictError ? 'regulatory_mapping_batch_scope_changed' : 'regulatory_mapping_batch_failed',
|
||
errorDetail: error instanceof RepositoryConflictError ? error.message : 'The batch stopped safely before all rules were scheduled.'
|
||
});
|
||
} catch { /* Preserve the original safe batch failure. */ }
|
||
}
|
||
}
|
||
|
||
private async availability(): Promise<{ enabled: boolean; reason?: string }> {
|
||
if (!this.generationEnabled) return { enabled: false, reason: this.options.disabledReason ?? 'Regulatory mapping is not configured.' };
|
||
try {
|
||
const sample = await this.options.index!.list({
|
||
corpusId: this.options.corpusId,
|
||
corpusVersion: this.options.corpusVersion,
|
||
sourceStandard: 'ISO/IEC 27002',
|
||
retrievalRole: 'semantic'
|
||
}, 1);
|
||
return sample.length ? { enabled: true } : { enabled: false, reason: 'The active regulatory corpus is not indexed.' };
|
||
} catch {
|
||
return { enabled: false, reason: 'Qdrant or the active regulatory corpus is unavailable.' };
|
||
}
|
||
}
|
||
|
||
private async finding(projectId: string, resultProjectionId: string) {
|
||
if (!resultProjectionId?.trim()) throw new RepositoryValidationError('A resultProjectionId is required.');
|
||
const finding = await this.repository.getUnifiedResultById(resultProjectionId);
|
||
if (!finding || finding.projectId !== projectId) throw new RepositoryNotFoundError('The selected finding does not exist in this project.');
|
||
return finding;
|
||
}
|
||
|
||
private async groupCandidates(hits: RegulatoryRetrievalCandidate[]): Promise<RegulatoryMappingCandidate[]> {
|
||
const byControl = new Map<string, RegulatoryRetrievalCandidate[]>();
|
||
for (const hit of hits) {
|
||
if (!hit.chunk.annexAControlId || !hit.chunk.sectionKind || !hit.chunk.sourcePageStart || !hit.chunk.sourcePageEnd || !hit.chunk.sourceSha256) continue;
|
||
const values = byControl.get(hit.chunk.controlId) ?? [];
|
||
values.push(hit);
|
||
byControl.set(hit.chunk.controlId, values);
|
||
}
|
||
const selected = [...byControl.entries()]
|
||
.map(([controlId, values]) => ({ controlId, values: values.sort((left, right) => right.score - left.score), score: Math.max(...values.map(value => value.score)) }))
|
||
.sort((left, right) => right.score - left.score || left.controlId.localeCompare(right.controlId, 'en', { numeric: true }))
|
||
.slice(0, MAPPING_CONTROL_SHORTLIST_LIMIT);
|
||
return Promise.all(selected.map(async selection => {
|
||
const semantic = selection.values.slice(0, 2).map(value => evidence(value.chunk));
|
||
const references = await this.options.index!.list({
|
||
corpusId: this.options.corpusId,
|
||
corpusVersion: this.options.corpusVersion,
|
||
controlId: selection.controlId,
|
||
sourceStandard: 'ISO/IEC 27001',
|
||
retrievalRole: 'reference'
|
||
}, 1);
|
||
if (references.length !== 1 || !references[0].sourceSha256) {
|
||
throw new RegulatoryMappingServiceError('regulatory_reference_missing', `The active corpus is missing the ISO 27001 reference for ${selection.controlId}.`, 503);
|
||
}
|
||
const first = selection.values[0].chunk;
|
||
return {
|
||
controlId: selection.controlId,
|
||
annexAControlId: first.annexAControlId!,
|
||
title: first.title,
|
||
score: selection.score,
|
||
evidence: [...semantic, ...references.map(evidence)]
|
||
};
|
||
}));
|
||
}
|
||
|
||
private async process(queued: RegulatoryMappingRun, finding: UnifiedResultProjection): Promise<'succeeded' | 'failed'> {
|
||
let run = queued;
|
||
try {
|
||
let retrieval;
|
||
try {
|
||
retrieval = await this.options.findingRetrieval!.retrieve(finding, MAPPING_CONTROL_SHORTLIST_LIMIT, {
|
||
corpusId: this.options.corpusId,
|
||
corpusVersion: this.options.corpusVersion,
|
||
sourceStandard: 'ISO/IEC 27002',
|
||
retrievalRole: 'semantic'
|
||
});
|
||
} catch {
|
||
throw new RegulatoryMappingServiceError('regulatory_retrieval_unavailable', 'Regulatory retrieval is currently unavailable.', 503);
|
||
}
|
||
if (retrieval.status !== 'candidates' || !retrieval.preparedQuery) {
|
||
throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422);
|
||
}
|
||
const candidates = await this.groupCandidates(retrieval.candidates);
|
||
if (!candidates.length) throw new RegulatoryMappingServiceError('regulatory_corpus_empty', 'The active regulatory corpus returned no eligible controls.', 503);
|
||
const startedAt = this.now().toISOString();
|
||
run = await this.repository.saveRegulatoryMappingRun({
|
||
...queued,
|
||
status: 'running',
|
||
preparedQuery: retrieval.preparedQuery,
|
||
queryHash: sha256(JSON.stringify(retrieval.preparedQuery)),
|
||
sourceHashes: [...new Set(candidates.flatMap(candidate => candidate.evidence.map(evidence => evidence.sourceSha256)))].sort(),
|
||
candidates,
|
||
startedAt,
|
||
updatedAt: startedAt
|
||
});
|
||
const prompt = mappingPrompt(run);
|
||
const completion = await this.options.client!.complete(SYSTEM_PROMPT, prompt);
|
||
const allowed = new Map(run.candidates.map(candidate => [
|
||
candidate.controlId, new Set(candidate.evidence.map(item => item.chunkId))
|
||
]));
|
||
const grounded = groundRegulatoryCandidateAssessmentResult(completion.value, allowed);
|
||
const errors = regulatoryCandidateAssessmentValidationErrors(grounded, allowed);
|
||
if (errors.length) throw new RegulatoryMappingServiceError('regulatory_mapping_invalid_output', errors.join(' '), 502);
|
||
const selected = selectRegulatoryMappingsFromAssessments(
|
||
grounded as RegulatoryCandidateAssessmentResult,
|
||
run.candidates.map(candidate => candidate.controlId)
|
||
);
|
||
const selectionErrors = regulatoryMappingResultValidationErrors(selected, allowed);
|
||
if (selectionErrors.length) throw new RegulatoryMappingServiceError('regulatory_mapping_invalid_output', selectionErrors.join(' '), 502);
|
||
const completedAt = this.now().toISOString();
|
||
await this.repository.saveRegulatoryMappingRun({
|
||
...run,
|
||
status: 'succeeded',
|
||
result: selected as RegulatoryMappingResult,
|
||
...(completion.usage ? { tokenUsage: completion.usage } : {}),
|
||
updatedAt: completedAt,
|
||
completedAt
|
||
});
|
||
return 'succeeded';
|
||
} catch (error) {
|
||
const completedAt = this.now().toISOString();
|
||
const failure = safeFailure(error);
|
||
const detail = safeFailureDetail(error);
|
||
try {
|
||
await this.repository.saveRegulatoryMappingRun({
|
||
...run, status: 'failed', updatedAt: completedAt, completedAt, errorCode: failure,
|
||
...(detail ? { errorDetail: detail } : {})
|
||
});
|
||
} catch { /* Preserve the original safe failure code. */ }
|
||
return 'failed';
|
||
}
|
||
}
|
||
}
|
||
|
||
export class RegulatoryMappingServiceError extends Error {
|
||
constructor(readonly code: string, message: string, readonly status: number) { super(message); this.name = 'RegulatoryMappingServiceError'; }
|
||
}
|
||
|
||
// Response projection: the persisted run keeps the full ISO clause text (the model reads it),
|
||
// but the copyrighted text is never shipped to the browser — only the citation fields are.
|
||
function withoutEvidenceText(run: RegulatoryMappingRun): RegulatoryMappingRun {
|
||
return {
|
||
...run,
|
||
candidates: run.candidates.map(candidate => ({
|
||
...candidate,
|
||
evidence: candidate.evidence.map(({ text, ...reference }) => reference)
|
||
}))
|
||
};
|
||
}
|
||
|
||
function evidence(chunk: RegulatoryRetrievalCandidate['chunk']): RegulatoryMappingEvidence {
|
||
return {
|
||
chunkId: chunk.chunkId,
|
||
textSha256: chunk.textSha256,
|
||
sourceSha256: chunk.sourceSha256!,
|
||
sourceStandard: chunk.sourceStandard!,
|
||
sectionKind: chunk.sectionKind!,
|
||
pageStart: chunk.sourcePageStart!,
|
||
pageEnd: chunk.sourcePageEnd!,
|
||
text: chunk.text
|
||
};
|
||
}
|
||
|
||
function mappingPrompt(run: RegulatoryMappingRun): string {
|
||
// The Annex A id is always "A." + controlId, so it is redundant for the model and
|
||
// sending it invites a reply that uses the Annex A form; expose only the controlId it must return.
|
||
const candidates = run.candidates.map(({ annexAControlId, ...candidate }) => candidate);
|
||
const payload = JSON.stringify({ finding: run.preparedQuery, candidates });
|
||
if (payload.length > MAXIMUM_PROMPT_CHARACTERS) throw new RegulatoryMappingServiceError('regulatory_mapping_prompt_too_large', 'The bounded regulatory prompt exceeded its limit.', 422);
|
||
return payload;
|
||
}
|
||
|
||
const SYSTEM_PROMPT = `Assess every supplied ISO control candidate independently against one public security finding query. Relevance means the control materially governs, prevents, limits, detects, or supports recovery from the finding; it never proves compliance. Return every supplied candidate exactly once, even when it is irrelevant. Label a candidate primary only when it directly governs the finding's root security subject or remediation. Label it supporting only when it adds a distinct material requirement, prerequisite, detection, restriction, or recovery contribution. Label it irrelevant when it merely shares a technology, protocol, account type, security domain, or broad words such as security, access, monitoring, policy, or configuration. ISO controls are principle-level and rarely repeat a finding's exact product wording, parameter, threshold, or value, so judge the governed subject from the supplied evidence. Assess candidates independently rather than stopping after one or two relevant controls; a deterministic reducer will select zero or one primary and zero to three supporting controls. Return JSON only with schemaVersion 1, outcome assessed|insufficient_context, optional bounded reason, and assessments. For outcome assessed, assessments must contain every supplied candidate exactly once. Each assessment requires controlId, relevance primary|supporting|irrelevant, confidence low|medium|high, a concise rationale under 600 characters, and one to three evidenceChunkIds copied only from that control's supplied evidence. Use insufficient_context only when the finding query itself lacks enough meaning to assess any candidate, and then return no assessments. Do not invent controls or evidence.`;
|
||
|
||
function safeFailure(error: unknown): string {
|
||
if (error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError) return error.code;
|
||
return 'regulatory_mapping_failed';
|
||
}
|
||
|
||
// Our own service and client error messages are developer-authored and structural
|
||
// (field paths, ISO control numbers) — never finding, tenant, or report content.
|
||
// Clamp the length and strip any markup-like sequence before it is stored or shown.
|
||
function safeFailureDetail(error: unknown): string | undefined {
|
||
if (!(error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError)) return undefined;
|
||
const detail = error.message.replace(/<\/?[a-z][^>]*>/gi, ' ').replace(/\s+/g, ' ').trim().slice(0, 600);
|
||
return detail || undefined;
|
||
}
|
||
|
||
function sha256(value: string): string {
|
||
return createHash('sha256').update(value, 'utf8').digest('hex');
|
||
}
|