1197 lines
72 KiB
TypeScript
1197 lines
72 KiB
TypeScript
import { createHash, randomUUID } from 'node:crypto';
|
||
import type { AiCompletionClient } from '../ai/litellm-client';
|
||
import { LiteLlmClientError } from '../ai/litellm-client';
|
||
import type { ComplianceReadPort, ComplianceStore } from '../../src/compliance/compliance-store';
|
||
import { RepositoryConflictError, RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors';
|
||
import {
|
||
REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION,
|
||
REGULATORY_MAPPING_PROMPT_VERSION,
|
||
REGULATORY_RETRIEVAL_VERSION,
|
||
REGULATORY_GPO_QUERY_VERSION,
|
||
emptyRegulatoryMappingBatchCounts,
|
||
groundRegulatoryMappingResult,
|
||
normalizeRegulatoryMappingBatchScope,
|
||
regulatoryMappingResultValidationErrors,
|
||
type RegulatoryMappingBatchCreateRequest,
|
||
type RegulatoryMappingBatchResponse,
|
||
type RegulatoryMappingBatchRun,
|
||
type RegulatoryMappingBatchScope,
|
||
type RegulatoryMappingCandidate,
|
||
type RegulatoryMappingCreateRequest,
|
||
type RegulatoryMappingDecision,
|
||
type RegulatoryEarlierControl,
|
||
type RegulatoryMappingDecisionCreateRequest,
|
||
type RegulatoryMappingEvidence,
|
||
type RegulatoryMappingResponse,
|
||
type RegulatoryMappingResult,
|
||
type RegulatoryMappingRun,
|
||
type RegulatoryMappingProposal,
|
||
type RegulatoryControlFinding,
|
||
type RegulatoryControlOverviewItem,
|
||
type RegulatoryControlOverviewResponse,
|
||
acceptedControlIds,
|
||
regulatoryControlTheme,
|
||
AUTO_MAPPING_BATCH_KEY_PREFIXES,
|
||
MAPPING_HISTORY_LIMIT,
|
||
BSI_CORPUS_ID,
|
||
BSI_MAPPING_PROMPT_VERSION,
|
||
BSI_RETRIEVAL_VERSION,
|
||
BSI_SOURCE_STANDARD,
|
||
frameworkField,
|
||
mappingFramework,
|
||
type RegulatoryFramework,
|
||
type RegulatoryMappingHistoryEntry
|
||
} from '../../src/compliance/regulatory-mapping';
|
||
import type {
|
||
RegulatoryChunk,
|
||
RegulatoryFindingQuery,
|
||
RegulatoryQueryExpansion,
|
||
RegulatoryRetrievalCandidate,
|
||
RegulatoryRetrievalFilters
|
||
} from '../../src/compliance/regulatory-rag';
|
||
import { fuseControlRankings, RegulatoryLexicalIndex, type FusedControl } from './regulatory-lexical-index';
|
||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||
import { privacySafeFinding } from '../../src/domain/privacy-safe-finding';
|
||
import type { RegulatoryFindingRetrievalService } from './regulatory-finding-retrieval';
|
||
import { bundledBsiCatalogs, type BsiCatalogs } from './bsi-snapshots';
|
||
import { bsiFootprint, bsiOverview, type BsiOverview } from '../../src/compliance/bsi-grundschutz';
|
||
import {
|
||
bsiConsistency,
|
||
bsiConsistencyFingerprintSource,
|
||
type RegulatoryConsistencyItem,
|
||
type RegulatoryConsistencyResponse,
|
||
type RegulatoryConsistencyReviewCreateRequest
|
||
} from '../../src/compliance/bsi-consistency';
|
||
import { bsiCorpusVersion, type BsiCorpus } from './bsi-corpus';
|
||
import type { RegulatoryChunkIndex } from './qdrant-regulatory-index';
|
||
import { buildRegulatoryFindingQuery, isRegulatoryMappableResult } from '../../src/compliance/regulatory-query';
|
||
import { canonicalJson } from '../../src/shared/canonical-json';
|
||
|
||
const POLL_AFTER_MS = 1_500;
|
||
const UNSUPPORTED_KIND_MESSAGE = 'Only canonical scanner findings and GPO baseline comparisons can be mapped in this version.';
|
||
// Prompt 1.3.0 sends each candidate's full control text: about 3,700 characters per control in the median, at most 8,000
|
||
// each, so eight long controls with their JSON escaping still fit.
|
||
const MAXIMUM_PROMPT_CHARACTERS = 96_000;
|
||
const MAXIMUM_CONTROL_TEXT_CHARACTERS = 8_000;
|
||
const SECTION_ORDER: readonly string[] = ['control', 'purpose', 'guidance', 'other-information', 'attributes'];
|
||
const CORE_SECTIONS = ['control', 'purpose', 'guidance'] as const;
|
||
const BATCH_PAGE_SIZE = 500;
|
||
const DEFAULT_MAXIMUM_CONCURRENT_RUNS = 2;
|
||
// User decision 2026-10-05 (provisional), narrowed 2026-10-09: of a finished mapping, at most the first two strong
|
||
// proposals (primary with high or medium confidence) are accepted automatically; weaker ones wait for a person.
|
||
export const AUTO_ACCEPTED_PROPOSALS = 2;
|
||
export const AUTO_ACCEPT_COMMENT = 'Accepted automatically: one of the first two strong proposals (primary, high or medium confidence) of the mapping.';
|
||
|
||
export function isStrongProposal(proposal: Pick<RegulatoryMappingProposal, 'relationship' | 'confidence'>): boolean {
|
||
return proposal.relationship === 'primary' && (proposal.confidence === 'high' || proposal.confidence === 'medium');
|
||
}
|
||
|
||
export interface RegulatoryMappingServiceOptions {
|
||
findingRetrieval?: RegulatoryFindingRetrievalService;
|
||
index?: RegulatoryChunkIndex;
|
||
client?: AiCompletionClient;
|
||
corpusId?: string;
|
||
corpusVersion?: string;
|
||
embeddingModel?: string;
|
||
disabledReason?: string;
|
||
now?: () => Date;
|
||
maximumConcurrentRuns?: number;
|
||
/** The active BSI catalog and mapping (`Update from BSI`); the pinned snapshots by default. */
|
||
bsiCatalogs?: () => Promise<BsiCatalogs>;
|
||
/** The BSI Kernel retrieval corpus for direct BSI mappings (stage 2); without it BSI mapping is unavailable. */
|
||
bsiCorpus?: BsiCorpus;
|
||
/** The model that restates a query in German for the hybrid search; without it only the English dense search runs. */
|
||
queryExpansionClient?: AiCompletionClient;
|
||
}
|
||
|
||
/** The versions that identify one mapping of a rule in one framework. */
|
||
interface MappingPipeline {
|
||
framework: RegulatoryFramework;
|
||
corpusId: string;
|
||
corpusVersion: string;
|
||
retrievalVersion: string;
|
||
promptVersion: string;
|
||
}
|
||
|
||
const FRAMEWORKS: readonly RegulatoryFramework[] = ['iso27001', 'bsi-kernel'];
|
||
const ISO_CANDIDATE_LIMIT = 8;
|
||
// Hybrid search (user decision 2026-10-09): 16 BSI candidates found the reference requirement for 12 of 20 test rules, 12 for 10.
|
||
const BSI_CANDIDATE_LIMIT = 16;
|
||
const BSI_EXCLUDED_PRACTICES = /^(BES|DLS|GEB|PERS)\./;
|
||
const MAXIMUM_CACHED_EXPANSIONS = 500;
|
||
const QUERY_EXPANSION_VERSION = 'de-expansion-1.0.0';
|
||
const QUERY_EXPANSION_PROMPT = 'Rewrite one public security finding as a German search query for the German editions of ISO/IEC 27001/27002 and the BSI Grundschutz++ Kernel. Return JSON only: {"query": "<one German sentence naming the general security subject, as a standard would phrase it>", "terms": ["<8 to 15 German terms a standard would use for this subject, plus technical terms that stay English>"]}. Do not add customer data.';
|
||
|
||
function retrievalFilters(pipeline: MappingPipeline): RegulatoryRetrievalFilters {
|
||
return pipeline.framework === 'bsi-kernel' ? { corpusId: pipeline.corpusId, corpusVersion: pipeline.corpusVersion }
|
||
: { corpusId: pipeline.corpusId, corpusVersion: pipeline.corpusVersion, sourceStandard: 'ISO/IEC 27002', retrievalRole: 'semantic' };
|
||
}
|
||
|
||
/** The model's German restatement, bounded plain text only; anything else means no hybrid search for this run. */
|
||
function queryExpansion(value: unknown): Omit<RegulatoryQueryExpansion, 'searches'> | undefined {
|
||
if (!value || typeof value !== 'object') return undefined;
|
||
const item = value as { query?: unknown; terms?: unknown };
|
||
const plain = (text: unknown, maximum: number) => typeof text === 'string' && text.trim() && text.length <= maximum && !/<\/?[a-z][^>]*>/i.test(text) ? text.trim() : undefined;
|
||
const query = plain(item.query, 600);
|
||
const terms = Array.isArray(item.terms) ? item.terms.map(term => plain(term, 80)).filter((term): term is string => Boolean(term)).slice(0, 20) : [];
|
||
return query ? { language: 'de', version: QUERY_EXPANSION_VERSION, query, terms } : undefined;
|
||
}
|
||
|
||
function requestedFramework(value: unknown): RegulatoryFramework {
|
||
if (value === undefined || value === null || value === '') return 'iso27001';
|
||
if (!FRAMEWORKS.includes(value as RegulatoryFramework)) throw new RepositoryValidationError('framework must be iso27001 or bsi-kernel.');
|
||
return value as RegulatoryFramework;
|
||
}
|
||
|
||
export class RegulatoryMappingService {
|
||
private readonly activeTasks = new Set<Promise<void>>();
|
||
private readonly now: () => Date;
|
||
private readonly maximumConcurrentRuns: number;
|
||
private readonly workQueue: Array<{
|
||
task: () => Promise<'succeeded' | 'failed'>;
|
||
resolve: (status: 'succeeded' | 'failed') => void;
|
||
}> = [];
|
||
private activeWorkerCount = 0;
|
||
private activationPending = false;
|
||
private titleCache?: { key: string; titles: Record<string, string> };
|
||
private readonly expansions = new Map<string, Promise<Omit<RegulatoryQueryExpansion, 'searches'> | undefined>>();
|
||
private readonly lexicalIndexes = new Map<string, Promise<RegulatoryLexicalIndex | undefined>>();
|
||
|
||
constructor(
|
||
private readonly read: ComplianceReadPort,
|
||
private readonly store: ComplianceStore,
|
||
private readonly options: RegulatoryMappingServiceOptions = {}
|
||
) {
|
||
this.now = options.now ?? (() => new Date());
|
||
this.maximumConcurrentRuns = options.maximumConcurrentRuns ?? DEFAULT_MAXIMUM_CONCURRENT_RUNS;
|
||
if (!Number.isInteger(this.maximumConcurrentRuns) || this.maximumConcurrentRuns < 1 || this.maximumConcurrentRuns > 8) {
|
||
throw new Error('Regulatory mapping concurrency must be an integer between 1 and 8.');
|
||
}
|
||
}
|
||
|
||
get generationEnabled(): boolean {
|
||
return !this.activationPending && Boolean(this.options.findingRetrieval && this.options.index && this.options.client
|
||
&& this.options.corpusId && this.options.corpusVersion && this.options.embeddingModel);
|
||
}
|
||
|
||
async activateCorpus(corpusId: string, corpusVersion: string): Promise<void> {
|
||
this.activationPending = true;
|
||
try {
|
||
await this.close();
|
||
this.options.corpusId = corpusId;
|
||
this.options.corpusVersion = corpusVersion;
|
||
} finally {
|
||
this.activationPending = false;
|
||
}
|
||
}
|
||
|
||
async recoverInterruptedRuns(): Promise<void> {
|
||
const completedAt = this.now().toISOString();
|
||
for (const run of (await this.store.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) {
|
||
await this.store.saveRegulatoryMappingRun({
|
||
...run,
|
||
status: 'failed',
|
||
updatedAt: completedAt,
|
||
completedAt,
|
||
errorCode: 'regulatory_mapping_interrupted'
|
||
});
|
||
}
|
||
for (const project of await this.read.listProjects()) {
|
||
for (const run of (await this.store.listRegulatoryMappingBatchRuns(project.id)).filter(item => ['queued', 'running'].includes(item.status))) {
|
||
await this.store.updateRegulatoryMappingBatchRun({
|
||
...run,
|
||
status: 'failed',
|
||
updatedAt: completedAt,
|
||
completedAt,
|
||
errorCode: 'regulatory_mapping_batch_interrupted',
|
||
errorDetail: 'Batch processing was interrupted before completion. Start a new batch to retry remaining rules.'
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
/** True when mapping can run for the framework; ISO also needs its active licensed corpus. */
|
||
private frameworkEnabled(framework: RegulatoryFramework): boolean {
|
||
if (framework === 'iso27001') return this.generationEnabled;
|
||
return !this.activationPending && Boolean(this.options.findingRetrieval && this.options.index && this.options.client
|
||
&& this.options.embeddingModel && this.options.bsiCorpus?.configured);
|
||
}
|
||
|
||
private async pipeline(framework: RegulatoryFramework): Promise<MappingPipeline> {
|
||
if (framework === 'iso27001') return { framework, corpusId: this.options.corpusId ?? '', corpusVersion: this.options.corpusVersion ?? '',
|
||
retrievalVersion: REGULATORY_RETRIEVAL_VERSION, promptVersion: REGULATORY_MAPPING_PROMPT_VERSION };
|
||
const { catalog } = await this.bsiCatalogs();
|
||
return { framework, corpusId: BSI_CORPUS_ID, corpusVersion: bsiCorpusVersion(catalog), retrievalVersion: BSI_RETRIEVAL_VERSION,
|
||
promptVersion: BSI_MAPPING_PROMPT_VERSION };
|
||
}
|
||
|
||
async get(projectId: string, resultProjectionId: string, requested?: RegulatoryFramework): Promise<RegulatoryMappingResponse> {
|
||
const framework = requestedFramework(requested);
|
||
const finding = await this.finding(projectId, resultProjectionId);
|
||
const supported = Boolean(buildRegulatoryFindingQuery(finding));
|
||
const availability = supported ? await this.availability(framework) : { enabled: false };
|
||
const enabled = availability.enabled && supported;
|
||
const ruleKey = supported ? this.ruleKey(finding) : undefined;
|
||
const pipeline = await this.pipeline(framework);
|
||
const run = ruleKey && this.options.client ? await this.store.getRegulatoryMappingRun(this.mappingId(ruleKey, pipeline)) : null;
|
||
const decisions = ruleKey ? await this.store.listRegulatoryMappingDecisions(ruleKey, framework) : [];
|
||
const earlierControls = ruleKey ? await this.earlierControls(ruleKey, run, decisions, framework) : [];
|
||
const isoDecisions = framework === 'iso27001' ? decisions : ruleKey ? await this.store.listRegulatoryMappingDecisions(ruleKey) : [];
|
||
const bsi = await this.bsiOverview(acceptedControlIds(isoDecisions));
|
||
const consistency = framework === 'bsi-kernel' && ruleKey ? await this.ruleConsistency(ruleKey, acceptedControlIds(isoDecisions), acceptedControlIds(decisions)) : undefined;
|
||
return {
|
||
version: 1,
|
||
projectId,
|
||
resultProjectionId,
|
||
framework,
|
||
generationEnabled: enabled,
|
||
state: run?.status ?? (enabled ? 'empty' : 'unsupported'),
|
||
...(!supported ? { disabledReason: !isRegulatoryMappableResult(finding)
|
||
? UNSUPPORTED_KIND_MESSAGE
|
||
: 'This finding has insufficient public context for regulatory mapping.' }
|
||
: !availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}),
|
||
...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}),
|
||
...(run ? { run: withoutEvidenceText(run) } : {}),
|
||
decisions,
|
||
...(earlierControls.length ? { earlierControls } : {}),
|
||
...(bsi ? { bsi } : {}),
|
||
...(consistency ? { consistency: { state: consistency.state, reviewed: consistency.reviewed } } : {})
|
||
};
|
||
}
|
||
|
||
/**
|
||
* The run that grounds a decision: the current one, or for an earlier decision the newest older succeeded
|
||
* run of the rule that retrieved the control, so it stays reviewable after a pipeline version change.
|
||
*/
|
||
private async groundingMappingId(ruleKey: string, controlId: string, pipeline: MappingPipeline): Promise<string> {
|
||
const currentId = this.mappingId(ruleKey, pipeline);
|
||
const current = await this.store.getRegulatoryMappingRun(currentId);
|
||
if (!current || current.candidates.some(candidate => candidate.controlId === controlId)) return currentId;
|
||
const older = (await this.store.listRegulatoryMappingRuns()).find(run => run.ruleKey === ruleKey && run.status === 'succeeded'
|
||
&& mappingFramework(run) === pipeline.framework && run.candidates.some(candidate => candidate.controlId === controlId));
|
||
return older?.id ?? currentId;
|
||
}
|
||
|
||
/** Decided controls the current run did not retrieve, titled from the newest older run of the rule that did. */
|
||
private async earlierControls(ruleKey: string, run: RegulatoryMappingRun | null, decisions: RegulatoryMappingDecision[],
|
||
framework: RegulatoryFramework): Promise<RegulatoryEarlierControl[]> {
|
||
const current = new Set(run?.candidates.map(candidate => candidate.controlId) ?? []);
|
||
const missing = [...new Set(decisions.map(decision => decision.controlId))].filter(controlId => !current.has(controlId));
|
||
if (!missing.length) return [];
|
||
const known = new Map<string, RegulatoryMappingCandidate>();
|
||
for (const older of await this.store.listRegulatoryMappingRuns()) {
|
||
if (older.ruleKey !== ruleKey || mappingFramework(older) !== framework) continue;
|
||
for (const candidate of older.candidates) if (!known.has(candidate.controlId)) known.set(candidate.controlId, candidate);
|
||
}
|
||
return missing.sort((left, right) => left.localeCompare(right, 'en', { numeric: true })).map(controlId => {
|
||
const candidate = known.get(controlId);
|
||
return { controlId, annexAControlId: candidate?.annexAControlId ?? (framework === 'iso27001' ? `A.${controlId}` : controlId),
|
||
...(candidate?.title ? { title: candidate.title } : {}) };
|
||
});
|
||
}
|
||
|
||
private disabledError(framework: RegulatoryFramework): RegulatoryMappingServiceError {
|
||
return new RegulatoryMappingServiceError('regulatory_mapping_disabled', framework === 'bsi-kernel' && this.generationEnabled
|
||
? 'BSI mapping is not configured.' : this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503);
|
||
}
|
||
|
||
async start(projectId: string, request: RegulatoryMappingCreateRequest): Promise<RegulatoryMappingResponse> {
|
||
const framework = requestedFramework(request.framework);
|
||
if (!this.frameworkEnabled(framework)) throw this.disabledError(framework);
|
||
const finding = await this.finding(projectId, request.resultProjectionId);
|
||
if (!isRegulatoryMappableResult(finding)) throw new RegulatoryMappingServiceError('unsupported_finding_kind', UNSUPPORTED_KIND_MESSAGE, 422);
|
||
if (!buildRegulatoryFindingQuery(finding)) throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422);
|
||
const pipeline = await this.pipeline(framework);
|
||
const claimed = await this.claim(finding, true, 'single', pipeline, await this.origin(projectId, request.origin));
|
||
if (claimed.claimed) this.track(this.enqueue(() => this.process(claimed.run, finding, pipeline)).then(() => undefined));
|
||
return this.get(projectId, request.resultProjectionId, framework);
|
||
}
|
||
|
||
async getBatch(projectId: string, requestedScope: RegulatoryMappingBatchScope, requested?: RegulatoryFramework): Promise<RegulatoryMappingBatchResponse> {
|
||
const framework = requestedFramework(requested);
|
||
const scope = await this.validateBatchScope(projectId, requestedScope);
|
||
const scopeKey = this.batchScopeKey(projectId, scope, framework);
|
||
const run = (await this.store.listRegulatoryMappingBatchRuns(projectId, scopeKey))[0];
|
||
const availability = await this.availability(framework);
|
||
return {
|
||
version: 1,
|
||
projectId,
|
||
scope,
|
||
generationEnabled: availability.enabled,
|
||
state: run?.status ?? 'empty',
|
||
...(!availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}),
|
||
...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}),
|
||
...(run ? { run } : {})
|
||
};
|
||
}
|
||
|
||
async startBatch(projectId: string, request: RegulatoryMappingBatchCreateRequest): Promise<RegulatoryMappingBatchResponse> {
|
||
const framework = requestedFramework(request.framework);
|
||
if (!this.frameworkEnabled(framework)) throw this.disabledError(framework);
|
||
if (typeof request.idempotencyKey !== 'string' || !/^[A-Za-z0-9._:-]{8,128}$/.test(request.idempotencyKey)) {
|
||
throw new RepositoryValidationError('idempotencyKey must contain 8 to 128 safe characters.');
|
||
}
|
||
const scope = await this.validateBatchScope(projectId, request);
|
||
const timestamp = this.now().toISOString();
|
||
const candidate: RegulatoryMappingBatchRun = {
|
||
version: 1,
|
||
id: `regulatory-mapping-batch-${randomUUID()}`,
|
||
...frameworkField(framework),
|
||
projectId,
|
||
scopeKey: this.batchScopeKey(projectId, scope, framework),
|
||
filterHash: sha256(canonicalJson(scope.filters)),
|
||
assessmentIds: scope.assessmentIds,
|
||
scanRunIds: scope.scanRunIds,
|
||
filters: scope.filters,
|
||
idempotencyKey: request.idempotencyKey,
|
||
status: 'queued',
|
||
counts: emptyRegulatoryMappingBatchCounts(),
|
||
createdAt: timestamp,
|
||
updatedAt: timestamp
|
||
};
|
||
const run = await this.store.createRegulatoryMappingBatchRun(candidate);
|
||
if (run.id === candidate.id) this.track(this.processBatch(run));
|
||
return this.getBatch(projectId, {
|
||
assessmentIds: run.assessmentIds,
|
||
scanRunIds: run.scanRunIds,
|
||
filters: run.filters
|
||
}, framework);
|
||
}
|
||
|
||
async decide(projectId: string, request: RegulatoryMappingDecisionCreateRequest): Promise<RegulatoryMappingResponse> {
|
||
const framework = requestedFramework(request.framework);
|
||
if (!this.frameworkEnabled(framework)) throw this.disabledError(framework);
|
||
const finding = await this.finding(projectId, request.resultProjectionId);
|
||
if (!isRegulatoryMappableResult(finding)) throw new RegulatoryMappingServiceError('unsupported_finding_kind', UNSUPPORTED_KIND_MESSAGE, 422);
|
||
const ruleKey = this.ruleKey(finding);
|
||
const comment = request.comment?.trim();
|
||
const mappingId = await this.groundingMappingId(ruleKey, request.controlId, await this.pipeline(framework));
|
||
const decision: RegulatoryMappingDecision = {
|
||
version: 1,
|
||
id: `regulatory-decision-${randomUUID()}`,
|
||
...frameworkField(framework),
|
||
ruleKey,
|
||
sourceTool: finding.sourceTool ?? 'unknown',
|
||
...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}),
|
||
mappingId,
|
||
controlId: request.controlId,
|
||
decision: request.decision,
|
||
...(comment ? { comment } : {}),
|
||
createdAt: this.now().toISOString()
|
||
};
|
||
await this.store.createRegulatoryMappingDecision(decision, request.expectedLatestDecisionId);
|
||
return this.get(projectId, request.resultProjectionId, framework);
|
||
}
|
||
|
||
/** The consistency of one rule with whether exactly this result was kept by `Mark reviewed`. */
|
||
private async ruleConsistency(ruleKey: string, acceptedIso: string[], acceptedBsi: string[], reviews?: Map<string, Set<string>>) {
|
||
const { crosswalk } = await this.bsiCatalogs();
|
||
const consistency = bsiConsistency(acceptedIso, acceptedBsi, crosswalk);
|
||
if (!consistency) return undefined;
|
||
const fingerprint = sha256(bsiConsistencyFingerprintSource(acceptedIso, acceptedBsi, crosswalk.version));
|
||
const reviewed = reviews ? reviews.get(ruleKey)?.has(fingerprint) ?? false
|
||
: (await this.store.listRegulatoryConsistencyReviews()).some(review => review.ruleKey === ruleKey && review.fingerprint === fingerprint);
|
||
return { ...consistency, fingerprint, reviewed };
|
||
}
|
||
|
||
/**
|
||
* The Compliance tab review list (Phase 13F, BSI stage 2, user decision 2026-10-09): every rule of the
|
||
* timeslot's mappable deviations whose accepted ISO controls and directly accepted BSI requirements can be
|
||
* compared. Derived only; nothing here changes a decision. Rules are named by one Result DB row of the timeslot.
|
||
*/
|
||
async consistency(projectId: string, requested: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>): Promise<RegulatoryConsistencyResponse> {
|
||
const scope = await this.validateBatchScope(projectId, { assessmentIds: requested.assessmentIds, scanRunIds: requested.scanRunIds, filters: { deviation: true } });
|
||
const results = (await this.queryBatchResults(projectId, scope)).filter(isRegulatoryMappableResult)
|
||
.sort((left, right) => left.resultProjectionId.localeCompare(right.resultProjectionId));
|
||
const settingIds = await this.gpoSettingIds(projectId, scope, results);
|
||
const byRule = new Map<string, { representative: UnifiedResultProjection; identities: Set<string> }>();
|
||
for (const finding of results) {
|
||
const ruleKey = this.ruleKey(finding);
|
||
const entry = byRule.get(ruleKey) ?? { representative: finding, identities: new Set<string>() };
|
||
entry.identities.add(findingIdentity(finding, settingIds.get(finding.resultProjectionId)));
|
||
byRule.set(ruleKey, entry);
|
||
}
|
||
const { catalog, crosswalk } = await this.bsiCatalogs();
|
||
const requirements = new Map(catalog.requirements.map(requirement => [requirement.id, requirement]));
|
||
const reachable = new Set(crosswalk.entries.flatMap(entry => entry.targets));
|
||
const reviews = new Map<string, Set<string>>();
|
||
for (const review of await this.store.listRegulatoryConsistencyReviews()) reviews.set(review.ruleKey, (reviews.get(review.ruleKey) ?? new Set()).add(review.fingerprint));
|
||
// ISO control titles and retrieved candidates from the latest succeeded ISO run of each rule.
|
||
const isoRuns = new Map<string, RegulatoryMappingRun>();
|
||
for (const run of await this.store.listRegulatoryMappingRuns()) {
|
||
if (!byRule.has(run.ruleKey) || mappingFramework(run) !== 'iso27001' || run.status !== 'succeeded') continue;
|
||
const latest = isoRuns.get(run.ruleKey);
|
||
if (!latest || run.createdAt > latest.createdAt || run.createdAt === latest.createdAt && run.id > latest.id) isoRuns.set(run.ruleKey, run);
|
||
}
|
||
const isoPipeline = await this.pipeline('iso27001');
|
||
const counts = { consistent: 0, inconsistent: 0, noReference: 0, notCompared: 0, reviewed: 0 };
|
||
const items: RegulatoryConsistencyItem[] = [];
|
||
for (const [ruleKey, { representative, identities }] of byRule) {
|
||
const acceptedIso = acceptedControlIds(await this.store.listRegulatoryMappingDecisions(ruleKey));
|
||
const acceptedBsi = acceptedControlIds(await this.store.listRegulatoryMappingDecisions(ruleKey, 'bsi-kernel'));
|
||
const result = await this.ruleConsistency(ruleKey, acceptedIso, acceptedBsi, reviews);
|
||
if (!result) { if (acceptedIso.length) counts.notCompared += 1; continue; }
|
||
counts[result.state === 'no-reference' ? 'noReference' : result.state] += 1;
|
||
if (result.reviewed) counts.reviewed += 1;
|
||
// A reverse candidate is decidable here only where the current ISO run retrieved it, as the decision needs grounding.
|
||
const currentIso = this.options.client && isoPipeline.corpusVersion
|
||
? await this.store.getRegulatoryMappingRun(this.mappingId(ruleKey, isoPipeline)) : null;
|
||
const titles = new Map((isoRuns.get(ruleKey)?.candidates ?? []).map(candidate => [candidate.controlId, candidate.title]));
|
||
const derived = new Set(result.derivedRequirementIds);
|
||
items.push({
|
||
resultProjectionId: representative.resultProjectionId,
|
||
title: representative.title,
|
||
sourceTool: representative.sourceTool ?? 'unknown',
|
||
...(representative.sourceRuleId ? { sourceRuleId: representative.sourceRuleId } : {}),
|
||
findingCount: identities.size,
|
||
state: result.state,
|
||
reviewed: result.reviewed,
|
||
iso: [...acceptedIso].sort(byNumber).map(controlId => ({ controlId, ...(titles.get(controlId) ? { title: titles.get(controlId) } : {}) })),
|
||
direct: result.directRequirementIds.map(requirementId => ({ requirementId,
|
||
...(requirements.get(requirementId) ? { title: requirements.get(requirementId)!.title, practice: requirements.get(requirementId)!.practice } : {}),
|
||
derived: derived.has(requirementId), reachable: reachable.has(requirementId) })),
|
||
derivedCount: result.derivedRequirementIds.length,
|
||
reverse: result.reverseCandidates.map(candidate => ({ controlId: candidate.isoControlId,
|
||
...(titles.get(candidate.isoControlId) ? { title: titles.get(candidate.isoControlId) } : {}), requirementIds: candidate.requirementIds,
|
||
decidable: currentIso?.status === 'succeeded' && currentIso.candidates.some(item => item.controlId === candidate.isoControlId) }))
|
||
});
|
||
}
|
||
const order = { inconsistent: 0, 'no-reference': 1, consistent: 2 } as const;
|
||
items.sort((left, right) => Number(left.reviewed) - Number(right.reviewed) || order[left.state] - order[right.state]
|
||
|| right.findingCount - left.findingCount || left.title.localeCompare(right.title) || left.resultProjectionId.localeCompare(right.resultProjectionId));
|
||
return { version: 1, projectId, scope: { assessmentIds: scope.assessmentIds, scanRunIds: scope.scanRunIds },
|
||
crosswalkVersion: crosswalk.version, catalogVersion: catalog.version, counts, items };
|
||
}
|
||
|
||
/** Records that a person keeps the current ISO <-> BSI result of the finding's rule as it is. */
|
||
async markConsistencyReviewed(projectId: string, request: RegulatoryConsistencyReviewCreateRequest): Promise<{ reviewed: true }> {
|
||
const finding = await this.finding(projectId, request?.resultProjectionId);
|
||
if (!isRegulatoryMappableResult(finding)) throw new RegulatoryMappingServiceError('unsupported_finding_kind', UNSUPPORTED_KIND_MESSAGE, 422);
|
||
const ruleKey = this.ruleKey(finding);
|
||
const acceptedIso = acceptedControlIds(await this.store.listRegulatoryMappingDecisions(ruleKey));
|
||
const acceptedBsi = acceptedControlIds(await this.store.listRegulatoryMappingDecisions(ruleKey, 'bsi-kernel'));
|
||
const result = await this.ruleConsistency(ruleKey, acceptedIso, acceptedBsi);
|
||
if (!result) throw new RepositoryValidationError('This rule has no accepted BSI requirement to compare.');
|
||
const comment = typeof request.comment === 'string' ? request.comment.trim() : '';
|
||
if (!result.reviewed) await this.store.createRegulatoryConsistencyReview({ version: 1, id: `regulatory-consistency-review-${randomUUID()}`,
|
||
ruleKey, fingerprint: result.fingerprint, ...(comment ? { comment } : {}), createdAt: this.now().toISOString() });
|
||
return { reviewed: true };
|
||
}
|
||
|
||
/** BSI Kernel requirements derived from accepted ISO controls through the active BSI mapping (Phase 13F, stage 1). */
|
||
private async bsiOverview(acceptedIsoControlIds: readonly string[]): Promise<BsiOverview | undefined> {
|
||
const { catalog, crosswalk } = await this.bsiCatalogs();
|
||
return bsiOverview(acceptedIsoControlIds, crosswalk, catalog);
|
||
}
|
||
|
||
/**
|
||
* Annex A control titles of the active licensed corpus (one ISO/IEC 27001 reference document per control), for the
|
||
* Client Info footprint; only titles, never clause text. Cached per corpus version; an unavailable index yields none.
|
||
*/
|
||
private async annexATitles(): Promise<Record<string, string>> {
|
||
const { index, corpusId, corpusVersion } = this.options;
|
||
if (!index || !corpusId || !corpusVersion) return {};
|
||
const key = `${corpusId}\u0000${corpusVersion}`;
|
||
if (this.titleCache?.key === key) return this.titleCache.titles;
|
||
try {
|
||
const chunks = await index.list({ corpusId, corpusVersion, sourceStandard: 'ISO/IEC 27001', retrievalRole: 'reference' }, 100);
|
||
const titles: Record<string, string> = {};
|
||
for (const chunk of chunks) if (chunk.controlId && chunk.title?.trim() && !titles[chunk.controlId]) titles[chunk.controlId] = chunk.title.trim();
|
||
if (Object.keys(titles).length) this.titleCache = { key, titles };
|
||
return titles;
|
||
} catch {
|
||
return {};
|
||
}
|
||
}
|
||
|
||
private async bsiCatalogs(): Promise<BsiCatalogs> {
|
||
return this.options.bsiCatalogs ? this.options.bsiCatalogs() : bundledBsiCatalogs;
|
||
}
|
||
|
||
/** Accepted controls of every mappable finding in a client timeslot, for the Work `ISO control` and `ISO theme` filters. */
|
||
async controlAssignments(projectId: string, requested: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>): Promise<RegulatoryControlOverviewResponse> {
|
||
return this.controlOverview(projectId, requested, { deviationsOnly: false });
|
||
}
|
||
|
||
/**
|
||
* Accepted controls of the deviations in one client timeslot. Decisions are global per scanner rule;
|
||
* a GPO setting counts once, like its rolled-up Work record, however many baseline comparisons it has.
|
||
*/
|
||
async controlOverview(projectId: string, requested: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>,
|
||
{ deviationsOnly = true }: { deviationsOnly?: boolean } = {}): Promise<RegulatoryControlOverviewResponse> {
|
||
const scope = await this.validateBatchScope(projectId, { assessmentIds: requested.assessmentIds, scanRunIds: requested.scanRunIds,
|
||
filters: deviationsOnly ? { deviation: true } : {} });
|
||
const byRule = new Map<string, Map<string, RegulatoryControlFinding>>();
|
||
const results = (await this.queryBatchResults(projectId, scope)).filter(isRegulatoryMappableResult);
|
||
const settingIds = await this.gpoSettingIds(projectId, scope, results);
|
||
for (const finding of results) {
|
||
const settingId = settingIds.get(finding.resultProjectionId);
|
||
const identity = findingIdentity(finding, settingId);
|
||
const findings = byRule.get(this.ruleKey(finding)) ?? new Map<string, RegulatoryControlFinding>();
|
||
// The setting ID only lets Work match its rolled-up GPO setting; it is never displayed or exported.
|
||
if (!findings.has(identity)) findings.set(identity, { resultProjectionId: finding.resultProjectionId, sourceKind: finding.sourceKind,
|
||
sourceResultId: finding.sourceResultId, sourceRuleId: settingId ?? finding.sourceRuleId, policyScope: finding.policyScope });
|
||
byRule.set(this.ruleKey(finding), findings);
|
||
}
|
||
// Control titles come from the newest succeeded run of each rule that retrieved the control.
|
||
const titles = new Map<string, Map<string, RegulatoryMappingCandidate>>();
|
||
const mapped = new Set<string>();
|
||
const latestRun = new Map<string, RegulatoryMappingRun>();
|
||
for (const run of await this.store.listRegulatoryMappingRuns()) {
|
||
if (!byRule.has(run.ruleKey) || mappingFramework(run) !== 'iso27001') continue;
|
||
const latest = latestRun.get(run.ruleKey);
|
||
if (!latest || run.createdAt > latest.createdAt || run.createdAt === latest.createdAt && run.id > latest.id) latestRun.set(run.ruleKey, run);
|
||
if (run.status !== 'succeeded') continue;
|
||
mapped.add(run.ruleKey);
|
||
const known = titles.get(run.ruleKey) ?? new Map<string, RegulatoryMappingCandidate>();
|
||
for (const candidate of run.candidates) if (!known.has(candidate.controlId)) known.set(candidate.controlId, candidate);
|
||
titles.set(run.ruleKey, known);
|
||
}
|
||
const controls = new Map<string, { item: RegulatoryControlOverviewItem; findings: Map<string, RegulatoryControlFinding> }>();
|
||
const covered = new Set<string>();
|
||
const unaccepted: Array<[string, Map<string, RegulatoryControlFinding>]> = [];
|
||
for (const [ruleKey, findings] of byRule) {
|
||
const accepted = acceptedControlIds(await this.store.listRegulatoryMappingDecisions(ruleKey));
|
||
if (!accepted.length) unaccepted.push([ruleKey, findings]);
|
||
for (const controlId of accepted) {
|
||
const candidate = titles.get(ruleKey)?.get(controlId);
|
||
const entry = controls.get(controlId) ?? { findings: new Map(), item: {
|
||
controlId, annexAControlId: candidate?.annexAControlId ?? `A.${controlId}`, findings: [],
|
||
...(candidate?.title ? { title: candidate.title } : {}),
|
||
...(regulatoryControlTheme(controlId) ? { theme: regulatoryControlTheme(controlId) } : {}) } };
|
||
if (!entry.item.title && candidate?.title) entry.item.title = candidate.title;
|
||
for (const [identity, finding] of findings) { entry.findings.set(identity, finding); covered.add(identity); }
|
||
controls.set(controlId, entry);
|
||
}
|
||
}
|
||
const bsi = await this.bsiOverview([...controls.keys()]);
|
||
// A finding counts once, in the strongest state of its rule: accepted, mapped, failed, else not mapped.
|
||
const mappedNoneAccepted = new Set<string>();
|
||
const failed = new Set<string>();
|
||
const notMapped = new Set<string>();
|
||
const failedFindings: RegulatoryControlFinding[] = [];
|
||
for (const [ruleKey, findings] of unaccepted) {
|
||
const state = mapped.has(ruleKey) ? mappedNoneAccepted : latestRun.get(ruleKey)?.status === 'failed' ? failed : notMapped;
|
||
for (const [identity, finding] of findings) {
|
||
if (covered.has(identity)) continue;
|
||
state.add(identity);
|
||
if (state === failed) failedFindings.push(finding);
|
||
}
|
||
}
|
||
const coverage = { deviations: covered.size + mappedNoneAccepted.size + failed.size + notMapped.size, accepted: covered.size,
|
||
mappedNoneAccepted: mappedNoneAccepted.size, failed: failed.size, notMapped: notMapped.size };
|
||
return {
|
||
version: 1, projectId, scope: { assessmentIds: scope.assessmentIds, scanRunIds: scope.scanRunIds }, findingCount: covered.size,
|
||
controls: [...controls.values()].map(({ item, findings }) => ({ ...item, findings: [...findings.values()] }))
|
||
.sort((left, right) => right.findings.length - left.findings.length
|
||
|| left.controlId.localeCompare(right.controlId, 'en', { numeric: true })),
|
||
...(deviationsOnly ? { mappingHistory: await this.mappingHistory(projectId, scope), coverage, annexATitles: await this.annexATitles(),
|
||
bsiFootprint: await this.bsiCatalogs().then(({ catalog, crosswalk }) => bsiFootprint(crosswalk, catalog)) }
|
||
// The Work `ISO mapping` filter: findings whose rule's latest mapping failed, matched like the control filters.
|
||
: { failedFindings }),
|
||
...(bsi ? { bsi } : {})
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Newest first, globally (user decision 2026-10-08), because mappings and decisions are global per rule:
|
||
* every batch of every client and every single finding mapping, each marked by where it was started
|
||
* relative to the shown timeslot. Entries carry dates, trigger, that mark and counts only, never a client,
|
||
* rule or scope id. Runs from before 2.21.0 record no trigger; a run a batch started is counted by its batch.
|
||
*/
|
||
private async mappingHistory(projectId: string, scope: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>): Promise<RegulatoryMappingHistoryEntry[]> {
|
||
const assessments = new Set(scope.assessmentIds);
|
||
const scanRuns = new Set(scope.scanRunIds);
|
||
const viewed = originTokens(projectId, scope);
|
||
const viewedTimeslot = new Set(viewed.timeslot);
|
||
const batchRuns = (await Promise.all((await this.read.listProjects())
|
||
.map(project => this.store.listRegulatoryMappingBatchRuns(project.id)))).flat();
|
||
// User decision 2026-10-09: the history lists ISO and direct BSI mappings, each named by its framework.
|
||
const batches: RegulatoryMappingHistoryEntry[] = batchRuns.map(run => ({
|
||
id: run.id, kind: 'batch', framework: mappingFramework(run),
|
||
scope: run.projectId !== projectId ? 'other-client'
|
||
: run.assessmentIds.some(id => assessments.has(id)) || run.scanRunIds.some(id => scanRuns.has(id)) ? 'timeslot' : 'other-timeslot',
|
||
trigger: AUTO_MAPPING_BATCH_KEY_PREFIXES.some(prefix => run.idempotencyKey.startsWith(prefix)) ? 'automatic' : 'manual',
|
||
status: run.status, createdAt: run.createdAt, ...(run.completedAt ? { completedAt: run.completedAt } : {}),
|
||
counts: { findings: run.counts.eligibleFindingCount, sentToModel: run.counts.scheduledRuleCount,
|
||
reused: run.counts.reusedSucceededRuleCount + run.counts.reusedActiveRuleCount,
|
||
succeeded: run.counts.succeededRuleCount, failed: run.counts.failedRuleCount }
|
||
}));
|
||
// A single mapping maps one rule for every client; it has no finding count of its own.
|
||
const singles: RegulatoryMappingHistoryEntry[] = (await this.store.listRegulatoryMappingRuns())
|
||
.filter(run => run.trigger === 'single')
|
||
.map(run => ({
|
||
id: run.id, kind: 'single', framework: mappingFramework(run), trigger: 'manual',
|
||
scope: !run.origin ? 'unknown' : run.origin.timeslot.some(token => viewedTimeslot.has(token)) ? 'timeslot'
|
||
: run.origin.project === viewed.project ? 'other-timeslot' : 'other-client',
|
||
status: run.status === 'succeeded' ? 'completed' : run.status,
|
||
createdAt: run.createdAt, ...(run.completedAt ? { completedAt: run.completedAt } : {}),
|
||
counts: { sentToModel: 1, reused: 0, succeeded: run.status === 'succeeded' ? 1 : 0, failed: run.status === 'failed' ? 1 : 0 }
|
||
}));
|
||
return [...batches, ...singles]
|
||
.sort((left, right) => right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id))
|
||
.slice(0, MAPPING_HISTORY_LIMIT);
|
||
}
|
||
|
||
/**
|
||
* Accepts the first two strong proposals of a finished mapping, in the model's order. A control that already has a
|
||
* decision keeps it, so a manual reject is never overridden; a failure here never fails the mapping.
|
||
*/
|
||
private async autoAcceptFirstProposals(run: RegulatoryMappingRun, result: RegulatoryMappingResult): Promise<void> {
|
||
if (result.outcome !== 'mapped') return;
|
||
try {
|
||
// User decision 2026-10-09: direct BSI mappings are accepted automatically like ISO ones, for now.
|
||
const framework = mappingFramework(run);
|
||
const decided = new Set((await this.store.listRegulatoryMappingDecisions(run.ruleKey, framework)).map(item => item.controlId));
|
||
for (const proposal of result.mappings.filter(isStrongProposal).slice(0, AUTO_ACCEPTED_PROPOSALS)) {
|
||
if (decided.has(proposal.controlId)) continue;
|
||
await this.store.createRegulatoryMappingDecision({
|
||
version: 1, id: `regulatory-decision-${randomUUID()}`, ...frameworkField(framework), ruleKey: run.ruleKey, sourceTool: run.sourceTool,
|
||
...(run.sourceRuleId ? { sourceRuleId: run.sourceRuleId } : {}), mappingId: run.id, controlId: proposal.controlId,
|
||
decision: 'accepted', comment: AUTO_ACCEPT_COMMENT, createdAt: this.now().toISOString()
|
||
});
|
||
}
|
||
} catch { /* A concurrent manual decision wins; the mapping itself has succeeded. */ }
|
||
}
|
||
|
||
// The global sharing key: the rule identity `tool:ruleId`, shared across every tenant.
|
||
// Rules without an id fall back to a stable hash of the non-personal check title so
|
||
// identical findings still share, while never keying on tenant or occurrence.
|
||
private ruleKey(finding: UnifiedResultProjection): string {
|
||
const tool = finding.sourceTool?.trim() || 'unknown';
|
||
const ruleId = finding.sourceRuleId?.trim();
|
||
if (ruleId) return `${tool}:${ruleId}`.slice(0, 200);
|
||
const hash = createHash('sha256').update(privacySafeFinding(finding).title.trim(), 'utf8').digest('hex').slice(0, 24);
|
||
return `${tool}#title:${hash}`.slice(0, 200);
|
||
}
|
||
|
||
// Deterministic id per (ruleKey, pipeline versions) so one row exists per rule per
|
||
// version tuple: regeneration reuses it, and a version bump yields a new mapping.
|
||
// ISO ids keep their earlier tuple; a BSI id also names its framework.
|
||
private mappingId(ruleKey: string, pipeline: MappingPipeline): string {
|
||
const tuple = [...(pipeline.framework === 'bsi-kernel' ? [pipeline.framework] : []),
|
||
ruleKey, pipeline.corpusVersion, pipeline.retrievalVersion, pipeline.promptVersion, this.options.client!.model,
|
||
// GPO rules carry the baseline context of their query version; scanner mapping ids stay unchanged.
|
||
...(isGpoRuleKey(ruleKey) ? [REGULATORY_GPO_QUERY_VERSION] : [])].join(' ');
|
||
return `regulatory-mapping-${createHash('sha256').update(tuple, 'utf8').digest('hex').slice(0, 40)}`;
|
||
}
|
||
|
||
// The stored prompt version must differ wherever the id differs: the table's unique
|
||
// identity tuple has no column for the GPO query version, so GPO runs record it here.
|
||
private promptVersion(ruleKey: string, pipeline: MappingPipeline): string {
|
||
return isGpoRuleKey(ruleKey) ? `${pipeline.promptVersion}+${REGULATORY_GPO_QUERY_VERSION}` : pipeline.promptVersion;
|
||
}
|
||
|
||
async close(): Promise<void> {
|
||
while (this.activeTasks.size) await Promise.allSettled([...this.activeTasks]);
|
||
}
|
||
|
||
private track(task: Promise<void>): void {
|
||
this.activeTasks.add(task);
|
||
void task.finally(() => this.activeTasks.delete(task));
|
||
}
|
||
|
||
private enqueue(task: () => Promise<'succeeded' | 'failed'>): Promise<'succeeded' | 'failed'> {
|
||
const promise = new Promise<'succeeded' | 'failed'>(resolve => this.workQueue.push({ task, resolve }));
|
||
this.drainQueue();
|
||
return promise;
|
||
}
|
||
|
||
private drainQueue(): void {
|
||
while (this.activeWorkerCount < this.maximumConcurrentRuns && this.workQueue.length) {
|
||
const work = this.workQueue.shift()!;
|
||
this.activeWorkerCount += 1;
|
||
void work.task().then(work.resolve, () => work.resolve('failed')).finally(() => {
|
||
this.activeWorkerCount -= 1;
|
||
this.drainQueue();
|
||
});
|
||
}
|
||
}
|
||
|
||
private queuedRun(finding: UnifiedResultProjection, trigger: 'single' | 'batch', pipeline: MappingPipeline, origin?: RegulatoryMappingRun['origin']): RegulatoryMappingRun {
|
||
const preparedQuery = buildRegulatoryFindingQuery(finding);
|
||
if (!preparedQuery) throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422);
|
||
const timestamp = this.now().toISOString();
|
||
const ruleKey = this.ruleKey(finding);
|
||
return {
|
||
version: 1,
|
||
id: this.mappingId(ruleKey, pipeline),
|
||
...frameworkField(pipeline.framework),
|
||
ruleKey,
|
||
sourceTool: finding.sourceTool ?? 'unknown',
|
||
...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}),
|
||
status: 'queued',
|
||
queryHash: sha256(JSON.stringify(preparedQuery)),
|
||
preparedQuery,
|
||
corpusId: pipeline.corpusId,
|
||
corpusVersion: pipeline.corpusVersion,
|
||
sourceHashes: [],
|
||
embeddingModel: this.options.embeddingModel!,
|
||
retrievalVersion: pipeline.retrievalVersion,
|
||
promptVersion: this.promptVersion(ruleKey, pipeline),
|
||
outputSchemaVersion: REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION,
|
||
model: this.options.client!.model,
|
||
trigger,
|
||
...(origin ? { origin } : {}),
|
||
createdAt: timestamp,
|
||
updatedAt: timestamp,
|
||
candidates: []
|
||
};
|
||
}
|
||
|
||
private async claim(finding: UnifiedResultProjection, forceRegeneration: boolean, trigger: 'single' | 'batch', pipeline: MappingPipeline,
|
||
origin?: RegulatoryMappingRun['origin']) {
|
||
return this.store.claimRegulatoryMappingRun(this.queuedRun(finding, trigger, pipeline, origin), forceRegeneration);
|
||
}
|
||
|
||
/** The client always; the timeslot only when it is a valid timeslot of that client. */
|
||
private async origin(projectId: string, requested: RegulatoryMappingCreateRequest['origin']): Promise<RegulatoryMappingRun['origin']> {
|
||
let timeslot: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'> = { assessmentIds: [], scanRunIds: [] };
|
||
if (requested) {
|
||
try { timeslot = await this.validateBatchScope(projectId, { assessmentIds: requested.assessmentIds, scanRunIds: requested.scanRunIds, filters: {} }); }
|
||
catch { /* An unusable timeslot leaves only the client as origin. */ }
|
||
}
|
||
return originTokens(projectId, timeslot);
|
||
}
|
||
|
||
private async validateBatchScope(projectId: string, requested: RegulatoryMappingBatchScope): Promise<RegulatoryMappingBatchScope> {
|
||
if (!await this.read.getProject(projectId)) throw new RepositoryNotFoundError('The selected project does not exist.');
|
||
const scope = normalizeRegulatoryMappingBatchScope(requested);
|
||
if (!scope.assessmentIds.length && !scope.scanRunIds.length) throw new RepositoryValidationError('A selected timeslot scope is required.');
|
||
const assessments = await this.read.listAssessments(projectId);
|
||
const allowedAssessmentIds = new Set(assessments.map(item => item.id));
|
||
if (scope.assessmentIds.some(id => !allowedAssessmentIds.has(id))) throw new RepositoryValidationError('The batch scope contains an assessment outside this project.');
|
||
const scanRuns = await this.read.listProjectScanRuns(projectId);
|
||
const allowedRuns = new Map(scanRuns.filter(item => !item.supersededByScanRunId).map(item => [item.id, item]));
|
||
if (scope.scanRunIds.some(id => !allowedRuns.has(id))) throw new RepositoryValidationError('The batch scope contains an unavailable scan run.');
|
||
if (scope.assessmentIds.length && scope.scanRunIds.some(id => !scope.assessmentIds.includes(allowedRuns.get(id)!.assessmentId))) {
|
||
throw new RepositoryValidationError('The assessment and scan-run scope do not describe the same timeslot.');
|
||
}
|
||
return scope;
|
||
}
|
||
|
||
// ISO keys keep their earlier input, so an active ISO batch and a BSI batch of the same scope never collide.
|
||
private batchScopeKey(projectId: string, scope: RegulatoryMappingBatchScope, framework: RegulatoryFramework = 'iso27001'): string {
|
||
return sha256(canonicalJson({ projectId, ...scope, ...frameworkField(framework) }));
|
||
}
|
||
|
||
private async queryBatchResults(projectId: string, scope: RegulatoryMappingBatchScope): Promise<UnifiedResultProjection[]> {
|
||
const query = {
|
||
...scope.filters,
|
||
audience: 'result-db' as const,
|
||
projectId,
|
||
assessmentIds: scope.assessmentIds,
|
||
scanRunIds: scope.scanRunIds
|
||
};
|
||
const items: UnifiedResultProjection[] = [];
|
||
let expectedTotal: number | undefined;
|
||
for (let offset = 0; ; offset += BATCH_PAGE_SIZE) {
|
||
const page = await this.read.queryUnifiedResults({ ...query, offset, limit: BATCH_PAGE_SIZE });
|
||
expectedTotal ??= page.total;
|
||
if (page.total !== expectedTotal) throw new RepositoryConflictError('The filtered result set changed while the ISO mapping batch was being prepared.');
|
||
items.push(...page.items);
|
||
if (items.length >= page.total || page.items.length === 0) break;
|
||
}
|
||
if (items.length !== expectedTotal) throw new RepositoryConflictError('The complete filtered result set could not be read consistently.');
|
||
return items;
|
||
}
|
||
|
||
/**
|
||
* The internal technical setting ID of each GPO comparison the Result DB delivered without one
|
||
* (user decision 2026-10-05). Read-only, limited to policy-evaluation rows of the same timeslot,
|
||
* and used only to identify the rolled-up Work setting; no other internal field is kept.
|
||
*/
|
||
private async gpoSettingIds(projectId: string, scope: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>,
|
||
results: UnifiedResultProjection[]): Promise<Map<string, string>> {
|
||
const wanted = new Set(results.filter(item => item.sourceKind === 'policy-evaluation' && !item.sourceRuleId?.trim())
|
||
.map(item => item.resultProjectionId));
|
||
const settingIds = new Map<string, string>();
|
||
if (!wanted.size) return settingIds;
|
||
const query = { audience: 'internal-analysis' as const, projectId, assessmentIds: scope.assessmentIds, scanRunIds: scope.scanRunIds,
|
||
sourceKind: ['policy-evaluation' as const] };
|
||
for (let offset = 0; ; offset += BATCH_PAGE_SIZE) {
|
||
const page = await this.read.queryUnifiedResults({ ...query, offset, limit: BATCH_PAGE_SIZE });
|
||
for (const item of page.items) {
|
||
const settingId = item.sourceRuleId?.trim();
|
||
if (settingId && wanted.has(item.resultProjectionId)) settingIds.set(item.resultProjectionId, settingId);
|
||
}
|
||
if (offset + page.items.length >= page.total || page.items.length === 0) break;
|
||
}
|
||
return settingIds;
|
||
}
|
||
|
||
private async processBatch(queued: RegulatoryMappingBatchRun): Promise<void> {
|
||
const startedAt = this.now().toISOString();
|
||
let run = await this.store.updateRegulatoryMappingBatchRun({ ...queued, status: 'running', startedAt, updatedAt: startedAt });
|
||
try {
|
||
const results = (await this.queryBatchResults(run.projectId, run)).slice()
|
||
.sort((left, right) => left.resultProjectionId.localeCompare(right.resultProjectionId));
|
||
const representatives = new Map<string, UnifiedResultProjection>();
|
||
const counts = { ...emptyRegulatoryMappingBatchCounts(), filteredResultCount: results.length };
|
||
// Findings count like Work records: a GPO setting once, however many baseline comparisons it has.
|
||
const eligible = new Set<string>();
|
||
const unsupported = new Set<string>();
|
||
const settingIds = await this.gpoSettingIds(run.projectId, run, results);
|
||
for (const finding of results) {
|
||
if (!isRegulatoryMappableResult(finding)) { counts.excludedNonScannerCount += 1; continue; }
|
||
const identity = findingIdentity(finding, settingIds.get(finding.resultProjectionId));
|
||
if (!buildRegulatoryFindingQuery(finding)) { unsupported.add(identity); continue; }
|
||
eligible.add(identity);
|
||
const ruleKey = this.ruleKey(finding);
|
||
if (!representatives.has(ruleKey)) representatives.set(ruleKey, finding);
|
||
}
|
||
counts.eligibleFindingCount = eligible.size;
|
||
counts.unsupportedFindingCount = unsupported.size;
|
||
counts.distinctRuleCount = representatives.size;
|
||
const scheduled: Array<Promise<'succeeded' | 'failed'>> = [];
|
||
const pipeline = await this.pipeline(mappingFramework(run));
|
||
for (const finding of representatives.values()) {
|
||
const claimed = await this.claim(finding, false, 'batch', pipeline);
|
||
if (!claimed.claimed) {
|
||
if (claimed.run.status === 'succeeded') counts.reusedSucceededRuleCount += 1;
|
||
else counts.reusedActiveRuleCount += 1;
|
||
continue;
|
||
}
|
||
counts.scheduledRuleCount += 1;
|
||
const completion = this.enqueue(() => this.process(claimed.run, finding, pipeline));
|
||
this.track(completion.then(() => undefined));
|
||
scheduled.push(completion);
|
||
}
|
||
run = await this.store.updateRegulatoryMappingBatchRun({ ...run, counts, updatedAt: this.now().toISOString() });
|
||
let progress = Promise.resolve();
|
||
const completions = scheduled.map(completion => completion.then(status => {
|
||
progress = progress.then(async () => {
|
||
if (status === 'succeeded') counts.succeededRuleCount += 1;
|
||
else counts.failedRuleCount += 1;
|
||
run = await this.store.updateRegulatoryMappingBatchRun({ ...run, counts: { ...counts }, updatedAt: this.now().toISOString() });
|
||
});
|
||
return progress;
|
||
}));
|
||
await Promise.all(completions);
|
||
await progress;
|
||
const completedAt = this.now().toISOString();
|
||
await this.store.updateRegulatoryMappingBatchRun({ ...run, status: 'completed', counts, updatedAt: completedAt, completedAt });
|
||
} catch (error) {
|
||
const completedAt = this.now().toISOString();
|
||
try {
|
||
await this.store.updateRegulatoryMappingBatchRun({
|
||
...run,
|
||
status: 'failed',
|
||
updatedAt: completedAt,
|
||
completedAt,
|
||
errorCode: error instanceof RepositoryConflictError ? 'regulatory_mapping_batch_scope_changed' : 'regulatory_mapping_batch_failed',
|
||
errorDetail: error instanceof RepositoryConflictError ? error.message : 'The batch stopped safely before all rules were scheduled.'
|
||
});
|
||
} catch { /* Preserve the original safe batch failure. */ }
|
||
}
|
||
}
|
||
|
||
private async availability(framework: RegulatoryFramework = 'iso27001'): Promise<{ enabled: boolean; reason?: string }> {
|
||
if (framework === 'bsi-kernel') {
|
||
// The BSI corpus is built from the bundled or updated catalog when the first BSI mapping needs it.
|
||
return this.frameworkEnabled(framework) ? { enabled: true } : { enabled: false, reason: this.disabledError(framework).message };
|
||
}
|
||
if (!this.generationEnabled) return { enabled: false, reason: this.options.disabledReason ?? 'Regulatory mapping is not configured.' };
|
||
try {
|
||
const sample = await this.options.index!.list({
|
||
corpusId: this.options.corpusId,
|
||
corpusVersion: this.options.corpusVersion,
|
||
sourceStandard: 'ISO/IEC 27002',
|
||
retrievalRole: 'semantic'
|
||
}, 1);
|
||
return sample.length ? { enabled: true } : { enabled: false, reason: 'The active regulatory corpus is not indexed.' };
|
||
} catch {
|
||
return { enabled: false, reason: 'Qdrant or the active regulatory corpus is unavailable.' };
|
||
}
|
||
}
|
||
|
||
private async finding(projectId: string, resultProjectionId: string) {
|
||
if (!resultProjectionId?.trim()) throw new RepositoryValidationError('A resultProjectionId is required.');
|
||
// The Result DB row carries no real scope ids; the repository scopes the lookup to the project.
|
||
const finding = await this.read.getUnifiedResultById(resultProjectionId, projectId);
|
||
if (!finding) throw new RepositoryNotFoundError('The selected finding does not exist in this project.');
|
||
return finding;
|
||
}
|
||
|
||
/**
|
||
* The further ranked lists of the hybrid search (user decision 2026-10-09): a dense search with a German restatement
|
||
* of the finding and a keyword search with its German terms. The norm texts are German, so an English query alone
|
||
* matches them weakly. Without a German restatement the mapping keeps the English dense search only.
|
||
*/
|
||
private async hybridRankings(query: RegulatoryFindingQuery, pipeline: MappingPipeline, limit: number) {
|
||
const expansion = this.options.queryExpansionClient ? await this.expandQuery(query) : undefined;
|
||
if (!expansion) return { lists: [] as RegulatoryRetrievalCandidate[][] };
|
||
const german = `${expansion.query} ${expansion.terms.join(' ')}`;
|
||
const lists = [await this.options.findingRetrieval!.retrieveText(german, limit, retrievalFilters(pipeline))];
|
||
const lexical = await this.lexicalIndex(pipeline);
|
||
if (lexical) lists.push(lexical.search(german, limit));
|
||
return { expansion: { ...expansion, searches: lexical ? ['dense-en', 'dense-de', 'keyword-de'] : ['dense-en', 'dense-de'] }, lists };
|
||
}
|
||
|
||
/** One German restatement per public query, shared by the ISO and BSI mapping of a rule; a failure is retried next time. */
|
||
private expandQuery(query: RegulatoryFindingQuery): Promise<Omit<RegulatoryQueryExpansion, 'searches'> | undefined> {
|
||
const key = sha256(query.query);
|
||
const cached = this.expansions.get(key);
|
||
if (cached) return cached;
|
||
const task = this.options.queryExpansionClient!.complete(QUERY_EXPANSION_PROMPT, query.query)
|
||
.then(completion => queryExpansion(completion.value)).catch(() => undefined);
|
||
this.expansions.set(key, task);
|
||
void task.then(value => { if (!value) this.expansions.delete(key); });
|
||
while (this.expansions.size > MAXIMUM_CACHED_EXPANSIONS) this.expansions.delete(this.expansions.keys().next().value!);
|
||
return task;
|
||
}
|
||
|
||
/** The keyword index of one corpus version, built once from Qdrant; unavailable without a full listing. */
|
||
private lexicalIndex(pipeline: MappingPipeline): Promise<RegulatoryLexicalIndex | undefined> {
|
||
const key = `${pipeline.corpusId}\u0000${pipeline.corpusVersion}`;
|
||
const cached = this.lexicalIndexes.get(key);
|
||
if (cached) return cached;
|
||
const listAll = this.options.index!.listAll?.bind(this.options.index);
|
||
const task = listAll ? listAll(retrievalFilters(pipeline)).then(chunks => new RegulatoryLexicalIndex(chunks)).catch(() => undefined) : Promise.resolve(undefined);
|
||
this.lexicalIndexes.set(key, task);
|
||
void task.then(value => { if (!value) this.lexicalIndexes.delete(key); });
|
||
return task;
|
||
}
|
||
|
||
/** The eight best ISO controls of the fused ranking, each with its 27001 statement and full 27002 text. */
|
||
private async groupCandidates(fused: FusedControl[]): Promise<RegulatoryMappingCandidate[]> {
|
||
const usable = (chunk: RegulatoryChunk) => Boolean(chunk.annexAControlId && chunk.sectionKind && chunk.sourcePageStart && chunk.sourcePageEnd && chunk.sourceSha256);
|
||
const selected = fused.map(control => ({ ...control, chunks: control.chunks.filter(usable) }))
|
||
.filter(control => control.chunks.length > 0).slice(0, ISO_CANDIDATE_LIMIT);
|
||
return Promise.all(selected.map(async selection => {
|
||
const semantic = await this.fullControlText(selection.controlId, selection.chunks);
|
||
const references = await this.options.index!.list({
|
||
corpusId: this.options.corpusId,
|
||
corpusVersion: this.options.corpusVersion,
|
||
controlId: selection.controlId,
|
||
sourceStandard: 'ISO/IEC 27001',
|
||
retrievalRole: 'reference'
|
||
}, 1);
|
||
if (references.length !== 1 || !references[0].sourceSha256) {
|
||
throw new RegulatoryMappingServiceError('regulatory_reference_missing', `The active corpus is missing the ISO 27001 reference for ${selection.controlId}.`, 503);
|
||
}
|
||
const first = selection.chunks[0];
|
||
return {
|
||
controlId: selection.controlId,
|
||
annexAControlId: first.annexAControlId!,
|
||
title: first.title,
|
||
score: selection.score,
|
||
evidence: [...references.map(evidence), ...semantic]
|
||
};
|
||
}));
|
||
}
|
||
|
||
/**
|
||
* The ISO/IEC 27002 text of one candidate control for the model (prompt 1.3.0, user decision 2026-10-09): control,
|
||
* purpose and guidance in document order instead of only the two best-matching passages, then other information
|
||
* while it fits. Retrieval only selects the control. The passages that matched are always kept; the rest stops at
|
||
* a per-control cap, so a very long control cannot crowd out the other candidates.
|
||
*/
|
||
private async fullControlText(controlId: string, matched: RegulatoryChunk[]): Promise<RegulatoryMappingEvidence[]> {
|
||
const listed = await this.options.index!.list({ corpusId: this.options.corpusId, corpusVersion: this.options.corpusVersion,
|
||
controlId, sourceStandard: 'ISO/IEC 27002' }, 100);
|
||
const usable = (chunk: RegulatoryChunk) => chunk.controlId === controlId && chunk.sourceStandard === 'ISO/IEC 27002'
|
||
&& Boolean(chunk.sectionKind && chunk.sourcePageStart && chunk.sourcePageEnd && chunk.sourceSha256);
|
||
const byId = new Map([...listed, ...matched].filter(usable).map(chunk => [chunk.chunkId, chunk]));
|
||
const ordered = [...byId.values()].sort((left, right) => SECTION_ORDER.indexOf(left.sectionKind!) - SECTION_ORDER.indexOf(right.sectionKind!)
|
||
|| left.documentId.localeCompare(right.documentId) || left.chunkIndex - right.chunkIndex);
|
||
// Matched passages first in rank order, the best one always; the hybrid search can match many, so they count
|
||
// toward the cap like every other passage.
|
||
const kept = new Set<string>();
|
||
let length = 0;
|
||
for (const chunk of matched.filter(usable)) {
|
||
if (kept.has(chunk.chunkId) || kept.size > 0 && length + chunk.text.length > MAXIMUM_CONTROL_TEXT_CHARACTERS) continue;
|
||
kept.add(chunk.chunkId);
|
||
length += chunk.text.length;
|
||
}
|
||
// Core sections next, other information only with the space left.
|
||
for (const pass of [CORE_SECTIONS, ['other-information']] as const) {
|
||
for (const chunk of ordered) {
|
||
if (kept.has(chunk.chunkId) || !pass.includes(chunk.sectionKind as never)) continue;
|
||
if (length + chunk.text.length > MAXIMUM_CONTROL_TEXT_CHARACTERS) continue;
|
||
kept.add(chunk.chunkId);
|
||
length += chunk.text.length;
|
||
}
|
||
}
|
||
return ordered.filter(chunk => kept.has(chunk.chunkId)).map(evidence);
|
||
}
|
||
|
||
/**
|
||
* The best Kernel requirements of the fused ranking, each with every chunk of its requirement as evidence,
|
||
* so the model always reads the statement even when only its guidance was retrieved.
|
||
*/
|
||
private async groupBsiCandidates(fused: FusedControl[], corpusVersion: string): Promise<RegulatoryMappingCandidate[]> {
|
||
const { catalog } = await this.bsiCatalogs();
|
||
const known = new Map(catalog.requirements.map(requirement => [requirement.id, requirement]));
|
||
const selected = fused.filter(control => known.has(control.controlId) && control.chunks.some(chunk => chunk.corpusId === BSI_CORPUS_ID))
|
||
.slice(0, BSI_CANDIDATE_LIMIT).map(control => [control.controlId, control.score] as const);
|
||
return Promise.all(selected.map(async ([controlId, score]) => {
|
||
const chunks = (await this.options.index!.list({ corpusId: BSI_CORPUS_ID, corpusVersion, controlId }, 3))
|
||
.sort((left, right) => left.chunkIndex - right.chunkIndex);
|
||
if (!chunks.length) throw new RegulatoryMappingServiceError('regulatory_reference_missing', `The BSI corpus is missing requirement ${controlId}.`, 503);
|
||
return { controlId, annexAControlId: controlId, title: known.get(controlId)!.title, score,
|
||
evidence: chunks.map(chunk => bsiEvidence(chunk, catalog.sourceSha256)) };
|
||
}));
|
||
}
|
||
|
||
private async process(queued: RegulatoryMappingRun, finding: UnifiedResultProjection, pipeline: MappingPipeline): Promise<'succeeded' | 'failed'> {
|
||
let run = queued;
|
||
const bsi = pipeline.framework === 'bsi-kernel';
|
||
try {
|
||
let retrieval;
|
||
try {
|
||
// The BSI corpus is indexed from the active catalog the first time a BSI mapping needs it.
|
||
if (bsi && await this.options.bsiCorpus!.ensure() !== pipeline.corpusVersion) {
|
||
throw new Error('The BSI catalog changed while the mapping was queued.');
|
||
}
|
||
retrieval = await this.options.findingRetrieval!.retrieve(finding, bsi ? 40 : 32, retrievalFilters(pipeline));
|
||
} catch {
|
||
throw new RegulatoryMappingServiceError('regulatory_retrieval_unavailable', 'Regulatory retrieval is currently unavailable.', 503);
|
||
}
|
||
if (retrieval.status !== 'candidates' || !retrieval.preparedQuery) {
|
||
throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422);
|
||
}
|
||
const hybrid = await this.hybridRankings(retrieval.preparedQuery, pipeline, bsi ? 40 : 32);
|
||
// The run records the German restatement it searched with.
|
||
const preparedQuery: RegulatoryFindingQuery = hybrid.expansion ? { ...retrieval.preparedQuery, expansion: hybrid.expansion } : retrieval.preparedQuery;
|
||
// BSI only: procurement, service providers, buildings and personnel are not the subject of a technical finding.
|
||
const fused = fuseControlRankings([retrieval.candidates, ...hybrid.lists], bsi ? controlId => BSI_EXCLUDED_PRACTICES.test(controlId) : undefined);
|
||
const candidates = bsi ? await this.groupBsiCandidates(fused, pipeline.corpusVersion) : await this.groupCandidates(fused);
|
||
if (!candidates.length) throw new RegulatoryMappingServiceError('regulatory_corpus_empty', 'The active regulatory corpus returned no eligible controls.', 503);
|
||
const startedAt = this.now().toISOString();
|
||
run = await this.store.saveRegulatoryMappingRun({
|
||
...queued,
|
||
status: 'running',
|
||
preparedQuery,
|
||
queryHash: sha256(JSON.stringify(preparedQuery)),
|
||
sourceHashes: [...new Set(candidates.flatMap(candidate => candidate.evidence.map(evidence => evidence.sourceSha256)))].sort(),
|
||
candidates,
|
||
startedAt,
|
||
updatedAt: startedAt
|
||
});
|
||
const prompt = mappingPrompt(run);
|
||
const completion = await this.options.client!.complete(bsi ? BSI_SYSTEM_PROMPT : SYSTEM_PROMPT, prompt);
|
||
const allowed = new Map(run.candidates.map(candidate => [
|
||
candidate.controlId, new Set(candidate.evidence.map(item => item.chunkId))
|
||
]));
|
||
const grounded = groundRegulatoryMappingResult(completion.value, allowed);
|
||
const errors = regulatoryMappingResultValidationErrors(grounded, allowed);
|
||
if (errors.length) throw new RegulatoryMappingServiceError('regulatory_mapping_invalid_output', errors.join(' '), 502);
|
||
const completedAt = this.now().toISOString();
|
||
await this.store.saveRegulatoryMappingRun({
|
||
...run,
|
||
status: 'succeeded',
|
||
result: grounded as RegulatoryMappingResult,
|
||
...(completion.usage ? { tokenUsage: completion.usage } : {}),
|
||
updatedAt: completedAt,
|
||
completedAt
|
||
});
|
||
await this.autoAcceptFirstProposals(run, grounded as RegulatoryMappingResult);
|
||
return 'succeeded';
|
||
} catch (error) {
|
||
const completedAt = this.now().toISOString();
|
||
const failure = safeFailure(error);
|
||
const detail = safeFailureDetail(error);
|
||
try {
|
||
await this.store.saveRegulatoryMappingRun({
|
||
...run, status: 'failed', updatedAt: completedAt, completedAt, errorCode: failure,
|
||
...(detail ? { errorDetail: detail } : {})
|
||
});
|
||
} catch { /* Preserve the original safe failure code. */ }
|
||
return 'failed';
|
||
}
|
||
}
|
||
}
|
||
|
||
export class RegulatoryMappingServiceError extends Error {
|
||
constructor(readonly code: string, message: string, readonly status: number) { super(message); this.name = 'RegulatoryMappingServiceError'; }
|
||
}
|
||
|
||
// Response projection: the persisted run keeps the full ISO clause text (the model reads it),
|
||
// but the copyrighted text is never shipped to the browser — only the citation fields are.
|
||
function withoutEvidenceText(run: RegulatoryMappingRun): RegulatoryMappingRun {
|
||
return {
|
||
...run,
|
||
candidates: run.candidates.map(candidate => ({
|
||
...candidate,
|
||
evidence: candidate.evidence.map(({ text, ...reference }) => reference)
|
||
}))
|
||
};
|
||
}
|
||
|
||
function evidence(chunk: RegulatoryRetrievalCandidate['chunk']): RegulatoryMappingEvidence {
|
||
return {
|
||
chunkId: chunk.chunkId,
|
||
textSha256: chunk.textSha256,
|
||
sourceSha256: chunk.sourceSha256!,
|
||
sourceStandard: chunk.sourceStandard!,
|
||
sectionKind: chunk.sectionKind!,
|
||
pageStart: chunk.sourcePageStart!,
|
||
pageEnd: chunk.sourcePageEnd!,
|
||
text: chunk.text
|
||
};
|
||
}
|
||
|
||
function bsiEvidence(chunk: RegulatoryChunk, sourceSha256: string): RegulatoryMappingEvidence {
|
||
return { chunkId: chunk.chunkId, textSha256: chunk.textSha256, sourceSha256, sourceStandard: BSI_SOURCE_STANDARD, sectionKind: 'control', text: chunk.text };
|
||
}
|
||
|
||
function mappingPrompt(run: RegulatoryMappingRun): string {
|
||
// The Annex A id is always "A." + controlId, so it is redundant for the model and
|
||
// sending it invites a reply that uses the Annex A form; expose only the controlId it must return.
|
||
// A BSI candidate repeats its requirement id there, and the BSI evidence has no pages.
|
||
const candidates = run.candidates.map(({ annexAControlId, ...candidate }) => candidate);
|
||
const payload = JSON.stringify({ finding: run.preparedQuery, candidates });
|
||
if (payload.length > MAXIMUM_PROMPT_CHARACTERS) throw new RegulatoryMappingServiceError('regulatory_mapping_prompt_too_large', 'The bounded regulatory prompt exceeded its limit.', 422);
|
||
return payload;
|
||
}
|
||
|
||
const SYSTEM_PROMPT = `Map one public security finding query to the supplied ISO control candidates. Each candidate carries its ISO/IEC 27001 control statement and the ISO/IEC 27002 control, purpose and guidance text of that control in document order; read the whole text before judging. A mapping means the control is topically relevant to the finding, never proof of compliance. Map a candidate when it governs the same security subject as the finding (for example password or authentication management, logging, or access restriction), even when the control text does not state the finding's specific value, threshold, parameter, or configuration; ISO controls are principle-level and rarely restate such specifics. Judge relevance from the control's subject and its supplied evidence, not from whether the exact requirement text appears. Return JSON only with schemaVersion 1, outcome mapped|no_relevant_control|insufficient_context, optional bounded reason, and mappings (maximum five). Each mapping requires controlId, relationship primary|supporting, confidence low|medium|high, a concise rationale under 600 characters, and one to three evidenceChunkIds copied only from that control's supplied evidence. Use no_relevant_control only when no candidate governs the finding's subject, and insufficient_context only when the finding query itself lacks enough meaning to judge. Do not invent controls or evidence.`;
|
||
|
||
const BSI_SYSTEM_PROMPT = `Map one public security finding query to the supplied BSI Grundschutz++ Kernel requirement candidates. The requirements are published in German; MUSS, SOLLTE and KANN state their obligation level. A mapping means the requirement is topically relevant to the finding, never proof of compliance. Map a candidate when it governs the same security subject as the finding (for example authentication, hardening of a configuration, logging, or restriction of access), even when the requirement does not restate the finding's specific value, threshold, parameter, or configuration. Prefer the requirements that address the finding's specific subject over general management or process requirements. Judge relevance from the requirement's subject and its supplied evidence. Return JSON only with schemaVersion 1, outcome mapped|no_relevant_control|insufficient_context, optional bounded reason, and mappings (maximum five). Each mapping requires controlId (the requirement id exactly as supplied, for example KONF.2.1), relationship primary|supporting, confidence low|medium|high, a concise English rationale under 600 characters, and one to three evidenceChunkIds copied only from that requirement's supplied evidence. Use no_relevant_control only when no candidate governs the finding's subject, and insufficient_context only when the finding query itself lacks enough meaning to judge. Do not invent requirements or evidence.`;
|
||
|
||
function byNumber(left: string, right: string): number {
|
||
return left.localeCompare(right, 'en', { numeric: true });
|
||
}
|
||
|
||
function safeFailure(error: unknown): string {
|
||
if (error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError) return error.code;
|
||
return 'regulatory_mapping_failed';
|
||
}
|
||
|
||
// Our own service and client error messages are developer-authored and structural
|
||
// (field paths, ISO control numbers) — never finding, tenant, or report content.
|
||
// Clamp the length and strip any markup-like sequence before it is stored or shown.
|
||
function safeFailureDetail(error: unknown): string | undefined {
|
||
if (!(error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError)) return undefined;
|
||
const detail = error.message.replace(/<\/?[a-z][^>]*>/gi, ' ').replace(/\s+/g, ' ').trim().slice(0, 600);
|
||
return detail || undefined;
|
||
}
|
||
|
||
/**
|
||
* One finding as Work counts it: a scanner finding by occurrence, a GPO setting by scope and technical
|
||
* setting identity. The Result DB withholds ASSESSOR's internal identity hash, which the engine reads
|
||
* separately; only if that is unavailable does the disclosed title, the technical setting identity, stand in.
|
||
*/
|
||
function findingIdentity(finding: UnifiedResultProjection, settingId?: string): string {
|
||
if (finding.sourceKind === 'scanner-finding') return `scanner:${finding.sourceResultId}`;
|
||
return `gpo:${finding.policyScope ?? ''}:${settingId ?? (finding.sourceRuleId?.trim() || `title:${finding.title.trim()}`)}`;
|
||
}
|
||
|
||
function isGpoRuleKey(ruleKey: string): boolean {
|
||
return ruleKey.startsWith('GPOs:') || ruleKey.startsWith('GPOs#');
|
||
}
|
||
|
||
/** Content-free origin tokens of a client and its timeslot; equal inputs give equal tokens. */
|
||
function originTokens(projectId: string, scope: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>): { project: string; timeslot: string[] } {
|
||
return {
|
||
project: sha256(`origin-project:${projectId}`),
|
||
timeslot: [...scope.assessmentIds.map(id => sha256(`origin-assessment:${projectId}:${id}`)),
|
||
...scope.scanRunIds.map(id => sha256(`origin-scan-run:${projectId}:${id}`))].sort()
|
||
};
|
||
}
|
||
|
||
function sha256(value: string): string {
|
||
return createHash('sha256').update(value, 'utf8').digest('hex');
|
||
}
|