Assessor_2.0/src/compliance/regulatory-mapping.ts

468 lines
23 KiB
TypeScript

import type { AiOverviewTokenUsage } from './ai-overview';
import type { RegulatoryFindingQuery, RegulatorySectionKind } from './regulatory-rag';
import type { ResultDbFilters } from '../domain/result-db-query';
import type { UnifiedResultProjection } from '../domain/unified-result';
export const REGULATORY_MAPPING_PROMPT_VERSION = '1.2.0';
export const REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION = 1 as const;
export const REGULATORY_RETRIEVAL_VERSION = 'dense-control-grouping-1.1.0';
/** Version of the GPO comparison context in the query; only GPO rules are regenerated when it changes. */
export const REGULATORY_GPO_QUERY_VERSION = 'gpo-baseline-context-1.1.0';
export type RegulatoryMappingRunStatus = 'queued' | 'running' | 'succeeded' | 'failed';
export type RegulatoryMappingRelationship = 'primary' | 'supporting';
export type RegulatoryMappingConfidence = 'low' | 'medium' | 'high';
export type RegulatoryMappingDecisionValue = 'accepted' | 'rejected';
export type RegulatoryMappingBatchStatus = 'queued' | 'running' | 'completed' | 'failed';
export type RegulatoryMappingBatchFilters = ResultDbFilters;
export interface RegulatoryMappingBatchScope {
assessmentIds: string[];
scanRunIds: string[];
filters: RegulatoryMappingBatchFilters;
}
export interface RegulatoryMappingBatchCounts {
filteredResultCount: number;
excludedNonScannerCount: number;
eligibleFindingCount: number;
unsupportedFindingCount: number;
distinctRuleCount: number;
reusedSucceededRuleCount: number;
reusedActiveRuleCount: number;
scheduledRuleCount: number;
succeededRuleCount: number;
failedRuleCount: number;
}
export interface RegulatoryMappingBatchRun extends RegulatoryMappingBatchScope {
version: 1;
id: string;
projectId: string;
scopeKey: string;
filterHash: string;
idempotencyKey: string;
status: RegulatoryMappingBatchStatus;
counts: RegulatoryMappingBatchCounts;
createdAt: string;
updatedAt: string;
startedAt?: string;
completedAt?: string;
errorCode?: string;
errorDetail?: string;
}
export interface RegulatoryMappingBatchResponse {
version: 1;
projectId: string;
scope: RegulatoryMappingBatchScope;
generationEnabled: boolean;
state: 'empty' | RegulatoryMappingBatchStatus;
disabledReason?: string;
pollAfterMs?: number;
run?: RegulatoryMappingBatchRun;
}
export interface RegulatoryMappingBatchCreateRequest extends RegulatoryMappingBatchScope {
idempotencyKey: string;
}
export interface RegulatoryMappingEvidence {
chunkId: string;
textSha256: string;
sourceSha256: string;
sourceStandard: 'ISO/IEC 27001' | 'ISO/IEC 27002';
sectionKind: RegulatorySectionKind;
pageStart: number;
pageEnd: number;
// Full ISO clause text. Required in the persisted run (the model reads it to reason),
// but omitted from the API response projection so the copyrighted text never reaches the browser.
text?: string;
}
export interface RegulatoryMappingCandidate {
controlId: string;
annexAControlId: string;
title: string;
score: number;
evidence: RegulatoryMappingEvidence[];
}
export interface RegulatoryMappingProposal {
controlId: string;
relationship: RegulatoryMappingRelationship;
confidence: RegulatoryMappingConfidence;
rationale: string;
evidenceChunkIds: string[];
}
export interface RegulatoryMappingResult {
schemaVersion: 1;
outcome: 'mapped' | 'no_relevant_control' | 'insufficient_context';
reason?: string;
mappings: RegulatoryMappingProposal[];
}
/**
* A global, tenant-independent mapping of one scanner rule to ISO controls. Keyed by
* `ruleKey` (the rule identity `tool:ruleId`, or a stable title hash for rules without
* an id) plus the pipeline versions, so the same rule maps once and every occurrence in
* every project reuses it. Never carries project, tenant, or occurrence identity.
*/
export interface RegulatoryMappingRun {
version: 1;
id: string;
ruleKey: string;
sourceTool: string;
sourceRuleId?: string;
status: RegulatoryMappingRunStatus;
queryHash: string;
preparedQuery: RegulatoryFindingQuery;
corpusId: string;
corpusVersion: string;
sourceHashes: string[];
embeddingModel: string;
retrievalVersion: string;
promptVersion: string;
outputSchemaVersion: 1;
model: string;
/** How the run was started: a single finding (`Map ISO controls`) or a batch. Absent on runs before 2.21.0. */
trigger?: 'single' | 'batch';
/**
* Where a single mapping was started, as SHA-256 tokens of the client and of each assessment and scan run of
* its timeslot. Runs are global and exported with every client, so no client or timeslot id is stored.
*/
origin?: { project: string; timeslot: string[] };
createdAt: string;
updatedAt: string;
startedAt?: string;
completedAt?: string;
tokenUsage?: AiOverviewTokenUsage;
errorCode?: string;
errorDetail?: string;
candidates: RegulatoryMappingCandidate[];
result?: RegulatoryMappingResult;
}
/** A global accept/reject decision, keyed by rule identity and control. Shared across all projects. */
export interface RegulatoryMappingDecision {
version: 1;
id: string;
ruleKey: string;
sourceTool: string;
sourceRuleId?: string;
mappingId: string;
controlId: string;
decision: RegulatoryMappingDecisionValue;
comment?: string;
previousDecisionId?: string;
createdAt: string;
}
/**
* Control ids whose latest decision is 'accepted'. Each stored decision names the one it replaced
* (`previousDecisionId`), so the latest is the one no other decision replaced; time and id only break
* a tie between unchained entries.
*/
export function acceptedControlIds(decisions: readonly RegulatoryMappingDecision[]): string[] {
const replaced = new Set(decisions.flatMap(item => item.previousDecisionId ? [item.previousDecisionId] : []));
const latest = new Map<string, RegulatoryMappingDecision>();
for (const decision of decisions) {
if (replaced.has(decision.id)) continue;
const current = latest.get(decision.controlId);
if (!current || decision.createdAt > current.createdAt || decision.createdAt === current.createdAt && decision.id > current.id) {
latest.set(decision.controlId, decision);
}
}
return [...latest.values()].filter(item => item.decision === 'accepted').map(item => item.controlId);
}
export type RegulatoryControlTheme = 'Organizational' | 'People' | 'Physical' | 'Technological';
/** ISO/IEC 27002:2022 groups its controls by clause: 5 Organizational, 6 People, 7 Physical, 8 Technological. */
export function regulatoryControlTheme(controlId: string): RegulatoryControlTheme | undefined {
return ({ 5: 'Organizational', 6: 'People', 7: 'Physical', 8: 'Technological' } as const)[Number(controlId.split('.')[0]) as 5 | 6 | 7 | 8];
}
/** A Result DB row named precisely enough to open its Work record. */
export type RegulatoryControlFinding = Pick<UnifiedResultProjection, 'resultProjectionId' | 'sourceKind' | 'sourceResultId' | 'sourceRuleId' | 'policyScope'>;
export interface RegulatoryControlOverviewItem {
controlId: string;
annexAControlId: string;
title?: string;
theme?: RegulatoryControlTheme;
findings: RegulatoryControlFinding[];
}
/** Accepted controls of the deviations in one client timeslot; decisions stay global per scanner rule. */
/** Idempotency-key prefixes of batches the automatic mapping starts; every other batch was started manually. */
export const AUTO_MAPPING_BATCH_KEY_PREFIXES = ['auto:', 'auto-gpo:'] as const;
export const MAPPING_HISTORY_LIMIT = 50;
/**
* Content-free entry of the Client Info `Last mapping` window, listed globally: a batch of any client or a
* single finding mapping, which is global per rule.
*/
export interface RegulatoryMappingHistoryEntry {
id: string;
kind: 'batch' | 'single';
/** Where the mapping was started, relative to the shown timeslot; `unknown` for a single mapping without a recorded origin. */
scope: 'timeslot' | 'other-timeslot' | 'other-client' | 'unknown';
/** Automatic only for batches the auto-mapping watcher started. */
trigger: 'automatic' | 'manual';
status: RegulatoryMappingBatchStatus;
createdAt: string;
completedAt?: string;
/** `findings` only for a batch: the eligible findings of its own scope. */
counts: { findings?: number; sentToModel: number; reused: number; succeeded: number; failed: number };
}
export interface RegulatoryControlOverviewResponse {
version: 1;
projectId: string;
scope: { assessmentIds: string[]; scanRunIds: string[] };
/** Distinct findings that carry at least one accepted control. */
findingCount: number;
controls: RegulatoryControlOverviewItem[];
/** Newest first, at most MAPPING_HISTORY_LIMIT; only on the Client Info overview. */
mappingHistory?: RegulatoryMappingHistoryEntry[];
}
export interface RegulatoryMappingResponse {
version: 1;
projectId: string;
resultProjectionId: string;
generationEnabled: boolean;
state: 'empty' | RegulatoryMappingRunStatus | 'unsupported';
disabledReason?: string;
pollAfterMs?: number;
run?: RegulatoryMappingRun;
decisions: RegulatoryMappingDecision[];
/**
* Controls decided for this rule that the current run did not retrieve (for example after a query
* version change), so an earlier accept stays reviewable. Titles come from older runs of the rule.
*/
earlierControls?: RegulatoryEarlierControl[];
}
export interface RegulatoryEarlierControl {
controlId: string;
annexAControlId: string;
title?: string;
}
export interface RegulatoryMappingCreateRequest {
resultProjectionId: string;
/** The timeslot the Finding Reader shows; the run keeps it only as hashes, for the mapping history scope. */
origin?: { assessmentIds: string[]; scanRunIds: string[] };
}
export interface RegulatoryMappingDecisionCreateRequest {
resultProjectionId: string;
controlId: string;
decision: RegulatoryMappingDecisionValue;
comment?: string;
expectedLatestDecisionId?: string;
}
/**
* Trims each mapping's evidence citations to the exact chunk ids supplied for that
* control, tolerating a model that miscopies or mis-scopes one of several long
* compound chunk ids. This only removes ungrounded ids — it never invents evidence —
* and drops a mapping that, for a known control, cited no real evidence at all.
* Malformed or unknown-control mappings are left untouched for the validator to report.
*/
export function groundRegulatoryMappingResult(
value: unknown,
allowedCandidates: ReadonlyMap<string, ReadonlySet<string>>
): unknown {
if (!isObject(value) || !Array.isArray(value.mappings)) return value;
const mappings: unknown[] = [];
for (const mapping of value.mappings) {
const allowed = isObject(mapping) && typeof mapping.controlId === 'string' ? allowedCandidates.get(mapping.controlId) : undefined;
if (!allowed || !Array.isArray((mapping as Record<string, unknown>).evidenceChunkIds)) { mappings.push(mapping); continue; }
const grounded = [...new Set(((mapping as Record<string, unknown>).evidenceChunkIds as unknown[])
.filter((id): id is string => typeof id === 'string' && allowed.has(id)))].slice(0, 3);
if (grounded.length) mappings.push({ ...(mapping as Record<string, unknown>), evidenceChunkIds: grounded });
}
return { ...value, mappings };
}
export function regulatoryMappingResultValidationErrors(
value: unknown,
allowedCandidates?: ReadonlyMap<string, ReadonlySet<string>>
): string[] {
const errors: string[] = [];
if (!isObject(value)) return ['Regulatory mapping output must be a JSON object.'];
if (value.schemaVersion !== 1) errors.push('schemaVersion must be 1.');
if (!['mapped', 'no_relevant_control', 'insufficient_context'].includes(String(value.outcome))) errors.push('outcome is invalid.');
if (!Array.isArray(value.mappings) || value.mappings.length > 5) errors.push('mappings must contain at most five entries.');
else {
const controls = new Set<string>();
value.mappings.forEach((mapping, index) => {
if (!isObject(mapping)) { errors.push(`mappings[${index}] must be an object.`); return; }
const controlId = boundedString(mapping.controlId, 20, `mappings[${index}].controlId`, errors);
enumeration(mapping.relationship, ['primary', 'supporting'], `mappings[${index}].relationship`, errors);
enumeration(mapping.confidence, ['low', 'medium', 'high'], `mappings[${index}].confidence`, errors);
boundedString(mapping.rationale, 600, `mappings[${index}].rationale`, errors);
if (controlId && controls.has(controlId)) errors.push(`mappings[${index}] duplicates control ${controlId}.`);
if (controlId) controls.add(controlId);
const allowedEvidence = controlId ? allowedCandidates?.get(controlId) : undefined;
if (!Array.isArray(mapping.evidenceChunkIds) || mapping.evidenceChunkIds.length < 1 || mapping.evidenceChunkIds.length > 3) {
errors.push(`mappings[${index}].evidenceChunkIds must contain one to three entries.`);
} else mapping.evidenceChunkIds.forEach((chunkId, evidenceIndex) => {
if (typeof chunkId !== 'string' || !allowedEvidence?.has(chunkId)) errors.push(`mappings[${index}].evidenceChunkIds[${evidenceIndex}] is not grounded in the supplied control.`);
});
if (controlId && allowedCandidates && !allowedCandidates.has(controlId)) errors.push(`mappings[${index}] references an unknown candidate control.`);
});
}
if (value.outcome === 'mapped' && (!Array.isArray(value.mappings) || !value.mappings.length)) errors.push('mapped outcome requires at least one mapping.');
if (value.outcome !== 'mapped' && Array.isArray(value.mappings) && value.mappings.length) errors.push('A refusal outcome must not include mappings.');
if (value.outcome !== 'mapped') boundedString(value.reason, 600, 'reason', errors);
return errors;
}
export function regulatoryMappingRunValidationErrors(run: RegulatoryMappingRun): string[] {
const errors: string[] = [];
for (const value of [run.id, run.ruleKey, run.sourceTool, run.corpusId, run.corpusVersion, run.embeddingModel, run.model]) {
if (!value?.trim()) errors.push('Regulatory mapping run identity is incomplete.');
}
if (run.ruleKey && run.ruleKey.length > 200) errors.push('Regulatory mapping ruleKey is too long.');
if (run.sourceRuleId !== undefined && (!run.sourceRuleId.trim() || run.sourceRuleId.length > 200)) errors.push('Regulatory mapping sourceRuleId is invalid.');
if (!['queued', 'running', 'succeeded', 'failed'].includes(run.status)) errors.push('Regulatory mapping run status is invalid.');
if (!/^[a-f0-9]{64}$/.test(run.queryHash)) errors.push('Regulatory mapping queryHash must be SHA-256.');
const candidatesRequired = run.status === 'running' || run.status === 'succeeded';
if ((!run.sourceHashes.length && candidatesRequired) || run.sourceHashes.some(value => !/^[a-f0-9]{64}$/.test(value))) errors.push('Regulatory mapping sourceHashes must contain SHA-256 digests.');
if (!Array.isArray(run.candidates) || (candidatesRequired && run.candidates.length < 1) || run.candidates.length > 8) errors.push('Regulatory mapping candidates must contain one to eight controls when processing has started.');
else {
const controls = new Set<string>();
run.candidates.forEach(candidate => {
if (!/^[5-8]\.\d{1,2}$/.test(candidate.controlId) || candidate.annexAControlId !== `A.${candidate.controlId}`) errors.push('Regulatory mapping candidate control identity is invalid.');
if (controls.has(candidate.controlId)) errors.push(`Regulatory mapping candidate ${candidate.controlId} is duplicated.`);
controls.add(candidate.controlId);
if (!candidate.title.trim() || candidate.title.length > 500 || !Number.isFinite(candidate.score)) errors.push(`Regulatory mapping candidate ${candidate.controlId} metadata is invalid.`);
if (!candidate.evidence.length || candidate.evidence.length > 3) errors.push(`Regulatory mapping candidate ${candidate.controlId} must contain one to three evidence chunks.`);
candidate.evidence.forEach(evidence => {
if (!evidence.chunkId.trim() || !/^[a-f0-9]{64}$/.test(evidence.textSha256) || !/^[a-f0-9]{64}$/.test(evidence.sourceSha256)
|| !Number.isInteger(evidence.pageStart) || !Number.isInteger(evidence.pageEnd) || evidence.pageStart < 1 || evidence.pageEnd < evidence.pageStart
|| typeof evidence.text !== 'string' || !evidence.text.trim() || evidence.text.length > 20_000) errors.push(`Regulatory mapping candidate ${candidate.controlId} evidence is invalid.`);
});
});
}
if (run.status === 'succeeded' && !run.result) errors.push('A succeeded regulatory mapping run requires a result.');
if (run.result) errors.push(...regulatoryMappingResultValidationErrors(run.result, candidateEvidence(run.candidates)));
return [...new Set(errors)];
}
export function normalizeRegulatoryMappingBatchScope(scope: RegulatoryMappingBatchScope): RegulatoryMappingBatchScope {
const values = (input: readonly string[] | undefined) => [...new Set((input ?? []).map(value => value.trim()).filter(Boolean))]
.sort((left, right) => left.localeCompare(right));
const text = (input: string | undefined) => input?.trim() || undefined;
const filters = scope.filters ?? {};
return {
assessmentIds: values(scope.assessmentIds),
scanRunIds: values(scope.scanRunIds),
filters: {
evaluationRunId: text(filters.evaluationRunId),
sourceKind: values(filters.sourceKind) as RegulatoryMappingBatchFilters['sourceKind'],
sourceTool: values(filters.sourceTool),
severity: values(filters.severity) as RegulatoryMappingBatchFilters['severity'],
outcome: values(filters.outcome) as RegulatoryMappingBatchFilters['outcome'],
technicalExecutionState: values(filters.technicalExecutionState) as RegulatoryMappingBatchFilters['technicalExecutionState'],
resultType: values(filters.resultType) as RegulatoryMappingBatchFilters['resultType'],
policyClassification: values(filters.policyClassification) as RegulatoryMappingBatchFilters['policyClassification'],
isIoe: filters.isIoe,
deviation: filters.deviation,
mappingRuleVersion: values(filters.mappingRuleVersion),
sourceProfileVersion: values(filters.sourceProfileVersion),
processingStatus: values(filters.processingStatus) as RegulatoryMappingBatchFilters['processingStatus'],
reviewDisposition: values(filters.reviewDisposition) as RegulatoryMappingBatchFilters['reviewDisposition'],
lifecycleState: values(filters.lifecycleState) as RegulatoryMappingBatchFilters['lifecycleState'],
categoryNamespace: values(filters.categoryNamespace),
categoryValue: values(filters.categoryValue),
affectedAsset: text(filters.affectedAsset),
environmentState: values(filters.environmentState) as RegulatoryMappingBatchFilters['environmentState'],
environmentProperty: values(filters.environmentProperty),
environmentSubjectType: values(filters.environmentSubjectType),
search: text(filters.search),
observedFrom: text(filters.observedFrom),
observedTo: text(filters.observedTo)
}
};
}
export function emptyRegulatoryMappingBatchCounts(): RegulatoryMappingBatchCounts {
return {
filteredResultCount: 0,
excludedNonScannerCount: 0,
eligibleFindingCount: 0,
unsupportedFindingCount: 0,
distinctRuleCount: 0,
reusedSucceededRuleCount: 0,
reusedActiveRuleCount: 0,
scheduledRuleCount: 0,
succeededRuleCount: 0,
failedRuleCount: 0
};
}
export function regulatoryMappingBatchRunValidationErrors(run: RegulatoryMappingBatchRun): string[] {
const errors: string[] = [];
for (const value of [run.id, run.projectId, run.scopeKey, run.filterHash, run.idempotencyKey]) {
if (!value?.trim()) errors.push('Regulatory mapping batch identity is incomplete.');
}
if (!/^[a-f0-9]{64}$/.test(run.filterHash)) errors.push('Regulatory mapping batch filterHash must be SHA-256.');
if (!/^[A-Za-z0-9._:-]{8,128}$/.test(run.idempotencyKey)) errors.push('Regulatory mapping batch idempotencyKey is invalid.');
if (!['queued', 'running', 'completed', 'failed'].includes(run.status)) errors.push('Regulatory mapping batch status is invalid.');
const normalized = normalizeRegulatoryMappingBatchScope(run);
if (!normalized.assessmentIds.length && !normalized.scanRunIds.length) errors.push('Regulatory mapping batch requires a selected timeslot.');
const counts = Object.values(run.counts);
if (counts.some(value => !Number.isInteger(value) || value < 0)) errors.push('Regulatory mapping batch counts must be non-negative integers.');
if (run.counts.filteredResultCount !== run.counts.excludedNonScannerCount + run.counts.eligibleFindingCount + run.counts.unsupportedFindingCount) {
errors.push('Regulatory mapping batch result counts are inconsistent.');
}
if (run.counts.distinctRuleCount !== run.counts.reusedSucceededRuleCount + run.counts.reusedActiveRuleCount + run.counts.scheduledRuleCount) {
errors.push('Regulatory mapping batch rule counts are inconsistent.');
}
if (run.counts.succeededRuleCount + run.counts.failedRuleCount > run.counts.scheduledRuleCount) {
errors.push('Regulatory mapping batch completion counts exceed scheduled rules.');
}
if (run.errorDetail !== undefined && (run.errorDetail.length > 600 || /<\/?[a-z][^>]*>/i.test(run.errorDetail))) {
errors.push('Regulatory mapping batch errorDetail must be plain text with at most 600 characters.');
}
return [...new Set(errors)];
}
export function regulatoryMappingDecisionValidationErrors(decision: RegulatoryMappingDecision): string[] {
const errors: string[] = [];
for (const value of [decision.id, decision.ruleKey, decision.sourceTool, decision.mappingId, decision.controlId]) {
if (!value?.trim()) errors.push('Regulatory mapping decision identity is incomplete.');
}
if (decision.sourceRuleId !== undefined && (!decision.sourceRuleId.trim() || decision.sourceRuleId.length > 200)) errors.push('Regulatory mapping decision sourceRuleId is invalid.');
if (!['accepted', 'rejected'].includes(decision.decision)) errors.push('Regulatory mapping decision is invalid.');
if (decision.comment !== undefined && (decision.comment.length > 1_000 || /<\/?[a-z][^>]*>/i.test(decision.comment))) errors.push('Regulatory mapping decision comment must be plain text with at most 1000 characters.');
return [...new Set(errors)];
}
function candidateEvidence(candidates: RegulatoryMappingCandidate[]): Map<string, Set<string>> {
return new Map(candidates.map(candidate => [candidate.controlId, new Set(candidate.evidence.map(evidence => evidence.chunkId))]));
}
function boundedString(value: unknown, maximum: number, path: string, errors: string[]): string | undefined {
if (typeof value !== 'string' || !value.trim() || value.length > maximum || /<\/?[a-z][^>]*>/i.test(value)) {
errors.push(`${path} must be non-empty plain text with at most ${maximum} characters.`);
return undefined;
}
return value;
}
function enumeration(value: unknown, allowed: readonly string[], path: string, errors: string[]): void {
if (typeof value !== 'string' || !allowed.includes(value)) errors.push(`${path} is invalid.`);
}
function isObject(value: unknown): value is Record<string, any> {
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
}