Assessor_2.0/src/comparison/comparison-engine.ts

426 lines
20 KiB
TypeScript

import type {
CanonicalAssessmentModel,
CanonicalFieldRole,
EvidenceRecord,
FindingDefinition,
FindingOccurrence,
JsonValue,
ScanRun
} from '../domain/canonical-model';
import type {
ComparisonMatch,
ComparisonMatchMethod,
ComparisonOptions,
ComparisonSummary,
FindingChange,
FindingComparison,
LocalComparisonRule,
ScanRunComparison
} from '../domain/comparison';
import { jaccardSimilarity, normalizeComparisonText, similarityTokens, stableJson } from './comparison-normalization';
import { localRuleConnects, validateLocalComparisonRules } from './local-mapping-rules';
import { definitionHasDurableIdentity } from '../domain/finding-identity';
import { resolvedCompatibilityKey } from '../domain/processing-compatibility';
interface ComparisonContext {
definitions: Map<string, FindingDefinition>;
evidence: Map<string, EvidenceRecord[]>;
options: ComparisonOptions;
}
interface MatchedPair {
baseline: FindingOccurrence;
target: FindingOccurrence;
match: ComparisonMatch;
}
interface SuggestedPair extends MatchedPair {
score: number;
}
type KeyFactory = (occurrence: FindingOccurrence, context: ComparisonContext) => string[];
const lifecycleOrder: Record<FindingComparison['lifecycle'], number> = {
persistent: 0,
reoccurred: 1,
new: 2,
resolved: 3,
'possibly-identical': 4,
'not-comparable': 5
};
function normalizedRuleId(occurrence: FindingOccurrence): string {
return normalizeComparisonText(occurrence.sourceIdentity.canonicalRuleId ?? occurrence.sourceIdentity.originalRuleId);
}
function normalizedToolId(occurrence: FindingOccurrence): string {
return normalizeComparisonText(occurrence.sourceIdentity.tool.id);
}
function mappedValue(occurrence: FindingOccurrence, role: CanonicalFieldRole): JsonValue | undefined {
return occurrence.fieldMappings.find(mapping => mapping.targetRole === role)?.normalizedValue;
}
function normalizedRoleValue(occurrence: FindingOccurrence, role: CanonicalFieldRole, context: ComparisonContext): string {
if (role === 'rule-id') return normalizedRuleId(occurrence);
if (role === 'title') return normalizeComparisonText(
mappedValue(occurrence, role)
?? context.definitions.get(occurrence.definitionId)?.title
?? occurrence.sourceIdentity.originalTitle
);
if (role === 'asset') return normalizeComparisonText(occurrence.affectedAsset?.normalizedValue ?? mappedValue(occurrence, role));
return normalizeComparisonText(mappedValue(occurrence, role));
}
function canonicalDefinitionKeys(occurrence: FindingOccurrence, context: ComparisonContext): string[] {
const definition = context.definitions.get(occurrence.definitionId);
if (!definitionHasDurableIdentity(definition, occurrence)) return [];
const canonicalKey = normalizeComparisonText(definition?.canonicalKey);
return [
occurrence.definitionId ? `definition:${occurrence.definitionId}` : '',
canonicalKey ? `canonical:${canonicalKey}` : ''
].filter(Boolean);
}
function toolRuleKeys(occurrence: FindingOccurrence, context?: ComparisonContext): string[] {
const definition = context?.definitions.get(occurrence.definitionId);
if (definition?.identity?.strategy === 'unresolved') return [];
const tool = normalizedToolId(occurrence);
const rule = normalizedRuleId(occurrence);
return tool && rule ? [`tool-rule:${tool}|${rule}`] : [];
}
function frameworkKeys(occurrence: FindingOccurrence, context: ComparisonContext): string[] {
const definition = context.definitions.get(occurrence.definitionId);
if (!definitionHasDurableIdentity(definition, occurrence)) return [];
return (definition?.frameworkReferences ?? [])
.map(reference => {
const framework = normalizeComparisonText(reference.framework);
const control = normalizeComparisonText(reference.controlId);
return framework && control ? `framework:${framework}|${control}` : '';
})
.filter(Boolean)
.sort();
}
function groupsByKey(occurrences: FindingOccurrence[], keyFactory: KeyFactory, context: ComparisonContext): Map<string, FindingOccurrence[]> {
const groups = new Map<string, FindingOccurrence[]>();
occurrences.forEach(occurrence => {
[...new Set(keyFactory(occurrence, context))].forEach(key => {
if (!key) return;
groups.set(key, [...(groups.get(key) ?? []), occurrence]);
});
});
return groups;
}
function match(method: ComparisonMatchMethod, confidence: number, reason: string, decision: ComparisonMatch['decision'] = 'automatic'): ComparisonMatch {
return { method, confidence, reasons: [reason], decision };
}
function uniqueStageMatches(
baseline: FindingOccurrence[],
target: FindingOccurrence[],
keyFactory: KeyFactory,
context: ComparisonContext,
method: ComparisonMatchMethod,
confidence: number,
reason: string
): MatchedPair[] {
const leftGroups = groupsByKey(baseline, keyFactory, context);
const rightGroups = groupsByKey(target, keyFactory, context);
const candidatePairs = [...leftGroups.keys()].sort().flatMap(key => {
const left = leftGroups.get(key) ?? [];
const right = rightGroups.get(key) ?? [];
return left.length === 1 && right.length === 1 ? [{ baseline: left[0], target: right[0], key }] : [];
});
const deduplicated = new Map<string, typeof candidatePairs[number]>();
candidatePairs.forEach(pair => deduplicated.set(`${pair.baseline.id}\u0000${pair.target.id}`, pair));
const candidates = [...deduplicated.values()];
const baselineCounts = new Map<string, number>();
const targetCounts = new Map<string, number>();
candidates.forEach(pair => {
baselineCounts.set(pair.baseline.id, (baselineCounts.get(pair.baseline.id) ?? 0) + 1);
targetCounts.set(pair.target.id, (targetCounts.get(pair.target.id) ?? 0) + 1);
});
return candidates
.filter(pair => baselineCounts.get(pair.baseline.id) === 1 && targetCounts.get(pair.target.id) === 1)
.sort((left, right) => left.baseline.id.localeCompare(right.baseline.id) || left.target.id.localeCompare(right.target.id))
.map(pair => ({ baseline: pair.baseline, target: pair.target, match: match(method, confidence, reason) }));
}
function localRuleMatches(
baseline: FindingOccurrence[],
target: FindingOccurrence[],
rules: LocalComparisonRule[],
context: ComparisonContext
): MatchedPair[] {
const candidates = rules.slice().sort((left, right) => left.id.localeCompare(right.id)).flatMap(rule => {
const pairs = baseline.flatMap(left => target
.filter(right => localRuleConnects(rule, left, right, context.definitions))
.map(right => ({ baseline: left, target: right })));
return pairs.length === 1 ? [{ ...pairs[0], rule }] : [];
});
const baselineCounts = new Map<string, number>();
const targetCounts = new Map<string, number>();
candidates.forEach(pair => {
baselineCounts.set(pair.baseline.id, (baselineCounts.get(pair.baseline.id) ?? 0) + 1);
targetCounts.set(pair.target.id, (targetCounts.get(pair.target.id) ?? 0) + 1);
});
return candidates
.filter(pair => baselineCounts.get(pair.baseline.id) === 1 && targetCounts.get(pair.target.id) === 1)
.map(pair => ({
baseline: pair.baseline,
target: pair.target,
match: match('local-rule', 1, `Lokal bestätigte Mappingregel ${pair.rule.id}.`, 'user-confirmed')
}));
}
function evidenceValue(records: EvidenceRecord[]): JsonValue {
return records
.map(record => ({ kind: record.kind, title: record.title ?? '', value: record.value }))
.sort((left, right) => stableJson(left).localeCompare(stableJson(right)));
}
function comparisonChanges(left: FindingOccurrence, right: FindingOccurrence, context: ComparisonContext): FindingChange[] {
const candidates: Array<{ kind: FindingChange['kind']; before: JsonValue; after: JsonValue }> = [
{ kind: 'severity', before: left.severity.normalizedValue, after: right.severity.normalizedValue },
{ kind: 'status', before: left.status.normalizedValue, after: right.status.normalizedValue },
{ kind: 'score', before: left.score?.normalizedValue ?? null, after: right.score?.normalizedValue ?? null },
{ kind: 'asset', before: left.affectedAsset?.normalizedValue ?? null, after: right.affectedAsset?.normalizedValue ?? null },
{ kind: 'evidence', before: evidenceValue(context.evidence.get(left.id) ?? []), after: evidenceValue(context.evidence.get(right.id) ?? []) }
];
return candidates.filter(candidate => stableJson(candidate.before) !== stableJson(candidate.after));
}
function similarityScore(left: FindingOccurrence, right: FindingOccurrence, context: ComparisonContext): number {
const titleScore = jaccardSimilarity(
similarityTokens(normalizedRoleValue(left, 'title', context)),
similarityTokens(normalizedRoleValue(right, 'title', context))
);
if (!titleScore) return 0;
const leftAsset = normalizedRoleValue(left, 'asset', context);
const rightAsset = normalizedRoleValue(right, 'asset', context);
const leftCategory = normalizedRoleValue(left, 'category', context);
const rightCategory = normalizedRoleValue(right, 'category', context);
const sameAsset = leftAsset && leftAsset === rightAsset ? 0.1 : 0;
const sameCategory = leftCategory && leftCategory === rightCategory ? 0.08 : 0;
const sameTool = normalizedToolId(left) === normalizedToolId(right) ? 0.07 : 0;
return Math.min(0.95, titleScore * 0.75 + sameAsset + sameCategory + sameTool);
}
function similaritySuggestions(
baseline: FindingOccurrence[],
target: FindingOccurrence[],
context: ComparisonContext
): SuggestedPair[] {
const threshold = context.options.similarityThreshold ?? 0.72;
const candidates = baseline.flatMap(left => target.map(right => ({
baseline: left,
target: right,
score: similarityScore(left, right, context)
}))).filter(candidate => candidate.score >= threshold)
.sort((left, right) => right.score - left.score || left.baseline.id.localeCompare(right.baseline.id) || left.target.id.localeCompare(right.target.id));
const usedBaseline = new Set<string>();
const usedTarget = new Set<string>();
return candidates.flatMap(candidate => {
if (usedBaseline.has(candidate.baseline.id) || usedTarget.has(candidate.target.id)) return [];
usedBaseline.add(candidate.baseline.id);
usedTarget.add(candidate.target.id);
return [{
...candidate,
match: match(
'similarity-suggestion',
Number(candidate.score.toFixed(4)),
'Titel und optionale Merkmale sind ähnlich; eine Benutzerbestätigung ist erforderlich.',
'suggested'
)
}];
});
}
function intersects(left: string[], right: string[]): boolean {
const rightKeys = new Set(right);
return left.some(key => rightKeys.has(key));
}
function exactHistoricalMatch(
historical: FindingOccurrence[],
current: FindingOccurrence,
context: ComparisonContext,
rules: LocalComparisonRule[],
runs: Map<string, ScanRun>
): FindingOccurrence | undefined {
return historical.filter(previous =>
rules.some(rule => localRuleConnects(rule, previous, current, context.definitions))
|| intersects(canonicalDefinitionKeys(previous, context), canonicalDefinitionKeys(current, context))
|| intersects(toolRuleKeys(previous, context), toolRuleKeys(current, context))
|| intersects(frameworkKeys(previous, context), frameworkKeys(current, context))
).sort((left, right) =>
(runs.get(right.scanRunId)?.sequence ?? 0) - (runs.get(left.scanRunId)?.sequence ?? 0)
|| left.id.localeCompare(right.id)
)[0];
}
function occurrenceHasIdentity(occurrence: FindingOccurrence, context: ComparisonContext): boolean {
return Boolean(
canonicalDefinitionKeys(occurrence, context).length
|| toolRuleKeys(occurrence, context).length
|| frameworkKeys(occurrence, context).length
);
}
function emptyMatch(reason: string): ComparisonMatch {
return { method: 'none', decision: 'unmatched', confidence: 0, reasons: [reason] };
}
function buildSummary(results: FindingComparison[]): ComparisonSummary {
return {
total: results.length,
new: results.filter(result => result.lifecycle === 'new').length,
persistent: results.filter(result => result.lifecycle === 'persistent').length,
resolved: results.filter(result => result.lifecycle === 'resolved').length,
reoccurred: results.filter(result => result.lifecycle === 'reoccurred').length,
possiblyIdentical: results.filter(result => result.lifecycle === 'possibly-identical').length,
notComparable: results.filter(result => result.lifecycle === 'not-comparable').length,
changedSeverity: results.filter(result => result.changes.some(change => change.kind === 'severity')).length,
changedStatus: results.filter(result => result.changes.some(change => change.kind === 'status')).length,
changedEvidence: results.filter(result => result.changes.some(change => change.kind === 'evidence')).length
};
}
export function compareScanRuns(
model: CanonicalAssessmentModel,
baselineRunId: string,
targetRunId: string,
options: ComparisonOptions = {}
): ScanRunComparison {
if (baselineRunId === targetRunId) throw new Error('Ein Scan-Lauf kann nicht mit sich selbst verglichen werden.');
const runs = new Map(model.scanRuns.map(run => [run.id, run]));
const baselineRun = runs.get(baselineRunId);
const targetRun = runs.get(targetRunId);
if (!baselineRun || !targetRun) throw new Error('Mindestens einer der gewählten Scan-Läufe ist nicht vorhanden.');
const definitions = new Map(model.findingDefinitions.map(definition => [definition.id, definition]));
const evidence = new Map<string, EvidenceRecord[]>();
model.evidence.forEach(record => evidence.set(record.occurrenceId, [...(evidence.get(record.occurrenceId) ?? []), record]));
const context: ComparisonContext = { definitions, evidence, options };
const rules = options.localRules ?? [];
const ruleIssues = validateLocalComparisonRules(rules);
if (ruleIssues.length) throw new Error(`Ungültige lokale Mappingregeln: ${ruleIssues.join(' ')}`);
const baseline = model.findingOccurrences.filter(occurrence => occurrence.scanRunId === baselineRunId).sort((left, right) => left.id.localeCompare(right.id));
const target = model.findingOccurrences.filter(occurrence => occurrence.scanRunId === targetRunId).sort((left, right) => left.id.localeCompare(right.id));
const diagnostics: string[] = [];
const sameAssessment = baselineRun.assessmentId === targetRun.assessmentId;
if (!sameAssessment) diagnostics.push('Die Scan-Läufe gehören zu unterschiedlichen Assessments; nicht zugeordnete Findings werden als nicht vergleichbar ausgewiesen.');
const sameTool = normalizeComparisonText(baselineRun.tool.id) === normalizeComparisonText(targetRun.tool.id);
if (!sameTool) diagnostics.push('Die Scan-Läufe stammen aus unterschiedlichen Werkzeugen; nur werkzeugübergreifend stabile Identitäten werden automatisch zugeordnet.');
// Persistierte Läufe desselben Werkzeugs dürfen nur verglichen werden, wenn
// ihr kanonischer Verarbeitungsvertrag bekannt und kompatibel ist. Das
// verhindert fachlich falsche Lifecycle-Aussagen nach semantischen Updates.
const baselineFiles = model.importFiles.filter(file => file.scanRunId === baselineRunId);
const targetFiles = model.importFiles.filter(file => file.scanRunId === targetRunId);
if (sameTool && baselineFiles.length && targetFiles.length) {
const keysFor = (files: typeof model.importFiles) => [...new Set(files.map(file =>
resolvedCompatibilityKey(file.detectedTool.id || file.processedWith.profileId, file.processedWith)
))];
const baselineKeys = keysFor(baselineFiles);
const targetKeys = keysFor(targetFiles);
const compatible = baselineKeys.length === 1 && targetKeys.length === 1
&& baselineKeys[0] !== undefined && baselineKeys[0] === targetKeys[0];
if (!compatible) {
const reason = baselineKeys.includes(undefined) || targetKeys.includes(undefined)
? 'Der Verarbeitungsvertrag mindestens eines Laufs ist unbekannt; der Vergleich wurde sicherheitshalber gesperrt.'
: 'Die Läufe wurden mit semantisch inkompatiblen Verarbeitungsverträgen erzeugt; ein Reimport in getrennte oder einheitlich aktuelle Timeslots ist erforderlich.';
diagnostics.push(reason);
const results: FindingComparison[] = [
...baseline.map(occurrence => ({ baselineOccurrenceId: occurrence.id, lifecycle: 'not-comparable' as const, match: emptyMatch(reason), changes: [] })),
...target.map(occurrence => ({ targetOccurrenceId: occurrence.id, lifecycle: 'not-comparable' as const, match: emptyMatch(reason), changes: [] }))
];
return { version: 1, baselineRunId, targetRunId, results, summary: buildSummary(results), diagnostics };
}
}
const unmatchedBaseline = new Map(baseline.map(occurrence => [occurrence.id, occurrence]));
const unmatchedTarget = new Map(target.map(occurrence => [occurrence.id, occurrence]));
const pairs: MatchedPair[] = [];
const applyPairs = (found: MatchedPair[]) => found.forEach(pair => {
if (!unmatchedBaseline.has(pair.baseline.id) || !unmatchedTarget.has(pair.target.id)) return;
unmatchedBaseline.delete(pair.baseline.id);
unmatchedTarget.delete(pair.target.id);
pairs.push(pair);
});
const remainingBaseline = () => [...unmatchedBaseline.values()];
const remainingTarget = () => [...unmatchedTarget.values()];
applyPairs(localRuleMatches(remainingBaseline(), remainingTarget(), rules, context));
applyPairs(uniqueStageMatches(remainingBaseline(), remainingTarget(), canonicalDefinitionKeys, context, 'canonical-definition', 1, 'Gemeinsame bestätigte Finding-Definition oder kanonischer Schlüssel.'));
applyPairs(uniqueStageMatches(remainingBaseline(), remainingTarget(), toolRuleKeys, context, 'tool-rule-id', 0.99, 'Werkzeug und Regel-ID stimmen eindeutig überein.'));
applyPairs(uniqueStageMatches(remainingBaseline(), remainingTarget(), frameworkKeys, context, 'framework-control', 0.96, 'Framework und Control-ID stimmen eindeutig überein.'));
const suggestions = sameAssessment ? similaritySuggestions(remainingBaseline(), remainingTarget(), context) : [];
suggestions.forEach(suggestion => {
unmatchedBaseline.delete(suggestion.baseline.id);
unmatchedTarget.delete(suggestion.target.id);
});
const historical = model.findingOccurrences.filter(occurrence => {
const run = runs.get(occurrence.scanRunId);
return run
&& run.assessmentId === targetRun.assessmentId
&& run.sequence < baselineRun.sequence
&& occurrence.scanRunId !== baselineRunId;
});
const results: FindingComparison[] = ([
...pairs.map(pair => ({
baselineOccurrenceId: pair.baseline.id,
targetOccurrenceId: pair.target.id,
lifecycle: 'persistent' as const,
match: pair.match,
changes: comparisonChanges(pair.baseline, pair.target, context)
})),
...suggestions.map(pair => ({
baselineOccurrenceId: pair.baseline.id,
targetOccurrenceId: pair.target.id,
lifecycle: 'possibly-identical' as const,
match: pair.match,
changes: comparisonChanges(pair.baseline, pair.target, context)
})),
...remainingTarget().map(occurrence => {
const previous = sameAssessment ? exactHistoricalMatch(historical, occurrence, context, rules, runs) : undefined;
const comparable = sameAssessment && (sameTool || previous) && occurrenceHasIdentity(occurrence, context);
return {
targetOccurrenceId: occurrence.id,
...(previous ? { historicalOccurrenceId: previous.id } : {}),
lifecycle: previous ? 'reoccurred' as const : comparable ? 'new' as const : 'not-comparable' as const,
match: previous
? match('canonical-definition', 0.98, 'Das Finding war im direkten Basislauf abwesend, wurde aber in einem früheren Lauf eindeutig gefunden.')
: emptyMatch(comparable ? 'Im Basislauf wurde kein entsprechendes Finding gefunden.' : 'Es fehlen ausreichend stabile Vergleichsmerkmale.'),
changes: []
};
}),
...remainingBaseline().map(occurrence => {
const comparable = sameAssessment && sameTool && occurrenceHasIdentity(occurrence, context);
return {
baselineOccurrenceId: occurrence.id,
lifecycle: comparable ? 'resolved' as const : 'not-comparable' as const,
match: emptyMatch(comparable ? 'Im Ziellauf wurde kein entsprechendes Finding gefunden.' : 'Es fehlen ausreichend stabile Vergleichsmerkmale.'),
changes: []
};
})
] as FindingComparison[]).sort((left, right) =>
lifecycleOrder[left.lifecycle] - lifecycleOrder[right.lifecycle]
|| (left.baselineOccurrenceId ?? '').localeCompare(right.baselineOccurrenceId ?? '')
|| (left.targetOccurrenceId ?? '').localeCompare(right.targetOccurrenceId ?? '')
);
return {
version: 1,
baselineRunId,
targetRunId,
results,
summary: buildSummary(results),
diagnostics
};
}