249 lines
14 KiB
TypeScript
249 lines
14 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
import type {
|
|
CanonicalAssessmentModel,
|
|
CanonicalFindingStatus,
|
|
CanonicalSeverity,
|
|
EvidenceRecord,
|
|
FindingDefinition,
|
|
FindingOccurrence,
|
|
ImportFileRecord,
|
|
ScanRun,
|
|
ToolIdentity
|
|
} from '../../../src/domain/canonical-model';
|
|
import { compareScanRuns } from '../../../src/evaluation/scan-run-comparison/comparison-engine';
|
|
import { createLocalComparisonRule } from '../../../src/evaluation/scan-run-comparison/local-mapping-rules';
|
|
|
|
const timestamp = '2026-07-17T08:00:00.000Z';
|
|
const tool = (id: string): ToolIdentity => ({ id, name: id });
|
|
const confidence = { level: 'high' as const, score: 1, reasons: ['synthetic'] };
|
|
const provenance = { decisionSource: 'source-native' as const, confidence, mappingVersion: '1.0.0' };
|
|
|
|
function definition(id: string, title: string, controlId?: string): FindingDefinition {
|
|
return {
|
|
id,
|
|
canonicalKey: `${id}-key`,
|
|
title,
|
|
frameworkReferences: controlId ? [{ framework: 'CIS', controlId }] : [],
|
|
createdAt: timestamp,
|
|
updatedAt: timestamp
|
|
};
|
|
}
|
|
|
|
function occurrence(options: {
|
|
id: string;
|
|
runId: string;
|
|
definitionId: string;
|
|
ruleId?: string;
|
|
title: string;
|
|
toolId?: string;
|
|
severity?: CanonicalSeverity;
|
|
status?: CanonicalFindingStatus;
|
|
category?: string;
|
|
asset?: string;
|
|
}): FindingOccurrence {
|
|
const severity = options.severity ?? 'high';
|
|
const status = options.status ?? 'open';
|
|
return {
|
|
id: options.id,
|
|
definitionId: options.definitionId,
|
|
scanRunId: options.runId,
|
|
importFileId: `file-${options.runId}`,
|
|
sourceIdentity: {
|
|
tool: tool(options.toolId ?? 'pingcastle'),
|
|
originalRuleId: options.ruleId,
|
|
canonicalRuleId: options.ruleId,
|
|
originalTitle: options.title
|
|
},
|
|
sourcePosition: { rowIndex: 1 },
|
|
originalFields: [{ key: 'title', label: 'Title', value: options.title, columnIndex: 0 }],
|
|
fieldMappings: [
|
|
{ targetRole: 'title', sourceFieldKeys: ['title'], normalizedValue: options.title, provenance },
|
|
...(options.category ? [{ targetRole: 'category' as const, sourceFieldKeys: ['category'], normalizedValue: options.category, provenance }] : [])
|
|
],
|
|
severity: { originalValue: severity, normalizedValue: severity, provenance },
|
|
status: { originalValue: status, normalizedValue: status, provenance },
|
|
...(options.asset ? { affectedAsset: { originalValue: options.asset, normalizedValue: options.asset, provenance } } : {})
|
|
};
|
|
}
|
|
|
|
function run(id: string, sequence: number, toolId = 'pingcastle', assessmentId = 'assessment'): ScanRun {
|
|
return { id, assessmentId, sequence, label: id, tool: tool(toolId), importedAt: timestamp };
|
|
}
|
|
|
|
function importFile(runId: string, profileVersion: string, mappingVersion: string): ImportFileRecord {
|
|
return {
|
|
id: `file-${runId}`, scanRunId: runId, fileName: `${runId}.json`, format: 'JSON', sizeBytes: 1,
|
|
importedAt: timestamp, detectedTool: tool('pingcastle'), detection: confidence,
|
|
processedWith: { parserId: 'json', parserVersion: '1.0.0', profileId: 'pingcastle', profileVersion, mappingVersion },
|
|
sourceArtifact: { kind: 'not-retained', reason: 'synthetic' }
|
|
};
|
|
}
|
|
|
|
function model(options: {
|
|
runs: ScanRun[];
|
|
definitions: FindingDefinition[];
|
|
occurrences: FindingOccurrence[];
|
|
evidence?: EvidenceRecord[];
|
|
importFiles?: ImportFileRecord[];
|
|
}): CanonicalAssessmentModel {
|
|
return {
|
|
schemaVersion: 1,
|
|
project: { id: 'project', name: 'Project', createdAt: timestamp, updatedAt: timestamp, attributes: {} },
|
|
assessment: { id: 'assessment', projectId: 'project', name: 'Assessment', createdAt: timestamp, updatedAt: timestamp },
|
|
scanRuns: options.runs,
|
|
importFiles: options.importFiles ?? [],
|
|
sourceTables: [],
|
|
findingDefinitions: options.definitions,
|
|
findingOccurrences: options.occurrences,
|
|
evidence: options.evidence ?? [],
|
|
reviews: [],
|
|
diagnostics: []
|
|
};
|
|
}
|
|
|
|
describe('comparison engine', () => {
|
|
it('fails closed for semantically incompatible persisted processing contracts', () => {
|
|
const comparisonModel = model({
|
|
runs: [run('left', 1), run('right', 2)],
|
|
definitions: [definition('def-a', 'Synthetic finding')],
|
|
occurrences: [
|
|
occurrence({ id: 'left-a', runId: 'left', definitionId: 'def-a', ruleId: 'A-1', title: 'Synthetic finding' }),
|
|
occurrence({ id: 'right-a', runId: 'right', definitionId: 'def-a', ruleId: 'A-1', title: 'Synthetic finding' })
|
|
],
|
|
importFiles: [importFile('left', '1.0.0', '1.0.0'), importFile('right', '1.2.0', '1.1.0')]
|
|
});
|
|
|
|
const result = compareScanRuns(comparisonModel, 'left', 'right');
|
|
expect(result.summary).toMatchObject({ notComparable: 2, persistent: 0, new: 0, resolved: 0 });
|
|
expect(result.diagnostics.join(' ')).toContain('semantisch inkompatiblen');
|
|
});
|
|
|
|
it('reports persistent, new, resolved, reoccurred and suggested findings with field changes', () => {
|
|
const definitions = [
|
|
definition('def-a', 'Weak password policy'),
|
|
definition('def-b', 'Legacy protocol enabled'),
|
|
definition('def-c', 'New risky delegation'),
|
|
definition('def-r', 'Recurring stale account'),
|
|
definition('def-old-name', 'Password policy permits weak settings'),
|
|
definition('def-new-name', 'Password policy permits weak settings')
|
|
];
|
|
const occurrences = [
|
|
occurrence({ id: 'history-r', runId: 'run-1', definitionId: 'def-r', ruleId: 'R-1', title: 'Recurring stale account' }),
|
|
occurrence({ id: 'baseline-a', runId: 'run-2', definitionId: 'def-a', ruleId: 'A-1', title: 'Weak password policy', severity: 'high', status: 'open' }),
|
|
occurrence({ id: 'baseline-b', runId: 'run-2', definitionId: 'def-b', ruleId: 'B-1', title: 'Legacy protocol enabled' }),
|
|
occurrence({ id: 'baseline-s', runId: 'run-2', definitionId: 'def-old-name', ruleId: 'LEGACY-1', title: 'Password policy permits weak settings' }),
|
|
occurrence({ id: 'target-a', runId: 'run-3', definitionId: 'def-a', ruleId: 'A-1', title: 'Weak password policy', severity: 'critical', status: 'failed' }),
|
|
occurrence({ id: 'target-c', runId: 'run-3', definitionId: 'def-c', ruleId: 'C-1', title: 'New risky delegation' }),
|
|
occurrence({ id: 'target-r', runId: 'run-3', definitionId: 'def-r', ruleId: 'R-1', title: 'Recurring stale account' }),
|
|
occurrence({ id: 'target-s', runId: 'run-3', definitionId: 'def-new-name', ruleId: 'MODERN-1', title: 'Password policy permits weak settings' })
|
|
];
|
|
const evidence: EvidenceRecord[] = [
|
|
{ id: 'ev-1', occurrenceId: 'baseline-a', kind: 'text', value: 'old evidence', sourceFieldKeys: ['evidence'] },
|
|
{ id: 'ev-2', occurrenceId: 'target-a', kind: 'text', value: 'new evidence', sourceFieldKeys: ['evidence'] }
|
|
];
|
|
const result = compareScanRuns(model({
|
|
runs: [run('run-1', 1), run('run-2', 2), run('run-3', 3)],
|
|
definitions,
|
|
occurrences,
|
|
evidence
|
|
}), 'run-2', 'run-3');
|
|
|
|
expect(result.summary).toMatchObject({ persistent: 1, new: 1, resolved: 1, reoccurred: 1, possiblyIdentical: 1, notComparable: 0 });
|
|
const persistent = result.results.find(item => item.targetOccurrenceId === 'target-a');
|
|
expect(persistent?.match.method).toBe('canonical-definition');
|
|
expect(persistent?.changes.map(change => change.kind)).toEqual(['severity', 'status', 'evidence']);
|
|
expect(result.results.find(item => item.targetOccurrenceId === 'target-r')).toMatchObject({ lifecycle: 'reoccurred', historicalOccurrenceId: 'history-r' });
|
|
expect(result.results.find(item => item.targetOccurrenceId === 'target-s')).toMatchObject({ lifecycle: 'possibly-identical', match: { decision: 'suggested' } });
|
|
});
|
|
|
|
it('is reproducible regardless of input order', () => {
|
|
const definitions = [definition('def-a', 'A'), definition('def-b', 'B')];
|
|
const occurrences = [
|
|
occurrence({ id: 'left-a', runId: 'left', definitionId: 'def-a', ruleId: 'A-1', title: 'A' }),
|
|
occurrence({ id: 'left-b', runId: 'left', definitionId: 'def-b', ruleId: 'B-1', title: 'B' }),
|
|
occurrence({ id: 'right-b', runId: 'right', definitionId: 'def-b', ruleId: 'B-1', title: 'B' }),
|
|
occurrence({ id: 'right-a', runId: 'right', definitionId: 'def-a', ruleId: 'A-1', title: 'A' })
|
|
];
|
|
const forward = model({ runs: [run('left', 1), run('right', 2)], definitions, occurrences });
|
|
const reversed = model({ runs: [...forward.scanRuns].reverse(), definitions: [...definitions].reverse(), occurrences: [...occurrences].reverse() });
|
|
expect(compareScanRuns(reversed, 'left', 'right')).toEqual(compareScanRuns(forward, 'left', 'right'));
|
|
});
|
|
|
|
it('matches cross-tool controls only when the framework identity is unique', () => {
|
|
const definitions = [definition('left-def', 'Control from scanner A', '1.1.1'), definition('right-def', 'Control from scanner B', '1.1.1')];
|
|
const result = compareScanRuns(model({
|
|
runs: [run('left', 1, 'cis-cat'), run('right', 2, 'maester')],
|
|
definitions,
|
|
occurrences: [
|
|
occurrence({ id: 'left-control', runId: 'left', definitionId: 'left-def', title: 'Control from scanner A', toolId: 'cis-cat' }),
|
|
occurrence({ id: 'right-control', runId: 'right', definitionId: 'right-def', title: 'Control from scanner B', toolId: 'maester' })
|
|
]
|
|
}), 'left', 'right');
|
|
expect(result.results[0]).toMatchObject({ lifecycle: 'persistent', match: { method: 'framework-control', decision: 'automatic' } });
|
|
});
|
|
|
|
it('never turns an ambiguous duplicate identity into an automatic match', () => {
|
|
const definitions = [definition('left-1-def', 'Duplicate title'), definition('left-2-def', 'Duplicate title'), definition('right-def', 'Duplicate title')];
|
|
const result = compareScanRuns(model({
|
|
runs: [run('left', 1), run('right', 2)],
|
|
definitions,
|
|
occurrences: [
|
|
occurrence({ id: 'left-1', runId: 'left', definitionId: 'left-1-def', ruleId: 'DUP-1', title: 'Duplicate title' }),
|
|
occurrence({ id: 'left-2', runId: 'left', definitionId: 'left-2-def', ruleId: 'DUP-1', title: 'Duplicate title' }),
|
|
occurrence({ id: 'right-1', runId: 'right', definitionId: 'right-def', ruleId: 'DUP-1', title: 'Duplicate title' })
|
|
]
|
|
}), 'left', 'right');
|
|
expect(result.results.some(item => item.lifecycle === 'persistent')).toBe(false);
|
|
expect(result.results.find(item => item.targetOccurrenceId === 'right-1')?.match.decision).toBe('suggested');
|
|
});
|
|
|
|
it('reuses an explicitly confirmed suggestion as a local mapping rule', () => {
|
|
const definitions = [definition('old-def', 'Renamed security check'), definition('new-def', 'Renamed security check')];
|
|
const left = occurrence({ id: 'left-old', runId: 'left', definitionId: 'old-def', ruleId: 'OLD-42', title: 'Renamed security check' });
|
|
const right = occurrence({ id: 'right-new', runId: 'right', definitionId: 'new-def', ruleId: 'NEW-42', title: 'Renamed security check' });
|
|
const comparisonModel = model({ runs: [run('left', 1), run('right', 2)], definitions, occurrences: [left, right] });
|
|
expect(compareScanRuns(comparisonModel, 'left', 'right').results[0].match.decision).toBe('suggested');
|
|
const rule = createLocalComparisonRule('confirmed-rename-42', left, right, definitions, 'Vom Benutzer bestätigt.');
|
|
const confirmed = compareScanRuns(comparisonModel, 'left', 'right', { localRules: [rule] });
|
|
expect(confirmed.results[0]).toMatchObject({ lifecycle: 'persistent', match: { method: 'local-rule', decision: 'user-confirmed', confidence: 1 } });
|
|
});
|
|
|
|
it('keeps exact title and asset matches as suggestions until a local rule is confirmed', () => {
|
|
const definitions = [definition('left-title', 'Same title'), definition('right-title', 'Same title')];
|
|
const left = occurrence({ id: 'left-title-only', runId: 'left', definitionId: 'left-title', title: 'Same title', asset: 'Synthetic host' });
|
|
const right = occurrence({ id: 'right-title-only', runId: 'right', definitionId: 'right-title', title: 'Same title', asset: 'Synthetic host' });
|
|
const comparisonModel = model({ runs: [run('left', 1), run('right', 2)], definitions, occurrences: [left, right] });
|
|
|
|
const suggested = compareScanRuns(comparisonModel, 'left', 'right');
|
|
expect(suggested.results[0]).toMatchObject({
|
|
lifecycle: 'possibly-identical',
|
|
match: { method: 'similarity-suggestion', decision: 'suggested' }
|
|
});
|
|
|
|
const localRule = createLocalComparisonRule('confirmed-title-and-asset', left, right, definitions);
|
|
expect(localRule).toMatchObject({ source: 'user-confirmed', left: { title: 'same title', asset: 'synthetic host' } });
|
|
const confirmed = compareScanRuns(comparisonModel, 'left', 'right', { localRules: [localRule] });
|
|
expect(confirmed.results[0]).toMatchObject({
|
|
lifecycle: 'persistent', match: { method: 'local-rule', decision: 'user-confirmed' }
|
|
});
|
|
|
|
const duplicate = occurrence({ id: 'left-title-duplicate', runId: 'left', definitionId: 'left-title', title: 'Same title', asset: 'Synthetic host' });
|
|
const ambiguousModel = model({ runs: [run('left', 1), run('right', 2)], definitions, occurrences: [left, duplicate, right] });
|
|
expect(compareScanRuns(ambiguousModel, 'left', 'right', { localRules: [localRule] }).results
|
|
.some(item => item.match.method === 'local-rule')).toBe(false);
|
|
});
|
|
it('marks unrelated assessment runs as not comparable', () => {
|
|
const definitions = [definition('left-def', 'Left'), definition('right-def', 'Right')];
|
|
const result = compareScanRuns(model({
|
|
runs: [run('left', 1, 'pingcastle', 'assessment-a'), run('right', 2, 'pingcastle', 'assessment-b')],
|
|
definitions,
|
|
occurrences: [
|
|
occurrence({ id: 'left-item', runId: 'left', definitionId: 'left-def', ruleId: 'LEFT-1', title: 'Left' }),
|
|
occurrence({ id: 'right-item', runId: 'right', definitionId: 'right-def', ruleId: 'RIGHT-1', title: 'Right' })
|
|
]
|
|
}), 'left', 'right');
|
|
expect(result.summary.notComparable).toBe(2);
|
|
expect(result.diagnostics[0]).toContain('unterschiedlichen Assessments');
|
|
});
|
|
});
|