245 lines
16 KiB
TypeScript
245 lines
16 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
import type {
|
|
CanonicalAssessmentModel,
|
|
CanonicalFindingStatus,
|
|
FindingOccurrence,
|
|
ImportFileRecord
|
|
} from '../../src/domain/canonical-model';
|
|
import type { EnvironmentObservation } from '../../src/domain/environment-context';
|
|
import { compareProjectTimeslotEvidence } from '../../src/comparison/timeslot-comparison-engine';
|
|
|
|
const observedA = '2026-01-01T10:00:00.000Z';
|
|
const observedB = '2026-02-01T10:00:00.000Z';
|
|
const provenance = {
|
|
decisionSource: 'source-native' as const,
|
|
confidence: { level: 'high' as const, score: 1, reasons: ['synthetic'] },
|
|
mappingVersion: '1'
|
|
};
|
|
|
|
function occurrence(id: string, runId: string, ruleId: string, status: CanonicalFindingStatus): FindingOccurrence {
|
|
return {
|
|
id, definitionId: `definition-${ruleId}`, scanRunId: runId, importFileId: `file-${runId}`,
|
|
sourceIdentity: { tool: { id: 'cis-cat', name: 'CIS-CAT Pro' }, canonicalRuleId: ruleId },
|
|
sourcePosition: { rowIndex: 1 }, originalFields: [], fieldMappings: [],
|
|
severity: { originalValue: 'high', normalizedValue: 'high', provenance },
|
|
status: { originalValue: status, normalizedValue: status, provenance },
|
|
outcome: {
|
|
originalValue: status === 'passed' ? 'pass' : 'fail',
|
|
normalizedValue: status === 'passed' ? 'pass' : 'fail', provenance
|
|
}
|
|
};
|
|
}
|
|
|
|
function importFile(runId: string): ImportFileRecord {
|
|
return {
|
|
id: `file-${runId}`, scanRunId: runId, fileName: `${runId}.html`, format: 'HTML',
|
|
sizeBytes: 10, importedAt: observedA, detectedTool: { id: 'cis-cat', name: 'CIS-CAT Pro' },
|
|
detection: { level: 'high', score: 1, reasons: ['synthetic'] },
|
|
processedWith: {
|
|
parserId: 'html', parserVersion: '1', profileId: 'cis-cat', profileVersion: '1.4.0',
|
|
mappingVersion: '1.3.0', normalizationVersion: '1.3.0', presentationVersion: '1.3.0',
|
|
compatibilityKey: 'cis-cat-canonical-v1'
|
|
},
|
|
sourceArtifact: { kind: 'not-retained', reason: 'synthetic' }
|
|
};
|
|
}
|
|
|
|
function observation(runId: string, property: string, value: string): EnvironmentObservation {
|
|
return {
|
|
version: 1, id: `${runId}-${property}`, scanRunId: runId, importFileId: `file-${runId}`,
|
|
tool: { id: 'cis-cat', name: 'CIS-CAT Pro' }, subject: { type: 'host' },
|
|
property, label: property, originalValue: value, normalizedValue: value,
|
|
valueType: property === 'report.generated_at' ? 'datetime' : 'string', state: 'observed',
|
|
source: {}, mappingVersion: '1', attributes: {}
|
|
};
|
|
}
|
|
|
|
function model(runId: string, observedAt: string, findings: FindingOccurrence[]): CanonicalAssessmentModel {
|
|
return {
|
|
schemaVersion: 1,
|
|
project: { id: 'project', name: 'Synthetic', createdAt: observedA, updatedAt: observedB, attributes: {} },
|
|
assessment: { id: `assessment-${runId}`, projectId: 'project', name: runId, createdAt: observedAt, updatedAt: observedAt },
|
|
scanRuns: [{
|
|
id: runId, assessmentId: `assessment-${runId}`, sequence: 1, label: runId,
|
|
tool: { id: 'cis-cat', name: 'CIS-CAT Pro' }, importedAt: observedAt
|
|
}],
|
|
importFiles: [importFile(runId)], sourceTables: [],
|
|
environmentObservations: [
|
|
observation(runId, 'host.name', 'server-a'),
|
|
observation(runId, 'benchmark.name', 'benchmark-a'),
|
|
observation(runId, 'report.generated_at', observedAt)
|
|
],
|
|
findingDefinitions: [...new Set(findings.map(item => item.definitionId))].map(id => ({
|
|
id, canonicalKey: id, title: id, frameworkReferences: [], createdAt: observedA, updatedAt: observedAt
|
|
})),
|
|
findingOccurrences: findings, evidence: [], reviews: [], diagnostics: []
|
|
};
|
|
}
|
|
|
|
describe('conservative project timeslot comparison', () => {
|
|
it('counts explicit fail-to-pass, keeps absence unresolved and separates new deviations', () => {
|
|
const baseline = model('run-a', observedA, [
|
|
occurrence('a-1', 'run-a', 'RULE-1', 'failed'),
|
|
occurrence('a-2', 'run-a', 'RULE-2', 'failed')
|
|
]);
|
|
const target = model('run-b', observedB, [
|
|
occurrence('b-1', 'run-b', 'RULE-1', 'passed'),
|
|
occurrence('b-3', 'run-b', 'RULE-3', 'failed')
|
|
]);
|
|
const result = compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']);
|
|
expect(result.summary).toEqual({
|
|
baselineDeviations: 2, verifiedRemediated: 1, confirmedOpen: 0,
|
|
unverifiedAbsent: 1, notComparable: 0, notApplicable: 0, newDeviations: 1, regressedDeviations: 0
|
|
});
|
|
expect(result.sources[0].decisions.map(item => item.state).sort())
|
|
.toEqual(['unverified-absent', 'verified-remediated']);
|
|
});
|
|
|
|
it('separates a regression after a single verified pass from a new deviation and ambiguous history', () => {
|
|
const baseline = model('run-a', observedA, [
|
|
occurrence('a-1', 'run-a', 'RULE-1', 'passed'),
|
|
occurrence('a-2', 'run-a', 'RULE-2', 'passed'),
|
|
occurrence('a-2b', 'run-a', 'RULE-2', 'passed')
|
|
]);
|
|
const target = model('run-b', observedB, [
|
|
occurrence('b-1', 'run-b', 'RULE-1', 'failed'),
|
|
occurrence('b-2', 'run-b', 'RULE-2', 'failed'),
|
|
occurrence('b-3', 'run-b', 'RULE-3', 'failed')
|
|
]);
|
|
const result = compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']);
|
|
expect(result.version).toBe('1.2.0');
|
|
expect(result.sources[0].regressedDeviationOccurrenceIds).toEqual(['b-1']);
|
|
expect(result.sources[0].newDeviationOccurrenceIds).toEqual(['b-3']);
|
|
expect(result.summary).toMatchObject({ newDeviations: 1, regressedDeviations: 1, baselineDeviations: 0 });
|
|
});
|
|
|
|
it('splits a run imported from several scanners by the scanner of each file', () => {
|
|
const baseline = model('run-a', observedA, [occurrence('a-1', 'run-a', 'RULE-1', 'failed')]);
|
|
const target = model('run-b', observedB, [occurrence('b-1', 'run-b', 'RULE-1', 'passed')]);
|
|
// One batch with CIS-CAT and Maester: the run is labelled 'multiple', each file keeps its scanner.
|
|
target.scanRuns[0].tool = { id: 'multiple', name: 'Multiple Tools' };
|
|
const maesterFile = { ...importFile('run-b'), id: 'file-run-b-maester', fileName: 'maester.csv',
|
|
detectedTool: { id: 'maester', name: 'Maester' }, processedWith: { ...importFile('run-b').processedWith, profileId: 'maester' } };
|
|
target.importFiles.push(maesterFile);
|
|
target.findingOccurrences.push({ ...occurrence('m-1', 'run-b', 'MT.1001', 'failed'), importFileId: maesterFile.id,
|
|
sourceIdentity: { tool: { id: 'maester', name: 'Maester' }, canonicalRuleId: 'MT.1001' } });
|
|
const result = compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']);
|
|
expect(result.sources.map(source => [source.toolId, source.state])).toEqual([['cis-cat', 'compared'], ['maester', 'not-comparable']]);
|
|
expect(result.summary).toMatchObject({ verifiedRemediated: 1, newDeviations: 0 });
|
|
expect(result.sources[1]).toMatchObject({ reasons: ['baseline-source-missing'], targetFileIds: ['file-run-b-maester'] });
|
|
});
|
|
|
|
it('pairs several sources of one scanner by their verified scope', () => {
|
|
const withHost = (built: CanonicalAssessmentModel, host: string) => {
|
|
built.environmentObservations = built.environmentObservations?.map(item => item.property === 'host.name' ? { ...item, normalizedValue: host, originalValue: host } : item);
|
|
return built;
|
|
};
|
|
const merge = (left: CanonicalAssessmentModel, right: CanonicalAssessmentModel): CanonicalAssessmentModel => ({
|
|
...left, scanRuns: [...left.scanRuns, { ...right.scanRuns[0], assessmentId: left.assessment.id, sequence: 2 }],
|
|
importFiles: [...left.importFiles, ...right.importFiles], environmentObservations: [...left.environmentObservations!, ...right.environmentObservations!],
|
|
findingDefinitions: [...left.findingDefinitions, ...right.findingDefinitions], findingOccurrences: [...left.findingOccurrences, ...right.findingOccurrences]
|
|
});
|
|
const baseline = merge(withHost(model('run-a1', observedA, [occurrence('a-1', 'run-a1', 'RULE-1', 'failed')]), 'server-a'),
|
|
withHost(model('run-a2', observedA, [occurrence('a-2', 'run-a2', 'RULE-1', 'failed')]), 'server-b'));
|
|
const target = merge(withHost(model('run-b2', observedB, [occurrence('b-2', 'run-b2', 'RULE-1', 'failed')]), 'server-b'),
|
|
withHost(model('run-b1', observedB, [occurrence('b-1', 'run-b1', 'RULE-1', 'passed')]), 'server-a'));
|
|
const result = compareProjectTimeslotEvidence([baseline, target], ['run-a1', 'run-a2'], ['run-b1', 'run-b2']);
|
|
const decision = (id: string) => result.sources.flatMap(source => source.decisions).find(item => item.baselineOccurrenceId === id);
|
|
// The same rule is resolved on server A and still open on server B; never crossed between hosts.
|
|
expect(decision('a-1')).toMatchObject({ state: 'verified-remediated', targetOccurrenceId: 'b-1' });
|
|
expect(decision('a-2')).toMatchObject({ state: 'confirmed-open', targetOccurrenceId: 'b-2' });
|
|
});
|
|
|
|
it('blocks only diagnostics whose impact can reach the results', () => {
|
|
const diagnostic = (runId: string, code: string) => ({ id: `${runId}-${code}`, importFileId: `file-${runId}`, severity: 'warning' as const, code, message: 'synthetic' });
|
|
const baseline = model('run-a', observedA, [occurrence('a-1', 'run-a', 'RULE-1', 'failed')]);
|
|
const target = model('run-b', observedB, [occurrence('b-1', 'run-b', 'RULE-1', 'passed')]);
|
|
target.diagnostics = [diagnostic('run-b', 'html-active-content-removed')] as never;
|
|
expect(compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']).summary.verifiedRemediated).toBe(1);
|
|
target.diagnostics = [diagnostic('run-b', 'cis-cat-summary-mismatch')] as never;
|
|
expect(compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']).summary).toMatchObject({ verifiedRemediated: 0, unverifiedAbsent: 1 });
|
|
});
|
|
|
|
it('uses the timeslot order when a report states no time of its own', () => {
|
|
const withoutTime = (built: CanonicalAssessmentModel) => ({ ...built,
|
|
environmentObservations: built.environmentObservations?.filter(item => item.property !== 'report.generated_at') });
|
|
const baseline = withoutTime(model('run-a', observedA, [occurrence('a-1', 'run-a', 'RULE-1', 'failed')]));
|
|
const target = withoutTime(model('run-b', observedB, [occurrence('b-1', 'run-b', 'RULE-1', 'passed')]));
|
|
expect(compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']).summary.verifiedRemediated).toBe(1);
|
|
// An older report in B stays not comparable.
|
|
const older = compareProjectTimeslotEvidence([model('run-c', observedB, [occurrence('c-1', 'run-c', 'RULE-1', 'failed')]),
|
|
model('run-d', observedA, [occurrence('d-1', 'run-d', 'RULE-1', 'passed')])], ['run-c'], ['run-d']);
|
|
expect(older.summary).toMatchObject({ verifiedRemediated: 0, notComparable: 1 });
|
|
});
|
|
|
|
it('accepts a PingCastle rule missing from a complete report of the same domain and engine version as remediated', () => {
|
|
const pingCastle = (runId: string, observedAt: string, engine: string, rules: string[]): CanonicalAssessmentModel => {
|
|
const built = model(runId, observedAt, rules.map(rule => ({ ...occurrence(`${runId}-${rule}`, runId, rule, 'failed'),
|
|
sourceIdentity: { tool: { id: 'pingcastle', name: 'PingCastle' }, canonicalRuleId: rule } })));
|
|
built.scanRuns[0].tool = { id: 'pingcastle', name: 'PingCastle' };
|
|
built.importFiles[0] = { ...built.importFiles[0], detectedTool: { id: 'pingcastle', name: 'PingCastle' },
|
|
processedWith: { ...built.importFiles[0].processedWith, profileId: 'pingcastle', compatibilityKey: 'pingcastle-canonical-v1' } };
|
|
built.environmentObservations = [
|
|
{ ...observation(runId, 'domain.fqdn', 'synthetic.example'), tool: { id: 'pingcastle', name: 'PingCastle' } },
|
|
{ ...observation(runId, 'report.engine_version', engine), tool: { id: 'pingcastle', name: 'PingCastle' } }
|
|
];
|
|
return built;
|
|
};
|
|
const baseline = pingCastle('run-a', observedA, '3.3.0', ['A-Krbtgt', 'P-AdminLogin']);
|
|
const same = compareProjectTimeslotEvidence([baseline, pingCastle('run-b', observedB, '3.3.0', ['P-AdminLogin'])], ['run-a'], ['run-b']);
|
|
expect(same.summary).toMatchObject({ verifiedRemediated: 1, confirmedOpen: 1, unverifiedAbsent: 0 });
|
|
// Another PingCastle version may have renamed or dropped the rule: absence stays unverified.
|
|
const other = compareProjectTimeslotEvidence([baseline, pingCastle('run-c', observedB, '3.4.0', ['P-AdminLogin'])], ['run-a'], ['run-c']);
|
|
expect(other.summary).toMatchObject({ verifiedRemediated: 0, confirmedOpen: 1, unverifiedAbsent: 1 });
|
|
});
|
|
|
|
it('uses the client as scope when each timeslot holds one report that states no scope', () => {
|
|
const withoutScope = (built: CanonicalAssessmentModel) => ({ ...built,
|
|
environmentObservations: built.environmentObservations?.filter(item => item.property !== 'host.name') });
|
|
const baseline = withoutScope(model('run-a', observedA, [occurrence('a-1', 'run-a', 'RULE-1', 'failed')]));
|
|
const target = withoutScope(model('run-b', observedB, [occurrence('b-1', 'run-b', 'RULE-1', 'passed')]));
|
|
expect(compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']).summary.verifiedRemediated).toBe(1);
|
|
// A scope stated on one side only never pairs.
|
|
const stated = model('run-c', observedB, [occurrence('c-1', 'run-c', 'RULE-1', 'passed')]);
|
|
expect(compareProjectTimeslotEvidence([baseline, stated], ['run-a'], ['run-c']).summary).toMatchObject({ verifiedRemediated: 0, notComparable: 1 });
|
|
});
|
|
|
|
it('reports a source of only one timeslot as not comparable instead of failing', () => {
|
|
const target = model('run-b', observedB, [occurrence('b-1', 'run-b', 'RULE-1', 'failed')]);
|
|
const result = compareProjectTimeslotEvidence([target], [], ['run-b']);
|
|
expect(result.sources).toMatchObject([{ state: 'not-comparable', reasons: ['baseline-source-missing'] }]);
|
|
expect(result.summary).toMatchObject({ newDeviations: 0, regressedDeviations: 0 });
|
|
});
|
|
|
|
it('does not treat a missing scanner source or a same-time reimport as remediation', () => {
|
|
const baseline = model('run-a', observedA, [occurrence('a-1', 'run-a', 'RULE-1', 'failed')]);
|
|
const otherBaseline = model('run-c', observedA, []);
|
|
otherBaseline.scanRuns[0].tool = { id: 'pingcastle', name: 'PingCastle' };
|
|
otherBaseline.importFiles[0].detectedTool = { id: 'pingcastle', name: 'PingCastle' };
|
|
otherBaseline.importFiles[0].processedWith.profileId = 'pingcastle';
|
|
otherBaseline.environmentObservations = [];
|
|
const missing = compareProjectTimeslotEvidence([baseline, otherBaseline], ['run-a'], ['run-c']);
|
|
expect(missing.summary).toMatchObject({ verifiedRemediated: 0, notComparable: 1 });
|
|
|
|
const target = model('run-b', observedA, [occurrence('b-1', 'run-b', 'RULE-1', 'passed')]);
|
|
const sameTime = compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']);
|
|
// The same report again (unchanged report time) is never progress (user decision 2026-10-13: not comparable).
|
|
expect(sameTime.summary).toMatchObject({ verifiedRemediated: 0, notComparable: 1 });
|
|
expect(sameTime.sources[0].reasons).toEqual(['target-observation-not-newer']);
|
|
});
|
|
|
|
it('blocks scope changes and incompatible processing before lifecycle claims', () => {
|
|
const baseline = model('run-a', observedA, [occurrence('a-1', 'run-a', 'RULE-1', 'failed')]);
|
|
const changedScope = model('run-b', observedB, [occurrence('b-1', 'run-b', 'RULE-1', 'passed')]);
|
|
const host = changedScope.environmentObservations?.find(item => item.property === 'host.name');
|
|
if (host) host.normalizedValue = 'server-b';
|
|
expect(compareProjectTimeslotEvidence([baseline, changedScope], ['run-a'], ['run-b']).summary)
|
|
.toMatchObject({ verifiedRemediated: 0, notComparable: 1 });
|
|
|
|
const incompatible = model('run-c', observedB, [occurrence('c-1', 'run-c', 'RULE-1', 'passed')]);
|
|
incompatible.importFiles[0].processedWith.compatibilityKey = 'different-contract';
|
|
expect(compareProjectTimeslotEvidence([baseline, incompatible], ['run-a'], ['run-c']).summary)
|
|
.toMatchObject({ verifiedRemediated: 0, notComparable: 1 });
|
|
});
|
|
});
|