import { describe, expect, it } from 'vitest'; import type { CanonicalFindingStatus, FindingOccurrence, ImportFileRecord, ScanRun } from '../../src/domain/canonical-model'; import type { EnvironmentObservation } from '../../src/domain/environment-context'; import { LocalAssessorRepository } from '../../src/persistence/local-assessor-repository'; import { compareProjectTimeslotEvidence } from '../../src/comparison/timeslot-comparison-engine'; // Permanent on/off invariance of the Comparison Engine (AGENTS.md, "Core Engine and engines"): the Core // works unchanged with the engine switched off, and running a comparison of two stored timeslots leaves // every stored ASSESSOR value, the repository revision and the compared models exactly as they were. const timestamp = '2026-01-01T08:00:00.000Z'; const provenance = { decisionSource: 'source-native' as const, confidence: { level: 'high' as const, score: 1, reasons: ['synthetic'] }, mappingVersion: '1' }; const tool = { id: 'cis-cat', name: 'CIS-CAT Pro' }; function run(id: string, assessmentId: string, importedAt: string): ScanRun { return { id, assessmentId, sequence: 1, label: id, tool, importedAt }; } function importFile(runId: string, importedAt: string): ImportFileRecord { return { id: `file-${runId}`, scanRunId: runId, fileName: `${runId}.html`, format: 'HTML', sizeBytes: 10, importedAt, detectedTool: tool, detection: { level: 'high', score: 1, reasons: ['synthetic'] }, processedWith: { parserId: 'html', parserVersion: '1', profileId: 'cis-cat', profileVersion: '1.4.0', mappingVersion: '1.3.0', normalizationVersion: '1.3.0', presentationVersion: '1.3.0', compatibilityKey: 'cis-cat-canonical-v1' }, sourceArtifact: { kind: 'not-retained', reason: 'synthetic' } }; } function observation(runId: string, property: string, value: string): EnvironmentObservation { return { version: 1, id: `${runId}-${property}`, scanRunId: runId, importFileId: `file-${runId}`, tool, subject: { type: 'host' }, property, label: property, originalValue: value, normalizedValue: value, valueType: property === 'report.generated_at' ? 'datetime' : 'string', state: 'observed', source: {}, mappingVersion: '1', attributes: {} }; } function occurrence(runId: string, ruleId: string, status: CanonicalFindingStatus): FindingOccurrence { return { id: `${runId}-${ruleId}`, definitionId: `definition-${ruleId}`, scanRunId: runId, importFileId: `file-${runId}`, sourceIdentity: { tool, canonicalRuleId: ruleId }, sourcePosition: { rowIndex: 1 }, originalFields: [{ key: 'rule', label: 'Rule', columnIndex: 0, value: ruleId }], fieldMappings: [], severity: { originalValue: 'high', normalizedValue: 'high', provenance }, status: { originalValue: status, normalizedValue: status, provenance }, outcome: { originalValue: status === 'passed' ? 'pass' : 'fail', normalizedValue: status === 'passed' ? 'pass' : 'fail', provenance } }; } async function commitTimeslot(repository: LocalAssessorRepository, name: string, observedAt: string, findings: Array<[string, CanonicalFindingStatus]>) { const assessmentId = `assessment-${name}`; await repository.saveAssessment({ id: assessmentId, projectId: 'dummy-project', name, createdAt: observedAt, updatedAt: observedAt }); const scanRun = run(`run-${name}`, assessmentId, observedAt); const session = await repository.beginImport({ sessionId: `session-${name}`, projectId: 'dummy-project', assessmentId, scanRun }); await session.stage({ importFiles: [importFile(scanRun.id, observedAt)], environmentObservations: [ observation(scanRun.id, 'host.name', 'server-a'), observation(scanRun.id, 'benchmark.name', 'benchmark-a'), observation(scanRun.id, 'report.generated_at', observedAt) ], findingDefinitions: findings.map(([ruleId]) => ({ id: `definition-${ruleId}`, canonicalKey: `cis-cat:${ruleId}`, title: ruleId, frameworkReferences: [], createdAt: timestamp, updatedAt: timestamp })), findingOccurrences: findings.map(([ruleId, status]) => occurrence(scanRun.id, ruleId, status)) }); await session.commit(); return { assessmentId, runId: scanRun.id }; } describe('Comparison Engine on/off invariance', () => { it('leaves every stored ASSESSOR value, the repository revision and the compared models unchanged', async () => { const repository = new LocalAssessorRepository(); await repository.saveProject({ id: 'dummy-project', name: 'Dummy Customer', createdAt: timestamp, updatedAt: timestamp, attributes: {} }); const a = await commitTimeslot(repository, 'a', '2026-01-01T10:00:00.000Z', [['1.1.1', 'open'], ['1.1.2', 'open']]); const b = await commitTimeslot(repository, 'b', '2026-02-01T10:00:00.000Z', [['1.1.1', 'passed'], ['1.1.2', 'open']]); const off = { revision: repository.revision, data: JSON.stringify(await repository.exportProject('dummy-project')) }; const models = [(await repository.loadAssessmentModel(a.assessmentId))!, (await repository.loadAssessmentModel(b.assessmentId))!]; const modelsBefore = JSON.stringify(models); const comparison = compareProjectTimeslotEvidence(models, [a.runId], [b.runId]); // The engine really ran: the stored fail-to-pass of the same identity is a verified remediation. expect(comparison.summary.verifiedRemediated).toBe(1); expect(comparison.summary.confirmedOpen).toBe(1); expect(JSON.stringify(models)).toBe(modelsBefore); expect(repository.revision).toBe(off.revision); expect(JSON.stringify(await repository.exportProject('dummy-project'))).toBe(off.data); }); });