From 24c0f959aa55de131cc14649c18e649bbae7e528 Mon Sep 17 00:00:00 2001 From: Binh Date: Tue, 13 Oct 2026 11:52:56 +0200 Subject: [PATCH] feat(comparison): compare scanners, GPO settings and score basis across timeslots Separate new from regressed scanner deviations and accept a timeslot without scanner runs on one side. Pair rolled-up GPO Work verdicts by scope and technical identity with a newer backup set and the same baselines; a changed value of a still deviating setting stays neutral. compareTimeslots yields the change categories for the UI and a score delta over comparable sources only. --- docs/IMPLEMENTATION_PLAN.md | 6 +- docs/TIMESLOT_COMPARISON_PLAN.md | 4 +- src/comparison/gpo-timeslot-comparison.ts | 164 +++++++++++++ src/comparison/timeslot-comparison-engine.ts | 40 ++- src/comparison/timeslot-comparison-report.ts | 227 ++++++++++++++++++ .../gpo-timeslot-comparison.test.ts | 122 ++++++++++ .../timeslot-comparison-engine.test.ts | 27 ++- .../timeslot-comparison-report.test.ts | 145 +++++++++++ 8 files changed, 719 insertions(+), 16 deletions(-) create mode 100644 src/comparison/gpo-timeslot-comparison.ts create mode 100644 src/comparison/timeslot-comparison-report.ts create mode 100644 tests/comparison/gpo-timeslot-comparison.test.ts create mode 100644 tests/comparison/timeslot-comparison-report.test.ts diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index f1efaeb..7de1204 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -2,7 +2,7 @@ ## Comparison Engine: decision and staging (planned 2026-10-13, Phase 8 timeslot comparison as an engine) -Status: **steps 1 and 2 done and verified (2026-10-13), steps 3-4 open**. Explicit user decision 2026-10-13: the timeslot comparison of [TIMESLOT_COMPARISON_PLAN.md](TIMESLOT_COMPARISON_PLAN.md) (P1-P6) becomes a separately switchable **Comparison Engine** that owns its comparison logic. Its folders follow the other engines without an `-engine` suffix: `src/comparison/`, `server/comparison/`, `tests/comparison/`. It reads the Core only through a declared read-only port and never writes, overrides or re-derives a Core verdict, score, review, risk acceptance or Result DB row. All contracts, the acceptance matrix and the conservative rules of the timeslot plan stay binding unchanged (object/scope identity, no `absent = resolved` without a declared profile negative-evidence contract, a missing source keeps its open deviations as unverified, score delta only over the shared population, UI is a consumer, never its own matcher). +Status: **steps 1, 2 and 3a done and verified (2026-10-13), steps 3b and 4 open**. Explicit user decision 2026-10-13: the timeslot comparison of [TIMESLOT_COMPARISON_PLAN.md](TIMESLOT_COMPARISON_PLAN.md) (P1-P6) becomes a separately switchable **Comparison Engine** that owns its comparison logic. Its folders follow the other engines without an `-engine` suffix: `src/comparison/`, `server/comparison/`, `tests/comparison/`. It reads the Core only through a declared read-only port and never writes, overrides or re-derives a Core verdict, score, review, risk acceptance or Result DB row. All contracts, the acceptance matrix and the conservative rules of the timeslot plan stay binding unchanged (object/scope identity, no `absent = resolved` without a declared profile negative-evidence contract, a missing source keeps its open deviations as unverified, score delta only over the shared population, UI is a consumer, never its own matcher). UI decision 2026-10-13 (mockups A and B in the local, git-ignored `artifacts/compare-engine/`; synthetic Timeslot A): entry is a Client Info tab `Compare` with A/B timeslot selection, change counts, Security Domain and source deltas, each count opening Work; Work receives a `Change` facet and change badges, and the Finding Reader a `Compare` section with A and B values side by side. A retained value change of a still deviating GPO setting is shown neutral (`Value changed`), neither better nor worse. Mockup C (better/worse bridge) is not selected. @@ -14,7 +14,9 @@ Staging and required Core releases: Steps 1-2 (Core release Governance, Core Tests, Evaluation, Store, Presentation): the Phase 8 files moved unchanged to `src/evaluation/scan-run-comparison/` (test to `tests/evaluation/scan-run-comparison/`); only import paths changed in the three Core importers and two tests. `assessor_comparisons`, its routes and data are untouched (0 rows). `.claude/hooks/core-areas.mjs` registers `Comparison Engine` and drops `src/comparison/` from Evaluation; `AGENTS.md`, the boundary test and `docs/ONBOARDING.md` name the engine; new permanent invariance test `tests/comparison/comparison-engine-invariance.test.ts` (synthetic, real local repository: a comparison leaves export, revision and models unchanged; one verified fail-to-pass, one confirmed open). Verified: import gate 528/528 (twice); main suite 2,451 passed, 18 optional skipped, 1 failed: the pre-existing `db-structure-layout.test.ts` tile-radius check, unrelated (no CSS or DB Structure file changed). The first gate run hit the documented 5-second timeout of the combined Node import test once; it passed alone (757 ms) and in the rerun, limit unchanged. Both production builds pass (existing chunk-size warning only). No product version change: no visible behaviour or contract changed. -Change log 2026-10-13: recorded the Comparison Engine decision, naming and staging; relocated the Core Phase 8 scan-run comparison and registered the Comparison Engine with its invariance test. +Step 3a (engine paths only, no Core release): `src/comparison/timeslot-comparison-engine.ts` (result 1.1.0) separates new from regressed scanner deviations and accepts a timeslot without scanner runs on one side; new `gpo-timeslot-comparison.ts` pairs rolled-up GPO Work verdicts across timeslots; new `timeslot-comparison-report.ts` (`compareTimeslots`, 1.0.0) yields the UI change categories (resolved, unchanged, value-changed, unverified, not-comparable, not-applicable, new, regressed; target-side deviations of a one-sided source are not comparable) and the A/B score basis over comparable sources only. Contract details in TIMESLOT_COMPARISON_PLAN.md P2/P4. Open for 3b: a read port that builds both timeslots' models and GPO Work results (a timeslot can hold several assessments; the GPO backup time comes from `backupCreatedAt`), Security Domain deltas, the backend route and the UI. 15 synthetic engine tests. Verified: import gate 528/528; main suite 2,462 passed, 18 optional skipped, 1 failed (the same pre-existing, unrelated `db-structure-layout.test.ts` tile-radius check); local-only and design-language checks pass; both production builds pass. No product version change: no visible behaviour. + +Change log 2026-10-13: recorded the Comparison Engine decision, naming and staging; relocated the Core Phase 8 scan-run comparison and registered the Comparison Engine with its invariance test; step 3a engine comparison of scanners, GPO settings and score basis. ## Results cluster foreground correction (2026-10-13, Phase 7 Presentation, 2.25.6) diff --git a/docs/TIMESLOT_COMPARISON_PLAN.md b/docs/TIMESLOT_COMPARISON_PLAN.md index 4df38f5..f26cd02 100644 --- a/docs/TIMESLOT_COMPARISON_PLAN.md +++ b/docs/TIMESLOT_COMPARISON_PLAN.md @@ -89,9 +89,9 @@ Die Paketstände beziehen sich ausschließlich auf nachgewiesene Ergebnisse dies |---|---|---| | P0 · abgeschlossen | Synthetisches Bestandsaudit je Scanner/Format/GPO: Regelgranularität, bereits erhaltene Objekt-/Scopefelder, Quellenpfade, Vollständigkeitssignale und fehlende Informationen dokumentieren; notwendige Vertragsänderungen und Reimportbetroffenheit bestimmen | [TIMESLOT_COMPARISON_AUDIT.md](TIMESLOT_COMPARISON_AUDIT.md); keine echten Kundendaten; Fähigkeiten und Grenzen je unterstütztem Pfad dokumentiert | | P1 · in Arbeit | Versionierte Vergleichsidentität, Gegenprüfungs-/Diagnosevertrag und getrennte Ergebnis-/Verpflichtungszustände in der Domäne spezifizieren; Persistenz-/API-Entwurf und Übergangsregeln festlegen | Domänen- und Profilverträge sowie synthetische Scope-/Zeitfälle umgesetzt; Persistenz-/API- und Übergangsentwurf bleiben offen | -| P2 · in Arbeit | Scanner-/GPO-Paarung anhand echter Quellreferenzen; vollständige Kompatibilitätsprüfungen; objektgenauer Lebenszyklus, Fail-zu-Pass, konservative Abwesenheit und einheitliche aktuelle Abweichungsmetriken | Sicherer Scannerpfad mit echter Tool-/Run-/File-Paarung, Processing-, Scope-, Reportzeit- und Diagnosegrenzen umgesetzt; GPO, Historie, Produktintegration und vollständige Matrix bleiben offen | +| P2 · in Arbeit | Scanner-/GPO-Paarung anhand echter Quellreferenzen; vollständige Kompatibilitätsprüfungen; objektgenauer Lebenszyklus, Fail-zu-Pass, konservative Abwesenheit und einheitliche aktuelle Abweichungsmetriken | Sicherer Scannerpfad mit echter Tool-/Run-/File-Paarung, Processing-, Scope-, Reportzeit- und Diagnosegrenzen umgesetzt. 2026-10-13 in der Comparison Engine: Scanner trennen `neu` und `wieder aufgetreten` (genau ein belegtes Pass derselben Identität in A; mehrdeutige Historie behauptet nichts); einseitige Quellen sind nicht vergleichbar; GPO-Paarung über die unverändert gelesenen, wie in Work hochgerollten Verdikte je Scope und technischer Identität, nur mit neuerer Backupzeit und gleicher Baselinemenge; fehlende oder nicht entschiedene Einstellung bleibt ungeklärt; geänderter Istwert einer weiter abweichenden Einstellung ist neutral (`value-changed`). Historie, Produktintegration und vollständige Matrix bleiben offen | | P3 | Verpflichtungsbezogene Fortschreibung über alle Zwischenstände, getrennt von der Score-Baseline; atomare/idempotente Ablage und API gemäß P1, Wiederladen und auditierter Scope-Ausstieg | P1/P2; nichts verschwindet bei Teilimport, Reload, Reimport oder mehrdeutiger Zuordnung | -| P4 | Explizite A/B-Scorebasis, objekt-/scopebewusste Population und getrennte Freigaben für Delta/Gegenprüfung; native Scoregrenzen und Coverageausgabe | P1/P2/P3; keine Verbesserung durch Quellenauslassung, Objektwechsel oder historische Baselinevermischung | +| P4 · begonnen | Explizite A/B-Scorebasis, objekt-/scopebewusste Population und getrennte Freigaben für Delta/Gegenprüfung; native Scoregrenzen und Coverageausgabe | P1/P2/P3; keine Verbesserung durch Quellenauslassung, Objektwechsel oder historische Baselinevermischung. 2026-10-13: Delta nur über Quellen mit vergleichbarem Paar in A und B (Scanner über `compareSecurityScores`, GPO als Pass-Rate über in beiden Timeslots entschiedene Einstellungen derselben Baselines); jede ausgeschlossene Quelle mit Grund. Security-Domain-Deltas, Coverageausgabe und P3-Abhängigkeit offen | | P5 | A/B-Auswahl und dreiteiliger Bericht in der vorhandenen Oberfläche; Gründe, Messbasis, Quellen/Versionen und Belege pro Aussage; aktueller Workspace bleibt unverändert nutzbar | P2-P4 fachlich verifiziert; UI ist Konsument der Domänenlogik, kein eigener Matcher | | P6 | Gesamtabnahme, Dokumentationsabgleich und kontrollierte Aktivierung | Alle unten genannten Fälle; keine Freigabe allein wegen vorhandener Funktion oder grüner Buildausgabe | diff --git a/src/comparison/gpo-timeslot-comparison.ts b/src/comparison/gpo-timeslot-comparison.ts new file mode 100644 index 0000000..c3aa37e --- /dev/null +++ b/src/comparison/gpo-timeslot-comparison.ts @@ -0,0 +1,164 @@ +import type { RemediationEvidenceState } from '../domain/timeslot-comparison'; +import type { PolicyWorkResult, PolicyWorkStatus } from '../evaluation/policy-work-results'; +import { policyWorkVerdictPriority } from '../evaluation/policy-work-results'; +import { canonicalJson } from '../shared/canonical-json'; + +export const GPO_TIMESLOT_COMPARISON_VERSION = '1.0.0' as const; + +/** One timeslot's rolled-up GPO settings exactly as Work reads them, plus its backup observation time. */ +export interface GpoTimeslotInput { + results: PolicyWorkResult[]; + /** Latest backup creation time of the timeslot's GPO backups; never the import time. */ + observedAt?: string; +} + +type SettingVerdict = Exclude; + +interface TimeslotSetting { + key: string; + scope: string; + identity: string; + verdict?: SettingVerdict; + baselineIds: string; + actualValues: string; + resultIds: string[]; + ambiguous: boolean; +} + +export interface GpoSettingDecision { + /** Cross-timeslot setting identity: scope and technical identity only, never a run or assessment. */ + settingKey: string; + scope: string; + identity: string; + baselineVerdict: SettingVerdict; + targetVerdict?: SettingVerdict; + state: RemediationEvidenceState; + /** Retained configured value differs while the setting still deviates. Neutral, never better or worse. */ + valueChanged: boolean; + reasons: string[]; + baselineResultIds: string[]; + targetResultIds: string[]; +} + +export interface GpoTargetChange { + settingKey: string; + baselineVerdict?: SettingVerdict; + targetResultIds: string[]; +} + +export interface GpoTimeslotComparison { + version: typeof GPO_TIMESLOT_COMPARISON_VERSION; + state: 'compared' | 'not-comparable'; + reasons: string[]; + decisions: GpoSettingDecision[]; + newDeviations: GpoTargetChange[]; + regressedDeviations: GpoTargetChange[]; + /** Pass rate over settings decided (Pass or Fail) in both timeslots against the same baselines. */ + sharedPassRate?: { settings: number; baseline: number; target: number }; +} + +const settingKey = (scope: string, identity: string) => JSON.stringify([scope, identity]); + +/** + * Rolls each timeslot up like Work: one verdict per technical setting, chosen by the shared + * verdict priority. Conflict and duplicate entries are diagnostics and never a verdict. Several + * aggregation runs holding the same setting in one timeslot make it ambiguous. + */ +function rollUp(results: PolicyWorkResult[]): Map { + const byIdentity = new Map(); + for (const result of results) { + const key = settingKey(result.scope, result.identity); + byIdentity.set(key, [...(byIdentity.get(key) ?? []), result]); + } + const settings = new Map(); + for (const [key, group] of byIdentity) { + const verdicts = group.filter((item): item is PolicyWorkResult & { status: SettingVerdict } => + item.status !== 'Conflicts' && item.status !== 'Duplicates'); + const verdict = verdicts.reduce((best, item) => + !best || policyWorkVerdictPriority[item.status] > policyWorkVerdictPriority[best] ? item.status : best, undefined); + const decisive = verdicts.filter(item => item.status === verdict); + settings.set(key, { + key, scope: group[0].scope, identity: group[0].identity, verdict, + baselineIds: canonicalJson([...new Set(verdicts.flatMap(item => item.comparison?.baselineId ?? []))].sort()), + actualValues: canonicalJson([...new Set(decisive.flatMap(item => + item.comparison?.comparator.actual === undefined ? [] : [canonicalJson(item.comparison.comparator.actual)]))].sort()), + resultIds: group.map(item => item.id).sort(), + ambiguous: new Set(group.flatMap(item => item.settingKey ?? [])).size > 1 + }); + } + return settings; +} + +const newer = (baseline?: string, target?: string) => Boolean(baseline && target + && Number.isFinite(Date.parse(baseline)) && Number.isFinite(Date.parse(target)) && Date.parse(target) > Date.parse(baseline)); + +/** + * Conservative A/B comparison of rolled-up GPO settings. The authoritative verdicts are read as + * they are and never re-derived. Remediation needs the same setting with an explicit Pass against the + * same baselines in a newer backup set; an absent or undecided setting stays unverified, because a + * GPO backup set is no proof of a removed configuration. + */ +export function compareGpoTimeslots(baseline: GpoTimeslotInput, target: GpoTimeslotInput): GpoTimeslotComparison { + const version = GPO_TIMESLOT_COMPARISON_VERSION; + const baselineSettings = rollUp(baseline.results); + const targetSettings = rollUp(target.results); + const baselineFails = [...baselineSettings.values()].filter(item => item.verdict === 'Fail') + .sort((left, right) => left.key.localeCompare(right.key)); + const sourceReasons = [ + ...(baselineSettings.size ? [] : ['baseline-source-missing']), + ...(targetSettings.size ? [] : ['target-source-missing']) + ]; + const decision = (item: TimeslotSetting, state: RemediationEvidenceState, reasons: string[], other?: TimeslotSetting, + valueChanged = false): GpoSettingDecision => ({ + settingKey: item.key, scope: item.scope, identity: item.identity, baselineVerdict: 'Fail', + ...(other?.verdict ? { targetVerdict: other.verdict } : {}), state, valueChanged, reasons, + baselineResultIds: item.resultIds, targetResultIds: other?.resultIds ?? [] + }); + if (sourceReasons.length) return { + version, state: 'not-comparable', reasons: sourceReasons, + decisions: baselineFails.map(item => decision(item, 'not-comparable', sourceReasons)), + newDeviations: [], regressedDeviations: [] + }; + const timeVerified = newer(baseline.observedAt, target.observedAt); + const decisions = baselineFails.map(item => { + if (item.ambiguous) return decision(item, 'not-comparable', ['gpo-setting-ambiguous']); + const other = targetSettings.get(item.key); + if (!timeVerified) return decision(item, 'unverified-absent', ['newer-observation-not-verified'], other); + if (!other) return decision(item, 'unverified-absent', ['target-setting-absent']); + if (other.ambiguous) return decision(item, 'unverified-absent', ['gpo-setting-ambiguous'], other); + if (other.baselineIds !== item.baselineIds) return decision(item, 'not-comparable', ['baseline-set-changed'], other); + if (other.verdict === 'Fail') return decision(item, 'confirmed-open', ['same-setting-current-deviation'], other, + other.actualValues !== item.actualValues); + if (other.verdict === 'Pass') return decision(item, 'verified-remediated', ['same-setting-explicit-pass'], other); + return decision(item, 'unverified-absent', [`target-verdict-not-decisive:${other.verdict ?? 'none'}`], other); + }); + + const newDeviations: GpoTargetChange[] = []; + const regressedDeviations: GpoTargetChange[] = []; + if (timeVerified) { + for (const item of [...targetSettings.values()].sort((left, right) => left.key.localeCompare(right.key))) { + if (item.verdict !== 'Fail' || item.ambiguous) continue; + const before = baselineSettings.get(item.key); + if (before?.ambiguous || before?.verdict === 'Fail') continue; + const change = { settingKey: item.key, ...(before?.verdict ? { baselineVerdict: before.verdict } : {}), targetResultIds: item.resultIds }; + // A regression needs an explicit Pass against the same baselines; undecided history claims nothing. + if (before?.verdict === 'Pass') { if (before.baselineIds === item.baselineIds) regressedDeviations.push(change); } + else if (!before?.verdict || before.verdict === 'Not configured' || before.verdict === 'No baseline rule') newDeviations.push(change); + } + } + + const shared = [...baselineSettings.values()].flatMap(item => { + const other = targetSettings.get(item.key); + return other && !item.ambiguous && !other.ambiguous && item.baselineIds === other.baselineIds + && (item.verdict === 'Pass' || item.verdict === 'Fail') && (other.verdict === 'Pass' || other.verdict === 'Fail') + ? [{ before: item.verdict, after: other.verdict }] : []; + }); + const rate = (values: string[]) => values.filter(value => value === 'Pass').length / values.length * 100; + return { + version, state: 'compared', reasons: timeVerified ? [] : ['newer-observation-not-verified'], + decisions, newDeviations, regressedDeviations, + ...(timeVerified && shared.length ? { sharedPassRate: { + settings: shared.length, baseline: rate(shared.map(item => item.before)), target: rate(shared.map(item => item.after)) + } } : {}) + }; +} diff --git a/src/comparison/timeslot-comparison-engine.ts b/src/comparison/timeslot-comparison-engine.ts index 296a49c..053e93b 100644 --- a/src/comparison/timeslot-comparison-engine.ts +++ b/src/comparison/timeslot-comparison-engine.ts @@ -27,11 +27,14 @@ export interface TimeslotSourceComparison { state: 'compared' | 'not-comparable'; reasons: string[]; decisions: TimeslotOccurrenceDecision[]; + /** Current deviations whose identity was not a deviation and not a verified pass in A. */ newDeviationOccurrenceIds: string[]; + /** Current deviations whose identity was a single verified pass in A. */ + regressedDeviationOccurrenceIds: string[]; } export interface ProjectTimeslotEvidenceComparison { - version: '1.0.0'; + version: '1.1.0'; projectId: string; baselineRunIds: string[]; targetRunIds: string[]; @@ -44,6 +47,7 @@ export interface ProjectTimeslotEvidenceComparison { notComparable: number; notApplicable: number; newDeviations: number; + regressedDeviations: number; }; } @@ -108,7 +112,8 @@ export function compareProjectTimeslotEvidence( if (!models.length) throw new Error('Timeslot comparison requires project data.'); const projectId = models[0].project.id; if (models.some(model => model.project.id !== projectId)) throw new Error('Timeslots must belong to the same project.'); - if (!baselineRunIds.length || !targetRunIds.length) throw new Error('Both timeslots require scanner runs.'); + // One side may lack scanner runs; its sources are then reported as not comparable, never resolved. + if (!baselineRunIds.length && !targetRunIds.length) throw new Error('At least one timeslot requires scanner runs.'); if (baselineRunIds.some(id => targetRunIds.includes(id))) throw new Error('Baseline and target timeslots must use different runs.'); const runs = models.flatMap(model => model.scanRuns); @@ -148,7 +153,8 @@ export function compareProjectTimeslotEvidence( if (!baselineRun || !targetRun || !profile || profile.timeslotComparison.kind === 'unavailable') return { toolId, baselineRunId: baselineRun?.id, targetRunId: targetRun?.id, state: 'not-comparable', reasons: pairingReasons, - decisions: unmatchedDecisions(baselineOccurrences, pairingReasons), newDeviationOccurrenceIds: [] + decisions: unmatchedDecisions(baselineOccurrences, pairingReasons), newDeviationOccurrenceIds: [], + regressedDeviationOccurrenceIds: [] }; const definition = profile.timeslotComparison; @@ -180,7 +186,8 @@ export function compareProjectTimeslotEvidence( || baselineTime.state !== 'verified' || targetTime.state !== 'verified') return { toolId, baselineRunId: baselineRun.id, targetRunId: targetRun.id, state: 'not-comparable', reasons: [...new Set(sourceReasons)], - decisions: unmatchedDecisions(baselineOccurrences, [...new Set(sourceReasons)]), newDeviationOccurrenceIds: [] + decisions: unmatchedDecisions(baselineOccurrences, [...new Set(sourceReasons)]), newDeviationOccurrenceIds: [], + regressedDeviationOccurrenceIds: [] }; const targetOccurrences = occurrences.filter(item => item.scanRunId === targetRun.id); @@ -225,20 +232,30 @@ export function compareProjectTimeslotEvidence( }; }); const baselineKeys = new Set(baselineIdentities.flatMap(entry => entry.identity.state === 'verified' ? [entry.identity.identity.key] : [])); - const newDeviationOccurrenceIds = targetIdentities.flatMap(entry => { - if (entry.identity.state !== 'verified' || !isDeviation(Boolean(entry.item.isIoe), occurrenceOutcome(entry.item))) return []; - if ((targetByIdentity.get(entry.identity.identity.key) ?? []).length !== 1) return []; - return baselineKeys.has(entry.identity.identity.key) ? [] : [entry.item.id]; + // A regression needs exactly one explicit pass of the same identity in A. + const baselinePassCounts = new Map(); + baselineOccurrences.filter(item => occurrenceOutcome(item) === 'pass').forEach(item => { + const identity = identityFor(item, definition, baselineScope.scope); + if (identity.state === 'verified') baselinePassCounts.set(identity.identity.key, (baselinePassCounts.get(identity.identity.key) ?? 0) + 1); }); + const currentDeviations = targetIdentities.filter(entry => entry.identity.state === 'verified' + && isDeviation(Boolean(entry.item.isIoe), occurrenceOutcome(entry.item)) + && (targetByIdentity.get(entry.identity.identity.key) ?? []).length === 1 + && !baselineKeys.has(entry.identity.identity.key)); + const keyOf = (entry: typeof targetIdentities[number]) => entry.identity.state === 'verified' ? entry.identity.identity.key : ''; + const regressed = currentDeviations.filter(entry => baselinePassCounts.get(keyOf(entry)) === 1); + // Several passes of one identity in A are ambiguous history: neither new nor regressed. + const newDeviationOccurrenceIds = currentDeviations.filter(entry => !baselinePassCounts.has(keyOf(entry))).map(entry => entry.item.id); return { toolId, baselineRunId: baselineRun.id, targetRunId: targetRun.id, - state: 'compared', reasons: [...new Set(blockingDiagnostics)], decisions, newDeviationOccurrenceIds + state: 'compared', reasons: [...new Set(blockingDiagnostics)], decisions, newDeviationOccurrenceIds, + regressedDeviationOccurrenceIds: regressed.map(entry => entry.item.id) }; }); const decisions = sources.flatMap(source => source.decisions); return { - version: '1.0.0', projectId, baselineRunIds: [...baselineRunIds], targetRunIds: [...targetRunIds], sources, + version: '1.1.0', projectId, baselineRunIds: [...baselineRunIds], targetRunIds: [...targetRunIds], sources, summary: { baselineDeviations: decisions.length, verifiedRemediated: decisions.filter(item => item.state === 'verified-remediated').length, @@ -246,7 +263,8 @@ export function compareProjectTimeslotEvidence( unverifiedAbsent: decisions.filter(item => item.state === 'unverified-absent').length, notComparable: decisions.filter(item => item.state === 'not-comparable').length, notApplicable: decisions.filter(item => item.state === 'not-applicable').length, - newDeviations: sources.reduce((sum, source) => sum + source.newDeviationOccurrenceIds.length, 0) + newDeviations: sources.reduce((sum, source) => sum + source.newDeviationOccurrenceIds.length, 0), + regressedDeviations: sources.reduce((sum, source) => sum + source.regressedDeviationOccurrenceIds.length, 0) } }; } diff --git a/src/comparison/timeslot-comparison-report.ts b/src/comparison/timeslot-comparison-report.ts new file mode 100644 index 0000000..069d8a2 --- /dev/null +++ b/src/comparison/timeslot-comparison-report.ts @@ -0,0 +1,227 @@ +import type { CanonicalAssessmentModel, FindingOccurrence } from '../domain/canonical-model'; +import type { RemediationEvidenceState } from '../domain/timeslot-comparison'; +import { isDeviation } from '../domain/deviation-predicate'; +import { outcomeFromLegacyStatus } from '../domain/unified-result'; +import type { SecurityScanScore } from '../domain/security-scoring'; +import { compareSecurityScores, defaultSecurityScoringPolicy, weightedSecurityScore } from '../domain/security-scoring'; +import { canonicalSecurityScores } from '../scoring/canonical-security-score'; +import { compareProjectTimeslotEvidence, type ProjectTimeslotEvidenceComparison } from './timeslot-comparison-engine'; +import { compareGpoTimeslots, type GpoTimeslotComparison, type GpoTimeslotInput } from './gpo-timeslot-comparison'; + +export const TIMESLOT_COMPARISON_REPORT_VERSION = '1.0.0' as const; +export const GPO_SOURCE_ID = 'group-policy'; + +/** + * Change categories shown by the Comparison Engine. `value-changed` is a still deviating setting + * with a different retained value and is neutral (user decision 2026-10-13). Only `resolved` is better; + * only `new` and `regressed` are worse. + */ +export type TimeslotChange = 'resolved' | 'unchanged' | 'value-changed' | 'unverified' | 'not-comparable' + | 'not-applicable' | 'new' | 'regressed'; + +export interface TimeslotChangeEntry { + sourceId: string; + change: TimeslotChange; + /** Which timeslot the referenced records belong to. */ + side: 'baseline' | 'target'; + /** Scanner occurrence IDs or rolled-up GPO Work result IDs. */ + recordIds: string[]; + /** Counterpart records in the other timeslot, when paired. */ + counterpartIds: string[]; + identityKey?: string; + reasons: string[]; +} + +export interface TimeslotSourceScore { + sourceId: string; + baselineScore?: number; + targetScore?: number; + change?: number; + weight: number; + method: 'paired-scans' | 'shared-gpo-pass-rate'; + reasons: string[]; +} + +export interface TimeslotComparisonReport { + version: typeof TIMESLOT_COMPARISON_REPORT_VERSION; + projectId: string; + baselineRunIds: string[]; + targetRunIds: string[]; + sources: Array<{ sourceId: string; state: 'compared' | 'not-comparable'; reasons: string[] }>; + changes: TimeslotChangeEntry[]; + summary: Record & { targetNotComparable: number }; + score: { + /** Weighted only over sources with a comparable score in both timeslots. */ + baselineScore?: number; + targetScore?: number; + change?: number; + sources: TimeslotSourceScore[]; + excludedSources: Array<{ sourceId: string; reasons: string[] }>; + }; + scanner: ProjectTimeslotEvidenceComparison; + gpo?: GpoTimeslotComparison; +} + +const changeOf: Record = { + 'verified-remediated': 'resolved', + 'confirmed-open': 'unchanged', + 'unverified-absent': 'unverified', + 'not-comparable': 'not-comparable', + 'not-applicable': 'not-applicable' +}; + +const outcomeOf = (item: FindingOccurrence) => item.outcome?.normalizedValue + ?? outcomeFromLegacyStatus(item.status.normalizedValue ?? 'unknown'); + +function scannerChanges(models: CanonicalAssessmentModel[], scanner: ProjectTimeslotEvidenceComparison): TimeslotChangeEntry[] { + const occurrences = models.flatMap(model => model.findingOccurrences); + return scanner.sources.flatMap(source => [ + ...source.decisions.map((item): TimeslotChangeEntry => ({ + sourceId: source.toolId, change: changeOf[item.state], side: 'baseline', recordIds: [item.baselineOccurrenceId], + counterpartIds: item.targetOccurrenceId ? [item.targetOccurrenceId] : [], + ...(item.identityKey ? { identityKey: item.identityKey } : {}), reasons: item.reasons + })), + ...source.newDeviationOccurrenceIds.map((id): TimeslotChangeEntry => ({ + sourceId: source.toolId, change: 'new', side: 'target', recordIds: [id], counterpartIds: [], reasons: [] + })), + ...source.regressedDeviationOccurrenceIds.map((id): TimeslotChangeEntry => ({ + sourceId: source.toolId, change: 'regressed', side: 'target', recordIds: [id], counterpartIds: [], reasons: [] + })), + // Deviations of a source that cannot be compared are neither new nor worse. + ...(source.state === 'not-comparable' && source.targetRunId ? occurrences + .filter(item => item.scanRunId === source.targetRunId && isDeviation(Boolean(item.isIoe), outcomeOf(item))) + .map((item): TimeslotChangeEntry => ({ + sourceId: source.toolId, change: 'not-comparable', side: 'target', recordIds: [item.id], counterpartIds: [], + reasons: source.reasons + })) : []) + ]); +} + +function gpoChanges(gpo: GpoTimeslotComparison, target?: GpoTimeslotInput): TimeslotChangeEntry[] { + const changes = [ + ...gpo.decisions.map((item): TimeslotChangeEntry => ({ + sourceId: GPO_SOURCE_ID, + change: item.state === 'confirmed-open' && item.valueChanged ? 'value-changed' : changeOf[item.state], + side: 'baseline', recordIds: item.baselineResultIds, counterpartIds: item.targetResultIds, + identityKey: item.settingKey, reasons: item.reasons + })), + ...gpo.newDeviations.map((item): TimeslotChangeEntry => ({ + sourceId: GPO_SOURCE_ID, change: 'new', side: 'target', recordIds: item.targetResultIds, counterpartIds: [], + identityKey: item.settingKey, reasons: [] + })), + ...gpo.regressedDeviations.map((item): TimeslotChangeEntry => ({ + sourceId: GPO_SOURCE_ID, change: 'regressed', side: 'target', recordIds: item.targetResultIds, counterpartIds: [], + identityKey: item.settingKey, reasons: [] + })) + ]; + if (gpo.state === 'not-comparable' && target) { + const keys = new Set(target.results.filter(item => item.status === 'Fail').map(item => JSON.stringify([item.scope, item.identity]))); + for (const key of [...keys].sort()) changes.push({ + sourceId: GPO_SOURCE_ID, change: 'not-comparable', side: 'target', identityKey: key, counterpartIds: [], + recordIds: target.results.filter(item => JSON.stringify([item.scope, item.identity]) === key).map(item => item.id).sort(), + reasons: gpo.reasons + }); + } + return changes; +} + +function scannerScores(models: CanonicalAssessmentModel[], baselineRunIds: string[], targetRunIds: string[]) { + const scans = canonicalSecurityScores(models); + const pick = (ids: string[]) => scans.filter(scan => scan.runIds.some(id => ids.includes(id))); + const baseline = pick(baselineRunIds); + const target = pick(targetRunIds); + const byTool = (items: SecurityScanScore[]) => { + const map = new Map(); + items.forEach(item => map.set(item.toolId, [...(map.get(item.toolId) ?? []), item])); + return map; + }; + const baselineByTool = byTool(baseline); + const targetByTool = byTool(target); + return [...new Set([...baselineByTool.keys(), ...targetByTool.keys()])].sort().map((toolId): TimeslotSourceScore => { + const left = baselineByTool.get(toolId) ?? []; + const right = targetByTool.get(toolId) ?? []; + const policy = (left[0] ?? right[0]).policy; + const weight = policy.weights[toolId] ?? policy.defaultWeight; + if (!left.length || !right.length) return { + sourceId: toolId, weight, method: 'paired-scans', reasons: [left.length ? 'target-source-missing' : 'baseline-source-missing'] + }; + const compared = compareSecurityScores(left, right); + const reasons = [ + ...(compared.comparable === 0 ? ['no-comparable-scan-pair'] : []), + ...(compared.scopeChanged ? ['scan-scope-changed'] : []) + ]; + return { + sourceId: toolId, weight, method: 'paired-scans', reasons, + ...(compared.change !== undefined && !compared.scopeChanged ? { + baselineScore: compared.baselineScore, targetScore: compared.currentScore, change: compared.change + } : {}) + }; + }); +} + +/** + * Explicit A/B comparison of two timeslots of one client. Scanner sources use the conservative + * evidence comparison, GPO settings their rolled-up Work verdicts. Score deltas cover only sources + * with a comparable score in both timeslots; every other source is listed as excluded with its reason. + */ +export function compareTimeslots(input: { + projectId: string; + models: CanonicalAssessmentModel[]; + baselineRunIds: string[]; + targetRunIds: string[]; + gpo?: { baseline: GpoTimeslotInput; target: GpoTimeslotInput }; +}): TimeslotComparisonReport { + if (input.models.some(model => model.project.id !== input.projectId)) throw new Error('Timeslots must belong to the same project.'); + const scanner: ProjectTimeslotEvidenceComparison = input.baselineRunIds.length || input.targetRunIds.length + ? compareProjectTimeslotEvidence(input.models, input.baselineRunIds, input.targetRunIds) + : { version: '1.1.0', projectId: input.projectId, baselineRunIds: [], targetRunIds: [], sources: [], summary: { + baselineDeviations: 0, verifiedRemediated: 0, confirmedOpen: 0, unverifiedAbsent: 0, notComparable: 0, + notApplicable: 0, newDeviations: 0, regressedDeviations: 0 + } }; + const gpo = input.gpo && (input.gpo.baseline.results.length || input.gpo.target.results.length) + ? compareGpoTimeslots(input.gpo.baseline, input.gpo.target) : undefined; + const changes = [...scannerChanges(input.models, scanner), ...(gpo ? gpoChanges(gpo, input.gpo?.target) : [])]; + + const sourceScores = scannerScores(input.models, input.baselineRunIds, input.targetRunIds); + if (gpo) { + const policy = input.models[0]?.assessment.scoringPolicy ?? defaultSecurityScoringPolicy(); + const rate = gpo.sharedPassRate; + sourceScores.push({ + sourceId: GPO_SOURCE_ID, weight: policy.weights[GPO_SOURCE_ID] ?? policy.defaultWeight, method: 'shared-gpo-pass-rate', + reasons: rate ? [] : gpo.state === 'not-comparable' ? gpo.reasons : ['no-shared-decided-settings'], + ...(rate ? { baselineScore: rate.baseline, targetScore: rate.target, change: rate.target - rate.baseline } : {}) + }); + } + const scored = sourceScores.filter(item => item.change !== undefined); + const baselineScore = weightedSecurityScore(scored.map(item => ({ score: item.baselineScore, weight: item.weight }))); + const targetScore = weightedSecurityScore(scored.map(item => ({ score: item.targetScore, weight: item.weight }))); + + const count = (change: TimeslotChange, side?: 'baseline' | 'target') => + changes.filter(item => item.change === change && (!side || item.side === side)).length; + return { + version: TIMESLOT_COMPARISON_REPORT_VERSION, + projectId: input.projectId, + baselineRunIds: [...input.baselineRunIds], + targetRunIds: [...input.targetRunIds], + sources: [ + ...scanner.sources.map(source => ({ sourceId: source.toolId, state: source.state, reasons: source.reasons })), + ...(gpo ? [{ sourceId: GPO_SOURCE_ID, state: gpo.state, reasons: gpo.reasons }] : []) + ], + changes, + summary: { + resolved: count('resolved'), unchanged: count('unchanged'), 'value-changed': count('value-changed'), + unverified: count('unverified'), 'not-comparable': count('not-comparable', 'baseline'), + 'not-applicable': count('not-applicable'), new: count('new'), regressed: count('regressed'), + targetNotComparable: count('not-comparable', 'target') + }, + score: { + ...(baselineScore !== undefined && targetScore !== undefined + ? { baselineScore, targetScore, change: targetScore - baselineScore } : {}), + sources: sourceScores, + excludedSources: sourceScores.filter(item => item.change === undefined) + .map(item => ({ sourceId: item.sourceId, reasons: item.reasons })) + }, + scanner, + ...(gpo ? { gpo } : {}) + }; +} diff --git a/tests/comparison/gpo-timeslot-comparison.test.ts b/tests/comparison/gpo-timeslot-comparison.test.ts new file mode 100644 index 0000000..b18e601 --- /dev/null +++ b/tests/comparison/gpo-timeslot-comparison.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from 'vitest'; +import type { PolicyWorkResult, PolicyWorkStatus } from '../../src/evaluation/policy-work-results'; +import { compareGpoTimeslots } from '../../src/comparison/gpo-timeslot-comparison'; + +// Synthetic rolled-up GPO Work results. Only the fields the comparison reads are populated. +const observedA = '2026-01-01T10:00:00.000Z'; +const observedB = '2026-02-01T10:00:00.000Z'; + +function setting(id: string, identity: string, status: PolicyWorkStatus, options: { + actual?: unknown; baselineId?: string; aggregationRun?: string; scope?: string; +} = {}): PolicyWorkResult { + const scope = options.scope ?? 'computer'; + return { + id, status, title: identity, scope, identity, importedAt: observedA, sources: [], + settingKey: JSON.stringify(['dummy-project', options.aggregationRun ?? 'aggregation-1', scope, identity]), + comparison: { + baselineId: options.baselineId ?? 'baseline-server', + comparator: { actual: options.actual === undefined ? undefined : { kind: 'number', value: options.actual } } + } as unknown as PolicyWorkResult['comparison'] + }; +} + +describe('GPO timeslot comparison', () => { + it('pairs rolled-up settings by scope and identity and keeps a value change neutral', () => { + const result = compareGpoTimeslots( + { observedAt: observedA, results: [ + setting('a-1', 'MinimumPasswordLength', 'Fail', { actual: 7 }), + setting('a-2', 'LockoutBadCount', 'Fail', { actual: 0 }), + setting('a-3', 'NoLMHash', 'Fail', { actual: 0 }), + setting('a-4', 'PasswordComplexity', 'Fail', { actual: 0 }) + ] }, + { observedAt: observedB, results: [ + setting('b-1', 'MinimumPasswordLength', 'Fail', { actual: 10 }), + setting('b-2', 'LockoutBadCount', 'Fail', { actual: 0 }), + setting('b-3', 'NoLMHash', 'Pass', { actual: 1 }), + setting('b-4', 'PasswordComplexity', 'Not configured') + ] } + ); + const byIdentity = Object.fromEntries(result.decisions.map(item => [item.identity, item])); + expect(byIdentity.MinimumPasswordLength).toMatchObject({ state: 'confirmed-open', valueChanged: true, targetResultIds: ['b-1'] }); + expect(byIdentity.LockoutBadCount).toMatchObject({ state: 'confirmed-open', valueChanged: false }); + expect(byIdentity.NoLMHash).toMatchObject({ state: 'verified-remediated', reasons: ['same-setting-explicit-pass'] }); + // A setting missing from the newer backup set is not proof of a removed configuration. + expect(byIdentity.PasswordComplexity).toMatchObject({ state: 'unverified-absent', reasons: ['target-verdict-not-decisive:Not configured'] }); + }); + + it('rolls several baseline releases of one setting up with the Work verdict order', () => { + const result = compareGpoTimeslots( + { observedAt: observedA, results: [ + setting('a-1', 'SeBackupPrivilege', 'Pass', { baselineId: 'baseline-dc' }), + setting('a-2', 'SeBackupPrivilege', 'Fail', { baselineId: 'baseline-server' }) + ] }, + { observedAt: observedB, results: [ + setting('b-1', 'SeBackupPrivilege', 'Pass', { baselineId: 'baseline-dc' }), + setting('b-2', 'SeBackupPrivilege', 'Pass', { baselineId: 'baseline-server' }) + ] } + ); + expect(result.decisions).toHaveLength(1); + expect(result.decisions[0]).toMatchObject({ state: 'verified-remediated', baselineResultIds: ['a-1', 'a-2'] }); + }); + + it('separates new and regressed settings and claims nothing from undecided history', () => { + const result = compareGpoTimeslots( + { observedAt: observedA, results: [ + setting('a-1', 'SeEnableDelegationPrivilege', 'Not configured'), + setting('a-2', 'SeRestorePrivilege', 'Pass'), + setting('a-3', 'SeAuditPrivilege', 'Review required'), + setting('a-4', 'SeLoadDriverPrivilege', 'Pass', { baselineId: 'baseline-old' }) + ] }, + { observedAt: observedB, results: [ + setting('b-1', 'SeEnableDelegationPrivilege', 'Fail'), + setting('b-2', 'SeRestorePrivilege', 'Fail'), + setting('b-3', 'SeAuditPrivilege', 'Fail'), + setting('b-4', 'SeLoadDriverPrivilege', 'Fail'), + setting('b-5', 'DnsPolicyConfig', 'Fail') + ] } + ); + expect(result.newDeviations.map(item => item.targetResultIds[0]).sort()).toEqual(['b-1', 'b-5']); + expect(result.regressedDeviations.map(item => item.targetResultIds[0])).toEqual(['b-2']); + }); + + it('requires a newer backup set, the same baselines and a GPO source in both timeslots', () => { + const fail = { observedAt: observedA, results: [setting('a-1', 'NoLMHash', 'Fail')] }; + const sameTime = compareGpoTimeslots(fail, { observedAt: observedA, results: [setting('b-1', 'NoLMHash', 'Pass')] }); + expect(sameTime.decisions[0]).toMatchObject({ state: 'unverified-absent', reasons: ['newer-observation-not-verified'] }); + expect(sameTime.sharedPassRate).toBeUndefined(); + + const changedBaseline = compareGpoTimeslots(fail, { observedAt: observedB, results: [setting('b-1', 'NoLMHash', 'Pass', { baselineId: 'baseline-new' })] }); + expect(changedBaseline.decisions[0]).toMatchObject({ state: 'not-comparable', reasons: ['baseline-set-changed'] }); + + const missing = compareGpoTimeslots(fail, { observedAt: observedB, results: [] }); + expect(missing).toMatchObject({ state: 'not-comparable', reasons: ['target-source-missing'] }); + expect(missing.decisions[0].state).toBe('not-comparable'); + }); + + it('marks a setting held by several aggregation runs of one timeslot as ambiguous', () => { + const result = compareGpoTimeslots( + { observedAt: observedA, results: [ + setting('a-1', 'NoLMHash', 'Fail', { aggregationRun: 'aggregation-1' }), + setting('a-2', 'NoLMHash', 'Fail', { aggregationRun: 'aggregation-2' }) + ] }, + { observedAt: observedB, results: [setting('b-1', 'NoLMHash', 'Pass')] } + ); + expect(result.decisions[0]).toMatchObject({ state: 'not-comparable', reasons: ['gpo-setting-ambiguous'] }); + }); + + it('computes the shared pass rate only over settings decided in both timeslots', () => { + const result = compareGpoTimeslots( + { observedAt: observedA, results: [ + setting('a-1', 'One', 'Fail'), setting('a-2', 'Two', 'Fail'), setting('a-3', 'Three', 'Pass'), + setting('a-4', 'OnlyInA', 'Fail') + ] }, + { observedAt: observedB, results: [ + setting('b-1', 'One', 'Pass'), setting('b-2', 'Two', 'Fail'), setting('b-3', 'Three', 'Pass'), + setting('b-5', 'OnlyInB', 'Pass') + ] } + ); + expect(result.sharedPassRate?.settings).toBe(3); + expect(result.sharedPassRate?.baseline).toBeCloseTo(100 / 3); + expect(result.sharedPassRate?.target).toBeCloseTo(200 / 3); + }); +}); diff --git a/tests/comparison/timeslot-comparison-engine.test.ts b/tests/comparison/timeslot-comparison-engine.test.ts index 32ec215..fb7e90b 100644 --- a/tests/comparison/timeslot-comparison-engine.test.ts +++ b/tests/comparison/timeslot-comparison-engine.test.ts @@ -89,12 +89,37 @@ describe('conservative project timeslot comparison', () => { const result = compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']); expect(result.summary).toEqual({ baselineDeviations: 2, verifiedRemediated: 1, confirmedOpen: 0, - unverifiedAbsent: 1, notComparable: 0, notApplicable: 0, newDeviations: 1 + unverifiedAbsent: 1, notComparable: 0, notApplicable: 0, newDeviations: 1, regressedDeviations: 0 }); expect(result.sources[0].decisions.map(item => item.state).sort()) .toEqual(['unverified-absent', 'verified-remediated']); }); + it('separates a regression after a single verified pass from a new deviation and ambiguous history', () => { + const baseline = model('run-a', observedA, [ + occurrence('a-1', 'run-a', 'RULE-1', 'passed'), + occurrence('a-2', 'run-a', 'RULE-2', 'passed'), + occurrence('a-2b', 'run-a', 'RULE-2', 'passed') + ]); + const target = model('run-b', observedB, [ + occurrence('b-1', 'run-b', 'RULE-1', 'failed'), + occurrence('b-2', 'run-b', 'RULE-2', 'failed'), + occurrence('b-3', 'run-b', 'RULE-3', 'failed') + ]); + const result = compareProjectTimeslotEvidence([baseline, target], ['run-a'], ['run-b']); + expect(result.version).toBe('1.1.0'); + expect(result.sources[0].regressedDeviationOccurrenceIds).toEqual(['b-1']); + expect(result.sources[0].newDeviationOccurrenceIds).toEqual(['b-3']); + expect(result.summary).toMatchObject({ newDeviations: 1, regressedDeviations: 1, baselineDeviations: 0 }); + }); + + it('reports a source of only one timeslot as not comparable instead of failing', () => { + const target = model('run-b', observedB, [occurrence('b-1', 'run-b', 'RULE-1', 'failed')]); + const result = compareProjectTimeslotEvidence([target], [], ['run-b']); + expect(result.sources).toMatchObject([{ state: 'not-comparable', reasons: ['baseline-source-missing'] }]); + expect(result.summary).toMatchObject({ newDeviations: 0, regressedDeviations: 0 }); + }); + it('does not treat a missing scanner source or a same-time reimport as remediation', () => { const baseline = model('run-a', observedA, [occurrence('a-1', 'run-a', 'RULE-1', 'failed')]); const otherBaseline = model('run-c', observedA, []); diff --git a/tests/comparison/timeslot-comparison-report.test.ts b/tests/comparison/timeslot-comparison-report.test.ts new file mode 100644 index 0000000..2e3d987 --- /dev/null +++ b/tests/comparison/timeslot-comparison-report.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from 'vitest'; +import type { + CanonicalAssessmentModel, + CanonicalFindingStatus, + FindingOccurrence, + ImportFileRecord +} from '../../src/domain/canonical-model'; +import type { EnvironmentObservation } from '../../src/domain/environment-context'; +import type { PolicyWorkResult, PolicyWorkStatus } from '../../src/evaluation/policy-work-results'; +import { compareTimeslots } from '../../src/comparison/timeslot-comparison-report'; + +// Synthetic two-timeslot client: CIS-CAT in both, Maester only in B, GPO settings in both. +const observedA = '2026-01-01T10:00:00.000Z'; +const observedB = '2026-02-01T10:00:00.000Z'; +const provenance = { + decisionSource: 'source-native' as const, + confidence: { level: 'high' as const, score: 1, reasons: ['synthetic'] }, + mappingVersion: '1' +}; +const tools = { 'cis-cat': { id: 'cis-cat', name: 'CIS-CAT Pro' }, maester: { id: 'maester', name: 'Maester' } }; + +function occurrence(id: string, runId: string, toolId: keyof typeof tools, ruleId: string, status: CanonicalFindingStatus): FindingOccurrence { + const outcome = status === 'passed' ? 'pass' : 'fail'; + return { + id, definitionId: `definition-${ruleId}`, scanRunId: runId, importFileId: `file-${runId}`, + sourceIdentity: { tool: tools[toolId], canonicalRuleId: ruleId }, sourcePosition: { rowIndex: 1 }, + originalFields: [], fieldMappings: [], + severity: { originalValue: 'high', normalizedValue: 'high', provenance }, + status: { originalValue: status, normalizedValue: status, provenance }, + outcome: { originalValue: outcome, normalizedValue: outcome, provenance } + }; +} + +function importFile(runId: string, toolId: keyof typeof tools): ImportFileRecord { + return { + id: `file-${runId}`, scanRunId: runId, fileName: `${runId}.html`, format: 'HTML', sizeBytes: 10, importedAt: observedA, + detectedTool: tools[toolId], detection: { level: 'high', score: 1, reasons: ['synthetic'] }, + processedWith: { + parserId: 'html', parserVersion: '1', profileId: toolId, profileVersion: '1.4.0', mappingVersion: '1.3.0', + normalizationVersion: '1.3.0', presentationVersion: '1.3.0', compatibilityKey: `${toolId}-canonical-v1` + }, + sourceArtifact: { kind: 'not-retained', reason: 'synthetic' } + }; +} + +function observation(runId: string, toolId: keyof typeof tools, property: string, value: string): EnvironmentObservation { + return { + version: 1, id: `${runId}-${property}`, scanRunId: runId, importFileId: `file-${runId}`, tool: tools[toolId], + subject: { type: 'host' }, property, label: property, originalValue: value, normalizedValue: value, + valueType: property === 'report.generated_at' ? 'datetime' : 'string', state: 'observed', + source: {}, mappingVersion: '1', attributes: {} + }; +} + +function model(slot: 'a' | 'b', runs: Array<{ runId: string; toolId: keyof typeof tools; findings: FindingOccurrence[] }>): CanonicalAssessmentModel { + const observedAt = slot === 'a' ? observedA : observedB; + const assessmentId = `assessment-${slot}`; + const findings = runs.flatMap(run => run.findings); + return { + schemaVersion: 1, + project: { id: 'dummy-project', name: 'Dummy Customer', createdAt: observedA, updatedAt: observedB, attributes: {} }, + assessment: { id: assessmentId, projectId: 'dummy-project', name: slot, createdAt: observedAt, updatedAt: observedAt }, + scanRuns: runs.map((run, index) => ({ id: run.runId, assessmentId, sequence: index + 1, label: run.runId, tool: tools[run.toolId], importedAt: observedAt })), + importFiles: runs.map(run => importFile(run.runId, run.toolId)), sourceTables: [], + environmentObservations: runs.flatMap(run => [ + observation(run.runId, run.toolId, 'host.name', 'server-a'), + observation(run.runId, run.toolId, 'benchmark.name', 'benchmark-a'), + observation(run.runId, run.toolId, 'report.generated_at', observedAt), + observation(run.runId, run.toolId, 'report.engine_version', '4.0.0') + ]), + findingDefinitions: [...new Set(findings.map(item => item.definitionId))].map(id => ({ + id, canonicalKey: id, title: id, frameworkReferences: [], createdAt: observedA, updatedAt: observedAt + })), + findingOccurrences: findings, evidence: [], reviews: [], diagnostics: [] + }; +} + +function setting(id: string, identity: string, status: PolicyWorkStatus, actual?: number): PolicyWorkResult { + return { + id, status, title: identity, scope: 'computer', identity, importedAt: observedA, sources: [], + settingKey: JSON.stringify(['dummy-project', 'aggregation', 'computer', identity]), + comparison: { baselineId: 'baseline-server', comparator: { actual: actual === undefined ? undefined : { kind: 'number', value: actual } } } as unknown as PolicyWorkResult['comparison'] + }; +} + +function report(reversed = false) { + const order = (items: T[]) => reversed ? [...items].reverse() : items; + const a = model('a', [{ runId: 'cis-a', toolId: 'cis-cat', findings: [ + occurrence('a-1', 'cis-a', 'cis-cat', '1.1.1', 'failed'), + occurrence('a-2', 'cis-a', 'cis-cat', '1.1.2', 'failed'), + occurrence('a-3', 'cis-a', 'cis-cat', '5.1', 'passed') + ] }]); + const b = model('b', [ + { runId: 'cis-b', toolId: 'cis-cat', findings: [ + occurrence('b-1', 'cis-b', 'cis-cat', '1.1.1', 'passed'), + occurrence('b-2', 'cis-b', 'cis-cat', '1.1.2', 'failed'), + occurrence('b-3', 'cis-b', 'cis-cat', '5.1', 'failed') + ] }, + { runId: 'maester-b', toolId: 'maester', findings: [occurrence('m-1', 'maester-b', 'maester', 'MT.1001', 'failed')] } + ]); + return compareTimeslots({ + projectId: 'dummy-project', models: order([a, b]), baselineRunIds: ['cis-a'], targetRunIds: order(['cis-b', 'maester-b']), + gpo: { + baseline: { observedAt: observedA, results: order([setting('ga-1', 'MinimumPasswordLength', 'Fail', 7), setting('ga-2', 'NoLMHash', 'Fail', 0)]) }, + target: { observedAt: observedB, results: order([setting('gb-1', 'MinimumPasswordLength', 'Fail', 10), setting('gb-2', 'NoLMHash', 'Pass', 1)]) } + } + }); +} + +describe('Timeslot comparison report', () => { + it('assigns every change category from scanner evidence and rolled-up GPO verdicts', () => { + const result = report(); + expect(result.summary).toEqual({ + resolved: 2, unchanged: 1, 'value-changed': 1, unverified: 0, 'not-comparable': 0, 'not-applicable': 0, + new: 0, regressed: 1, targetNotComparable: 1 + }); + const find = (change: string) => result.changes.filter(item => item.change === change).map(item => item.recordIds).flat().sort(); + expect(find('resolved')).toEqual(['a-1', 'ga-2']); + expect(find('value-changed')).toEqual(['ga-1']); + expect(find('regressed')).toEqual(['b-3']); + // A source only in B is neither new nor worse. + expect(result.changes.find(item => item.change === 'not-comparable')).toMatchObject({ sourceId: 'maester', side: 'target', recordIds: ['m-1'] }); + }); + + it('limits the score delta to sources comparable in both timeslots and lists every excluded source', () => { + const result = report(); + const gpo = result.score.sources.find(item => item.sourceId === 'group-policy'); + expect(gpo).toMatchObject({ method: 'shared-gpo-pass-rate', baselineScore: 0, targetScore: 50, change: 50 }); + expect(result.score.excludedSources).toContainEqual({ sourceId: 'maester', reasons: ['baseline-source-missing'] }); + const scored = result.score.sources.filter(item => item.change !== undefined); + expect(scored.map(item => item.sourceId)).toEqual(['cis-cat', 'group-policy']); + // Same three evaluated rules in A and B: one pass each, so the paired CIS-CAT pass rate is unchanged. + expect(result.score.sources.find(item => item.sourceId === 'cis-cat')).toMatchObject({ method: 'paired-scans', change: 0 }); + const weight = scored.reduce((sum, item) => sum + item.weight, 0); + expect(result.score.targetScore).toBeCloseTo(scored.reduce((sum, item) => sum + item.targetScore! * item.weight, 0) / weight); + }); + + it('is deterministic and independent of the input order', () => { + const first = report(); + const reversed = report(true); + expect(reversed.summary).toEqual(first.summary); + expect(reversed.changes).toEqual(first.changes); + expect(reversed.score).toEqual(first.score); + }); +});