From 32c67baccede80b08a81538283ec4e2ae7bcce13 Mon Sep 17 00:00:00 2001 From: Binh Date: Tue, 6 Oct 2026 10:45:24 +0200 Subject: [PATCH] feat: extend Compliance Engine and ISO workflows for v2.19.0 - Move AI overview and privacy-safe JSON export into Compliance Engine - Add ISO control and theme facets to Work, including GPO settings - Enrich ISO queries with baseline context, Windows code meanings and technical explanations - Count GPO settings once and preserve editable decisions across mapping runs - Fix GPO mapping version conflicts and backup domain identity resolution - Add regression coverage and update architecture and reimport documentation --- AGENTS.md | 4 +- docs/IMPLEMENTATION_PLAN.md | 14 ++ docs/REIMPORT_POLICY.md | 2 + package-lock.json | 4 +- package.json | 2 +- server/ai/litellm-client.ts | 11 +- server/assessor-http-server.ts | 10 +- .../{ai => compliance}/ai-overview-service.ts | 68 ++++---- .../compliance/compliance-engine-runtime.ts | 8 +- .../compliance/postgres-compliance-store.ts | 60 ++++++- .../compliance/regulatory-mapping-service.ts | 120 ++++++++++++-- server/gpo/gpo-backup-job-processor.ts | 2 +- server/gpo/gpo-backup-submission-service.ts | 2 +- server/index.ts | 19 +-- .../postgres/postgres-assessor-repository.ts | 62 +------ src/api/assessor-api-client.ts | 7 + src/api/assessor-api-contract.ts | 2 +- src/api/request-activity.ts | 1 + src/app/AppProvider.tsx | 3 +- src/{domain => compliance}/ai-overview.ts | 15 ++ src/compliance/compliance-store.ts | 10 +- src/compliance/local-compliance-store.ts | 41 ++++- .../privacy-safe-export.ts} | 43 +++++ src/compliance/regulatory-mapping.ts | 15 +- src/compliance/regulatory-query.ts | 96 ++++++++++- .../ui}/AiOverviewPanel.tsx | 12 +- src/compliance/ui/ClientIsoControls.tsx | 9 +- src/compliance/ui/RegulatoryMappingPanel.tsx | 46 ++++-- src/{styles => compliance/ui}/ai-overview.css | 0 src/compliance/ui/compliance-work-facets.ts | 126 +++++++++++++++ src/compliance/ui/compliance.css | 3 + src/compliance/ui/use-regulatory-mapping.ts | 6 +- src/domain/gpo-metadata.ts | 15 +- src/domain/privacy-safe-finding.ts | 5 +- src/domain/result-db-disclosure.ts | 9 +- src/domain/result-db-identity-guard.ts | 2 +- src/domain/unified-result.ts | 13 ++ src/exports/unified-result-csv.ts | 2 +- src/exports/unified-result-excel.ts | 4 +- .../actionlist/ActionlistControls.tsx | 4 +- src/features/dashboard/DashboardPanel.tsx | 18 ++- .../results-browser/ResultsBrowser.tsx | 14 +- .../results-browser/work-record-lookup.ts | 22 ++- .../search-filter/SearchFilterBar.tsx | 10 +- src/features/search-filter/filter-records.ts | 9 +- .../search-filter/use-filtered-records.ts | 7 +- src/features/search-filter/work-facets.ts | 5 +- .../sidebar/SidebarSecurityDomains.tsx | 16 +- src/main.tsx | 1 - src/persistence/assessor-repository.ts | 5 - src/persistence/local-assessor-repository.ts | 56 +------ tests/app/app-version.test.ts | 2 +- .../architecture/core-engine-boundary.test.ts | 15 +- .../ai-overview-fixtures.ts} | 2 +- .../ai-overview-panel.test.tsx | 8 +- .../ai-overview-schema.test.ts | 0 .../ai-overview-scope-interaction.test.tsx | 4 +- .../ai-overview-service.test.ts | 51 ++++-- .../ai-overview.test.ts | 6 +- tests/compliance/client-iso-controls.test.tsx | 16 +- .../compliance-engine-switch.test.tsx | 29 +++- tests/compliance/iso-work-facets.test.tsx | 153 ++++++++++++++++++ .../privacy-safe-export.test.ts} | 32 +++- .../regulatory-finding-retrieval.test.ts | 56 +++++++ ...gulatory-mapping-earlier-decision.test.tsx | 41 +++++ .../regulatory-mapping-service.test.ts | 96 ++++++++++- tests/exports/unified-result-jsonl.test.ts | 2 +- .../results-browser-timeslot-scope.test.ts | 2 +- tests/local-only-check.mjs | 4 +- .../technical-explanation-result-db.test.ts | 79 +++++++++ tests/server/assessor-http-server.test.ts | 2 +- tests/server/gpo-backup-ingestion.test.ts | 37 +++++ tests/server/gpo-backup-job-processor.test.ts | 4 +- 73 files changed, 1374 insertions(+), 307 deletions(-) rename server/{ai => compliance}/ai-overview-service.ts (87%) rename src/{domain => compliance}/ai-overview.ts (91%) rename src/{exports/result-db-copy.ts => compliance/privacy-safe-export.ts} (71%) rename src/{features/dashboard => compliance/ui}/AiOverviewPanel.tsx (94%) rename src/{styles => compliance/ui}/ai-overview.css (100%) create mode 100644 src/compliance/ui/compliance-work-facets.ts rename tests/{fixtures/ai-overview.ts => compliance/ai-overview-fixtures.ts} (94%) rename tests/{features => compliance}/ai-overview-panel.test.tsx (91%) rename tests/{postgres => compliance}/ai-overview-schema.test.ts (100%) rename tests/{features => compliance}/ai-overview-scope-interaction.test.tsx (93%) rename tests/{server => compliance}/ai-overview-service.test.ts (66%) rename tests/{domain => compliance}/ai-overview.test.ts (91%) create mode 100644 tests/compliance/iso-work-facets.test.tsx rename tests/{exports/result-db-copy.test.ts => compliance/privacy-safe-export.test.ts} (73%) create mode 100644 tests/compliance/regulatory-mapping-earlier-decision.test.tsx create mode 100644 tests/persistence/technical-explanation-result-db.test.ts diff --git a/AGENTS.md b/AGENTS.md index 457925b..aebb7c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ ## Core Engine and engines - Explicit user decision 2026-10-01: ASSESSOR with all its existing functions is the **Core Engine**. The context enrichment (Phase 13E, "context engine"/"Kontext-Engine" in earlier text) is the **Context Engine**. The read-only Report Engine (Phase 7, `src/reporting/`, `server/reporting/`) is an engine in the same sense. Future features arrive as further engines that provide a capability without changing the Core. The UI labels `Context`/`Kontext` stay unchanged. -- Explicit user decision 2026-10-05: the regulatory control mapping (Phase 13F, ISO mapping) is the **Compliance Engine** (`src/compliance/`, `server/compliance/`, `tests/compliance/`). It is framework-neutral by intent: ISO/IEC 27001/27002 is its first corpus, later frameworks such as BSI or DORA arrive inside this engine without a Core change. It reads the Core only through `ComplianceReadPort` (single Result DB row by project, filtered Result DB pages, project, assessment and scan-run lists) and stores mapping runs, accept/reject decisions and batches only through its own `ComplianceStore`. The existing tables of migration 026 keep their names and data and are owned by the engine; its writes never advance the Core repository revision. The Core project export receives the engine data beside the Core export at the HTTP route. Its extension points are the Finding Reader `Map ISO controls` action and its window, the Finding Reader `ISO controls` section, the Result DB `Map all ISO controls` batch action, the Data Hub ISO corpus tile and the Client Info `ISO controls` section. Explicit user decision 2026-10-05: `Map ISO controls` opens a window like `Accepted Risk` with Accept and Reject per control (no third decision value). The Finding Reader lists the controls whose latest decision is accepted in an `ISO controls` section read like `Description`, after the summary fields. Client Info shows the accepted controls of the selected timeslot directly below the Context heatmap in the same tile layout without severity colour, each with its ISO 27002 theme and the number of deviation findings (a GPO setting counts once); a tile opens exactly those findings in Work. Decisions stay global per scanner rule. Provisional explicit user decision 2026-10-05 (may change later): after a mapping succeeds, single or batch, the engine accepts its first two proposals in the model's order automatically, only the one if there is a single proposal, with the comment `Accepted automatically: one of the first two proposals of the mapping.`; a control that already has a decision keeps it, so a manual reject is never overridden, and a failure of this step never fails the mapping. +- Explicit user decision 2026-10-05: the regulatory control mapping (Phase 13F, ISO mapping) is the **Compliance Engine** (`src/compliance/`, `server/compliance/`, `tests/compliance/`). It is framework-neutral by intent: ISO/IEC 27001/27002 is its first corpus, later frameworks such as BSI or DORA arrive inside this engine without a Core change. It reads the Core only through `ComplianceReadPort` (single Result DB row by project, filtered Result DB pages, project, assessment and scan-run lists; explicit user decision 2026-10-05: additionally the internal-analysis `policy-evaluation` rows of the same project and timeslot, used solely to read the GPO technical setting ID that the Result DB withholds, so a GPO setting counts once like its rolled-up Work record and the Work ISO facets match it; that ID is never displayed or exported and no other internal field is read) and stores mapping runs, accept/reject decisions, batches and AI overview runs only through its own `ComplianceStore`. Explicit user decision 2026-10-05: all AI integration belongs to this engine. The AI overview (service, model, Client Info panel) moved from the Core into it, and the Result DB `Export JSON` is the engine's 1:1 copy of what its AI features receive: the exact AI overview input built by the same function the overview sends to the model, and every distinct ISO mapping query with the Result DB rows that produce it (the retrieved candidate controls depend on the index and are not exported). The shared model client `server/ai/litellm-client.ts` stays Core platform code. The existing tables of migrations 025 (AI overview runs) and 026 keep their names and data and are owned by the engine; its writes never advance the Core repository revision. Because the Core deletes scan runs and assessments without knowing the engine, the engine removes AI overview runs of a deleted timeslot from its own table on start and on read. The Core project export receives the engine data, including succeeded AI overview runs, beside the Core export at the HTTP route. Its extension points are the Finding Reader `Map ISO controls` action and its window, the Finding Reader `ISO controls` section, the Result DB `Map all ISO controls` batch action and `Export JSON` action, the Data Hub ISO corpus tile, the Client Info `ISO controls` section and `AI Overview` tab, and the Work `ISO control` and `ISO theme` facets. Switched off, the engine also removes the AI overview tab and its requests and the `Export JSON` action. Explicit user decision 2026-10-05: `Map ISO controls` opens a window like `Accepted Risk` with Accept and Reject per control (no third decision value). The Finding Reader lists the controls whose latest decision is accepted in an `ISO controls` section read like `Description`, after the summary fields. Client Info shows the accepted controls of the selected timeslot directly below the Context heatmap in the same tile layout without severity colour, each with its ISO 27002 theme and the number of deviation findings (a GPO setting counts once); a tile opens Work with the `Deviation` status and that control selected in the `ISO control` facet, like a Context heatmap cell. Explicit user decision 2026-10-05: Work offers the Compliance Engine facets `ISO control` (one value per accepted control, `A.8.20 · Networks security`) and `ISO theme` (Organizational, People, Physical, Technological) in mixed Work and in both GPO entry paths, directly before the permanent final `Source: Scan`; this extends the GPO-only facet list. They read the accepted controls of every mappable finding of the timeslot (scanner findings by occurrence, GPO settings by scope and technical identity), offer only values with hits, count against the other active filters, keep an active selection while loading, and disappear with their filter cleared when the engine is off. Decisions stay global per scanner rule. Provisional explicit user decision 2026-10-05 (may change later): after a mapping succeeds, single or batch, the engine accepts its first two proposals in the model's order automatically, only the one if there is a single proposal, with the comment `Accepted automatically: one of the first two proposals of the mapping.`; a control that already has a decision keeps it, so a manual reject is never overridden, and a failure of this step never fails the mapping. - Engines may read the Core without restriction but never write to, change or override it. They read only through a declared read-only port and the read-only database access, never through Core tables or internal modules directly. What an engine shows or exports still obeys the Core's privacy and Result DB disclosure rules. - Dependencies point one way: an engine depends on the Core, the Core never depends on an engine. The Core works unchanged with every engine switched off; each engine keeps a permanent on/off invariance test like the Context layer's. Engines do not build on each other except through another engine's published results. - An engine that needs storage gets its own schema or tables after separate approval with backup and migration, never columns in Core tables. An engine cannot create findings: new scanner profiles, importers and parsers write Core data and are always Core changes. @@ -51,7 +51,7 @@ ## Context layer (Phase 13E) -- Latest explicit user decision 2026-10-01 supersedes the preceding permanent-Context decision: the Work toolbar has an `Engines` button at the former Context-button position. It stays marked blue, including on hover and while the popup is closed (an explicit colour-role exception for this control only). The shared popup lists Core Engine, Context Engine, Report Engine and, since the explicit user decision of 2026-10-05, Compliance Engine with checkboxes. All default to active; Core Engine is always checked and cannot be disabled. User choices for optional engines are stored locally under the shared Core-owned engine-activation contract. Disabling Context removes its UI, stops new context requests and clears the active Context Cluster filter; disabling Report blocks new report generation from the UI; disabling Compliance removes the Finding Reader ISO action and panel, the Result DB ISO batch action and the Data Hub ISO corpus tile and stops their requests, and re-enabling starts the Finding Reader ISO panel closed. Already submitted backend work is not cancelled by this local preference; its late UI output is suppressed. Cluster badges still select the exact Work filter, and the Context detail section still starts collapsed. Engine independence and Core invariance remain binding. +- Latest explicit user decision 2026-10-01 supersedes the preceding permanent-Context decision: the Work toolbar has an `Engines` button at the former Context-button position. It stays marked blue, including on hover and while the popup is closed (an explicit colour-role exception for this control only). The shared popup lists Core Engine, Context Engine, Report Engine and, since the explicit user decision of 2026-10-05, Compliance Engine with checkboxes. All default to active; Core Engine is always checked and cannot be disabled. User choices for optional engines are stored locally under the shared Core-owned engine-activation contract. Disabling Context removes its UI, stops new context requests and clears the active Context Cluster filter; disabling Report blocks new report generation from the UI; disabling Compliance removes the Finding Reader ISO action and panel, the Result DB ISO batch action and `Export JSON`, the Client Info `AI Overview` tab and the Data Hub ISO corpus tile and stops their requests, and re-enabling starts the Finding Reader ISO panel closed. Already submitted backend work is not cancelled by this local preference; its late UI output is suppressed. Cluster badges still select the exact Work filter, and the Context detail section still starts collapsed. Engine independence and Core invariance remain binding. - Explicit user decision 2026-10-01 supersedes the earlier Context on/off, default-off, locally saved switch, hover-button and badge-scroll agreements: Context is permanently active and the global toolbar Context button is removed. Ignore obsolete locally saved on/off preferences. Clicking a Finding Reader Context cluster badge selects that exact `Context Cluster` Work facet value, replacing the previous cluster selection while preserving other filters. The Context detail section still starts collapsed for each finding and opens through its own summary. All read-only, provenance and verdict/score boundaries remain binding; only an explicit cluster-filter selection restricts the Work record set. diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index c3855de..2040e3d 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -3404,6 +3404,18 @@ Festgelegtes Betriebsrisiko vom 2026-09-06: Verzeichnisabhaengige Compose-Projek ## 12. Änderungsprotokoll +2026-10-05 - Phase 13F / AI-Integration in die Compliance Engine verlegt (Nutzerentscheidung; Core-Freigabe Platform, Presentation, Disclosure, Store, Normalization, Core Tests, Governance; per Test verifiziert; in Produktversion 2.19.0 enthalten, keine weitere Versionserhoehung): Die AI-Uebersicht (Dienst `server/compliance/ai-overview-service.ts`, Modell `src/compliance/ai-overview.ts`, Panel und CSS unter `src/compliance/ui/`) und der Privacy-Safe-Export (`src/compliance/privacy-safe-export.ts`, vorher `src/exports/result-db-copy.ts`) gehoeren jetzt zur Compliance Engine. Der Dienst liest ueber `ComplianceReadPort` und speichert ueber `ComplianceStore` (Postgres und lokal); die Tabelle aus Migration 025 behaelt Namen und Daten und gehoert der Engine, ohne Migration oder Schemaaenderung. Das Core-Repository verliert die AI-Methoden, das Feld im Core-Projektexport und die Loeschkaskade; die Engine liefert die erfolgreichen Laeufe neben dem Core-Export und entfernt Laeufe geloeschter Timeslots beim Start und beim Lesen selbst. `server/ai/litellm-client.ts` bleibt Core und hat einen eigenen Tokenzaehler-Typ. Neue Erweiterungspunkte: Client-Info-Tab `AI Overview` und Result-DB-Aktion `Export JSON`, beide nur bei eingeschalteter Engine. `Export JSON` (Schema 2) enthaelt 1:1, was die KI erhaelt: `aiOverviewInput` aus derselben Funktion, die die AI-Uebersicht hasht und sendet (deterministische Reihenfolge), und `isoMappingQueries` mit jeder verschiedenen ISO-Anfrage und ihren Result-DB-Zeilen; die abgerufenen Kandidatenkontrollen haengen vom Index ab und sind nicht enthalten. Tests verschoben nach `tests/compliance/`; neu: 1:1-Export, Engine-Schalter fuer AI-Tab und Export, Bereinigung geloeschter Timeslots, Core-Export ohne AI-Laeufe. Gesamtsuite gruen bis auf einen lastbedingten Timeout von `sheet-selection-cost` (einzeln 0,7 s gruen); beide Builds, Local-only- und Designpruefung bestanden. + +2026-10-05 - Phase 13F / GPO-Einstellungen in den Work-ISO-Facetten (Engine-Arbeit, Nutzerentscheidung Option 1 mit Core-Freigabe Governance, per Test verifiziert): Die Engine liest ueber `ComplianceReadPort` zusaetzlich die internen `policy-evaluation`-Zeilen desselben Projekts und Timeslots, ausschliesslich um die von der Result DB zurueckgehaltene technische GPO-Einstellungs-ID zu erhalten. Damit zaehlt eine GPO-Einstellung exakt wie ihr Work-Datensatz (auch bei gleichem Titel verschiedener Einstellungen), und die Work-Facetten `ISO control`/`ISO theme` finden GPO-Einstellungen. Die ID wird nie angezeigt oder exportiert; Mapping und Entscheidungen bleiben je offengelegtem Titel. Die zugehoerige AGENTS.md-Regel wurde am selben Tag mit erneuter Governance-Freigabe nachgetragen. + +2026-10-05 - Phase 13F / GPO-Kontext in der ISO-Mapping-Anfrage (Engine-Arbeit, per Test verifiziert): Die Anfrage einer GPO-Abweichung bestand bisher praktisch nur aus Einstellungstitel und einem festen Beschreibungssatz. Sie enthaelt jetzt aus offengelegten Result-DB-Feldern Einstellungsart (Scope, Auslieferungsmechanismus), Policy-Abschnitt bzw. Registry-Schluessel, den in Woerter zerlegten Einstellungsnamen (`MinimumPasswordLength` -> `Minimum Password Length`, damit auch die Themenerkennung greift), Baseline-Familie, empfohlenen Baselinewert und den bewiesenen Befund (nicht konfiguriert, widerspruechlich oder abweichend). Eine Richtung (schwaecher/staerker) wird nur genannt, wenn der Vergleich sie festhaelt; alle aktuellen Vergleiche sind Gleichheitsvergleiche und beweisen nur `different`, daher wird z. B. nicht behauptet, eine Passwortlaenge liege unter 14. Der Kundenwert wird nie gelesen. Neue Version `REGULATORY_GPO_QUERY_VERSION` (gpo-baseline-context-1.0.0) geht nur in die Mapping-ID von GPO-Regeln ein; der naechste Batch erzeugt GPO-Mappings neu, Scanner-Mappings bleiben wiederverwendet. Bestehende Entscheidungen bleiben je Regel erhalten; die vorlaeufige Auto-Akzeptanz nimmt bei neuen Laeufen bis zu zwei noch unentschiedene Vorschlaege hinzu. + +2026-10-05 - Phase 13F / Compliance Engine zaehlt eine GPO-Einstellung einmal (Engine-Arbeit, per Test verifiziert): Die Result DB fuehrt je Baselinevergleich eine Zeile (Beispiel: 127 Zeilen fuer 25 abweichende Einstellungen) und haelt die interne Kennung `gpo-policy-identity-*` zurueck. Die Engine fiel dadurch fuer GPO auf die vergleichsbezogene `sourceResultId` zurueck und zaehlte im Client-Info-Bereich `ISO controls` und im Batch (`Eligible/Unsupported findings`) je Vergleich. Neu identifiziert die Engine eine GPO-Einstellung ueber Scope und offengelegten Titel (die technische Einstellungsidentitaet), wenn keine oeffentliche Kennung vorliegt; `Filtered results` bleibt die Result-DB-Zeilenzahl. Keine Core-Aenderung. Offener Befund: Die Work-Facetten `ISO control`/`ISO theme` ordnen GPO-Einstellungen ueber `sourceRuleId` zu, die in Result-DB-Zeilen fehlt; GPO-Einstellungen erscheinen dort deshalb nicht. Eine Loesung erfordert eine Nutzerentscheidung (Read-Port-Vertrag oder Result-DB-Offenlegung). + +2026-10-05 - Phase 13B / GPO-Domaenenschluessel aus Backup-GPO-Ordnern (Core-Freigabe Normalization, Ingest, Core Tests; per Test und am gespeicherten Artefakt inhaltsfrei verifiziert): Ein Standard-`Backup-GPO`-Ordner nennt die Domaene in `bkupInfo.xml` per `GPODomainGuid` und in `Backup.xml`/`GPReport.xml` per DNS-Name. `aggregateGpoBackupMetadata` verglich `domain-guid:` und `dns:` als einen Wert und meldete `gpo-metadata-domainKey-conflict`; jede GPO blieb mit `metadata-conflict` unaufgeloest und blockiert, Data Hub zeigte `Limited analysis`. Seit dd92aa2 (2026-09-06, DNS auch aus `Backup.xml`/`GPReport.xml`) betraf das jeden Standardexport. Korrektur: GUID und DNS-Name sind zwei Kennungsarten derselben Domaene; ein Konflikt besteht nur bei zwei verschiedenen Werten derselben Art, die GUID hat Vorrang. Append-only `gpo-backup-policy-analysis` und Jobprozessor 2.2.5 -> 2.2.6; betroffene Backups zeigen `Reprocessing required` (kein Upload, keine SQL-Migration, Metadatenparser 1.4.0 unveraendert). Lokale Backup-Zeiten ohne Zeitzone bleiben wie bisher unaufgeloest. Zwei synthetische Regressionstests (Standardlayout aufgeloest; abweichende DNS-Namen neben GUID weiterhin blockiert). Gesamtsuite: ein nicht zugehoeriger Fehlschlag (`unified-result-jsonl` erwartet Disclosure-Policy 3.5.0, paralleler Arbeitsstand 3.6.0). Produktversion nicht angehoben, da der Bereich Platform nicht freigegeben war. + +2026-10-06 - Phase 13F / Windows-Code-Bedeutungen in der GPO-ISO-Mapping-Anfrage (Engine-Arbeit, per Test verifiziert): Die Anfrage nennt zu Einstellungsname und empfohlenem Baselinewert jetzt die dokumentierte Bedeutung standardisierter Windows-Codes aus derselben Liste, die der Finding Reader anzeigt (`src/shared/windows-codes.ts`): `Setting name: Se Backup Privilege (Back up files and directories)`, `*S-1-5-32-544 (Administrators)`. Nur feste oeffentliche Microsoft-Namen; Domaenen-SIDs werden wie bisher nur ueber ihre Standard-RID benannt, der Kundenwert wird nie gelesen. `REGULATORY_GPO_QUERY_VERSION` 1.0.0 -> 1.1.0; der naechste Batch erzeugt GPO-Mappings neu, Scanner-Mappings und bestehende Entscheidungen bleiben. Keine Core-Aenderung, keine Produktversionsanhebung (Engine-Verfeinerung im laufenden Stand 2.19.0). Korrektur desselben Tages: Die GPO-Query-Version ging nur in die Mapping-ID ein, nicht in das eindeutige Identitaetstupel der Tabelle `assessor_regulatory_mappings` (rule_key, corpus_version, retrieval_version, prompt_version, model). Ein Neulauf alter GPO-Regeln scheiterte daher mit `Regulatory mapping identity conflicted with another pipeline tuple.`. GPO-Laeufe speichern ihre `promptVersion` jetzt als `1.2.0+gpo-baseline-context-1.1.0`; Scanner-Laeufe und alle IDs bleiben unveraendert, keine Migration. Zweite Korrektur: Entscheidungen gelten global je Regel, das Fenster `Map ISO controls` listete aber nur Kontrollen des aktuellen Laufs. Eine von einem frueheren Lauf automatisch akzeptierte Kontrolle, die der neue Lauf nicht mehr abruft, blieb in der Finding-Reader-Sektion `ISO controls` (ohne Titel) und liess sich nicht ablehnen. Die Antwort traegt jetzt `earlierControls` (entschiedene, im aktuellen Lauf nicht abgerufene Kontrollen, Titel aus aelteren Laeufen der Regel); das Fenster zeigt sie unter `Earlier decisions` mit Accept/Reject, die Sektion uebernimmt ihren Titel. Synthetischer Interaktionstest (Reject auf einer frueheren Kontrolle) und Servicetest. Dritte Korrektur: Der Store verlangt, dass eine Entscheidung auf einen Lauf zeigt, der die Kontrolle abgerufen hat (`Decision control was not a grounded candidate.`); eine Entscheidung zu einer frueheren Kontrolle verweist deshalb auf den neuesten aelteren erfolgreichen Lauf der Regel, der sie abgerufen hat. Der Servicetest lehnt die fruehere Kontrolle jetzt ab. + 2026-10-02 - Nutzerwunsch fuer spaeter dokumentiert: automatische Security-Baseline-Updates als zurueckgestelltes Arbeitspaket in Phase 13B, mit Phase-10-Backend und Phase-7-Anzeige. Datenqualitaet, Herkunft/Version/Format/Vollstaendigkeit, Erhalt der letzten gueltigen Baseline und unveraenderte historische Auswertungen sind Abnahmekriterien; Quellen und Intervall bleiben offen. Ausschliesslich Planaufnahme, keine Implementierung, Automatisierung oder Produktversionsaenderung. @@ -3657,6 +3669,8 @@ Verifiziert 2026-09-09 (Phase 7): Bereichswechsel beschleunigt und Performance-V Verifiziert 2026-09-08 (Phase 7/13B): GPO-Quellenkorrektur fuer Work und Client Info. Aktuelle Evaluationen aus dem vorhandenen autoritativen Readmodell statt ergebnisabhaengiger Backupgruppierung; Import- und Auswertungszeit getrennt, Source-Scan-Provenienz auch auf GPO-only-Blaettern. 1.014 Tests bestanden, acht optionale PostgreSQL-Tests uebersprungen; Local-only-/Designpruefung und beide Builds erfolgreich. Dummy lesend bestaetigt: eine statt drei GPO-Quellenoptionen, 3.805 aktuelle Zeilen, elf historische Evaluationen unveraendert. Keine Persistenzmutation oder Neuanalyse. | Datum | Änderung | +| 2026-10-05 | Phasen 6/13F, in Produktversion 2.19.0 enthalten, Disclosure Policy 3.6.0 (Core-Freigabe Normalization, Disclosure, Presentation, Core Tests): Neues allgemeines Result-DB-Feld `technicalExplanation` fuer die technische Erklaerung einer Regel, wo ein Scanner sie liefert (heute PingCastle `Technical explanation`; andere Werkzeuge lassen es leer). Es wird beim Aufbau der Result DB aus dem bereits importierten Originalfeld gelesen (Schluessel `technical-explanation` oder Bezeichnung `Technical explanation`), daher ohne Reimport, in Local- und PostgreSQL-Repository ueber dieselbe Funktion. Es ist oeffentliche Regelanleitung wie `Description`, wird genauso streng redigiert (nur bekannte Scanner, unbekannte Quellen fail-closed, GPO ohne Feld), vom Identitaetsschutz geprueft und im inhaltsfreien Trace gefuehrt; `Details` mit Umgebungswerten bleibt zurueckgehalten. Die Policy-Erhoehung baut die gespeicherte Result DB neu auf. Result-DB-Details zeigen das Feld, Excel und CSV fuehren die Spalte nach `Description`. Die ISO-Zuordnung nimmt die redigierte technische Erklaerung in Suche und Modellanfrage auf; bestehende Zuordnungen behalten ihre ID. Tests: beide Importwege, Redaktion, Trace, Identitaetsschutz, fail-closed, interne Analyse mit Original, ISO-Abfrage. | +| 2026-10-05 | Phase 13F, Produktversion 2.19.0 (Minor: neue Work-Faehigkeit; Core-Freigabe Governance, Presentation, Platform, Core Tests): Work erhaelt die Compliance-Engine-Facetten `ISO control` (je akzeptierter Kontrolle, `A.8.20 · Networks security`) und `ISO theme` (Organizational, People, Physical, Technological) wie die Context-Cluster-Facette, direkt vor dem permanenten `Source: Scan`, in gemischtem Work und beiden GPO-Einstiegen (Erweiterung der GPO-Facettenliste per Nutzerentscheidung). Grundlage ist der neue Endpunkt `GET /projects/:id/regulatory-control-assignments` mit den akzeptierten Kontrollen aller zuordenbaren Findings des Timeslots (nicht nur Abweichungen), damit der Work-Statusfilter wie gewohnt wirkt; Scanner-Findings ueber das Vorkommen, GPO-Einstellungen ueber Scope und technische Identitaet (gemeinsamer Helfer `gpoWorkSettingIdentity`). Eine Anfrage je Timeslot fuer alle fuenf Work-Verbraucher, Neuladen nach jeder Entscheidung, aktive Auswahl bleibt waehrend des Ladens; ausgeschaltet entfallen die Facetten und ein aktiver ISO-Filter wird entfernt. Themenwerte werden kleingeschrieben verglichen, die Anzeige bleibt ISO-Schreibweise. Die Client-Info-Kachel oeffnet Work jetzt mit Status `Deviation` und der Kontrolle im `ISO control`-Filter statt einer festen Record-Liste. Aktivitaetslabel `Loading ISO control filters`. Tests: Werte, Zaehlung gegen andere Filter, Scanner- und GPO-Abgleich, Thema, Position vor `Source: Scan`, geteilte Anfrage, Neuladen, Engine aus mit Filterentfernung, Kachel-Sprung; Version-UI/API-Tests und beide Builds. | | 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten (nur Engine-Code), vorlaeufige Nutzerentscheidung: Nach einer erfolgreichen Zuordnung werden die ersten zwei Vorschlaege in der Reihenfolge des Modells automatisch akzeptiert (bei nur einem Vorschlag nur dieser), mit dem Kommentar `Accepted automatically: one of the first two proposals of the mapping.`; eine Kontrolle mit bestehender Entscheidung behaelt sie, ein manuelles Reject wird also nie ueberschrieben, und ein Fehler dabei laesst die Zuordnung erfolgreich. Gilt fuer Einzel- und Batchzuordnung. Einmalige Nachholung ueber die normale Entscheidungs-API fuer die bestehenden 130 Regeln mit erfolgreichem Lauf (129 mit Vorschlaegen, 1 ohne): 210 Entscheidungen fuer 112 Regeln angelegt (98 mit zwei, 14 mit einer akzeptierten Kontrolle), 33 bereits entschiedene Kontrollen uebersprungen, keine Fehler. Die Regel steht seit der Governance-Freigabe vom selben Tag als vorlaeufige Nutzerentscheidung in den verbindlichen Projektregeln. | | 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten (nur Engine-Code): Das Aktivitaetspanel sprang waehrend laufender ISO-Arbeit, weil jede Statusabfrage (alle 1,5 s bei Zuordnung und Batch, alle 2 s beim Korpus-Embedding) einen neuen Eintrag startete; das Panel oeffnet sich bei jedem neuen Eintrag und klappt die Historie zu, kurz sichtbar wurden dabei die Core-Gruppen der Historie. Wiederholte Statusabfragen laufen jetzt ueber einen eigenen Client ohne Protokolleintrag (`describeActivity`); der eine Eintrag je laufender Arbeit bleibt, das erste Laden jeder Ansicht wird weiter protokolliert. Tests ersetzen die Anfrage unterhalb der Protokollierung und pruefen, dass waehrend des Laufs kein neuer Eintrag startet; ohne die Korrektur scheitern sie. | | 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten (nur Engine-Code): Seit `Map ISO controls` ein Fenster ist, endete der Protokolleintrag `Mapping ISO controls` beim Schliessen des Fensters mit `continues in the backend`, und das Protokoll meldete `All tasks completed`, waehrend das Modell noch lief. Ein Lauf-Tracker der Engine haelt den Eintrag jetzt je laufender Zuordnung bis zum Endzustand des Backends, unabhaengig von Fenster, Finding und Ansicht; Fenster und Reader-Abschnitt uebernehmen seine Statusaktualisierungen, sodass nur einmal gepollt wird. Ausschalten der Engine beendet nur die Beobachtung. Tests fuer geschlossenes Fenster bis Abschluss und fuer einen fehlgeschlagenen Lauf. | diff --git a/docs/REIMPORT_POLICY.md b/docs/REIMPORT_POLICY.md index 53b0d83..8a9ce2b 100644 --- a/docs/REIMPORT_POLICY.md +++ b/docs/REIMPORT_POLICY.md @@ -39,6 +39,8 @@ Clarification 2026-09-17 (baseline analyses of the written form): The Microsoft Clarification 2026-09-17 (written security-template section and key): The Security Template parser folded the section name to lower case and discarded the written form, so `[Privilege Rights]` was retained only as `privilege rights`. The parser now keeps the written section and key; the case-folded forms stay internal to grammar selection, grouping and the identity digest, which keeps existing observation and identity IDs stable. The native target resolver stores the written section and key in the retained target, matching what the registry resolver already did, so Work shows `Privilege Rights - SeCreateTokenPrivilege` and the value stays searchable against Microsoft documentation and external tables. This changes persisted parsing output, so it is an append-only processing-version change, not a presentation change: `gpo-security-template-inf` 1.1.0 -> 1.2.0, `gpo-backup-policy-analysis` 2.2.3 -> 2.2.4 and the GPO job processor 2.2.3 -> 2.2.4. Historical imports keep their retained values unchanged and are marked for reprocessing: Data Hub reports an affected GPO backup as `Reprocessing required` with the analyzer versions and offers the existing reprocess action on the stored source, so no upload is needed. Reprocessing precedes re-evaluation, and prior runs remain retained as history. No SQL migration and no evaluator change. +Clarification 2026-10-05 (GPO domain key): A standard `Backup-GPO` folder names the domain by GUID in `bkupInfo.xml` and by DNS name in `Backup.xml`/`GPReport.xml`. The metadata aggregate treated these two identifier kinds as conflicting values, so every such backup stayed unresolved and blocked (`Limited analysis`). A GUID and a DNS name now count as two kinds of the same domain; only two different values of one kind conflict, and the GUID is preferred. This changes persisted identity resolution, so it is an append-only processing-version change: `gpo-backup-policy-analysis` and the GPO job processor 2.2.5 -> 2.2.6 (metadata parser 1.4.0 unchanged). Affected backups show `Reprocessing required`; the existing reprocess action on the stored source applies the fix. No upload, SQL migration or evaluator change. + Clarification 2026-09-25 (GPO link facts): The GPO metadata parser retains, from each backup's own `GPReport.xml`, the number of `LinksTo` entries and how many are enabled (never the linked OU or domain names). A backup with no enabled link is presented as `Not linked`: the Finding Reader marks its source sections and states `No effect` in the problem summary when every contributing GPO is unlinked, and GPO Processing shows the count. This is a backup-report fact at backup time, not a claim about effective device state, and verdicts and score are unchanged. Because parsing now retains a new fact, it is an append-only processing-version change: `gpo-backup-metadata-xml` 1.2.0 -> 1.4.0 (the Microsoft baseline template variant keeps 1.3.0, so baseline packages need no re-analysis), `gpo-backup-policy-analysis` 2.2.4 -> 2.2.5 and the GPO job processor 2.2.4 -> 2.2.5. Historical imports are marked `Reprocessing required` and the existing reprocess action on the stored source adds the link facts; no upload, SQL migration or evaluator change is required. Until then no link status is shown. Clarification 2026-09-17 (Not comparable verdict): A retained customer setting whose delivery mechanism has no comparable technical policy identity - today always a Group Policy Preference - reaches Work under the score-neutral verdict `Not comparable` instead of being dropped. It is never resolved into a policy identity and never compared with a Microsoft baseline. The unified read projection additionally exposes the already retained preference CSE, action and item-level targeting status so that context stays readable in the agreed GPO columns; no new column, facet or persisted field is introduced. Snapshot-aggregation review hints that restate an unresolved identity already reported by their source analysis are no longer emitted a second time. All of this projects facts the importer already retained, so no processing-manifest version, source reimport, evaluator change or SQL migration is required. diff --git a/package-lock.json b/package-lock.json index 77c89c2..896b853 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "assessor", - "version": "2.18.0", + "version": "2.19.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "assessor", - "version": "2.18.0", + "version": "2.19.0", "dependencies": { "@fontsource-variable/inter": "^5.3.0", "jsdom": "^29.1.1", diff --git a/package.json b/package.json index 5f1351b..f3923fc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assessor", - "version": "2.18.0", + "version": "2.19.0", "private": true, "type": "module", "scripts": { diff --git a/server/ai/litellm-client.ts b/server/ai/litellm-client.ts index 19615d3..45a1cec 100644 --- a/server/ai/litellm-client.ts +++ b/server/ai/litellm-client.ts @@ -1,4 +1,9 @@ -import type { AiOverviewTokenUsage } from '../../src/domain/ai-overview'; +/** Token counts the provider reports for one completion. */ +export interface AiCompletionTokenUsage { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; +} export interface LiteLlmEnvironment { ASSESSOR_AI_MODE?: string; @@ -11,7 +16,7 @@ export interface LiteLlmEnvironment { export interface AiCompletion { value: unknown; - usage?: AiOverviewTokenUsage; + usage?: AiCompletionTokenUsage; } export interface AiCompletionClient { @@ -206,7 +211,7 @@ function completionContent(value: unknown): string { return value.choices[0].message.content; } -function completionUsage(value: unknown): AiOverviewTokenUsage | undefined { +function completionUsage(value: unknown): AiCompletionTokenUsage | undefined { if (!isObject(value) || !isObject(value.usage)) return undefined; const number = (entry: unknown) => typeof entry === 'number' && Number.isFinite(entry) ? entry : undefined; const usage = { diff --git a/server/assessor-http-server.ts b/server/assessor-http-server.ts index 1aaaeac..09a9988 100644 --- a/server/assessor-http-server.ts +++ b/server/assessor-http-server.ts @@ -29,9 +29,9 @@ import type { GpoPolicySnapshotService } from './gpo/gpo-policy-snapshot-service import type { GpoPolicyEvaluationService } from './gpo/gpo-policy-evaluation-service'; import type { GpoPolicyFindingService } from './gpo/gpo-policy-finding-service'; import type { GpoPolicyAnalysisOverviewService } from './gpo/gpo-policy-analysis-overview-service'; -import type { AiOverviewService } from './ai/ai-overview-service'; -import { AiOverviewServiceError } from './ai/ai-overview-service'; -import type { AiOverviewCreateRequest, AiOverviewScope } from '../src/domain/ai-overview'; +import type { AiOverviewService } from './compliance/ai-overview-service'; +import { AiOverviewServiceError } from './compliance/ai-overview-service'; +import type { AiOverviewCreateRequest, AiOverviewScope } from '../src/compliance/ai-overview'; import type { RegulatoryMappingBatchCreateRequest, RegulatoryMappingBatchScope, @@ -1175,6 +1175,10 @@ export function createAssessorHttpServer(options: AssessorHttpServerOptions): Se const body = await readJson(request, maximumBodyBytes); status = 200; sendJson(response, status, await options.regulatoryMappingService.decide(segments[1], body), requestId); return; } + if (segments[0] === 'projects' && segments[2] === 'regulatory-control-assignments' && segments.length === 3 && method === 'GET') { + if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.'); + status = 200; sendJson(response, status, await options.regulatoryMappingService.controlAssignments(segments[1], regulatoryMappingBatchScopeFrom(url)), requestId); return; + } if (segments[0] === 'projects' && segments[2] === 'regulatory-control-overview' && segments.length === 3 && method === 'GET') { if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.'); status = 200; sendJson(response, status, await options.regulatoryMappingService.controlOverview(segments[1], regulatoryMappingBatchScopeFrom(url)), requestId); return; diff --git a/server/ai/ai-overview-service.ts b/server/compliance/ai-overview-service.ts similarity index 87% rename from server/ai/ai-overview-service.ts rename to server/compliance/ai-overview-service.ts index d560213..bee91d5 100644 --- a/server/ai/ai-overview-service.ts +++ b/server/compliance/ai-overview-service.ts @@ -1,7 +1,7 @@ import { createHash, randomUUID } from 'node:crypto'; -import type { AssessorRepository } from '../../src/persistence/assessor-repository'; +import type { ComplianceReadPort, ComplianceStore } from '../../src/compliance/compliance-store'; import { RepositoryConflictError, RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors'; -import { resultsToPrivacySafeExportJson } from '../../src/exports/result-db-copy'; +import { aiOverviewInputJson } from '../../src/compliance/privacy-safe-export'; import type { UnifiedResultProjection } from '../../src/domain/unified-result'; import { AI_OVERVIEW_OUTPUT_SCHEMA_VERSION, @@ -15,9 +15,9 @@ import { type AiOverviewRun, type AiOverviewScope, type AiOverviewTokenUsage -} from '../../src/domain/ai-overview'; -import type { AiCompletionClient } from './litellm-client'; -import { LiteLlmClientError } from './litellm-client'; +} from '../../src/compliance/ai-overview'; +import type { AiCompletionClient } from '../ai/litellm-client'; +import { LiteLlmClientError } from '../ai/litellm-client'; const PAGE_SIZE = 500; const ACTIVE_POLL_MS = 1_500; @@ -57,7 +57,7 @@ export class AiOverviewService { private readonly workerId: string; private readonly activeTasks = new Set>(); - constructor(private readonly repository: AssessorRepository, options: AiOverviewServiceOptions = {}) { + constructor(private readonly read: ComplianceReadPort, private readonly store: ComplianceStore, options: AiOverviewServiceOptions = {}) { this.client = options.client; this.now = options.now ?? (() => new Date()); this.maximumChunkCharacters = options.maximumChunkCharacters ?? 80_000; @@ -76,10 +76,11 @@ export class AiOverviewService { async recoverInterruptedRuns(): Promise { const now = this.now().toISOString(); - for (const project of await this.repository.listProjects()) { - const runs = await this.repository.listAiOverviewRuns(project.id); + for (const project of await this.read.listProjects()) { + await this.pruneDeletedTimeslots(project.id); + const runs = await this.store.listAiOverviewRuns(project.id); for (const run of runs.filter(item => ['queued', 'running'].includes(item.status))) { - await this.repository.updateAiOverviewRun({ + await this.store.updateAiOverviewRun({ ...run, status: 'failed', updatedAt: now, @@ -93,10 +94,27 @@ export class AiOverviewService { } } + /** + * The Core deletes scan runs and assessments without knowing this engine. Their AI overviews are + * derived from deleted data, so the engine removes them from its own table on start and on read. + */ + async pruneDeletedTimeslots(projectId: string): Promise { + const [assessments, scanRuns, runs] = await Promise.all([ + this.read.listAssessments(projectId), this.read.listProjectScanRuns(projectId), this.store.listAiOverviewRuns(projectId) + ]); + const assessmentIds = new Set(assessments.map(item => item.id)); + const scanRunIds = new Set(scanRuns.map(item => item.id)); + const orphaned = runs.filter(run => !['queued', 'running'].includes(run.status) + && (run.assessmentIds.some(id => !assessmentIds.has(id)) || run.scanRunIds.some(id => !scanRunIds.has(id)))); + await this.store.deleteAiOverviewRuns(projectId, orphaned.map(run => run.id)); + } + async get(projectId: string, requestedScope: AiOverviewScope): Promise { + if (!await this.read.getProject(projectId)) throw new RepositoryNotFoundError(`Projekt ${projectId} ist nicht vorhanden.`); + await this.pruneDeletedTimeslots(projectId); const scope = await this.validateScope(projectId, requestedScope); const scopeKey = aiOverviewScopeKey(projectId, scope); - const runs = await this.repository.listAiOverviewRuns(projectId, scopeKey); + const runs = await this.store.listAiOverviewRuns(projectId, scopeKey); const run = runs[0]; const previousSuccessfulRun = run?.status === 'succeeded' ? undefined : runs.find(item => item.status === 'succeeded'); let stale = false; @@ -136,13 +154,13 @@ export class AiOverviewService { updatedAt: timestamp, callCount: 0 }; - const run = await this.repository.createAiOverviewRun(candidate); + const run = await this.store.createAiOverviewRun(candidate); if (run.id === candidate.id) { const task = this.process(run, prepared).finally(() => this.activeTasks.delete(task)); this.activeTasks.add(task); } const previousSuccessfulRun = run.status === 'succeeded' ? undefined - : (await this.repository.listAiOverviewRuns(projectId, run.scopeKey)).find(item => item.status === 'succeeded'); + : (await this.store.listAiOverviewRuns(projectId, run.scopeKey)).find(item => item.status === 'succeeded'); return responseFor(projectId, scope, true, run, false, previousSuccessfulRun); } @@ -161,7 +179,7 @@ export class AiOverviewService { leaseExpiresAt: new Date(this.now().getTime() + this.leaseDurationMs).toISOString() }; try { - run = await this.repository.updateAiOverviewRun(run); + run = await this.store.updateAiOverviewRun(run); const completion = await generateAiOverview(prepared.parsed, { client: this.client!, maximumChunkCharacters: this.maximumChunkCharacters, @@ -171,7 +189,7 @@ export class AiOverviewService { factProperties: prepared.factProperties, onCall: async usage => { const updatedAt = this.now().toISOString(); - run = await this.repository.updateAiOverviewRun({ + run = await this.store.updateAiOverviewRun({ ...run, callCount: run.callCount + 1, tokenUsage: addUsage(run.tokenUsage, usage), @@ -181,7 +199,7 @@ export class AiOverviewService { } }); const completedAt = this.now().toISOString(); - await this.repository.updateAiOverviewRun({ + await this.store.updateAiOverviewRun({ ...run, status: 'succeeded', result: completion, @@ -194,7 +212,7 @@ export class AiOverviewService { const completedAt = this.now().toISOString(); const failure = safeGenerationFailure(error); try { - await this.repository.updateAiOverviewRun({ + await this.store.updateAiOverviewRun({ ...run, status: 'failed', updatedAt: completedAt, @@ -211,13 +229,13 @@ export class AiOverviewService { } private async validateScope(projectId: string, requested: AiOverviewScope): Promise { - if (!await this.repository.getProject(projectId)) throw new RepositoryNotFoundError(`Projekt ${projectId} ist nicht vorhanden.`); + if (!await this.read.getProject(projectId)) throw new RepositoryNotFoundError(`Projekt ${projectId} ist nicht vorhanden.`); const scope = normalizeAiOverviewScope(requested); if (!scope.assessmentIds.length && !scope.scanRunIds.length) throw new RepositoryValidationError('A selected timeslot scope is required.'); - const assessments = await this.repository.listAssessments(projectId); + const assessments = await this.read.listAssessments(projectId); const allowedAssessmentIds = new Set(assessments.map(item => item.id)); if (scope.assessmentIds.some(id => !allowedAssessmentIds.has(id))) throw new RepositoryValidationError('The AI overview scope contains an assessment outside this project.'); - const scanRuns = await this.repository.listProjectScanRuns(projectId); + const scanRuns = await this.read.listProjectScanRuns(projectId); const allowedRuns = new Map(scanRuns.filter(item => !item.supersededByScanRunId).map(item => [item.id, item])); if (scope.scanRunIds.some(id => !allowedRuns.has(id))) throw new RepositoryValidationError('The AI overview scope contains an unavailable scan run.'); if (scope.assessmentIds.length && scope.scanRunIds.some(id => !scope.assessmentIds.includes(allowedRuns.get(id)!.assessmentId))) { @@ -227,9 +245,9 @@ export class AiOverviewService { } private async prepareInput(projectId: string, scope: AiOverviewScope): Promise { - const items = await queryAllUnifiedResults(this.repository, projectId, scope); - const ordered = items.slice().sort((left, right) => stableResultKey(left).localeCompare(stableResultKey(right))); - const json = resultsToPrivacySafeExportJson(ordered); + const items = await queryAllUnifiedResults(this.read, projectId, scope); + // The same function builds the Result DB `Export JSON`, so the export is exactly what the model receives. + const json = aiOverviewInputJson(items); const parsed = parsePrivacySafeInput(json); const findingIds = new Set(parsed.findings.flatMap(item => typeof item.id === 'string' ? [item.id] : [])); const gapIds = new Set([ @@ -309,7 +327,7 @@ export function splitPrivacySafeInput(input: PrivacySafeInput, maximumCharacters return chunks; } -async function queryAllUnifiedResults(repository: AssessorRepository, projectId: string, scope: AiOverviewScope): Promise { +async function queryAllUnifiedResults(repository: ComplianceReadPort, projectId: string, scope: AiOverviewScope): Promise { const items: UnifiedResultProjection[] = []; let expectedTotal: number | undefined; for (let offset = 0; ; offset += PAGE_SIZE) { @@ -351,10 +369,6 @@ function parsePrivacySafeInput(json: string): PrivacySafeInput { return value as PrivacySafeInput; } -function stableResultKey(item: UnifiedResultProjection): string { - return [item.sourceKind, item.sourceTool ?? '', item.sourceRuleId ?? '', item.resultProjectionId].join('\u0000'); -} - function responseFor(projectId: string, scope: AiOverviewScope, enabled: boolean, run?: AiOverviewRun, stale = false, previousSuccessfulRun?: AiOverviewRun): AiOverviewResponse { return { version: 1, diff --git a/server/compliance/compliance-engine-runtime.ts b/server/compliance/compliance-engine-runtime.ts index 9485ee4..0aa596a 100644 --- a/server/compliance/compliance-engine-runtime.ts +++ b/server/compliance/compliance-engine-runtime.ts @@ -9,10 +9,12 @@ import { PostgresComplianceStore } from './postgres-compliance-store'; import { createRegulatoryRagRuntime } from './regulatory-rag-runtime'; import { RegulatoryMappingService } from './regulatory-mapping-service'; import { RegulatoryCorpusUploadService } from './regulatory-corpus-upload-service'; +import { AiOverviewService, type AiOverviewServiceOptions } from './ai-overview-service'; export interface ComplianceEngineRuntime { mappingService: RegulatoryMappingService; corpusUploadService: RegulatoryCorpusUploadService; + aiOverviewService: AiOverviewService; store: ComplianceStore; /** Stops running work; ASSESSOR keeps its own repository open. */ stop(): Promise; @@ -25,6 +27,7 @@ export interface ComplianceEngineRuntimeOptions { aiClient?: AiCompletionClient; artifactRoot: string; environment?: RepositoryEnvironment; + aiOverview?: Omit; } /** @@ -65,11 +68,14 @@ export async function createComplianceEngineRuntime(options: ComplianceEngineRun disabledReason }); await corpusUploadService.initialize(); + const aiOverviewService = new AiOverviewService(read, store, { ...options.aiOverview, client: options.aiClient }); + await aiOverviewService.recoverInterruptedRuns(); return { mappingService, corpusUploadService, + aiOverviewService, store, - stop: () => corpusUploadService.stop().then(() => mappingService.close()), + stop: () => Promise.all([corpusUploadService.stop().then(() => mappingService.close()), aiOverviewService.close()]).then(() => undefined), close: async () => { await pool?.end(); } }; } diff --git a/server/compliance/postgres-compliance-store.ts b/server/compliance/postgres-compliance-store.ts index f49d29f..c32ad31 100644 --- a/server/compliance/postgres-compliance-store.ts +++ b/server/compliance/postgres-compliance-store.ts @@ -10,6 +10,7 @@ import { type RegulatoryMappingDecision, type RegulatoryMappingRun } from '../../src/compliance/regulatory-mapping'; +import { aiOverviewRunValidationErrors, assertAiOverviewTransition, type AiOverviewRun } from '../../src/compliance/ai-overview'; interface DataRow { data: T } @@ -30,7 +31,7 @@ function dataRow(result: QueryResult>): T | null { } /** - * The Compliance Engine's own tables (migration 026). Writes run in the engine's own transactions: + * The Compliance Engine's own tables (migration 026, and the AI overview runs of migration 025). Writes run in the engine's own transactions: * they never advance the ASSESSOR repository revision and never trigger a Result DB rebuild. */ export class PostgresComplianceStore implements ComplianceStore { @@ -211,13 +212,68 @@ export class PostgresComplianceStore implements ComplianceStore { `, [ruleKey])); } + async createAiOverviewRun(run: AiOverviewRun): Promise { + const errors = aiOverviewRunValidationErrors(run); + if (errors.length) throw new RepositoryValidationError(errors.join(' ')); + return this.transaction(async client => { + const idempotent = dataRow(await client.query>(` + SELECT data FROM assessor_ai_overview_runs + WHERE project_id = $1 AND scope_key = $2 AND idempotency_key = $3 + `, [run.projectId, run.scopeKey, run.idempotencyKey])); + if (idempotent) return idempotent; + await client.query(` + INSERT INTO assessor_ai_overview_runs + (id, project_id, scope_key, assessment_ids, scan_run_ids, idempotency_key, run_status, + input_hash, prompt_version, output_schema_version, model, created_at, updated_at, data) + VALUES ($1, $2, $3, $4::text[], $5::text[], $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb) + `, [run.id, run.projectId, run.scopeKey, run.assessmentIds, run.scanRunIds, run.idempotencyKey, + run.status, run.inputHash, run.promptVersion, run.outputSchemaVersion, run.model, + run.createdAt, run.updatedAt, json(run)]); + return clone(run); + }); + } + + async updateAiOverviewRun(run: AiOverviewRun): Promise { + const errors = aiOverviewRunValidationErrors(run); + if (errors.length) throw new RepositoryValidationError(errors.join(' ')); + return this.transaction(async client => { + const existing = dataRow(await client.query>( + 'SELECT data FROM assessor_ai_overview_runs WHERE id = $1 FOR UPDATE', [run.id] + )); + if (!existing) throw new RepositoryNotFoundError(`AI overview run ${run.id} does not exist.`); + assertAiOverviewTransition(existing, run); + await client.query(` + UPDATE assessor_ai_overview_runs SET + run_status = $2, updated_at = $3, data = $4::jsonb + WHERE id = $1 + `, [run.id, run.status, run.updatedAt, json(run)]); + return clone(run); + }); + } + + async listAiOverviewRuns(projectId: string, scopeKey?: string): Promise { + return dataRows(await this.pool.query>(` + SELECT data FROM assessor_ai_overview_runs + WHERE project_id = $1 AND ($2::text IS NULL OR scope_key = $2) + ORDER BY created_at DESC, id DESC + `, [projectId, scopeKey ?? null])); + } + + async deleteAiOverviewRuns(projectId: string, runIds: string[]): Promise { + if (!runIds.length) return; + await this.transaction(client => client.query( + 'DELETE FROM assessor_ai_overview_runs WHERE project_id = $1 AND id = ANY($2::text[])', [projectId, runIds] + )); + } + async exportProject(projectId: string): Promise { return { regulatoryMappingBatchRuns: await this.listRegulatoryMappingBatchRuns(projectId), regulatoryMappingRuns: (await this.listRegulatoryMappingRuns()).filter(item => item.status === 'succeeded'), regulatoryMappingDecisions: dataRows(await this.pool.query>( 'SELECT data FROM assessor_regulatory_mapping_decisions ORDER BY created_at, id' - )) + )), + aiOverviewRuns: (await this.listAiOverviewRuns(projectId)).filter(item => item.status === 'succeeded') }; } } diff --git a/server/compliance/regulatory-mapping-service.ts b/server/compliance/regulatory-mapping-service.ts index 7b53d8c..ba05afb 100644 --- a/server/compliance/regulatory-mapping-service.ts +++ b/server/compliance/regulatory-mapping-service.ts @@ -7,6 +7,7 @@ import { REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, REGULATORY_RETRIEVAL_VERSION, + REGULATORY_GPO_QUERY_VERSION, emptyRegulatoryMappingBatchCounts, groundRegulatoryMappingResult, normalizeRegulatoryMappingBatchScope, @@ -18,6 +19,7 @@ import { type RegulatoryMappingCandidate, type RegulatoryMappingCreateRequest, type RegulatoryMappingDecision, + type RegulatoryEarlierControl, type RegulatoryMappingDecisionCreateRequest, type RegulatoryMappingEvidence, type RegulatoryMappingResponse, @@ -130,6 +132,7 @@ export class RegulatoryMappingService { const ruleKey = supported ? this.ruleKey(finding) : undefined; const run = ruleKey && this.options.client ? await this.store.getRegulatoryMappingRun(this.mappingId(ruleKey)) : null; const decisions = ruleKey ? await this.store.listRegulatoryMappingDecisions(ruleKey) : []; + const earlierControls = ruleKey ? await this.earlierControls(ruleKey, run, decisions) : []; return { version: 1, projectId, @@ -142,10 +145,40 @@ export class RegulatoryMappingService { : !availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}), ...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}), ...(run ? { run: withoutEvidenceText(run) } : {}), - decisions + decisions, + ...(earlierControls.length ? { earlierControls } : {}) }; } + /** + * The run that grounds a decision: the current one, or for an earlier decision the newest older succeeded + * run of the rule that retrieved the control, so it stays reviewable after a pipeline version change. + */ + private async groundingMappingId(ruleKey: string, controlId: string): Promise { + const currentId = this.mappingId(ruleKey); + const current = await this.store.getRegulatoryMappingRun(currentId); + if (!current || current.candidates.some(candidate => candidate.controlId === controlId)) return currentId; + const older = (await this.store.listRegulatoryMappingRuns()).find(run => run.ruleKey === ruleKey && run.status === 'succeeded' + && run.candidates.some(candidate => candidate.controlId === controlId)); + return older?.id ?? currentId; + } + + /** Decided controls the current run did not retrieve, titled from the newest older run of the rule that did. */ + private async earlierControls(ruleKey: string, run: RegulatoryMappingRun | null, decisions: RegulatoryMappingDecision[]): Promise { + const current = new Set(run?.candidates.map(candidate => candidate.controlId) ?? []); + const missing = [...new Set(decisions.map(decision => decision.controlId))].filter(controlId => !current.has(controlId)); + if (!missing.length) return []; + const known = new Map(); + for (const older of await this.store.listRegulatoryMappingRuns()) { + if (older.ruleKey !== ruleKey) continue; + for (const candidate of older.candidates) if (!known.has(candidate.controlId)) known.set(candidate.controlId, candidate); + } + return missing.sort((left, right) => left.localeCompare(right, 'en', { numeric: true })).map(controlId => { + const candidate = known.get(controlId); + return { controlId, annexAControlId: candidate?.annexAControlId ?? `A.${controlId}`, ...(candidate?.title ? { title: candidate.title } : {}) }; + }); + } + async start(projectId: string, request: RegulatoryMappingCreateRequest): Promise { if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503); const finding = await this.finding(projectId, request.resultProjectionId); @@ -210,13 +243,14 @@ export class RegulatoryMappingService { if (!isRegulatoryMappableResult(finding)) throw new RegulatoryMappingServiceError('unsupported_finding_kind', UNSUPPORTED_KIND_MESSAGE, 422); const ruleKey = this.ruleKey(finding); const comment = request.comment?.trim(); + const mappingId = await this.groundingMappingId(ruleKey, request.controlId); const decision: RegulatoryMappingDecision = { version: 1, id: `regulatory-decision-${randomUUID()}`, ruleKey, sourceTool: finding.sourceTool ?? 'unknown', ...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}), - mappingId: this.mappingId(ruleKey), + mappingId, controlId: request.controlId, decision: request.decision, ...(comment ? { comment } : {}), @@ -226,18 +260,29 @@ export class RegulatoryMappingService { return this.get(projectId, request.resultProjectionId); } + /** Accepted controls of every mappable finding in a client timeslot, for the Work `ISO control` and `ISO theme` filters. */ + async controlAssignments(projectId: string, requested: Pick): Promise { + return this.controlOverview(projectId, requested, { deviationsOnly: false }); + } + /** * Accepted controls of the deviations in one client timeslot. Decisions are global per scanner rule; * a GPO setting counts once, like its rolled-up Work record, however many baseline comparisons it has. */ - async controlOverview(projectId: string, requested: Pick): Promise { - const scope = await this.validateBatchScope(projectId, { assessmentIds: requested.assessmentIds, scanRunIds: requested.scanRunIds, filters: { deviation: true } }); + async controlOverview(projectId: string, requested: Pick, + { deviationsOnly = true }: { deviationsOnly?: boolean } = {}): Promise { + const scope = await this.validateBatchScope(projectId, { assessmentIds: requested.assessmentIds, scanRunIds: requested.scanRunIds, + filters: deviationsOnly ? { deviation: true } : {} }); const byRule = new Map>(); - for (const finding of (await this.queryBatchResults(projectId, scope)).filter(isRegulatoryMappableResult)) { - const identity = finding.sourceKind === 'scanner-finding' ? `scanner:${finding.sourceResultId}` : `gpo:${finding.policyScope ?? ''}:${finding.sourceRuleId ?? finding.sourceResultId}`; + const results = (await this.queryBatchResults(projectId, scope)).filter(isRegulatoryMappableResult); + const settingIds = await this.gpoSettingIds(projectId, scope, results); + for (const finding of results) { + const settingId = settingIds.get(finding.resultProjectionId); + const identity = findingIdentity(finding, settingId); const findings = byRule.get(this.ruleKey(finding)) ?? new Map(); + // The setting ID only lets Work match its rolled-up GPO setting; it is never displayed or exported. if (!findings.has(identity)) findings.set(identity, { resultProjectionId: finding.resultProjectionId, sourceKind: finding.sourceKind, - sourceResultId: finding.sourceResultId, sourceRuleId: finding.sourceRuleId, policyScope: finding.policyScope }); + sourceResultId: finding.sourceResultId, sourceRuleId: settingId ?? finding.sourceRuleId, policyScope: finding.policyScope }); byRule.set(this.ruleKey(finding), findings); } // Control titles come from the newest succeeded run of each rule that retrieved the control. @@ -303,10 +348,18 @@ export class RegulatoryMappingService { // Deterministic id per (ruleKey, pipeline versions) so one row exists per rule per // version tuple: regeneration reuses it, and a version bump yields a new mapping. private mappingId(ruleKey: string): string { - const tuple = [ruleKey, this.options.corpusVersion, REGULATORY_RETRIEVAL_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, this.options.client!.model].join(''); + const tuple = [ruleKey, this.options.corpusVersion, REGULATORY_RETRIEVAL_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, this.options.client!.model, + // GPO rules carry the baseline context of their query version; scanner mapping ids stay unchanged. + ...(isGpoRuleKey(ruleKey) ? [REGULATORY_GPO_QUERY_VERSION] : [])].join(''); return `regulatory-mapping-${createHash('sha256').update(tuple, 'utf8').digest('hex').slice(0, 40)}`; } + // The stored prompt version must differ wherever the id differs: the table's unique + // identity tuple has no column for the GPO query version, so GPO runs record it here. + private promptVersion(ruleKey: string): string { + return isGpoRuleKey(ruleKey) ? `${REGULATORY_MAPPING_PROMPT_VERSION}+${REGULATORY_GPO_QUERY_VERSION}` : REGULATORY_MAPPING_PROMPT_VERSION; + } + async close(): Promise { while (this.activeTasks.size) await Promise.allSettled([...this.activeTasks]); } @@ -352,7 +405,7 @@ export class RegulatoryMappingService { sourceHashes: [], embeddingModel: this.options.embeddingModel!, retrievalVersion: REGULATORY_RETRIEVAL_VERSION, - promptVersion: REGULATORY_MAPPING_PROMPT_VERSION, + promptVersion: this.promptVersion(ruleKey), outputSchemaVersion: REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION, model: this.options.client!.model, createdAt: timestamp, @@ -406,6 +459,30 @@ export class RegulatoryMappingService { return items; } + /** + * The internal technical setting ID of each GPO comparison the Result DB delivered without one + * (user decision 2026-10-05). Read-only, limited to policy-evaluation rows of the same timeslot, + * and used only to identify the rolled-up Work setting; no other internal field is kept. + */ + private async gpoSettingIds(projectId: string, scope: Pick, + results: UnifiedResultProjection[]): Promise> { + const wanted = new Set(results.filter(item => item.sourceKind === 'policy-evaluation' && !item.sourceRuleId?.trim()) + .map(item => item.resultProjectionId)); + const settingIds = new Map(); + if (!wanted.size) return settingIds; + const query = { audience: 'internal-analysis' as const, projectId, assessmentIds: scope.assessmentIds, scanRunIds: scope.scanRunIds, + sourceKind: ['policy-evaluation' as const] }; + for (let offset = 0; ; offset += BATCH_PAGE_SIZE) { + const page = await this.read.queryUnifiedResults({ ...query, offset, limit: BATCH_PAGE_SIZE }); + for (const item of page.items) { + const settingId = item.sourceRuleId?.trim(); + if (settingId && wanted.has(item.resultProjectionId)) settingIds.set(item.resultProjectionId, settingId); + } + if (offset + page.items.length >= page.total || page.items.length === 0) break; + } + return settingIds; + } + private async processBatch(queued: RegulatoryMappingBatchRun): Promise { const startedAt = this.now().toISOString(); let run = await this.store.updateRegulatoryMappingBatchRun({ ...queued, status: 'running', startedAt, updatedAt: startedAt }); @@ -414,13 +491,20 @@ export class RegulatoryMappingService { .sort((left, right) => left.resultProjectionId.localeCompare(right.resultProjectionId)); const representatives = new Map(); const counts = { ...emptyRegulatoryMappingBatchCounts(), filteredResultCount: results.length }; + // Findings count like Work records: a GPO setting once, however many baseline comparisons it has. + const eligible = new Set(); + const unsupported = new Set(); + const settingIds = await this.gpoSettingIds(run.projectId, run, results); for (const finding of results) { if (!isRegulatoryMappableResult(finding)) { counts.excludedNonScannerCount += 1; continue; } - if (!buildRegulatoryFindingQuery(finding)) { counts.unsupportedFindingCount += 1; continue; } - counts.eligibleFindingCount += 1; + const identity = findingIdentity(finding, settingIds.get(finding.resultProjectionId)); + if (!buildRegulatoryFindingQuery(finding)) { unsupported.add(identity); continue; } + eligible.add(identity); const ruleKey = this.ruleKey(finding); if (!representatives.has(ruleKey)) representatives.set(ruleKey, finding); } + counts.eligibleFindingCount = eligible.size; + counts.unsupportedFindingCount = unsupported.size; counts.distinctRuleCount = representatives.size; const scheduled: Array> = []; for (const finding of representatives.values()) { @@ -640,6 +724,20 @@ function safeFailureDetail(error: unknown): string | undefined { return detail || undefined; } +/** + * One finding as Work counts it: a scanner finding by occurrence, a GPO setting by scope and technical + * setting identity. The Result DB withholds ASSESSOR's internal identity hash, which the engine reads + * separately; only if that is unavailable does the disclosed title, the technical setting identity, stand in. + */ +function findingIdentity(finding: UnifiedResultProjection, settingId?: string): string { + if (finding.sourceKind === 'scanner-finding') return `scanner:${finding.sourceResultId}`; + return `gpo:${finding.policyScope ?? ''}:${settingId ?? (finding.sourceRuleId?.trim() || `title:${finding.title.trim()}`)}`; +} + +function isGpoRuleKey(ruleKey: string): boolean { + return ruleKey.startsWith('GPOs:') || ruleKey.startsWith('GPOs#'); +} + function sha256(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex'); } diff --git a/server/gpo/gpo-backup-job-processor.ts b/server/gpo/gpo-backup-job-processor.ts index 22c1edb..ac5cf5f 100644 --- a/server/gpo/gpo-backup-job-processor.ts +++ b/server/gpo/gpo-backup-job-processor.ts @@ -9,7 +9,7 @@ import type { BackendArtifactStore } from '../artifacts/backend-artifact-store'; import { GPO_BACKUP_ANALYZER_VERSION } from './gpo-backup-submission-service'; import type { GpoBackupWorkerExecutor } from './gpo-inventory-executor'; -export const GPO_BACKUP_JOB_PROCESSOR_VERSION = '2.2.5'; +export const GPO_BACKUP_JOB_PROCESSOR_VERSION = '2.2.6'; export interface GpoBackupJobProcessorOptions { leaseDurationMs?: number; diff --git a/server/gpo/gpo-backup-submission-service.ts b/server/gpo/gpo-backup-submission-service.ts index 36e565e..5c0e0ba 100644 --- a/server/gpo/gpo-backup-submission-service.ts +++ b/server/gpo/gpo-backup-submission-service.ts @@ -7,7 +7,7 @@ import type { BackendArtifactStore } from '../artifacts/backend-artifact-store'; import type { GpoInventoryExecutor } from './gpo-inventory-executor'; import { gpoSourceAvailability, requireGpoSource } from './gpo-source-availability'; -export const GPO_BACKUP_ANALYZER_VERSION = 'gpo-backup-policy-analysis:2.2.5'; +export const GPO_BACKUP_ANALYZER_VERSION = 'gpo-backup-policy-analysis:2.2.6'; export interface SubmitGpoBackupRequest { restoreArtifactId?: string; diff --git a/server/index.ts b/server/index.ts index c78b399..7724223 100644 --- a/server/index.ts +++ b/server/index.ts @@ -21,7 +21,6 @@ import { MitreCatalogService } from './reference-data/mitre-catalog-service'; import { FileSystemReportArchive, resolveReportArchiveRoot, type ReportArchiveEnvironment } from './reporting/report-archive'; import { createPostgresReportReadRuntime, memoryReportReadRuntime, type ReportReadRuntime } from './reporting/report-read-runtime'; import { createLiteLlmClientFromEnvironment } from './ai/litellm-client'; -import { AiOverviewService } from './ai/ai-overview-service'; import { createComplianceEngineRuntime } from './compliance/compliance-engine-runtime'; function boundedInteger(value: string | undefined, fallback: number, minimum: number, maximum: number, name: string): number { @@ -77,19 +76,17 @@ async function main(): Promise { console.warn('Report Engine nicht verfügbar:', error instanceof Error ? error.message : 'Unbekannter Fehler'); } const aiClient = createLiteLlmClientFromEnvironment(); - const aiOverviewService = new AiOverviewService(runtime.repository, { - client: aiClient, - maximumChunkCharacters: boundedInteger(process.env.ASSESSOR_AI_CHUNK_CHARS, 80_000, 10_000, 500_000, 'ASSESSOR_AI_CHUNK_CHARS'), - maximumCalls: boundedInteger(process.env.ASSESSOR_AI_MAX_CALLS, 64, 1, 64, 'ASSESSOR_AI_MAX_CALLS'), - leaseDurationMs: boundedInteger(process.env.ASSESSOR_AI_JOB_LEASE_MS, 5 * 60_000, 10_000, 30 * 60_000, 'ASSESSOR_AI_JOB_LEASE_MS') - }); - await aiOverviewService.recoverInterruptedRuns(); // The Compliance Engine runs beside ASSESSOR with its own store and reads the Core only through its read-only port. const complianceEngine = await createComplianceEngineRuntime({ repository: runtime.repository, persistence: runtime.persistence, aiClient, - artifactRoot + artifactRoot, + aiOverview: { + maximumChunkCharacters: boundedInteger(process.env.ASSESSOR_AI_CHUNK_CHARS, 80_000, 10_000, 500_000, 'ASSESSOR_AI_CHUNK_CHARS'), + maximumCalls: boundedInteger(process.env.ASSESSOR_AI_MAX_CALLS, 64, 1, 64, 'ASSESSOR_AI_MAX_CALLS'), + leaseDurationMs: boundedInteger(process.env.ASSESSOR_AI_JOB_LEASE_MS, 5 * 60_000, 10_000, 30 * 60_000, 'ASSESSOR_AI_JOB_LEASE_MS') + } }); const artifactStore = new FileSystemBackendArtifactStore(artifactRoot); const mitreCatalogService = new MitreCatalogService(join(artifactRoot, 'reference-data', 'mitre-enterprise.json')); @@ -151,7 +148,7 @@ async function main(): Promise { gpoPolicyEvaluationService, gpoPolicyFindingService, gpoPolicyAnalysisOverviewService, - aiOverviewService, + aiOverviewService: complianceEngine.aiOverviewService, regulatoryMappingService: complianceEngine.mappingService, regulatoryCorpusUploadService: complianceEngine.corpusUploadService, complianceExport: complianceEngine.store, @@ -171,7 +168,7 @@ async function main(): Promise { if (shuttingDown) return; shuttingDown = true; console.log(`${signal}: ASSESSOR wird beendet.`); - void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop(), aiOverviewService.close(), complianceEngine.stop()]).then(() => { + void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop(), complianceEngine.stop()]).then(() => { server.close(async error => { try { await reportRuntime?.close(); diff --git a/server/postgres/postgres-assessor-repository.ts b/server/postgres/postgres-assessor-repository.ts index 60c3a00..a35aa61 100644 --- a/server/postgres/postgres-assessor-repository.ts +++ b/server/postgres/postgres-assessor-repository.ts @@ -34,8 +34,6 @@ import { POLICY_EVALUATION_ENGINE, storedPolicyEvaluationValidationErrors } from import type { StoredPolicyFindingRun } from '../../src/domain/policy-finding'; import type { EnvironmentObservation, EnvironmentSnapshot, EnvironmentSnapshotQuery, ScannerReportSection } from '../../src/domain/environment-context'; import { environmentCoverage } from '../../src/domain/environment-context'; -import type { AiOverviewRun } from '../../src/domain/ai-overview'; -import { aiOverviewRunValidationErrors } from '../../src/domain/ai-overview'; import { storedPolicyFindingRunValidationErrors } from '../../src/domain/policy-finding'; import { latestReferenceRuns, type ReferenceRecognitionRun, type ReferenceCatalogSnapshot } from '../../src/domain/reference-recognition'; import { validateReferenceRun } from '../../src/persistence/reference-run-validation'; @@ -652,61 +650,6 @@ export class PostgresAssessorRepository implements AssessorRepository { WHERE r.project_id = $1 AND rc.catalog_sha256 = c.sha256) ORDER BY c.sha256`, [projectId])); } - async createAiOverviewRun(run: AiOverviewRun): Promise { - const errors = aiOverviewRunValidationErrors(run); - if (errors.length) throw new RepositoryValidationError(errors.join(' ')); - return this.mutate(async client => { - const idempotent = dataRow(await client.query>(` - SELECT data FROM assessor_ai_overview_runs - WHERE project_id = $1 AND scope_key = $2 AND idempotency_key = $3 - `, [run.projectId, run.scopeKey, run.idempotencyKey])); - if (idempotent) return idempotent; - await client.query(` - INSERT INTO assessor_ai_overview_runs - (id, project_id, scope_key, assessment_ids, scan_run_ids, idempotency_key, run_status, - input_hash, prompt_version, output_schema_version, model, created_at, updated_at, data) - VALUES ($1, $2, $3, $4::text[], $5::text[], $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb) - `, [run.id, run.projectId, run.scopeKey, run.assessmentIds, run.scanRunIds, run.idempotencyKey, - run.status, run.inputHash, run.promptVersion, run.outputSchemaVersion, run.model, - run.createdAt, run.updatedAt, json(run)]); - return clone(run); - }, undefined, result => result.id === run.id); - } - - async updateAiOverviewRun(run: AiOverviewRun): Promise { - const errors = aiOverviewRunValidationErrors(run); - if (errors.length) throw new RepositoryValidationError(errors.join(' ')); - return this.mutate(async client => { - const existing = dataRow(await client.query>( - 'SELECT data FROM assessor_ai_overview_runs WHERE id = $1 FOR UPDATE', [run.id] - )); - if (!existing) throw new RepositoryNotFoundError(`AI overview run ${run.id} does not exist.`); - if (existing.projectId !== run.projectId || existing.scopeKey !== run.scopeKey || existing.inputHash !== run.inputHash - || existing.idempotencyKey !== run.idempotencyKey || existing.model !== run.model) { - throw new RepositoryConflictError('Immutable AI overview run identity cannot be changed.'); - } - if (['succeeded', 'failed'].includes(existing.status)) throw new RepositoryConflictError('A completed AI overview run is immutable.'); - if (existing.status === 'queued' && !['running', 'failed'].includes(run.status) - || existing.status === 'running' && !['running', 'succeeded', 'failed'].includes(run.status)) { - throw new RepositoryConflictError(`Invalid AI overview transition from ${existing.status} to ${run.status}.`); - } - await client.query(` - UPDATE assessor_ai_overview_runs SET - run_status = $2, updated_at = $3, data = $4::jsonb - WHERE id = $1 - `, [run.id, run.status, run.updatedAt, json(run)]); - return clone(run); - }); - } - - async listAiOverviewRuns(projectId: string, scopeKey?: string): Promise { - return dataRows(await this.pool.query>(` - SELECT data FROM assessor_ai_overview_runs - WHERE project_id = $1 AND ($2::text IS NULL OR scope_key = $2) - ORDER BY created_at DESC, id DESC - `, [projectId, scopeKey ?? null])); - } - async beginImport(start: ImportSessionStart): Promise { if (!start.sessionId.trim()) throw new RepositoryValidationError('Eine Importsitzung benötigt eine ID.'); if (this.activeImportSessions.has(start.sessionId)) throw new RepositoryConflictError(`Importsitzung ${start.sessionId} ist bereits aktiv.`); @@ -2778,8 +2721,7 @@ export class PostgresAssessorRepository implements AssessorRepository { environmentProfiles: await this.listEnvironmentProfiles(projectId), policySnapshotAggregations: await this.listPolicySnapshotAggregations(projectId), policyEvaluations: await this.listPolicyEvaluations(projectId), - policyFindingRuns, - aiOverviewRuns: (await this.listAiOverviewRuns(projectId)).filter(item => item.status === 'succeeded') + policyFindingRuns }; } @@ -2803,7 +2745,6 @@ export class PostgresAssessorRepository implements AssessorRepository { [scanRunId] ); await client.query('DELETE FROM assessor_comparisons WHERE assessment_id = $1', [row.assessment_id]); - await client.query('DELETE FROM assessor_ai_overview_runs WHERE project_id = $1 AND $2 = ANY(scan_run_ids)', [projectId, scanRunId]); const deleted = await client.query('DELETE FROM assessor_scan_runs WHERE id = $1 RETURNING id', [scanRunId]); if (!deleted.rowCount) throw new RepositoryConflictError('Scanbericht wurde zwischen Prüfung und Löschung bereits entfernt.'); await client.query(` @@ -2848,7 +2789,6 @@ export class PostgresAssessorRepository implements AssessorRepository { await client.query('DELETE FROM assessor_asset_edges WHERE assessment_id = $1', [assessmentId]); await client.query('DELETE FROM assessor_edge_coverage WHERE assessment_id = $1', [assessmentId]); await client.query('DELETE FROM assessor_canonical_assets WHERE assessment_id = $1', [assessmentId]); - await client.query('DELETE FROM assessor_ai_overview_runs WHERE project_id = $1 AND $2 = ANY(assessment_ids)', [projectId, assessmentId]); const deleted = await client.query('DELETE FROM assessor_assessments WHERE id = $1 RETURNING id', [assessmentId]); if (!deleted.rowCount) throw new RepositoryConflictError('The timeslot was removed between validation and deletion.'); await client.query(` diff --git a/src/api/assessor-api-client.ts b/src/api/assessor-api-client.ts index 1e1fd5f..892870c 100644 --- a/src/api/assessor-api-client.ts +++ b/src/api/assessor-api-client.ts @@ -260,6 +260,13 @@ export class AssessorApiClient { })); } + getRegulatoryControlAssignments(projectId: string, scope: { assessmentIds: string[]; scanRunIds: string[] }): Promise { + return this.request(unifiedQueryPath(`/projects/${encodeURIComponent(projectId)}/regulatory-control-assignments`, { + assessmentIds: scope.assessmentIds, + scanRunIds: scope.scanRunIds + })); + } + getRegulatoryControlOverview(projectId: string, scope: { assessmentIds: string[]; scanRunIds: string[] }): Promise { return this.request(unifiedQueryPath(`/projects/${encodeURIComponent(projectId)}/regulatory-control-overview`, { assessmentIds: scope.assessmentIds, diff --git a/src/api/assessor-api-contract.ts b/src/api/assessor-api-contract.ts index d8174ce..5957d20 100644 --- a/src/api/assessor-api-contract.ts +++ b/src/api/assessor-api-contract.ts @@ -17,7 +17,7 @@ import type { PolicyEvaluationRuleCatalog } from '../domain/policy-evaluation-ru import type { PolicyAnalysisOverview } from '../domain/policy-analysis-overview'; import type { MitreGraphData } from '../features/mitre-graph/mitre-graph'; import type { AssessmentContext } from '../assessment-context/context-engine'; -export type { AiOverviewCreateRequest, AiOverviewResponse } from '../domain/ai-overview'; +export type { AiOverviewCreateRequest, AiOverviewResponse } from '../compliance/ai-overview'; export type { RegulatoryMappingBatchCreateRequest, RegulatoryMappingBatchResponse, diff --git a/src/api/request-activity.ts b/src/api/request-activity.ts index 101d813..c0561b0 100644 --- a/src/api/request-activity.ts +++ b/src/api/request-activity.ts @@ -28,6 +28,7 @@ export function requestActivity(path: string, method = 'GET'): { area: string; l /** Compliance Engine routes; checked before the generic project patterns so they never read as client work. */ function complianceActivity(route: string, method: string): string | undefined { const write = method !== 'GET'; + if (route.includes('/regulatory-control-assignments')) return 'Loading ISO control filters'; if (route.includes('/regulatory-control-overview')) return 'Loading ISO control overview'; if (route.includes('/regulatory-mapping-batches')) return write ? 'Starting ISO mapping batch' : 'Loading ISO mapping batch status'; if (route.includes('/regulatory-mapping-runs')) return 'Generating ISO control proposals'; diff --git a/src/app/AppProvider.tsx b/src/app/AppProvider.tsx index 5008110..1d1da47 100644 --- a/src/app/AppProvider.tsx +++ b/src/app/AppProvider.tsx @@ -119,10 +119,11 @@ export function AppProvider({ children }: PropsWithChildren) { const facetFilters = { ...current.facetFilters }; if (!engines.context) delete facetFilters['context:cluster']; if (!engines.context || !engines.report) delete facetFilters['report:business-impact']; + if (!engines.compliance) { delete facetFilters['compliance:iso-control']; delete facetFilters['compliance:iso-theme']; } if (Object.keys(facetFilters).length === Object.keys(current.facetFilters).length) return current; return { ...current, facetFilters }; }); - }, [engines.context, engines.report, state.facetFilters]); + }, [engines.compliance, engines.context, engines.report, state.facetFilters]); useEffect(() => { let active = true; diff --git a/src/domain/ai-overview.ts b/src/compliance/ai-overview.ts similarity index 91% rename from src/domain/ai-overview.ts rename to src/compliance/ai-overview.ts index 9a4d5fc..38e62ef 100644 --- a/src/domain/ai-overview.ts +++ b/src/compliance/ai-overview.ts @@ -1,3 +1,5 @@ +import { RepositoryConflictError } from '../persistence/repository-errors'; + export const AI_OVERVIEW_PROMPT_VERSION = '1.1.0'; export const AI_OVERVIEW_OUTPUT_SCHEMA_VERSION = 1; @@ -239,6 +241,19 @@ function containsDirectIdentifier(value: unknown): boolean { return isObject(value) && Object.values(value).some(containsDirectIdentifier); } +/** A stored run keeps its identity; queued -> running|failed, running -> running|succeeded|failed, completed is final. */ +export function assertAiOverviewTransition(existing: AiOverviewRun, run: AiOverviewRun): void { + if (existing.projectId !== run.projectId || existing.scopeKey !== run.scopeKey || existing.inputHash !== run.inputHash + || existing.idempotencyKey !== run.idempotencyKey || existing.model !== run.model) { + throw new RepositoryConflictError('Immutable AI overview run identity cannot be changed.'); + } + if (['succeeded', 'failed'].includes(existing.status)) throw new RepositoryConflictError('A completed AI overview run is immutable.'); + if (existing.status === 'queued' && !['running', 'failed'].includes(run.status) + || existing.status === 'running' && !['running', 'succeeded', 'failed'].includes(run.status)) { + throw new RepositoryConflictError(`Invalid AI overview transition from ${existing.status} to ${run.status}.`); + } +} + export function aiOverviewRunValidationErrors(run: AiOverviewRun): string[] { const errors: string[] = []; if (!run.id.trim() || !run.projectId.trim() || !run.scopeKey.trim() || !run.idempotencyKey.trim()) errors.push('AI overview run identity is incomplete.'); diff --git a/src/compliance/compliance-store.ts b/src/compliance/compliance-store.ts index 0cd89eb..8be58ef 100644 --- a/src/compliance/compliance-store.ts +++ b/src/compliance/compliance-store.ts @@ -1,5 +1,6 @@ import type { AssessorRepository } from '../persistence/assessor-repository'; import type { RegulatoryMappingBatchRun, RegulatoryMappingDecision, RegulatoryMappingRun } from './regulatory-mapping'; +import type { AiOverviewRun } from './ai-overview'; /** * The only Core reads the Compliance Engine performs (AGENTS.md, "Core Engine and engines"). The engine @@ -25,10 +26,17 @@ export interface ComplianceProjectExport { /** Mappings and decisions are global and keyed by scanner rule, not by project. */ regulatoryMappingRuns: RegulatoryMappingRun[]; regulatoryMappingDecisions: RegulatoryMappingDecision[]; + /** Succeeded AI overview runs of the project (table of migration 025, owned by the engine). */ + aiOverviewRuns: AiOverviewRun[]; } -/** The engine's own storage: mapping runs, accept/reject decisions and batches. */ +/** The engine's own storage: mapping runs, accept/reject decisions, batches and AI overview runs. */ export interface ComplianceStore { + createAiOverviewRun(run: AiOverviewRun): Promise; + updateAiOverviewRun(run: AiOverviewRun): Promise; + listAiOverviewRuns(projectId: string, scopeKey?: string): Promise; + /** Removes runs of a project; the engine prunes runs whose timeslot no longer exists in the Core. */ + deleteAiOverviewRuns(projectId: string, runIds: string[]): Promise; getRegulatoryMappingRun(id: string): Promise; claimRegulatoryMappingRun(run: RegulatoryMappingRun, forceRegeneration: boolean): Promise<{ run: RegulatoryMappingRun; claimed: boolean }>; saveRegulatoryMappingRun(run: RegulatoryMappingRun): Promise; diff --git a/src/compliance/local-compliance-store.ts b/src/compliance/local-compliance-store.ts index 52ce98f..2479e42 100644 --- a/src/compliance/local-compliance-store.ts +++ b/src/compliance/local-compliance-store.ts @@ -8,6 +8,7 @@ import { type RegulatoryMappingDecision, type RegulatoryMappingRun } from './regulatory-mapping'; +import { aiOverviewRunValidationErrors, assertAiOverviewTransition, type AiOverviewRun } from './ai-overview'; function clone(value: T): T { return structuredClone(value); @@ -21,6 +22,7 @@ export class LocalComplianceStore implements ComplianceStore { private runs: RegulatoryMappingRun[] = []; private batches: RegulatoryMappingBatchRun[] = []; private decisions: RegulatoryMappingDecision[] = []; + private aiOverviewRuns: AiOverviewRun[] = []; async getRegulatoryMappingRun(id: string): Promise { const run = this.runs.find(item => item.id === id); @@ -103,11 +105,48 @@ export class LocalComplianceStore implements ComplianceStore { return this.decisions.filter(item => item.ruleKey === ruleKey).map(clone).sort(newestFirst); } + async createAiOverviewRun(run: AiOverviewRun): Promise { + const errors = aiOverviewRunValidationErrors(run); + if (errors.length) throw new RepositoryValidationError(errors.join(' ')); + const idempotent = this.aiOverviewRuns.find(item => + item.projectId === run.projectId && item.scopeKey === run.scopeKey && item.idempotencyKey === run.idempotencyKey); + if (idempotent) return clone(idempotent); + if (this.aiOverviewRuns.some(item => item.projectId === run.projectId && item.scopeKey === run.scopeKey + && ['queued', 'running'].includes(item.status))) { + throw new RepositoryConflictError('For this timeslot, an AI overview is already being generated.'); + } + if (this.aiOverviewRuns.some(item => item.id === run.id)) throw new RepositoryConflictError(`AI overview run ${run.id} already exists.`); + this.aiOverviewRuns.push(clone(run)); + return clone(run); + } + + async updateAiOverviewRun(run: AiOverviewRun): Promise { + const errors = aiOverviewRunValidationErrors(run); + if (errors.length) throw new RepositoryValidationError(errors.join(' ')); + const index = this.aiOverviewRuns.findIndex(item => item.id === run.id); + if (index < 0) throw new RepositoryNotFoundError(`AI overview run ${run.id} does not exist.`); + assertAiOverviewTransition(this.aiOverviewRuns[index], run); + this.aiOverviewRuns[index] = clone(run); + return clone(run); + } + + async listAiOverviewRuns(projectId: string, scopeKey?: string): Promise { + return this.aiOverviewRuns.filter(item => item.projectId === projectId && (!scopeKey || item.scopeKey === scopeKey)) + .map(clone).sort(newestFirst); + } + + async deleteAiOverviewRuns(projectId: string, runIds: string[]): Promise { + const ids = new Set(runIds); + this.aiOverviewRuns = this.aiOverviewRuns.filter(item => item.projectId !== projectId || !ids.has(item.id)); + } + async exportProject(projectId: string): Promise { return { regulatoryMappingBatchRuns: this.batches.filter(item => item.projectId === projectId).map(clone), regulatoryMappingRuns: this.runs.filter(item => item.status === 'succeeded').map(clone), - regulatoryMappingDecisions: this.decisions.map(clone) + regulatoryMappingDecisions: this.decisions.map(clone), + aiOverviewRuns: this.aiOverviewRuns.filter(item => item.projectId === projectId && item.status === 'succeeded') + .map(clone).sort(newestFirst) }; } } diff --git a/src/exports/result-db-copy.ts b/src/compliance/privacy-safe-export.ts similarity index 71% rename from src/exports/result-db-copy.ts rename to src/compliance/privacy-safe-export.ts index 9efcb43..524ea84 100644 --- a/src/exports/result-db-copy.ts +++ b/src/compliance/privacy-safe-export.ts @@ -2,6 +2,9 @@ import type { UnifiedResultProjection } from '../domain/unified-result'; import { POLICY_CLASSIFICATION_LABELS } from '../domain/policy-status-labels'; import { discloseResultDbProjection } from '../domain/result-db-disclosure'; import { privacySafeFinding, privacySafeText } from '../domain/privacy-safe-finding'; +import { canonicalJson } from '../shared/canonical-json'; +import type { RegulatoryFindingQuery } from './regulatory-rag'; +import { buildRegulatoryFindingQuery } from './regulatory-query'; const SAFE_ENVIRONMENT_PROPERTIES = new Set([ 'domain.controller_count', @@ -18,6 +21,46 @@ export function privacySafeResultJsonFileName(generatedAt: string): string { return `assessor-result-db-privacy-safe-${timestamp}.json`; } +function stableResultKey(item: UnifiedResultProjection): string { + return [item.sourceKind, item.sourceTool ?? '', item.sourceRuleId ?? '', item.resultProjectionId].join('\u0000'); +} + +/** The exact JSON the AI overview sends to the model, in its deterministic order. */ +export function aiOverviewInputJson(items: UnifiedResultProjection[]): string { + return resultsToPrivacySafeExportJson(items.slice().sort((left, right) => stableResultKey(left).localeCompare(stableResultKey(right)))); +} + +export interface ComplianceAiExport { + schemaVersion: 2; + /** Exactly what the AI overview receives for these rows (it splits the same object into chunks when it is large). */ + aiOverviewInput: unknown; + /** Each distinct finding query the ISO mapping model receives, with the Result DB rows that produce it. */ + isoMappingQueries: Array<{ query: RegulatoryFindingQuery; resultProjectionIds: string[] }>; +} + +/** + * The Result DB `Export JSON` of the Compliance Engine: a 1:1 copy of what its AI features receive, + * built by the same functions they use. The ISO model additionally receives the retrieved candidate + * controls, which depend on the index and are not part of this offline export. + */ +export function complianceAiExportJson(items: UnifiedResultProjection[]): string { + const queries = new Map(); + for (const item of items.slice().sort((left, right) => left.resultProjectionId.localeCompare(right.resultProjectionId))) { + const query = buildRegulatoryFindingQuery(item); + if (!query) continue; + const key = canonicalJson(query); + const entry = queries.get(key) ?? { query, resultProjectionIds: [] }; + entry.resultProjectionIds.push(item.resultProjectionId); + queries.set(key, entry); + } + const value: ComplianceAiExport = { + schemaVersion: 2, + aiOverviewInput: JSON.parse(aiOverviewInputJson(items)) as unknown, + isoMappingQueries: [...queries.values()] + }; + return JSON.stringify(value, null, 2); +} + export function resultsToPrivacySafeExportJson(items: UnifiedResultProjection[]): string { // A raw GPO observation is context for its rolled-up comparison, not a finding of its own. const findings = items.filter(item => item.sourceKind !== 'environment-observation' && item.sourceKind !== 'policy-observation'); diff --git a/src/compliance/regulatory-mapping.ts b/src/compliance/regulatory-mapping.ts index bc5788b..4819085 100644 --- a/src/compliance/regulatory-mapping.ts +++ b/src/compliance/regulatory-mapping.ts @@ -1,4 +1,4 @@ -import type { AiOverviewTokenUsage } from '../domain/ai-overview'; +import type { AiOverviewTokenUsage } from './ai-overview'; import type { RegulatoryFindingQuery, RegulatorySectionKind } from './regulatory-rag'; import type { ResultDbFilters } from '../domain/result-db-query'; import type { UnifiedResultProjection } from '../domain/unified-result'; @@ -6,6 +6,8 @@ import type { UnifiedResultProjection } from '../domain/unified-result'; export const REGULATORY_MAPPING_PROMPT_VERSION = '1.2.0'; export const REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION = 1 as const; export const REGULATORY_RETRIEVAL_VERSION = 'dense-control-grouping-1.1.0'; +/** Version of the GPO comparison context in the query; only GPO rules are regenerated when it changes. */ +export const REGULATORY_GPO_QUERY_VERSION = 'gpo-baseline-context-1.1.0'; export type RegulatoryMappingRunStatus = 'queued' | 'running' | 'succeeded' | 'failed'; export type RegulatoryMappingRelationship = 'primary' | 'supporting'; @@ -207,6 +209,17 @@ export interface RegulatoryMappingResponse { pollAfterMs?: number; run?: RegulatoryMappingRun; decisions: RegulatoryMappingDecision[]; + /** + * Controls decided for this rule that the current run did not retrieve (for example after a query + * version change), so an earlier accept stays reviewable. Titles come from older runs of the rule. + */ + earlierControls?: RegulatoryEarlierControl[]; +} + +export interface RegulatoryEarlierControl { + controlId: string; + annexAControlId: string; + title?: string; } export interface RegulatoryMappingCreateRequest { diff --git a/src/compliance/regulatory-query.ts b/src/compliance/regulatory-query.ts index 6f77dae..99f73ba 100644 --- a/src/compliance/regulatory-query.ts +++ b/src/compliance/regulatory-query.ts @@ -1,14 +1,16 @@ import type { RegulatoryFindingQuery } from './regulatory-rag'; import { REGULATORY_FINDING_QUERY_SCHEMA_VERSION } from './regulatory-rag'; import type { UnifiedResultProjection } from '../domain/unified-result'; -import { privacySafeFinding } from '../domain/privacy-safe-finding'; +import { privacySafeFinding, privacySafeText } from '../domain/privacy-safe-finding'; import { discloseResultDbProjection } from '../domain/result-db-disclosure'; +import { annotateWindowsCodes } from '../shared/windows-codes'; import catalog from '../profiles/public-search/catalog.json'; import { publicSecurityTopics } from '../profiles/public-search/public-security-topics'; const MAXIMUM_TITLE_CHARACTERS = 500; const MAXIMUM_DETAIL_CHARACTERS = 1_200; const MAXIMUM_RECOMMENDATION_CHARACTERS = 600; +const MAXIMUM_TECHNICAL_CHARACTERS = 1_200; const publicTools: Record = { 'cis-cat': 'CIS-CAT / CIS Benchmarks', @@ -35,21 +37,27 @@ const publicRuleIds: Record = { export function buildRegulatoryFindingQuery(finding: UnifiedResultProjection): RegulatoryFindingQuery | undefined { if (!isRegulatoryMappableResult(finding)) return undefined; // A GPO comparison crosses only through the Result DB disclosure policy, which withholds the customer value. - const safe = privacySafeFinding(finding.sourceKind === 'policy-evaluation' ? discloseResultDbProjection(finding) : finding); + const disclosed = finding.sourceKind === 'policy-evaluation' ? discloseResultDbProjection(finding) : finding; + const safe = privacySafeFinding(disclosed); const title = safe.title.trim().slice(0, MAXIMUM_TITLE_CHARACTERS); if (!title) return undefined; const toolId = finding.sourceTool?.trim().toLowerCase() ?? ''; const publicTool = publicTools[toolId]; const publicRuleId = recognizedRuleId(toolId, finding.sourceRuleId); const detail = safe.description?.slice(0, MAXIMUM_DETAIL_CHARACTERS); + // A scanner's technical explanation names the mechanism behind the check and sharpens retrieval. + const technical = safe.technicalExplanation?.slice(0, MAXIMUM_TECHNICAL_CHARACTERS); const recommendation = safe.recommendation?.slice(0, MAXIMUM_RECOMMENDATION_CHARACTERS); - const topics = publicSecurityTopics([title, detail, recommendation].filter(Boolean).join('\n')); + const policy = finding.sourceKind === 'policy-evaluation' ? policyContext(disclosed, title) : []; + const topics = publicSecurityTopics([title, finding.sourceKind === 'policy-evaluation' && readableName(title), detail, technical, recommendation].filter(Boolean).join('\n')); const query = [ 'Retrieve regulatory controls whose normative requirements directly address this security finding.', publicTool && `Public assessment tool: ${publicTool}.`, publicRuleId && `Public rule ID: ${publicRuleId}.`, `Security check: ${title}.`, + ...policy, detail && `Check detail: ${detail}.`, + technical && `Technical explanation: ${technical}.`, recommendation && `Recommended action: ${recommendation}.`, topics.length && `Public technical topics: ${topics.join('; ')}.`, 'Prefer controls with direct requirement text over merely related guidance. Do not infer customer scope, assets, configuration values, or compliance status.' @@ -64,6 +72,88 @@ export function buildRegulatoryFindingQuery(finding: UnifiedResultProjection): R }; } +const MAXIMUM_POLICY_VALUE_CHARACTERS = 300; +const MAXIMUM_POLICY_SET_ENTRIES = 12; + +const deliveryMechanisms: Record = { + 'registry-policy': 'administrative template (Registry.pol)', + 'security-template': 'security template (GptTmpl.inf)', + 'advanced-audit-policy': 'advanced audit policy (audit.csv)', + 'group-policy-preference': 'Group Policy preference' +}; + +/** `MinimumPasswordLength` -> `Minimum Password Length`, so the setting reads as words and topics match. */ +function readableName(title: string): string | undefined { + const separator = title.lastIndexOf(' · '); + const name = (separator < 0 ? title : title.slice(separator + 3)).trim(); + const words = name.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2'); + return words !== name ? words : undefined; +} + +/** + * `*S-1-5-32-544` -> `*S-1-5-32-544 (Administrators)`: the documented public meaning of a standard + * Windows code, from the same list the Finding Reader shows; unknown codes stay plain. + */ +function withCodeMeanings(text: string): string { + return annotateWindowsCodes(text).map(segment => segment.meaning ? `${segment.text} (${segment.meaning})` : segment.text).join(''); +} + +/** A disclosed baseline value as text; customer values never reach this function. */ +function baselineValueText(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined; + const item = value as Record; + let text: string | undefined; + if ((item.kind === 'integer' || item.kind === 'text') && typeof item.value === 'string') text = item.value; + else if (item.kind === 'string-set' && Array.isArray(item.values)) { + const values = item.values.filter((entry): entry is string => typeof entry === 'string'); + text = values.length ? values.slice(0, MAXIMUM_POLICY_SET_ENTRIES).join(', ') + + (values.length > MAXIMUM_POLICY_SET_ENTRIES ? ', …' : '') : 'no entries'; + } else if (item.kind === 'registry-state' && typeof item.state === 'string') { + text = { absent: 'value absent', 'key-empty': 'key present without values', 'key-present': 'key present' }[item.state]; + } else if (item.kind === 'audit-flags') { + text = [item.success === true && 'Success', item.failure === true && 'Failure'].filter(Boolean).join(' and ') || 'No auditing'; + } else if (item.kind === 'none') text = 'not configured'; + return text ? withCodeMeanings(privacySafeText(text)).slice(0, MAXIMUM_POLICY_VALUE_CHARACTERS) : undefined; +} + +/** + * Public context of a GPO baseline comparison, read only from fields the Result DB discloses: the + * delivery mechanism and section, the baseline family and recommended value, and what the comparison + * proves. The customer's configured value is never read. A direction (weaker/stronger) is stated only + * where every baseline variant's comparator recorded it; an exact comparison proves only a difference. + */ +function policyContext(item: UnifiedResultProjection, title: string): string[] { + const separator = title.lastIndexOf(' · '); + const section = separator > 0 ? title.slice(0, separator).trim() : undefined; + const mechanism = item.policyDeliveryMechanism ? deliveryMechanisms[item.policyDeliveryMechanism] : undefined; + const scope = item.policyScope === 'computer' ? 'Computer Configuration' : item.policyScope === 'user' ? 'User Configuration' : undefined; + // `SeBackupPrivilege` adds its documented name `Back up files and directories`. + const settingName = (separator < 0 ? title : title.slice(separator + 3)).trim(); + const meaning = annotateWindowsCodes(settingName).find(segment => segment.meaning)?.meaning; + const name = [readableName(title) ?? (meaning && settingName), meaning && `(${meaning})`].filter(Boolean).join(' ') || undefined; + const family = item.policyBaselineLabel?.trim(); + const expected = baselineValueText(item.policyBaselineValue); + const relations = new Set((item.policyBaselineVariantAssessment?.variants ?? []).map(variant => variant.comparator.relation)); + const relation = relations.size === 1 ? [...relations][0] : item.policyComparisonRelation; + const state = item.policyCustomerState === 'missing' + ? 'The recommended setting is not configured in the analysed Group Policy backups.' + : item.policyCustomerState === 'conflicted' + ? 'Group Policy backups configure conflicting values for this setting.' + : item.policyCustomerState === 'configured' && item.isDeviation + ? relation === 'weaker' ? 'The configured value is weaker than the baseline recommendation.' + : relation === 'stronger' ? 'The configured value is stronger than the baseline recommendation but does not match it.' + : 'The configured value differs from the baseline recommendation.' + : undefined; + return [ + (mechanism || scope) && `Group Policy setting type: ${[scope, mechanism].filter(Boolean).join(', ')}.`, + section && `${item.policyDeliveryMechanism === 'registry-policy' ? 'Registry key' : 'Policy section'}: ${privacySafeText(section)}.`, + name && `Setting name: ${name}.`, + family && `Baseline: ${privacySafeText(family)}.`, + expected && `Baseline recommended value: ${expected}.`, + state + ].filter((value): value is string => Boolean(value)); +} + function recognizedRuleId(toolId: string, value: string | undefined): string | undefined { const candidate = value?.trim(); if (!candidate) return undefined; diff --git a/src/features/dashboard/AiOverviewPanel.tsx b/src/compliance/ui/AiOverviewPanel.tsx similarity index 94% rename from src/features/dashboard/AiOverviewPanel.tsx rename to src/compliance/ui/AiOverviewPanel.tsx index 51c49a3..f7bab7c 100644 --- a/src/features/dashboard/AiOverviewPanel.tsx +++ b/src/compliance/ui/AiOverviewPanel.tsx @@ -1,11 +1,15 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { AssessorApiClient } from '../../api/assessor-api-client'; -import type { AiOverviewResponse, AiOverviewResult } from '../../domain/ai-overview'; +import { useEnginePreferences } from '../../app/engine-preferences'; +import type { AiOverviewResponse, AiOverviewResult } from '../ai-overview'; +import './ai-overview.css'; import type { ReactNode } from 'react'; const assessorApi = new AssessorApiClient(); export interface AiOverviewController { + /** False while the Compliance Engine is switched off: no request, no tab. */ + available: boolean; response?: AiOverviewResponse; loading: boolean; generating: boolean; @@ -13,7 +17,9 @@ export interface AiOverviewController { generate(): Promise; } -export function useAiOverview(projectId: string | undefined, assessmentIds: string[] = [], scanRunIds: string[] = []): AiOverviewController { +export function useAiOverview(requestedProjectId: string | undefined, assessmentIds: string[] = [], scanRunIds: string[] = []): AiOverviewController { + const available = useEnginePreferences().compliance; + const projectId = available ? requestedProjectId : undefined; const [storedResponse, setStoredResponse] = useState<{ scope: { key: string }; value: AiOverviewResponse }>(); const [loading, setLoading] = useState(false); const [generating, setGenerating] = useState(false); @@ -83,7 +89,7 @@ export function useAiOverview(projectId: string | undefined, assessmentIds: stri } }, [normalizedAssessmentIds, normalizedScanRunIds, projectId, requestScope]); - return { response, loading, generating, error, generate }; + return { available, response, loading, generating, error, generate }; } export function AiOverviewPanel({ controller, navigation }: { controller: AiOverviewController; navigation?: ReactNode }) { diff --git a/src/compliance/ui/ClientIsoControls.tsx b/src/compliance/ui/ClientIsoControls.tsx index ac0db37..6f094ca 100644 --- a/src/compliance/ui/ClientIsoControls.tsx +++ b/src/compliance/ui/ClientIsoControls.tsx @@ -2,7 +2,7 @@ import { useEffect, useId, useState } from 'react'; import { AssessorApiClient } from '../../api/assessor-api-client'; import type { BackendStatus } from '../../app/app-state'; import { useEnginePreferences } from '../../app/engine-preferences'; -import type { RegulatoryControlFinding, RegulatoryControlOverviewResponse } from '../regulatory-mapping'; +import type { RegulatoryControlOverviewResponse } from '../regulatory-mapping'; import { useRegulatoryDecisionRevision } from './decision-revision'; import './compliance.css'; @@ -13,13 +13,14 @@ interface Props { assessmentIds?: readonly string[]; scanRunIds?: readonly string[]; backendStatus: BackendStatus; - onOpenFindings(findings: RegulatoryControlFinding[]): void; + /** Opens Work filtered to this accepted control, like a Context heatmap cell. */ + onOpenControl(controlId: string): void; } type OverviewState = { status: 'idle' | 'loading' } | { status: 'error' } | { status: 'ready'; overview: RegulatoryControlOverviewResponse }; /** Client Info `ISO controls`: accepted controls of the timeslot's deviations, laid out like the Context heatmap. */ -export function ClientIsoControls({ projectId, assessmentIds = [], scanRunIds = [], backendStatus, onOpenFindings }: Props) { +export function ClientIsoControls({ projectId, assessmentIds = [], scanRunIds = [], backendStatus, onOpenControl }: Props) { const available = useEnginePreferences().compliance; const revision = useRegulatoryDecisionRevision(); const id = useId(); @@ -50,7 +51,7 @@ export function ClientIsoControls({ projectId, assessmentIds = [], scanRunIds = {state.overview.controls.map(control => - + + ; } diff --git a/src/styles/ai-overview.css b/src/compliance/ui/ai-overview.css similarity index 100% rename from src/styles/ai-overview.css rename to src/compliance/ui/ai-overview.css diff --git a/src/compliance/ui/compliance-work-facets.ts b/src/compliance/ui/compliance-work-facets.ts new file mode 100644 index 0000000..e99ca48 --- /dev/null +++ b/src/compliance/ui/compliance-work-facets.ts @@ -0,0 +1,126 @@ +import { useEffect, useState } from 'react'; +import { AssessorApiClient } from '../../api/assessor-api-client'; +import { useApp } from '../../app/AppProvider'; +import { useEnginePreferences } from '../../app/engine-preferences'; +import type { FindingRecord } from '../../domain/finding'; +import { gpoWorkSettingIdentity } from '../../features/results-browser/work-record-lookup'; +import type { DynamicFacet } from '../../features/search-filter/dynamic-facets'; +import type { RegulatoryControlOverviewResponse, RegulatoryControlTheme } from '../regulatory-mapping'; +import { useRegulatoryDecisionRevision } from './decision-revision'; + +/** Work facets of the Compliance Engine; present only while the engine is on and its assignments are loaded. */ +export const ISO_CONTROL_FACET_ID = 'compliance:iso-control'; +export const ISO_THEME_FACET_ID = 'compliance:iso-theme'; +export const COMPLIANCE_FACET_IDS = [ISO_CONTROL_FACET_ID, ISO_THEME_FACET_ID] as const; +const SOURCE_SCAN_FACET_ID = 'provenance:source-scan'; +const THEMES: RegulatoryControlTheme[] = ['Organizational', 'People', 'Physical', 'Technological']; + +export interface ComplianceWorkFacetIndex { + facets: [DynamicFacet, DynamicFacet]; +} + +const indexes = new WeakMap(); + +/** One shared index per loaded response, so every Work consumer uses identical cache keys. */ +export function complianceWorkFacetIndex(response: RegulatoryControlOverviewResponse): ComplianceWorkFacetIndex { + const existing = indexes.get(response); + if (existing) return existing; + const byOccurrence = new Map>(); + const bySetting = new Map>(); + const add = (map: Map>, key: string, controlId: string) => map.set(key, (map.get(key) ?? new Set()).add(controlId)); + for (const control of response.controls) { + for (const finding of control.findings) { + if (finding.sourceKind === 'scanner-finding') add(byOccurrence, finding.sourceResultId, control.controlId); + else if (finding.sourceRuleId) add(bySetting, `${finding.policyScope ?? ''}\u0000${finding.sourceRuleId}`, control.controlId); + } + } + // Scanner findings match by occurrence; GPO settings by scope and technical setting identity, like Result DB drill-downs. + const controlsFor = (record: FindingRecord): string[] => { + if (record.findingOccurrenceId) return [...byOccurrence.get(record.findingOccurrenceId) ?? []]; + const identity = gpoWorkSettingIdentity(record.id); + return identity ? [...bySetting.get(`${identity.scope ?? ''}\u0000${identity.settingId}`) ?? []] : []; + }; + const themeOf = new Map(response.controls.map(control => [control.controlId, control.theme])); + const controls = [...response.controls].sort((left, right) => left.controlId.localeCompare(right.controlId, 'en', { numeric: true })); + const index: ComplianceWorkFacetIndex = { + facets: [{ + id: ISO_CONTROL_FACET_ID, + label: 'ISO control', + source: 'derived', + kind: 'category', + description: 'Compliance Engine: findings and GPO settings with this accepted ISO/IEC 27002 control. It does not change results.', + values: controls.map(control => ({ value: control.controlId, label: `${control.annexAControlId}${control.title ? ` · ${control.title}` : ''}`, count: 0 })), + valuesFor: controlsFor + }, { + id: ISO_THEME_FACET_ID, + label: 'ISO theme', + source: 'derived', + kind: 'category', + description: 'Compliance Engine: findings and GPO settings with an accepted ISO/IEC 27002 control of this theme. It does not change results.', + // Facet values are matched in lower case; the label keeps the ISO spelling. + values: THEMES.filter(theme => response.controls.some(control => control.theme === theme)) + .map(theme => ({ value: theme.toLowerCase(), label: theme, count: 0 })), + valuesFor: record => [...new Set(controlsFor(record).flatMap(controlId => themeOf.get(controlId)?.toLowerCase() ?? []))] + }] + }; + indexes.set(response, index); + return index; +} + +/** Adds the ISO facets directly before the permanent final `Source: Scan` control. */ +export function withComplianceFacets(facets: DynamicFacet[], index: ComplianceWorkFacetIndex | undefined): DynamicFacet[] { + if (!index) return facets; + const sourceScan = facets.findIndex(facet => facet.id === SOURCE_SCAN_FACET_ID); + return sourceScan < 0 ? [...facets, ...index.facets] + : [...facets.slice(0, sourceScan), ...index.facets, ...facets.slice(sourceScan)]; +} + +const assessorApi = new AssessorApiClient(); +// One request per timeslot and decision revision, shared by every Work consumer. +const requests = new Map>(); +const ready = new Map(); +const lastReadyByScope = new Map(); + +/** The ISO facet index for Work, or undefined while the engine is off or nothing is loaded yet. */ +export function useComplianceWorkFacets(): ComplianceWorkFacetIndex | undefined { + const app = useApp(); + const available = useEnginePreferences().compliance; + const revision = useRegulatoryDecisionRevision(); + const assessmentIds = [...(app.activeAssessmentIds ?? [])].sort(); + const scanRunIds = [...(app.activeScanRunIds ?? [])].sort(); + const projectId = app.targetProjectId; + const scope = JSON.stringify([projectId, assessmentIds, scanRunIds]); + const key = `${scope}\u0000${revision}`; + const enabled = available && app.backendStatus === 'connected' && Boolean(projectId) && (assessmentIds.length > 0 || scanRunIds.length > 0); + const [, setLoaded] = useState(0); + + useEffect(() => { + if (!enabled || !projectId || ready.has(key)) return; + let active = true; + let request = requests.get(key); + if (!request) { + request = assessorApi.getRegulatoryControlAssignments(projectId, { assessmentIds, scanRunIds }); + requests.set(key, request); + request.then(response => { + ready.set(key, response); + lastReadyByScope.set(scope, response); + while (ready.size > 8) ready.delete(ready.keys().next().value!); + while (lastReadyByScope.size > 4) lastReadyByScope.delete(lastReadyByScope.keys().next().value!); + }, () => undefined).finally(() => requests.delete(key)); + } + request.then(() => { if (active) setLoaded(value => value + 1); }, () => undefined); + return () => { active = false; }; + }, [enabled, key]); // eslint-disable-line react-hooks/exhaustive-deps + + if (!enabled) return undefined; + // While a new decision revision of the same timeslot loads, the last loaded state keeps an active filter valid. + const response = ready.get(key) ?? lastReadyByScope.get(scope); + return response ? complianceWorkFacetIndex(response) : undefined; +} + +/** Test support. */ +export function resetComplianceWorkFacetsForTests(): void { + requests.clear(); + ready.clear(); + lastReadyByScope.clear(); +} diff --git a/src/compliance/ui/compliance.css b/src/compliance/ui/compliance.css index 92cdc67..daa8670 100644 --- a/src/compliance/ui/compliance.css +++ b/src/compliance/ui/compliance.css @@ -32,3 +32,6 @@ .iso-controls__cell strong { font-size: var(--type-body-size); font-weight: var(--fw-semibold); line-height: var(--lh-ui); } .iso-controls__category { font-size: var(--type-table-body-size); font-weight: var(--fw-regular); } .iso-controls__amount { margin-top: auto; font-size: var(--type-body-size); font-weight: var(--fw-semibold); } +.regulatory-earlier-controls { margin-top: 16px; } +.regulatory-earlier-controls > h4 { margin-bottom: 4px; } +.regulatory-earlier-controls > p { color: var(--text-muted); } diff --git a/src/compliance/ui/use-regulatory-mapping.ts b/src/compliance/ui/use-regulatory-mapping.ts index 46ac6f2..f546d9e 100644 --- a/src/compliance/ui/use-regulatory-mapping.ts +++ b/src/compliance/ui/use-regulatory-mapping.ts @@ -90,7 +90,7 @@ export function useRegulatoryMapping( } }, [projectId, requestScope, resultProjectionId]); - const decide = useCallback(async (candidate: RegulatoryMappingCandidate, decision: RegulatoryMappingDecisionValue, comment?: string) => { + const decide = useCallback(async (candidate: Pick, decision: RegulatoryMappingDecisionValue, comment?: string) => { if (!projectId || !resultProjectionId || !response?.run || activeScope.current !== requestScope) return; const latest = response.decisions.find(item => item.controlId === candidate.controlId); setLoading(true); @@ -116,7 +116,9 @@ export function useRegulatoryMapping( const acceptedControls = useMemo(() => available && response ? acceptedControlIds(response.decisions).map(controlId => { - const candidate = response.run?.candidates.find(item => item.controlId === controlId); + // A control decided in an earlier run keeps its title after the current run stopped retrieving it. + const candidate = response.run?.candidates.find(item => item.controlId === controlId) + ?? response.earlierControls?.find(item => item.controlId === controlId); return { controlId, annexAControlId: candidate?.annexAControlId ?? `A.${controlId}`, ...(candidate?.title ? { title: candidate.title } : {}) }; }).sort((left, right) => left.controlId.localeCompare(right.controlId, 'en', { numeric: true })) : [], [available, response]); diff --git a/src/domain/gpo-metadata.ts b/src/domain/gpo-metadata.ts index 072c00f..a6fd13d 100644 --- a/src/domain/gpo-metadata.ts +++ b/src/domain/gpo-metadata.ts @@ -56,9 +56,22 @@ export function aggregateGpoBackupMetadata(documents: GpoBackupMetadataDocument[ } return values.length === 1 ? values[0] : undefined; }; + // A Backup-GPO folder names its domain by GUID in bkupInfo.xml and by DNS name in + // Backup.xml/GPReport.xml. Different kinds identify the same domain; only two values + // of one kind conflict. The GUID is preferred because it survives a domain rename. + const domainKeyOf = (kind: 'domain-guid' | 'dns'): string | undefined => { + const values = unique(documents.map(document => document.domainKey?.startsWith(`${kind}:`) ? document.domainKey : undefined)); + if (values.length > 1) { + diagnostics.add('gpo-metadata-domainKey-conflict'); + invalid = true; + } + return values.length === 1 ? values[0] : undefined; + }; const gpoId = field('gpoId'); const backupId = field('backupId'); - const domainKey = field('domainKey'); + const domainGuidKey = domainKeyOf('domain-guid'); + const domainDnsKey = domainKeyOf('dns'); + const domainKey = diagnostics.has('gpo-metadata-domainKey-conflict') ? undefined : domainGuidKey ?? domainDnsKey; const backupCreatedAt = field('backupCreatedAt'); const complete = Boolean(gpoId && backupId && domainKey && backupCreatedAt); // Several reports with link facts cannot be ordered; no link status is claimed then. diff --git a/src/domain/privacy-safe-finding.ts b/src/domain/privacy-safe-finding.ts index 8636375..9eed937 100644 --- a/src/domain/privacy-safe-finding.ts +++ b/src/domain/privacy-safe-finding.ts @@ -3,7 +3,7 @@ import type { UnifiedResultProjection } from './unified-result'; /** * The single privacy boundary shared by the AI overview and the ISO control mapping. * It projects a canonical finding to its rule-level check fields only — title, - * description, recommendation and normalized metadata — and never includes the + * description, technical explanation, recommendation and normalized metadata — and never includes the * result-table / occurrence data (affectedAsset, source locators, per-object values). * Definitions are deduplicated per rule (canonicalKey = tool:ruleId), so this text * describes the check, not the tenant. Email addresses are redacted defensively. @@ -12,6 +12,7 @@ export interface PrivacySafeFinding { id: string | null; title: string; description?: string; + technicalExplanation?: string; recommendation?: string; severity: string; outcome: string; @@ -32,11 +33,13 @@ export function privacySafeCheckText(value: string | undefined): string | undefi export function privacySafeFinding(item: UnifiedResultProjection): PrivacySafeFinding { const description = privacySafeCheckText(item.description); + const technicalExplanation = privacySafeCheckText(item.technicalExplanation); const recommendation = privacySafeCheckText(item.recommendation); return { id: item.sourceRuleId ?? null, title: privacySafeText(item.title), ...(description ? { description } : {}), + ...(technicalExplanation ? { technicalExplanation } : {}), ...(recommendation ? { recommendation } : {}), severity: item.severityNormalized, outcome: item.outcomeNormalized, diff --git a/src/domain/result-db-disclosure.ts b/src/domain/result-db-disclosure.ts index fd68964..5673465 100644 --- a/src/domain/result-db-disclosure.ts +++ b/src/domain/result-db-disclosure.ts @@ -6,7 +6,7 @@ import { wellKnownSidName } from '../shared/windows-codes'; import { applyResultDbIdentityGuard, RESULT_DB_IDENTITY_GUARD_VERSION, type ResultDbIdentityGuard } from './result-db-identity-guard'; import { addedPlaceholders, type ResultDbDisclosureTrace } from './result-db-trace'; -export const RESULT_DB_DISCLOSURE_POLICY_VERSION = '3.5.0'; +export const RESULT_DB_DISCLOSURE_POLICY_VERSION = '3.6.0'; const SAFE_TIMESTAMP = '1970-01-01T00:00:00.000Z'; @@ -668,7 +668,8 @@ const RESULT_DB_FIELD_DISCLOSURE: Record; @@ -799,6 +800,8 @@ function disclosedValues(item: UnifiedResultProjection): UnifiedResultProjection recommendation: policyRedactor ? safePolicyNarrative(item, policyRedactor, item.recommendation) : redactNarrative(item, item.recommendation, 'recommendation'), + // Public rule guidance like the description, so it is redacted as strictly as a description. + technicalExplanation: policyEvaluation ? undefined : redactNarrative(item, item.technicalExplanation, 'description'), affectedAsset: undefined, severityOriginal: item.severityNormalized, outcomeOriginal: item.outcomeNormalized, diff --git a/src/domain/result-db-identity-guard.ts b/src/domain/result-db-identity-guard.ts index 45bf254..b9d9513 100644 --- a/src/domain/result-db-identity-guard.ts +++ b/src/domain/result-db-identity-guard.ts @@ -205,7 +205,7 @@ export function buildResultDbIdentityGuards(input: { /** Content fields of a disclosed row; identifiers, enums and version fields are never rewritten. */ const guardedFields = [ - 'title', 'description', 'recommendation', 'sourceRuleId', 'policyBaselineLabel', 'policyBaselineValue', + 'title', 'description', 'recommendation', 'technicalExplanation', 'sourceRuleId', 'policyBaselineLabel', 'policyBaselineValue', 'policyBaselineVariantAssessment', 'environmentNormalizedValue', 'categories' ] as const satisfies ReadonlyArray; diff --git a/src/domain/unified-result.ts b/src/domain/unified-result.ts index b8a9a8c..e14376f 100644 --- a/src/domain/unified-result.ts +++ b/src/domain/unified-result.ts @@ -146,6 +146,8 @@ export interface UnifiedResultProjection { title: string; description?: string; recommendation?: string; + /** The rule's technical explanation, where the scanner report provides one (for example PingCastle). */ + technicalExplanation?: string; affectedAsset?: string; score?: number; severityOriginal: JsonValue; @@ -273,6 +275,16 @@ function scannerSourceToolScope(source: ScannerUnifiedResultSource): UnifiedResu return undefined; } +/** + * A scanner's technical explanation of a rule, read from the retained original fields: the extractor key + * `technical-explanation` or the source label "Technical explanation". Tools without one leave it empty. + */ +export function technicalExplanation(fields: ReadonlyArray<{ key: string; label: string; value: unknown }>): string | undefined { + const field = fields.find(item => item.key === 'technical-explanation') + ?? fields.find(item => item.label.trim().toLocaleLowerCase('en-US') === 'technical explanation'); + return typeof field?.value === 'string' ? field.value.trim() || undefined : undefined; +} + export function resultSourceToolLabel(value?: string): string { const labels: Record = { 'purple-knight-ad': 'Purple Knight for Active Directory', @@ -442,6 +454,7 @@ export function createScannerUnifiedResult(source: ScannerUnifiedResultSource): title: source.definition.title, description: source.definition.description, recommendation: source.definition.recommendation, + technicalExplanation: technicalExplanation(source.occurrence.originalFields), affectedAsset: source.occurrence.affectedAsset?.normalizedValue ?? undefined, score: source.occurrence.score?.normalizedValue ?? undefined, severityOriginal: source.occurrence.severity.originalValue, diff --git a/src/exports/unified-result-csv.ts b/src/exports/unified-result-csv.ts index 1c6ae0b..627fe54 100644 --- a/src/exports/unified-result-csv.ts +++ b/src/exports/unified-result-csv.ts @@ -11,7 +11,7 @@ export function buildUnifiedResultCsv(query: UnifiedResultQuery, results: Unifie const [manifest, ...entries] = buildUnifiedResultJsonl(query, results, generatedAt).trimEnd().split('\n') .map(line => JSON.parse(line) as Record); const records = entries.map(entry => entry.result as Record); - const preferred = ['title', 'sourceRuleId', 'sourceTool', 'severityNormalized', 'outcomeNormalized', 'resultType', 'isIoe', 'isDeviation', 'description', 'recommendation']; + const preferred = ['title', 'sourceRuleId', 'sourceTool', 'severityNormalized', 'outcomeNormalized', 'resultType', 'isIoe', 'isDeviation', 'description', 'technicalExplanation', 'recommendation']; const fields = [...new Set(records.flatMap(record => Object.keys(record)))]; const rank = (field: string) => preferred.includes(field) ? preferred.indexOf(field) : preferred.length; fields.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b)); diff --git a/src/exports/unified-result-excel.ts b/src/exports/unified-result-excel.ts index bcbd42b..6bb56cd 100644 --- a/src/exports/unified-result-excel.ts +++ b/src/exports/unified-result-excel.ts @@ -15,7 +15,7 @@ export function buildUnifiedResultExcel(query: UnifiedResultQuery, results: Unif const [manifest, ...entries] = buildUnifiedResultJsonl(query, results, generatedAt).trimEnd().split('\n') .map(line => JSON.parse(line) as Record); const records = entries.map(entry => entry.result as Record); - const preferred = ['title', 'sourceRuleId', 'sourceTool', 'severityNormalized', 'outcomeNormalized', 'resultType', 'isIoe', 'isDeviation', 'description', 'recommendation']; + const preferred = ['title', 'sourceRuleId', 'sourceTool', 'severityNormalized', 'outcomeNormalized', 'resultType', 'isIoe', 'isDeviation', 'description', 'technicalExplanation', 'recommendation']; const fields = [...new Set(records.flatMap(record => Object.keys(record)))]; fields.sort((a, b) => { const rank = (field: string) => preferred.includes(field) ? preferred.indexOf(field) : preferred.length; @@ -47,7 +47,7 @@ export function buildUnifiedResultExcel(query: UnifiedResultQuery, results: Unif sheet['!autofilter'] = { ref: sheet['!ref']! }; utils.book_append_sheet(workbook, sheet, name); }; - append('Results', rows, [12, ...fields.map(field => ['title', 'description', 'recommendation'].includes(field) ? 65 : 25)]); + append('Results', rows, [12, ...fields.map(field => ['title', 'description', 'technicalExplanation', 'recommendation'].includes(field) ? 65 : 25)]); append('Export info', [['Property', 'Value'], ...Object.entries(manifest).map(([key, value]) => [key, cell(value, 0, key)]), ['Excel notes', 'Nested values are JSON text. Long values are complete in Long text, ordered by Position, Field and Part. Position 0 refers to Export info.']], [28, 100]); if (overflow.length > 1) append('Long text', overflow, [12, 30, 10, 100]); diff --git a/src/features/actionlist/ActionlistControls.tsx b/src/features/actionlist/ActionlistControls.tsx index 20f832a..dfe3b11 100644 --- a/src/features/actionlist/ActionlistControls.tsx +++ b/src/features/actionlist/ActionlistControls.tsx @@ -8,18 +8,20 @@ import { browserDownloadAdapter } from '../../platform/browser/browser-download- import { useFilteredRecords } from '../search-filter/use-filtered-records'; import { selectWorkFacets } from '../search-filter/work-facets'; import { useContextClusterIndex } from '../context-layer/use-context-cluster-index'; +import { useComplianceWorkFacets } from '../../compliance/ui/compliance-work-facets'; export function ActionlistControls() { const app = useApp(); const records = useFilteredRecords(); const contextClusters = useContextClusterIndex(); + const complianceFacets = useComplianceWorkFacets(); const [failed, setFailed] = useState(false); const exportView = async (format: 'csv' | 'md' | 'html') => { if (!app.dataset || !app.activeSheet || !records.length) return; const activity = beginActivity('Core Engine', `Preparing Work ${format.toUpperCase()} export`); try { await showActivityFrame(); - const facets = selectWorkFacets(app.activeSheet, { ...app, contextClusters }); + const facets = selectWorkFacets(app.activeSheet, { ...app, contextClusters, complianceFacets }); const statusLabel = app.activeProfile.presentation.statusFilters.find(filter => filter.value === app.statusFilter)?.label ?? app.statusFilter; const filters = Object.entries(app.facetFilters).map(([id, filter]) => { const facet = facets.find(item => item.id === id); diff --git a/src/features/dashboard/DashboardPanel.tsx b/src/features/dashboard/DashboardPanel.tsx index ec6c1e2..0edbff6 100644 --- a/src/features/dashboard/DashboardPanel.tsx +++ b/src/features/dashboard/DashboardPanel.tsx @@ -1,7 +1,7 @@ import { readEnginePreferences, useEnginePreferences } from '../../app/engine-preferences'; import { ClientContextHeatmap } from '../context-layer/ContextHeatmap'; import { ClientIsoControls } from '../../compliance/ui/ClientIsoControls'; -import { workRecordIdsForResult } from '../results-browser/work-record-lookup'; +import { ISO_CONTROL_FACET_ID } from '../../compliance/ui/compliance-work-facets'; import { CONTEXT_CLUSTER_FACET_ID } from '../context-layer/context-cluster-facet'; import { useAssessmentContext } from '../context-layer/use-assessment-context'; import { publishContextReport } from '../../assessment-context/context-report-publication'; @@ -40,7 +40,7 @@ import { WorkTimeslotControl } from '../search-filter/WorkTimeslotControl'; import { dashboardResultState, dashboardScoreFromStates, type DashboardResultState } from './dashboard-result-state'; export { dashboardResultState } from './dashboard-result-state'; -import { AiOverviewPanel, useAiOverview } from './AiOverviewPanel'; +import { AiOverviewPanel, useAiOverview } from '../../compliance/ui/AiOverviewPanel'; import { DEFENDER_FOR_IOT_PROFILE_ID } from '../../domain/defender-for-iot'; import { DefenderVulnerabilityPanel } from '../defender-for-iot/DefenderVulnerabilityPanel'; import { defenderModelForDataset } from '../defender-for-iot/defender-dataset-model'; @@ -1420,7 +1420,7 @@ export function DashboardPanel({ onOpenWork }: DashboardPanelProps) { const pureDefender = dataset?.profileId === DEFENDER_FOR_IOT_PROFILE_ID; const environment = useEnvironmentSnapshot(app.targetProjectId, app.activeScanRunIds); const aiOverview = useAiOverview(app.targetProjectId, app.activeAssessmentIds ?? [], app.activeScanRunIds ?? []); - const showAiOverview = Boolean(aiOverview.loading || aiOverview.response?.generationEnabled || aiOverview.response?.run); + const showAiOverview = aiOverview.available && Boolean(aiOverview.loading || aiOverview.response?.generationEnabled || aiOverview.response?.run); useEffect(() => { if (pureDefender && clientInfoView === 'overview') setClientInfoView('defender'); @@ -1678,11 +1678,15 @@ export function DashboardPanel({ onOpenWork }: DashboardPanelProps) { setFacetFilter(CONTEXT_CLUSTER_FACET_ID, { values: [clusterId] }); }, false)} /> - {/* Compliance Engine: accepted ISO controls of this timeslot; a tile opens exactly its findings in Work. */} + {/* Compliance Engine: accepted ISO controls of this timeslot; a tile selects its control in the Work ISO control filter. */} jump(() => { - setStatusFilter('all'); - setPathFindingIds([...new Set(findings.flatMap(finding => workRecordIdsForResult(dataset, finding)))]); + backendStatus={app.backendStatus} onOpenControl={controlId => jump(() => { + const targetSheet = sourceScanNavigationSheet(dataset, activeSheet); + if (targetSheet) { + app.setActiveSheet(targetSheet.id); + setStatusFilter('deviation'); + } + setFacetFilter(ISO_CONTROL_FACET_ID, { values: [controlId] }); }, false)} /> {scoringOpen && setScoringOpen(false)} />} diff --git a/src/features/results-browser/ResultsBrowser.tsx b/src/features/results-browser/ResultsBrowser.tsx index 79aaa6d..1f0ae52 100644 --- a/src/features/results-browser/ResultsBrowser.tsx +++ b/src/features/results-browser/ResultsBrowser.tsx @@ -11,7 +11,7 @@ import { TriangleIcon } from '../../components/TriangleIcon'; import { browserDownloadAdapter, downloadBrowserBlob } from '../../platform/browser/browser-download-adapter'; import { downloadText } from '../../exports/text-download'; import { buildUnifiedResultJsonl, queryAllUnifiedResults, unifiedResultJsonlFileName } from '../../exports/unified-result-jsonl'; -import { privacySafeResultJsonFileName, resultsToPrivacySafeExportJson } from '../../exports/result-db-copy'; +import { complianceAiExportJson, privacySafeResultJsonFileName } from '../../compliance/privacy-safe-export'; import { hasSelectedTimeslotScope } from '../../app/work-scope'; import { POLICY_CLASSIFICATION_LABELS } from '../../domain/policy-status-labels'; import { BackendConnectionDialog } from '../../components/BackendConnectionDialog/BackendConnectionDialog'; @@ -124,7 +124,8 @@ export function ResultsBrowser({ backendStatus, projectId, scanRunIds, assessmen downloadText( browserDownloadAdapter, privacySafeResultJsonFileName(generatedAt), - resultsToPrivacySafeExportJson(results), + // Compliance Engine extension point: exactly what its AI features receive for these rows. + complianceAiExportJson(results), 'application/json;charset=utf-8' ); } else downloadText( @@ -189,9 +190,9 @@ export function ResultsBrowser({ backendStatus, projectId, scanRunIds, assessmen onClick={() => void exportFilteredResults('excel')}>{exporting === 'excel' ? 'Exporting …' : 'Export Excel'} - + {regulatoryBatch.available && } {!queryResult?.items.length ? updateFilters({ ...DEFAULT_RESULT_DB_FILTERS })} onSearchAll={() => updateFilters({})}>No result matches the current filter combination. :
@@ -248,6 +249,7 @@ export function ResultRow({ item: sourceItem, expanded, onToggle, workRecordIds

{displayEnvironmentValue(item.policyValues)}

} {item.description &&

{item.description}

} + {item.technicalExplanation &&

Technical explanation: {item.technicalExplanation}

} {item.recommendation &&

Recommendation: {item.recommendation}

} {policyComparison &&
Expected baseline value @@ -329,7 +331,7 @@ function processingStatusLabel(value: string | undefined): string { } const traceFieldLabels: Record = { - title: 'Title', description: 'Description', recommendation: 'Recommendation', sourceRuleId: 'Source Rule ID', + title: 'Title', description: 'Description', technicalExplanation: 'Technical explanation', recommendation: 'Recommendation', sourceRuleId: 'Source Rule ID', policyBaselineLabel: 'Baseline', policyBaselineValue: 'Expected baseline value', policyBaselineVariantAssessment: 'Baseline expectations', environmentNormalizedValue: 'Value', categories: 'Categories' }; diff --git a/src/features/results-browser/work-record-lookup.ts b/src/features/results-browser/work-record-lookup.ts index 6a586d3..ea3b23e 100644 --- a/src/features/results-browser/work-record-lookup.ts +++ b/src/features/results-browser/work-record-lookup.ts @@ -1,6 +1,18 @@ import type { ImportedDataset } from '../../domain/dataset'; import type { UnifiedResultProjection } from '../../domain/unified-result'; +/** Scope and technical setting identity of a rolled-up GPO Work setting, from its record id. */ +export function gpoWorkSettingIdentity(recordId: string): { scope: unknown; settingId: unknown } | undefined { + if (!recordId.startsWith('gpo-work-setting:')) return undefined; + try { + const key = JSON.parse(decodeURIComponent(recordId.slice('gpo-work-setting:'.length))) as unknown[]; + return { scope: key[2], settingId: key[3] }; + } catch { + // Not an evaluated setting key; unevaluated diagnostics have no comparison original. + return undefined; + } +} + /** * Work records holding the original of a Result DB row (user decision 2026-09-28: the Result DB shows * only the checked content; the original stays in Work). Scanner rows match their finding occurrence; @@ -13,13 +25,9 @@ export function workRecordIdsForResult(dataset: ImportedDataset | null | undefin for (const sheet of dataset.sheets) { for (const record of sheet.analysis.records) { if (row.sourceKind === 'scanner-finding' && record.findingOccurrenceId === row.sourceResultId) ids.add(record.id); - if (row.sourceKind === 'policy-evaluation' && row.sourceRuleId && record.id.startsWith('gpo-work-setting:')) { - try { - const key = JSON.parse(decodeURIComponent(record.id.slice('gpo-work-setting:'.length))) as unknown[]; - if (key[2] === row.policyScope && key[3] === row.sourceRuleId) ids.add(record.id); - } catch { - // Not an evaluated setting key; unevaluated diagnostics have no comparison original. - } + if (row.sourceKind === 'policy-evaluation' && row.sourceRuleId) { + const identity = gpoWorkSettingIdentity(record.id); + if (identity && identity.scope === row.policyScope && identity.settingId === row.sourceRuleId) ids.add(record.id); } } } diff --git a/src/features/search-filter/SearchFilterBar.tsx b/src/features/search-filter/SearchFilterBar.tsx index ce0a5d1..a02c91c 100644 --- a/src/features/search-filter/SearchFilterBar.tsx +++ b/src/features/search-filter/SearchFilterBar.tsx @@ -6,6 +6,7 @@ import { GPO_FAILING_BACKUP_FACET_ID, sourceScanNavigationSheet } from './dynami import { DynamicFilterControls } from './DynamicFilterControls'; import { selectWorkFacets } from './work-facets'; import { useContextClusterIndex } from '../context-layer/use-context-cluster-index'; +import { COMPLIANCE_FACET_IDS, useComplianceWorkFacets } from '../../compliance/ui/compliance-work-facets'; import { CONTEXT_CLUSTER_FACET_ID } from '../context-layer/context-cluster-facet'; import { measureNavigationStep } from '../../app/navigation-diagnostics'; export { WorkTimeslotControl } from './WorkTimeslotControl'; @@ -92,11 +93,12 @@ export function SearchFilterBar({ hiddenFacetLabels = [] }: SearchFilterBarProps const engines = useEnginePreferences(); // The facet uses the shared context for the current scope. const contextClusters = useContextClusterIndex(); + const complianceFacets = useComplianceWorkFacets(); const countedFacets = useMemo( () => measureNavigationStep('filter-counts', () => selectWorkFacets(activeSheet, { - activeProfile, search, statusFilter, facetFilters, dataset, actionOnly, reviews, pathFindingIds, contextClusters + activeProfile, search, statusFilter, facetFilters, dataset, actionOnly, reviews, pathFindingIds, contextClusters, complianceFacets })), - [actionOnly, activeProfile, activeSheet, contextClusters, dataset, facetFilters, pathFindingIds, reviews, search, statusFilter] + [actionOnly, activeProfile, activeSheet, complianceFacets, contextClusters, dataset, facetFilters, pathFindingIds, reviews, search, statusFilter] ); const visibleFacets = useMemo( () => countedFacets.filter(facet => !hiddenFacetLabels.includes(facet.label)), @@ -123,6 +125,8 @@ export function SearchFilterBar({ hiddenFacetLabels = [] }: SearchFilterBarProps Object.keys(facetFilters).forEach(facetId => { // Keep the selected context filter while its context is loading. if (facetId === CONTEXT_CLUSTER_FACET_ID && engines.context) return; + // Likewise keep a selected ISO filter while the Compliance Engine assignments load. + if ((COMPLIANCE_FACET_IDS as readonly string[]).includes(facetId) && engines.compliance && !complianceFacets) return; if (!contextualFacetIds.has(facetId)) setFacetFilter(facetId, null); }); countedFacets.forEach(facet => { @@ -137,7 +141,7 @@ export function SearchFilterBar({ hiddenFacetLabels = [] }: SearchFilterBarProps if (statusFilter !== 'all' && statusFacet && !statusFacet.values.some(value => value.value === statusFilter)) { setStatusFilter('all'); } - }, [engines.context, countedFacets, facetFilters, setFacetFilter, setStatusFilter, statusFilter]); + }, [complianceFacets, engines.compliance, engines.context, countedFacets, facetFilters, setFacetFilter, setStatusFilter, statusFilter]); return (
diff --git a/src/features/search-filter/filter-records.ts b/src/features/search-filter/filter-records.ts index 09f6c94..e625808 100644 --- a/src/features/search-filter/filter-records.ts +++ b/src/features/search-filter/filter-records.ts @@ -13,6 +13,7 @@ import { import { createSheetDerivationCache } from '../../shared/sheet-derivation-cache'; import { withContextClusterFacet, type ContextClusterIndex } from '../context-layer/context-cluster-facet'; import { withBusinessImpactFacet } from './business-impact-controls'; +import { withComplianceFacets, type ComplianceWorkFacetIndex } from '../../compliance/ui/compliance-work-facets'; import { readEnginePreferences } from '../../app/engine-preferences'; export interface RecordFilterOptions { @@ -28,6 +29,8 @@ export interface RecordFilterOptions { reviews: Record; /** Context layer facet (Phase 13E); absent while the layer is off, so Work filters exactly as without it. */ contextClusters?: ContextClusterIndex; + /** Compliance Engine ISO control and ISO theme facets; absent while the engine is off. */ + complianceFacets?: ComplianceWorkFacetIndex; } const cachedRecords = createSheetDerivationCache(); @@ -37,7 +40,7 @@ export function recordFilterKeys(options: RecordFilterOptions): readonly unknown return [options.activeProfile, options.dataset, options.reviews, options.search, options.sortColumn, options.sortDirection, options.statusFilter, options.actionOnly, options.pathFindingIds?.length ? options.pathFindingIds : undefined, - JSON.stringify(options.facetFilters ?? {}), options.contextClusters, readEnginePreferences()]; + JSON.stringify(options.facetFilters ?? {}), options.contextClusters, options.complianceFacets, readEnginePreferences()]; } export function filterAndSortRecords(sheet: DatasetSheet | undefined, options: RecordFilterOptions): FindingRecord[] { @@ -50,9 +53,9 @@ function deriveFilteredRecords(sheet: DatasetSheet, options: RecordFilterOptions const normalizedSearch = options.search.trim().toLocaleLowerCase('en-US'); const baseFacets = deriveDynamicFacets(sheet, options.dataset, options.activeProfile, options.reviews); const contextToolId = selectedFacetContextToolId(sheet, baseFacets, options.facetFilters, options.reviews); - const facets = withBusinessImpactFacet(withContextClusterFacet(contextToolId === 'group-policy' + const facets = withComplianceFacets(withBusinessImpactFacet(withContextClusterFacet(contextToolId === 'group-policy' ? deriveDynamicFacets(sheet, options.dataset, options.activeProfile, options.reviews, 'group-policy') - : baseFacets, options.contextClusters), options.contextClusters); + : baseFacets, options.contextClusters), options.contextClusters), options.complianceFacets); const pathFindingIds = options.pathFindingIds?.length ? new Set(options.pathFindingIds) : undefined; const filtered = sheet.analysis.records.filter(record => { if (pathFindingIds && !pathFindingIds.has(record.id)) return false; diff --git a/src/features/search-filter/use-filtered-records.ts b/src/features/search-filter/use-filtered-records.ts index ff1dce4..baba1b8 100644 --- a/src/features/search-filter/use-filtered-records.ts +++ b/src/features/search-filter/use-filtered-records.ts @@ -2,6 +2,7 @@ import { useMemo } from 'react'; import { useApp } from '../../app/AppProvider'; import { filterAndSortRecords } from './filter-records'; import { useContextClusterIndex } from '../context-layer/use-context-cluster-index'; +import { useComplianceWorkFacets } from '../../compliance/ui/compliance-work-facets'; export function useFilteredRecords(statusFilterOverride?: string) { const { @@ -18,6 +19,7 @@ export function useFilteredRecords(statusFilterOverride?: string) { reviews } = useApp(); const contextClusters = useContextClusterIndex(); + const complianceFacets = useComplianceWorkFacets(); const effectiveStatusFilter = statusFilterOverride ?? statusFilter; return useMemo(() => filterAndSortRecords(activeSheet, { @@ -31,6 +33,7 @@ export function useFilteredRecords(statusFilterOverride?: string) { pathFindingIds, actionOnly, reviews, - contextClusters - }), [activeProfile, activeSheet, actionOnly, contextClusters, dataset, effectiveStatusFilter, facetFilters, pathFindingIds, reviews, search, sortColumn, sortDirection]); + contextClusters, + complianceFacets + }), [activeProfile, activeSheet, actionOnly, complianceFacets, contextClusters, dataset, effectiveStatusFilter, facetFilters, pathFindingIds, reviews, search, sortColumn, sortDirection]); } diff --git a/src/features/search-filter/work-facets.ts b/src/features/search-filter/work-facets.ts index f827ab0..ec84b67 100644 --- a/src/features/search-filter/work-facets.ts +++ b/src/features/search-filter/work-facets.ts @@ -9,6 +9,7 @@ import { import { filterAndSortRecords, recordFilterKeys, type RecordFilterOptions } from './filter-records'; import { withContextClusterFacet } from '../context-layer/context-cluster-facet'; import { withBusinessImpactFacet } from './business-impact-controls'; +import { withComplianceFacets } from '../../compliance/ui/compliance-work-facets'; export type WorkFacetOptions = Omit; const cache = createSheetDerivationCache(); @@ -22,9 +23,9 @@ export function selectWorkFacets(sheet: DatasetSheet | undefined, options: WorkF return cache(sheet, recordFilterKeys(query), () => { const baseFacets = deriveDynamicFacets(sheet, options.dataset, options.activeProfile, options.reviews); const contextToolId = selectedFacetContextToolId(sheet, baseFacets, options.facetFilters, options.reviews); - const facets = withBusinessImpactFacet(withContextClusterFacet(contextToolId === 'group-policy' + const facets = withComplianceFacets(withBusinessImpactFacet(withContextClusterFacet(contextToolId === 'group-policy' ? deriveDynamicFacets(sheet, options.dataset, options.activeProfile, options.reviews, 'group-policy') - : baseFacets, options.contextClusters), options.contextClusters); + : baseFacets, options.contextClusters), options.contextClusters), options.complianceFacets); const records = filterAndSortRecords(sheet, { ...query, statusFilter: 'all', facetFilters: emptyFilters }); // The engine retains its complete vocabulary; the Work control offers only reachable clusters. const selectableFacets = facets.map(facet => facet.derivedKind === 'context-cluster' diff --git a/src/features/sidebar/SidebarSecurityDomains.tsx b/src/features/sidebar/SidebarSecurityDomains.tsx index 2acce71..3198a60 100644 --- a/src/features/sidebar/SidebarSecurityDomains.tsx +++ b/src/features/sidebar/SidebarSecurityDomains.tsx @@ -9,6 +9,7 @@ import { type DynamicFacetFilters } from '../search-filter/dynamic-facets'; import { selectWorkFacets } from '../search-filter/work-facets'; import type { ContextClusterIndex } from '../context-layer/context-cluster-facet'; import { useContextClusterIndex } from '../context-layer/use-context-cluster-index'; +import { useComplianceWorkFacets, type ComplianceWorkFacetIndex } from '../../compliance/ui/compliance-work-facets'; import { businessImpactControls, BUSINESS_IMPACT_FACET_ID } from '../search-filter/business-impact-controls'; import { useEnginePreferences } from '../../app/engine-preferences'; @@ -23,6 +24,7 @@ interface SecurityDomainOptions { search: string; actionOnly: boolean; contextClusters?: ContextClusterIndex; + complianceFacets?: ComplianceWorkFacetIndex; } export interface SidebarSecurityDomain { @@ -38,11 +40,11 @@ export interface SidebarSecurityDomainSummary { /** Uses the exact shared Client Info facet, ordering and self-excluding counts. */ export function summarizeSecurityDomains(options: SecurityDomainOptions): SidebarSecurityDomainSummary | undefined { - const { sheet, dataset, activeProfile, reviews, statusFilter, facetFilters, pathFindingIds, search, actionOnly, contextClusters } = options; + const { sheet, dataset, activeProfile, reviews, statusFilter, facetFilters, pathFindingIds, search, actionOnly, contextClusters, complianceFacets } = options; if (!sheet) return undefined; const facets = selectWorkFacets(sheet, { - dataset, activeProfile, reviews, statusFilter, facetFilters, pathFindingIds, search, actionOnly, contextClusters + dataset, activeProfile, reviews, statusFilter, facetFilters, pathFindingIds, search, actionOnly, contextClusters, complianceFacets }); const domainFacet = facets.find(facet => facet.id === SECURITY_DOMAIN_FACET_ID); if (!domainFacet) return undefined; @@ -74,15 +76,16 @@ export function SidebarSecurityDomains({ sheet }: { sheet: DatasetSheet | undefi setStatusFilter } = useApp(); const contextClusters = useContextClusterIndex(); + const complianceFacets = useComplianceWorkFacets(); const engines = useEnginePreferences(); const impactCounts = useMemo(() => { if (!engines.report || !engines.context || !contextClusters) return undefined; const facet = selectWorkFacets(sheet, { dataset, activeProfile, reviews, statusFilter: 'deviation', facetFilters, - pathFindingIds, search, actionOnly, contextClusters }).find(item => item.id === BUSINESS_IMPACT_FACET_ID); + pathFindingIds, search, actionOnly, contextClusters, complianceFacets }).find(item => item.id === BUSINESS_IMPACT_FACET_ID); if (!facet) return undefined; const counts = new Map(facet.values.map(value => [value.value, value.count])); return businessImpactControls.map(theme => ({ ...theme, count: counts.get(theme.id) ?? 0 })); - }, [engines.report, engines.context, contextClusters, sheet, dataset, activeProfile, reviews, facetFilters, pathFindingIds, search, actionOnly]); + }, [engines.report, engines.context, complianceFacets, contextClusters, sheet, dataset, activeProfile, reviews, facetFilters, pathFindingIds, search, actionOnly]); const summary = useMemo(() => measureNavigationStep('security-domain-summary', () => summarizeSecurityDomains({ sheet, dataset, @@ -93,8 +96,9 @@ export function SidebarSecurityDomains({ sheet }: { sheet: DatasetSheet | undefi pathFindingIds, search, actionOnly, - contextClusters - })), [actionOnly, activeProfile, contextClusters, dataset, facetFilters, pathFindingIds, reviews, search, sheet, statusFilter]); + contextClusters, + complianceFacets + })), [actionOnly, activeProfile, complianceFacets, contextClusters, dataset, facetFilters, pathFindingIds, reviews, search, sheet, statusFilter]); if (!summary) return null; const selected = facetFilters[summary.facetId]?.values ?? []; diff --git a/src/main.tsx b/src/main.tsx index 43e8f2f..7ca478f 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -28,7 +28,6 @@ import './styles/obsidian-graph.css'; import './styles/functional-icons.css'; import './styles/tile-contract.css'; import './styles/filter-empty-state.css'; -import './styles/ai-overview.css'; import './styles/defender-for-iot.css'; if (import.meta.env.DEV && new URLSearchParams(location.search).get('measureNavigation') === '1') { diff --git a/src/persistence/assessor-repository.ts b/src/persistence/assessor-repository.ts index 02a1481..9be2f15 100644 --- a/src/persistence/assessor-repository.ts +++ b/src/persistence/assessor-repository.ts @@ -35,7 +35,6 @@ import type { UnifiedResultQueryResult } from '../domain/unified-result'; import type { EnvironmentObservation, EnvironmentSnapshot, EnvironmentSnapshotQuery, ScannerReportSection } from '../domain/environment-context'; -import type { AiOverviewRun } from '../domain/ai-overview'; export interface FindingOccurrenceQuery { projectId?: string; @@ -147,7 +146,6 @@ export interface ProjectDataExport { gpoPolicyAnalyses: StoredGpoPolicyAnalysis[]; environmentProfiles: EnvironmentProfile[]; policySnapshotAggregations: StoredPolicySnapshotAggregation[]; - aiOverviewRuns: AiOverviewRun[]; } export interface ControlledDeleteConfirmation { @@ -222,9 +220,6 @@ export interface AssessorRepository { getResultDbStatus(): Promise; listResultCategories(query?: UnifiedResultQuery): Promise; getEnvironmentSnapshot(query: EnvironmentSnapshotQuery): Promise; - createAiOverviewRun(run: AiOverviewRun): Promise; - updateAiOverviewRun(run: AiOverviewRun): Promise; - listAiOverviewRuns(projectId: string, scopeKey?: string): Promise; saveImportArtifact(artifact: ImportArtifact): Promise; saveImportArtifactAndEnqueueJob(artifact: ImportArtifact, job: IngestionJob): Promise; diff --git a/src/persistence/local-assessor-repository.ts b/src/persistence/local-assessor-repository.ts index b1559f8..0c3a7c2 100644 --- a/src/persistence/local-assessor-repository.ts +++ b/src/persistence/local-assessor-repository.ts @@ -33,8 +33,6 @@ import { storedPolicyEvaluationValidationErrors } from '../domain/policy-evaluat import type { StoredPolicyFindingRun } from '../domain/policy-finding'; import type { EnvironmentObservation, EnvironmentSnapshot, EnvironmentSnapshotQuery, ScannerReportSection } from '../domain/environment-context'; import { environmentCoverage } from '../domain/environment-context'; -import type { AiOverviewRun } from '../domain/ai-overview'; -import { aiOverviewRunValidationErrors } from '../domain/ai-overview'; import { storedPolicyFindingRunValidationErrors } from '../domain/policy-finding'; import { environmentProfileValidationErrors, @@ -110,7 +108,6 @@ interface LocalRepositoryState { environmentObservations: EnvironmentObservation[]; findingDefinitions: FindingDefinition[]; findingOccurrences: FindingOccurrence[]; - aiOverviewRuns: AiOverviewRun[]; evidence: EvidenceRecord[]; reviews: FindingReview[]; scopedRiskAcceptances: ScopedRiskAcceptance[]; @@ -146,7 +143,6 @@ const emptyState = (): LocalRepositoryState => ({ environmentObservations: [], findingDefinitions: [], findingOccurrences: [], - aiOverviewRuns: [], evidence: [], reviews: [], scopedRiskAcceptances: [], @@ -390,52 +386,6 @@ export class LocalAssessorRepository implements AssessorRepository { return clone(this.state.referenceCatalogs.filter(catalog => hashes.has(catalog.sha256))); } - async createAiOverviewRun(run: AiOverviewRun): Promise { - const errors = aiOverviewRunValidationErrors(run); - if (errors.length) throw new RepositoryValidationError(errors.join(' ')); - if (!this.state.projects.some(project => project.id === run.projectId)) { - throw new RepositoryNotFoundError(`Projekt ${run.projectId} ist nicht vorhanden.`); - } - const idempotent = this.state.aiOverviewRuns.find(item => - item.projectId === run.projectId && item.scopeKey === run.scopeKey && item.idempotencyKey === run.idempotencyKey); - if (idempotent) return clone(idempotent); - if (this.state.aiOverviewRuns.some(item => item.scopeKey === run.scopeKey && ['queued', 'running'].includes(item.status))) { - throw new RepositoryConflictError('For this timeslot, an AI overview is already being generated.'); - } - if (this.state.aiOverviewRuns.some(item => item.id === run.id)) throw new RepositoryConflictError(`AI overview run ${run.id} already exists.`); - const next = clone(this.state); - next.aiOverviewRuns.push(clone(run)); - this.replaceState(next); - return clone(run); - } - - async updateAiOverviewRun(run: AiOverviewRun): Promise { - const errors = aiOverviewRunValidationErrors(run); - if (errors.length) throw new RepositoryValidationError(errors.join(' ')); - const existing = this.state.aiOverviewRuns.find(item => item.id === run.id); - if (!existing) throw new RepositoryNotFoundError(`AI overview run ${run.id} does not exist.`); - if (existing.projectId !== run.projectId || existing.scopeKey !== run.scopeKey || existing.inputHash !== run.inputHash - || existing.idempotencyKey !== run.idempotencyKey || existing.model !== run.model) { - throw new RepositoryConflictError('Immutable AI overview run identity cannot be changed.'); - } - if (['succeeded', 'failed'].includes(existing.status)) throw new RepositoryConflictError('A completed AI overview run is immutable.'); - if (existing.status === 'queued' && !['running', 'failed'].includes(run.status) - || existing.status === 'running' && !['running', 'succeeded', 'failed'].includes(run.status)) { - throw new RepositoryConflictError(`Invalid AI overview transition from ${existing.status} to ${run.status}.`); - } - const next = clone(this.state); - next.aiOverviewRuns[next.aiOverviewRuns.findIndex(item => item.id === run.id)] = clone(run); - this.replaceState(next); - return clone(run); - } - - async listAiOverviewRuns(projectId: string, scopeKey?: string): Promise { - return this.state.aiOverviewRuns - .filter(item => item.projectId === projectId && (!scopeKey || item.scopeKey === scopeKey)) - .map(clone) - .sort((left, right) => right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id)); - } - async beginImport(start: ImportSessionStart): Promise { if (!start.sessionId.trim()) throw new RepositoryValidationError('Eine Importsitzung benötigt eine ID.'); if (this.activeImportSessions.has(start.sessionId)) throw new RepositoryConflictError(`Importsitzung ${start.sessionId} ist bereits aktiv.`); @@ -1613,8 +1563,7 @@ export class LocalAssessorRepository implements AssessorRepository { environmentProfiles: this.state.environmentProfiles.filter(item => item.projectId === projectId).map(clone), policySnapshotAggregations: this.state.policySnapshotAggregations.filter(item => item.run.projectId === projectId).map(clone), policyEvaluations: this.state.policyEvaluations.filter(item => item.run.projectId === projectId).map(clone), - policyFindingRuns: policyFindingRuns.map(clone), - aiOverviewRuns: this.state.aiOverviewRuns.filter(item => item.projectId === projectId && item.status === 'succeeded').map(clone) + policyFindingRuns: policyFindingRuns.map(clone) }); } @@ -1676,7 +1625,6 @@ export class LocalAssessorRepository implements AssessorRepository { next.gpoBackupInstances = next.gpoBackupInstances.filter(item => !processingRunIds.has(item.processingRunId)); next.gpoPolicyAnalyses = next.gpoPolicyAnalyses.filter(item => !processingRunIds.has(item.run.sourceProcessingRunId)); next.artifactDeletionRequests = next.artifactDeletionRequests.filter(item => !artifactIds.has(item.artifactId)); - next.aiOverviewRuns = next.aiOverviewRuns.filter(item => !item.scanRunIds.includes(scanRunId)); const remainingHashes = new Set(next.importFiles.map(item => item.sha256).filter(Boolean) as string[]); next.importFingerprints = next.importFingerprints.filter(item => !removedHashes.has(item.sha256) || remainingHashes.has(item.sha256)); this.replaceState(next); @@ -1743,7 +1691,6 @@ export class LocalAssessorRepository implements AssessorRepository { next.policyEvaluations = next.policyEvaluations.filter(item => !evaluationRunIds.has(item.run.id)); next.policyFindingRuns = next.policyFindingRuns.filter(item => !evaluationRunIds.has(item.run.evaluationRunId)); next.artifactDeletionRequests = next.artifactDeletionRequests.filter(item => !artifactIds.has(item.artifactId)); - next.aiOverviewRuns = next.aiOverviewRuns.filter(item => !item.assessmentIds.includes(assessmentId)); const remainingDefinitionIds = new Set([ ...next.findingOccurrences.map(item => item.definitionId), ...next.policyFindingRuns.flatMap(item => item.occurrences.map(occurrence => occurrence.definitionId)) @@ -1799,7 +1746,6 @@ export class LocalAssessorRepository implements AssessorRepository { next.environmentProfiles = next.environmentProfiles.filter(item => item.projectId !== projectId); next.policySnapshotAggregations = next.policySnapshotAggregations.filter(item => item.run.projectId !== projectId); next.policyEvaluations = next.policyEvaluations.filter(item => item.run.projectId !== projectId); - next.aiOverviewRuns = next.aiOverviewRuns.filter(item => item.projectId !== projectId); next.scopedRiskAcceptances = next.scopedRiskAcceptances.filter(item => item.projectId !== projectId); this.replaceState(next); return true; diff --git a/tests/app/app-version.test.ts b/tests/app/app-version.test.ts index dd91a6a..6e4d85c 100644 --- a/tests/app/app-version.test.ts +++ b/tests/app/app-version.test.ts @@ -9,7 +9,7 @@ const components = readFileSync(new URL('../../src/styles/components.css', impor describe('ASSESSOR version in the brand header', () => { it('shows the centrally maintained product version and opens About from the brand', () => { - expect(ASSESSOR_PRODUCT_VERSION).toBe('2.18.0'); + expect(ASSESSOR_PRODUCT_VERSION).toBe('2.19.0'); expect(app).toContain("'db-structure': 'Data Flow'"); expect(app).toContain("['db-structure', 'Data Flow']"); expect(app).not.toContain("'DB Structure'"); diff --git a/tests/architecture/core-engine-boundary.test.ts b/tests/architecture/core-engine-boundary.test.ts index 433701d..f01706a 100644 --- a/tests/architecture/core-engine-boundary.test.ts +++ b/tests/architecture/core-engine-boundary.test.ts @@ -53,7 +53,8 @@ const allSources = () => [...sourceFiles(join(root, 'src')), ...sourceFiles(join /** * Pinned Core -> Engine dependencies: the documented extension points ( * `Context Cluster` Work facet, Finding Reader `Context` section, Pathfinder and sidebar cluster - * readers; Finding Reader ISO mapping, Result DB ISO batch and Data Hub ISO corpus upload), the + * readers; Finding Reader ISO mapping, Result DB ISO batch and `Export JSON`, Data Hub ISO corpus upload, + * the Client Info `AI Overview` tab and the Work `ISO control`/`ISO theme` facets), the * type-only API contract and client, and the server wiring of the engines. */ const EXTENSION_POINTS: Record = { @@ -64,15 +65,15 @@ const EXTENSION_POINTS: Record { @@ -31,7 +31,7 @@ describe('Client Info AI Overview', () => { run: { ...base, id: 'run-2', status: 'failed', result: undefined, errorMessage: 'Generation failed safely.' }, previousSuccessfulRun: base }; - const html = renderToStaticMarkup( {} }} />); + const html = renderToStaticMarkup( {} }} />); expect(html).toContain('Generation failed safely.'); expect(html).toContain('Synthetic executive summary.'); expect(html).toContain('previous saved overview'); diff --git a/tests/postgres/ai-overview-schema.test.ts b/tests/compliance/ai-overview-schema.test.ts similarity index 100% rename from tests/postgres/ai-overview-schema.test.ts rename to tests/compliance/ai-overview-schema.test.ts diff --git a/tests/features/ai-overview-scope-interaction.test.tsx b/tests/compliance/ai-overview-scope-interaction.test.tsx similarity index 93% rename from tests/features/ai-overview-scope-interaction.test.tsx rename to tests/compliance/ai-overview-scope-interaction.test.tsx index d3bf15d..65d92e7 100644 --- a/tests/features/ai-overview-scope-interaction.test.tsx +++ b/tests/compliance/ai-overview-scope-interaction.test.tsx @@ -4,8 +4,8 @@ import { act, type ReactNode } from 'react'; import { createRoot } from 'react-dom/client'; import { afterEach, expect, it, vi } from 'vitest'; import { AssessorApiClient } from '../../src/api/assessor-api-client'; -import type { AiOverviewResponse } from '../../src/domain/ai-overview'; -import { useAiOverview } from '../../src/features/dashboard/AiOverviewPanel'; +import type { AiOverviewResponse } from '../../src/compliance/ai-overview'; +import { useAiOverview } from '../../src/compliance/ui/AiOverviewPanel'; afterEach(() => { vi.restoreAllMocks(); diff --git a/tests/server/ai-overview-service.test.ts b/tests/compliance/ai-overview-service.test.ts similarity index 66% rename from tests/server/ai-overview-service.test.ts rename to tests/compliance/ai-overview-service.test.ts index 02ff6db..2380683 100644 --- a/tests/server/ai-overview-service.test.ts +++ b/tests/compliance/ai-overview-service.test.ts @@ -1,8 +1,9 @@ import { describe, expect, it, vi } from 'vitest'; -import { AiOverviewService } from '../../server/ai/ai-overview-service'; +import { AiOverviewService } from '../../server/compliance/ai-overview-service'; import { LocalAssessorRepository } from '../../src/persistence/local-assessor-repository'; +import { LocalComplianceStore } from '../../src/compliance/local-compliance-store'; import type { UnifiedResultProjection } from '../../src/domain/unified-result'; -import { validAiOverviewResult } from '../fixtures/ai-overview'; +import { validAiOverviewResult } from './ai-overview-fixtures'; const stamp = '2026-09-14T08:00:00.000Z'; @@ -21,6 +22,7 @@ function syntheticResult(): UnifiedResultProjection { describe('AI overview service', () => { it('uses the complete timeslot query, persists a successful run, and honors idempotency', async () => { const repository = new LocalAssessorRepository(); + const store = new LocalComplianceStore(); await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp }); await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp }); const query = vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] }); @@ -28,7 +30,7 @@ describe('AI overview service', () => { serviceResult.coverageGaps = []; serviceResult.environmentOverview.factsUsed = []; const complete = vi.fn(async () => ({ value: serviceResult, usage: { totalTokens: 30 } })); - const service = new AiOverviewService(repository, { + const service = new AiOverviewService(repository, store, { client: { model: 'work-model', complete }, now: () => new Date(stamp) }); @@ -42,16 +44,19 @@ describe('AI overview service', () => { expect(query).toHaveBeenCalledWith({ audience: 'result-db', projectId: 'project-1', assessmentIds: ['assessment-1'], scanRunIds: [], offset: 0, limit: 500 }); - expect((await repository.exportProject('project-1')).aiOverviewRuns).toHaveLength(1); + // The runs live in the engine store and reach the project export beside the Core export, never inside it. + expect((await store.exportProject('project-1')).aiOverviewRuns).toHaveLength(1); + expect('aiOverviewRuns' in await repository.exportProject('project-1')).toBe(false); await service.start('project-1', request); await service.close(); expect(complete).toHaveBeenCalledTimes(1); - expect(await repository.listAiOverviewRuns('project-1')).toHaveLength(1); + expect(await store.listAiOverviewRuns('project-1')).toHaveLength(1); }); it('keeps the previous success when a later regeneration fails', async () => { const repository = new LocalAssessorRepository(); + const store = new LocalComplianceStore(); await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp }); await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp }); vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] }); @@ -62,7 +67,7 @@ describe('AI overview service', () => { .mockResolvedValueOnce({ value: serviceResult }) .mockResolvedValueOnce({ value: { schemaVersion: 1 } }); let currentTime = stamp; - const service = new AiOverviewService(repository, { client: { model: 'work-model', complete }, now: () => new Date(currentTime) }); + const service = new AiOverviewService(repository, store, { client: { model: 'work-model', complete }, now: () => new Date(currentTime) }); await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-good' }); await service.close(); @@ -70,7 +75,7 @@ describe('AI overview service', () => { await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-fail' }); await service.close(); - const runs = await repository.listAiOverviewRuns('project-1'); + const runs = await store.listAiOverviewRuns('project-1'); expect(runs.map(run => run.status).sort()).toEqual(['failed', 'succeeded']); expect(runs.find(run => run.status === 'succeeded')?.result?.executiveSummary).toBe('Synthetic executive summary.'); expect(runs.find(run => run.status === 'failed')?.errorCode).toBe('ai_output_invalid'); @@ -83,6 +88,7 @@ describe('AI overview service', () => { it('marks the previous success stale while a regeneration is queued', async () => { const repository = new LocalAssessorRepository(); + const store = new LocalComplianceStore(); await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp }); await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp }); vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] }); @@ -90,11 +96,11 @@ describe('AI overview service', () => { serviceResult.coverageGaps = []; serviceResult.environmentOverview.factsUsed = []; const complete = vi.fn().mockResolvedValueOnce({ value: serviceResult }); - const service = new AiOverviewService(repository, { client: { model: 'work-model', complete }, now: () => new Date(stamp) }); + const service = new AiOverviewService(repository, store, { client: { model: 'work-model', complete }, now: () => new Date(stamp) }); await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-good' }); await service.close(); - const [saved] = await repository.listAiOverviewRuns('project-1'); - await repository.createAiOverviewRun({ + const [saved] = await store.listAiOverviewRuns('project-1'); + await store.createAiOverviewRun({ ...saved, id: 'ai-overview-queued', idempotencyKey: 'request-queued', status: 'queued', inputHash: 'b'.repeat(64), result: undefined, completedAt: undefined, createdAt: '2026-09-14T09:00:00.000Z', updatedAt: '2026-09-14T09:00:00.000Z', callCount: 0 }); @@ -103,4 +109,29 @@ describe('AI overview service', () => { const response = await service.get('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [] }); expect(response).toMatchObject({ state: 'queued', stale: true, previousSuccessfulRun: { id: saved.id } }); }); + + it('removes the overview of a deleted timeslot from the engine store', async () => { + const repository = new LocalAssessorRepository(); + const store = new LocalComplianceStore(); + await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp }); + await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp }); + await repository.saveAssessment({ id: 'assessment-2', projectId: 'project-1', name: 'Second timeslot', createdAt: stamp, updatedAt: stamp }); + vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] }); + const serviceResult = validAiOverviewResult(); + serviceResult.coverageGaps = []; + serviceResult.environmentOverview.factsUsed = []; + const service = new AiOverviewService(repository, store, { + client: { model: 'work-model', complete: vi.fn(async () => ({ value: serviceResult })) }, now: () => new Date(stamp) + }); + for (const assessmentId of ['assessment-1', 'assessment-2']) { + await service.start('project-1', { assessmentIds: [assessmentId], scanRunIds: [], idempotencyKey: `request-${assessmentId}` }); + await service.close(); + } + expect(await store.listAiOverviewRuns('project-1')).toHaveLength(2); + + // The Core deletes the timeslot without knowing the engine; the engine prunes its own derived run. + await repository.deleteAssessment('project-1', 'assessment-1', { projectId: 'project-1', assessmentId: 'assessment-1', cascade: true }); + await service.get('project-1', { assessmentIds: ['assessment-2'], scanRunIds: [] }); + expect((await store.listAiOverviewRuns('project-1')).map(run => run.assessmentIds)).toEqual([['assessment-2']]); + }); }); diff --git a/tests/domain/ai-overview.test.ts b/tests/compliance/ai-overview.test.ts similarity index 91% rename from tests/domain/ai-overview.test.ts rename to tests/compliance/ai-overview.test.ts index 704e466..7fc631b 100644 --- a/tests/domain/ai-overview.test.ts +++ b/tests/compliance/ai-overview.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { aiOverviewResultValidationErrors, aiOverviewScopeKey, normalizeAiOverviewScope } from '../../src/domain/ai-overview'; -import { splitPrivacySafeInput, type PrivacySafeInput } from '../../server/ai/ai-overview-service'; -import { validAiOverviewResult } from '../fixtures/ai-overview'; +import { aiOverviewResultValidationErrors, aiOverviewScopeKey, normalizeAiOverviewScope } from '../../src/compliance/ai-overview'; +import { splitPrivacySafeInput, type PrivacySafeInput } from '../../server/compliance/ai-overview-service'; +import { validAiOverviewResult } from './ai-overview-fixtures'; describe('AI overview contract', () => { it('normalizes timeslot identity deterministically', () => { diff --git a/tests/compliance/client-iso-controls.test.tsx b/tests/compliance/client-iso-controls.test.tsx index 2e0fb3e..346d2de 100644 --- a/tests/compliance/client-iso-controls.test.tsx +++ b/tests/compliance/client-iso-controls.test.tsx @@ -28,10 +28,10 @@ const overview: RegulatoryControlOverviewResponse = { let host: HTMLDivElement; let root: Root; -const onOpenFindings = vi.fn(); +const onOpenControl = vi.fn(); const render = () => act(async () => { root.render(); + backendStatus="connected" onOpenControl={onOpenControl}/>); await new Promise(resolve => setTimeout(resolve, 0)); }); @@ -39,7 +39,7 @@ beforeEach(() => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); localStorage.clear(); resetEnginePreferencesForTests(); - onOpenFindings.mockReset(); + onOpenControl.mockReset(); host = document.createElement('div'); document.body.append(host); root = createRoot(host); @@ -65,7 +65,7 @@ describe('Client Info ISO controls', () => { ['Organizational', 'A.5.15', '1 finding'] ]); await act(async () => tiles[0].click()); - expect(onOpenFindings).toHaveBeenCalledWith(overview.controls[0].findings); + expect(onOpenControl).toHaveBeenCalledWith('8.20'); }); it('reloads after a control decision and says so when nothing is accepted', async () => { @@ -88,13 +88,17 @@ describe('Client Info ISO controls', () => { it('names the overview request in the activity log as Compliance Engine work', () => { expect(requestActivity('/projects/p/regulatory-control-overview?assessmentIds=a')).toEqual({ area: 'Compliance Engine', label: 'Loading ISO control overview' }); + expect(requestActivity('/projects/p/regulatory-control-assignments?assessmentIds=a')).toEqual({ area: 'Compliance Engine', label: 'Loading ISO control filters' }); }); - it('places the section below the Context heatmap and opens the findings by their Work records', () => { + it('places the section below the Context heatmap and opens Work through the ISO control filter', () => { const source = readFileSync(join(process.cwd(), 'src/features/dashboard/DashboardPanel.tsx'), 'utf8'); expect(source.indexOf(' workRecordIdsForResult(dataset, finding)))])'); + const drilldown = source.slice(source.indexOf(' { }); // Compliance Engine switch (AGENTS.md, "Core Engine and engines"): switched off, the Finding Reader, -// Result DB and Data Hub show none of its controls and send none of its requests; switched on again, +// Result DB, Client Info AI overview and Data Hub show none of its controls and send none of its requests; switched on again, // the Finding Reader ISO panel starts closed. let host: HTMLDivElement; let root: Root; @@ -51,6 +52,11 @@ function BatchProbe(): ReactNode { return {batch.available ? 'available' : 'off'}; } +function AiOverviewProbe(): ReactNode { + const overview = useAiOverview('dummy-project', ['dummy-assessment'], []); + return {overview.available ? 'available' : 'off'}; +} + beforeEach(() => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); localStorage.clear(); @@ -134,6 +140,27 @@ describe('Compliance Engine switch', () => { expect(source).toContain('{batchDialogOpen && regulatoryBatch.available && { + const request = vi.spyOn(AssessorApiClient.prototype, 'getAiOverview').mockResolvedValue({ + version: 1, projectId: 'dummy-project', scope: { assessmentIds: ['dummy-assessment'], scanRunIds: [] }, + generationEnabled: true, state: 'empty', stale: false + }); + setEngineEnabled('compliance', false); + await act(async () => { root.render(); await new Promise(resolve => setTimeout(resolve, 0)); }); + expect(host.textContent).toBe('off'); + expect(request).not.toHaveBeenCalled(); + await act(async () => { setEngineEnabled('compliance', true); await new Promise(resolve => setTimeout(resolve, 0)); }); + expect(host.textContent).toBe('available'); + expect(request).toHaveBeenCalledTimes(1); + + const dashboard = readFileSync(join(process.cwd(), 'src/features/dashboard/DashboardPanel.tsx'), 'utf8'); + expect(dashboard).toContain('const showAiOverview = aiOverview.available && '); + const browser = readFileSync(join(process.cwd(), 'src/features/results-browser/ResultsBrowser.tsx'), 'utf8'); + const exportButton = browser.slice(browser.lastIndexOf('{regulatoryBatch.available && void app.openProject('dummy-project', [], ['dummy-assessment'])}>Open + + {JSON.stringify(app.facetFilters)} + {withScope && <>} + ; +} + +describe('Work ISO facet loading and the engine switch', () => { + it('shares one request for every Work consumer and reloads after a control decision', async () => { + const { loadCanonicalProjectWorkspace } = await import('../../src/features/project-import/load-canonical-project-workspace'); + vi.mocked(loadCanonicalProjectWorkspace).mockResolvedValue({ assessmentIds: ['dummy-assessment'], models: [], + dataset: { fileName: 'Dummy', scanTool: 'Generic', sheets: [workSheet()] } } as never); + vi.spyOn(AssessorApiClient.prototype, 'getAssessmentContext').mockResolvedValue({} as never); + const request = vi.spyOn(AssessorApiClient.prototype, 'getRegulatoryControlAssignments').mockResolvedValue(assignments); + await act(async () => root.render()); + await act(async () => { host.querySelector('button')!.click(); await new Promise(resolve => setTimeout(resolve, 0)); }); + await act(async () => { await new Promise(resolve => setTimeout(resolve, 0)); }); + expect(request).toHaveBeenCalledTimes(1); + expect([...host.querySelectorAll('[data-iso-facets]')].map(node => node.textContent)).toEqual(['2,2', '2,2']); + await act(async () => { markRegulatoryDecisionSaved(); await new Promise(resolve => setTimeout(resolve, 0)); }); + expect(request).toHaveBeenCalledTimes(2); + }); + + it('loads nothing and clears an active ISO filter when the Compliance Engine is switched off', async () => { + const request = vi.spyOn(AssessorApiClient.prototype, 'getRegulatoryControlAssignments').mockResolvedValue(assignments); + await act(async () => root.render()); + await act(async () => host.querySelectorAll('button')[1].click()); + expect(JSON.parse(host.querySelector('[data-filters]')!.textContent!)).toEqual({ [ISO_CONTROL_FACET_ID]: { values: ['8.20'] } }); + await act(async () => { setEngineEnabled('compliance', false); }); + expect(JSON.parse(host.querySelector('[data-filters]')!.textContent!)).toEqual({}); + await act(async () => root.render()); + expect(host.querySelector('[data-iso-facets]')?.textContent).toBe('none'); + expect(request).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/exports/result-db-copy.test.ts b/tests/compliance/privacy-safe-export.test.ts similarity index 73% rename from tests/exports/result-db-copy.test.ts rename to tests/compliance/privacy-safe-export.test.ts index 0880101..f9c7665 100644 --- a/tests/exports/result-db-copy.test.ts +++ b/tests/compliance/privacy-safe-export.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import type { UnifiedResultProjection } from '../../src/domain/unified-result'; -import { resultsToPrivacySafeExportJson } from '../../src/exports/result-db-copy'; +import { aiOverviewInputJson, complianceAiExportJson, resultsToPrivacySafeExportJson, type ComplianceAiExport } from '../../src/compliance/privacy-safe-export'; +import { buildRegulatoryFindingQuery } from '../../src/compliance/regulatory-query'; function result(overrides: Partial): UnifiedResultProjection { return { @@ -108,3 +109,32 @@ describe('Result DB compact privacy-safe JSON export', () => { expect(JSON.stringify(copied)).not.toMatch(/PRIVATE/); }); }); + +describe('Compliance Engine AI export', () => { + const gpo = (id: string) => result({ + resultProjectionId: `policy-evaluation-comparison:${id}`, sourceResultId: `comparison-${id}`, sourceKind: 'policy-evaluation', + sourceTool: 'GPOs', title: 'System Access · MinimumPasswordLength', isDeviation: true, outcomeNormalized: 'fail', + policyScope: 'computer', policyDeliveryMechanism: 'security-template', policyCustomerState: 'configured', + policyComparisonRelation: 'different', policyBaselineLabel: 'Microsoft Security Baseline - Windows Server', + policyBaselineValue: { kind: 'integer', value: '14' } + }); + const scanner = result({ + resultProjectionId: 'scanner-finding:one', sourceResultId: 'one', sourceRuleId: 'PK-1', title: 'Scanner deviation', + description: 'Scanner evidence', outcomeNormalized: 'fail', isDeviation: true + }); + + it('contains exactly the AI overview input and every ISO query the model receives', () => { + const items = [gpo('b'), scanner, gpo('a')]; + const exported = JSON.parse(complianceAiExportJson(items)) as ComplianceAiExport; + expect(exported.schemaVersion).toBe(2); + // The AI overview hashes and sends this same JSON; the order of the rows does not change it. + expect(exported.aiOverviewInput).toEqual(JSON.parse(aiOverviewInputJson(items))); + expect(aiOverviewInputJson([...items].reverse())).toBe(aiOverviewInputJson(items)); + // The two baseline comparisons of one GPO setting produce one query, as the model receives it. + expect(exported.isoMappingQueries).toEqual([ + { query: buildRegulatoryFindingQuery(gpo('a')), resultProjectionIds: ['policy-evaluation-comparison:a', 'policy-evaluation-comparison:b'] }, + { query: buildRegulatoryFindingQuery(scanner), resultProjectionIds: ['scanner-finding:one'] } + ]); + expect(exported.isoMappingQueries[0].query.query).toContain('Baseline recommended value: 14.'); + }); +}); diff --git a/tests/compliance/regulatory-finding-retrieval.test.ts b/tests/compliance/regulatory-finding-retrieval.test.ts index db9b1f5..73907ab 100644 --- a/tests/compliance/regulatory-finding-retrieval.test.ts +++ b/tests/compliance/regulatory-finding-retrieval.test.ts @@ -120,3 +120,59 @@ describe('regulatory finding retrieval boundary', () => { expect(retrieve).not.toHaveBeenCalled(); }); }); + +describe('GPO baseline context in the regulatory query', () => { + const gpo = (overrides: Partial = {}) => finding({ + resultProjectionId: 'policy-evaluation-comparison:one', sourceKind: 'policy-evaluation', sourceTool: 'GPOs', + sourceRuleId: 'gpo-policy-identity-0123456789abcdef', title: 'System Access · MinimumPasswordLength', + description: undefined, recommendation: undefined, affectedAsset: undefined, + policyScope: 'computer', policyDeliveryMechanism: 'security-template', policyCustomerState: 'configured', + policyComparisonRelation: 'different', policyBaselineLabel: 'Microsoft Security Baseline - Windows Server 2025 v2506', + policyBaselineValue: { kind: 'integer', value: '14' }, + // The customer value is a synthetic secret that must never reach the query. + policyActualValue: { kind: 'integer', value: '7' }, + ...overrides + }); + + it('adds section, readable name, baseline family and value and the proven difference, never the configured value', () => { + const query = buildRegulatoryFindingQuery(gpo())!.query; + expect(query).toContain('Group Policy setting type: Computer Configuration, security template (GptTmpl.inf).'); + expect(query).toContain('Policy section: System Access.'); + expect(query).toContain('Setting name: Minimum Password Length.'); + // Only the baseline family crosses, never the concrete release. + expect(query).toContain('Baseline: Microsoft Security Baseline - Windows Server.'); + expect(query).not.toContain('v2506'); + expect(query).toContain('Baseline recommended value: 14.'); + // An exact comparison proves a difference, not a direction. + expect(query).toContain('The configured value differs from the baseline recommendation.'); + expect(query).not.toMatch(/below|weaker|lower/i); + expect(query).not.toMatch(/\b7\b/); + expect(buildRegulatoryFindingQuery(gpo())!.publicTopics).toContain('Password policy'); + }); + + it('states a missing setting and a recorded weaker comparison as such', () => { + expect(buildRegulatoryFindingQuery(gpo({ policyCustomerState: 'missing', policyActualValue: undefined }))!.query) + .toContain('The recommended setting is not configured in the analysed Group Policy backups.'); + const weaker = gpo({ policyComparisonRelation: 'weaker', policyBaselineVariantAssessment: undefined }); + expect(buildRegulatoryFindingQuery(weaker)!.query).toContain('The configured value is weaker than the baseline recommendation.'); + }); + + it('adds the documented meaning of a user right and a well-known SID, never resolving a domain SID', () => { + const query = buildRegulatoryFindingQuery(gpo({ title: 'Privilege Rights · SeBackupPrivilege', + policyBaselineValue: { kind: 'string-set', values: ['*S-1-5-32-544', '*S-1-5-32-551'] }, + policyActualValue: { kind: 'string-set', values: ['*S-1-5-21-1111111111-2222222222-3333333333-1105'] } }))!.query; + expect(query).toContain('Setting name: Se Backup Privilege (Back up files and directories).'); + expect(query).toContain('Baseline recommended value: *S-1-5-32-544 (Administrators), *S-1-5-32-551 (Backup Operators).'); + expect(query).not.toContain('1105'); + // A setting without a documented code keeps its plain readable name. + expect(buildRegulatoryFindingQuery(gpo())!.query).toContain('Setting name: Minimum Password Length.'); + }); + + it('names a registry key and leaves scanner queries unchanged', () => { + const registry = buildRegulatoryFindingQuery(gpo({ policyDeliveryMechanism: 'registry-policy', + title: 'Software\Policies\Microsoft\Windows\System · EnableSmartScreen', policyBaselineValue: { kind: 'integer', value: '1' } }))!.query; + expect(registry).toContain('Registry key: Software\Policies\Microsoft\Windows\System.'); + expect(registry).toContain('Setting name: Enable Smart Screen.'); + expect(buildRegulatoryFindingQuery(finding())!.query).not.toContain('Group Policy setting type'); + }); +}); diff --git a/tests/compliance/regulatory-mapping-earlier-decision.test.tsx b/tests/compliance/regulatory-mapping-earlier-decision.test.tsx new file mode 100644 index 0000000..046edef --- /dev/null +++ b/tests/compliance/regulatory-mapping-earlier-decision.test.tsx @@ -0,0 +1,41 @@ +// @vitest-environment jsdom + +import { act } from 'react'; +import { createRoot } from 'react-dom/client'; +import { expect, it, vi } from 'vitest'; +import type { RegulatoryMappingResponse } from '../../src/compliance/regulatory-mapping'; +import { RegulatoryMappingPanel } from '../../src/compliance/ui/RegulatoryMappingPanel'; + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +// Synthetic: A.8.3 was auto-accepted by an earlier run of the rule; the current run no longer retrieves it. +const response: RegulatoryMappingResponse = { + version: 1, projectId: 'project-1', resultProjectionId: 'policy-evaluation-comparison:one', generationEnabled: true, state: 'succeeded', + decisions: [{ version: 1, id: 'decision-1', ruleKey: 'GPOs:synthetic', sourceTool: 'GPOs', mappingId: 'regulatory-mapping-old', + controlId: '8.3', decision: 'accepted', createdAt: '2026-10-05T10:00:00.000Z' }], + earlierControls: [{ controlId: '8.3', annexAControlId: 'A.8.3', title: 'Synthetic earlier title' }], + run: { + version: 1, id: 'regulatory-mapping-new', ruleKey: 'GPOs:synthetic', sourceTool: 'GPOs', status: 'succeeded', queryHash: 'a'.repeat(64), + preparedQuery: { schemaVersion: 1, query: 'Synthetic public query.', specificity: 'topic', publicTopics: [] }, + corpusId: 'iso-test', corpusVersion: 'v2', sourceHashes: [], embeddingModel: 'bge-m3', retrievalVersion: 'test', promptVersion: 'test', + outputSchemaVersion: 1, model: 'work-model', createdAt: '2026-10-06T10:00:00.000Z', updatedAt: '2026-10-06T10:00:00.000Z', + candidates: [{ controlId: '8.2', annexAControlId: 'A.8.2', title: 'Synthetic current title', score: 0.9, evidence: [] }], + result: { schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: '8.2', relationship: 'primary', confidence: 'high', rationale: 'Synthetic.', evidenceChunkIds: [] }] } + } +}; + +it('rejects a control accepted by an earlier run that the current run no longer retrieves', () => { + const onDecision = vi.fn(); + const container = document.createElement('div'); + const root = createRoot(container); + act(() => root.render( {}} onDecision={onDecision}/>)); + + const earlier = container.querySelector('[aria-label="Earlier decisions"]')!; + expect(earlier.textContent).toContain('A.8.3 — Synthetic earlier title'); + expect(earlier.textContent).toContain('accepted'); + expect(earlier.textContent).not.toContain('A.8.2'); + const reject = [...earlier.querySelectorAll('button')].find(button => button.textContent === 'Reject')!; + act(() => reject.click()); + expect(onDecision).toHaveBeenCalledWith(expect.objectContaining({ controlId: '8.3' }), 'rejected', ''); + act(() => root.unmount()); +}); diff --git a/tests/compliance/regulatory-mapping-service.test.ts b/tests/compliance/regulatory-mapping-service.test.ts index 0244e63..f41c729 100644 --- a/tests/compliance/regulatory-mapping-service.test.ts +++ b/tests/compliance/regulatory-mapping-service.test.ts @@ -1,13 +1,15 @@ import { describe, expect, it, vi } from 'vitest'; import { AUTO_ACCEPT_COMMENT, RegulatoryMappingService } from '../../server/compliance/regulatory-mapping-service'; import { RegulatoryFindingRetrievalService } from '../../server/compliance/regulatory-finding-retrieval'; -import { acceptedControlIds, type RegulatoryMappingBatchRun, type RegulatoryMappingDecision, type RegulatoryMappingRun } from '../../src/compliance/regulatory-mapping'; +import { acceptedControlIds, REGULATORY_GPO_QUERY_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, type RegulatoryMappingBatchRun, type RegulatoryMappingDecision, type RegulatoryMappingRun } from '../../src/compliance/regulatory-mapping'; import type { RegulatoryChunk, RegulatoryRetrievalRequest } from '../../src/compliance/regulatory-rag'; import { filterUnifiedResults, type UnifiedResultProjection } from '../../src/domain/unified-result'; import type { ComplianceReadPort, ComplianceStore } from '../../src/compliance/compliance-store'; type EngineRepository = ComplianceReadPort & ComplianceStore; import { removeResultDbScopeIdentifiers } from '../../src/domain/result-db-disclosure'; +import { complianceWorkFacetIndex } from '../../src/compliance/ui/compliance-work-facets'; +import type { FindingRecord } from '../../src/domain/finding'; const stamp = '2026-09-15T10:00:00.000Z'; // Result-table / occurrence data that must never reach the model or the query. @@ -132,6 +134,7 @@ describe('regulatory mapping service', () => { const shared = await svc.get('project-2', 'scanner-finding:occurrence-2'); expect(shared).toMatchObject({ state: 'succeeded', run: { ruleKey: 'purple-knight:A-SmbSigning', sourceTool: 'purple-knight', embeddingModel: 'bge-m3', tokenUsage: { totalTokens: 12 } } }); expect(shared.run?.result?.mappings[0].controlId).toBe('5.1'); + expect(shared.run?.promptVersion).toBe(REGULATORY_MAPPING_PROMPT_VERSION); expect(complete).toHaveBeenCalledOnce(); // The copyrighted ISO clause text is stripped from the response but still reaches the model. @@ -205,10 +208,66 @@ describe('regulatory mapping service', () => { .toEqual(['scanner-finding:occurrence-1', 'scanner-finding:occurrence-2', 'policy-evaluation-comparison:one']); // Only the timeslot deviations are read, and only through the read-only port. expect(vi.mocked(repository.queryUnifiedResults).mock.calls.at(-1)![0]).toMatchObject({ audience: 'result-db', deviation: true, projectId: 'project-1' }); + // The Work filters read every mappable finding, so the status filter in Work decides what is shown. + const assignments = await svc.controlAssignments('project-1', { assessmentIds: ['assessment-1'], scanRunIds: ['run-1'] }); + expect(assignments.findingCount).toBe(4); + expect(assignments.controls[0].findings.map(item => item.resultProjectionId)).toContain('scanner-finding:not-deviation'); + expect(vi.mocked(repository.queryUnifiedResults).mock.calls.at(-1)?.[0]?.deviation).toBeUndefined(); await expect(svc.controlOverview('project-1', { assessmentIds: [], scanRunIds: [] })).rejects.toThrow('timeslot scope is required'); await expect(svc.controlOverview('project-1', { assessmentIds: ['foreign-assessment'], scanRunIds: [] })).rejects.toThrow('outside this project'); }); + it('counts a GPO setting once and gives Work its setting ID although the Result DB withholds it', async () => { + // Internal comparisons carry the setting ID; the Result DB rows of the same comparisons do not. + const comparison = (id: string, settingId: string, title: string, policyScope: 'computer' | 'user') => finding({ + resultProjectionId: `policy-evaluation-comparison:${id}`, sourceResultId: `comparison-${id}`, sourceKind: 'policy-evaluation', + sourceTool: 'GPOs', sourceRuleId: settingId, policyScope, title }); + const findings = [ + // Three baseline packages of one setting, a second setting, the same title in the other scope and + // a different setting that shares the first setting's title. + comparison('a1', 'gpo-policy-identity-aaaa', 'Synthetic · Mode', 'computer'), + comparison('a2', 'gpo-policy-identity-aaaa', 'Synthetic · Mode', 'computer'), + comparison('a3', 'gpo-policy-identity-aaaa', 'Synthetic · Mode', 'computer'), + comparison('b1', 'gpo-policy-identity-bbbb', 'Security Options · SyntheticKey', 'computer'), + comparison('b2', 'gpo-policy-identity-bbbb', 'Security Options · SyntheticKey', 'computer'), + comparison('c1', 'gpo-policy-identity-cccc', 'Synthetic · Mode', 'user'), + comparison('d1', 'gpo-policy-identity-dddd', 'Synthetic · Mode', 'computer') + ]; + const repository = mockRepository(findings); + const internal = vi.mocked(repository.queryUnifiedResults).getMockImplementation()!; + vi.mocked(repository.queryUnifiedResults).mockImplementation(async (query = {}) => { + const page = await internal(query); + return query.audience === 'internal-analysis' ? page + : { ...page, items: page.items.map(item => ({ ...item, sourceRuleId: undefined })) }; + }); + const svc = service(repository, vi.fn(async () => ({ value: { + schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: '5.1', relationship: 'primary', confidence: 'high', + rationale: 'Synthetic rationale.', evidenceChunkIds: ['semantic-5.1', 'reference-5.1'] }] + } }))); + const scope = { assessmentIds: ['assessment-1'], scanRunIds: ['run-1'], filters: { deviation: true } }; + await svc.startBatch('project-1', { ...scope, idempotencyKey: 'batch-gpo-settings' }); + await svc.close(); + + // Mapping stays one global rule per disclosed title; counting follows the Work settings. + expect((await svc.getBatch('project-1', scope)).run?.counts) + .toMatchObject({ filteredResultCount: 7, eligibleFindingCount: 4, distinctRuleCount: 2 }); + const overview = await svc.controlOverview('project-1', { assessmentIds: ['assessment-1'], scanRunIds: ['run-1'] }); + expect(overview.findingCount).toBe(4); + expect(overview.controls[0].findings.map(item => `${item.policyScope}:${item.sourceRuleId}`).sort()).toEqual([ + 'computer:gpo-policy-identity-aaaa', 'computer:gpo-policy-identity-bbbb', + 'computer:gpo-policy-identity-dddd', 'user:gpo-policy-identity-cccc']); + // The internal read is limited to GPO comparisons of the same timeslot. + const internalQueries = vi.mocked(repository.queryUnifiedResults).mock.calls.map(call => call[0] ?? {}) + .filter(query => query.audience === 'internal-analysis'); + expect(internalQueries.length).toBeGreaterThan(0); + expect(internalQueries.every(query => query.projectId === 'project-1' && query.sourceKind?.join() === 'policy-evaluation' + && query.assessmentIds?.join() === 'assessment-1' && query.scanRunIds?.join() === 'run-1')).toBe(true); + + // The Work ISO filter finds the rolled-up GPO setting by scope and setting ID. + const workSetting = { id: `gpo-work-setting:${encodeURIComponent(JSON.stringify(['project-1', 'aggregation-1', 'computer', 'gpo-policy-identity-aaaa']))}` } as FindingRecord; + expect(complianceWorkFacetIndex(overview).facets[0].valuesFor!(workSetting)).toEqual(['5.1']); + }); + it('accepts the first two proposals automatically, keeps existing decisions and accepts nothing without a proposal', async () => { const semantic = (controlId: string): RegulatoryChunk => ({ ...chunk('ISO/IEC 27002'), controlId, annexAControlId: `A.${controlId}`, documentId: `27002-${controlId}`, chunkId: `semantic-${controlId}` }); @@ -251,6 +310,36 @@ describe('regulatory mapping service', () => { expect((await none.get('project-1', 'scanner-finding:occurrence-1')).decisions).toEqual([]); }); + it('keeps a control decided by an earlier run reviewable after a new pipeline version stops retrieving it', async () => { + const semantic = (controlId: string): RegulatoryChunk => ({ ...chunk('ISO/IEC 27002'), controlId, annexAControlId: `A.${controlId}`, + documentId: `27002-${controlId}`, chunkId: `semantic-${controlId}` }); + const repository = mockRepository([finding()]); + const build = (corpusVersion: string, controlId: string) => new RegulatoryMappingService(repository, repository, { + findingRetrieval: new RegulatoryFindingRetrievalService({ retrieve: vi.fn(async (_r: RegulatoryRetrievalRequest) => [{ score: 0.9, chunk: semantic(controlId) }]) }), + index: { ensureCollection: vi.fn(), replaceDocuments: vi.fn(), searchDense: vi.fn(), list: vi.fn(async () => [chunk('ISO/IEC 27001')]) }, + client: { model: 'work-model', complete: vi.fn(async () => ({ value: { schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId, + relationship: 'primary', confidence: 'high', rationale: 'Synthetic rationale.', evidenceChunkIds: [`semantic-${controlId}`] }] } })) as never }, + corpusId: 'iso-test', corpusVersion, embeddingModel: 'bge-m3', now: () => new Date(stamp) + }); + const older = build('v1', '8.3'); + await older.start('project-1', { resultProjectionId: 'scanner-finding:occurrence-1' }); + await older.close(); + const current = build('v2', '8.2'); + await current.start('project-1', { resultProjectionId: 'scanner-finding:occurrence-1' }); + await current.close(); + + const response = await current.get('project-1', 'scanner-finding:occurrence-1'); + expect(acceptedControlIds(response.decisions).sort()).toEqual(['8.2', '8.3']); + expect(response.run?.candidates.map(item => item.controlId)).toEqual(['8.2']); + expect(response.earlierControls).toEqual([{ controlId: '8.3', annexAControlId: 'A.8.3', title: semantic('8.3').title }]); + + // Rejecting the earlier control is grounded by the older run that retrieved it. + const earlier = response.decisions.find(item => item.controlId === '8.3')!; + const rejected = await current.decide('project-1', { resultProjectionId: 'scanner-finding:occurrence-1', controlId: '8.3', decision: 'rejected', expectedLatestDecisionId: earlier.id }); + expect(acceptedControlIds(rejected.decisions)).toEqual(['8.2']); + expect(rejected.decisions.find(item => item.controlId === '8.3' && item.previousDecisionId === earlier.id)).toMatchObject({ decision: 'rejected', mappingId: earlier.mappingId }); + }); + it('keeps a saved mapping readable when the retrieval index is unavailable', async () => { const repository = mockRepository([finding()]); const complete = vi.fn(async (_system: string, _input: string) => ({ value: { @@ -328,7 +417,8 @@ describe('regulatory mapping service', () => { expect(response.run).toMatchObject({ status: 'completed', counts: { filteredResultCount: 7, excludedNonScannerCount: 1, - eligibleFindingCount: 5, + // The two baseline comparisons of one GPO setting are one eligible finding, like its Work record. + eligibleFindingCount: 4, unsupportedFindingCount: 1, distinctRuleCount: 3, scheduledRuleCount: 3, @@ -354,6 +444,8 @@ describe('regulatory mapping service', () => { await svc.close(); const mapped = await svc.get('project-1', gpo.resultProjectionId); expect(mapped).toMatchObject({ generationEnabled: true, state: 'succeeded', run: { ruleKey: 'GPOs:registry:machine:software\policies\synthetic:mode' } }); + // The stored identity tuple must differ per GPO query version, or a version bump collides with the older row. + expect(mapped.run?.promptVersion).toBe(`${REGULATORY_MAPPING_PROMPT_VERSION}+${REGULATORY_GPO_QUERY_VERSION}`); expect(complete).toHaveBeenCalledTimes(1); expect(String(complete.mock.calls[0][1])).not.toContain('synthetic.account'); diff --git a/tests/exports/unified-result-jsonl.test.ts b/tests/exports/unified-result-jsonl.test.ts index cb2e23c..84e6ea8 100644 --- a/tests/exports/unified-result-jsonl.test.ts +++ b/tests/exports/unified-result-jsonl.test.ts @@ -49,7 +49,7 @@ describe('filtered Result DB JSONL export', () => { recordType: 'manifest', format: RESULT_DB_JSONL_FORMAT, schemaVersion: RESULT_DB_JSONL_SCHEMA_VERSION, - disclosurePolicyVersion: '3.5.0', + disclosurePolicyVersion: '3.6.0', handling: RESULT_DB_EXPORT_HANDLING, generatedOn: '2026-09-03', resultCount: 2, diff --git a/tests/features/results-browser-timeslot-scope.test.ts b/tests/features/results-browser-timeslot-scope.test.ts index 50c80c2..b33759d 100644 --- a/tests/features/results-browser-timeslot-scope.test.ts +++ b/tests/features/results-browser-timeslot-scope.test.ts @@ -26,7 +26,7 @@ describe('Result DB timeslot scope', () => { expect(browserSource).toContain("'Export JSON'"); expect(browserSource).toContain('queryAllUnifiedResults'); expect(browserSource).toContain('buildUnifiedResultJsonl(scopedQuery, results, generatedAt)'); - expect(browserSource).toContain('resultsToPrivacySafeExportJson(results)'); + expect(browserSource).toContain('complianceAiExportJson(results)'); expect(browserSource).toContain('while (loadAll && items.length < firstPage.total)'); expect(browserSource).toContain('limit: MAX_QUERY_PAGE_SIZE'); expect(browserSource).toContain('>Load more'); diff --git a/tests/local-only-check.mjs b/tests/local-only-check.mjs index 71905c8..b838838 100644 --- a/tests/local-only-check.mjs +++ b/tests/local-only-check.mjs @@ -121,8 +121,8 @@ assert.match(liteLlmClient, /Authorization: `Bearer \$\{this\.apiKey\}`/, 'the w assert.match(liteLlmClient, /maximumOutputBytes/, 'work LiteLLM responses must remain size-bounded'); assert.match(liteLlmClient, /controller\.abort\(\)/, 'work LiteLLM requests must retain an enforced timeout'); assert.doesNotMatch(apiClient, /ASSESSOR_AI_(?:BASE_URL|API_KEY)/, 'the browser API client must not receive the LiteLLM origin or secret'); -const aiOverviewService = read(join('server', 'ai', 'ai-overview-service.ts')); -assert.match(aiOverviewService, /resultsToPrivacySafeExportJson/, 'AI generation must reuse the privacy-safe Result DB export'); +const aiOverviewService = read(join('server', 'compliance', 'ai-overview-service.ts')); +assert.match(aiOverviewService, /aiOverviewInputJson/, 'AI generation must reuse the privacy-safe input that the Result DB Export JSON copies'); assert.doesNotMatch(aiOverviewService, /console\.(?:log|warn|error)/, 'AI generation must not log prompts, provider bodies, or secrets'); const postgresRepository = read(join('server', 'postgres', 'postgres-assessor-repository.ts')); assert.doesNotMatch(postgresRepository, /\\$\\{query\\./, 'PostgreSQL queries must not interpolate fields directly from request queries'); diff --git a/tests/persistence/technical-explanation-result-db.test.ts b/tests/persistence/technical-explanation-result-db.test.ts new file mode 100644 index 0000000..79530b0 --- /dev/null +++ b/tests/persistence/technical-explanation-result-db.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest'; +import type { FindingOccurrence, ScanRun } from '../../src/domain/canonical-model'; +import { discloseResultDbProjectionWithTrace } from '../../src/domain/result-db-disclosure'; +import { technicalExplanation, type UnifiedResultProjection } from '../../src/domain/unified-result'; +import { buildRegulatoryFindingQuery } from '../../src/compliance/regulatory-query'; +import { LocalAssessorRepository } from '../../src/persistence/local-assessor-repository'; + +// A scanner's technical explanation (PingCastle today) is public rule guidance. It enters the Result DB +// as its own field, redacted as strictly as a description, guarded against client names and traced; +// tools without one leave it empty, and unknown sources stay fail-closed. +const stamp = '2026-10-05T08:00:00.000Z'; +const confidence = { level: 'high' as const, score: 1, reasons: ['synthetic'] }; +const provenance = { decisionSource: 'source-native' as const, confidence, mappingVersion: '1.0.0' }; +const explanation = 'The AdminSDHolder ACL is copied to protected accounts every 60 minutes; Dummy Customer changed it on 2026-01-03.'; + +describe('technical explanation field', () => { + it('reads the extractor key or the source label and ignores other or empty values', () => { + expect(technicalExplanation([{ key: 'technical-explanation', label: 'Technical explanation', value: ' Native text. ' }])).toBe('Native text.'); + expect(technicalExplanation([{ key: 'column-3', label: 'Technical Explanation', value: 'Table text.' }])).toBe('Table text.'); + expect(technicalExplanation([{ key: 'description', label: 'Description', value: 'Other text.' }])).toBeUndefined(); + expect(technicalExplanation([{ key: 'technical-explanation', label: 'Technical explanation', value: ' ' }])).toBeUndefined(); + expect(technicalExplanation([{ key: 'technical-explanation', label: 'Technical explanation', value: null }])).toBeUndefined(); + }); + + it('is redacted like a description, traced, and withheld for unknown or GPO sources', () => { + const row = { resultProjectionId: 'scanner-finding:dummy', sourceKind: 'scanner-finding', sourceResultId: 'dummy', projectId: 'dummy-project', + collectionRunId: 'dummy-run', sourceTool: 'pingcastle', sourceRuleId: 'P-AdminSDHolder', title: 'AdminSDHolder ACL modified', + technicalExplanation: explanation, severityOriginal: 'high', severityNormalized: 'high', outcomeOriginal: 'fail', outcomeNormalized: 'fail', + technicalExecutionState: 'completed', resultType: 'compliance_check', isIoe: false, isDeviation: true, lifecycleState: 'new', + mappingConfidence: 1, importedAt: stamp, createdAt: stamp, frameworkReferences: [], categories: [] } as unknown as UnifiedResultProjection; + const disclosed = discloseResultDbProjectionWithTrace(row); + expect(disclosed.technicalExplanation).toContain('AdminSDHolder ACL is copied to protected accounts'); + expect(disclosed.technicalExplanation).not.toContain('60'); + expect(disclosed.technicalExplanation).not.toContain('2026-01-03'); + expect(disclosed.disclosureTrace?.redactions.some(entry => entry.field === 'technicalExplanation')).toBe(true); + expect(JSON.stringify(disclosed.disclosureTrace)).not.toContain('AdminSDHolder'); + + expect(discloseResultDbProjectionWithTrace({ ...row, sourceTool: 'unknown-scanner' }).technicalExplanation).toBeUndefined(); + expect(discloseResultDbProjectionWithTrace({ ...row, sourceKind: 'policy-evaluation' } as UnifiedResultProjection).technicalExplanation).toBeUndefined(); + }); + + it('reaches the stored Result DB row of a PingCastle finding with the client name withheld, and the ISO query', async () => { + const repository = new LocalAssessorRepository(); + await repository.saveProject({ id: 'dummy-project', name: 'Dummy Customer', createdAt: stamp, updatedAt: stamp, attributes: {} }); + await repository.saveAssessment({ id: 'dummy-assessment', projectId: 'dummy-project', name: 'Dummy', createdAt: stamp, updatedAt: stamp }); + const scanRun: ScanRun = { id: 'dummy-run', assessmentId: 'dummy-assessment', sequence: 1, label: 'Dummy', + tool: { id: 'pingcastle', name: 'PingCastle' }, importedAt: stamp }; + const occurrence: FindingOccurrence = { id: 'dummy-occurrence', definitionId: 'dummy-definition', scanRunId: scanRun.id, importFileId: 'dummy-file', + sourceIdentity: { tool: scanRun.tool, originalRuleId: 'P-AdminSDHolder', canonicalRuleId: 'P-AdminSDHolder', originalTitle: 'AdminSDHolder ACL modified' }, + sourcePosition: { rowIndex: 1 }, + originalFields: [{ key: 'technical-explanation', label: 'Technical explanation', value: explanation }], + fieldMappings: [], severity: { originalValue: 'High', normalizedValue: 'high', provenance }, + status: { originalValue: 'Matched', normalizedValue: 'failed', provenance } }; + const session = await repository.beginImport({ sessionId: 'dummy-session', projectId: 'dummy-project', assessmentId: 'dummy-assessment', scanRun }); + await session.stage({ + importFiles: [{ id: 'dummy-file', scanRunId: scanRun.id, fileName: 'dummy.html', format: 'HTML', sizeBytes: 10, importedAt: stamp, + detectedTool: scanRun.tool, detection: confidence, + processedWith: { parserId: 'html', parserVersion: '1.0.0', profileId: 'pingcastle', profileVersion: '1.0.0', mappingVersion: '1.0.0' }, + sourceArtifact: { kind: 'not-retained', reason: 'synthetic' } }], + findingDefinitions: [{ id: 'dummy-definition', canonicalKey: 'pingcastle:P-AdminSDHolder', + identity: { version: 1, strategy: 'tool-rule-id', durable: true, evidence: ['synthetic'] }, + title: 'AdminSDHolder ACL modified', category: 'Privileged Accounts', frameworkReferences: [], createdAt: stamp, updatedAt: stamp }], + findingOccurrences: [occurrence] + }); + await session.commit(); + + const stored = await repository.getUnifiedResultById('scanner-finding:dummy-occurrence', 'dummy-project'); + expect(stored?.technicalExplanation).toContain('AdminSDHolder ACL is copied to protected accounts'); + expect(stored?.technicalExplanation).not.toContain('Dummy Customer'); + expect(stored?.technicalExplanation).not.toContain('2026-01-03'); + // Internal analysis keeps the complete original. + const internal = await repository.queryUnifiedResults({ audience: 'internal-analysis', projectId: 'dummy-project' }); + expect(internal.items[0].technicalExplanation).toBe(explanation); + // The ISO query reads the disclosed row, so it carries the redacted technical explanation. + const query = buildRegulatoryFindingQuery(stored!)!.query; + expect(query).toContain('Technical explanation: The AdminSDHolder ACL is copied to protected accounts'); + expect(query).not.toContain('Dummy Customer'); + }); +}); diff --git a/tests/server/assessor-http-server.test.ts b/tests/server/assessor-http-server.test.ts index d0fc98a..41a718d 100644 --- a/tests/server/assessor-http-server.test.ts +++ b/tests/server/assessor-http-server.test.ts @@ -59,7 +59,7 @@ describe('local ASSESSOR HTTP server', () => { const health = await fetch(`${baseUrl}/api/v1/health`); expect(await health.json()).toEqual({ - status: 'ok', application: 'ASSESSOR', productVersion: '2.18.0', apiVersion: 1, repositoryRevision: 0, persistence: 'memory' + status: 'ok', application: 'ASSESSOR', productVersion: '2.19.0', apiVersion: 1, repositoryRevision: 0, persistence: 'memory' }); expect(health.headers.get('access-control-allow-origin')).toBeNull(); }); diff --git a/tests/server/gpo-backup-ingestion.test.ts b/tests/server/gpo-backup-ingestion.test.ts index d1f797b..043497a 100644 --- a/tests/server/gpo-backup-ingestion.test.ts +++ b/tests/server/gpo-backup-ingestion.test.ts @@ -275,6 +275,43 @@ describe('GPO backup metadata ingestion', () => { expect(result.backupInstances[0].inventory.diagnosticCodes).toContain('gpo-metadata-gpoId-conflict'); }); + describe('domain GUID beside DNS name in a standard Backup-GPO folder', () => { + const domainGuid = '55555555-5555-4555-8555-555555555555'; + const backupXml = (domain: string) => ``; + const bkupInfo = (guid: string) => ``; + const gpreport = `{${gpoId}}synthetic.exampleSynthetic`; + const folder = (files: Entry[]) => ingest(zip([...files, + { path: `{${backupId}}/DomainSysvol/GPO/Machine/Microsoft/Windows NT/SecEdit/GptTmpl.inf`, content: '[System Access]\r\nMinimumPasswordLength=14' }])); + + it('resolves the GPO with the domain GUID instead of blocking it as a conflict', () => { + const result = folder([ + { path: `{${backupId}}/Backup.xml`, content: backupXml('synthetic.example') }, + { path: `{${backupId}}/bkupInfo.xml`, content: bkupInfo(domainGuid) }, + { path: `{${backupId}}/gpreport.xml`, content: gpreport } + ]); + expect(result.backupInstances[0]).toMatchObject({ + identity: { status: 'resolved', gpoId, backupId }, + domain: { status: 'resolved', key: `domain-guid:${domainGuid}` }, + metadataStatus: 'complete', + inventory: { status: 'complete' } + }); + expect(result.backupInstances[0].inventory.entries.every(entry => entry.disposition === 'supported')).toBe(true); + expect(result.backupInstances[0].inventory.diagnosticCodes).not.toContain('gpo-metadata-domainKey-conflict'); + }); + + it('still blocks two different DNS names beside a domain GUID', () => { + const result = folder([ + { path: `{${backupId}}/Backup.xml`, content: backupXml('other.example') }, + { path: `{${backupId}}/bkupInfo.xml`, content: bkupInfo(domainGuid) }, + { path: `{${backupId}}/gpreport.xml`, content: gpreport } + ]); + expect(result.backupInstances[0]).toMatchObject({ + identity: { status: 'unresolved', reasonCode: 'metadata-conflict' }, domain: { status: 'unknown' } + }); + expect(result.backupInstances[0].inventory.diagnosticCodes).toContain('gpo-metadata-domainKey-conflict'); + }); + }); + it('detects but never returns a GPP-style secret value from metadata XML', () => { const result = ingest(zip([ { path: 'one/Backup.xml', content: metadata(gpoId, backupId, '') }, diff --git a/tests/server/gpo-backup-job-processor.test.ts b/tests/server/gpo-backup-job-processor.test.ts index 0cb1d3b..d6c2eb3 100644 --- a/tests/server/gpo-backup-job-processor.test.ts +++ b/tests/server/gpo-backup-job-processor.test.ts @@ -71,7 +71,7 @@ describe('lease-based GPO backup job processor', () => { await expect(repository.listProcessingRuns(artifacts[0].id)).resolves.toMatchObject([{ id: result?.processingRunId, projectId: project.id, artifactId: artifacts[0].id, status: 'partial', processors: [ - { kind: 'artifact-parser', id: 'gpo-backup-ingestion', version: '2.2.5' }, + { kind: 'artifact-parser', id: 'gpo-backup-ingestion', version: '2.2.6' }, { kind: 'normalizer', id: 'gpo-technical-registry-target', version: '1.0.0' } ] }]); @@ -171,7 +171,7 @@ describe('lease-based GPO backup job processor', () => { expect(policyAnalyses[0]).toMatchObject({ run: { sourceProcessingRunId: result?.processingRunId, - analyzer: { id: 'gpo-policy-identity-analysis', version: 'gpo-backup-policy-analysis:2.2.5' }, + analyzer: { id: 'gpo-policy-identity-analysis', version: 'gpo-backup-policy-analysis:2.2.6' }, identityResolver: { id: 'gpo-technical-registry-target', version: '1.0.0' }, observationCount: expect.any(Number), resolvedCount: 3,