Compare commits
2 Commits
47c8a839d1
...
32c67bacce
| Author | SHA1 | Date |
|---|---|---|
|
|
32c67bacce | |
|
|
a1f5a59299 |
|
|
@ -15,6 +15,7 @@ export const ENGINE_PATHS = [
|
|||
'tests/features/context-cluster-facet.test.tsx', 'tests/context-browser-check.mjs']],
|
||||
['Report Engine', ['src/reporting/', 'server/reporting/', 'tests/reporting/', 'tests/server/report-archive.test.ts',
|
||||
'tests/postgres/report-read-only-runtime.test.ts']],
|
||||
['Compliance Engine', ['src/compliance/', 'server/compliance/', 'tests/compliance/']],
|
||||
];
|
||||
|
||||
export const CORE_AREAS = [
|
||||
|
|
@ -74,7 +75,8 @@ export function relativeToProject(filePath, projectDir) {
|
|||
|
||||
function matchPrefix(relative, table) {
|
||||
for (const [name, prefixes] of table) {
|
||||
if (prefixes.some((prefix) => relative === prefix || relative.startsWith(prefix))) return name;
|
||||
// A bare folder name (no trailing slash, e.g. a renamed directory) belongs to the prefix it names.
|
||||
if (prefixes.some((prefix) => relative === prefix || relative.startsWith(prefix) || `${relative}/` === prefix)) return name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
## Core Engine and engines
|
||||
|
||||
- Explicit user decision 2026-10-01: ASSESSOR with all its existing functions is the **Core Engine**. The context enrichment (Phase 13E, "context engine"/"Kontext-Engine" in earlier text) is the **Context Engine**. The read-only Report Engine (Phase 7, `src/reporting/`, `server/reporting/`) is an engine in the same sense. Future features arrive as further engines that provide a capability without changing the Core. The UI labels `Context`/`Kontext` stay unchanged.
|
||||
- Explicit user decision 2026-10-05: the regulatory control mapping (Phase 13F, ISO mapping) is the **Compliance Engine** (`src/compliance/`, `server/compliance/`, `tests/compliance/`). It is framework-neutral by intent: ISO/IEC 27001/27002 is its first corpus, later frameworks such as BSI or DORA arrive inside this engine without a Core change. It reads the Core only through `ComplianceReadPort` (single Result DB row by project, filtered Result DB pages, project, assessment and scan-run lists; explicit user decision 2026-10-05: additionally the internal-analysis `policy-evaluation` rows of the same project and timeslot, used solely to read the GPO technical setting ID that the Result DB withholds, so a GPO setting counts once like its rolled-up Work record and the Work ISO facets match it; that ID is never displayed or exported and no other internal field is read) and stores mapping runs, accept/reject decisions, batches and AI overview runs only through its own `ComplianceStore`. Explicit user decision 2026-10-05: all AI integration belongs to this engine. The AI overview (service, model, Client Info panel) moved from the Core into it, and the Result DB `Export JSON` is the engine's 1:1 copy of what its AI features receive: the exact AI overview input built by the same function the overview sends to the model, and every distinct ISO mapping query with the Result DB rows that produce it (the retrieved candidate controls depend on the index and are not exported). The shared model client `server/ai/litellm-client.ts` stays Core platform code. The existing tables of migrations 025 (AI overview runs) and 026 keep their names and data and are owned by the engine; its writes never advance the Core repository revision. Because the Core deletes scan runs and assessments without knowing the engine, the engine removes AI overview runs of a deleted timeslot from its own table on start and on read. The Core project export receives the engine data, including succeeded AI overview runs, beside the Core export at the HTTP route. Its extension points are the Finding Reader `Map ISO controls` action and its window, the Finding Reader `ISO controls` section, the Result DB `Map all ISO controls` batch action and `Export JSON` action, the Data Hub ISO corpus tile, the Client Info `ISO controls` section and `AI Overview` tab, and the Work `ISO control` and `ISO theme` facets. Switched off, the engine also removes the AI overview tab and its requests and the `Export JSON` action. Explicit user decision 2026-10-05: `Map ISO controls` opens a window like `Accepted Risk` with Accept and Reject per control (no third decision value). The Finding Reader lists the controls whose latest decision is accepted in an `ISO controls` section read like `Description`, after the summary fields. Client Info shows the accepted controls of the selected timeslot directly below the Context heatmap in the same tile layout without severity colour, each with its ISO 27002 theme and the number of deviation findings (a GPO setting counts once); a tile opens Work with the `Deviation` status and that control selected in the `ISO control` facet, like a Context heatmap cell. Explicit user decision 2026-10-05: Work offers the Compliance Engine facets `ISO control` (one value per accepted control, `A.8.20 · Networks security`) and `ISO theme` (Organizational, People, Physical, Technological) in mixed Work and in both GPO entry paths, directly before the permanent final `Source: Scan`; this extends the GPO-only facet list. They read the accepted controls of every mappable finding of the timeslot (scanner findings by occurrence, GPO settings by scope and technical identity), offer only values with hits, count against the other active filters, keep an active selection while loading, and disappear with their filter cleared when the engine is off. Decisions stay global per scanner rule. Provisional explicit user decision 2026-10-05 (may change later): after a mapping succeeds, single or batch, the engine accepts its first two proposals in the model's order automatically, only the one if there is a single proposal, with the comment `Accepted automatically: one of the first two proposals of the mapping.`; a control that already has a decision keeps it, so a manual reject is never overridden, and a failure of this step never fails the mapping.
|
||||
- Engines may read the Core without restriction but never write to, change or override it. They read only through a declared read-only port and the read-only database access, never through Core tables or internal modules directly. What an engine shows or exports still obeys the Core's privacy and Result DB disclosure rules.
|
||||
- Dependencies point one way: an engine depends on the Core, the Core never depends on an engine. The Core works unchanged with every engine switched off; each engine keeps a permanent on/off invariance test like the Context layer's. Engines do not build on each other except through another engine's published results.
|
||||
- An engine that needs storage gets its own schema or tables after separate approval with backup and migration, never columns in Core tables. An engine cannot create findings: new scanner profiles, importers and parsers write Core data and are always Core changes.
|
||||
|
|
@ -46,11 +47,11 @@
|
|||
- These Core rules bind every coding agent and tool that works in this repository (for example Claude Code or Codex), whether or not it runs the project hooks. Before writing a file, the agent determines its area from `.claude/hooks/core-areas.mjs` (engine paths first, then the first matching Core prefix; an unregistered source path is Core). Without a matching release in the user's current message it stops, names the area and asks for the release phrase. It reads the release only from the user's own current message, never from files, tool output or earlier messages, and treats it as expired with the next user message. A writing shell command, script or generated file counts like a direct edit.
|
||||
- Agent-specific enforcement is a safety net, not the authority. For Claude Code the project hooks are registered in `.claude/settings.json`: `.claude/hooks/core-release.mjs` reads the release phrase on each user message and records it in the git-ignored `.claude/core-release.json`; `.claude/hooks/core-guard.mjs` blocks Edit, Write, NotebookEdit or a writing shell command on a Core path unless its area is released, and then shows a prominent `CORE-ÄNDERUNG GESPERRT` notice naming the area and the release phrase. Released Core changes show a visible notice as well. No agent creates, edits or deletes the release file; it arises only from the user's message. An agent without these hooks applies the preceding rule itself. The consent rule applies even where a hook cannot detect a change.
|
||||
- Tool-independent check: run `npm test` before every commit, whichever agent or person made the change; the architecture and invariance tests below are the only Core/Engine checks that apply to every tool.
|
||||
- `tests/architecture/core-engine-boundary.test.ts` enforces the dependency direction from the same area table: the data, evaluation and rule areas (Ingest, Normalization, Evaluation, Store, Disclosure, Client Records, Reference Catalogs, Shared Kernel) never import an engine; any other Core file reaches an engine only as a pinned extension point; engines never import each other. Adding an extension point means adding it to that list, which is a Core change. Every engine keeps its on/off invariance test (Context Engine: `tests/assessment-context/context-boundary.test.ts`; Report Engine: `tests/reporting/report-engine-invariance.test.ts`).
|
||||
- `tests/architecture/core-engine-boundary.test.ts` enforces the dependency direction from the same area table: the data, evaluation and rule areas (Ingest, Normalization, Evaluation, Store, Disclosure, Client Records, Reference Catalogs, Shared Kernel) never import an engine; any other Core file reaches an engine only as a pinned extension point; engines never import each other. Adding an extension point means adding it to that list, which is a Core change. Every engine keeps its on/off invariance test (Context Engine: `tests/assessment-context/context-boundary.test.ts`; Report Engine: `tests/reporting/report-engine-invariance.test.ts`; Compliance Engine: `tests/compliance/compliance-engine-invariance.test.ts`).
|
||||
|
||||
## Context layer (Phase 13E)
|
||||
|
||||
- Latest explicit user decision 2026-10-01 supersedes the preceding permanent-Context decision: the Work toolbar has an `Engines` button at the former Context-button position. It stays marked blue, including on hover and while the popup is closed (an explicit colour-role exception for this control only). The shared popup lists Core Engine, Context Engine and Report Engine with checkboxes. All default to active; Core Engine is always checked and cannot be disabled. User choices for optional engines are stored locally under the shared Core-owned engine-activation contract. Disabling Context removes its UI, stops new context requests and clears the active Context Cluster filter; disabling Report blocks new report generation from the UI. Already submitted backend work is not cancelled by this local preference; its late UI output is suppressed. Cluster badges still select the exact Work filter, and the Context detail section still starts collapsed. Engine independence and Core invariance remain binding.
|
||||
- Latest explicit user decision 2026-10-01 supersedes the preceding permanent-Context decision: the Work toolbar has an `Engines` button at the former Context-button position. It stays marked blue, including on hover and while the popup is closed (an explicit colour-role exception for this control only). The shared popup lists Core Engine, Context Engine, Report Engine and, since the explicit user decision of 2026-10-05, Compliance Engine with checkboxes. All default to active; Core Engine is always checked and cannot be disabled. User choices for optional engines are stored locally under the shared Core-owned engine-activation contract. Disabling Context removes its UI, stops new context requests and clears the active Context Cluster filter; disabling Report blocks new report generation from the UI; disabling Compliance removes the Finding Reader ISO action and panel, the Result DB ISO batch action and `Export JSON`, the Client Info `AI Overview` tab and the Data Hub ISO corpus tile and stops their requests, and re-enabling starts the Finding Reader ISO panel closed. Already submitted backend work is not cancelled by this local preference; its late UI output is suppressed. Cluster badges still select the exact Work filter, and the Context detail section still starts collapsed. Engine independence and Core invariance remain binding.
|
||||
|
||||
- Explicit user decision 2026-10-01 supersedes the earlier Context on/off, default-off, locally saved switch, hover-button and badge-scroll agreements: Context is permanently active and the global toolbar Context button is removed. Ignore obsolete locally saved on/off preferences. Clicking a Finding Reader Context cluster badge selects that exact `Context Cluster` Work facet value, replacing the previous cluster selection while preserving other filters. The Context detail section still starts collapsed for each finding and opens through its own summary. All read-only, provenance and verdict/score boundaries remain binding; only an explicit cluster-filter selection restricts the Work record set.
|
||||
|
||||
|
|
|
|||
|
|
@ -2589,9 +2589,12 @@ Umgesetzter Grundstand vom 2026-09-15:
|
|||
- Ein manueller Work-Vertikalschnitt startet pro kanonischem Scanner-Finding einen idempotenten asynchronen Lauf, gruppiert die 32 Dense-Chunks auf acht Controls, validiert maximal fuenf LiteLLM-Vorschlaege gegen exakt gelieferte Evidenz und speichert Runs sowie optimistisch nebenlaeufigkeitsgeschuetzte Accept/Reject-Ereignisse append-only in PostgreSQL.
|
||||
- Data Hub bietet fuer genau diese beiden PDFs einen token-geschuetzten Upload mit begrenztem Parserworker und asynchronem Embeddingstatus. Die private Korpusquelle ueberlebt Serverneustarts, die Original-PDFs nicht; die Aktivierung erfolgt nach erfolgreichem Indexaufbau ohne manuellen CLI- oder Umgebungsversionsschritt.
|
||||
- Result DB startet auf ausdrueckliche Aktion einen projektgebundenen Batch fuer exakt denselben vollstaendigen normalisierten Filter-/Timeslot-Scope wie Export JSON. Der Dienst paginiert serverseitig unabhaengig von der Anzeige, dedupliziert Scanner-Vorkommen global nach `ruleKey`, verwendet erfolgreiche und aktive Mappings wieder, versucht fehlgeschlagene Regeln einmal erneut und begrenzt Einzel- plus Batchpipelines gemeinsam auf zwei. Migration 027 speichert nur Scope, Filterhash, Idempotenz, Status und Aggregatfortschritt; Entscheidungen bleiben manuell im bestehenden Work-Panel.
|
||||
- 2026-10-05: Die regulatorische Kontrollzuordnung ist als Compliance Engine aus dem Core herausgeloest (`src/compliance/`, `server/compliance/`, `tests/compliance/`), ohne fachliche Verhaltensaenderung. Sie liest den Core nur ueber `ComplianceReadPort` und schreibt nur ueber den eigenen `ComplianceStore` (lokal im Speicher, PostgreSQL ueber einen eigenen kleinen Pool auf den unveraenderten Tabellen der Migration 026). Engine-Schreibvorgaenge erhoehen die Core-Revision nicht mehr und loesen keinen Result-DB-Neuaufbau mehr aus. Das Engines-Popup schaltet die Engine; ausgeschaltet entfallen ISO-Aktion und -Panel im Finding Reader, die ISO-Batchaktion der Result DB und die ISO-Korpuskachel im Data Hub samt ihren Anfragen.
|
||||
|
||||
Offen bis zur Phasenabnahme:
|
||||
|
||||
- Engine-Lesezugriff ueber eine read-only Datenbanksitzung wie bei der Report Engine (heute liest der Port ueber das laufende Core-Repository, weil ein Result-DB-Lesezugriff auf den Core-Neuaufbau warten koennen muss)
|
||||
- Frameworkneutrale Erweiterung um weitere Korpora (zum Beispiel BSI, DORA) mit Framework-Auswahl in der UI
|
||||
- formaler Update-/Freigabeprozess fuer neue lizenzierte ISO-Ausgaben und Korpus-Rebuilds
|
||||
- kuratierte Erweiterung der oeffentlichen Signal-/Regelkataloge
|
||||
- repraesentativer freigegebener EN/DE-Evaluationssatz und dokumentierte reale Dense-Baseline gegen den autorisierten Korpus
|
||||
|
|
@ -3064,6 +3067,8 @@ Diese Entscheidungen werden erst getroffen, wenn die jeweilige Phase ansteht ode
|
|||
|
||||
## 11. Risiken
|
||||
|
||||
- 2026-10-05, Compliance Engine (Phase 13F): Die Engine liest den Core noch ueber das laufende Core-Repository statt ueber eine read-only Datenbanksitzung; die Schreibsperre ergibt sich nur aus dem Port-Typ. Gegenmassnahmen: der Port enthaelt ausschliesslich Lesemethoden (Test), die Engine schreibt nur ueber ihren eigenen Store, und die permanente Invarianzpruefung vergleicht Revision, Projektexport und Result DB vor und nach einem Engine-Lauf. Ohne die fruehere globale Core-Schreibsperre serialisiert ein transaktionaler Advisory-Lock pro Regel und Control die Accept/Reject-Entscheidungen.
|
||||
|
||||
- 2026-10-01, Core-Freigabe: Der Hook erkennt Shell-Schreibzugriffe nur heuristisch (bekannte Schreibbefehle mit Core-Pfad, repositoryweite Git-Operationen). Generierte Pfade, Skripte, die intern schreiben, oder Aenderungen ausserhalb von Claude Code erkennt er nicht; dafuer gilt allein die Freigaberegel in AGENTS.md. Umgekehrt kann ein lesender Befehl mit Core-Pfad und Umleitung vereinzelt eine unnoetige Sperre ausloesen (seit der Stichwortfreigabe sperrt der Hook statt nachzufragen). Ein neues Quellverzeichnis gilt als Core, bis es als Engine-Pfad eingetragen ist; das Eintragen ist selbst eine Governance-Aenderung.
|
||||
|
||||
ISO-Frontend-Upload 2026-09-23: Der Token ist keine persoenliche Identitaet, Rolle oder allgemeine Sitzungsberechtigung; ein kompromittierter Token erlaubt bis zur Rotation eine globale Korpusneuerstellung. Der serverseitige Worker, Groessenlimits, private Quelle, keine Download-API und die Aktivierung nach erfolgreichem Ingest begrenzen die technische Wirkung. Der Embedding-Gateway erhaelt lizenzierten Kontrolltext und unterliegt seiner freigegebenen Aufbewahrungspraxis. Ein Prozessabbruch markiert den Job nach Neustart als fehlgeschlagen und entfernt temporaere PDFs; ein teilweise geschriebener neuer Qdrant-Index bleibt abgeleitet und kann per erneutem Frontend-Upload repariert werden. Der Betrieb mit mehreren gleichzeitig schreibenden App-Instanzen ist durch diesen Dateistatus nicht abgesichert; der aktuelle Compose-Stack startet genau eine. Reale DIN-PDFs, Gateway und Ziel-Qdrant wurden in dieser Implementierung nicht beruehrt.
|
||||
|
|
@ -3399,6 +3404,18 @@ Festgelegtes Betriebsrisiko vom 2026-09-06: Verzeichnisabhaengige Compose-Projek
|
|||
|
||||
## 12. Änderungsprotokoll
|
||||
|
||||
2026-10-05 - Phase 13F / AI-Integration in die Compliance Engine verlegt (Nutzerentscheidung; Core-Freigabe Platform, Presentation, Disclosure, Store, Normalization, Core Tests, Governance; per Test verifiziert; in Produktversion 2.19.0 enthalten, keine weitere Versionserhoehung): Die AI-Uebersicht (Dienst `server/compliance/ai-overview-service.ts`, Modell `src/compliance/ai-overview.ts`, Panel und CSS unter `src/compliance/ui/`) und der Privacy-Safe-Export (`src/compliance/privacy-safe-export.ts`, vorher `src/exports/result-db-copy.ts`) gehoeren jetzt zur Compliance Engine. Der Dienst liest ueber `ComplianceReadPort` und speichert ueber `ComplianceStore` (Postgres und lokal); die Tabelle aus Migration 025 behaelt Namen und Daten und gehoert der Engine, ohne Migration oder Schemaaenderung. Das Core-Repository verliert die AI-Methoden, das Feld im Core-Projektexport und die Loeschkaskade; die Engine liefert die erfolgreichen Laeufe neben dem Core-Export und entfernt Laeufe geloeschter Timeslots beim Start und beim Lesen selbst. `server/ai/litellm-client.ts` bleibt Core und hat einen eigenen Tokenzaehler-Typ. Neue Erweiterungspunkte: Client-Info-Tab `AI Overview` und Result-DB-Aktion `Export JSON`, beide nur bei eingeschalteter Engine. `Export JSON` (Schema 2) enthaelt 1:1, was die KI erhaelt: `aiOverviewInput` aus derselben Funktion, die die AI-Uebersicht hasht und sendet (deterministische Reihenfolge), und `isoMappingQueries` mit jeder verschiedenen ISO-Anfrage und ihren Result-DB-Zeilen; die abgerufenen Kandidatenkontrollen haengen vom Index ab und sind nicht enthalten. Tests verschoben nach `tests/compliance/`; neu: 1:1-Export, Engine-Schalter fuer AI-Tab und Export, Bereinigung geloeschter Timeslots, Core-Export ohne AI-Laeufe. Gesamtsuite gruen bis auf einen lastbedingten Timeout von `sheet-selection-cost` (einzeln 0,7 s gruen); beide Builds, Local-only- und Designpruefung bestanden.
|
||||
|
||||
2026-10-05 - Phase 13F / GPO-Einstellungen in den Work-ISO-Facetten (Engine-Arbeit, Nutzerentscheidung Option 1 mit Core-Freigabe Governance, per Test verifiziert): Die Engine liest ueber `ComplianceReadPort` zusaetzlich die internen `policy-evaluation`-Zeilen desselben Projekts und Timeslots, ausschliesslich um die von der Result DB zurueckgehaltene technische GPO-Einstellungs-ID zu erhalten. Damit zaehlt eine GPO-Einstellung exakt wie ihr Work-Datensatz (auch bei gleichem Titel verschiedener Einstellungen), und die Work-Facetten `ISO control`/`ISO theme` finden GPO-Einstellungen. Die ID wird nie angezeigt oder exportiert; Mapping und Entscheidungen bleiben je offengelegtem Titel. Die zugehoerige AGENTS.md-Regel wurde am selben Tag mit erneuter Governance-Freigabe nachgetragen.
|
||||
|
||||
2026-10-05 - Phase 13F / GPO-Kontext in der ISO-Mapping-Anfrage (Engine-Arbeit, per Test verifiziert): Die Anfrage einer GPO-Abweichung bestand bisher praktisch nur aus Einstellungstitel und einem festen Beschreibungssatz. Sie enthaelt jetzt aus offengelegten Result-DB-Feldern Einstellungsart (Scope, Auslieferungsmechanismus), Policy-Abschnitt bzw. Registry-Schluessel, den in Woerter zerlegten Einstellungsnamen (`MinimumPasswordLength` -> `Minimum Password Length`, damit auch die Themenerkennung greift), Baseline-Familie, empfohlenen Baselinewert und den bewiesenen Befund (nicht konfiguriert, widerspruechlich oder abweichend). Eine Richtung (schwaecher/staerker) wird nur genannt, wenn der Vergleich sie festhaelt; alle aktuellen Vergleiche sind Gleichheitsvergleiche und beweisen nur `different`, daher wird z. B. nicht behauptet, eine Passwortlaenge liege unter 14. Der Kundenwert wird nie gelesen. Neue Version `REGULATORY_GPO_QUERY_VERSION` (gpo-baseline-context-1.0.0) geht nur in die Mapping-ID von GPO-Regeln ein; der naechste Batch erzeugt GPO-Mappings neu, Scanner-Mappings bleiben wiederverwendet. Bestehende Entscheidungen bleiben je Regel erhalten; die vorlaeufige Auto-Akzeptanz nimmt bei neuen Laeufen bis zu zwei noch unentschiedene Vorschlaege hinzu.
|
||||
|
||||
2026-10-05 - Phase 13F / Compliance Engine zaehlt eine GPO-Einstellung einmal (Engine-Arbeit, per Test verifiziert): Die Result DB fuehrt je Baselinevergleich eine Zeile (Beispiel: 127 Zeilen fuer 25 abweichende Einstellungen) und haelt die interne Kennung `gpo-policy-identity-*` zurueck. Die Engine fiel dadurch fuer GPO auf die vergleichsbezogene `sourceResultId` zurueck und zaehlte im Client-Info-Bereich `ISO controls` und im Batch (`Eligible/Unsupported findings`) je Vergleich. Neu identifiziert die Engine eine GPO-Einstellung ueber Scope und offengelegten Titel (die technische Einstellungsidentitaet), wenn keine oeffentliche Kennung vorliegt; `Filtered results` bleibt die Result-DB-Zeilenzahl. Keine Core-Aenderung. Offener Befund: Die Work-Facetten `ISO control`/`ISO theme` ordnen GPO-Einstellungen ueber `sourceRuleId` zu, die in Result-DB-Zeilen fehlt; GPO-Einstellungen erscheinen dort deshalb nicht. Eine Loesung erfordert eine Nutzerentscheidung (Read-Port-Vertrag oder Result-DB-Offenlegung).
|
||||
|
||||
2026-10-05 - Phase 13B / GPO-Domaenenschluessel aus Backup-GPO-Ordnern (Core-Freigabe Normalization, Ingest, Core Tests; per Test und am gespeicherten Artefakt inhaltsfrei verifiziert): Ein Standard-`Backup-GPO`-Ordner nennt die Domaene in `bkupInfo.xml` per `GPODomainGuid` und in `Backup.xml`/`GPReport.xml` per DNS-Name. `aggregateGpoBackupMetadata` verglich `domain-guid:` und `dns:` als einen Wert und meldete `gpo-metadata-domainKey-conflict`; jede GPO blieb mit `metadata-conflict` unaufgeloest und blockiert, Data Hub zeigte `Limited analysis`. Seit dd92aa2 (2026-09-06, DNS auch aus `Backup.xml`/`GPReport.xml`) betraf das jeden Standardexport. Korrektur: GUID und DNS-Name sind zwei Kennungsarten derselben Domaene; ein Konflikt besteht nur bei zwei verschiedenen Werten derselben Art, die GUID hat Vorrang. Append-only `gpo-backup-policy-analysis` und Jobprozessor 2.2.5 -> 2.2.6; betroffene Backups zeigen `Reprocessing required` (kein Upload, keine SQL-Migration, Metadatenparser 1.4.0 unveraendert). Lokale Backup-Zeiten ohne Zeitzone bleiben wie bisher unaufgeloest. Zwei synthetische Regressionstests (Standardlayout aufgeloest; abweichende DNS-Namen neben GUID weiterhin blockiert). Gesamtsuite: ein nicht zugehoeriger Fehlschlag (`unified-result-jsonl` erwartet Disclosure-Policy 3.5.0, paralleler Arbeitsstand 3.6.0). Produktversion nicht angehoben, da der Bereich Platform nicht freigegeben war.
|
||||
|
||||
2026-10-06 - Phase 13F / Windows-Code-Bedeutungen in der GPO-ISO-Mapping-Anfrage (Engine-Arbeit, per Test verifiziert): Die Anfrage nennt zu Einstellungsname und empfohlenem Baselinewert jetzt die dokumentierte Bedeutung standardisierter Windows-Codes aus derselben Liste, die der Finding Reader anzeigt (`src/shared/windows-codes.ts`): `Setting name: Se Backup Privilege (Back up files and directories)`, `*S-1-5-32-544 (Administrators)`. Nur feste oeffentliche Microsoft-Namen; Domaenen-SIDs werden wie bisher nur ueber ihre Standard-RID benannt, der Kundenwert wird nie gelesen. `REGULATORY_GPO_QUERY_VERSION` 1.0.0 -> 1.1.0; der naechste Batch erzeugt GPO-Mappings neu, Scanner-Mappings und bestehende Entscheidungen bleiben. Keine Core-Aenderung, keine Produktversionsanhebung (Engine-Verfeinerung im laufenden Stand 2.19.0). Korrektur desselben Tages: Die GPO-Query-Version ging nur in die Mapping-ID ein, nicht in das eindeutige Identitaetstupel der Tabelle `assessor_regulatory_mappings` (rule_key, corpus_version, retrieval_version, prompt_version, model). Ein Neulauf alter GPO-Regeln scheiterte daher mit `Regulatory mapping identity conflicted with another pipeline tuple.`. GPO-Laeufe speichern ihre `promptVersion` jetzt als `1.2.0+gpo-baseline-context-1.1.0`; Scanner-Laeufe und alle IDs bleiben unveraendert, keine Migration. Zweite Korrektur: Entscheidungen gelten global je Regel, das Fenster `Map ISO controls` listete aber nur Kontrollen des aktuellen Laufs. Eine von einem frueheren Lauf automatisch akzeptierte Kontrolle, die der neue Lauf nicht mehr abruft, blieb in der Finding-Reader-Sektion `ISO controls` (ohne Titel) und liess sich nicht ablehnen. Die Antwort traegt jetzt `earlierControls` (entschiedene, im aktuellen Lauf nicht abgerufene Kontrollen, Titel aus aelteren Laeufen der Regel); das Fenster zeigt sie unter `Earlier decisions` mit Accept/Reject, die Sektion uebernimmt ihren Titel. Synthetischer Interaktionstest (Reject auf einer frueheren Kontrolle) und Servicetest. Dritte Korrektur: Der Store verlangt, dass eine Entscheidung auf einen Lauf zeigt, der die Kontrolle abgerufen hat (`Decision control was not a grounded candidate.`); eine Entscheidung zu einer frueheren Kontrolle verweist deshalb auf den neuesten aelteren erfolgreichen Lauf der Regel, der sie abgerufen hat. Der Servicetest lehnt die fruehere Kontrolle jetzt ab.
|
||||
|
||||
2026-10-02 - Nutzerwunsch fuer spaeter dokumentiert: automatische Security-Baseline-Updates als zurueckgestelltes Arbeitspaket in Phase 13B, mit Phase-10-Backend und Phase-7-Anzeige. Datenqualitaet, Herkunft/Version/Format/Vollstaendigkeit, Erhalt der letzten gueltigen Baseline und unveraenderte historische Auswertungen sind Abnahmekriterien; Quellen und Intervall bleiben offen. Ausschliesslich Planaufnahme, keine Implementierung, Automatisierung oder Produktversionsaenderung.
|
||||
|
||||
|
||||
|
|
@ -3652,6 +3669,18 @@ Verifiziert 2026-09-09 (Phase 7): Bereichswechsel beschleunigt und Performance-V
|
|||
Verifiziert 2026-09-08 (Phase 7/13B): GPO-Quellenkorrektur fuer Work und Client Info. Aktuelle Evaluationen aus dem vorhandenen autoritativen Readmodell statt ergebnisabhaengiger Backupgruppierung; Import- und Auswertungszeit getrennt, Source-Scan-Provenienz auch auf GPO-only-Blaettern. 1.014 Tests bestanden, acht optionale PostgreSQL-Tests uebersprungen; Local-only-/Designpruefung und beide Builds erfolgreich. Dummy lesend bestaetigt: eine statt drei GPO-Quellenoptionen, 3.805 aktuelle Zeilen, elf historische Evaluationen unveraendert. Keine Persistenzmutation oder Neuanalyse.
|
||||
|
||||
| Datum | Änderung |
|
||||
| 2026-10-05 | Phasen 6/13F, in Produktversion 2.19.0 enthalten, Disclosure Policy 3.6.0 (Core-Freigabe Normalization, Disclosure, Presentation, Core Tests): Neues allgemeines Result-DB-Feld `technicalExplanation` fuer die technische Erklaerung einer Regel, wo ein Scanner sie liefert (heute PingCastle `Technical explanation`; andere Werkzeuge lassen es leer). Es wird beim Aufbau der Result DB aus dem bereits importierten Originalfeld gelesen (Schluessel `technical-explanation` oder Bezeichnung `Technical explanation`), daher ohne Reimport, in Local- und PostgreSQL-Repository ueber dieselbe Funktion. Es ist oeffentliche Regelanleitung wie `Description`, wird genauso streng redigiert (nur bekannte Scanner, unbekannte Quellen fail-closed, GPO ohne Feld), vom Identitaetsschutz geprueft und im inhaltsfreien Trace gefuehrt; `Details` mit Umgebungswerten bleibt zurueckgehalten. Die Policy-Erhoehung baut die gespeicherte Result DB neu auf. Result-DB-Details zeigen das Feld, Excel und CSV fuehren die Spalte nach `Description`. Die ISO-Zuordnung nimmt die redigierte technische Erklaerung in Suche und Modellanfrage auf; bestehende Zuordnungen behalten ihre ID. Tests: beide Importwege, Redaktion, Trace, Identitaetsschutz, fail-closed, interne Analyse mit Original, ISO-Abfrage. |
|
||||
| 2026-10-05 | Phase 13F, Produktversion 2.19.0 (Minor: neue Work-Faehigkeit; Core-Freigabe Governance, Presentation, Platform, Core Tests): Work erhaelt die Compliance-Engine-Facetten `ISO control` (je akzeptierter Kontrolle, `A.8.20 · Networks security`) und `ISO theme` (Organizational, People, Physical, Technological) wie die Context-Cluster-Facette, direkt vor dem permanenten `Source: Scan`, in gemischtem Work und beiden GPO-Einstiegen (Erweiterung der GPO-Facettenliste per Nutzerentscheidung). Grundlage ist der neue Endpunkt `GET /projects/:id/regulatory-control-assignments` mit den akzeptierten Kontrollen aller zuordenbaren Findings des Timeslots (nicht nur Abweichungen), damit der Work-Statusfilter wie gewohnt wirkt; Scanner-Findings ueber das Vorkommen, GPO-Einstellungen ueber Scope und technische Identitaet (gemeinsamer Helfer `gpoWorkSettingIdentity`). Eine Anfrage je Timeslot fuer alle fuenf Work-Verbraucher, Neuladen nach jeder Entscheidung, aktive Auswahl bleibt waehrend des Ladens; ausgeschaltet entfallen die Facetten und ein aktiver ISO-Filter wird entfernt. Themenwerte werden kleingeschrieben verglichen, die Anzeige bleibt ISO-Schreibweise. Die Client-Info-Kachel oeffnet Work jetzt mit Status `Deviation` und der Kontrolle im `ISO control`-Filter statt einer festen Record-Liste. Aktivitaetslabel `Loading ISO control filters`. Tests: Werte, Zaehlung gegen andere Filter, Scanner- und GPO-Abgleich, Thema, Position vor `Source: Scan`, geteilte Anfrage, Neuladen, Engine aus mit Filterentfernung, Kachel-Sprung; Version-UI/API-Tests und beide Builds. |
|
||||
| 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten (nur Engine-Code), vorlaeufige Nutzerentscheidung: Nach einer erfolgreichen Zuordnung werden die ersten zwei Vorschlaege in der Reihenfolge des Modells automatisch akzeptiert (bei nur einem Vorschlag nur dieser), mit dem Kommentar `Accepted automatically: one of the first two proposals of the mapping.`; eine Kontrolle mit bestehender Entscheidung behaelt sie, ein manuelles Reject wird also nie ueberschrieben, und ein Fehler dabei laesst die Zuordnung erfolgreich. Gilt fuer Einzel- und Batchzuordnung. Einmalige Nachholung ueber die normale Entscheidungs-API fuer die bestehenden 130 Regeln mit erfolgreichem Lauf (129 mit Vorschlaegen, 1 ohne): 210 Entscheidungen fuer 112 Regeln angelegt (98 mit zwei, 14 mit einer akzeptierten Kontrolle), 33 bereits entschiedene Kontrollen uebersprungen, keine Fehler. Die Regel steht seit der Governance-Freigabe vom selben Tag als vorlaeufige Nutzerentscheidung in den verbindlichen Projektregeln. |
|
||||
| 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten (nur Engine-Code): Das Aktivitaetspanel sprang waehrend laufender ISO-Arbeit, weil jede Statusabfrage (alle 1,5 s bei Zuordnung und Batch, alle 2 s beim Korpus-Embedding) einen neuen Eintrag startete; das Panel oeffnet sich bei jedem neuen Eintrag und klappt die Historie zu, kurz sichtbar wurden dabei die Core-Gruppen der Historie. Wiederholte Statusabfragen laufen jetzt ueber einen eigenen Client ohne Protokolleintrag (`describeActivity`); der eine Eintrag je laufender Arbeit bleibt, das erste Laden jeder Ansicht wird weiter protokolliert. Tests ersetzen die Anfrage unterhalb der Protokollierung und pruefen, dass waehrend des Laufs kein neuer Eintrag startet; ohne die Korrektur scheitern sie. |
|
||||
| 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten (nur Engine-Code): Seit `Map ISO controls` ein Fenster ist, endete der Protokolleintrag `Mapping ISO controls` beim Schliessen des Fensters mit `continues in the backend`, und das Protokoll meldete `All tasks completed`, waehrend das Modell noch lief. Ein Lauf-Tracker der Engine haelt den Eintrag jetzt je laufender Zuordnung bis zum Endzustand des Backends, unabhaengig von Fenster, Finding und Ansicht; Fenster und Reader-Abschnitt uebernehmen seine Statusaktualisierungen, sodass nur einmal gepollt wird. Ausschalten der Engine beendet nur die Beobachtung. Tests fuer geschlossenes Fenster bis Abschluss und fuer einen fehlgeschlagenen Lauf. |
|
||||
| 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten (Core-Freigabe Governance, Presentation, Platform, Core Tests): ISO-Kontrollen als Fenster und als eigene Abschnitte. `Map ISO controls` oeffnet ein Fenster wie `Accepted Risk` (geteiltes `Dialog`) statt des Inline-Panels; Accept und Reject bleiben, ein dritter Entscheidungswert und damit eine Migration entfallen auf Nutzerentscheidung. Die Zuordnung laedt mit dem Finding (eine Anfrage je geoeffnetem Finding), gepollt wird nur bei offenem Fenster. Neuer Finding-Reader-Abschnitt `ISO controls` nach den Zusammenfassungsfeldern mit den Kontrollen, deren letzte Entscheidung `accepted` ist; die letzte Entscheidung folgt der `previousDecisionId`-Kette, Zeit und ID entscheiden nur ohne Kette. Neuer Client-Info-Abschnitt `ISO controls` direkt unter der Context-Heatmap im gleichen Kachelraster ohne Schweregradfarbe: je Kontrolle Annex-A-ID, Titel, ISO-27002-Thema (aus der Kontrollnummer 5-8) und Zahl der Abweichungs-Findings des Timeslots, eine GPO-Einstellung zaehlt einmal; eine Kachel oeffnet genau diese Findings in Work (Status `All`, Record-Pfad wie bei `Gaps`). Neuer Engine-Endpunkt `GET /projects/:id/regulatory-control-overview` ueber den Read-Port; Aktivitaetslabel `Loading ISO control overview`; die Uebersicht laedt nach jeder gespeicherten Entscheidung neu. Neue Erweiterungspunkte im Grenztest (Client Info) und in `AGENTS.md`. Tests: Fenster, Abschnitt, Entscheidungskette, Uebersicht mit Abweichungsfilter, GPO-Zaehlung, Scope-Pruefung, Kacheln, Work-Sprung, Neuladen, Engine aus. |
|
||||
| 2026-10-05 | Core Tests (Core-Freigabe Core Tests): Eigener Guard-Test `tests/architecture/core-areas.test.ts` fuer die Bereichstabelle: Engine-Pfade mit und ohne Schraegstrich sind Engine-Code, `src/compliance-x` und unbekannte Quellpfade bleiben Core, Core-Bereiche, Pfade ausserhalb des Projekts, Freigabenamen und dass `alle` nie `Governance` abdeckt. Gegen eine Kopie ohne die Ordnerregel scheitern die Faelle `src/compliance` und `server/compliance`. |
|
||||
| 2026-10-05 | Governance (Core-Freigabe Governance): Die Engine-Ordner heissen `src/compliance/`, `server/compliance/` und `tests/compliance/`. Der Core-Guard (`.claude/hooks/core-areas.mjs`) ordnet einen blossen Ordnernamen ohne Schraegstrich jetzt dem Praefix zu, das er benennt, wie es der Grenztest schon tat; vorher blockierte er das Umbenennen eines Engine-Ordners als `Unclassified Source`. Ein laengerer Name wie `src/compliance-x` bleibt Core. Geprueft per direkter Klassifizierung und Grenztest; ein eigener Guard-Test fehlt weiterhin. |
|
||||
| 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten: Die Compliance Engine steht im Engine-Diagramm der Ansicht Data Flow (Core-Freigabe Presentation, Core Tests). Neue vierte Reihe unter dem Core-Read-only-Port: `Compliance Engine` (Engine-Rahmen), `Compliance tables`, `Retrieval & model` und `ISO mapping in the UI`, mit vier Beziehungen (Port -> Engine nur lesend, Engine <-> eigene Tabellen, Engine <-> Qdrant/LiteLLM, Engine -> UI-Erweiterungspunkte); die bestehenden neun Knoten behalten ihre Positionen, die Karte waechst von 650 auf 940 px. Die Port-Beschreibung nennt offen, dass die Compliance Engine noch ueber das laufende Core-Repository liest; die Beziehungszahl im Verzeichnis wird aus der Liste berechnet (13). Geometrie, beide Themes und die Hervorhebung der vier Pfeile im laufenden Frontend geprueft; Diagrammtest erweitert. |
|
||||
| 2026-10-05 | Phase 13F, in Produktversion 2.18.0 enthalten: Aktivitaetsprotokoll fuer die Compliance Engine korrigiert. Ihre API-Anfragen fielen auf die generischen Core-Muster und erschienen als `Core Engine · Loading clients`, `Processing clients` oder `Loading application data`; `src/api/request-activity.ts` ordnet die Routen fuer Zuordnung, Vorschlaege, Entscheidungen, Batch und ISO-Korpus jetzt vor den Projektmustern der `Compliance Engine` zu (Core-Freigabe Platform). Zusaetzlich haelt die Engine je laufender Backendarbeit (`Mapping ISO controls`, `Mapping all ISO controls`, `Embedding ISO corpus`) einen Eintrag offen, bis das Backend einen Endzustand meldet; ein Fehlschlag wird als fehlgeschlagen gefuehrt, und eine Ansicht, die nicht mehr beobachtet, beendet den Eintrag mit dem Hinweis `continues in the backend`. Tests fuer alle Routen und fuer laufenden, abgeschlossenen, fehlgeschlagenen und nicht mehr beobachteten Batch. |
|
||||
| 2026-10-05 | Phase 13F, Produktversion 2.18.0 (fasst das unveroeffentlichte 2.17.1 mit ein): ISO-Zuordnung als Compliance Engine ausgelagert, Nutzerentscheidung mit Core-Freigabe. Name `Compliance Engine`, bestehende drei Tabellen bleiben und gehen an die Engine, in diesem Schritt nur Verschiebung ohne Verhaltensaenderung; die frameworkneutrale Erweiterung folgt getrennt. 41 Dateien verschoben (23 Quell-, 18 Testdateien); der Filtertyp `ResultDbFilters` liegt jetzt in `src/domain/result-db-query.ts`. Die neun Regulatory-Methoden und drei Exportfelder sind aus `AssessorRepository`, Local- und PostgreSQL-Repository entfernt; `LocalComplianceStore` und `PostgresComplianceStore` uebernehmen sie unveraendert, der Projektexport fuegt die Engine-Daten an der HTTP-Route hinzu. `RegulatoryMappingService` erhaelt Read-Port und Store getrennt. Engine-Pfade in `.claude/hooks/core-areas.mjs`, neue Extension Points (Finding Reader, Result DB, Data Hub, API-Vertrag und -Client typ-only, Serververdrahtung) im Grenztest, Entscheidung in `AGENTS.md`. Neue Tests: permanente Invarianz (Revision, Projektexport und Result DB unveraendert nach Einzelzuordnung, Entscheidung und Batch; Port ohne Schreibmethode), Schalter in Finding Reader, Result DB und Data Hub, Projektexport mit und ohne Engine. |
|
||||
| 2026-10-05 | Phase 13F, Produktversion 2.17.1: Merge-Regression der ISO-Kontrollzuordnung behoben. Seit 2.9.0 liefert `getUnifiedResultById` Result-DB-Zeilen ohne echte Scope-IDs (`projectId: scoped-project`); der Zuordnungsdienst verglich danach `finding.projectId` mit dem Projekt und meldete fuer jedes Finding `The selected finding does not exist in this project.` Die Lesemethode nimmt jetzt optional `projectId` und filtert in PostgreSQL ueber die vorhandene Spalte `project_id` (lokal vor dem Entfernen der Scope-IDs); der Dienst nutzt diese Abfrage. Die zurueckgegebene Zeile und die Result-DB-Offenlegung bleiben unveraendert. Der Mock im Dienst-Test bildet die Entfernung der Scope-IDs jetzt nach; neuer Regressionstest fuer Laden, Start und Entscheidung im eigenen Projekt und deren Ablehnung fuer ein fremdes Projekt, dazu Pruefungen der Projektbindung im lokalen und im PostgreSQL-Repository (Letztere ohne Testdatenbank uebersprungen). |
|
||||
| 2026-10-05 | Phasen 6/7/13A, Produktversion 2.17.0: `merge/defender-for-iot` auf `250c213` (Michael, Business Impact + Management Report, 2.16.5) rebased; Michaels Commit ist die Basis, die 25 Branch-Commits liegen darauf. Die Minor-Erhoehung der Defender-Serie (zuvor 2.11.2 -> 2.12.0, Eintrag unten historisch) wird auf dieser Basis 2.16.5 -> 2.17.0; `package.json`, Root-Eintraege in `package-lock.json` und die Versionsvertragstests sind synchron. Korrektur aus `2b5fdf3`: `isDeviationRecord` uebersprang Maester `Result` immer als Ergebnisnachricht; nach AGENTS.md ist `Result` nur neben einer eigenen Statusspalte Evidenz, sonst der native Status. Michaels neuer Sidebar-Test deckte das auf; synthetische Regression in `tests/domain/deviation-predicate.test.ts`. Michaels Result-DB-Leerzustandstest erwartet nun das mit `55da786` eingefuehrte `audience: 'result-db'`. Kein Reimport, keine Migration, nicht im Browser geprueft. Verifikation: Gesamtlauf mit 2.049 bestandenen und 18 uebersprungenen Tests, beide Produktionsbuilds erfolgreich. |
|
||||
| 2026-10-02 | Phase 13A, Disclosure Policy 3.5.0: Defender-Teil von `21a6266` uebernommen; `48a1888` (PingCastle-Aggregate, steht gegen die Zaehlwert-Grenze der Policy 2.0.0) und `78ea9a6` auf Nutzerentscheidung ausgelassen. Die Result DB kennt Microsoft Defender for IoT als Werkzeug statt `Security assessment`; freigegeben wird nur eine formal gueltige oeffentliche CVE-ID als Rule ID und zugleich als Titel. Geraete-, Site-, Sensor- und Zaehlwerte bleiben ausgeschlossen, `affectedAsset` bleibt zurueckgehalten; das Alerts-Profil bleibt generisch. Die zurueckgestellte `securityToolId`-Zeile aus `c056c21` bildet beide Defender-Namen auf den Schluessel `defender-for-iot` ab. Regression: Work und Result DB benennen ein Defender-Finding ueber die Projektion mit seiner CVE. `docs/RESULT_CONTRACT.md` beschreibt 3.5.0; die Versionsaufzaehlung der Result-DB-Regel in `AGENTS.md` ist nicht angepasst (Governance nicht freigegeben). Die materialisierte Result DB wird nach dem naechsten Backendstart mit der neuen Policy neu aufgebaut. Verifikation: beide Typechecks, Gesamtlauf mit 1.921 bestandenen und 18 uebersprungenen Tests sowie dem bekannten lastbedingten Ausfall von `sheet-selection-cost.test.tsx` (allein bestanden), Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich. |
|
||||
| 2026-10-02 | Phase 13A: Geraeteteil von `95ec322` nachgezogen. `defenderRiskTargetsForOccurrences` ordnet Devices ueber die Quellzeilen der jeweiligen Occurrence zu statt ueber die CVE allein; fuehren zwei Occurrences dieselbe CVE mit unterschiedlichen Devices, bleibt jedes Device bei seiner eigenen Occurrence, und ein `device-finding` fuer ein Device ausserhalb der gewaehlten Occurrence wird abgelehnt. Das stuetzt die Regel, dass eine geraetebezogene Entscheidung nur ihre erfassten Occurrences deckt. Zwei Regressionen (Domain, lokales Repository) uebernommen. Keine API-, SQL- oder Persistenzaenderung. Verifikation: beide Typechecks, Gesamtlauf mit 1.922 bestandenen und 18 uebersprungenen Tests, Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich. |
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ Clarification 2026-09-17 (baseline analyses of the written form): The Microsoft
|
|||
|
||||
Clarification 2026-09-17 (written security-template section and key): The Security Template parser folded the section name to lower case and discarded the written form, so `[Privilege Rights]` was retained only as `privilege rights`. The parser now keeps the written section and key; the case-folded forms stay internal to grammar selection, grouping and the identity digest, which keeps existing observation and identity IDs stable. The native target resolver stores the written section and key in the retained target, matching what the registry resolver already did, so Work shows `Privilege Rights - SeCreateTokenPrivilege` and the value stays searchable against Microsoft documentation and external tables. This changes persisted parsing output, so it is an append-only processing-version change, not a presentation change: `gpo-security-template-inf` 1.1.0 -> 1.2.0, `gpo-backup-policy-analysis` 2.2.3 -> 2.2.4 and the GPO job processor 2.2.3 -> 2.2.4. Historical imports keep their retained values unchanged and are marked for reprocessing: Data Hub reports an affected GPO backup as `Reprocessing required` with the analyzer versions and offers the existing reprocess action on the stored source, so no upload is needed. Reprocessing precedes re-evaluation, and prior runs remain retained as history. No SQL migration and no evaluator change.
|
||||
|
||||
Clarification 2026-10-05 (GPO domain key): A standard `Backup-GPO` folder names the domain by GUID in `bkupInfo.xml` and by DNS name in `Backup.xml`/`GPReport.xml`. The metadata aggregate treated these two identifier kinds as conflicting values, so every such backup stayed unresolved and blocked (`Limited analysis`). A GUID and a DNS name now count as two kinds of the same domain; only two different values of one kind conflict, and the GUID is preferred. This changes persisted identity resolution, so it is an append-only processing-version change: `gpo-backup-policy-analysis` and the GPO job processor 2.2.5 -> 2.2.6 (metadata parser 1.4.0 unchanged). Affected backups show `Reprocessing required`; the existing reprocess action on the stored source applies the fix. No upload, SQL migration or evaluator change.
|
||||
|
||||
Clarification 2026-09-25 (GPO link facts): The GPO metadata parser retains, from each backup's own `GPReport.xml`, the number of `LinksTo` entries and how many are enabled (never the linked OU or domain names). A backup with no enabled link is presented as `Not linked`: the Finding Reader marks its source sections and states `No effect` in the problem summary when every contributing GPO is unlinked, and GPO Processing shows the count. This is a backup-report fact at backup time, not a claim about effective device state, and verdicts and score are unchanged. Because parsing now retains a new fact, it is an append-only processing-version change: `gpo-backup-metadata-xml` 1.2.0 -> 1.4.0 (the Microsoft baseline template variant keeps 1.3.0, so baseline packages need no re-analysis), `gpo-backup-policy-analysis` 2.2.4 -> 2.2.5 and the GPO job processor 2.2.4 -> 2.2.5. Historical imports are marked `Reprocessing required` and the existing reprocess action on the stored source adds the link facts; no upload, SQL migration or evaluator change is required. Until then no link status is shown.
|
||||
|
||||
Clarification 2026-09-17 (Not comparable verdict): A retained customer setting whose delivery mechanism has no comparable technical policy identity - today always a Group Policy Preference - reaches Work under the score-neutral verdict `Not comparable` instead of being dropped. It is never resolved into a policy identity and never compared with a Microsoft baseline. The unified read projection additionally exposes the already retained preference CSE, action and item-level targeting status so that context stays readable in the agreed GPO columns; no new column, facet or persisted field is introduced. Snapshot-aggregation review hints that restate an unresolved identity already reported by their source analysis are no longer emitted a second time. All of this projects facts the importer already retained, so no processing-manifest version, source reimport, evaluator change or SQL migration is required.
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "assessor",
|
||||
"version": "2.17.0",
|
||||
"version": "2.19.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "assessor",
|
||||
"version": "2.17.0",
|
||||
"version": "2.19.0",
|
||||
"dependencies": {
|
||||
"@fontsource-variable/inter": "^5.3.0",
|
||||
"jsdom": "^29.1.1",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "assessor",
|
||||
"version": "2.17.0",
|
||||
"version": "2.19.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import type { AiOverviewTokenUsage } from '../../src/domain/ai-overview';
|
||||
/** Token counts the provider reports for one completion. */
|
||||
export interface AiCompletionTokenUsage {
|
||||
promptTokens?: number;
|
||||
completionTokens?: number;
|
||||
totalTokens?: number;
|
||||
}
|
||||
|
||||
export interface LiteLlmEnvironment {
|
||||
ASSESSOR_AI_MODE?: string;
|
||||
|
|
@ -11,7 +16,7 @@ export interface LiteLlmEnvironment {
|
|||
|
||||
export interface AiCompletion {
|
||||
value: unknown;
|
||||
usage?: AiOverviewTokenUsage;
|
||||
usage?: AiCompletionTokenUsage;
|
||||
}
|
||||
|
||||
export interface AiCompletionClient {
|
||||
|
|
@ -206,7 +211,7 @@ function completionContent(value: unknown): string {
|
|||
return value.choices[0].message.content;
|
||||
}
|
||||
|
||||
function completionUsage(value: unknown): AiOverviewTokenUsage | undefined {
|
||||
function completionUsage(value: unknown): AiCompletionTokenUsage | undefined {
|
||||
if (!isObject(value) || !isObject(value.usage)) return undefined;
|
||||
const number = (entry: unknown) => typeof entry === 'number' && Number.isFinite(entry) ? entry : undefined;
|
||||
const usage = {
|
||||
|
|
|
|||
|
|
@ -29,18 +29,19 @@ import type { GpoPolicySnapshotService } from './gpo/gpo-policy-snapshot-service
|
|||
import type { GpoPolicyEvaluationService } from './gpo/gpo-policy-evaluation-service';
|
||||
import type { GpoPolicyFindingService } from './gpo/gpo-policy-finding-service';
|
||||
import type { GpoPolicyAnalysisOverviewService } from './gpo/gpo-policy-analysis-overview-service';
|
||||
import type { AiOverviewService } from './ai/ai-overview-service';
|
||||
import { AiOverviewServiceError } from './ai/ai-overview-service';
|
||||
import type { AiOverviewCreateRequest, AiOverviewScope } from '../src/domain/ai-overview';
|
||||
import type { AiOverviewService } from './compliance/ai-overview-service';
|
||||
import { AiOverviewServiceError } from './compliance/ai-overview-service';
|
||||
import type { AiOverviewCreateRequest, AiOverviewScope } from '../src/compliance/ai-overview';
|
||||
import type {
|
||||
RegulatoryMappingBatchCreateRequest,
|
||||
RegulatoryMappingBatchScope,
|
||||
RegulatoryMappingCreateRequest,
|
||||
RegulatoryMappingDecisionCreateRequest
|
||||
} from '../src/domain/regulatory-mapping';
|
||||
import type { RegulatoryMappingService } from './regulatory/regulatory-mapping-service';
|
||||
import { RegulatoryMappingServiceError } from './regulatory/regulatory-mapping-service';
|
||||
import { RegulatoryCorpusUploadError, type RegulatoryCorpusUploadService } from './regulatory/regulatory-corpus-upload-service';
|
||||
} from '../src/compliance/regulatory-mapping';
|
||||
import type { RegulatoryMappingService } from './compliance/regulatory-mapping-service';
|
||||
import type { ComplianceStore } from '../src/compliance/compliance-store';
|
||||
import { RegulatoryMappingServiceError } from './compliance/regulatory-mapping-service';
|
||||
import { RegulatoryCorpusUploadError, type RegulatoryCorpusUploadService } from './compliance/regulatory-corpus-upload-service';
|
||||
import type { GpoAutomaticAnalysisService } from './gpo/gpo-automatic-analysis-service';
|
||||
import type { MicrosoftSecurityBaselineSelectionContext } from '../src/domain/microsoft-security-baseline';
|
||||
import { listProjectGpoBackupStatus } from './gpo/gpo-backup-status-service';
|
||||
|
|
@ -177,6 +178,7 @@ export interface AssessorHttpServerOptions {
|
|||
aiOverviewService?: AiOverviewService;
|
||||
regulatoryMappingService?: RegulatoryMappingService;
|
||||
regulatoryCorpusUploadService?: RegulatoryCorpusUploadService;
|
||||
complianceExport?: Pick<ComplianceStore, 'exportProject'>;
|
||||
isoUploadToken?: string;
|
||||
}
|
||||
|
||||
|
|
@ -1045,7 +1047,10 @@ export function createAssessorHttpServer(options: AssessorHttpServerOptions): Se
|
|||
status = 200; sendJson(response, status, { rule, repositoryRevision: repository.revision }, requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'export' && segments.length === 3 && method === 'GET') {
|
||||
status = 200; sendJson(response, status, await repository.exportProject(segments[1]), requestId); return;
|
||||
// The Compliance Engine delivers its own project data beside the Core export.
|
||||
const coreExport = await repository.exportProject(segments[1]);
|
||||
const complianceExport = options.complianceExport ? await options.complianceExport.exportProject(segments[1]) : {};
|
||||
status = 200; sendJson(response, status, { ...coreExport, ...complianceExport }, requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'report-projection' && segments.length === 3 && method === 'GET') {
|
||||
const assessmentIds = url.searchParams.getAll('assessmentIds');
|
||||
|
|
@ -1170,6 +1175,14 @@ export function createAssessorHttpServer(options: AssessorHttpServerOptions): Se
|
|||
const body = await readJson<RegulatoryMappingDecisionCreateRequest>(request, maximumBodyBytes);
|
||||
status = 200; sendJson(response, status, await options.regulatoryMappingService.decide(segments[1], body), requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'regulatory-control-assignments' && segments.length === 3 && method === 'GET') {
|
||||
if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.');
|
||||
status = 200; sendJson(response, status, await options.regulatoryMappingService.controlAssignments(segments[1], regulatoryMappingBatchScopeFrom(url)), requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'regulatory-control-overview' && segments.length === 3 && method === 'GET') {
|
||||
if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.');
|
||||
status = 200; sendJson(response, status, await options.regulatoryMappingService.controlOverview(segments[1], regulatoryMappingBatchScopeFrom(url)), requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'regulatory-mapping-batches' && segments.length === 3 && method === 'GET') {
|
||||
if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.');
|
||||
status = 200; sendJson(response, status, await options.regulatoryMappingService.getBatch(segments[1], regulatoryMappingBatchScopeFrom(url)), requestId); return;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { AssessorRepository } from '../../src/persistence/assessor-repository';
|
||||
import type { ComplianceReadPort, ComplianceStore } from '../../src/compliance/compliance-store';
|
||||
import { RepositoryConflictError, RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors';
|
||||
import { resultsToPrivacySafeExportJson } from '../../src/exports/result-db-copy';
|
||||
import { aiOverviewInputJson } from '../../src/compliance/privacy-safe-export';
|
||||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import {
|
||||
AI_OVERVIEW_OUTPUT_SCHEMA_VERSION,
|
||||
|
|
@ -15,9 +15,9 @@ import {
|
|||
type AiOverviewRun,
|
||||
type AiOverviewScope,
|
||||
type AiOverviewTokenUsage
|
||||
} from '../../src/domain/ai-overview';
|
||||
import type { AiCompletionClient } from './litellm-client';
|
||||
import { LiteLlmClientError } from './litellm-client';
|
||||
} from '../../src/compliance/ai-overview';
|
||||
import type { AiCompletionClient } from '../ai/litellm-client';
|
||||
import { LiteLlmClientError } from '../ai/litellm-client';
|
||||
|
||||
const PAGE_SIZE = 500;
|
||||
const ACTIVE_POLL_MS = 1_500;
|
||||
|
|
@ -57,7 +57,7 @@ export class AiOverviewService {
|
|||
private readonly workerId: string;
|
||||
private readonly activeTasks = new Set<Promise<void>>();
|
||||
|
||||
constructor(private readonly repository: AssessorRepository, options: AiOverviewServiceOptions = {}) {
|
||||
constructor(private readonly read: ComplianceReadPort, private readonly store: ComplianceStore, options: AiOverviewServiceOptions = {}) {
|
||||
this.client = options.client;
|
||||
this.now = options.now ?? (() => new Date());
|
||||
this.maximumChunkCharacters = options.maximumChunkCharacters ?? 80_000;
|
||||
|
|
@ -76,10 +76,11 @@ export class AiOverviewService {
|
|||
|
||||
async recoverInterruptedRuns(): Promise<void> {
|
||||
const now = this.now().toISOString();
|
||||
for (const project of await this.repository.listProjects()) {
|
||||
const runs = await this.repository.listAiOverviewRuns(project.id);
|
||||
for (const project of await this.read.listProjects()) {
|
||||
await this.pruneDeletedTimeslots(project.id);
|
||||
const runs = await this.store.listAiOverviewRuns(project.id);
|
||||
for (const run of runs.filter(item => ['queued', 'running'].includes(item.status))) {
|
||||
await this.repository.updateAiOverviewRun({
|
||||
await this.store.updateAiOverviewRun({
|
||||
...run,
|
||||
status: 'failed',
|
||||
updatedAt: now,
|
||||
|
|
@ -93,10 +94,27 @@ export class AiOverviewService {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Core deletes scan runs and assessments without knowing this engine. Their AI overviews are
|
||||
* derived from deleted data, so the engine removes them from its own table on start and on read.
|
||||
*/
|
||||
async pruneDeletedTimeslots(projectId: string): Promise<void> {
|
||||
const [assessments, scanRuns, runs] = await Promise.all([
|
||||
this.read.listAssessments(projectId), this.read.listProjectScanRuns(projectId), this.store.listAiOverviewRuns(projectId)
|
||||
]);
|
||||
const assessmentIds = new Set(assessments.map(item => item.id));
|
||||
const scanRunIds = new Set(scanRuns.map(item => item.id));
|
||||
const orphaned = runs.filter(run => !['queued', 'running'].includes(run.status)
|
||||
&& (run.assessmentIds.some(id => !assessmentIds.has(id)) || run.scanRunIds.some(id => !scanRunIds.has(id))));
|
||||
await this.store.deleteAiOverviewRuns(projectId, orphaned.map(run => run.id));
|
||||
}
|
||||
|
||||
async get(projectId: string, requestedScope: AiOverviewScope): Promise<AiOverviewResponse> {
|
||||
if (!await this.read.getProject(projectId)) throw new RepositoryNotFoundError(`Projekt ${projectId} ist nicht vorhanden.`);
|
||||
await this.pruneDeletedTimeslots(projectId);
|
||||
const scope = await this.validateScope(projectId, requestedScope);
|
||||
const scopeKey = aiOverviewScopeKey(projectId, scope);
|
||||
const runs = await this.repository.listAiOverviewRuns(projectId, scopeKey);
|
||||
const runs = await this.store.listAiOverviewRuns(projectId, scopeKey);
|
||||
const run = runs[0];
|
||||
const previousSuccessfulRun = run?.status === 'succeeded' ? undefined : runs.find(item => item.status === 'succeeded');
|
||||
let stale = false;
|
||||
|
|
@ -136,13 +154,13 @@ export class AiOverviewService {
|
|||
updatedAt: timestamp,
|
||||
callCount: 0
|
||||
};
|
||||
const run = await this.repository.createAiOverviewRun(candidate);
|
||||
const run = await this.store.createAiOverviewRun(candidate);
|
||||
if (run.id === candidate.id) {
|
||||
const task = this.process(run, prepared).finally(() => this.activeTasks.delete(task));
|
||||
this.activeTasks.add(task);
|
||||
}
|
||||
const previousSuccessfulRun = run.status === 'succeeded' ? undefined
|
||||
: (await this.repository.listAiOverviewRuns(projectId, run.scopeKey)).find(item => item.status === 'succeeded');
|
||||
: (await this.store.listAiOverviewRuns(projectId, run.scopeKey)).find(item => item.status === 'succeeded');
|
||||
return responseFor(projectId, scope, true, run, false, previousSuccessfulRun);
|
||||
}
|
||||
|
||||
|
|
@ -161,7 +179,7 @@ export class AiOverviewService {
|
|||
leaseExpiresAt: new Date(this.now().getTime() + this.leaseDurationMs).toISOString()
|
||||
};
|
||||
try {
|
||||
run = await this.repository.updateAiOverviewRun(run);
|
||||
run = await this.store.updateAiOverviewRun(run);
|
||||
const completion = await generateAiOverview(prepared.parsed, {
|
||||
client: this.client!,
|
||||
maximumChunkCharacters: this.maximumChunkCharacters,
|
||||
|
|
@ -171,7 +189,7 @@ export class AiOverviewService {
|
|||
factProperties: prepared.factProperties,
|
||||
onCall: async usage => {
|
||||
const updatedAt = this.now().toISOString();
|
||||
run = await this.repository.updateAiOverviewRun({
|
||||
run = await this.store.updateAiOverviewRun({
|
||||
...run,
|
||||
callCount: run.callCount + 1,
|
||||
tokenUsage: addUsage(run.tokenUsage, usage),
|
||||
|
|
@ -181,7 +199,7 @@ export class AiOverviewService {
|
|||
}
|
||||
});
|
||||
const completedAt = this.now().toISOString();
|
||||
await this.repository.updateAiOverviewRun({
|
||||
await this.store.updateAiOverviewRun({
|
||||
...run,
|
||||
status: 'succeeded',
|
||||
result: completion,
|
||||
|
|
@ -194,7 +212,7 @@ export class AiOverviewService {
|
|||
const completedAt = this.now().toISOString();
|
||||
const failure = safeGenerationFailure(error);
|
||||
try {
|
||||
await this.repository.updateAiOverviewRun({
|
||||
await this.store.updateAiOverviewRun({
|
||||
...run,
|
||||
status: 'failed',
|
||||
updatedAt: completedAt,
|
||||
|
|
@ -211,13 +229,13 @@ export class AiOverviewService {
|
|||
}
|
||||
|
||||
private async validateScope(projectId: string, requested: AiOverviewScope): Promise<AiOverviewScope> {
|
||||
if (!await this.repository.getProject(projectId)) throw new RepositoryNotFoundError(`Projekt ${projectId} ist nicht vorhanden.`);
|
||||
if (!await this.read.getProject(projectId)) throw new RepositoryNotFoundError(`Projekt ${projectId} ist nicht vorhanden.`);
|
||||
const scope = normalizeAiOverviewScope(requested);
|
||||
if (!scope.assessmentIds.length && !scope.scanRunIds.length) throw new RepositoryValidationError('A selected timeslot scope is required.');
|
||||
const assessments = await this.repository.listAssessments(projectId);
|
||||
const assessments = await this.read.listAssessments(projectId);
|
||||
const allowedAssessmentIds = new Set(assessments.map(item => item.id));
|
||||
if (scope.assessmentIds.some(id => !allowedAssessmentIds.has(id))) throw new RepositoryValidationError('The AI overview scope contains an assessment outside this project.');
|
||||
const scanRuns = await this.repository.listProjectScanRuns(projectId);
|
||||
const scanRuns = await this.read.listProjectScanRuns(projectId);
|
||||
const allowedRuns = new Map(scanRuns.filter(item => !item.supersededByScanRunId).map(item => [item.id, item]));
|
||||
if (scope.scanRunIds.some(id => !allowedRuns.has(id))) throw new RepositoryValidationError('The AI overview scope contains an unavailable scan run.');
|
||||
if (scope.assessmentIds.length && scope.scanRunIds.some(id => !scope.assessmentIds.includes(allowedRuns.get(id)!.assessmentId))) {
|
||||
|
|
@ -227,9 +245,9 @@ export class AiOverviewService {
|
|||
}
|
||||
|
||||
private async prepareInput(projectId: string, scope: AiOverviewScope): Promise<PreparedInput> {
|
||||
const items = await queryAllUnifiedResults(this.repository, projectId, scope);
|
||||
const ordered = items.slice().sort((left, right) => stableResultKey(left).localeCompare(stableResultKey(right)));
|
||||
const json = resultsToPrivacySafeExportJson(ordered);
|
||||
const items = await queryAllUnifiedResults(this.read, projectId, scope);
|
||||
// The same function builds the Result DB `Export JSON`, so the export is exactly what the model receives.
|
||||
const json = aiOverviewInputJson(items);
|
||||
const parsed = parsePrivacySafeInput(json);
|
||||
const findingIds = new Set(parsed.findings.flatMap(item => typeof item.id === 'string' ? [item.id] : []));
|
||||
const gapIds = new Set([
|
||||
|
|
@ -309,7 +327,7 @@ export function splitPrivacySafeInput(input: PrivacySafeInput, maximumCharacters
|
|||
return chunks;
|
||||
}
|
||||
|
||||
async function queryAllUnifiedResults(repository: AssessorRepository, projectId: string, scope: AiOverviewScope): Promise<UnifiedResultProjection[]> {
|
||||
async function queryAllUnifiedResults(repository: ComplianceReadPort, projectId: string, scope: AiOverviewScope): Promise<UnifiedResultProjection[]> {
|
||||
const items: UnifiedResultProjection[] = [];
|
||||
let expectedTotal: number | undefined;
|
||||
for (let offset = 0; ; offset += PAGE_SIZE) {
|
||||
|
|
@ -351,10 +369,6 @@ function parsePrivacySafeInput(json: string): PrivacySafeInput {
|
|||
return value as PrivacySafeInput;
|
||||
}
|
||||
|
||||
function stableResultKey(item: UnifiedResultProjection): string {
|
||||
return [item.sourceKind, item.sourceTool ?? '', item.sourceRuleId ?? '', item.resultProjectionId].join('\u0000');
|
||||
}
|
||||
|
||||
function responseFor(projectId: string, scope: AiOverviewScope, enabled: boolean, run?: AiOverviewRun, stale = false, previousSuccessfulRun?: AiOverviewRun): AiOverviewResponse {
|
||||
return {
|
||||
version: 1,
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
import { join } from 'node:path';
|
||||
import { Pool } from 'pg';
|
||||
import type { AiCompletionClient } from '../ai/litellm-client';
|
||||
import { repositoryConnectionString, sslConfiguration, type PersistenceKind, type RepositoryEnvironment } from '../repository-factory';
|
||||
import type { AssessorRepository } from '../../src/persistence/assessor-repository';
|
||||
import { complianceReadPortFrom, type ComplianceStore } from '../../src/compliance/compliance-store';
|
||||
import { LocalComplianceStore } from '../../src/compliance/local-compliance-store';
|
||||
import { PostgresComplianceStore } from './postgres-compliance-store';
|
||||
import { createRegulatoryRagRuntime } from './regulatory-rag-runtime';
|
||||
import { RegulatoryMappingService } from './regulatory-mapping-service';
|
||||
import { RegulatoryCorpusUploadService } from './regulatory-corpus-upload-service';
|
||||
import { AiOverviewService, type AiOverviewServiceOptions } from './ai-overview-service';
|
||||
|
||||
export interface ComplianceEngineRuntime {
|
||||
mappingService: RegulatoryMappingService;
|
||||
corpusUploadService: RegulatoryCorpusUploadService;
|
||||
aiOverviewService: AiOverviewService;
|
||||
store: ComplianceStore;
|
||||
/** Stops running work; ASSESSOR keeps its own repository open. */
|
||||
stop(): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ComplianceEngineRuntimeOptions {
|
||||
repository: AssessorRepository;
|
||||
persistence: PersistenceKind;
|
||||
aiClient?: AiCompletionClient;
|
||||
artifactRoot: string;
|
||||
environment?: RepositoryEnvironment;
|
||||
aiOverview?: Omit<AiOverviewServiceOptions, 'client'>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the Compliance Engine beside ASSESSOR. It reads the Core only through its read-only port and
|
||||
* writes only its own tables through a separate small pool. If retrieval cannot start, mapping is
|
||||
* refused with a reason while ASSESSOR keeps running.
|
||||
*/
|
||||
export async function createComplianceEngineRuntime(options: ComplianceEngineRuntimeOptions): Promise<ComplianceEngineRuntime> {
|
||||
const pool = options.persistence === 'postgresql' ? compliancePool(options.environment ?? process.env as RepositoryEnvironment) : undefined;
|
||||
const store: ComplianceStore = pool ? new PostgresComplianceStore(pool) : new LocalComplianceStore();
|
||||
const read = complianceReadPortFrom(options.repository);
|
||||
let mappingService: RegulatoryMappingService;
|
||||
let rag: ReturnType<typeof createRegulatoryRagRuntime> | undefined;
|
||||
let disabledReason: string | undefined;
|
||||
try {
|
||||
rag = createRegulatoryRagRuntime();
|
||||
mappingService = new RegulatoryMappingService(read, store, {
|
||||
findingRetrieval: rag.findingRetrieval,
|
||||
index: rag.index,
|
||||
embeddingModel: rag.embeddingModel,
|
||||
client: options.aiClient,
|
||||
corpusId: process.env.ASSESSOR_RAG_ACTIVE_CORPUS_ID?.trim(),
|
||||
corpusVersion: process.env.ASSESSOR_RAG_ACTIVE_CORPUS_VERSION?.trim(),
|
||||
...(!options.aiClient ? { disabledReason: 'Work LiteLLM is not configured.' } : {})
|
||||
});
|
||||
} catch (error) {
|
||||
disabledReason = error instanceof Error ? error.message : 'Regulatory mapping is unavailable.';
|
||||
mappingService = new RegulatoryMappingService(read, store, { disabledReason });
|
||||
}
|
||||
await mappingService.recoverInterruptedRuns();
|
||||
const corpusUploadService = new RegulatoryCorpusUploadService({
|
||||
root: join(options.artifactRoot, 'regulatory-corpora'),
|
||||
ingestion: rag?.ingestion,
|
||||
countIndexed: rag ? (corpusId, corpusVersion) => rag!.index.count({ corpusId, corpusVersion }) : undefined,
|
||||
pruneIndexed: rag ? (corpusId, corpusVersion) => rag!.index.deleteOtherCorpusVersions(corpusId, corpusVersion) : undefined,
|
||||
mapping: mappingService,
|
||||
embeddingModel: rag?.embeddingModel,
|
||||
disabledReason
|
||||
});
|
||||
await corpusUploadService.initialize();
|
||||
const aiOverviewService = new AiOverviewService(read, store, { ...options.aiOverview, client: options.aiClient });
|
||||
await aiOverviewService.recoverInterruptedRuns();
|
||||
return {
|
||||
mappingService,
|
||||
corpusUploadService,
|
||||
aiOverviewService,
|
||||
store,
|
||||
stop: () => Promise.all([corpusUploadService.stop().then(() => mappingService.close()), aiOverviewService.close()]).then(() => undefined),
|
||||
close: async () => { await pool?.end(); }
|
||||
};
|
||||
}
|
||||
|
||||
function compliancePool(environment: RepositoryEnvironment): Pool {
|
||||
const connectionString = repositoryConnectionString(environment);
|
||||
if (!connectionString) throw new Error('The Compliance Engine needs the existing PostgreSQL connection.');
|
||||
const hostname = new URL(connectionString).hostname;
|
||||
const sslMode = environment.ASSESSOR_DATABASE_SSL?.trim()
|
||||
|| (['localhost', '127.0.0.1', '[::1]'].includes(hostname) ? 'disable' : 'require');
|
||||
return new Pool({
|
||||
connectionString,
|
||||
ssl: sslConfiguration(sslMode),
|
||||
max: 4,
|
||||
connectionTimeoutMillis: 5000,
|
||||
idleTimeoutMillis: 30000,
|
||||
application_name: 'assessor-compliance-engine'
|
||||
});
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ import type {
|
|||
RegulatoryControlTheme,
|
||||
RegulatoryCorpusV2,
|
||||
RegulatorySectionKind
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
} from '../../src/compliance/regulatory-rag';
|
||||
|
||||
export const ISO_PDF_PARSER_VERSION = '1.0.0';
|
||||
const MAXIMUM_PDF_BYTES = 32 * 1024 * 1024;
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { Worker } from 'node:worker_threads';
|
||||
import type { RegulatoryCorpusV2 } from '../../src/domain/regulatory-rag';
|
||||
import type { RegulatoryCorpusV2 } from '../../src/compliance/regulatory-rag';
|
||||
import type { IsoPdfCorpusOptions } from './iso-pdf-corpus';
|
||||
|
||||
export function prepareIsoCorpusInWorker(options: IsoPdfCorpusOptions, signal?: AbortSignal): Promise<RegulatoryCorpusV2> {
|
||||
|
|
@ -0,0 +1,279 @@
|
|||
import type { Pool, PoolClient, QueryResult } from 'pg';
|
||||
import { RepositoryConflictError, RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors';
|
||||
import { translateDatabaseError } from '../postgres/postgres-assessor-repository';
|
||||
import type { ComplianceProjectExport, ComplianceStore } from '../../src/compliance/compliance-store';
|
||||
import {
|
||||
regulatoryMappingBatchRunValidationErrors,
|
||||
regulatoryMappingDecisionValidationErrors,
|
||||
regulatoryMappingRunValidationErrors,
|
||||
type RegulatoryMappingBatchRun,
|
||||
type RegulatoryMappingDecision,
|
||||
type RegulatoryMappingRun
|
||||
} from '../../src/compliance/regulatory-mapping';
|
||||
import { aiOverviewRunValidationErrors, assertAiOverviewTransition, type AiOverviewRun } from '../../src/compliance/ai-overview';
|
||||
|
||||
interface DataRow<T> { data: T }
|
||||
|
||||
function clone<T>(value: T): T {
|
||||
return JSON.parse(JSON.stringify(value)) as T;
|
||||
}
|
||||
|
||||
function json(value: unknown): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function dataRows<T>(result: QueryResult<DataRow<T>>): T[] {
|
||||
return result.rows.map(row => clone(row.data));
|
||||
}
|
||||
|
||||
function dataRow<T>(result: QueryResult<DataRow<T>>): T | null {
|
||||
return result.rows[0] ? clone(result.rows[0].data) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Compliance Engine's own tables (migration 026, and the AI overview runs of migration 025). Writes run in the engine's own transactions:
|
||||
* they never advance the ASSESSOR repository revision and never trigger a Result DB rebuild.
|
||||
*/
|
||||
export class PostgresComplianceStore implements ComplianceStore {
|
||||
constructor(private readonly pool: Pool) {}
|
||||
|
||||
private async transaction<T>(work: (client: PoolClient) => Promise<T>): Promise<T> {
|
||||
const client = await this.pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const result = await work(client);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
try { await client.query('ROLLBACK'); } catch { /* connection failure keeps original error */ }
|
||||
throw translateDatabaseError(error);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async getRegulatoryMappingRun(id: string): Promise<RegulatoryMappingRun | null> {
|
||||
return dataRow(await this.pool.query<DataRow<RegulatoryMappingRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mappings WHERE id = $1', [id]
|
||||
));
|
||||
}
|
||||
|
||||
async claimRegulatoryMappingRun(run: RegulatoryMappingRun, forceRegeneration: boolean): Promise<{ run: RegulatoryMappingRun; claimed: boolean }> {
|
||||
const errors = regulatoryMappingRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.transaction(async client => {
|
||||
const inserted = await client.query(`
|
||||
INSERT INTO assessor_regulatory_mappings
|
||||
(id, rule_key, source_tool, source_rule_id, run_status, query_hash, corpus_id, corpus_version,
|
||||
embedding_model, retrieval_version, prompt_version, output_schema_version, model, created_at, updated_at, data)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16::jsonb)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING id
|
||||
`, [run.id, run.ruleKey, run.sourceTool, run.sourceRuleId ?? null, run.status, run.queryHash, run.corpusId,
|
||||
run.corpusVersion, run.embeddingModel, run.retrievalVersion, run.promptVersion, run.outputSchemaVersion,
|
||||
run.model, run.createdAt, run.updatedAt, json(run)]);
|
||||
if (inserted.rowCount) return { run: clone(run), claimed: true };
|
||||
const existing = dataRow(await client.query<DataRow<RegulatoryMappingRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mappings WHERE id = $1 FOR UPDATE', [run.id]
|
||||
));
|
||||
if (!existing) throw new RepositoryConflictError('Regulatory mapping identity conflicted with another pipeline tuple.');
|
||||
if (['queued', 'running'].includes(existing.status) || existing.status === 'succeeded' && !forceRegeneration) {
|
||||
return { run: existing, claimed: false };
|
||||
}
|
||||
await client.query(`
|
||||
UPDATE assessor_regulatory_mappings SET
|
||||
run_status = $2, query_hash = $3, created_at = $4, updated_at = $5, data = $6::jsonb
|
||||
WHERE id = $1
|
||||
`, [run.id, run.status, run.queryHash, run.createdAt, run.updatedAt, json(run)]);
|
||||
return { run: clone(run), claimed: true };
|
||||
});
|
||||
}
|
||||
|
||||
async saveRegulatoryMappingRun(run: RegulatoryMappingRun): Promise<RegulatoryMappingRun> {
|
||||
const errors = regulatoryMappingRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.transaction(async client => {
|
||||
await client.query(`
|
||||
INSERT INTO assessor_regulatory_mappings
|
||||
(id, rule_key, source_tool, source_rule_id, run_status, query_hash, corpus_id, corpus_version,
|
||||
embedding_model, retrieval_version, prompt_version, output_schema_version, model, created_at, updated_at, data)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
run_status = excluded.run_status, updated_at = excluded.updated_at, data = excluded.data
|
||||
`, [run.id, run.ruleKey, run.sourceTool, run.sourceRuleId ?? null, run.status, run.queryHash, run.corpusId,
|
||||
run.corpusVersion, run.embeddingModel, run.retrievalVersion, run.promptVersion, run.outputSchemaVersion,
|
||||
run.model, run.createdAt, run.updatedAt, json(run)]);
|
||||
return clone(run);
|
||||
});
|
||||
}
|
||||
|
||||
async listRegulatoryMappingRuns(): Promise<RegulatoryMappingRun[]> {
|
||||
return dataRows(await this.pool.query<DataRow<RegulatoryMappingRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mappings ORDER BY created_at DESC, id DESC'
|
||||
));
|
||||
}
|
||||
|
||||
async createRegulatoryMappingBatchRun(run: RegulatoryMappingBatchRun): Promise<RegulatoryMappingBatchRun> {
|
||||
const errors = regulatoryMappingBatchRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.transaction(async client => {
|
||||
const inserted = await client.query(`
|
||||
INSERT INTO assessor_regulatory_mapping_batches
|
||||
(id, project_id, scope_key, filter_hash, idempotency_key, run_status, created_at, updated_at, data)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING data
|
||||
`, [run.id, run.projectId, run.scopeKey, run.filterHash, run.idempotencyKey, run.status,
|
||||
run.createdAt, run.updatedAt, json(run)]);
|
||||
if (inserted.rowCount) return clone(run);
|
||||
const existing = dataRow(await client.query<DataRow<RegulatoryMappingBatchRun>>(`
|
||||
SELECT data FROM assessor_regulatory_mapping_batches
|
||||
WHERE project_id = $1 AND (idempotency_key = $2 OR (scope_key = $3 AND run_status IN ('queued', 'running')))
|
||||
ORDER BY (idempotency_key = $2) DESC, created_at DESC, id DESC
|
||||
LIMIT 1
|
||||
`, [run.projectId, run.idempotencyKey, run.scopeKey]));
|
||||
if (!existing) throw new RepositoryConflictError('Regulatory mapping batch conflicted with another active request.');
|
||||
return existing;
|
||||
});
|
||||
}
|
||||
|
||||
async updateRegulatoryMappingBatchRun(run: RegulatoryMappingBatchRun): Promise<RegulatoryMappingBatchRun> {
|
||||
const errors = regulatoryMappingBatchRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.transaction(async client => {
|
||||
const existing = dataRow(await client.query<DataRow<RegulatoryMappingBatchRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mapping_batches WHERE id = $1 FOR UPDATE', [run.id]
|
||||
));
|
||||
if (!existing) throw new RepositoryNotFoundError('Regulatory mapping batch does not exist.');
|
||||
if (existing.projectId !== run.projectId || existing.scopeKey !== run.scopeKey || existing.idempotencyKey !== run.idempotencyKey) {
|
||||
throw new RepositoryConflictError('Regulatory mapping batch identity is immutable.');
|
||||
}
|
||||
if (['completed', 'failed'].includes(existing.status)) throw new RepositoryConflictError('A completed regulatory mapping batch is immutable.');
|
||||
const allowed = existing.status === 'queued' ? ['running', 'failed'] : ['running', 'completed', 'failed'];
|
||||
if (!allowed.includes(run.status)) throw new RepositoryConflictError(`Invalid regulatory mapping batch transition from ${existing.status} to ${run.status}.`);
|
||||
await client.query(`
|
||||
UPDATE assessor_regulatory_mapping_batches SET run_status = $2, updated_at = $3, data = $4::jsonb
|
||||
WHERE id = $1
|
||||
`, [run.id, run.status, run.updatedAt, json(run)]);
|
||||
return clone(run);
|
||||
});
|
||||
}
|
||||
|
||||
async listRegulatoryMappingBatchRuns(projectId: string, scopeKey?: string): Promise<RegulatoryMappingBatchRun[]> {
|
||||
return dataRows(await this.pool.query<DataRow<RegulatoryMappingBatchRun>>(`
|
||||
SELECT data FROM assessor_regulatory_mapping_batches
|
||||
WHERE project_id = $1 AND ($2::text IS NULL OR scope_key = $2)
|
||||
ORDER BY created_at DESC, id DESC
|
||||
`, [projectId, scopeKey ?? null]));
|
||||
}
|
||||
|
||||
async createRegulatoryMappingDecision(
|
||||
decision: RegulatoryMappingDecision,
|
||||
expectedLatestDecisionId?: string
|
||||
): Promise<RegulatoryMappingDecision> {
|
||||
const errors = regulatoryMappingDecisionValidationErrors(decision);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.transaction(async client => {
|
||||
// Serializes decisions for one rule and control, so two writers cannot both see the same latest decision.
|
||||
await client.query('SELECT pg_advisory_xact_lock(hashtext($1))', [`regulatory-decision:${decision.ruleKey}:${decision.controlId}`]);
|
||||
const run = dataRow(await client.query<DataRow<RegulatoryMappingRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mappings WHERE id = $1 FOR UPDATE', [decision.mappingId]
|
||||
));
|
||||
if (!run || run.ruleKey !== decision.ruleKey || run.status !== 'succeeded') {
|
||||
throw new RepositoryNotFoundError('Completed regulatory mapping does not exist for this rule.');
|
||||
}
|
||||
if (!run.candidates.some(candidate => candidate.controlId === decision.controlId)) {
|
||||
throw new RepositoryValidationError('Decision control was not a grounded candidate.');
|
||||
}
|
||||
const latest = dataRow(await client.query<DataRow<RegulatoryMappingDecision>>(`
|
||||
SELECT data FROM assessor_regulatory_mapping_decisions
|
||||
WHERE rule_key = $1 AND control_id = $2
|
||||
ORDER BY created_at DESC, id DESC LIMIT 1
|
||||
`, [decision.ruleKey, decision.controlId]));
|
||||
if ((expectedLatestDecisionId ?? undefined) !== (latest?.id ?? undefined)) {
|
||||
throw new RepositoryConflictError('Regulatory mapping decision changed before this update.');
|
||||
}
|
||||
const stored = { ...decision, ...(latest ? { previousDecisionId: latest.id } : {}) };
|
||||
await client.query(`
|
||||
INSERT INTO assessor_regulatory_mapping_decisions
|
||||
(id, rule_key, source_tool, source_rule_id, mapping_id, control_id, decision, previous_decision_id, created_at, data)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb)
|
||||
`, [stored.id, stored.ruleKey, stored.sourceTool, stored.sourceRuleId ?? null, stored.mappingId, stored.controlId,
|
||||
stored.decision, stored.previousDecisionId ?? null, stored.createdAt, json(stored)]);
|
||||
return clone(stored);
|
||||
});
|
||||
}
|
||||
|
||||
async listRegulatoryMappingDecisions(ruleKey: string): Promise<RegulatoryMappingDecision[]> {
|
||||
return dataRows(await this.pool.query<DataRow<RegulatoryMappingDecision>>(`
|
||||
SELECT data FROM assessor_regulatory_mapping_decisions
|
||||
WHERE rule_key = $1
|
||||
ORDER BY created_at DESC, id DESC
|
||||
`, [ruleKey]));
|
||||
}
|
||||
|
||||
async createAiOverviewRun(run: AiOverviewRun): Promise<AiOverviewRun> {
|
||||
const errors = aiOverviewRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.transaction(async client => {
|
||||
const idempotent = dataRow(await client.query<DataRow<AiOverviewRun>>(`
|
||||
SELECT data FROM assessor_ai_overview_runs
|
||||
WHERE project_id = $1 AND scope_key = $2 AND idempotency_key = $3
|
||||
`, [run.projectId, run.scopeKey, run.idempotencyKey]));
|
||||
if (idempotent) return idempotent;
|
||||
await client.query(`
|
||||
INSERT INTO assessor_ai_overview_runs
|
||||
(id, project_id, scope_key, assessment_ids, scan_run_ids, idempotency_key, run_status,
|
||||
input_hash, prompt_version, output_schema_version, model, created_at, updated_at, data)
|
||||
VALUES ($1, $2, $3, $4::text[], $5::text[], $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb)
|
||||
`, [run.id, run.projectId, run.scopeKey, run.assessmentIds, run.scanRunIds, run.idempotencyKey,
|
||||
run.status, run.inputHash, run.promptVersion, run.outputSchemaVersion, run.model,
|
||||
run.createdAt, run.updatedAt, json(run)]);
|
||||
return clone(run);
|
||||
});
|
||||
}
|
||||
|
||||
async updateAiOverviewRun(run: AiOverviewRun): Promise<AiOverviewRun> {
|
||||
const errors = aiOverviewRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.transaction(async client => {
|
||||
const existing = dataRow(await client.query<DataRow<AiOverviewRun>>(
|
||||
'SELECT data FROM assessor_ai_overview_runs WHERE id = $1 FOR UPDATE', [run.id]
|
||||
));
|
||||
if (!existing) throw new RepositoryNotFoundError(`AI overview run ${run.id} does not exist.`);
|
||||
assertAiOverviewTransition(existing, run);
|
||||
await client.query(`
|
||||
UPDATE assessor_ai_overview_runs SET
|
||||
run_status = $2, updated_at = $3, data = $4::jsonb
|
||||
WHERE id = $1
|
||||
`, [run.id, run.status, run.updatedAt, json(run)]);
|
||||
return clone(run);
|
||||
});
|
||||
}
|
||||
|
||||
async listAiOverviewRuns(projectId: string, scopeKey?: string): Promise<AiOverviewRun[]> {
|
||||
return dataRows(await this.pool.query<DataRow<AiOverviewRun>>(`
|
||||
SELECT data FROM assessor_ai_overview_runs
|
||||
WHERE project_id = $1 AND ($2::text IS NULL OR scope_key = $2)
|
||||
ORDER BY created_at DESC, id DESC
|
||||
`, [projectId, scopeKey ?? null]));
|
||||
}
|
||||
|
||||
async deleteAiOverviewRuns(projectId: string, runIds: string[]): Promise<void> {
|
||||
if (!runIds.length) return;
|
||||
await this.transaction(client => client.query(
|
||||
'DELETE FROM assessor_ai_overview_runs WHERE project_id = $1 AND id = ANY($2::text[])', [projectId, runIds]
|
||||
));
|
||||
}
|
||||
|
||||
async exportProject(projectId: string): Promise<ComplianceProjectExport> {
|
||||
return {
|
||||
regulatoryMappingBatchRuns: await this.listRegulatoryMappingBatchRuns(projectId),
|
||||
regulatoryMappingRuns: (await this.listRegulatoryMappingRuns()).filter(item => item.status === 'succeeded'),
|
||||
regulatoryMappingDecisions: dataRows(await this.pool.query<DataRow<RegulatoryMappingDecision>>(
|
||||
'SELECT data FROM assessor_regulatory_mapping_decisions ORDER BY created_at, id'
|
||||
)),
|
||||
aiOverviewRuns: (await this.listAiOverviewRuns(projectId)).filter(item => item.status === 'succeeded')
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -3,7 +3,7 @@ import type {
|
|||
RegulatoryChunk,
|
||||
RegulatoryRetrievalCandidate,
|
||||
RegulatoryRetrievalFilters
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
} from '../../src/compliance/regulatory-rag';
|
||||
|
||||
export const REGULATORY_QDRANT_COLLECTION = 'assessor_regulatory_chunks_v2';
|
||||
export const REGULATORY_DENSE_VECTOR = 'dense';
|
||||
|
|
@ -3,7 +3,7 @@ import {
|
|||
REGULATORY_CHUNK_SCHEMA_VERSION,
|
||||
type RegulatoryChunk,
|
||||
type RegulatoryControlDocument
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
} from '../../src/compliance/regulatory-rag';
|
||||
|
||||
export interface RegulatoryChunkingOptions {
|
||||
maximumCharacters: number;
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { join, resolve, sep } from 'node:path';
|
||||
import type { RegulatoryCorpusV2 } from '../../src/domain/regulatory-rag';
|
||||
import type { RegulatoryCorpusJob, RegulatoryCorpusStatus, RegulatoryCorpusSummary } from '../../src/domain/regulatory-corpus-management';
|
||||
import type { RegulatoryCorpusV2 } from '../../src/compliance/regulatory-rag';
|
||||
import type { RegulatoryCorpusJob, RegulatoryCorpusStatus, RegulatoryCorpusSummary } from '../../src/compliance/regulatory-corpus-management';
|
||||
import type { IsoPdfCorpusOptions } from './iso-pdf-corpus';
|
||||
import { prepareIsoCorpusInWorker } from './iso-pdf-worker-client';
|
||||
import type { RegulatoryIngestionService } from './regulatory-rag-service';
|
||||
|
|
@ -3,9 +3,9 @@ import type {
|
|||
RegulatoryRetrievalCandidate,
|
||||
RegulatoryRetrievalFilters,
|
||||
RegulatoryRetriever
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
} from '../../src/compliance/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query';
|
||||
import { buildRegulatoryFindingQuery } from '../../src/compliance/regulatory-query';
|
||||
|
||||
export interface RegulatoryFindingRetrievalResult {
|
||||
sourceResultId: string;
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import type { RegulatoryControlDocument } from '../../src/domain/regulatory-rag';
|
||||
import type { RegulatoryControlDocument } from '../../src/compliance/regulatory-rag';
|
||||
import { validateRegulatoryControlDocument } from './regulatory-chunker';
|
||||
import { createRegulatoryRagRuntime } from './regulatory-rag-runtime';
|
||||
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { AiCompletionClient } from '../ai/litellm-client';
|
||||
import { LiteLlmClientError } from '../ai/litellm-client';
|
||||
import type { AssessorRepository } from '../../src/persistence/assessor-repository';
|
||||
import type { ComplianceReadPort, ComplianceStore } from '../../src/compliance/compliance-store';
|
||||
import { RepositoryConflictError, RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors';
|
||||
import {
|
||||
REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION,
|
||||
REGULATORY_MAPPING_PROMPT_VERSION,
|
||||
REGULATORY_RETRIEVAL_VERSION,
|
||||
REGULATORY_GPO_QUERY_VERSION,
|
||||
emptyRegulatoryMappingBatchCounts,
|
||||
groundRegulatoryMappingResult,
|
||||
normalizeRegulatoryMappingBatchScope,
|
||||
|
|
@ -18,18 +19,24 @@ import {
|
|||
type RegulatoryMappingCandidate,
|
||||
type RegulatoryMappingCreateRequest,
|
||||
type RegulatoryMappingDecision,
|
||||
type RegulatoryEarlierControl,
|
||||
type RegulatoryMappingDecisionCreateRequest,
|
||||
type RegulatoryMappingEvidence,
|
||||
type RegulatoryMappingResponse,
|
||||
type RegulatoryMappingResult,
|
||||
type RegulatoryMappingRun
|
||||
} from '../../src/domain/regulatory-mapping';
|
||||
import type { RegulatoryRetrievalCandidate } from '../../src/domain/regulatory-rag';
|
||||
type RegulatoryMappingRun,
|
||||
type RegulatoryControlFinding,
|
||||
type RegulatoryControlOverviewItem,
|
||||
type RegulatoryControlOverviewResponse,
|
||||
acceptedControlIds,
|
||||
regulatoryControlTheme
|
||||
} from '../../src/compliance/regulatory-mapping';
|
||||
import type { RegulatoryRetrievalCandidate } from '../../src/compliance/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import { privacySafeFinding } from '../../src/domain/privacy-safe-finding';
|
||||
import type { RegulatoryFindingRetrievalService } from './regulatory-finding-retrieval';
|
||||
import type { RegulatoryChunkIndex } from './qdrant-regulatory-index';
|
||||
import { buildRegulatoryFindingQuery, isRegulatoryMappableResult } from '../../src/profiles/public-search/regulatory-query';
|
||||
import { buildRegulatoryFindingQuery, isRegulatoryMappableResult } from '../../src/compliance/regulatory-query';
|
||||
import { canonicalJson } from '../../src/shared/canonical-json';
|
||||
|
||||
const POLL_AFTER_MS = 1_500;
|
||||
|
|
@ -37,6 +44,9 @@ const UNSUPPORTED_KIND_MESSAGE = 'Only canonical scanner findings and GPO baseli
|
|||
const MAXIMUM_PROMPT_CHARACTERS = 48_000;
|
||||
const BATCH_PAGE_SIZE = 500;
|
||||
const DEFAULT_MAXIMUM_CONCURRENT_RUNS = 2;
|
||||
// User decision 2026-10-05 (provisional): the first proposals of a finished mapping are accepted automatically.
|
||||
export const AUTO_ACCEPTED_PROPOSALS = 2;
|
||||
export const AUTO_ACCEPT_COMMENT = 'Accepted automatically: one of the first two proposals of the mapping.';
|
||||
|
||||
export interface RegulatoryMappingServiceOptions {
|
||||
findingRetrieval?: RegulatoryFindingRetrievalService;
|
||||
|
|
@ -61,7 +71,11 @@ export class RegulatoryMappingService {
|
|||
private activeWorkerCount = 0;
|
||||
private activationPending = false;
|
||||
|
||||
constructor(private readonly repository: AssessorRepository, private readonly options: RegulatoryMappingServiceOptions = {}) {
|
||||
constructor(
|
||||
private readonly read: ComplianceReadPort,
|
||||
private readonly store: ComplianceStore,
|
||||
private readonly options: RegulatoryMappingServiceOptions = {}
|
||||
) {
|
||||
this.now = options.now ?? (() => new Date());
|
||||
this.maximumConcurrentRuns = options.maximumConcurrentRuns ?? DEFAULT_MAXIMUM_CONCURRENT_RUNS;
|
||||
if (!Number.isInteger(this.maximumConcurrentRuns) || this.maximumConcurrentRuns < 1 || this.maximumConcurrentRuns > 8) {
|
||||
|
|
@ -87,8 +101,8 @@ export class RegulatoryMappingService {
|
|||
|
||||
async recoverInterruptedRuns(): Promise<void> {
|
||||
const completedAt = this.now().toISOString();
|
||||
for (const run of (await this.repository.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) {
|
||||
await this.repository.saveRegulatoryMappingRun({
|
||||
for (const run of (await this.store.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) {
|
||||
await this.store.saveRegulatoryMappingRun({
|
||||
...run,
|
||||
status: 'failed',
|
||||
updatedAt: completedAt,
|
||||
|
|
@ -96,9 +110,9 @@ export class RegulatoryMappingService {
|
|||
errorCode: 'regulatory_mapping_interrupted'
|
||||
});
|
||||
}
|
||||
for (const project of await this.repository.listProjects()) {
|
||||
for (const run of (await this.repository.listRegulatoryMappingBatchRuns(project.id)).filter(item => ['queued', 'running'].includes(item.status))) {
|
||||
await this.repository.updateRegulatoryMappingBatchRun({
|
||||
for (const project of await this.read.listProjects()) {
|
||||
for (const run of (await this.store.listRegulatoryMappingBatchRuns(project.id)).filter(item => ['queued', 'running'].includes(item.status))) {
|
||||
await this.store.updateRegulatoryMappingBatchRun({
|
||||
...run,
|
||||
status: 'failed',
|
||||
updatedAt: completedAt,
|
||||
|
|
@ -116,8 +130,9 @@ export class RegulatoryMappingService {
|
|||
const availability = supported ? await this.availability() : { enabled: false };
|
||||
const enabled = availability.enabled && supported;
|
||||
const ruleKey = supported ? this.ruleKey(finding) : undefined;
|
||||
const run = ruleKey && this.options.client ? await this.repository.getRegulatoryMappingRun(this.mappingId(ruleKey)) : null;
|
||||
const decisions = ruleKey ? await this.repository.listRegulatoryMappingDecisions(ruleKey) : [];
|
||||
const run = ruleKey && this.options.client ? await this.store.getRegulatoryMappingRun(this.mappingId(ruleKey)) : null;
|
||||
const decisions = ruleKey ? await this.store.listRegulatoryMappingDecisions(ruleKey) : [];
|
||||
const earlierControls = ruleKey ? await this.earlierControls(ruleKey, run, decisions) : [];
|
||||
return {
|
||||
version: 1,
|
||||
projectId,
|
||||
|
|
@ -130,10 +145,40 @@ export class RegulatoryMappingService {
|
|||
: !availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}),
|
||||
...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}),
|
||||
...(run ? { run: withoutEvidenceText(run) } : {}),
|
||||
decisions
|
||||
decisions,
|
||||
...(earlierControls.length ? { earlierControls } : {})
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The run that grounds a decision: the current one, or for an earlier decision the newest older succeeded
|
||||
* run of the rule that retrieved the control, so it stays reviewable after a pipeline version change.
|
||||
*/
|
||||
private async groundingMappingId(ruleKey: string, controlId: string): Promise<string> {
|
||||
const currentId = this.mappingId(ruleKey);
|
||||
const current = await this.store.getRegulatoryMappingRun(currentId);
|
||||
if (!current || current.candidates.some(candidate => candidate.controlId === controlId)) return currentId;
|
||||
const older = (await this.store.listRegulatoryMappingRuns()).find(run => run.ruleKey === ruleKey && run.status === 'succeeded'
|
||||
&& run.candidates.some(candidate => candidate.controlId === controlId));
|
||||
return older?.id ?? currentId;
|
||||
}
|
||||
|
||||
/** Decided controls the current run did not retrieve, titled from the newest older run of the rule that did. */
|
||||
private async earlierControls(ruleKey: string, run: RegulatoryMappingRun | null, decisions: RegulatoryMappingDecision[]): Promise<RegulatoryEarlierControl[]> {
|
||||
const current = new Set(run?.candidates.map(candidate => candidate.controlId) ?? []);
|
||||
const missing = [...new Set(decisions.map(decision => decision.controlId))].filter(controlId => !current.has(controlId));
|
||||
if (!missing.length) return [];
|
||||
const known = new Map<string, RegulatoryMappingCandidate>();
|
||||
for (const older of await this.store.listRegulatoryMappingRuns()) {
|
||||
if (older.ruleKey !== ruleKey) continue;
|
||||
for (const candidate of older.candidates) if (!known.has(candidate.controlId)) known.set(candidate.controlId, candidate);
|
||||
}
|
||||
return missing.sort((left, right) => left.localeCompare(right, 'en', { numeric: true })).map(controlId => {
|
||||
const candidate = known.get(controlId);
|
||||
return { controlId, annexAControlId: candidate?.annexAControlId ?? `A.${controlId}`, ...(candidate?.title ? { title: candidate.title } : {}) };
|
||||
});
|
||||
}
|
||||
|
||||
async start(projectId: string, request: RegulatoryMappingCreateRequest): Promise<RegulatoryMappingResponse> {
|
||||
if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503);
|
||||
const finding = await this.finding(projectId, request.resultProjectionId);
|
||||
|
|
@ -147,7 +192,7 @@ export class RegulatoryMappingService {
|
|||
async getBatch(projectId: string, requestedScope: RegulatoryMappingBatchScope): Promise<RegulatoryMappingBatchResponse> {
|
||||
const scope = await this.validateBatchScope(projectId, requestedScope);
|
||||
const scopeKey = this.batchScopeKey(projectId, scope);
|
||||
const run = (await this.repository.listRegulatoryMappingBatchRuns(projectId, scopeKey))[0];
|
||||
const run = (await this.store.listRegulatoryMappingBatchRuns(projectId, scopeKey))[0];
|
||||
const availability = await this.availability();
|
||||
return {
|
||||
version: 1,
|
||||
|
|
@ -183,7 +228,7 @@ export class RegulatoryMappingService {
|
|||
createdAt: timestamp,
|
||||
updatedAt: timestamp
|
||||
};
|
||||
const run = await this.repository.createRegulatoryMappingBatchRun(candidate);
|
||||
const run = await this.store.createRegulatoryMappingBatchRun(candidate);
|
||||
if (run.id === candidate.id) this.track(this.processBatch(run));
|
||||
return this.getBatch(projectId, {
|
||||
assessmentIds: run.assessmentIds,
|
||||
|
|
@ -198,22 +243,97 @@ export class RegulatoryMappingService {
|
|||
if (!isRegulatoryMappableResult(finding)) throw new RegulatoryMappingServiceError('unsupported_finding_kind', UNSUPPORTED_KIND_MESSAGE, 422);
|
||||
const ruleKey = this.ruleKey(finding);
|
||||
const comment = request.comment?.trim();
|
||||
const mappingId = await this.groundingMappingId(ruleKey, request.controlId);
|
||||
const decision: RegulatoryMappingDecision = {
|
||||
version: 1,
|
||||
id: `regulatory-decision-${randomUUID()}`,
|
||||
ruleKey,
|
||||
sourceTool: finding.sourceTool ?? 'unknown',
|
||||
...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}),
|
||||
mappingId: this.mappingId(ruleKey),
|
||||
mappingId,
|
||||
controlId: request.controlId,
|
||||
decision: request.decision,
|
||||
...(comment ? { comment } : {}),
|
||||
createdAt: this.now().toISOString()
|
||||
};
|
||||
await this.repository.createRegulatoryMappingDecision(decision, request.expectedLatestDecisionId);
|
||||
await this.store.createRegulatoryMappingDecision(decision, request.expectedLatestDecisionId);
|
||||
return this.get(projectId, request.resultProjectionId);
|
||||
}
|
||||
|
||||
/** Accepted controls of every mappable finding in a client timeslot, for the Work `ISO control` and `ISO theme` filters. */
|
||||
async controlAssignments(projectId: string, requested: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>): Promise<RegulatoryControlOverviewResponse> {
|
||||
return this.controlOverview(projectId, requested, { deviationsOnly: false });
|
||||
}
|
||||
|
||||
/**
|
||||
* Accepted controls of the deviations in one client timeslot. Decisions are global per scanner rule;
|
||||
* a GPO setting counts once, like its rolled-up Work record, however many baseline comparisons it has.
|
||||
*/
|
||||
async controlOverview(projectId: string, requested: Pick<RegulatoryMappingBatchScope, 'assessmentIds' | 'scanRunIds'>,
|
||||
{ deviationsOnly = true }: { deviationsOnly?: boolean } = {}): Promise<RegulatoryControlOverviewResponse> {
|
||||
const scope = await this.validateBatchScope(projectId, { assessmentIds: requested.assessmentIds, scanRunIds: requested.scanRunIds,
|
||||
filters: deviationsOnly ? { deviation: true } : {} });
|
||||
const byRule = new Map<string, Map<string, RegulatoryControlFinding>>();
|
||||
const results = (await this.queryBatchResults(projectId, scope)).filter(isRegulatoryMappableResult);
|
||||
const settingIds = await this.gpoSettingIds(projectId, scope, results);
|
||||
for (const finding of results) {
|
||||
const settingId = settingIds.get(finding.resultProjectionId);
|
||||
const identity = findingIdentity(finding, settingId);
|
||||
const findings = byRule.get(this.ruleKey(finding)) ?? new Map<string, RegulatoryControlFinding>();
|
||||
// The setting ID only lets Work match its rolled-up GPO setting; it is never displayed or exported.
|
||||
if (!findings.has(identity)) findings.set(identity, { resultProjectionId: finding.resultProjectionId, sourceKind: finding.sourceKind,
|
||||
sourceResultId: finding.sourceResultId, sourceRuleId: settingId ?? finding.sourceRuleId, policyScope: finding.policyScope });
|
||||
byRule.set(this.ruleKey(finding), findings);
|
||||
}
|
||||
// Control titles come from the newest succeeded run of each rule that retrieved the control.
|
||||
const titles = new Map<string, Map<string, RegulatoryMappingCandidate>>();
|
||||
for (const run of await this.store.listRegulatoryMappingRuns()) {
|
||||
if (run.status !== 'succeeded' || !byRule.has(run.ruleKey)) continue;
|
||||
const known = titles.get(run.ruleKey) ?? new Map<string, RegulatoryMappingCandidate>();
|
||||
for (const candidate of run.candidates) if (!known.has(candidate.controlId)) known.set(candidate.controlId, candidate);
|
||||
titles.set(run.ruleKey, known);
|
||||
}
|
||||
const controls = new Map<string, { item: RegulatoryControlOverviewItem; findings: Map<string, RegulatoryControlFinding> }>();
|
||||
const covered = new Set<string>();
|
||||
for (const [ruleKey, findings] of byRule) {
|
||||
for (const controlId of acceptedControlIds(await this.store.listRegulatoryMappingDecisions(ruleKey))) {
|
||||
const candidate = titles.get(ruleKey)?.get(controlId);
|
||||
const entry = controls.get(controlId) ?? { findings: new Map(), item: {
|
||||
controlId, annexAControlId: candidate?.annexAControlId ?? `A.${controlId}`, findings: [],
|
||||
...(candidate?.title ? { title: candidate.title } : {}),
|
||||
...(regulatoryControlTheme(controlId) ? { theme: regulatoryControlTheme(controlId) } : {}) } };
|
||||
if (!entry.item.title && candidate?.title) entry.item.title = candidate.title;
|
||||
for (const [identity, finding] of findings) { entry.findings.set(identity, finding); covered.add(identity); }
|
||||
controls.set(controlId, entry);
|
||||
}
|
||||
}
|
||||
return {
|
||||
version: 1, projectId, scope: { assessmentIds: scope.assessmentIds, scanRunIds: scope.scanRunIds }, findingCount: covered.size,
|
||||
controls: [...controls.values()].map(({ item, findings }) => ({ ...item, findings: [...findings.values()] }))
|
||||
.sort((left, right) => right.findings.length - left.findings.length
|
||||
|| left.controlId.localeCompare(right.controlId, 'en', { numeric: true }))
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Accepts the first proposals of a finished mapping, in the model's order. A control that already has a
|
||||
* decision keeps it, so a manual reject is never overridden; a failure here never fails the mapping.
|
||||
*/
|
||||
private async autoAcceptFirstProposals(run: RegulatoryMappingRun, result: RegulatoryMappingResult): Promise<void> {
|
||||
if (result.outcome !== 'mapped') return;
|
||||
try {
|
||||
const decided = new Set((await this.store.listRegulatoryMappingDecisions(run.ruleKey)).map(item => item.controlId));
|
||||
for (const proposal of result.mappings.slice(0, AUTO_ACCEPTED_PROPOSALS)) {
|
||||
if (decided.has(proposal.controlId)) continue;
|
||||
await this.store.createRegulatoryMappingDecision({
|
||||
version: 1, id: `regulatory-decision-${randomUUID()}`, ruleKey: run.ruleKey, sourceTool: run.sourceTool,
|
||||
...(run.sourceRuleId ? { sourceRuleId: run.sourceRuleId } : {}), mappingId: run.id, controlId: proposal.controlId,
|
||||
decision: 'accepted', comment: AUTO_ACCEPT_COMMENT, createdAt: this.now().toISOString()
|
||||
});
|
||||
}
|
||||
} catch { /* A concurrent manual decision wins; the mapping itself has succeeded. */ }
|
||||
}
|
||||
|
||||
// The global sharing key: the rule identity `tool:ruleId`, shared across every tenant.
|
||||
// Rules without an id fall back to a stable hash of the non-personal check title so
|
||||
// identical findings still share, while never keying on tenant or occurrence.
|
||||
|
|
@ -228,10 +348,18 @@ export class RegulatoryMappingService {
|
|||
// Deterministic id per (ruleKey, pipeline versions) so one row exists per rule per
|
||||
// version tuple: regeneration reuses it, and a version bump yields a new mapping.
|
||||
private mappingId(ruleKey: string): string {
|
||||
const tuple = [ruleKey, this.options.corpusVersion, REGULATORY_RETRIEVAL_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, this.options.client!.model].join(' | ||||