diff --git a/.env.postgres.example b/.env.postgres.example index 74e5e86..c35b306 100644 --- a/.env.postgres.example +++ b/.env.postgres.example @@ -10,7 +10,7 @@ ASSESSOR_GPO_JOB_LEASE_MS=60000 ASSESSOR_GPO_JOB_RETRY_MS=5000 ASSESSOR_GPO_JOB_POLL_MS=1000 -# Qdrant is a rebuildable regulatory retrieval index. This key is local/deployment secret material. +# Qdrant is a rebuildable regulatory retrieval index. A key is optional for an unsecured loopback/internal self-host, but required for remote hosts. ASSESSOR_QDRANT_URL=http://127.0.0.1:6333 ASSESSOR_QDRANT_PORT=6333 ASSESSOR_QDRANT_API_KEY=replace-with-a-long-random-local-qdrant-key @@ -26,6 +26,9 @@ ASSESSOR_QDRANT_API_KEY=replace-with-a-long-random-local-qdrant-key # ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS=60000 # ASSESSOR_RAG_CHUNK_CHARACTERS=1600 # ASSESSOR_RAG_EMBEDDING_BATCH_SIZE=32 +# The mapping API is enabled only for this explicit, already-ingested corpus. +# ASSESSOR_RAG_ACTIVE_CORPUS_ID=iso-27001-27002-de +# ASSESSOR_RAG_ACTIVE_CORPUS_VERSION=din-2024-de-example # Optional work LiteLLM AI Overview. Leave ASSESSOR_AI_MODE unset to keep outbound AI disabled. # Store real values only in the ignored .env.postgres or deployment environment secret. diff --git a/.gitignore b/.gitignore index 7eef6e7..56515a8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ Mock-Defender/ .claude/scheduled_tasks.lock .codex-patches/ .tmp/ +/ISO27001 2024.PDF +/ISO27002 2024.PDF +backups/ diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index cf7d0b9..6a28793 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -2,6 +2,8 @@ Status: lebendes Projektdokument +ISO-27002-Mapping-Vertikalschnitt 2026-09-15 (Phase 13D, ausdruecklicher Nutzerauftrag): Die beiden lokal lizenzierten deutschen DIN-2024-PDFs werden serverseitig mit PDF.js und Positionsdaten in einen nicht eingecheckten Korpus-v2 ueberfuehrt. Passwortpflichtige, kopiergeschuetzte, textlose, uebergrosse oder strukturell unvollstaendige Dokumente scheitern geschlossen; ISO 27001 und 27002 muessen dieselben 93 Kontrollen mit 37/8/14/34 Themenkontrollen liefern. `assessor_regulatory_chunks_v2` enthaelt 705 mit lokalem BGE-M3 erzeugte, abschnitts- und kontrollsichere Chunks aus 558 Dokumenten, einschliesslich ISO-27001-Referenzen, Seiten-, Editions-, Parser- und Hashprovenienz; Sparse bleibt reserviert. Der Einzel-Finding-Dienst gruppiert 32 Dense-Treffer auf acht Kontrollen, bindet ISO-27001-Evidenz an, uebergibt nur die minimierte Public-Signal-Query und hoechstens 48.000 Zeichen ISO-Evidenz an das vorhandene Work-LiteLLM und akzeptiert ausschliesslich lokal validierte, evidenzgebundene JSON-Vorschlaege. Migration 025, beide Repositories und drei projektgebundene APIs speichern Runs und Accept/Reject-Entscheidungsereignisse append-only; Work zeigt den manuellen Ablauf inline mit Polling und Seitenbelegen. Ein realer synthetischer End-to-End-Test ueber kanonisches PostgreSQL, llama.cpp/BGE-M3, Qdrant und den freigegebenen LiteLLM-Gateway bestand einschliesslich Optimistic-Conflict und Cascade-Cleanup. Der laufende Entwicklungsserver wendete die additive Migration um 13:42 vor dem um 13:52 erstellten und verifizierten Backup automatisch an; beide neuen Tabellen waren leer. Diese Abweichung ist offen dokumentiert und wird nicht als Vorabbackup ausgegeben. Scannerverarbeitung blieb unveraendert; kein Manifestbump oder Reimport. + Regulatory-RAG-Llama-Livebaseline 2026-09-15 (Phase 13D, Nutzerstart): Der lokal gestartete llama.cpp-Server Build 10472 meldet Health 200 und bedient BGE-M3 FP16 ueber den OpenAI-kompatiblen Endpoint. Die produktiv gebauten CLIs indexierten den eingecheckten ausschliesslich synthetischen EN/DE-Korpus mit sechs Dokumenten als sechs 1.024-dimensionale Chunks in den kanonischen Qdrant 1.19.1. Drei ausdruecklich sprachuebergreifende Smoke-Cases erreichten bei K=3 HitRate 1,0, Recall 1,0 und MRR 1,0; alle erwarteten Kontrollen standen auf Rang 1. Das belegt End-to-End-Verkabelung und grundlegende Sprachueberquerung, wegen nur drei klar getrennten Kontrollthemen aber keine produktive Retrievalqualitaet. Qdrant enthaelt sechs synthetische Punkte; PostgreSQL und Realdaten wurden nicht beruehrt. 1.285 Standardtests bestanden, acht optionale PostgreSQL- und ein optionaler Qdrant-Test uebersprungen; Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich (bestehender Bundlehinweis). Ein repraesentativer autorisierter Korpus und unabhaengig gelabelter EN/DE-Satz bleiben fuer die reale Baseline erforderlich. Regulatory-RAG-Local-Embeddings 2026-09-15 (Phase 13D, Nutzerentscheidung): Die autorisierte Legacy-Collection `regulation_catalog_v1` wurde nach exakter Ein-Collection-Pruefung geloescht; der leere alte Qdrant-1.12.4-Container wurde entfernt, sein bisheriges Volume jedoch bewahrt. Der authentifizierte `assessor-2_0-qdrant-1` mit Qdrant 1.19.1 laeuft nun am kanonischen `127.0.0.1:6333`; Auth 401/200, gruene Collection und null Punkte sind bestaetigt. Fuer die lokale Dense-Entwicklung ist BGE-M3 mit llama.cpp festgelegt: multilingual, native 1.024 Dimensionen und damit passend zur Collection. Der Embeddingadapter erlaubt HTTP nur fuer Loopback, verlangt weiterhin einen Bearer-Key und lehnt Remote-HTTP ab. Ein neuer optionaler Request-Dimensionsschalter bleibt fuer einen spaeteren LiteLLM-Vergleich mit `text-embedding-3-large` standardmaessig aus, weil llama.cpp die native Modelldimension liefert. Das installierte llama.cpp Build 10472 ist vorhanden; ein BGE-M3-GGUF beziehungsweise dessen freigegebener Download und Serverstart fehlen noch. 1.285 Standardtests bestanden; acht optionale PostgreSQL- und ein optionaler Qdrant-Test uebersprungen, Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich (bestehender Bundlehinweis). Keine PostgreSQL-Mutation, keine Realdateninspektion und kein Reimport. @@ -1646,14 +1648,14 @@ Status: in Arbeit Ziele: - Regulatorische Kontrolltexte versioniert und reproduzierbar fuer mehrsprachige semantische Suche aufbereiten. -- Qdrant ausschliesslich als rebuildbaren Retrievalindex verwenden; PostgreSQL bleibt fachliche Quelle fuer Scans, Findings, Ergebnisse, Reviews und spaetere Mappingentscheidungen. +- Qdrant ausschliesslich als rebuildbaren Retrievalindex verwenden; PostgreSQL bleibt fachliche Quelle fuer Scans, Findings, Ergebnisse, Reviews, Mappinglaeufe und Entscheidungen. - Retrieval, Embedding, Chunking, spaetere Hybridfusion, Reranking und Mapping als getrennte Backendgrenzen halten. Umgesetzter Grundstand vom 2026-09-15: - Lokaler und Deployment-Compose-Stack enthalten das gepinnte Qdrant-Image 1.19.1, einen API-Key, deaktivierte ausgehende Nutzungstelemetrie und ein getrenntes persistentes Volume. Lokal ist nur die REST-Schnittstelle auf Loopback veroeffentlicht; im Deployment bleibt Qdrant intern. - `RegulatoryControlDocument` und `RegulatoryChunk` definieren einen versionierten, framework-/versions-/control-/chunk-/sprachgebundenen Vertrag. Chunk- und Inhaltsidentitaeten sind deterministisch; Originalkorpus und Korpusversion bleiben die Wiederaufbauquelle. -- Die Collection `assessor_regulatory_chunks_v1` verwendet benannte Dense-/Sparse-Vektoren, Cosine fuer Dense und Payloadindizes fuer alle Filterfelder. Sparse ist nur die vorbereitete Schema- und Schnittstellenposition, noch kein aktiver Retriever. +- Die aktive Collection `assessor_regulatory_chunks_v2` verwendet benannte Dense-/Sparse-Vektoren, Cosine fuer Dense und zwoelf Payloadindizes fuer alle Filterfelder. Sparse ist nur die vorbereitete Schema- und Schnittstellenposition, noch kein aktiver Retriever. - Ein abgesicherter OpenAI-kompatibler Embeddingadapter verlangt fuer Remoteziele HTTPS, erlaubt HTTP nur auf Loopback fuer den lokalen llama.cpp-Testbetrieb und bindet ein ausdruecklich freigegebenes mehrsprachiges Modell an dessen exakte Dimension. Redirects, unbeschraenkte Antworten und Dimensionsdrift werden abgewiesen. Ein opt-in `dimensions`-Request unterstuetzt spaetere OpenAI-/LiteLLM-Vergleiche, bleibt fuer llama.cpp aus. - CLI-Ingestion validiert und segmentiert einen begrenzten JSON-Korpus, bettet Chunks in Batches ein und ersetzt dokumentgebundene Qdrant-Punkte. Die Queryseite verwendet dasselbe Modell und kann Korpus, Framework, Version, Control und Sprache exakt filtern. - `RegulatoryRetriever`, `RegulatoryCandidateRetriever` und `RegulatoryReranker` trennen Verbraucher, Candidate Retrieval und optionale Nachsortierung. Spaetere Hybridfusion kann die Candidate-Stufe ersetzen, ohne Mappingcode an Qdrant zu binden. @@ -1662,15 +1664,17 @@ Umgesetzter Grundstand vom 2026-09-15: - Ein eingecheckter synthetischer EN/DE-Smoke-Korpus und drei sprachuebergreifende Labelcases sichern den ausfuehrbaren Vertrag. Der opt-in WSL-Qdrant-Integrationstest prueft mit deterministischen synthetischen Vektoren den realen 1.19.1-Collection-/Index-/Replace-/Filter-/Retrievalpfad; er bewertet ausdruecklich keine Embeddingqualitaet. - Der produktiv gebaute End-to-End-Pfad ist mit lokalem llama.cpp/BGE-M3 FP16 live gelaufen: sechs Dokumente/sechs Chunks; drei Cross-Language-Smoke-Cases bei K=3 jeweils Rang 1 und damit HitRate, Recall und MRR 1,0. Der Mini-Korpus ist nur ein Wiring-Smoke-Test, kein belastbarer Modellvergleich. - Der Grundstand ist in `docs/REGULATORY_RAG.md` dokumentiert und mit synthetischen Vertrags-, Client-, Ingestion-, Retrieval- und Deploymenttests abgesichert. +- Der PDF.js-Importer erzeugt aus den explizit uebergebenen ISO-27001-/27002-Pfaden einen lokalen Korpus-v2 mit Quellhashes, Ausgaben, 93 geteilten Kontrollidentitaeten, Abschnittsarten und Seitenbereichen. Nur ISO-27002-Abschnitte werden semantisch gerankt; die entsprechende ISO-27001-Aussage wird als autoritative Referenzevidenz angehaengt. +- Ein manueller Work-Vertikalschnitt startet pro kanonischem Scanner-Finding einen idempotenten asynchronen Lauf, gruppiert die 32 Dense-Chunks auf acht Controls, validiert maximal fuenf LiteLLM-Vorschlaege gegen exakt gelieferte Evidenz und speichert Runs sowie optimistisch nebenlaeufigkeitsgeschuetzte Accept/Reject-Ereignisse append-only in PostgreSQL. Offen bis zur Phasenabnahme: -- autorisierter und lizenzrechtlich geklaerter regulatorischer Quellkorpus samt Update-/Freigabeprozess -- organisatorische Datenschutzfreigabe fuer den Embeddingzielpfad und kuratierte Erweiterung der oeffentlichen Signal-/Regelkataloge +- formaler Update-/Freigabeprozess fuer neue lizenzierte ISO-Ausgaben und Korpus-Rebuilds +- kuratierte Erweiterung der oeffentlichen Signal-/Regelkataloge - repraesentativer freigegebener EN/DE-Evaluationssatz und dokumentierte reale Dense-Baseline gegen den autorisierten Korpus -- aktive Sparse-/BM25-Erzeugung, Hybridfusion, Kontrollgruppierung und evaluierter Reranker -- constrained Mappingmodell, Confidence-Kalibrierung, Human Review und append-only PostgreSQL-Provenienz -- API/UI, Mandanten-/Rollenabgrenzung, Betriebsmonitoring sowie praktischer Rebuild- und Lasttest +- aktive Sparse-/BM25-Erzeugung, Hybridfusion und evaluierter Reranker +- Confidence-Kalibrierung, Assessment-Batchlauf, Mandanten-/Rollenabgrenzung, Betriebsmonitoring und Lasttest +- Betriebsprozess, der vor jeder kuenftigen Migration einen laufenden Hot-Reload-Backendprozess stoppt oder die Migration bis nach dem verifizierten Backup sperrt Abnahmekriterien: @@ -1685,7 +1689,7 @@ Die naechste Umsetzung fuehrt Phase 13B fort und entwickelt parallel den ausdrue 1. Phase 13A ist abgeschlossen: Erkennung, technische Dokumenttypen, Source Extractors, Mappingprofile, Finding-Identität sowie allgemeine Artefakt-, Processing- und Jobgrenzen sind konsolidiert und die Scannerpersistenz verwendet den Candidate-Pfad. 2. Phase 13B ist in Arbeit: Der sichere Ingestion-Unterbau, technische Policy-Identitaeten, versionierte ADMX-/Baselineanalysen, deterministisches Reprocessing, Environment Profiles, konservative Snapshotaggregation, typisierte Evaluation, fail-closed Finding Factory, Unified-Result-Read-Projektion, projektgebundene Review-/Orchestrierungsoberflaechen und die aktuelle snapshotbezogene Ergebnisdeduplizierung sind umgesetzt. Als naechste vertikale Etappe folgen fachlich kuratierte Policy-Regeln und vollstaendige versionierte Windows-Inbox-ADMX-/ADML-Kataloge; danach weitere Preference-CSEs und Exportdetails. Ohne RSoP/GP Results bleibt die effektive Anwendung weiterhin ausdruecklich nicht verifiziert. -3. Phase 13D besitzt die rebuildbare Qdrant-, Chunking-, Embedding-, Ingestion- und Dense-Retrieval-Grundlage sowie einen fail-closed Public-Signal-Finding-Query-Builder und einen ausfuehrbaren Evaluationsvertrag. Als naechste Etappe folgen der freigegebene Korpus, organisatorische Datenschutzfreigabe, ein unabhaengig gelabelter EN/DE-Evaluationssatz, die reale Dense-Baseline und danach der persistierte Review-/Mappingvertrag. Sparse/Hybrid und Reranking werden erst nach messbarer Dense-Baseline aktiviert. +3. Phase 13D besitzt den lizenzierten ISO-Korpus-v2, die rebuildbare Qdrant-/Dense-Grundlage, den fail-closed Public-Signal-Pfad sowie den persistierten manuell reviewten Einzel-Finding-Workflow. Als naechste Etappe folgen ein unabhaengig gelabelter repraesentativer EN/DE-Evaluationssatz und die reale Dense-Baseline; Sparse/Hybrid, Reranking und Assessment-Batchmapping werden erst danach aktiviert. 4. Erst nach verifizierter Phase 13B den kontrollierten Firmenbetrieb aus Phase 14 mit SSO, Rollen, HTTPS, Monitoring, Backup/Restore und Aufbewahrung fortsetzen. Die technische Reihenfolge ist bis einschliesslich Phase 13A umgesetzt. Phase 13B baut auf dem verifizierten sicheren Ingestion-Unterbau auf und reicht inzwischen produktiv von der begrenzten Parser-/Workerverarbeitung ueber unveraenderliche Analyse-, Profil-, Aggregations- und Evaluationslaeufe bis zur streng evidenzgebundenen Finding Factory und ihrer gemeinsamen Unified-Result-Leseprojektion. Das Backend leitet projektgebundene Aktionsbereitschaft aus aktiven Revisionen und exakt passenden Referenzdaten ab; der Datahub fuehrt Environment-Profile-Anlage, Reprocessing, Aggregation, Evaluation und Finding Factory ueber die autoritativen idempotenten Services aus. Der produktive Regelkatalog bleibt bis fachlicher Kuratierung leer, und die vorhandenen Microsoft-Baselines bleiben ohne vollstaendige Windows-Inbox-ADMX-/ADML-Kataloge partial; dadurch erzeugt dieser reale Bestand weiterhin nur Review und keine Findings. @@ -1701,8 +1705,16 @@ Aktualisierung nach der Katalog-Ladegrenze vom 2026-09-03: Fachlich freigegebene ## 9. Bereits getroffene Architekturentscheidungen +Festlegung 2026-09-15 (Phase 13D, ISO-Mapping): ISO 27002 ist fuer alle 93 Controls der semantische Korpus; ISO 27001 Annex A bleibt autoritative Referenz und Vollstaendigkeitspruefung. Kanonische IDs sind `5.1` bis `8.34`, Aliaswerte `A.5.1` bis `A.8.34`. Ein Mapping bedeutet ausschliesslich fachliche Relevanz, niemals Compliance. Dense Retrieval gruppiert zuerst auf Control-Ebene; LiteLLM darf nur gelieferte Controls und Chunk-IDs ausgeben. Kein Confidencewert akzeptiert automatisch. Runs und Entscheidungen sind append-only in PostgreSQL, Qdrant bleibt jederzeit rebuildbar. Die erste UI bleibt beim einzelnen Finding; Batchmapping, Sparse/Hybrid und Reranking bleiben bis zur repraesentativen Retrievalbewertung ausgeschlossen. + Festlegung 2026-09-15 (Phase 13D, Finding-Retrievalgrenze): Nur Scanner-Findings mit einem exakt katalogisierten oeffentlichen Maester-/PingCastle-Regel-ID oder einem lokal erkannten fest definierten oeffentlichen Technikthema duerfen eine Embeddingquery erzeugen. Freitext, kundenspezifische Tool-/Regel-IDs, Assets, Pfade, Werte, Mandanten-/Projekt-/Review- und Provenienzdaten werden nie kopiert; ohne oeffentliches Signal bleibt Retrieval fail-closed. Der lokale Source-Result-Bezug bleibt ausserhalb des Embeddingrequests. Diese technische Minimierung ersetzt keine organisatorische Datenschutzfreigabe. Evaluationsmetriken werden auf Kontrollebene ermittelt und Retrievalscores bleiben Kandidatenranking, keine fachliche Confidence. +Festlegung 2026-09-15 (Phase 13D, Finding-Retrievalgrenze – Revision, ersetzt die vorstehende Festlegung auf ausdruecklichen Nutzerauftrag): Die ISO-Mappingquery wird nicht mehr aus einer festen Themenwhitelist erzeugt, sondern aus der gemeinsamen datenschutzsicheren Finding-Projektion (`src/domain/privacy-safe-finding.ts`, `privacySafeFinding`) – exakt derselben Grenze wie die KI-Uebersicht. Query-Quelle ist der regelbezogene Prueftext (Titel, sowie Beschreibung und Empfehlung, jeweils aus der je Regel deduplizierten Finding-Definition, mit E-Mail-Redaktion); katalogisierte Regel-IDs, oeffentlicher Toolname und erkannte Technikthemen bleiben nur noch Query-Anreicherung. Die Ergebnistabelle/Occurrence (affectedAsset, Source-Locator/Pfad, Objektwerte, Mandanten-/Review-/Provenienzdaten) wird nie gelesen. Dadurch mappt jedes Scanner-Finding mit Pruef-Titel; nur Nicht-Scanner-Datensaetze und Findings ohne Titel bleiben `insufficient-public-context`. `REGULATORY_RETRIEVAL_VERSION` auf `dense-control-grouping-1.1.0` erhoeht; kein Reimport, da Mappinglaeufe regenerierbar sind und keine persistierten Scannerdaten geaendert werden. Restrisiko: Die geteilte Redaktion entfernt nur E-Mail-Adressen und verlaesst sich auf regelgenerische Definitionen; eine haertere Redaktion (Host/IP/Pfad) waere eine gemeinsame Entscheidung fuer beide KI-Funktionen und bleibt offen. Folgeetappen: assessment-weites Batchmapping ueber alle Findings (dedupliziert je Regel/Definition, analog zum KI-Uebersicht-Scopelauf) und optional eine symmetrische Einzel-Finding-KI-Uebersicht. + +Praezisierung 2026-09-15 (Phase 13D, Mapping-Relevanz): Der Mapping-Systemprompt (`REGULATORY_MAPPING_PROMPT_VERSION` `1.2.0`) mappt jetzt ausdruecklich auf fachliche Themenrelevanz. Ein Kandidat wird zugeordnet, wenn er dasselbe Sicherheitsthema regelt (z. B. Passwort-/Authentisierungsverwaltung), auch wenn der ISO-Kontrolltext den konkreten Wert, Schwellwert oder Parameter des Findings nicht nennt; ISO-Controls sind prinzipienbasiert und wiederholen solche Spezifika selten. `no_relevant_control` gilt nur, wenn kein Kandidat das Thema regelt. Hintergrund: `A-MinPwdLen` wurde trotz korrekt abgerufenem Control `5.17` (Authentisierungsinformationen) faelschlich als nicht relevant zurueckgewiesen. Kein Reimport, da Mappinglaeufe regenerierbar sind. + +Festlegung 2026-09-15 (Phase 13D, globale ISO-Mappings, ausdruecklicher Nutzerauftrag): ISO-Mappings und Accept/Reject-Entscheidungen sind jetzt global und mandantenunabhaengig, analog zur CVE-Anreicherung. Migration `026` ersetzt die per-Occurrence-Tabellen `assessor_regulatory_mapping_runs`/`_decisions` (nur regenerierbare Testdaten, vor dem Drop gesichert) durch `assessor_regulatory_mappings` und ein globales `assessor_regulatory_mapping_decisions`. Schluessel ist die Regelidentitaet `ruleKey` (`tool:ruleId`, ersatzweise ein stabiler Titel-Hash) plus die Pipelineversionen (`UNIQUE(rule_key, corpus_version, retrieval_version, prompt_version, model)`); die Mapping-ID ist ein deterministisches Digest dieses Tupels, sodass ein Regel-Mapping genau einmal existiert und jede Occurrence in jedem Projekt es wiederverwendet. Entscheidungen sind append-only, per `(ruleKey, controlId)` gefuehrt, versionsunabhaengig und ueberleben Scan-, Assessment- und Projektloeschung. Der Dienst leitet nur die non-personale Regelidentitaet und den Pruefkontext aus dem Finding ab; Ergebnistabelle/Occurrence fliesst nie ein. Kein Reimport (kein Scanner-Import veraendert). Naechste Etappe: assessment-weites Batchmapping ueber distinct Regeln. + Festlegung 2026-09-15 (Phase 13D, ausdruecklicher Nutzerauftrag): Qdrant ist der rebuildbare Suchindex fuer versionierte regulatorische Chunks und niemals fachliche Quelle fuer Scannergebnisse oder Mappingentscheidungen. PostgreSQL bleibt System of Record fuer Scans, Findings, Ergebnisse, Reviews und spaetere append-only Mappingprovenienz. Dense und Sparse sind benannte Vektorraeume; zunaechst wird nur mehrsprachiges Dense Retrieval aktiviert. Candidate Retrieval und optionaler Reranker bleiben getrennte Schnittstellen. Ein Modellwechsel darf alte Vektoren nicht still weiterverwenden; Modellname und Dimension werden explizit gebunden, Abfragen filtern auf das aktive Modell, und ein Korpus-Rebuild bleibt Pflicht. Der Embeddingdienst ist eine eigene externe Vertrauensgrenze; Findinginhalte duerfen erst nach einer getrennten Datenschutzfreigabe als minimierte Query uebertragen werden. Festlegung 2026-09-10 (Purple-Knight-Begleitdateien): Ein nativer Score darf aus einer separaten Summary auf die Findings desselben Scanlaufs uebertragen werden, wenn genau ein nativer Traeger und eine eindeutige Ergebnispopulation denselben Werkzeug- und Verarbeitungskontrakt teilen. Datei- und Vorkommensherkunft bleiben gemeinsam erhalten; Mehrdeutigkeit verhindert die Verbindung. diff --git a/docs/PROJECT_OVERVIEW.md b/docs/PROJECT_OVERVIEW.md new file mode 100644 index 0000000..346c44d --- /dev/null +++ b/docs/PROJECT_OVERVIEW.md @@ -0,0 +1,374 @@ +# ASSESSOR — Project Overview + +ASSESSOR imports security-scanner reports (PingCastle, Purple Knight, CIS-CAT, +Microsoft Defender for IoT, Maester, Group Policy backups, and generic +CSV/JSON/XML/HTML/Excel exports), normalizes them into one canonical model, +scores the assessed environment, lets an analyst review and compare findings, +and presents an executive security posture. It also offers optional AI overview +generation and regulatory (ISO 27001/27002) mapping through a retrieval index. + +**This is the entry point for the whole project.** It explains what the software +is, how it is put together, how many containers it runs, how to develop it +locally, and how it is deployed. Detailed topic docs live alongside this file in +[`docs/`](.); this document links to the right one for each topic. + +> Status: the application runs today as a local browser app served by a Node.js +> backend, and is being deployed to an internal server behind an nginx reverse +> proxy. The full phased plan and its history are in +> [`docs/IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md); binding rules for +> anyone (human or AI) changing the code are in [`AGENTS.md`](../AGENTS.md). + +--- + +## 1. What runs where — the short answer + +There are **three long-running services**, defined in +[`docker-compose.deploy.yml`](../docker-compose.deploy.yml): + +| Service | Image | Role | Published on host? | +|------------|----------------------|------------------------------------------------------------------|--------------------| +| `app` | built from `Dockerfile` | Node.js server: serves the React app **and** the `/api/v1` API from one origin | Yes — the only exposed port | +| `postgres` | `postgres:16-alpine` | System of record (canonical findings, reviews, comparisons, jobs) | No | +| `qdrant` | `qdrant/qdrant:v1.19.1` | Rebuildable vector index for regulatory RAG retrieval | No | + +So **one deployed environment = 3 containers**. Only `app` is reachable from the +host; PostgreSQL and Qdrant are private on the internal Docker network. + +The server runs **two isolated environments in parallel** (see +[`docs/DEPLOYMENT.md`](DEPLOYMENT.md)), each its own set of 3 containers with +its own database, artifact volume, and Qdrant index: + +| Branch | Compose project | VPN address | Public host name (via nginx) | +|----------------|-------------------------|--------------------------|------------------------------------| +| `main` | `assessor_prod` | `http://10.0.0.154:4173` | `https://assessor.soc-ops.de` | +| `experimental` | `assessor_experimental` | `http://10.0.0.154:4174` | `https://staging.assessor.soc-ops.de` | + +That is **6 containers total** for the managed stacks. A separate, older HTTPS +installation exists on the same host and is intentionally *not* managed by this +workflow until an approved cutover. + +For **local development** you do not containerize the app. You run only the two +backing services in Docker (`postgres` + `qdrant`, project `assessor-2_0` — the +single canonical local datastore) and run the Node backend + Vite client as host +processes. See [§4](#4-local-development). + +--- + +## 2. Architecture + +### 2.1 Layered design + +ASSESSOR keeps a strict separation between format parsing, structural analysis, +semantic mapping, normalization, comparison, persistence, and presentation. The +core engine is **platform-neutral**: it never touches the DOM, `localStorage`, +browser `File` objects, SQL, or the network directly — those live behind +adapters. This is what lets the same domain logic run in the browser today and in +the server backend now / a future shell later. + +``` +┌─────────────────────────────────────────────────────────────┐ +│ React UI (src/features, src/components) │ presentation only +│ no source-specific parsing, no direct DB access │ +└───────────────┬───────────────────────────────────────────────┘ + │ HTTP (same origin) +┌───────────────▼───────────────────────────────────────────────┐ +│ Node backend (server/) — /api/v1 + static React app │ +│ import workers · GPO analysis · CVE enrichment · AI · RAG │ +└───────────────┬───────────────────────────────────────────────┘ + │ AssessorRepository contract (no SQL leaks upward) +┌───────────────▼───────────────────────────────────────────────┐ +│ Persistence adapter │ +│ • PostgresAssessorRepository (durable, production) │ +│ • LocalAssessorRepository (in-memory, dev only) │ +└───────────────┬───────────────────────────────────────────────┘ + │ +┌───────────────▼──────────┐ ┌──────────────────────────┐ +│ PostgreSQL 16 │ │ Qdrant (regulatory RAG) │ +│ system of record │ │ rebuildable index │ +└──────────────────────────┘ └──────────────────────────┘ +``` + +The **platform-neutral import engine** (`src/importers`, `src/domain`) turns raw +bytes into an `ImportSource`, parses it through injected runtime adapters, picks +a scanner profile via the shared recognition engine, and produces the canonical +model. Known tool profiles *extend* this generic pipeline rather than bypass it. + +Key boundary docs: +- [`docs/PLATFORM_NEUTRAL_ENGINE.md`](PLATFORM_NEUTRAL_ENGINE.md) — engine/adapter layering +- [`docs/PERSISTENCE_BOUNDARY.md`](PERSISTENCE_BOUNDARY.md) — the `AssessorRepository` contract and import transaction +- [`docs/CANONICAL_MODEL.md`](CANONICAL_MODEL.md) — the normalized data model +- [`docs/RECOGNITION_ENGINE.md`](RECOGNITION_ENGINE.md) — profile selection +- [`docs/UNIFIED_RESULT_MODEL.md`](UNIFIED_RESULT_MODEL.md) / [`docs/RESULT_CONTRACT.md`](RESULT_CONTRACT.md) — the read model behind the UI +- [`docs/SCORING_MODEL.md`](SCORING_MODEL.md) — native-score-first scoring engine + +### 2.2 The app service in detail + +The single `app` container (or local `node dist-server/server.mjs`) does three +jobs from one origin (port `4173`): + +1. Serves the built React SPA from `dist/`. +2. Serves the versioned JSON API under `/api/v1`. +3. Runs background work: isolated import parser workers, GPO inventory/analysis + workers, CVE enrichment, AI overview jobs, and regulatory retrieval. + +Same-origin is a deliberate security property: the server sets **no CORS**, and +every write requires the `X-Assessor-Request: 1` header with a matching `Origin`. +See [`docs/LOCAL_BACKEND.md`](LOCAL_BACKEND.md) and [`SECURITY.md`](../SECURITY.md). + +### 2.3 API surface + +All endpoints live under `/api/v1`. Top-level resource groups: + +`health` · `audit-events` · `db-structure` · `reference-data` · `projects` +(with nested `policy-analyses`, `environment-profiles`, `policy-snapshot-aggregations`, +`policy-evaluations`, `policy-finding-runs`, `gpo-backups`, `scan-runs`) · +`assessments` · `findings` · `results` · `reviews` · `profiles` · `comparisons` · +`cves` · `import-sessions`. + +Imports are **atomic sessions** (stage → commit → rollback). Errors always carry +`code`, `message`, and `requestId`. The contract is in +[`src/api/assessor-api-contract.ts`](../src/api/assessor-api-contract.ts); the server +in [`server/assessor-http-server.ts`](../server/assessor-http-server.ts). + +--- + +## 3. Repository layout + +``` +server/ Node backend + assessor-http-server.ts HTTP router (/api/v1 + static app) + index.ts process entry, wiring, env parsing + postgres/ repository adapter + migrations/ + gpo/ Group Policy backup ingestion & analysis + cve/ NVD CVE enrichment (opt-in, backend-only) + ai/ LiteLLM client + AI overview service + regulatory/ ISO corpus, chunking, Qdrant index, RAG runtime, mapping + artifacts/ filesystem artifact store +src/ Browser app + platform-neutral core + importers/ domain/ platform-neutral engine & canonical model + profiles/ scanner profiles (see §5) + recognition/ analysis/ comparison/ scoring/ evaluation/ + persistence/ repository contract + LocalAssessorRepository + platform/browser/ browser adapters (File, DOM, storage, download) + api/ typed API client & contract + features/ components/ styles/ React UI +docs/ all detailed documentation (see §9) +tests/ vitest suites + local-only & design-language checks +deploy/nginx/ reverse-proxy vhosts for prod + staging +.gitea/workflows/ CI/CD (deploy.yml) +Dockerfile multi-stage build for the app image +docker-compose.deploy.yml 3-service deployed stack (app+postgres+qdrant) +docker-compose.postgres.yml local backing services (postgres+qdrant) +``` + +--- + +## 4. Local development + +### Prerequisites +- Node.js 22 (matches the runtime image) and npm +- Docker + Docker Compose (for PostgreSQL + Qdrant) + +### First-time setup + +```bash +cp .env.postgres.example .env.postgres +# edit .env.postgres: set long, local-only ASSESSOR_POSTGRES_PASSWORD and ASSESSOR_QDRANT_API_KEY +npm ci +npm run db:up # starts the canonical postgres + qdrant (project assessor-2_0) +npm run dev # builds/starts backend on 127.0.0.1:4173, then Vite client on 127.0.0.1:5173 +``` + +`npm run dev` checks `/api/v1/health` first, brings up a missing backend, then +opens the Vite client (which proxies `/api` to `4173`). The backend binds to +`127.0.0.1` only — it is **not** exposed on the network in local mode. + +> The canonical local datastore is fixed: Compose project `assessor-2_0`, +> database `assessor`, volume `assessor-2_0_assessor-postgres-data`, on +> `127.0.0.1:5432`. Never create an alternative DB/volume/port for this +> workspace (see [`AGENTS.md`](../AGENTS.md) → "Canonical local PostgreSQL"). If +> port 5432 is taken by a different container, stop and report the conflict. + +### Useful scripts + +| Command | What it does | +|------------------------|-----------------------------------------------------------------| +| `npm run dev` | Full local stack (backend + Vite client) | +| `npm run dev:client` | Vite client only (backend managed separately) | +| `npm run build` | Build client (`dist/`) and server (`dist-server/`) | +| `npm run start:local` | Run the built server on 4173 using `.env.postgres` | +| `npm run serve:local` | `build` then `start:local` | +| `npm run db:up` / `db:down` / `db:logs` | Manage local postgres+qdrant | +| `npm run rag:prepare-iso` / `rag:ingest` / `rag:evaluate` | Build/populate/evaluate the regulatory index | +| `npm test` | vitest + local-only check + design-language check | + +Without `ASSESSOR_DATABASE_URL` the backend runs in an explicitly ephemeral +in-memory mode. If the URL is set but the DB is unreachable, startup **fails** +(no silent fallback), so a misconfiguration can never produce imports that look +saved but are not. Details: [`docs/POSTGRESQL.md`](POSTGRESQL.md). + +--- + +## 5. Supported scanner profiles + +Profiles live in [`src/profiles/`](../src/profiles/): `pingcastle`, `purple-knight`, +`cis-cat`, `defender-for-iot`, `maester`, `group-policy`, `generic`, and +`public-search` (regulatory). Each profile must declare its scoring semantics +(`securityScoring`) and extend the generic import pipeline — see +[`docs/TOOL_PROFILES.md`](TOOL_PROFILES.md) and the profile rules in +[`AGENTS.md`](../AGENTS.md). + +Any change to recognition, parsing, mapping, identity, scoring inputs, or +persisted evidence must follow [`docs/REIMPORT_POLICY.md`](REIMPORT_POLICY.md) +(add a processing-manifest version and mark affected historical imports for +reimport). + +--- + +## 6. Deployment + +Deployment is fully driven by the Gitea workflow +[`.gitea/workflows/deploy.yml`](../.gitea/workflows/deploy.yml). **You do not run +`docker compose` by hand on the server for a normal release** — a push does it. + +### How a release happens +1. Push to `main` (→ stable) or `experimental` (→ experimental). +2. The workflow loads that environment's complete env-file secret + (`PROD_ENV_FILE` / `STAGING_ENV_FILE`) and validates required keys are present. +3. Runs `npm ci`, the full `npm test`, and `npm run build`. +4. Acquires a per-environment Docker lock (Gitea ignores the `concurrency` key), + validates the resolved Compose config, and checks port/project isolation + (only `app`/`postgres`/`qdrant` may exist; the target port must not be + published by another project). +5. **Backs up** the existing environment (custom `pg_dump`, artifact tarball, + `SHA256SUMS`) into the external volume `assessor_deployment_backups`. +6. `docker compose ... up -d --build --wait` for that environment only. +7. Verifies health via `/api/v1/health` inside the container, then releases the lock. + +The two environments never share PostgreSQL, artifacts, or the Qdrant index — +Compose scopes all named volumes by project. `restart: unless-stopped` keeps both +alive across daemon/host restarts. Feature promotion is a `experimental → main` +merge; **databases and artifacts are never copied between environments.** + +### Reverse proxy / TLS +nginx terminates TLS and proxies to the VPN ports; vhosts are in +[`deploy/nginx/assessor.conf`](../deploy/nginx/assessor.conf). `proxy_set_header +Host $host` is mandatory — ASSESSOR rejects writes (403 `cross_origin_rejected`) +if the forwarded host does not match the browser origin. `client_max_body_size` +is 200m to allow ~100 MiB uploads (base64 overhead). + +### Backups & restore +The workflow writes a backup before every update and **never deletes backups**. +Restore tests must use an isolated temporary Compose project and synthetic data, +verify `SHA256SUMS`, and `pg_restore --exit-on-error` into a fresh DB. Never run +`docker compose down --volumes` on a permanent environment. Full procedure: +[`docs/DEPLOYMENT.md`](DEPLOYMENT.md) and +[`docs/POSTGRESQL.md`](POSTGRESQL.md). + +### Manual deploy (reference only) +```bash +docker compose --env-file .env -p assessor_prod -f docker-compose.deploy.yml \ + up -d --build --wait +``` + +--- + +## 7. Configuration + +Real secret values live only in the git-ignored `.env.postgres` (workstation) or +the deployment env-file secrets — never in the repo or in client-side config. +[`.env.postgres.example`](../.env.postgres.example) is the annotated template. + +| Variable | Purpose | +|----------------------------------|----------------------------------------------------------------| +| `ASSESSOR_HOST` / `ASSESSOR_PORT`| Bind address / port (default `127.0.0.1:4173`) | +| `ASSESSOR_DATABASE_URL` | PostgreSQL DSN; absent = ephemeral in-memory dev mode | +| `ASSESSOR_DATABASE_SSL` | `disable` only for local loopback; TLS required for remote DBs | +| `ASSESSOR_POSTGRES_*` | DB name/user/password (compose builds the DSN in deploy) | +| `ASSESSOR_ARTIFACT_ROOT` | Artifact store path (must be outside the public web root) | +| `ASSESSOR_QDRANT_URL` / `_API_KEY` | Regulatory index endpoint & key | +| `ASSESSOR_RAG_EMBEDDING_*` | Approved embedding model for regulatory ingestion/retrieval | +| `ASSESSOR_AI_MODE` (+ `ASSESSOR_AI_*`) | Enables the LiteLLM AI overview; unset = outbound AI off | +| `NVD_API_KEY` / `ASSESSOR_NVD_TIMEOUT_MS` | Optional CVE enrichment against the fixed NVD endpoint | +| `ASSESSOR_PARSER_*` / `ASSESSOR_GPO_*` | Import/GPO worker limits (timeout, memory, concurrency) | + +AI and CVE calls are the only outbound network paths, they are backend-only, +opt-in, and off by default. Trust boundaries: +[`docs/AI_OVERVIEW_SECURITY.md`](AI_OVERVIEW_SECURITY.md), +[`docs/REGULATORY_RAG.md`](REGULATORY_RAG.md), +[`docs/IMPORT_SECURITY_LIMITS.md`](IMPORT_SECURITY_LIMITS.md). + +--- + +## 8. Data & persistence + +- **PostgreSQL 16** is the system of record: relational keys/foreign keys for + identity plus the full versioned canonical object as JSONB (so new scanner + fields are captured losslessly). Full-text `tsvector` + GIN indexes back search. +- **Migrations** are versioned SQL files in + [`server/postgres/migrations/`](../server/postgres/migrations/) (currently through + `026_*`). On start the server takes an advisory lock, verifies each migration's + version + SHA-256, applies new ones in their own transaction, and refuses to + start if an already-applied migration was edited. Never edit an applied + migration — add a new numbered file. +- **Qdrant** holds the regulatory retrieval index. It is authenticated and + **rebuildable** from the versioned corpus — its volume speeds restarts but is + not a system-of-record backup. +- **Artifacts** (e.g. GPO raw backups) go to a filesystem store on a dedicated + volume, outside the public web root. + +Details: [`docs/POSTGRESQL.md`](POSTGRESQL.md), +[`docs/REGULATORY_RAG.md`](REGULATORY_RAG.md), [`MIGRATION.md`](../MIGRATION.md). + +--- + +## 9. Testing & verification + +```bash +npm test # vitest + local-only network check + design-language check +npm run build # type-check + build client and server (CI gate) +npm run test:postgres # integration tests against a real PostgreSQL 16 (ASSESSOR_TEST_DATABASE_URL) +``` + +Baseline verification for any relevant change is `npm test` + `npm run build` +(both run in CI before deploy). **Testing is privacy-safe:** use synthetic +fixtures; never read, log, screenshot, or quote real report content, file names, +finding titles, or tenant data. Against a running app, verify only content-free +signals (HTTP status, health, anonymized counts, boolean DOM state). See the +"Privacy-safe testing" section of [`AGENTS.md`](../AGENTS.md) and +[`docs/TEST_BASELINE.md`](TEST_BASELINE.md). + +--- + +## 10. Security model (summary) + +- Local/same-origin by default: no CORS, write requests need `X-Assessor-Request` + with a matching `Origin`; the backend binds loopback locally. +- Imported HTML/XML/Excel/JSON/CSV are **untrusted input** — active content is + never executed; HTML is inertized before table extraction; Excel formulas/VBA + stay disabled. Parsers run in isolated workers with time/memory limits. +- No external uploads, telemetry, or CDN assets. The only outbound paths are the + opt-in backend AI (LiteLLM) and CVE enrichment (NVD). +- Embedded report data is parsed with a bounded static grammar, never evaluated. + +Full model: [`SECURITY.md`](../SECURITY.md), [`docs/IMPORT_SECURITY_LIMITS.md`](IMPORT_SECURITY_LIMITS.md), +[`docs/architecture/gpo-threat-model.md`](architecture/gpo-threat-model.md). + +--- + +## 11. Documentation map + +**Start here → this overview.** Then, by topic: + +| Area | Documents | +|------|-----------| +| Rules for contributors | [`AGENTS.md`](../AGENTS.md) · [`CLAUDE.md`](../CLAUDE.md) | +| Plan & history | [`docs/IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md) · [`MIGRATION.md`](../MIGRATION.md) | +| Architecture | [`docs/PLATFORM_NEUTRAL_ENGINE.md`](PLATFORM_NEUTRAL_ENGINE.md) · [`docs/PERSISTENCE_BOUNDARY.md`](PERSISTENCE_BOUNDARY.md) · [`docs/architecture/`](architecture/) | +| Data model | [`docs/CANONICAL_MODEL.md`](CANONICAL_MODEL.md) · [`docs/UNIFIED_RESULT_MODEL.md`](UNIFIED_RESULT_MODEL.md) · [`docs/RESULT_CONTRACT.md`](RESULT_CONTRACT.md) · [`docs/SCORING_MODEL.md`](SCORING_MODEL.md) | +| Import | [`docs/IMPORT_WORKFLOW.md`](IMPORT_WORKFLOW.md) · [`docs/IMPORT_INTERMEDIATE_MODEL.md`](IMPORT_INTERMEDIATE_MODEL.md) · [`docs/RECOGNITION_ENGINE.md`](RECOGNITION_ENGINE.md) · [`docs/TOOL_PROFILES.md`](TOOL_PROFILES.md) · [`docs/REIMPORT_POLICY.md`](REIMPORT_POLICY.md) | +| Compare & filter | [`docs/COMPARISON_ENGINE.md`](COMPARISON_ENGINE.md) · [`docs/DYNAMIC_FILTERS.md`](DYNAMIC_FILTERS.md) | +| Backend & persistence | [`docs/LOCAL_BACKEND.md`](LOCAL_BACKEND.md) · [`docs/POSTGRESQL.md`](POSTGRESQL.md) | +| Deployment | [`docs/DEPLOYMENT.md`](DEPLOYMENT.md) · [`deploy/nginx/assessor.conf`](../deploy/nginx/assessor.conf) | +| AI & regulatory | [`docs/AI_OVERVIEW_SECURITY.md`](AI_OVERVIEW_SECURITY.md) · [`docs/LLM_DASHBOARD_PROMPT.md`](LLM_DASHBOARD_PROMPT.md) · [`docs/REGULATORY_RAG.md`](REGULATORY_RAG.md) | +| Security & testing | [`SECURITY.md`](../SECURITY.md) · [`docs/IMPORT_SECURITY_LIMITS.md`](IMPORT_SECURITY_LIMITS.md) · [`docs/TEST_BASELINE.md`](TEST_BASELINE.md) · [`docs/USER_FLOW_CHECKS.md`](USER_FLOW_CHECKS.md) | diff --git a/docs/REGULATORY_RAG.md b/docs/REGULATORY_RAG.md index 87d269c..5f88de1 100644 --- a/docs/REGULATORY_RAG.md +++ b/docs/REGULATORY_RAG.md @@ -1,14 +1,14 @@ # Regulatory-control RAG with Qdrant -Status: Phase 13D foundation. The current slice indexes and retrieves regulatory control text and can create privacy-minimized retrieval candidates for scanner findings. It does not map findings, call an LLM for classification, expose a browser API, or persist mapping decisions. +Status: Phase 13D one-finding vertical slice. The server prepares the licensed German ISO 27001/27002 corpus, indexes section-safe ISO 27002 semantic chunks plus ISO 27001 references, retrieves and groups candidates, requests a grounded LiteLLM proposal, persists runs and reviewer decisions, and exposes the workflow inline in Work. Batch mapping, sparse retrieval, reranking, and confidence calibration remain out of scope. ## Boundaries -- PostgreSQL remains the only system of record for ASSESSOR projects, assessments, scans, findings, results, reviews, and future accepted mappings. +- PostgreSQL remains the only system of record for ASSESSOR projects, assessments, scans, findings, results, reviews, mapping runs, and mapping decisions. - Qdrant is a derived, rebuildable search index. Losing or deleting its volume must not lose an ASSESSOR scan, finding, result, or review. - The versioned regulatory corpus JSON is the rebuild source for Qdrant. It must be retained and governed separately from the index. - React does not connect to Qdrant or the embedding provider. Both are backend-only boundaries. -- Regulatory retrieval produces candidates, not compliance decisions. A later constrained mapping stage must retain the retrieved chunk IDs, scores, corpus versions, model version, prompt/schema version, and final decision in PostgreSQL. +- Regulatory retrieval produces candidates, not compliance decisions. The constrained mapping stage retains retrieved chunk IDs, scores, corpus/source hashes, model, prompt/schema versions, and append-only reviewer decisions in PostgreSQL. ```text versioned regulatory corpus JSON @@ -17,21 +17,22 @@ versioned regulatory corpus JSON approved multilingual embedding service | dense vectors v -Qdrant assessor_regulatory_chunks_v1 +Qdrant assessor_regulatory_chunks_v2 ^ scanner finding -> public-signal query builder -> same embedding model -> dense candidates + -> group top 32 chunks into top 8 controls -> optional future hybrid fusion -> optional reranker - -> later constrained mapper -> PostgreSQL audit record + -> constrained mapper -> human decision -> PostgreSQL audit record ``` ## Collection contract -The collection is `assessor_regulatory_chunks_v1`. It has two named vector slots: +The active collection is `assessor_regulatory_chunks_v2`. It has two named vector slots: - `dense`: the configured embedding dimension with cosine distance. - `sparse`: reserved now so a later sparse/BM25-style encoder can be added without changing the retrieval abstractions. -Qdrant payloads use schema version 1 and contain: +Qdrant payloads use schema version 2 and contain: | Field | Purpose | |---|---| @@ -44,32 +45,21 @@ Qdrant payloads use schema version 1 and contain: | `title`, `text`, `text_sha256` | Retrieval content and integrity metadata | | `embedding_model` | Prevents querying vectors from a different model space | | `source_uri` | Optional public/source reference | +| `annex_a_control_id`, `control_theme` | Shared `5.1`–`8.34` identity, Annex A alias, and theme | +| `source_standard`, `source_edition`, `national_edition` | ISO/base and DIN source identity | +| `section_kind`, `source_page_start`, `source_page_end` | Section-safe retrieval and page provenance | +| `retrieval_role` | `semantic` for ISO 27002 or `reference` for ISO 27001 | +| `parser_version`, `source_sha256` | Reproducible parser/source provenance | All fields used by filters have Qdrant payload indexes. An existing collection with a different dense dimension, distance, or missing sparse slot fails closed and must be rebuilt from the corpus. -## Corpus format and ingestion +## ISO corpus preparation and ingestion -The CLI accepts a UTF-8 JSON file of no more than 64 MiB: +The server-only PDF.js CLI accepts explicit ISO 27001 and ISO 27002 paths and writes corpus-v2 JSON below the ignored `.assessor-data/regulatory-corpora/` directory: -```json -{ - "schemaVersion": 1, - "documents": [ - { - "schemaVersion": 1, - "corpusId": "synthetic-framework", - "corpusVersion": "2026-09-15", - "documentId": "synthetic-control-1-de", - "framework": "Synthetic Framework", - "frameworkVersion": "1", - "controlId": "CTRL-1", - "language": "de", - "title": "Synthetic control title", - "text": "Synthetic control text used only to demonstrate the contract.", - "sourceUri": "https://example.invalid/framework/control-1" - } - ] -} +```text +npm run rag:prepare-iso -- "ISO27001 2024.PDF" "ISO27002 2024.PDF" +npm run rag:ingest -- .assessor-data/regulatory-corpora/iso-27001-27002-de.json ``` The checked-in [`tests/fixtures/regulatory-corpus.synthetic.json`](../tests/fixtures/regulatory-corpus.synthetic.json) is an unlicensed synthetic EN/DE smoke corpus. It exists to verify rebuilding and retrieval mechanics only; it is not a regulatory source and must never be presented as compliance guidance. @@ -81,11 +71,11 @@ npm run db:up npm run rag:ingest -- path/to/regulatory-corpus.json ``` -Ingestion validates every document, splits text deterministically on paragraph/sentence/word boundaries, embeds all chunks in bounded batches, and only then replaces the indexed chunks for each `(corpusId, documentId)`. Replacement is not a cross-system transaction: if Qdrant fails after deletion, rerun ingestion from the unchanged corpus. +The parser accepts encrypted documents only when no password is required and copying is permitted. It rejects scanned/empty, oversized, copy-prohibited, structurally unexpected, incomplete, duplicate, or mismatched catalogs. The build must contain the matching 93 controls with theme counts 37/8/14/34. It retains hashes, editions, control/section metadata, and page provenance without committing licensed text. Ingestion splits only inside a control section, embeds all chunks in bounded batches, and then replaces indexed chunks per `(corpusId, documentId)`. If Qdrant fails after deletion, rerun ingestion from the unchanged corpus JSON. ## Configuration and trust -Qdrant 1.19.1 runs in Docker with a persistent named volume, an API key, and outbound usage telemetry disabled. Local REST access is loopback-only; deployment access stays on the Compose network. Remote plain HTTP Qdrant URLs are rejected. +Qdrant 1.19.1 runs in Docker with a persistent named volume and outbound usage telemetry disabled. Local REST access is loopback-only; deployment access stays on the Compose network. An API key is optional for an unsecured loopback/internal self-host and required for remote hosts. Remote plain HTTP Qdrant URLs are rejected. The embedding adapter uses an OpenAI-compatible `/v1/embeddings` endpoint. Remote endpoints must use HTTPS. Plain HTTP is accepted only for `localhost`, `127.0.0.1`, or `::1`, enabling a same-machine llama.cpp development server without weakening remote transport. Configuration must explicitly provide the model name and its exact native/requested dimensions. Redirects are rejected, response size and time are bounded, and API keys remain in backend environment variables. @@ -107,29 +97,44 @@ ASSESSOR_RAG_EMBEDDING_MODEL=bge-m3 ASSESSOR_RAG_EMBEDDING_API_KEY= ASSESSOR_RAG_EMBEDDING_DIMENSIONS=1024 ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS=false +ASSESSOR_RAG_ACTIVE_CORPUS_ID=iso-27001-27002-de +ASSESSOR_RAG_ACTIVE_CORPUS_VERSION= ``` For a later LiteLLM comparison with `text-embedding-3-large`, use the approved HTTPS gateway and set `ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS=true` if requesting 1,024-dimensional output. OpenAI supports the `dimensions` parameter for `text-embedding-3` models; llama.cpp currently returns the model's native dimension and should leave this flag false. Changing model identity or dimensions requires rebuilding the derived Qdrant corpus before comparing results. -Embedding is an external processing boundary. Regulatory text is sent during ingestion. The finding retrieval service can now send only the minimized public-signal query described below; unrestricted report text, paths, tenant identities, reviews, raw evidence, and local provenance IDs remain outside the embedding request. No browser endpoint or automatic mapping workflow activates this path. +Embedding is an external processing boundary. Regulatory text is sent during ingestion. The finding retrieval service sends only the privacy-safe check query described below; the result table, paths, tenant identities, reviews, raw evidence, and local provenance IDs remain outside the embedding and completion requests. ## Finding query privacy boundary -`buildRegulatoryFindingQuery` accepts the existing `UnifiedResultProjection` but emits only fixed public concepts: +`buildRegulatoryFindingQuery` accepts the existing `UnifiedResultProjection` and builds the query from the **shared privacy-safe finding projection** (`src/domain/privacy-safe-finding.ts`, `privacySafeFinding`) — the identical boundary used by the AI overview: -- exact Maester or PingCastle rule IDs that already exist in the checked-in public catalog; -- fixed public technical topic labels recognized locally from title, description, or recommendation; and -- a fixed public scanner name for known profiles. +- the rule-level **check text** — title, plus description and recommendation when present — which comes from the deduplicated finding *definition* (canonical key `tool:ruleId`) and describes the check, not the tenant; email addresses are redacted; +- a catalog Maester/PingCastle rule ID, a fixed public scanner name, and recognized public technical topics, all as optional query **enrichment**. -The query never copies raw title, description, recommendation, asset, source path, project/tenant identity, review owner, review text, original values, or provenance IDs. Unsupported result kinds and scanner findings without a recognized public rule or topic fail closed as `insufficient-public-context` and do not call the retriever. The local `sourceResultId` is returned only for in-process correlation and is not included in the embedding request. +The query never reads the **result table / occurrence data**: affected asset, source locator/path, per-object values, project/tenant identity, review owner, review text, original values, or provenance IDs. Because the query is built from the rule-level check text, **every scanner finding with a check title maps**; only non-scanner records and findings without a check title fail closed as `insufficient-public-context` and do not call the retriever. The local `sourceResultId` is returned only for in-process correlation and is not included in the embedding request. -This is a technical minimization boundary, not organizational approval to process production findings externally. The service remains backend-only and has no API/UI route. Any production activation still requires approval of the embedding destination, catalog coverage, retention terms, and the precise finding classes allowed to cross that boundary. +This is a technical minimization boundary. Activation still depends on the approved embedding/LiteLLM destinations, catalog coverage, retention terms, and the precise finding classes allowed to cross that boundary. Residual note: `privacySafeFinding` redacts emails only; it relies on definitions being rule-generic, so a scanner that folds an identifier (host, IP, path) into a *definition* title/description would carry it into the query. Strengthening that redaction is a shared decision affecting both AI features (see implementation plan). + +## One-finding mapping workflow + +The Work finding reader loads mapping state only after `Map ISO controls` is opened. Mappings are **global**: the finding contributes only its non-personal rule identity and check context, and the mapping is keyed by that identity plus the pipeline versions, so one rule is mapped once and every occurrence in every project reuses it. The API returns HTTP 202 and the UI polls while queued/running. The service retrieves 32 eligible ISO 27002 chunks, groups by maximum dense score, keeps eight controls, attaches up to two semantic passages and the matching ISO 27001 statement, and bounds the LiteLLM payload to 48,000 characters. + +The completion schema permits at most five `primary` or `supporting` proposals, advisory low/medium/high confidence, a bounded rationale, grounded evidence IDs, or an explicit `no_relevant_control`/`insufficient_context` outcome. Unknown controls, foreign evidence IDs, duplicates, malformed JSON, and oversized content fail closed. A reviewer can accept or reject any grounded top-eight candidate with an optional comment; each change appends a **global** decision event keyed by rule identity and control, using optimistic concurrency, shared across every project. Acceptance means relevant control, never compliance. + +Routes: + +- `GET /api/v1/projects/{projectId}/regulatory-mappings/{resultProjectionId}` +- `POST /api/v1/projects/{projectId}/regulatory-mapping-runs` +- `POST /api/v1/projects/{projectId}/regulatory-mapping-decisions` + +Migration 025 originally added occurrence-bound, project-scoped run and decision tables. Migration 026 supersedes it: it drops those tables and creates the global `assessor_regulatory_mappings` (keyed by `rule_key` + pipeline versions, `UNIQUE(rule_key, corpus_version, retrieval_version, prompt_version, model)`) and a global, rule-keyed `assessor_regulatory_mapping_decisions`; neither carries `project_id` or occurrence identity, and both survive project/finding deletion. Before 026 was applied, the 025 tables held only regenerable local test data (19 runs, 10 decisions); a verified backup of both tables was taken at `backups/regulatory_mapping_025_backup_20260915-155410.sql` and the data was then intentionally discarded per the user's decision. A live development backend applied 026 through its own migration runner (checksum recorded and re-verified against the source file); no reimport is required because mapping runs are regenerable and no scanner data changed. ### Local Qdrant integration check `tests/server/qdrant-regulatory-index.integration.test.ts` is opt-in and uses deterministic synthetic vectors. With `ASSESSOR_QDRANT_INTEGRATION=1` and the normal Qdrant/model identity environment variables, it verifies the real collection, payload indexes, document replacement, filters, and retrieval without calling an embedding provider. The standard suite skips it so tests do not unexpectedly mutate a developer's derived index. -On 2026-09-15 this check passed against Qdrant 1.19.1 in WSL: authenticated access returned HTTP 200, unauthenticated access returned HTTP 401, the collection was green, Dense used 1,024-dimensional cosine vectors, the Sparse slot existed, all nine filter payload indexes existed, and one synthetic integration point was retrieved and then removed. PostgreSQL was not accessed or changed. +On 2026-09-15 this check passed against Qdrant 1.19.1 in WSL: the collection was green, Dense used 1,024-dimensional cosine vectors, the Sparse slot existed, all 12 filter payload indexes existed, and one synthetic integration point was retrieved and then removed. ## Dense retrieval evaluation @@ -175,7 +180,7 @@ Live smoke result on 2026-09-15 with local llama.cpp build 10472, BGE-M3 FP16, a - `RegulatoryReranker` is optional and receives only the bounded candidate set. Without one, dense ordering is retained. - `RegulatoryRetriever` is the consumer-facing contract. Mapping code should depend on it, not on Qdrant. -Before enabling automatic finding mapping, complete organizational privacy approval, broaden public-signal coverage only through reviewed catalogs, establish the real dense baseline, and add control-level grouping, hybrid evaluation, reranker evaluation, calibrated confidence thresholds, a constrained output schema, human-review states, and append-only PostgreSQL mapping provenance. +Before assessment-wide batch mapping, broaden public-signal coverage only through reviewed catalogs, establish a representative independently labeled EN/DE baseline, and evaluate sparse/hybrid retrieval, reranking, and confidence calibration. The one-finding workflow must remain human-reviewed. ## Reference basis diff --git a/package-lock.json b/package-lock.json index b65bbc3..f46defa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@fontsource-variable/inter": "^5.3.0", "jsdom": "^29.1.1", + "pdfjs-dist": "^5.4.296", "pg": "^8.22.0", "react": "19.2.4", "react-dom": "19.2.4", @@ -1021,6 +1022,271 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz", + "integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.100", + "@napi-rs/canvas-darwin-arm64": "0.1.100", + "@napi-rs/canvas-darwin-x64": "0.1.100", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", + "@napi-rs/canvas-linux-arm64-musl": "0.1.100", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-musl": "0.1.100", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", + "@napi-rs/canvas-win32-x64-msvc": "0.1.100" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz", + "integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz", + "integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz", + "integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz", + "integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz", + "integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz", + "integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz", + "integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz", + "integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz", + "integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz", + "integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz", + "integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-rc.2", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.2.tgz", @@ -2281,6 +2547,18 @@ "node": ">= 14.16" } }, + "node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/pg": { "version": "8.22.0", "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", diff --git a/package.json b/package.json index ce8bc75..f7cc17f 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "db:down": "docker compose --env-file .env.postgres -f docker-compose.postgres.yml down", "db:logs": "docker compose --env-file .env.postgres -f docker-compose.postgres.yml logs -f postgres", "rag:ingest": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-ingest.mjs", + "rag:prepare-iso": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-prepare-iso.mjs", "rag:evaluate": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-evaluate.mjs", "test:postgres": "vitest run --config vitest.config.ts tests/postgres/postgres-repository.integration.test.ts && vitest run --config vitest.config.ts tests/postgres/gpo-backup-management.integration.test.ts tests/postgres/policy-analysis.integration.test.ts", "test": "vitest run --config vitest.config.ts && node tests/local-only-check.mjs && node tests/design-language-check.mjs", @@ -25,6 +26,7 @@ "dependencies": { "@fontsource-variable/inter": "^5.3.0", "jsdom": "^29.1.1", + "pdfjs-dist": "^5.4.296", "pg": "^8.22.0", "react": "19.2.4", "react-dom": "19.2.4", @@ -41,5 +43,8 @@ "typescript": "5.9.3", "vite": "^7.3.6", "vitest": "^3.2.7" + }, + "allowScripts": { + "esbuild@0.27.7": true } } diff --git a/server/assessor-http-server.ts b/server/assessor-http-server.ts index 09df6ab..f4ba5e2 100644 --- a/server/assessor-http-server.ts +++ b/server/assessor-http-server.ts @@ -30,6 +30,9 @@ import { NvdCveClientError } from './cve/nvd-cve-client'; import type { AiOverviewService } from './ai/ai-overview-service'; import { AiOverviewServiceError } from './ai/ai-overview-service'; import type { AiOverviewCreateRequest, AiOverviewScope } from '../src/domain/ai-overview'; +import type { RegulatoryMappingCreateRequest, RegulatoryMappingDecisionCreateRequest } from '../src/domain/regulatory-mapping'; +import type { RegulatoryMappingService } from './regulatory/regulatory-mapping-service'; +import { RegulatoryMappingServiceError } from './regulatory/regulatory-mapping-service'; import type { GpoAutomaticAnalysisService } from './gpo/gpo-automatic-analysis-service'; import type { MicrosoftSecurityBaselineSelectionContext } from '../src/domain/microsoft-security-baseline'; import { listProjectGpoBackupStatus } from './gpo/gpo-backup-status-service'; @@ -140,6 +143,7 @@ export interface AssessorHttpServerOptions { microsoftSecurityBaselineService?: MicrosoftSecurityBaselineService; cveEnrichmentService?: CveEnrichmentService; aiOverviewService?: AiOverviewService; + regulatoryMappingService?: RegulatoryMappingService; } class HttpError extends Error { @@ -351,6 +355,7 @@ function statusForError(error: unknown): { status: number; code: string; message return { status, code: error.code, message: error.message }; } if (error instanceof AiOverviewServiceError) return { status: error.status, code: error.code, message: error.message }; + if (error instanceof RegulatoryMappingServiceError) return { status: error.status, code: error.code, message: error.message }; return { status: 500, code: 'internal_error', message: 'Die Anfrage konnte nicht verarbeitet werden.' }; } @@ -908,6 +913,20 @@ export function createAssessorHttpServer(options: AssessorHttpServerOptions): Se const body = await readJson(request, maximumBodyBytes); status = 202; sendJson(response, status, await options.aiOverviewService.start(segments[1], body), requestId); return; } + if (segments[0] === 'projects' && segments[2] === 'regulatory-mappings' && segments.length === 4 && method === 'GET') { + if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.'); + status = 200; sendJson(response, status, await options.regulatoryMappingService.get(segments[1], segments[3]), requestId); return; + } + if (segments[0] === 'projects' && segments[2] === 'regulatory-mapping-runs' && segments.length === 3 && method === 'POST') { + if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.'); + const body = await readJson(request, maximumBodyBytes); + status = 202; sendJson(response, status, await options.regulatoryMappingService.start(segments[1], body), requestId); return; + } + if (segments[0] === 'projects' && segments[2] === 'regulatory-mapping-decisions' && segments.length === 3 && method === 'POST') { + if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.'); + const body = await readJson(request, maximumBodyBytes); + status = 200; sendJson(response, status, await options.regulatoryMappingService.decide(segments[1], body), requestId); return; + } if (segments.length === 1 && segments[0] === 'assessments' && method === 'POST') { const assessment = await readJson(request, maximumBodyBytes); await repository.saveAssessment(assessment); diff --git a/server/index.ts b/server/index.ts index b0cdd9e..6da7cc0 100644 --- a/server/index.ts +++ b/server/index.ts @@ -21,6 +21,8 @@ import { CveEnrichmentService } from './cve/cve-enrichment-service'; import { loadPolicyEvaluationRuleCatalog } from './gpo/policy-evaluation-rule-catalog-loader'; import { createLiteLlmClientFromEnvironment } from './ai/litellm-client'; import { AiOverviewService } from './ai/ai-overview-service'; +import { createRegulatoryRagRuntime } from './regulatory/regulatory-rag-runtime'; +import { RegulatoryMappingService } from './regulatory/regulatory-mapping-service'; function boundedInteger(value: string | undefined, fallback: number, minimum: number, maximum: number, name: string): number { if (value === undefined || value.trim() === '') return fallback; @@ -61,13 +63,35 @@ async function main(): Promise { expectedSha256: process.env.ASSESSOR_POLICY_RULE_CATALOG_SHA256 }); const runtime = await createRepositoryRuntime(); + const aiClient = createLiteLlmClientFromEnvironment(); const aiOverviewService = new AiOverviewService(runtime.repository, { - client: createLiteLlmClientFromEnvironment(), + client: aiClient, maximumChunkCharacters: boundedInteger(process.env.ASSESSOR_AI_CHUNK_CHARS, 80_000, 10_000, 500_000, 'ASSESSOR_AI_CHUNK_CHARS'), maximumCalls: boundedInteger(process.env.ASSESSOR_AI_MAX_CALLS, 64, 1, 64, 'ASSESSOR_AI_MAX_CALLS'), leaseDurationMs: boundedInteger(process.env.ASSESSOR_AI_JOB_LEASE_MS, 5 * 60_000, 10_000, 30 * 60_000, 'ASSESSOR_AI_JOB_LEASE_MS') }); await aiOverviewService.recoverInterruptedRuns(); + let regulatoryMappingService: RegulatoryMappingService; + try { + const corpusId = process.env.ASSESSOR_RAG_ACTIVE_CORPUS_ID?.trim(); + const corpusVersion = process.env.ASSESSOR_RAG_ACTIVE_CORPUS_VERSION?.trim(); + if (!corpusId || !corpusVersion) throw new Error('No active regulatory corpus is configured.'); + const rag = createRegulatoryRagRuntime(); + regulatoryMappingService = new RegulatoryMappingService(runtime.repository, { + findingRetrieval: rag.findingRetrieval, + index: rag.index, + embeddingModel: rag.embeddingModel, + client: aiClient, + corpusId, + corpusVersion, + ...(!aiClient ? { disabledReason: 'Work LiteLLM is not configured.' } : {}) + }); + } catch (error) { + regulatoryMappingService = new RegulatoryMappingService(runtime.repository, { + disabledReason: error instanceof Error ? error.message : 'Regulatory mapping is unavailable.' + }); + } + await regulatoryMappingService.recoverInterruptedRuns(); const artifactStore = new FileSystemBackendArtifactStore(artifactRoot); const gpoBackupSubmissionService = new GpoBackupSubmissionService( runtime.repository, @@ -129,6 +153,7 @@ async function main(): Promise { gpoPolicyAnalysisOverviewService, cveEnrichmentService, aiOverviewService, + regulatoryMappingService, automaticGpoAnalysis, importExecutor }); @@ -144,7 +169,7 @@ async function main(): Promise { if (shuttingDown) return; shuttingDown = true; console.log(`${signal}: ASSESSOR wird beendet.`); - void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop()]).then(() => { + void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop(), aiOverviewService.close(), regulatoryMappingService.close()]).then(() => { server.close(async error => { try { await runtime.close(); diff --git a/server/postgres/migration-catalog.ts b/server/postgres/migration-catalog.ts index 9150575..126bb99 100644 --- a/server/postgres/migration-catalog.ts +++ b/server/postgres/migration-catalog.ts @@ -20,6 +20,8 @@ import cveEnrichment from './migrations/021_cve_enrichment.sql?raw'; import vulnerabilityFindingResultType from './migrations/022_vulnerability_finding_result_type.sql?raw'; import defenderVulnerabilityBackfill from './migrations/023_defender_vulnerability_backfill.sql?raw'; import aiOverviewRuns from './migrations/024_ai_overview_runs.sql?raw'; +import regulatoryMapping from './migrations/025_regulatory_mapping.sql?raw'; +import globalRegulatoryMapping from './migrations/026_global_regulatory_mapping.sql?raw'; import nativePolicyTargets from './migrations/019_native_policy_targets.sql?raw'; import baselineDrivenPolicyEvaluation from './migrations/020_baseline_driven_policy_evaluation.sql?raw'; @@ -53,5 +55,7 @@ export const databaseMigrations: readonly DatabaseMigration[] = [ { version: '021', name: 'cve-enrichment', sql: cveEnrichment }, { version: '022', name: 'vulnerability-finding-result-type', sql: vulnerabilityFindingResultType }, { version: '023', name: 'defender-vulnerability-backfill', sql: defenderVulnerabilityBackfill }, - { version: '024', name: 'ai-overview-runs', sql: aiOverviewRuns } + { version: '024', name: 'ai-overview-runs', sql: aiOverviewRuns }, + { version: '025', name: 'regulatory-mapping', sql: regulatoryMapping }, + { version: '026', name: 'global-regulatory-mapping', sql: globalRegulatoryMapping } ]; diff --git a/server/postgres/migrations/025_regulatory_mapping.sql b/server/postgres/migrations/025_regulatory_mapping.sql new file mode 100644 index 0000000..7e5aca8 --- /dev/null +++ b/server/postgres/migrations/025_regulatory_mapping.sql @@ -0,0 +1,43 @@ +CREATE TABLE assessor_regulatory_mapping_runs ( + id text PRIMARY KEY, + project_id text NOT NULL REFERENCES assessor_projects(id) ON DELETE CASCADE, + source_result_id text NOT NULL REFERENCES assessor_finding_occurrences(id) ON DELETE CASCADE, + result_projection_id text NOT NULL, + idempotency_key text NOT NULL, + run_status text NOT NULL CHECK (run_status IN ('queued', 'running', 'succeeded', 'failed')), + query_hash text NOT NULL CHECK (query_hash ~ '^[a-f0-9]{64}$'), + corpus_id text NOT NULL, + corpus_version text NOT NULL, + embedding_model text NOT NULL, + retrieval_version text NOT NULL, + prompt_version text NOT NULL, + output_schema_version integer NOT NULL, + model text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + data jsonb NOT NULL, + CONSTRAINT assessor_regulatory_mapping_runs_idempotency_uq + UNIQUE (project_id, source_result_id, idempotency_key) +); + +CREATE UNIQUE INDEX assessor_regulatory_mapping_runs_active_finding_uq + ON assessor_regulatory_mapping_runs (project_id, source_result_id) + WHERE run_status IN ('queued', 'running'); + +CREATE INDEX assessor_regulatory_mapping_runs_finding_created_idx + ON assessor_regulatory_mapping_runs (project_id, source_result_id, created_at DESC, id DESC); + +CREATE TABLE assessor_regulatory_mapping_decisions ( + id text PRIMARY KEY, + project_id text NOT NULL REFERENCES assessor_projects(id) ON DELETE CASCADE, + source_result_id text NOT NULL REFERENCES assessor_finding_occurrences(id) ON DELETE CASCADE, + run_id text NOT NULL REFERENCES assessor_regulatory_mapping_runs(id) ON DELETE CASCADE, + control_id text NOT NULL, + decision text NOT NULL CHECK (decision IN ('accepted', 'rejected')), + previous_decision_id text REFERENCES assessor_regulatory_mapping_decisions(id), + created_at timestamptz NOT NULL, + data jsonb NOT NULL +); + +CREATE INDEX assessor_regulatory_mapping_decisions_finding_created_idx + ON assessor_regulatory_mapping_decisions (project_id, source_result_id, control_id, created_at DESC, id DESC); diff --git a/server/postgres/migrations/026_global_regulatory_mapping.sql b/server/postgres/migrations/026_global_regulatory_mapping.sql new file mode 100644 index 0000000..e81e857 --- /dev/null +++ b/server/postgres/migrations/026_global_regulatory_mapping.sql @@ -0,0 +1,47 @@ +-- Regulatory ISO mappings become global, tenant-independent knowledge keyed by rule +-- identity plus pipeline versions (see docs/REGULATORY_RAG.md). The prior per-occurrence +-- tables from migration 025 are replaced; they held only regenerable test data. +DROP TABLE IF EXISTS assessor_regulatory_mapping_decisions; +DROP TABLE IF EXISTS assessor_regulatory_mapping_runs; + +CREATE TABLE assessor_regulatory_mappings ( + id text PRIMARY KEY, + rule_key text NOT NULL, + source_tool text NOT NULL, + source_rule_id text, + run_status text NOT NULL CHECK (run_status IN ('queued', 'running', 'succeeded', 'failed')), + query_hash text NOT NULL CHECK (query_hash ~ '^[a-f0-9]{64}$'), + corpus_id text NOT NULL, + corpus_version text NOT NULL, + embedding_model text NOT NULL, + retrieval_version text NOT NULL, + prompt_version text NOT NULL, + output_schema_version integer NOT NULL, + model text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + data jsonb NOT NULL, + -- The id is a deterministic digest of (rule_key, corpus/retrieval/prompt versions, model), + -- so there is exactly one row per rule per pipeline version tuple. + CONSTRAINT assessor_regulatory_mappings_identity_uq + UNIQUE (rule_key, corpus_version, retrieval_version, prompt_version, model) +); + +CREATE INDEX assessor_regulatory_mappings_rule_idx + ON assessor_regulatory_mappings (rule_key, updated_at DESC, id DESC); + +CREATE TABLE assessor_regulatory_mapping_decisions ( + id text PRIMARY KEY, + rule_key text NOT NULL, + source_tool text NOT NULL, + source_rule_id text, + mapping_id text NOT NULL, + control_id text NOT NULL, + decision text NOT NULL CHECK (decision IN ('accepted', 'rejected')), + previous_decision_id text REFERENCES assessor_regulatory_mapping_decisions(id), + created_at timestamptz NOT NULL, + data jsonb NOT NULL +); + +CREATE INDEX assessor_regulatory_mapping_decisions_rule_control_idx + ON assessor_regulatory_mapping_decisions (rule_key, control_id, created_at DESC, id DESC); diff --git a/server/postgres/postgres-assessor-repository.ts b/server/postgres/postgres-assessor-repository.ts index e07e5e0..9706192 100644 --- a/server/postgres/postgres-assessor-repository.ts +++ b/server/postgres/postgres-assessor-repository.ts @@ -27,6 +27,8 @@ import { environmentCoverage } from '../../src/domain/environment-context'; import type { CveEnrichmentRecord } from '../../src/domain/cve-enrichment'; import type { AiOverviewRun } from '../../src/domain/ai-overview'; import { aiOverviewRunValidationErrors } from '../../src/domain/ai-overview'; +import type { RegulatoryMappingDecision, RegulatoryMappingRun } from '../../src/domain/regulatory-mapping'; +import { regulatoryMappingDecisionValidationErrors, regulatoryMappingRunValidationErrors } from '../../src/domain/regulatory-mapping'; import { cveEnrichmentValidationErrors, cveIdFromRuleId, normalizeCveId } from '../../src/domain/cve-enrichment'; import { storedPolicyFindingRunValidationErrors } from '../../src/domain/policy-finding'; import { @@ -366,6 +368,8 @@ const DB_STRUCTURE_LAYER_BY_TABLE: Record = { assessor_edge_coverage: 'canonical', assessor_finding_definitions: 'result', assessor_cve_enrichments: 'result', + assessor_regulatory_mappings: 'result', + assessor_regulatory_mapping_decisions: 'governance', assessor_ai_overview_runs: 'governance', assessor_finding_occurrences: 'result', assessor_result_categories: 'result', @@ -666,6 +670,79 @@ export class PostgresAssessorRepository implements AssessorRepository { `, [projectId, scopeKey ?? null])); } + async getRegulatoryMappingRun(id: string): Promise { + return dataRow(await this.pool.query>( + 'SELECT data FROM assessor_regulatory_mappings WHERE id = $1', [id] + )); + } + + async saveRegulatoryMappingRun(run: RegulatoryMappingRun): Promise { + const errors = regulatoryMappingRunValidationErrors(run); + if (errors.length) throw new RepositoryValidationError(errors.join(' ')); + return this.mutate(async client => { + await client.query(` + INSERT INTO assessor_regulatory_mappings + (id, rule_key, source_tool, source_rule_id, run_status, query_hash, corpus_id, corpus_version, + embedding_model, retrieval_version, prompt_version, output_schema_version, model, created_at, updated_at, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16::jsonb) + ON CONFLICT (id) DO UPDATE SET + run_status = excluded.run_status, updated_at = excluded.updated_at, data = excluded.data + `, [run.id, run.ruleKey, run.sourceTool, run.sourceRuleId ?? null, run.status, run.queryHash, run.corpusId, + run.corpusVersion, run.embeddingModel, run.retrievalVersion, run.promptVersion, run.outputSchemaVersion, + run.model, run.createdAt, run.updatedAt, json(run)]); + return clone(run); + }); + } + + async listRegulatoryMappingRuns(): Promise { + return dataRows(await this.pool.query>( + 'SELECT data FROM assessor_regulatory_mappings ORDER BY created_at DESC, id DESC' + )); + } + + async createRegulatoryMappingDecision( + decision: RegulatoryMappingDecision, + expectedLatestDecisionId?: string + ): Promise { + const errors = regulatoryMappingDecisionValidationErrors(decision); + if (errors.length) throw new RepositoryValidationError(errors.join(' ')); + return this.mutate(async client => { + const run = dataRow(await client.query>( + 'SELECT data FROM assessor_regulatory_mappings WHERE id = $1 FOR UPDATE', [decision.mappingId] + )); + if (!run || run.ruleKey !== decision.ruleKey || run.status !== 'succeeded') { + throw new RepositoryNotFoundError('Completed regulatory mapping does not exist for this rule.'); + } + if (!run.candidates.some(candidate => candidate.controlId === decision.controlId)) { + throw new RepositoryValidationError('Decision control was not a grounded candidate.'); + } + const latest = dataRow(await client.query>(` + SELECT data FROM assessor_regulatory_mapping_decisions + WHERE rule_key = $1 AND control_id = $2 + ORDER BY created_at DESC, id DESC LIMIT 1 + `, [decision.ruleKey, decision.controlId])); + if ((expectedLatestDecisionId ?? undefined) !== (latest?.id ?? undefined)) { + throw new RepositoryConflictError('Regulatory mapping decision changed before this update.'); + } + const stored = { ...decision, ...(latest ? { previousDecisionId: latest.id } : {}) }; + await client.query(` + INSERT INTO assessor_regulatory_mapping_decisions + (id, rule_key, source_tool, source_rule_id, mapping_id, control_id, decision, previous_decision_id, created_at, data) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb) + `, [stored.id, stored.ruleKey, stored.sourceTool, stored.sourceRuleId ?? null, stored.mappingId, stored.controlId, + stored.decision, stored.previousDecisionId ?? null, stored.createdAt, json(stored)]); + return clone(stored); + }); + } + + async listRegulatoryMappingDecisions(ruleKey: string): Promise { + return dataRows(await this.pool.query>(` + SELECT data FROM assessor_regulatory_mapping_decisions + WHERE rule_key = $1 + ORDER BY created_at DESC, id DESC + `, [ruleKey])); + } + async beginImport(start: ImportSessionStart): Promise { if (!start.sessionId.trim()) throw new RepositoryValidationError('Eine Importsitzung benötigt eine ID.'); if (this.activeImportSessions.has(start.sessionId)) throw new RepositoryConflictError(`Importsitzung ${start.sessionId} ist bereits aktiv.`); @@ -2447,7 +2524,12 @@ export class PostgresAssessorRepository implements AssessorRepository { policySnapshotAggregations: await this.listPolicySnapshotAggregations(projectId), policyEvaluations: await this.listPolicyEvaluations(projectId), policyFindingRuns, - aiOverviewRuns: (await this.listAiOverviewRuns(projectId)).filter(item => item.status === 'succeeded') + aiOverviewRuns: (await this.listAiOverviewRuns(projectId)).filter(item => item.status === 'succeeded'), + // Regulatory mappings and decisions are global reference data, not project-scoped. + regulatoryMappingRuns: (await this.listRegulatoryMappingRuns()).filter(item => item.status === 'succeeded'), + regulatoryMappingDecisions: dataRows(await this.pool.query>( + 'SELECT data FROM assessor_regulatory_mapping_decisions ORDER BY created_at, id' + )) }; } diff --git a/server/regulatory/iso-pdf-corpus.ts b/server/regulatory/iso-pdf-corpus.ts new file mode 100644 index 0000000..a53afcf --- /dev/null +++ b/server/regulatory/iso-pdf-corpus.ts @@ -0,0 +1,334 @@ +import { createHash } from 'node:crypto'; +import { readFile, stat } from 'node:fs/promises'; +import { getDocument, PermissionFlag } from 'pdfjs-dist/legacy/build/pdf.mjs'; +import type { + RegulatoryControlDocument, + RegulatoryControlTheme, + RegulatoryCorpusV2, + RegulatorySectionKind +} from '../../src/domain/regulatory-rag'; + +export const ISO_PDF_PARSER_VERSION = '1.0.0'; +const MAXIMUM_PDF_BYTES = 32 * 1024 * 1024; +const EXPECTED_THEME_COUNTS: Record = { + organizational: 37, + people: 8, + physical: 14, + technological: 34 +}; + +export interface PdfSpan { + text: string; + page: number; + x: number; + y: number; + width: number; + height: number; +} + +interface PdfJsTextItem { + str: string; + transform: unknown[]; + width: number; + height: number; +} + +export interface PdfLine { + text: string; + page: number; + y: number; + spans: PdfSpan[]; +} + +export interface ParsedPdf { + sha256: string; + pageCount: number; + pages: Array<{ page: number; lines: PdfLine[] }>; +} + +export interface ParsedSection { + kind: RegulatorySectionKind; + text: string; + pageStart: number; + pageEnd: number; +} + +export interface ParsedControl { + controlId: string; + title: string; + theme: RegulatoryControlTheme; + sections: ParsedSection[]; +} + +export interface IsoPdfCorpusOptions { + iso27001Path: string; + iso27002Path: string; + corpusId?: string; + corpusVersion?: string; + generatedAt?: string; +} + +export async function buildIsoPdfCorpus(options: IsoPdfCorpusOptions): Promise { + const [iso27001, iso27002] = await Promise.all([ + extractPdf(options.iso27001Path, 'ISO/IEC 27001'), + extractPdf(options.iso27002Path, 'ISO/IEC 27002') + ]); + const references = parseIso27001AnnexA(iso27001); + const semanticControls = parseIso27002Controls(iso27002); + validateIsoControlSets(references, semanticControls); + const corpusId = options.corpusId?.trim() || 'iso-27001-27002-de'; + const combinedHash = sha256(`${iso27001.sha256}\n${iso27002.sha256}`); + const corpusVersion = options.corpusVersion?.trim() || `din-2024-de-${combinedHash.slice(0, 12)}`; + const common = { corpusId, corpusVersion, language: 'de', parserVersion: ISO_PDF_PARSER_VERSION } as const; + const documents: RegulatoryControlDocument[] = []; + + for (const control of semanticControls) { + const reference = references.find(item => item.controlId === control.controlId)!; + documents.push(documentForSection(common, control, reference.sections[0], { + sourceStandard: 'ISO/IEC 27001', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27001:2024-01', + sourceSha256: iso27001.sha256, retrievalRole: 'reference' + })); + for (const section of control.sections) { + documents.push(documentForSection(common, control, section, { + sourceStandard: 'ISO/IEC 27002', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27002:2024-01', + sourceSha256: iso27002.sha256, retrievalRole: 'semantic' + })); + } + } + + return { + schemaVersion: 2, + corpusId, + corpusVersion, + language: 'de', + parserVersion: ISO_PDF_PARSER_VERSION, + generatedAt: options.generatedAt ?? new Date().toISOString(), + sources: [ + { standard: 'ISO/IEC 27001', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27001:2024-01', sha256: iso27001.sha256, pageCount: iso27001.pageCount }, + { standard: 'ISO/IEC 27002', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27002:2024-01', sha256: iso27002.sha256, pageCount: iso27002.pageCount } + ], + documents + }; +} + +function documentForSection( + common: { corpusId: string; corpusVersion: string; language: string; parserVersion: string }, + control: ParsedControl, + section: ParsedSection, + source: Pick +): RegulatoryControlDocument { + const sourceToken = source.sourceStandard === 'ISO/IEC 27001' ? '27001' : '27002'; + return { + schemaVersion: 2, + ...common, + documentId: `${common.corpusId}:${control.controlId}:${sourceToken}:${section.kind}`, + framework: 'ISO/IEC 27001 Annex A', + frameworkVersion: '2022', + controlId: control.controlId, + annexAControlId: `A.${control.controlId}`, + controlTheme: control.theme, + title: control.title, + text: section.text, + sectionKind: section.kind, + sourcePageStart: section.pageStart, + sourcePageEnd: section.pageEnd, + ...source + }; +} + +async function extractPdf(path: string, expectedStandard: string): Promise { + const file = await stat(path); + if (!file.isFile() || file.size < 1 || file.size > MAXIMUM_PDF_BYTES) { + throw new Error(`${expectedStandard} PDF must be a non-empty file no larger than 32 MiB.`); + } + const bytes = await readFile(path); + let document; + try { + document = await getDocument({ data: new Uint8Array(bytes), password: '', isEvalSupported: false, useSystemFonts: true }).promise; + } catch { + throw new Error(`${expectedStandard} PDF could not be opened without a password.`); + } + try { + const permissions = await document.getPermissions(); + if (permissions && !permissions.includes(PermissionFlag.COPY)) throw new Error(`${expectedStandard} PDF does not permit text extraction.`); + if (document.numPages < 1 || document.numPages > 1_000) throw new Error(`${expectedStandard} PDF page count is outside the supported range.`); + const pages: ParsedPdf['pages'] = []; + let characterCount = 0; + for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) { + const page = await document.getPage(pageNumber); + const viewport = page.getViewport({ scale: 1 }); + const content = await page.getTextContent({ disableNormalization: false }); + const spans = content.items.flatMap(item => toSpan(item, pageNumber, viewport.height)); + const lines = groupLines(spans); + characterCount += lines.reduce((sum, line) => sum + line.text.length, 0); + pages.push({ page: pageNumber, lines }); + page.cleanup(); + } + if (characterCount < document.numPages * 100) throw new Error(`${expectedStandard} PDF does not contain a usable text layer.`); + return { sha256: createHash('sha256').update(bytes).digest('hex'), pageCount: document.numPages, pages }; + } finally { + await document.destroy(); + } +} + +function toSpan(item: unknown, page: number, pageHeight: number): PdfSpan[] { + if (!isTextItem(item)) return []; + const [a, b, c, d, x, y] = item.transform.map(Number); + if (![a, b, c, d, x, y].every(Number.isFinite) || Math.abs(b) > 0.2 || Math.abs(c) > 0.2) return []; + if (y < 70 || y > pageHeight - 55) return []; + const text = normalizeInline(item.str); + return text ? [{ text, page, x, y, width: Number(item.width) || 0, height: Number(item.height) || Math.abs(d) }] : []; +} + +function groupLines(spans: PdfSpan[]): PdfLine[] { + const rows: PdfSpan[][] = []; + for (const span of spans.slice().sort((left, right) => right.y - left.y || left.x - right.x)) { + const row = rows.find(candidate => Math.abs(candidate[0].y - span.y) <= Math.max(1.5, span.height * 0.12)); + if (row) row.push(span); else rows.push([span]); + } + return rows.map(row => { + const ordered = row.sort((left, right) => left.x - right.x); + return { page: ordered[0].page, y: ordered[0].y, spans: ordered, text: joinSpans(ordered) }; + }).filter(line => line.text); +} + +function joinSpans(spans: PdfSpan[]): string { + let result = ''; + let previous: PdfSpan | undefined; + for (const span of spans) { + const gap = previous ? span.x - (previous.x + previous.width) : 0; + const separator = result && !/\s$/.test(result) && !/^\s/.test(span.text) && gap > Math.max(1.2, span.height * 0.12) ? ' ' : ''; + result += separator + span.text; + previous = span; + } + return normalizeInline(result); +} + +export function parseIso27002Controls(pdf: ParsedPdf): ParsedControl[] { + const lines = pdf.pages.flatMap(page => page.lines); + const starts: number[] = []; + for (let index = 0; index < lines.length; index += 1) { + if (!/^([5-8]\.\d{1,2})\s+\S/.test(lines[index].text)) continue; + const nearby = new Set(lines.slice(index + 1, index + 35).map(line => line.text)); + if (nearby.has('Maßnahme') && nearby.has('Zweck') && (nearby.has('Anleitung') || nearby.has('Leitfaden'))) starts.push(index); + } + const uniqueStarts = starts.filter((start, index) => !starts.slice(0, index).some(other => controlId(lines[other].text) === controlId(lines[start].text))); + return uniqueStarts.map((start, index) => parseIso27002Block(lines.slice(start, uniqueStarts[index + 1] ?? lines.length))); +} + +function parseIso27002Block(block: PdfLine[]): ParsedControl { + const match = /^([5-8]\.\d{1,2})\s+(.+)$/.exec(block[0].text); + if (!match) throw new Error('ISO/IEC 27002 control heading is malformed.'); + const labels: Array<[RegulatorySectionKind, readonly string[]]> = [ + ['control', ['Maßnahme']], ['purpose', ['Zweck']], ['guidance', ['Anleitung', 'Leitfaden']], ['other-information', ['Weitere Informationen']] + ]; + const positions = labels.map(([kind, aliases]) => ({ kind, index: block.findIndex(line => aliases.includes(line.text)) })); + if (positions.some(position => position.index < 0)) throw new Error(`ISO/IEC 27002 control ${match[1]} is missing a required section.`); + const attributesAt = block.slice(1, positions[0].index).findIndex(line => /Maßnahmenart/.test(line.text)); + if (attributesAt < 0) throw new Error(`ISO/IEC 27002 control ${match[1]} is missing its attribute table.`); + const titleLines = [match[2], ...block.slice(1, 1 + attributesAt).map(line => line.text)]; + const attributeLines = block.slice(1 + attributesAt, positions[0].index); + const sections: ParsedSection[] = [sectionFromLines('attributes', attributeLines)]; + positions.forEach((position, positionIndex) => { + const next = positions[positionIndex + 1]?.index ?? block.length; + sections.push(sectionFromLines(position.kind, block.slice(position.index + 1, next))); + }); + return { controlId: match[1], title: linesToText(titleLines), theme: themeFor(match[1]), sections }; +} + +export function parseIso27001AnnexA(pdf: ParsedPdf): ParsedControl[] { + const startPageIndex = pdf.pages.findIndex(page => page.lines.some(line => /Tabelle A\.1/.test(line.text)) + && page.lines.flatMap(line => line.spans).filter(span => span.text === 'Maßnahme').length >= 3); + if (startPageIndex < 0) throw new Error('ISO/IEC 27001 Annex A table was not found.'); + const controls: Array<{ controlId: string; pageStart: number; pageEnd: number; title: string[]; statement: string[] }> = []; + let current: (typeof controls)[number] | undefined; + for (const page of pdf.pages.slice(startPageIndex)) { + const statementX = Math.min(...page.lines.flatMap(line => line.spans) + .filter(span => span.text === 'Maßnahme').map(span => span.x)); + if (!Number.isFinite(statementX)) continue; + for (const line of page.lines) { + const idSpan = line.spans.find(span => span.x < 125 && /^[5-8]\.\d{1,2}$/.test(span.text)); + if (idSpan) { + current = { controlId: idSpan.text, pageStart: page.page, pageEnd: page.page, title: [], statement: [] }; + controls.push(current); + } + if (!current) continue; + current.pageEnd = page.page; + for (const span of line.spans) { + if (span === idSpan || span.text === 'Maßnahme' || /Tabelle A\.1|Informationssicherheitsmaßnahmen/.test(span.text)) continue; + if (span.x >= 65 && span.x < statementX) current.title.push(span.text); + else if (span.x >= statementX) current.statement.push(span.text); + } + } + } + return controls.map(control => ({ + controlId: control.controlId, + title: linesToText(control.title), + theme: themeFor(control.controlId), + sections: [{ + kind: 'control', text: linesToText(control.statement), + pageStart: control.pageStart, pageEnd: control.pageEnd + }] + })); +} + +function sectionFromLines(kind: RegulatorySectionKind, lines: PdfLine[]): ParsedSection { + const meaningful = lines.filter(line => !isFurniture(line.text)); + const text = linesToText(meaningful.map(line => line.text)); + if (!text) throw new Error(`ISO/IEC 27002 ${kind} section is empty.`); + return { kind, text, pageStart: meaningful[0].page, pageEnd: meaningful.at(-1)!.page }; +} + +export function validateIsoControlSets(references: ParsedControl[], semantic: ParsedControl[]): void { + const referenceIds = references.map(item => item.controlId); + const semanticIds = semantic.map(item => item.controlId); + for (const [label, ids] of [['ISO/IEC 27001', referenceIds], ['ISO/IEC 27002', semanticIds]] as const) { + if (ids.length !== 93 || new Set(ids).size !== ids.length) { + throw new Error(`${label} must contain exactly 93 unique controls; parsed ${ids.length} entries and ${new Set(ids).size} unique identifiers.`); + } + const counts = Object.fromEntries(Object.keys(EXPECTED_THEME_COUNTS).map(theme => [theme, ids.filter(id => themeFor(id) === theme).length])); + for (const [theme, expected] of Object.entries(EXPECTED_THEME_COUNTS)) { + if (counts[theme] !== expected) throw new Error(`${label} ${theme} control count must be ${expected}.`); + } + } + if (referenceIds.slice().sort().join('\n') !== semanticIds.slice().sort().join('\n')) { + throw new Error('ISO/IEC 27001 Annex A and ISO/IEC 27002 control identifiers do not match.'); + } +} + +function themeFor(controlIdValue: string): RegulatoryControlTheme { + if (controlIdValue.startsWith('5.')) return 'organizational'; + if (controlIdValue.startsWith('6.')) return 'people'; + if (controlIdValue.startsWith('7.')) return 'physical'; + if (controlIdValue.startsWith('8.')) return 'technological'; + throw new Error(`Unsupported ISO control identifier ${controlIdValue}.`); +} + +function controlId(value: string): string | undefined { + return /^([5-8]\.\d{1,2})\s+/.exec(value)?.[1]; +} + +function linesToText(lines: string[]): string { + return lines.map(normalizeInline).filter(Boolean).join('\n') + .replace(/(\p{L})-\n(?=\p{Ll})/gu, '$1') + .replace(/\n+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function normalizeInline(value: string): string { + return value.normalize('NFKC').replace(/\u00ad/g, '').replace(/[\u00a0\u2007\u202f]/g, ' ').replace(/\s+/g, ' ').trim(); +} + +function isFurniture(value: string): boolean { + return /^(?:DIN EN ISO\/IEC 2700[12]:|EN ISO\/IEC 2700[12]:|Normen-Download-|\d{1,3})/.test(value); +} + +function isTextItem(value: unknown): value is PdfJsTextItem { + return Boolean(value) && typeof value === 'object' && typeof (value as PdfJsTextItem).str === 'string' + && Array.isArray((value as PdfJsTextItem).transform) && (value as PdfJsTextItem).transform.length === 6; +} + +function sha256(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} diff --git a/server/regulatory/qdrant-regulatory-index.ts b/server/regulatory/qdrant-regulatory-index.ts index bb71616..dea520e 100644 --- a/server/regulatory/qdrant-regulatory-index.ts +++ b/server/regulatory/qdrant-regulatory-index.ts @@ -5,7 +5,7 @@ import type { RegulatoryRetrievalFilters } from '../../src/domain/regulatory-rag'; -export const REGULATORY_QDRANT_COLLECTION = 'assessor_regulatory_chunks_v1'; +export const REGULATORY_QDRANT_COLLECTION = 'assessor_regulatory_chunks_v2'; export const REGULATORY_DENSE_VECTOR = 'dense'; export const REGULATORY_SPARSE_VECTOR = 'sparse'; @@ -24,6 +24,7 @@ export interface RegulatoryChunkIndex { ensureCollection(): Promise; replaceDocuments(chunks: IndexedRegulatoryChunk[]): Promise; searchDense(vector: number[], limit: number, filters?: RegulatoryRetrievalFilters): Promise; + list(filters: RegulatoryRetrievalFilters, limit: number): Promise; } export class QdrantClientError extends Error { @@ -35,7 +36,7 @@ export class QdrantClientError extends Error { interface QdrantRegulatoryIndexOptions { baseUrl: URL; - apiKey: string; + apiKey?: string; dimensions: number; embeddingModel: string; timeoutMs: number; @@ -51,7 +52,10 @@ const payloadIndexes = [ ['framework_version', 'keyword'], ['control_id', 'keyword'], ['language', 'keyword'], - ['embedding_model', 'keyword'] + ['embedding_model', 'keyword'], + ['source_standard', 'keyword'], + ['section_kind', 'keyword'], + ['retrieval_role', 'keyword'] ] as const; export class QdrantRegulatoryIndex implements RegulatoryChunkIndex { @@ -129,6 +133,19 @@ export class QdrantRegulatoryIndex implements RegulatoryChunkIndex { return points.map(point => parseCandidate(point)); } + async list(filters: RegulatoryRetrievalFilters, limit: number): Promise { + if (!Number.isInteger(limit) || limit < 1 || limit > 100) throw new Error('Regulatory list limit must be between 1 and 100.'); + const response = await this.request('POST', `/collections/${REGULATORY_QDRANT_COLLECTION}/points/scroll`, { + filter: { must: [keywordCondition('embedding_model', this.options.embeddingModel), ...filterConditions(filters)] }, + limit, + with_payload: true, + with_vector: false + }); + const points = object(response.body)?.result?.points; + if (!Array.isArray(points)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant returned an invalid scroll response.'); + return points.map(point => parseChunkPayload(object(point)?.payload)); + } + private async request(method: string, path: string, body?: unknown, acceptedStatuses: number[] = [200]): Promise<{ status: number; body: unknown }> { const url = new URL(path, withTrailingSlash(this.options.baseUrl)); let response: Response; @@ -140,7 +157,7 @@ export class QdrantRegulatoryIndex implements RegulatoryChunkIndex { headers: { Accept: 'application/json', ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), - 'api-key': this.options.apiKey + ...(this.options.apiKey ? { 'api-key': this.options.apiKey } : {}) }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); @@ -167,7 +184,7 @@ export function createQdrantRegulatoryIndexFromEnvironment( ): QdrantRegulatoryIndex { const urlValue = environment.ASSESSOR_QDRANT_URL?.trim(); const apiKey = environment.ASSESSOR_QDRANT_API_KEY?.trim(); - if (!urlValue || !apiKey) throw new Error('Regulatory RAG requires ASSESSOR_QDRANT_URL and ASSESSOR_QDRANT_API_KEY.'); + if (!urlValue) throw new Error('Regulatory RAG requires ASSESSOR_QDRANT_URL.'); let baseUrl: URL; try { baseUrl = new URL(urlValue); } catch { throw new Error('ASSESSOR_QDRANT_URL must be an absolute URL.'); } if (!['http:', 'https:'].includes(baseUrl.protocol) || baseUrl.username || baseUrl.password || baseUrl.search || baseUrl.hash) { @@ -177,6 +194,9 @@ export function createQdrantRegulatoryIndexFromEnvironment( if (baseUrl.protocol !== 'https:' && !['localhost', '127.0.0.1', '[::1]', 'qdrant'].includes(baseUrl.hostname.toLowerCase())) { throw new Error('ASSESSOR_QDRANT_URL must use HTTPS unless it targets loopback or the internal qdrant service.'); } + if (!apiKey && !['localhost', '127.0.0.1', '[::1]', 'qdrant'].includes(baseUrl.hostname.toLowerCase())) { + throw new Error('Remote Qdrant requires ASSESSOR_QDRANT_API_KEY.'); + } return new QdrantRegulatoryIndex({ baseUrl, apiKey, @@ -219,7 +239,18 @@ function chunkPayload(chunk: RegulatoryChunk, embeddingModel: string): Record { const field = payload[name]; if (typeof field !== 'string' || !field) throw new QdrantClientError('qdrant_invalid_response', `Qdrant point is missing ${name}.`); return field; }; - if (payload.schema_version !== 1 || !Number.isInteger(payload.chunk_index)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant point has an unsupported regulatory schema.'); - return { - score, - chunk: { - schemaVersion: 1, + if (![1, 2].includes(Number(payload.schema_version)) || !Number.isInteger(payload.chunk_index)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant point has an unsupported regulatory schema.'); + const chunk: RegulatoryChunk = { + schemaVersion: Number(payload.schema_version) as 1 | 2, corpusId: stringField('corpus_id'), corpusVersion: stringField('corpus_version'), documentId: stringField('document_id'), @@ -250,9 +285,20 @@ function parseCandidate(value: unknown): RegulatoryRetrievalCandidate { title: stringField('title'), text: stringField('text'), textSha256: stringField('text_sha256'), - ...(typeof payload.source_uri === 'string' && payload.source_uri ? { sourceUri: payload.source_uri } : {}) - } + ...(typeof payload.source_uri === 'string' && payload.source_uri ? { sourceUri: payload.source_uri } : {}), + ...(typeof payload.annex_a_control_id === 'string' ? { annexAControlId: payload.annex_a_control_id } : {}), + ...(optionalEnum(payload.control_theme, ['organizational', 'people', 'physical', 'technological'] as const, 'control_theme') ? { controlTheme: payload.control_theme as RegulatoryChunk['controlTheme'] } : {}), + ...(optionalEnum(payload.source_standard, ['ISO/IEC 27001', 'ISO/IEC 27002'] as const, 'source_standard') ? { sourceStandard: payload.source_standard as RegulatoryChunk['sourceStandard'] } : {}), + ...(typeof payload.source_edition === 'string' ? { sourceEdition: payload.source_edition } : {}), + ...(typeof payload.national_edition === 'string' ? { nationalEdition: payload.national_edition } : {}), + ...(optionalEnum(payload.section_kind, ['attributes', 'control', 'purpose', 'guidance', 'other-information'] as const, 'section_kind') ? { sectionKind: payload.section_kind as RegulatoryChunk['sectionKind'] } : {}), + ...(Number.isInteger(payload.source_page_start) ? { sourcePageStart: Number(payload.source_page_start) } : {}), + ...(Number.isInteger(payload.source_page_end) ? { sourcePageEnd: Number(payload.source_page_end) } : {}), + ...(optionalEnum(payload.retrieval_role, ['semantic', 'reference'] as const, 'retrieval_role') ? { retrievalRole: payload.retrieval_role as RegulatoryChunk['retrievalRole'] } : {}), + ...(typeof payload.parser_version === 'string' ? { parserVersion: payload.parser_version } : {}), + ...(typeof payload.source_sha256 === 'string' ? { sourceSha256: payload.source_sha256 } : {}) }; + return chunk; } function filterConditions(filters: RegulatoryRetrievalFilters): Array> { @@ -262,7 +308,10 @@ function filterConditions(filters: RegulatoryRetrievalFilters): Array Boolean(entry[1])).map(([key, value]) => keywordCondition(key, value)); } @@ -321,3 +370,9 @@ async function readBoundedText(response: Response, maximumBytes: number): Promis function object(value: unknown): Record | undefined { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : undefined; } + +function optionalEnum(value: unknown, allowed: T, field: string): T[number] | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'string' || !allowed.includes(value)) throw new QdrantClientError('qdrant_invalid_response', `Qdrant point has invalid ${field}.`); + return value as T[number]; +} diff --git a/server/regulatory/regulatory-chunker.ts b/server/regulatory/regulatory-chunker.ts index 9af1221..0de62d3 100644 --- a/server/regulatory/regulatory-chunker.ts +++ b/server/regulatory/regulatory-chunker.ts @@ -10,7 +10,7 @@ export interface RegulatoryChunkingOptions { } export function validateRegulatoryControlDocument(value: unknown): RegulatoryControlDocument { - if (!isObject(value) || value.schemaVersion !== 1) throw new Error('Regulatory documents must use schemaVersion 1.'); + if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion))) throw new Error('Regulatory documents must use schemaVersion 1 or 2.'); const required = ['corpusId', 'corpusVersion', 'documentId', 'framework', 'frameworkVersion', 'controlId', 'language', 'title', 'text'] as const; const document = value as Record; for (const field of required) { @@ -22,8 +22,9 @@ export function validateRegulatoryControlDocument(value: unknown): RegulatoryCon if (document.sourceUri !== undefined && (typeof document.sourceUri !== 'string' || !document.sourceUri.trim())) { throw new Error('Regulatory document sourceUri must be a non-empty string when present.'); } - return { - schemaVersion: 1, + const schemaVersion = Number(document.schemaVersion) as 1 | 2; + const result: RegulatoryControlDocument = { + schemaVersion, corpusId: String(document.corpusId).trim(), corpusVersion: String(document.corpusVersion).trim(), documentId: String(document.documentId).trim(), @@ -35,6 +36,8 @@ export function validateRegulatoryControlDocument(value: unknown): RegulatoryCon text: normalizeText(String(document.text)), ...(document.sourceUri === undefined ? {} : { sourceUri: String(document.sourceUri).trim() }) }; + if (schemaVersion === 2) Object.assign(result, validateVersionTwoFields(document, result.controlId)); + return result; } export function chunkRegulatoryControl( @@ -63,7 +66,60 @@ export function chunkRegulatoryControl( } export function regulatoryChunkEmbeddingText(chunk: RegulatoryChunk): string { - return `${chunk.framework} ${chunk.frameworkVersion} | ${chunk.controlId} | ${chunk.title}\n\n${chunk.text}`; + const context = [ + `${chunk.framework} ${chunk.frameworkVersion}`, + chunk.annexAControlId ? `${chunk.controlId} (${chunk.annexAControlId})` : chunk.controlId, + chunk.controlTheme, + chunk.sectionKind, + chunk.title + ].filter(Boolean).join(' | '); + return `${context}\n\n${chunk.text}`; +} + +function validateVersionTwoFields(document: Record, controlId: string): Partial { + if (!/^[5-8]\.\d{1,2}$/.test(controlId)) throw new Error('Regulatory schemaVersion 2 controlId must use the canonical 5.x-8.x form.'); + const annexAControlId = requiredEnumLike(document, 'annexAControlId'); + if (annexAControlId !== `A.${controlId}`) throw new Error(`Regulatory annexAControlId must be A.${controlId}.`); + const controlTheme = requiredEnum(document, 'controlTheme', ['organizational', 'people', 'physical', 'technological'] as const); + const sourceStandard = requiredEnum(document, 'sourceStandard', ['ISO/IEC 27001', 'ISO/IEC 27002'] as const); + const sectionKind = requiredEnum(document, 'sectionKind', ['attributes', 'control', 'purpose', 'guidance', 'other-information'] as const); + const retrievalRole = requiredEnum(document, 'retrievalRole', ['semantic', 'reference'] as const); + const sourceEdition = requiredEnumLike(document, 'sourceEdition'); + const nationalEdition = requiredEnumLike(document, 'nationalEdition'); + const parserVersion = requiredEnumLike(document, 'parserVersion'); + const sourceSha256 = requiredEnumLike(document, 'sourceSha256'); + if (!/^[a-f0-9]{64}$/.test(sourceSha256)) throw new Error('Regulatory sourceSha256 must be a lower-case SHA-256 digest.'); + const sourcePageStart = requiredPage(document, 'sourcePageStart'); + const sourcePageEnd = requiredPage(document, 'sourcePageEnd'); + if (sourcePageEnd < sourcePageStart) throw new Error('Regulatory sourcePageEnd must not precede sourcePageStart.'); + if (sourceStandard === 'ISO/IEC 27001' && (sectionKind !== 'control' || retrievalRole !== 'reference')) { + throw new Error('ISO/IEC 27001 schemaVersion 2 documents must be reference control statements.'); + } + if (sourceStandard === 'ISO/IEC 27002' && retrievalRole !== 'semantic') { + throw new Error('ISO/IEC 27002 schemaVersion 2 documents must be semantic evidence.'); + } + return { + annexAControlId, controlTheme, sourceStandard, sourceEdition, nationalEdition, + sectionKind, sourcePageStart, sourcePageEnd, retrievalRole, parserVersion, sourceSha256 + }; +} + +function requiredEnumLike(document: Record, field: string): string { + const value = document[field]; + if (typeof value !== 'string' || !value.trim()) throw new Error(`Regulatory schemaVersion 2 ${field} must be a non-empty string.`); + return value.trim(); +} + +function requiredEnum(document: Record, field: string, values: T): T[number] { + const value = requiredEnumLike(document, field); + if (!values.includes(value)) throw new Error(`Regulatory schemaVersion 2 ${field} is not supported.`); + return value as T[number]; +} + +function requiredPage(document: Record, field: string): number { + const value = document[field]; + if (!Number.isInteger(value) || Number(value) < 1 || Number(value) > 10_000) throw new Error(`Regulatory schemaVersion 2 ${field} must be a positive page number.`); + return Number(value); } function splitText(text: string, maximumCharacters: number): string[] { diff --git a/server/regulatory/regulatory-ingest.ts b/server/regulatory/regulatory-ingest.ts index 647bc1e..26106c4 100644 --- a/server/regulatory/regulatory-ingest.ts +++ b/server/regulatory/regulatory-ingest.ts @@ -21,8 +21,8 @@ async function main(): Promise { } function corpusDocuments(value: unknown): RegulatoryControlDocument[] { - if (!isObject(value) || value.schemaVersion !== 1 || !Array.isArray(value.documents) || !value.documents.length) { - throw new Error('Regulatory corpus must use schemaVersion 1 and contain a non-empty documents array.'); + if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion)) || !Array.isArray(value.documents) || !value.documents.length) { + throw new Error('Regulatory corpus must use schemaVersion 1 or 2 and contain a non-empty documents array.'); } if (value.documents.length > 100_000) throw new Error('Regulatory corpus exceeds the 100000 document limit.'); return value.documents.map(validateRegulatoryControlDocument); diff --git a/server/regulatory/regulatory-mapping-service.ts b/server/regulatory/regulatory-mapping-service.ts new file mode 100644 index 0000000..827f3a8 --- /dev/null +++ b/server/regulatory/regulatory-mapping-service.ts @@ -0,0 +1,331 @@ +import { createHash, randomUUID } from 'node:crypto'; +import type { AiCompletionClient } from '../ai/litellm-client'; +import { LiteLlmClientError } from '../ai/litellm-client'; +import type { AssessorRepository } from '../../src/persistence/assessor-repository'; +import { RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors'; +import { + REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION, + REGULATORY_MAPPING_PROMPT_VERSION, + REGULATORY_RETRIEVAL_VERSION, + groundRegulatoryMappingResult, + regulatoryMappingResultValidationErrors, + type RegulatoryMappingCandidate, + type RegulatoryMappingCreateRequest, + type RegulatoryMappingDecision, + type RegulatoryMappingDecisionCreateRequest, + type RegulatoryMappingEvidence, + type RegulatoryMappingResponse, + type RegulatoryMappingResult, + type RegulatoryMappingRun +} from '../../src/domain/regulatory-mapping'; +import type { RegulatoryRetrievalCandidate } from '../../src/domain/regulatory-rag'; +import type { UnifiedResultProjection } from '../../src/domain/unified-result'; +import { privacySafeFinding } from '../../src/domain/privacy-safe-finding'; +import type { RegulatoryFindingRetrievalService } from './regulatory-finding-retrieval'; +import type { RegulatoryChunkIndex } from './qdrant-regulatory-index'; +import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query'; + +const POLL_AFTER_MS = 1_500; +const MAXIMUM_PROMPT_CHARACTERS = 48_000; + +export interface RegulatoryMappingServiceOptions { + findingRetrieval?: RegulatoryFindingRetrievalService; + index?: RegulatoryChunkIndex; + client?: AiCompletionClient; + corpusId?: string; + corpusVersion?: string; + embeddingModel?: string; + disabledReason?: string; + now?: () => Date; +} + +export class RegulatoryMappingService { + private readonly activeTasks = new Set>(); + private readonly now: () => Date; + + constructor(private readonly repository: AssessorRepository, private readonly options: RegulatoryMappingServiceOptions = {}) { + this.now = options.now ?? (() => new Date()); + } + + get generationEnabled(): boolean { + return Boolean(this.options.findingRetrieval && this.options.index && this.options.client + && this.options.corpusId && this.options.corpusVersion && this.options.embeddingModel); + } + + async recoverInterruptedRuns(): Promise { + const completedAt = this.now().toISOString(); + for (const run of (await this.repository.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) { + await this.repository.saveRegulatoryMappingRun({ + ...run, + status: 'failed', + updatedAt: completedAt, + completedAt, + errorCode: 'regulatory_mapping_interrupted' + }); + } + } + + async get(projectId: string, resultProjectionId: string): Promise { + const finding = await this.finding(projectId, resultProjectionId); + const supported = finding.sourceKind === 'scanner-finding' && Boolean(buildRegulatoryFindingQuery(finding)); + const availability = supported ? await this.availability() : { enabled: false }; + const enabled = availability.enabled && supported; + const ruleKey = supported ? this.ruleKey(finding) : undefined; + const run = enabled && ruleKey ? await this.repository.getRegulatoryMappingRun(this.mappingId(ruleKey)) : null; + const decisions = ruleKey ? await this.repository.listRegulatoryMappingDecisions(ruleKey) : []; + return { + version: 1, + projectId, + resultProjectionId, + generationEnabled: enabled, + state: run?.status ?? (enabled ? 'empty' : 'unsupported'), + ...(!supported ? { disabledReason: finding.sourceKind !== 'scanner-finding' + ? 'Only canonical scanner findings can be mapped in this version.' + : 'This finding has insufficient public context for regulatory mapping.' } + : !availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}), + ...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}), + ...(run ? { run: withoutEvidenceText(run) } : {}), + decisions + }; + } + + async start(projectId: string, request: RegulatoryMappingCreateRequest): Promise { + if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503); + const finding = await this.finding(projectId, request.resultProjectionId); + if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422); + const ruleKey = this.ruleKey(finding); + const id = this.mappingId(ruleKey); + const existing = await this.repository.getRegulatoryMappingRun(id); + if (existing && ['queued', 'running'].includes(existing.status)) return this.get(projectId, request.resultProjectionId); + let retrieval; + try { + retrieval = await this.options.findingRetrieval!.retrieve(finding, 32, { + corpusId: this.options.corpusId, + corpusVersion: this.options.corpusVersion, + sourceStandard: 'ISO/IEC 27002', + retrievalRole: 'semantic' + }); + } catch { + throw new RegulatoryMappingServiceError('regulatory_retrieval_unavailable', 'Regulatory retrieval is currently unavailable.', 503); + } + if (retrieval.status !== 'candidates' || !retrieval.preparedQuery) { + throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422); + } + const candidates = await this.groupCandidates(retrieval.candidates); + if (!candidates.length) throw new RegulatoryMappingServiceError('regulatory_corpus_empty', 'The active regulatory corpus returned no eligible controls.', 503); + const timestamp = this.now().toISOString(); + const queryHash = createHash('sha256').update(JSON.stringify(retrieval.preparedQuery), 'utf8').digest('hex'); + const proposed: RegulatoryMappingRun = { + version: 1, + id, + ruleKey, + sourceTool: finding.sourceTool ?? 'unknown', + ...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}), + status: 'queued', + queryHash, + preparedQuery: retrieval.preparedQuery, + corpusId: this.options.corpusId!, + corpusVersion: this.options.corpusVersion!, + sourceHashes: [...new Set(candidates.flatMap(candidate => candidate.evidence.map(evidence => evidence.sourceSha256)))].sort(), + embeddingModel: this.options.embeddingModel!, + retrievalVersion: REGULATORY_RETRIEVAL_VERSION, + promptVersion: REGULATORY_MAPPING_PROMPT_VERSION, + outputSchemaVersion: REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION, + model: this.options.client!.model, + createdAt: timestamp, + updatedAt: timestamp, + candidates + }; + await this.repository.saveRegulatoryMappingRun(proposed); + const task = this.process(proposed).finally(() => this.activeTasks.delete(task)); + this.activeTasks.add(task); + return this.get(projectId, request.resultProjectionId); + } + + async decide(projectId: string, request: RegulatoryMappingDecisionCreateRequest): Promise { + if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503); + const finding = await this.finding(projectId, request.resultProjectionId); + if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422); + const ruleKey = this.ruleKey(finding); + const comment = request.comment?.trim(); + const decision: RegulatoryMappingDecision = { + version: 1, + id: `regulatory-decision-${randomUUID()}`, + ruleKey, + sourceTool: finding.sourceTool ?? 'unknown', + ...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}), + mappingId: this.mappingId(ruleKey), + controlId: request.controlId, + decision: request.decision, + ...(comment ? { comment } : {}), + createdAt: this.now().toISOString() + }; + await this.repository.createRegulatoryMappingDecision(decision, request.expectedLatestDecisionId); + return this.get(projectId, request.resultProjectionId); + } + + // The global sharing key: the rule identity `tool:ruleId`, shared across every tenant. + // Rules without an id fall back to a stable hash of the non-personal check title so + // identical findings still share, while never keying on tenant or occurrence. + private ruleKey(finding: UnifiedResultProjection): string { + const tool = finding.sourceTool?.trim() || 'unknown'; + const ruleId = finding.sourceRuleId?.trim(); + if (ruleId) return `${tool}:${ruleId}`.slice(0, 200); + const hash = createHash('sha256').update(privacySafeFinding(finding).title.trim(), 'utf8').digest('hex').slice(0, 24); + return `${tool}#title:${hash}`.slice(0, 200); + } + + // Deterministic id per (ruleKey, pipeline versions) so one row exists per rule per + // version tuple: regeneration reuses it, and a version bump yields a new mapping. + private mappingId(ruleKey: string): string { + const tuple = [ruleKey, this.options.corpusVersion, REGULATORY_RETRIEVAL_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, this.options.client!.model].join(''); + return `regulatory-mapping-${createHash('sha256').update(tuple, 'utf8').digest('hex').slice(0, 40)}`; + } + + async close(): Promise { await Promise.allSettled([...this.activeTasks]); } + + private async availability(): Promise<{ enabled: boolean; reason?: string }> { + if (!this.generationEnabled) return { enabled: false, reason: this.options.disabledReason ?? 'Regulatory mapping is not configured.' }; + try { + const sample = await this.options.index!.list({ + corpusId: this.options.corpusId, + corpusVersion: this.options.corpusVersion, + sourceStandard: 'ISO/IEC 27002', + retrievalRole: 'semantic' + }, 1); + return sample.length ? { enabled: true } : { enabled: false, reason: 'The active regulatory corpus is not indexed.' }; + } catch { + return { enabled: false, reason: 'Qdrant or the active regulatory corpus is unavailable.' }; + } + } + + private async finding(projectId: string, resultProjectionId: string) { + if (!resultProjectionId?.trim()) throw new RepositoryValidationError('A resultProjectionId is required.'); + const finding = await this.repository.getUnifiedResultById(resultProjectionId); + if (!finding || finding.projectId !== projectId) throw new RepositoryNotFoundError('The selected finding does not exist in this project.'); + return finding; + } + + private async groupCandidates(hits: RegulatoryRetrievalCandidate[]): Promise { + const byControl = new Map(); + for (const hit of hits) { + if (!hit.chunk.annexAControlId || !hit.chunk.sectionKind || !hit.chunk.sourcePageStart || !hit.chunk.sourcePageEnd || !hit.chunk.sourceSha256) continue; + const values = byControl.get(hit.chunk.controlId) ?? []; + values.push(hit); + byControl.set(hit.chunk.controlId, values); + } + const selected = [...byControl.entries()] + .map(([controlId, values]) => ({ controlId, values: values.sort((left, right) => right.score - left.score), score: Math.max(...values.map(value => value.score)) })) + .sort((left, right) => right.score - left.score || left.controlId.localeCompare(right.controlId, 'en', { numeric: true })) + .slice(0, 8); + return Promise.all(selected.map(async selection => { + const semantic = selection.values.slice(0, 2).map(value => evidence(value.chunk)); + const references = await this.options.index!.list({ + corpusId: this.options.corpusId, + corpusVersion: this.options.corpusVersion, + controlId: selection.controlId, + sourceStandard: 'ISO/IEC 27001', + retrievalRole: 'reference' + }, 1); + if (references.length !== 1 || !references[0].sourceSha256) { + throw new RegulatoryMappingServiceError('regulatory_reference_missing', `The active corpus is missing the ISO 27001 reference for ${selection.controlId}.`, 503); + } + const first = selection.values[0].chunk; + return { + controlId: selection.controlId, + annexAControlId: first.annexAControlId!, + title: first.title, + score: selection.score, + evidence: [...semantic, ...references.map(evidence)] + }; + })); + } + + private async process(queued: RegulatoryMappingRun): Promise { + const startedAt = this.now().toISOString(); + const run = await this.repository.saveRegulatoryMappingRun({ ...queued, status: 'running', startedAt, updatedAt: startedAt }); + try { + const prompt = mappingPrompt(run); + const completion = await this.options.client!.complete(SYSTEM_PROMPT, prompt); + const allowed = new Map(run.candidates.map(candidate => [ + candidate.controlId, new Set(candidate.evidence.map(item => item.chunkId)) + ])); + const grounded = groundRegulatoryMappingResult(completion.value, allowed); + const errors = regulatoryMappingResultValidationErrors(grounded, allowed); + if (errors.length) throw new RegulatoryMappingServiceError('regulatory_mapping_invalid_output', errors.join(' '), 502); + const completedAt = this.now().toISOString(); + await this.repository.saveRegulatoryMappingRun({ + ...run, + status: 'succeeded', + result: grounded as RegulatoryMappingResult, + ...(completion.usage ? { tokenUsage: completion.usage } : {}), + updatedAt: completedAt, + completedAt + }); + } catch (error) { + const completedAt = this.now().toISOString(); + const failure = safeFailure(error); + const detail = safeFailureDetail(error); + try { + await this.repository.saveRegulatoryMappingRun({ + ...run, status: 'failed', updatedAt: completedAt, completedAt, errorCode: failure, + ...(detail ? { errorDetail: detail } : {}) + }); + } catch { /* Preserve the original safe failure code. */ } + } + } +} + +export class RegulatoryMappingServiceError extends Error { + constructor(readonly code: string, message: string, readonly status: number) { super(message); this.name = 'RegulatoryMappingServiceError'; } +} + +// Response projection: the persisted run keeps the full ISO clause text (the model reads it), +// but the copyrighted text is never shipped to the browser — only the citation fields are. +function withoutEvidenceText(run: RegulatoryMappingRun): RegulatoryMappingRun { + return { + ...run, + candidates: run.candidates.map(candidate => ({ + ...candidate, + evidence: candidate.evidence.map(({ text, ...reference }) => reference) + })) + }; +} + +function evidence(chunk: RegulatoryRetrievalCandidate['chunk']): RegulatoryMappingEvidence { + return { + chunkId: chunk.chunkId, + textSha256: chunk.textSha256, + sourceSha256: chunk.sourceSha256!, + sourceStandard: chunk.sourceStandard!, + sectionKind: chunk.sectionKind!, + pageStart: chunk.sourcePageStart!, + pageEnd: chunk.sourcePageEnd!, + text: chunk.text + }; +} + +function mappingPrompt(run: RegulatoryMappingRun): string { + // The Annex A id is always "A." + controlId, so it is redundant for the model and + // sending it invites a reply that uses the Annex A form; expose only the controlId it must return. + const candidates = run.candidates.map(({ annexAControlId, ...candidate }) => candidate); + const payload = JSON.stringify({ finding: run.preparedQuery, candidates }); + if (payload.length > MAXIMUM_PROMPT_CHARACTERS) throw new RegulatoryMappingServiceError('regulatory_mapping_prompt_too_large', 'The bounded regulatory prompt exceeded its limit.', 422); + return payload; +} + +const SYSTEM_PROMPT = `Map one public security finding query to the supplied ISO control candidates. A mapping means the control is topically relevant to the finding, never proof of compliance. Map a candidate when it governs the same security subject as the finding (for example password or authentication management, logging, or access restriction), even when the control text does not state the finding's specific value, threshold, parameter, or configuration; ISO controls are principle-level and rarely restate such specifics. Judge relevance from the control's subject and its supplied evidence, not from whether the exact requirement text appears. Return JSON only with schemaVersion 1, outcome mapped|no_relevant_control|insufficient_context, optional bounded reason, and mappings (maximum five). Each mapping requires controlId, relationship primary|supporting, confidence low|medium|high, a concise rationale under 600 characters, and one to three evidenceChunkIds copied only from that control's supplied evidence. Use no_relevant_control only when no candidate governs the finding's subject, and insufficient_context only when the finding query itself lacks enough meaning to judge. Do not invent controls or evidence.`; + +function safeFailure(error: unknown): string { + if (error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError) return error.code; + return 'regulatory_mapping_failed'; +} + +// Our own service and client error messages are developer-authored and structural +// (field paths, ISO control numbers) — never finding, tenant, or report content. +// Clamp the length and strip any markup-like sequence before it is stored or shown. +function safeFailureDetail(error: unknown): string | undefined { + if (!(error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError)) return undefined; + const detail = error.message.replace(/<\/?[a-z][^>]*>/gi, ' ').replace(/\s+/g, ' ').trim().slice(0, 600); + return detail || undefined; +} diff --git a/server/regulatory/regulatory-prepare-iso.ts b/server/regulatory/regulatory-prepare-iso.ts new file mode 100644 index 0000000..6ec3890 --- /dev/null +++ b/server/regulatory/regulatory-prepare-iso.ts @@ -0,0 +1,25 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { buildIsoPdfCorpus } from './iso-pdf-corpus'; + +async function main(): Promise { + const [iso27001Path, iso27002Path, outputPath, corpusVersion] = process.argv.slice(2); + if (!iso27001Path || !iso27002Path || !outputPath) { + throw new Error('Usage: npm run rag:prepare-iso -- [corpus-version]'); + } + const target = resolve(outputPath); + const corpus = await buildIsoPdfCorpus({ + iso27001Path: resolve(iso27001Path), + iso27002Path: resolve(iso27002Path), + ...(corpusVersion?.trim() ? { corpusVersion: corpusVersion.trim() } : {}) + }); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, `${JSON.stringify(corpus, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' }); + const controlCount = new Set(corpus.documents.map(document => document.controlId)).size; + console.log(`Prepared corpus ${corpus.corpusId}@${corpus.corpusVersion}: ${controlCount} controls, ${corpus.documents.length} section documents.`); +} + +void main().catch(error => { + console.error(error instanceof Error ? error.message : 'ISO corpus preparation failed.'); + process.exitCode = 1; +}); diff --git a/server/regulatory/regulatory-rag-runtime.ts b/server/regulatory/regulatory-rag-runtime.ts index 7da250a..c278916 100644 --- a/server/regulatory/regulatory-rag-runtime.ts +++ b/server/regulatory/regulatory-rag-runtime.ts @@ -30,6 +30,8 @@ export function createRegulatoryRagRuntime( ingestion: RegulatoryIngestionService; retrieval: RegulatoryRetrievalPipeline; findingRetrieval: RegulatoryFindingRetrievalService; + index: ReturnType; + embeddingModel: string; } { const embeddings = createEmbeddingClientFromEnvironment(environment); const index = createQdrantRegulatoryIndexFromEnvironment(embeddings.dimensions, embeddings.model, environment); @@ -41,7 +43,9 @@ export function createRegulatoryRagRuntime( embeddingBatchSize: boundedInteger(environment.ASSESSOR_RAG_EMBEDDING_BATCH_SIZE, 32, 1, 128, 'ASSESSOR_RAG_EMBEDDING_BATCH_SIZE') }), retrieval, - findingRetrieval: new RegulatoryFindingRetrievalService(retrieval) + findingRetrieval: new RegulatoryFindingRetrievalService(retrieval), + index, + embeddingModel: embeddings.model }; } diff --git a/src/api/assessor-api-client.ts b/src/api/assessor-api-client.ts index 4a3dbfe..722428d 100644 --- a/src/api/assessor-api-client.ts +++ b/src/api/assessor-api-client.ts @@ -48,7 +48,10 @@ import { type MicrosoftSecurityBaselineAnalysisListResponse, type CveEnrichmentResponse, type AiOverviewCreateRequest, - type AiOverviewResponse + type AiOverviewResponse, + type RegulatoryMappingCreateRequest, + type RegulatoryMappingDecisionCreateRequest, + type RegulatoryMappingResponse } from './assessor-api-contract'; export class AssessorApiError extends Error { @@ -164,6 +167,22 @@ export class AssessorApiClient { }); } + getRegulatoryMappings(projectId: string, resultProjectionId: string): Promise { + return this.request(`/projects/${encodeURIComponent(projectId)}/regulatory-mappings/${encodeURIComponent(resultProjectionId)}`); + } + + generateRegulatoryMappings(projectId: string, request: RegulatoryMappingCreateRequest): Promise { + return this.request(`/projects/${encodeURIComponent(projectId)}/regulatory-mapping-runs`, { + method: 'POST', body: JSON.stringify(request) + }); + } + + saveRegulatoryMappingDecision(projectId: string, request: RegulatoryMappingDecisionCreateRequest): Promise { + return this.request(`/projects/${encodeURIComponent(projectId)}/regulatory-mapping-decisions`, { + method: 'POST', body: JSON.stringify(request) + }); + } + listProjectImportFiles(projectId: string): Promise { return this.request(`/projects/${encodeURIComponent(projectId)}/import-files`); } diff --git a/src/api/assessor-api-contract.ts b/src/api/assessor-api-contract.ts index c478d92..7558a6d 100644 --- a/src/api/assessor-api-contract.ts +++ b/src/api/assessor-api-contract.ts @@ -17,6 +17,11 @@ import type { PolicyEvaluationRuleCatalog } from '../domain/policy-evaluation-ru import type { PolicyAnalysisOverview } from '../domain/policy-analysis-overview'; import type { CveEnrichmentRecord } from '../domain/cve-enrichment'; export type { AiOverviewCreateRequest, AiOverviewResponse } from '../domain/ai-overview'; +export type { + RegulatoryMappingCreateRequest, + RegulatoryMappingDecisionCreateRequest, + RegulatoryMappingResponse +} from '../domain/regulatory-mapping'; export const ASSESSOR_API_PREFIX = '/api/v1'; export const ASSESSOR_MUTATION_HEADER = 'X-Assessor-Request'; diff --git a/src/domain/privacy-safe-finding.ts b/src/domain/privacy-safe-finding.ts new file mode 100644 index 0000000..8636375 --- /dev/null +++ b/src/domain/privacy-safe-finding.ts @@ -0,0 +1,48 @@ +import type { UnifiedResultProjection } from './unified-result'; + +/** + * The single privacy boundary shared by the AI overview and the ISO control mapping. + * It projects a canonical finding to its rule-level check fields only — title, + * description, recommendation and normalized metadata — and never includes the + * result-table / occurrence data (affectedAsset, source locators, per-object values). + * Definitions are deduplicated per rule (canonicalKey = tool:ruleId), so this text + * describes the check, not the tenant. Email addresses are redacted defensively. + */ +export interface PrivacySafeFinding { + id: string | null; + title: string; + description?: string; + recommendation?: string; + severity: string; + outcome: string; + resultType: string; + isIoe: boolean; + deviation: boolean; + tool: string; +} + +export function privacySafeText(value: string): string { + return value.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, '[redacted-email]'); +} + +export function privacySafeCheckText(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return !trimmed || trimmed.toLocaleLowerCase('en-US') === 'none' ? undefined : privacySafeText(trimmed); +} + +export function privacySafeFinding(item: UnifiedResultProjection): PrivacySafeFinding { + const description = privacySafeCheckText(item.description); + const recommendation = privacySafeCheckText(item.recommendation); + return { + id: item.sourceRuleId ?? null, + title: privacySafeText(item.title), + ...(description ? { description } : {}), + ...(recommendation ? { recommendation } : {}), + severity: item.severityNormalized, + outcome: item.outcomeNormalized, + resultType: item.resultType, + isIoe: item.isIoe, + deviation: item.isDeviation, + tool: item.sourceTool ?? 'unknown' + }; +} diff --git a/src/domain/regulatory-mapping.ts b/src/domain/regulatory-mapping.ts new file mode 100644 index 0000000..c1a0cca --- /dev/null +++ b/src/domain/regulatory-mapping.ts @@ -0,0 +1,239 @@ +import type { AiOverviewTokenUsage } from './ai-overview'; +import type { RegulatoryFindingQuery, RegulatorySectionKind } from './regulatory-rag'; + +export const REGULATORY_MAPPING_PROMPT_VERSION = '1.2.0'; +export const REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION = 1 as const; +export const REGULATORY_RETRIEVAL_VERSION = 'dense-control-grouping-1.1.0'; + +export type RegulatoryMappingRunStatus = 'queued' | 'running' | 'succeeded' | 'failed'; +export type RegulatoryMappingRelationship = 'primary' | 'supporting'; +export type RegulatoryMappingConfidence = 'low' | 'medium' | 'high'; +export type RegulatoryMappingDecisionValue = 'accepted' | 'rejected'; + +export interface RegulatoryMappingEvidence { + chunkId: string; + textSha256: string; + sourceSha256: string; + sourceStandard: 'ISO/IEC 27001' | 'ISO/IEC 27002'; + sectionKind: RegulatorySectionKind; + pageStart: number; + pageEnd: number; + // Full ISO clause text. Required in the persisted run (the model reads it to reason), + // but omitted from the API response projection so the copyrighted text never reaches the browser. + text?: string; +} + +export interface RegulatoryMappingCandidate { + controlId: string; + annexAControlId: string; + title: string; + score: number; + evidence: RegulatoryMappingEvidence[]; +} + +export interface RegulatoryMappingProposal { + controlId: string; + relationship: RegulatoryMappingRelationship; + confidence: RegulatoryMappingConfidence; + rationale: string; + evidenceChunkIds: string[]; +} + +export interface RegulatoryMappingResult { + schemaVersion: 1; + outcome: 'mapped' | 'no_relevant_control' | 'insufficient_context'; + reason?: string; + mappings: RegulatoryMappingProposal[]; +} + +/** + * A global, tenant-independent mapping of one scanner rule to ISO controls. Keyed by + * `ruleKey` (the rule identity `tool:ruleId`, or a stable title hash for rules without + * an id) plus the pipeline versions, so the same rule maps once and every occurrence in + * every project reuses it. Never carries project, tenant, or occurrence identity. + */ +export interface RegulatoryMappingRun { + version: 1; + id: string; + ruleKey: string; + sourceTool: string; + sourceRuleId?: string; + status: RegulatoryMappingRunStatus; + queryHash: string; + preparedQuery: RegulatoryFindingQuery; + corpusId: string; + corpusVersion: string; + sourceHashes: string[]; + embeddingModel: string; + retrievalVersion: string; + promptVersion: string; + outputSchemaVersion: 1; + model: string; + createdAt: string; + updatedAt: string; + startedAt?: string; + completedAt?: string; + tokenUsage?: AiOverviewTokenUsage; + errorCode?: string; + errorDetail?: string; + candidates: RegulatoryMappingCandidate[]; + result?: RegulatoryMappingResult; +} + +/** A global accept/reject decision, keyed by rule identity and control. Shared across all projects. */ +export interface RegulatoryMappingDecision { + version: 1; + id: string; + ruleKey: string; + sourceTool: string; + sourceRuleId?: string; + mappingId: string; + controlId: string; + decision: RegulatoryMappingDecisionValue; + comment?: string; + previousDecisionId?: string; + createdAt: string; +} + +export interface RegulatoryMappingResponse { + version: 1; + projectId: string; + resultProjectionId: string; + generationEnabled: boolean; + state: 'empty' | RegulatoryMappingRunStatus | 'unsupported'; + disabledReason?: string; + pollAfterMs?: number; + run?: RegulatoryMappingRun; + decisions: RegulatoryMappingDecision[]; +} + +export interface RegulatoryMappingCreateRequest { + resultProjectionId: string; +} + +export interface RegulatoryMappingDecisionCreateRequest { + resultProjectionId: string; + controlId: string; + decision: RegulatoryMappingDecisionValue; + comment?: string; + expectedLatestDecisionId?: string; +} + +/** + * Trims each mapping's evidence citations to the exact chunk ids supplied for that + * control, tolerating a model that miscopies or mis-scopes one of several long + * compound chunk ids. This only removes ungrounded ids — it never invents evidence — + * and drops a mapping that, for a known control, cited no real evidence at all. + * Malformed or unknown-control mappings are left untouched for the validator to report. + */ +export function groundRegulatoryMappingResult( + value: unknown, + allowedCandidates: ReadonlyMap> +): unknown { + if (!isObject(value) || !Array.isArray(value.mappings)) return value; + const mappings: unknown[] = []; + for (const mapping of value.mappings) { + const allowed = isObject(mapping) && typeof mapping.controlId === 'string' ? allowedCandidates.get(mapping.controlId) : undefined; + if (!allowed || !Array.isArray((mapping as Record).evidenceChunkIds)) { mappings.push(mapping); continue; } + const grounded = [...new Set(((mapping as Record).evidenceChunkIds as unknown[]) + .filter((id): id is string => typeof id === 'string' && allowed.has(id)))].slice(0, 3); + if (grounded.length) mappings.push({ ...(mapping as Record), evidenceChunkIds: grounded }); + } + return { ...value, mappings }; +} + +export function regulatoryMappingResultValidationErrors( + value: unknown, + allowedCandidates?: ReadonlyMap> +): string[] { + const errors: string[] = []; + if (!isObject(value)) return ['Regulatory mapping output must be a JSON object.']; + if (value.schemaVersion !== 1) errors.push('schemaVersion must be 1.'); + if (!['mapped', 'no_relevant_control', 'insufficient_context'].includes(String(value.outcome))) errors.push('outcome is invalid.'); + if (!Array.isArray(value.mappings) || value.mappings.length > 5) errors.push('mappings must contain at most five entries.'); + else { + const controls = new Set(); + value.mappings.forEach((mapping, index) => { + if (!isObject(mapping)) { errors.push(`mappings[${index}] must be an object.`); return; } + const controlId = boundedString(mapping.controlId, 20, `mappings[${index}].controlId`, errors); + enumeration(mapping.relationship, ['primary', 'supporting'], `mappings[${index}].relationship`, errors); + enumeration(mapping.confidence, ['low', 'medium', 'high'], `mappings[${index}].confidence`, errors); + boundedString(mapping.rationale, 600, `mappings[${index}].rationale`, errors); + if (controlId && controls.has(controlId)) errors.push(`mappings[${index}] duplicates control ${controlId}.`); + if (controlId) controls.add(controlId); + const allowedEvidence = controlId ? allowedCandidates?.get(controlId) : undefined; + if (!Array.isArray(mapping.evidenceChunkIds) || mapping.evidenceChunkIds.length < 1 || mapping.evidenceChunkIds.length > 3) { + errors.push(`mappings[${index}].evidenceChunkIds must contain one to three entries.`); + } else mapping.evidenceChunkIds.forEach((chunkId, evidenceIndex) => { + if (typeof chunkId !== 'string' || !allowedEvidence?.has(chunkId)) errors.push(`mappings[${index}].evidenceChunkIds[${evidenceIndex}] is not grounded in the supplied control.`); + }); + if (controlId && allowedCandidates && !allowedCandidates.has(controlId)) errors.push(`mappings[${index}] references an unknown candidate control.`); + }); + } + if (value.outcome === 'mapped' && (!Array.isArray(value.mappings) || !value.mappings.length)) errors.push('mapped outcome requires at least one mapping.'); + if (value.outcome !== 'mapped' && Array.isArray(value.mappings) && value.mappings.length) errors.push('A refusal outcome must not include mappings.'); + if (value.outcome !== 'mapped') boundedString(value.reason, 600, 'reason', errors); + return errors; +} + +export function regulatoryMappingRunValidationErrors(run: RegulatoryMappingRun): string[] { + const errors: string[] = []; + for (const value of [run.id, run.ruleKey, run.sourceTool, run.corpusId, run.corpusVersion, run.embeddingModel, run.model]) { + if (!value?.trim()) errors.push('Regulatory mapping run identity is incomplete.'); + } + if (run.ruleKey && run.ruleKey.length > 200) errors.push('Regulatory mapping ruleKey is too long.'); + if (run.sourceRuleId !== undefined && (!run.sourceRuleId.trim() || run.sourceRuleId.length > 200)) errors.push('Regulatory mapping sourceRuleId is invalid.'); + if (!['queued', 'running', 'succeeded', 'failed'].includes(run.status)) errors.push('Regulatory mapping run status is invalid.'); + if (!/^[a-f0-9]{64}$/.test(run.queryHash)) errors.push('Regulatory mapping queryHash must be SHA-256.'); + if (!run.sourceHashes.length || run.sourceHashes.some(value => !/^[a-f0-9]{64}$/.test(value))) errors.push('Regulatory mapping sourceHashes must contain SHA-256 digests.'); + if (!Array.isArray(run.candidates) || run.candidates.length < 1 || run.candidates.length > 8) errors.push('Regulatory mapping candidates must contain one to eight controls.'); + else { + const controls = new Set(); + run.candidates.forEach(candidate => { + if (!/^[5-8]\.\d{1,2}$/.test(candidate.controlId) || candidate.annexAControlId !== `A.${candidate.controlId}`) errors.push('Regulatory mapping candidate control identity is invalid.'); + if (controls.has(candidate.controlId)) errors.push(`Regulatory mapping candidate ${candidate.controlId} is duplicated.`); + controls.add(candidate.controlId); + if (!candidate.title.trim() || candidate.title.length > 500 || !Number.isFinite(candidate.score)) errors.push(`Regulatory mapping candidate ${candidate.controlId} metadata is invalid.`); + if (!candidate.evidence.length || candidate.evidence.length > 3) errors.push(`Regulatory mapping candidate ${candidate.controlId} must contain one to three evidence chunks.`); + candidate.evidence.forEach(evidence => { + if (!evidence.chunkId.trim() || !/^[a-f0-9]{64}$/.test(evidence.textSha256) || !/^[a-f0-9]{64}$/.test(evidence.sourceSha256) + || !Number.isInteger(evidence.pageStart) || !Number.isInteger(evidence.pageEnd) || evidence.pageStart < 1 || evidence.pageEnd < evidence.pageStart + || typeof evidence.text !== 'string' || !evidence.text.trim() || evidence.text.length > 20_000) errors.push(`Regulatory mapping candidate ${candidate.controlId} evidence is invalid.`); + }); + }); + } + if (run.status === 'succeeded' && !run.result) errors.push('A succeeded regulatory mapping run requires a result.'); + if (run.result) errors.push(...regulatoryMappingResultValidationErrors(run.result, candidateEvidence(run.candidates))); + return [...new Set(errors)]; +} + +export function regulatoryMappingDecisionValidationErrors(decision: RegulatoryMappingDecision): string[] { + const errors: string[] = []; + for (const value of [decision.id, decision.ruleKey, decision.sourceTool, decision.mappingId, decision.controlId]) { + if (!value?.trim()) errors.push('Regulatory mapping decision identity is incomplete.'); + } + if (decision.sourceRuleId !== undefined && (!decision.sourceRuleId.trim() || decision.sourceRuleId.length > 200)) errors.push('Regulatory mapping decision sourceRuleId is invalid.'); + if (!['accepted', 'rejected'].includes(decision.decision)) errors.push('Regulatory mapping decision is invalid.'); + if (decision.comment !== undefined && (decision.comment.length > 1_000 || /<\/?[a-z][^>]*>/i.test(decision.comment))) errors.push('Regulatory mapping decision comment must be plain text with at most 1000 characters.'); + return [...new Set(errors)]; +} + +function candidateEvidence(candidates: RegulatoryMappingCandidate[]): Map> { + return new Map(candidates.map(candidate => [candidate.controlId, new Set(candidate.evidence.map(evidence => evidence.chunkId))])); +} + +function boundedString(value: unknown, maximum: number, path: string, errors: string[]): string | undefined { + if (typeof value !== 'string' || !value.trim() || value.length > maximum || /<\/?[a-z][^>]*>/i.test(value)) { + errors.push(`${path} must be non-empty plain text with at most ${maximum} characters.`); + return undefined; + } + return value; +} + +function enumeration(value: unknown, allowed: readonly string[], path: string, errors: string[]): void { + if (typeof value !== 'string' || !allowed.includes(value)) errors.push(`${path} is invalid.`); +} + +function isObject(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} diff --git a/src/domain/regulatory-rag.ts b/src/domain/regulatory-rag.ts index 0437436..5047a9d 100644 --- a/src/domain/regulatory-rag.ts +++ b/src/domain/regulatory-rag.ts @@ -1,8 +1,13 @@ -export const REGULATORY_CHUNK_SCHEMA_VERSION = 1 as const; +export const REGULATORY_CHUNK_SCHEMA_VERSION = 2 as const; export const REGULATORY_FINDING_QUERY_SCHEMA_VERSION = 1 as const; +export type RegulatoryDocumentSchemaVersion = 1 | 2; +export type RegulatoryControlTheme = 'organizational' | 'people' | 'physical' | 'technological'; +export type RegulatorySectionKind = 'attributes' | 'control' | 'purpose' | 'guidance' | 'other-information'; +export type RegulatoryRetrievalRole = 'semantic' | 'reference'; + export interface RegulatoryControlDocument { - schemaVersion: 1; + schemaVersion: RegulatoryDocumentSchemaVersion; corpusId: string; corpusVersion: string; documentId: string; @@ -13,10 +18,40 @@ export interface RegulatoryControlDocument { title: string; text: string; sourceUri?: string; + annexAControlId?: string; + controlTheme?: RegulatoryControlTheme; + sourceStandard?: 'ISO/IEC 27001' | 'ISO/IEC 27002'; + sourceEdition?: string; + nationalEdition?: string; + sectionKind?: RegulatorySectionKind; + sourcePageStart?: number; + sourcePageEnd?: number; + retrievalRole?: RegulatoryRetrievalRole; + parserVersion?: string; + sourceSha256?: string; +} + +export interface RegulatoryCorpusSource { + standard: 'ISO/IEC 27001' | 'ISO/IEC 27002'; + sourceEdition: string; + nationalEdition: string; + sha256: string; + pageCount: number; +} + +export interface RegulatoryCorpusV2 { + schemaVersion: 2; + corpusId: string; + corpusVersion: string; + language: string; + parserVersion: string; + generatedAt: string; + sources: RegulatoryCorpusSource[]; + documents: RegulatoryControlDocument[]; } export interface RegulatoryChunk { - schemaVersion: 1; + schemaVersion: RegulatoryDocumentSchemaVersion; corpusId: string; corpusVersion: string; documentId: string; @@ -30,6 +65,17 @@ export interface RegulatoryChunk { text: string; textSha256: string; sourceUri?: string; + annexAControlId?: string; + controlTheme?: RegulatoryControlTheme; + sourceStandard?: 'ISO/IEC 27001' | 'ISO/IEC 27002'; + sourceEdition?: string; + nationalEdition?: string; + sectionKind?: RegulatorySectionKind; + sourcePageStart?: number; + sourcePageEnd?: number; + retrievalRole?: RegulatoryRetrievalRole; + parserVersion?: string; + sourceSha256?: string; } export interface RegulatoryRetrievalFilters { @@ -39,6 +85,9 @@ export interface RegulatoryRetrievalFilters { frameworkVersion?: string; controlId?: string; language?: string; + sourceStandard?: string; + sectionKind?: RegulatorySectionKind; + retrievalRole?: RegulatoryRetrievalRole; } export interface RegulatoryRetrievalRequest { diff --git a/src/exports/result-db-copy.ts b/src/exports/result-db-copy.ts index 48d00a1..74a1416 100644 --- a/src/exports/result-db-copy.ts +++ b/src/exports/result-db-copy.ts @@ -1,4 +1,5 @@ import type { UnifiedResultProjection } from '../domain/unified-result'; +import { privacySafeFinding, privacySafeText } from '../domain/privacy-safe-finding'; const SAFE_ENVIRONMENT_PROPERTIES = new Set([ 'domain.controller_count', @@ -62,22 +63,7 @@ export function resultsToPrivacySafeExportJson(items: UnifiedResultProjection[]) findingTools: countBy(findings, item => item.sourceTool ?? 'unknown'), environmentStates: countBy(environment, item => item.environmentState ?? 'unknown') }, - findings: actionable.map(item => { - const description = copiedText(item.description); - const recommendation = copiedText(item.recommendation); - return { - id: item.sourceRuleId ?? null, - title: privacySafeText(item.title), - ...(description ? { description } : {}), - ...(recommendation ? { recommendation } : {}), - severity: item.severityNormalized, - outcome: item.outcomeNormalized, - resultType: item.resultType, - isIoe: item.isIoe, - deviation: item.isDeviation, - tool: item.sourceTool ?? 'unknown' - }; - }), + findings: actionable.map(privacySafeFinding), notRunChecks: notRun.map(item => ({ id: item.sourceRuleId ?? null, title: privacySafeText(item.title), @@ -122,15 +108,6 @@ function countBy(items: UnifiedResultProjection[], valueOf: (item: UnifiedResult return counts; } -function copiedText(value: string | undefined): string | undefined { - const trimmed = value?.trim(); - return !trimmed || trimmed.toLocaleLowerCase('en-US') === 'none' ? undefined : privacySafeText(trimmed); -} - -function privacySafeText(value: string): string { - return value.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, '[redacted-email]'); -} - function copiedEnvironmentProperty(item: UnifiedResultProjection): string | undefined { if (item.sourceKind !== 'environment-observation' || !item.environmentProperty) return undefined; if (SAFE_ENVIRONMENT_PROPERTIES.has(item.environmentProperty) || item.environmentProperty.endsWith('.row_count')) { diff --git a/src/features/dataset-viewer/DatasetViewer.tsx b/src/features/dataset-viewer/DatasetViewer.tsx index 58bdcc4..e5829c7 100644 --- a/src/features/dataset-viewer/DatasetViewer.tsx +++ b/src/features/dataset-viewer/DatasetViewer.tsx @@ -359,6 +359,8 @@ export function DatasetViewer() { onResults={hasResults(selected) ? () => openResults(selected) : undefined} resultsOpen={Boolean(resultColor(selected.id))} resultColor={resultColor(selected.id)} + projectId={app.projectId} + backendStatus={app.backendStatus} /> ) : view === 'table' ? ( diff --git a/src/features/finding-reader/FindingReader.tsx b/src/features/finding-reader/FindingReader.tsx index 2a3e1e2..8630436 100644 --- a/src/features/finding-reader/FindingReader.tsx +++ b/src/features/finding-reader/FindingReader.tsx @@ -19,6 +19,9 @@ import { formatPolicyComparisonValue, policyValueName } from '../../profiles/gro import { policyFindingCategories } from '../../profiles/group-policy/categories'; import { findingWebSearch } from '../../profiles/public-search/search'; import { policyEvidenceResultSheets, POLICY_CONFLICT_DETAILS, POLICY_DUPLICATE_DETAILS } from '../../profiles/group-policy/conflict-presentation'; +import type { BackendStatus } from '../../app/app-state'; +import { RegulatoryMappingPanel } from './RegulatoryMappingPanel'; +import { useRegulatoryMapping } from './use-regulatory-mapping'; interface Props { record: FindingRecord; @@ -39,6 +42,8 @@ interface Props { resultsOpen?: boolean; /** Farbe des geöffneten Result-Fensters; entspricht der Kachelmarkierung. */ resultColor?: string; + projectId?: string; + backendStatus?: BackendStatus; } const statuses = ['Open', 'In Review', 'Clarification Needed', 'Actionable', 'Blocked', 'Done', 'Not Relevant']; @@ -273,7 +278,7 @@ export function selectFindingFields(record: FindingRecord, title: string, hasStr return { statusField, badges, derivedCategories, mitre, summary, checkTitle, optionalFields, remainingFields, policyComparisonFields }; } -export function FindingReader({ record, title, review: storedReview, linkedSheets, resultReferences, search, position, recordCount, toolId, onPrevious, onNext, onReview, onResults, resultsOpen, resultColor }: Props) { +export function FindingReader({ record, title, review: storedReview, linkedSheets, resultReferences, search, position, recordCount, toolId, onPrevious, onNext, onReview, onResults, resultsOpen, resultColor, projectId = '', backendStatus = 'unavailable' }: Props) { const resultButtonStyle = resultsOpen && resultColor ? ({ '--result-color': resultColor } as CSSProperties) : undefined; const [openLinkedResultsFor, setOpenLinkedResultsFor] = useState(''); const linkedResultsOpen = openLinkedResultsFor === record.id; @@ -292,6 +297,7 @@ export function FindingReader({ record, title, review: storedReview, linkedSheet // sonst ihren gesamten Inhalt ein zweites Mal als "Referenz" dort stehen. const fileReferences = [...new Set(resultReferences.filter(isFileReference))]; const updateText = (field: keyof Review) => (event: ChangeEvent) => onReview({ [field]: event.target.value }); + const regulatoryMapping = useRegulatoryMapping(projectId, record.findingOccurrenceId, backendStatus); return (
@@ -327,11 +333,14 @@ export function FindingReader({ record, title, review: storedReview, linkedSheet
{webSearch && AI search} - Bookmark - + {/* Gleiche Kennzeichnung wie auf den Kacheln: ist das Ergebnisfenster zu diesem Fund offen, traegt die Schaltflaeche den Fensterton. */} + Bookmark +
@@ -359,6 +368,8 @@ export function FindingReader({ record, title, review: storedReview, linkedSheet )}
+ {regulatoryMapping.open && void regulatoryMapping.start()} onDecision={(candidate, decision, comment) => void regulatoryMapping.decide(candidate, decision, comment)} />}
{([['prerequisites', 'Prerequisites'], ['access', 'Required Access / Roles'], ['nextStep', 'Next Step'], ['evidence', 'Evidence / Acceptance Criteria'], ['comment', 'Comment']] as Array<[keyof Review, string]>).map(([field, label]) => (