feat: add regulatory mapping panel and hooks for managing regulatory mappings
This commit is contained in:
parent
82266befeb
commit
e31a19771e
|
|
@ -10,7 +10,7 @@ ASSESSOR_GPO_JOB_LEASE_MS=60000
|
|||
ASSESSOR_GPO_JOB_RETRY_MS=5000
|
||||
ASSESSOR_GPO_JOB_POLL_MS=1000
|
||||
|
||||
# Qdrant is a rebuildable regulatory retrieval index. This key is local/deployment secret material.
|
||||
# Qdrant is a rebuildable regulatory retrieval index. A key is optional for an unsecured loopback/internal self-host, but required for remote hosts.
|
||||
ASSESSOR_QDRANT_URL=http://127.0.0.1:6333
|
||||
ASSESSOR_QDRANT_PORT=6333
|
||||
ASSESSOR_QDRANT_API_KEY=replace-with-a-long-random-local-qdrant-key
|
||||
|
|
@ -26,6 +26,9 @@ ASSESSOR_QDRANT_API_KEY=replace-with-a-long-random-local-qdrant-key
|
|||
# ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS=60000
|
||||
# ASSESSOR_RAG_CHUNK_CHARACTERS=1600
|
||||
# ASSESSOR_RAG_EMBEDDING_BATCH_SIZE=32
|
||||
# The mapping API is enabled only for this explicit, already-ingested corpus.
|
||||
# ASSESSOR_RAG_ACTIVE_CORPUS_ID=iso-27001-27002-de
|
||||
# ASSESSOR_RAG_ACTIVE_CORPUS_VERSION=din-2024-de-example
|
||||
|
||||
# Optional work LiteLLM AI Overview. Leave ASSESSOR_AI_MODE unset to keep outbound AI disabled.
|
||||
# Store real values only in the ignored .env.postgres or deployment environment secret.
|
||||
|
|
|
|||
|
|
@ -22,3 +22,5 @@ vite.config.d.ts
|
|||
|
||||
# Lokale Download-Pakete von Reportentwuerfen (Inhalt liegt als HTML + reports/assets/ vor)
|
||||
reports/*.zip
|
||||
/ISO27001 2024.PDF
|
||||
/ISO27002 2024.PDF
|
||||
|
|
|
|||
|
|
@ -69,6 +69,8 @@ Referenzerkennung 2026-09-26 (Phase 13D, erster Umfang umgesetzt und verifiziert
|
|||
|
||||
Status: lebendes Projektdokument
|
||||
|
||||
ISO-27002-Mapping-Vertikalschnitt 2026-09-15 (Phase 13F, ausdruecklicher Nutzerauftrag): Die beiden lokal lizenzierten deutschen DIN-2024-PDFs werden serverseitig mit PDF.js und Positionsdaten in einen nicht eingecheckten Korpus-v2 ueberfuehrt. Passwortpflichtige, kopiergeschuetzte, textlose, uebergrosse oder strukturell unvollstaendige Dokumente scheitern geschlossen; ISO 27001 und 27002 muessen dieselben 93 Kontrollen mit 37/8/14/34 Themenkontrollen liefern. `assessor_regulatory_chunks_v2` enthaelt 705 mit lokalem BGE-M3 erzeugte, abschnitts- und kontrollsichere Chunks aus 558 Dokumenten, einschliesslich ISO-27001-Referenzen, Seiten-, Editions-, Parser- und Hashprovenienz; Sparse bleibt reserviert. Der Einzel-Finding-Dienst gruppiert 32 Dense-Treffer auf acht Kontrollen, bindet ISO-27001-Evidenz an, uebergibt nur die minimierte Public-Signal-Query und hoechstens 48.000 Zeichen ISO-Evidenz an das vorhandene Work-LiteLLM und akzeptiert ausschliesslich lokal validierte, evidenzgebundene JSON-Vorschlaege. Migration 025, beide Repositories und drei projektgebundene APIs speichern Runs und Accept/Reject-Entscheidungsereignisse append-only; Work zeigt den manuellen Ablauf inline mit Polling und Seitenbelegen. Ein realer synthetischer End-to-End-Test ueber kanonisches PostgreSQL, llama.cpp/BGE-M3, Qdrant und den freigegebenen LiteLLM-Gateway bestand einschliesslich Optimistic-Conflict und Cascade-Cleanup. Der laufende Entwicklungsserver wendete die additive Migration um 13:42 vor dem um 13:52 erstellten und verifizierten Backup automatisch an; beide neuen Tabellen waren leer. Diese Abweichung ist offen dokumentiert und wird nicht als Vorabbackup ausgegeben. Scannerverarbeitung blieb unveraendert; kein Manifestbump oder Reimport.
|
||||
|
||||
Regulatory-RAG-Llama-Livebaseline 2026-09-15 (Phase 13F, Nutzerstart): Der lokal gestartete llama.cpp-Server Build 10472 meldet Health 200 und bedient BGE-M3 FP16 ueber den OpenAI-kompatiblen Endpoint. Die produktiv gebauten CLIs indexierten den eingecheckten ausschliesslich synthetischen EN/DE-Korpus mit sechs Dokumenten als sechs 1.024-dimensionale Chunks in den kanonischen Qdrant 1.19.1. Drei ausdruecklich sprachuebergreifende Smoke-Cases erreichten bei K=3 HitRate 1,0, Recall 1,0 und MRR 1,0; alle erwarteten Kontrollen standen auf Rang 1. Das belegt End-to-End-Verkabelung und grundlegende Sprachueberquerung, wegen nur drei klar getrennten Kontrollthemen aber keine produktive Retrievalqualitaet. Qdrant enthaelt sechs synthetische Punkte; PostgreSQL und Realdaten wurden nicht beruehrt. 1.285 Standardtests bestanden, acht optionale PostgreSQL- und ein optionaler Qdrant-Test uebersprungen; Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich (bestehender Bundlehinweis). Ein repraesentativer autorisierter Korpus und unabhaengig gelabelter EN/DE-Satz bleiben fuer die reale Baseline erforderlich.
|
||||
|
||||
Regulatory-RAG-Local-Embeddings 2026-09-15 (Phase 13F, Nutzerentscheidung): Die autorisierte Legacy-Collection `regulation_catalog_v1` wurde nach exakter Ein-Collection-Pruefung geloescht; der leere alte Qdrant-1.12.4-Container wurde entfernt, sein bisheriges Volume jedoch bewahrt. Der authentifizierte `assessor-2_0-qdrant-1` mit Qdrant 1.19.1 laeuft nun am kanonischen `127.0.0.1:6333`; Auth 401/200, gruene Collection und null Punkte sind bestaetigt. Fuer die lokale Dense-Entwicklung ist BGE-M3 mit llama.cpp festgelegt: multilingual, native 1.024 Dimensionen und damit passend zur Collection. Der Embeddingadapter erlaubt HTTP nur fuer Loopback, verlangt weiterhin einen Bearer-Key und lehnt Remote-HTTP ab. Ein neuer optionaler Request-Dimensionsschalter bleibt fuer einen spaeteren LiteLLM-Vergleich mit `text-embedding-3-large` standardmaessig aus, weil llama.cpp die native Modelldimension liefert. Das installierte llama.cpp Build 10472 ist vorhanden; ein BGE-M3-GGUF beziehungsweise dessen freigegebener Download und Serverstart fehlen noch. 1.285 Standardtests bestanden; acht optionale PostgreSQL- und ein optionaler Qdrant-Test uebersprungen, Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich (bestehender Bundlehinweis). Keine PostgreSQL-Mutation, keine Realdateninspektion und kein Reimport.
|
||||
|
|
@ -2148,14 +2150,14 @@ Status: in Arbeit
|
|||
Ziele:
|
||||
|
||||
- Regulatorische Kontrolltexte versioniert und reproduzierbar fuer mehrsprachige semantische Suche aufbereiten.
|
||||
- Qdrant ausschliesslich als rebuildbaren Retrievalindex verwenden; PostgreSQL bleibt fachliche Quelle fuer Scans, Findings, Ergebnisse, Reviews und spaetere Mappingentscheidungen.
|
||||
- Qdrant ausschliesslich als rebuildbaren Retrievalindex verwenden; PostgreSQL bleibt fachliche Quelle fuer Scans, Findings, Ergebnisse, Reviews, Mappinglaeufe und Entscheidungen.
|
||||
- Retrieval, Embedding, Chunking, spaetere Hybridfusion, Reranking und Mapping als getrennte Backendgrenzen halten.
|
||||
|
||||
Umgesetzter Grundstand vom 2026-09-15:
|
||||
|
||||
- Lokaler und Deployment-Compose-Stack enthalten das gepinnte Qdrant-Image 1.19.1, einen API-Key, deaktivierte ausgehende Nutzungstelemetrie und ein getrenntes persistentes Volume. Lokal ist nur die REST-Schnittstelle auf Loopback veroeffentlicht; im Deployment bleibt Qdrant intern.
|
||||
- `RegulatoryControlDocument` und `RegulatoryChunk` definieren einen versionierten, framework-/versions-/control-/chunk-/sprachgebundenen Vertrag. Chunk- und Inhaltsidentitaeten sind deterministisch; Originalkorpus und Korpusversion bleiben die Wiederaufbauquelle.
|
||||
- Die Collection `assessor_regulatory_chunks_v1` verwendet benannte Dense-/Sparse-Vektoren, Cosine fuer Dense und Payloadindizes fuer alle Filterfelder. Sparse ist nur die vorbereitete Schema- und Schnittstellenposition, noch kein aktiver Retriever.
|
||||
- Die aktive Collection `assessor_regulatory_chunks_v2` verwendet benannte Dense-/Sparse-Vektoren, Cosine fuer Dense und zwoelf Payloadindizes fuer alle Filterfelder. Sparse ist nur die vorbereitete Schema- und Schnittstellenposition, noch kein aktiver Retriever.
|
||||
- Ein abgesicherter OpenAI-kompatibler Embeddingadapter verlangt fuer Remoteziele HTTPS, erlaubt HTTP nur auf Loopback fuer den lokalen llama.cpp-Testbetrieb und bindet ein ausdruecklich freigegebenes mehrsprachiges Modell an dessen exakte Dimension. Redirects, unbeschraenkte Antworten und Dimensionsdrift werden abgewiesen. Ein opt-in `dimensions`-Request unterstuetzt spaetere OpenAI-/LiteLLM-Vergleiche, bleibt fuer llama.cpp aus.
|
||||
- CLI-Ingestion validiert und segmentiert einen begrenzten JSON-Korpus, bettet Chunks in Batches ein und ersetzt dokumentgebundene Qdrant-Punkte. Die Queryseite verwendet dasselbe Modell und kann Korpus, Framework, Version, Control und Sprache exakt filtern.
|
||||
- `RegulatoryRetriever`, `RegulatoryCandidateRetriever` und `RegulatoryReranker` trennen Verbraucher, Candidate Retrieval und optionale Nachsortierung. Spaetere Hybridfusion kann die Candidate-Stufe ersetzen, ohne Mappingcode an Qdrant zu binden.
|
||||
|
|
@ -2164,15 +2166,17 @@ Umgesetzter Grundstand vom 2026-09-15:
|
|||
- Ein eingecheckter synthetischer EN/DE-Smoke-Korpus und drei sprachuebergreifende Labelcases sichern den ausfuehrbaren Vertrag. Der opt-in WSL-Qdrant-Integrationstest prueft mit deterministischen synthetischen Vektoren den realen 1.19.1-Collection-/Index-/Replace-/Filter-/Retrievalpfad; er bewertet ausdruecklich keine Embeddingqualitaet.
|
||||
- Der produktiv gebaute End-to-End-Pfad ist mit lokalem llama.cpp/BGE-M3 FP16 live gelaufen: sechs Dokumente/sechs Chunks; drei Cross-Language-Smoke-Cases bei K=3 jeweils Rang 1 und damit HitRate, Recall und MRR 1,0. Der Mini-Korpus ist nur ein Wiring-Smoke-Test, kein belastbarer Modellvergleich.
|
||||
- Der Grundstand ist in `docs/REGULATORY_RAG.md` dokumentiert und mit synthetischen Vertrags-, Client-, Ingestion-, Retrieval- und Deploymenttests abgesichert.
|
||||
- Der PDF.js-Importer erzeugt aus den explizit uebergebenen ISO-27001-/27002-Pfaden einen lokalen Korpus-v2 mit Quellhashes, Ausgaben, 93 geteilten Kontrollidentitaeten, Abschnittsarten und Seitenbereichen. Nur ISO-27002-Abschnitte werden semantisch gerankt; die entsprechende ISO-27001-Aussage wird als autoritative Referenzevidenz angehaengt.
|
||||
- Ein manueller Work-Vertikalschnitt startet pro kanonischem Scanner-Finding einen idempotenten asynchronen Lauf, gruppiert die 32 Dense-Chunks auf acht Controls, validiert maximal fuenf LiteLLM-Vorschlaege gegen exakt gelieferte Evidenz und speichert Runs sowie optimistisch nebenlaeufigkeitsgeschuetzte Accept/Reject-Ereignisse append-only in PostgreSQL.
|
||||
|
||||
Offen bis zur Phasenabnahme:
|
||||
|
||||
- autorisierter und lizenzrechtlich geklaerter regulatorischer Quellkorpus samt Update-/Freigabeprozess
|
||||
- organisatorische Datenschutzfreigabe fuer den Embeddingzielpfad und kuratierte Erweiterung der oeffentlichen Signal-/Regelkataloge
|
||||
- formaler Update-/Freigabeprozess fuer neue lizenzierte ISO-Ausgaben und Korpus-Rebuilds
|
||||
- kuratierte Erweiterung der oeffentlichen Signal-/Regelkataloge
|
||||
- repraesentativer freigegebener EN/DE-Evaluationssatz und dokumentierte reale Dense-Baseline gegen den autorisierten Korpus
|
||||
- aktive Sparse-/BM25-Erzeugung, Hybridfusion, Kontrollgruppierung und evaluierter Reranker
|
||||
- constrained Mappingmodell, Confidence-Kalibrierung, Human Review und append-only PostgreSQL-Provenienz
|
||||
- API/UI, Mandanten-/Rollenabgrenzung, Betriebsmonitoring sowie praktischer Rebuild- und Lasttest
|
||||
- aktive Sparse-/BM25-Erzeugung, Hybridfusion und evaluierter Reranker
|
||||
- Confidence-Kalibrierung, Assessment-Batchlauf, Mandanten-/Rollenabgrenzung, Betriebsmonitoring und Lasttest
|
||||
- Betriebsprozess, der vor jeder kuenftigen Migration einen laufenden Hot-Reload-Backendprozess stoppt oder die Migration bis nach dem verifizierten Backup sperrt
|
||||
|
||||
Abnahmekriterien:
|
||||
|
||||
|
|
@ -2199,7 +2203,7 @@ Die naechste Umsetzung fuehrt Phase 13B auf dem konsolidierten Import-, Policy-A
|
|||
|
||||
1. Phase 13A ist abgeschlossen: Erkennung, technische Dokumenttypen, Source Extractors, Mappingprofile, Finding-Identität sowie allgemeine Artefakt-, Processing- und Jobgrenzen sind konsolidiert und die Scannerpersistenz verwendet den Candidate-Pfad.
|
||||
2. Phase 13B ist in Arbeit: Der sichere Ingestion-Unterbau, technische Policy-Identitaeten, versionierte ADMX-/Baselineanalysen, deterministisches Reprocessing, Environment Profiles, konservative Snapshotaggregation, typisierte Evaluation, fail-closed Finding Factory, Unified-Result-Read-Projektion, projektgebundene Review-/Orchestrierungsoberflaechen und die aktuelle snapshotbezogene Ergebnisdeduplizierung sind umgesetzt. Als naechste vertikale Etappe folgen fachlich kuratierte Policy-Regeln und vollstaendige versionierte Windows-Inbox-ADMX-/ADML-Kataloge; danach weitere Preference-CSEs und Exportdetails. Ohne RSoP/GP Results bleibt die effektive Anwendung weiterhin ausdruecklich nicht verifiziert.
|
||||
3. Phase 13F besitzt die rebuildbare Qdrant-, Chunking-, Embedding-, Ingestion- und Dense-Retrieval-Grundlage sowie einen fail-closed Public-Signal-Finding-Query-Builder und einen ausfuehrbaren Evaluationsvertrag. Als naechste Etappe folgen der freigegebene Korpus, organisatorische Datenschutzfreigabe, ein unabhaengig gelabelter EN/DE-Evaluationssatz, die reale Dense-Baseline und danach der persistierte Review-/Mappingvertrag. Sparse/Hybrid und Reranking werden erst nach messbarer Dense-Baseline aktiviert.
|
||||
3. Phase 13F besitzt den lizenzierten ISO-Korpus-v2, die rebuildbare Qdrant-/Dense-Grundlage, den fail-closed Public-Signal-Pfad sowie den persistierten manuell reviewten Einzel-Finding-Workflow. Als naechste Etappe folgen ein unabhaengig gelabelter repraesentativer EN/DE-Evaluationssatz und die reale Dense-Baseline; Sparse/Hybrid, Reranking und Assessment-Batchmapping werden erst danach aktiviert.
|
||||
4. Erst nach verifizierter Phase 13B den kontrollierten Firmenbetrieb aus Phase 14 mit SSO, Rollen, HTTPS, Monitoring, Backup/Restore und Aufbewahrung fortsetzen.
|
||||
|
||||
Die technische Reihenfolge ist bis einschliesslich Phase 13A umgesetzt. Phase 13B baut auf dem verifizierten sicheren Ingestion-Unterbau auf und reicht inzwischen produktiv von der begrenzten Parser-/Workerverarbeitung ueber unveraenderliche Analyse-, Profil-, Aggregations- und Evaluationslaeufe bis zur streng evidenzgebundenen Finding Factory und ihrer gemeinsamen Unified-Result-Leseprojektion. Das Backend leitet projektgebundene Aktionsbereitschaft aus aktiven Revisionen und exakt passenden Referenzdaten ab; der Datahub fuehrt Environment-Profile-Anlage, Reprocessing, Aggregation, Evaluation und Finding Factory ueber die autoritativen idempotenten Services aus. Der produktive Regelkatalog bleibt bis fachlicher Kuratierung leer, und die vorhandenen Microsoft-Baselines bleiben ohne vollstaendige Windows-Inbox-ADMX-/ADML-Kataloge partial; dadurch erzeugt dieser reale Bestand weiterhin nur Review und keine Findings.
|
||||
|
|
@ -2266,6 +2270,14 @@ Festlegung 2026-09-15 (Phase 13F, Finding-Retrievalgrenze): Nur Scanner-Findings
|
|||
|
||||
Festlegung 2026-09-15 (Phase 13F, ausdruecklicher Nutzerauftrag): Qdrant ist der rebuildbare Suchindex fuer versionierte regulatorische Chunks und niemals fachliche Quelle fuer Scannergebnisse oder Mappingentscheidungen. PostgreSQL bleibt System of Record fuer Scans, Findings, Ergebnisse, Reviews und spaetere append-only Mappingprovenienz. Dense und Sparse sind benannte Vektorraeume; zunaechst wird nur mehrsprachiges Dense Retrieval aktiviert. Candidate Retrieval und optionaler Reranker bleiben getrennte Schnittstellen. Ein Modellwechsel darf alte Vektoren nicht still weiterverwenden; Modellname und Dimension werden explizit gebunden, Abfragen filtern auf das aktive Modell, und ein Korpus-Rebuild bleibt Pflicht. Der Embeddingdienst ist eine eigene externe Vertrauensgrenze; Findinginhalte duerfen erst nach einer getrennten Datenschutzfreigabe als minimierte Query uebertragen werden.
|
||||
|
||||
Festlegung 2026-09-15 (Phase 13F, ISO-Mapping): ISO 27002 ist fuer alle 93 Controls der semantische Korpus; ISO 27001 Annex A bleibt autoritative Referenz und Vollstaendigkeitspruefung. Kanonische IDs sind `5.1` bis `8.34`, Aliaswerte `A.5.1` bis `A.8.34`. Ein Mapping bedeutet ausschliesslich fachliche Relevanz, niemals Compliance. Dense Retrieval gruppiert zuerst auf Control-Ebene; LiteLLM darf nur gelieferte Controls und Chunk-IDs ausgeben. Kein Confidencewert akzeptiert automatisch. Runs und Entscheidungen sind append-only in PostgreSQL, Qdrant bleibt jederzeit rebuildbar. Die erste UI bleibt beim einzelnen Finding; Batchmapping, Sparse/Hybrid und Reranking bleiben bis zur repraesentativen Retrievalbewertung ausgeschlossen.
|
||||
|
||||
Festlegung 2026-09-15 (Phase 13F, Finding-Retrievalgrenze – Revision, ersetzt die vorstehende Festlegung auf ausdruecklichen Nutzerauftrag): Die ISO-Mappingquery wird nicht mehr aus einer festen Themenwhitelist erzeugt, sondern aus der gemeinsamen datenschutzsicheren Finding-Projektion (`src/domain/privacy-safe-finding.ts`, `privacySafeFinding`) – exakt derselben Grenze wie die KI-Uebersicht. Query-Quelle ist der regelbezogene Prueftext (Titel, sowie Beschreibung und Empfehlung, jeweils aus der je Regel deduplizierten Finding-Definition, mit E-Mail-Redaktion); katalogisierte Regel-IDs, oeffentlicher Toolname und erkannte Technikthemen bleiben nur noch Query-Anreicherung. Die Ergebnistabelle/Occurrence (affectedAsset, Source-Locator/Pfad, Objektwerte, Mandanten-/Review-/Provenienzdaten) wird nie gelesen. Dadurch mappt jedes Scanner-Finding mit Pruef-Titel; nur Nicht-Scanner-Datensaetze und Findings ohne Titel bleiben `insufficient-public-context`. `REGULATORY_RETRIEVAL_VERSION` auf `dense-control-grouping-1.1.0` erhoeht; kein Reimport, da Mappinglaeufe regenerierbar sind und keine persistierten Scannerdaten geaendert werden. Restrisiko: Die geteilte Redaktion entfernt nur E-Mail-Adressen und verlaesst sich auf regelgenerische Definitionen; eine haertere Redaktion (Host/IP/Pfad) waere eine gemeinsame Entscheidung fuer beide KI-Funktionen und bleibt offen. Folgeetappen: assessment-weites Batchmapping ueber alle Findings (dedupliziert je Regel/Definition, analog zum KI-Uebersicht-Scopelauf) und optional eine symmetrische Einzel-Finding-KI-Uebersicht.
|
||||
|
||||
Praezisierung 2026-09-15 (Phase 13F, Mapping-Relevanz): Der Mapping-Systemprompt (`REGULATORY_MAPPING_PROMPT_VERSION` `1.2.0`) mappt jetzt ausdruecklich auf fachliche Themenrelevanz. Ein Kandidat wird zugeordnet, wenn er dasselbe Sicherheitsthema regelt (z. B. Passwort-/Authentisierungsverwaltung), auch wenn der ISO-Kontrolltext den konkreten Wert, Schwellwert oder Parameter des Findings nicht nennt; ISO-Controls sind prinzipienbasiert und wiederholen solche Spezifika selten. `no_relevant_control` gilt nur, wenn kein Kandidat das Thema regelt. Hintergrund: `A-MinPwdLen` wurde trotz korrekt abgerufenem Control `5.17` (Authentisierungsinformationen) faelschlich als nicht relevant zurueckgewiesen. Kein Reimport, da Mappinglaeufe regenerierbar sind.
|
||||
|
||||
Festlegung 2026-09-15 (Phase 13F, globale ISO-Mappings, ausdruecklicher Nutzerauftrag): ISO-Mappings und Accept/Reject-Entscheidungen sind jetzt global und mandantenunabhaengig, analog zur CVE-Anreicherung. Migration `026` ersetzt die per-Occurrence-Tabellen `assessor_regulatory_mapping_runs`/`_decisions` (nur regenerierbare Testdaten, vor dem Drop gesichert) durch `assessor_regulatory_mappings` und ein globales `assessor_regulatory_mapping_decisions`. Schluessel ist die Regelidentitaet `ruleKey` (`tool:ruleId`, ersatzweise ein stabiler Titel-Hash) plus die Pipelineversionen (`UNIQUE(rule_key, corpus_version, retrieval_version, prompt_version, model)`); die Mapping-ID ist ein deterministisches Digest dieses Tupels, sodass ein Regel-Mapping genau einmal existiert und jede Occurrence in jedem Projekt es wiederverwendet. Entscheidungen sind append-only, per `(ruleKey, controlId)` gefuehrt, versionsunabhaengig und ueberleben Scan-, Assessment- und Projektloeschung. Der Dienst leitet nur die non-personale Regelidentitaet und den Pruefkontext aus dem Finding ab; Ergebnistabelle/Occurrence fliesst nie ein. Kein Reimport (kein Scanner-Import veraendert). Naechste Etappe: assessment-weites Batchmapping ueber distinct Regeln.
|
||||
|
||||
Festlegung 2026-09-10 (Purple-Knight-Begleitdateien): Ein nativer Score darf aus einer separaten Summary auf die Findings desselben Scanlaufs uebertragen werden, wenn genau ein nativer Traeger und eine eindeutige Ergebnispopulation denselben Werkzeug- und Verarbeitungskontrakt teilen. Datei- und Vorkommensherkunft bleiben gemeinsam erhalten; Mehrdeutigkeit verhindert die Verbindung.
|
||||
|
||||
Festlegung 2026-09-10 (weitere Scanner): Native Gesamtwerte werden bei jeder Profilerweiterung zuerst geprueft und deklarativ mit ihrer Bedeutung erfasst. Die Scoredefinition ist Pflicht, die Berechnung gemeinsam; ihre Version gehoert zur Vergleichskompatibilitaet. Extraktion und Originalherkunft bleiben im generischen Importpfad. AGENTS.md und docs/SCORING_MODEL.md halten die Erweiterungsregel fest.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,374 @@
|
|||
# ASSESSOR — Project Overview
|
||||
|
||||
ASSESSOR imports security-scanner reports (PingCastle, Purple Knight, CIS-CAT,
|
||||
Microsoft Defender for IoT, Maester, Group Policy backups, and generic
|
||||
CSV/JSON/XML/HTML/Excel exports), normalizes them into one canonical model,
|
||||
scores the assessed environment, lets an analyst review and compare findings,
|
||||
and presents an executive security posture. It also offers optional AI overview
|
||||
generation and regulatory (ISO 27001/27002) mapping through a retrieval index.
|
||||
|
||||
**This is the entry point for the whole project.** It explains what the software
|
||||
is, how it is put together, how many containers it runs, how to develop it
|
||||
locally, and how it is deployed. Detailed topic docs live alongside this file in
|
||||
[`docs/`](.); this document links to the right one for each topic.
|
||||
|
||||
> Status: the application runs today as a local browser app served by a Node.js
|
||||
> backend, and is being deployed to an internal server behind an nginx reverse
|
||||
> proxy. The full phased plan and its history are in
|
||||
> [`docs/IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md); binding rules for
|
||||
> anyone (human or AI) changing the code are in [`AGENTS.md`](../AGENTS.md).
|
||||
|
||||
---
|
||||
|
||||
## 1. What runs where — the short answer
|
||||
|
||||
There are **three long-running services**, defined in
|
||||
[`docker-compose.deploy.yml`](../docker-compose.deploy.yml):
|
||||
|
||||
| Service | Image | Role | Published on host? |
|
||||
|------------|----------------------|------------------------------------------------------------------|--------------------|
|
||||
| `app` | built from `Dockerfile` | Node.js server: serves the React app **and** the `/api/v1` API from one origin | Yes — the only exposed port |
|
||||
| `postgres` | `postgres:16-alpine` | System of record (canonical findings, reviews, comparisons, jobs) | No |
|
||||
| `qdrant` | `qdrant/qdrant:v1.19.1` | Rebuildable vector index for regulatory RAG retrieval | No |
|
||||
|
||||
So **one deployed environment = 3 containers**. Only `app` is reachable from the
|
||||
host; PostgreSQL and Qdrant are private on the internal Docker network.
|
||||
|
||||
The server runs **two isolated environments in parallel** (see
|
||||
[`docs/DEPLOYMENT.md`](DEPLOYMENT.md)), each its own set of 3 containers with
|
||||
its own database, artifact volume, and Qdrant index:
|
||||
|
||||
| Branch | Compose project | VPN address | Public host name (via nginx) |
|
||||
|----------------|-------------------------|--------------------------|------------------------------------|
|
||||
| `main` | `assessor_prod` | `http://10.0.0.154:4173` | `https://assessor.soc-ops.de` |
|
||||
| `experimental` | `assessor_experimental` | `http://10.0.0.154:4174` | `https://staging.assessor.soc-ops.de` |
|
||||
|
||||
That is **6 containers total** for the managed stacks. A separate, older HTTPS
|
||||
installation exists on the same host and is intentionally *not* managed by this
|
||||
workflow until an approved cutover.
|
||||
|
||||
For **local development** you do not containerize the app. You run only the two
|
||||
backing services in Docker (`postgres` + `qdrant`, project `assessor-2_0` — the
|
||||
single canonical local datastore) and run the Node backend + Vite client as host
|
||||
processes. See [§4](#4-local-development).
|
||||
|
||||
---
|
||||
|
||||
## 2. Architecture
|
||||
|
||||
### 2.1 Layered design
|
||||
|
||||
ASSESSOR keeps a strict separation between format parsing, structural analysis,
|
||||
semantic mapping, normalization, comparison, persistence, and presentation. The
|
||||
core engine is **platform-neutral**: it never touches the DOM, `localStorage`,
|
||||
browser `File` objects, SQL, or the network directly — those live behind
|
||||
adapters. This is what lets the same domain logic run in the browser today and in
|
||||
the server backend now / a future shell later.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ React UI (src/features, src/components) │ presentation only
|
||||
│ no source-specific parsing, no direct DB access │
|
||||
└───────────────┬───────────────────────────────────────────────┘
|
||||
│ HTTP (same origin)
|
||||
┌───────────────▼───────────────────────────────────────────────┐
|
||||
│ Node backend (server/) — /api/v1 + static React app │
|
||||
│ import workers · GPO analysis · CVE enrichment · AI · RAG │
|
||||
└───────────────┬───────────────────────────────────────────────┘
|
||||
│ AssessorRepository contract (no SQL leaks upward)
|
||||
┌───────────────▼───────────────────────────────────────────────┐
|
||||
│ Persistence adapter │
|
||||
│ • PostgresAssessorRepository (durable, production) │
|
||||
│ • LocalAssessorRepository (in-memory, dev only) │
|
||||
└───────────────┬───────────────────────────────────────────────┘
|
||||
│
|
||||
┌───────────────▼──────────┐ ┌──────────────────────────┐
|
||||
│ PostgreSQL 16 │ │ Qdrant (regulatory RAG) │
|
||||
│ system of record │ │ rebuildable index │
|
||||
└──────────────────────────┘ └──────────────────────────┘
|
||||
```
|
||||
|
||||
The **platform-neutral import engine** (`src/importers`, `src/domain`) turns raw
|
||||
bytes into an `ImportSource`, parses it through injected runtime adapters, picks
|
||||
a scanner profile via the shared recognition engine, and produces the canonical
|
||||
model. Known tool profiles *extend* this generic pipeline rather than bypass it.
|
||||
|
||||
Key boundary docs:
|
||||
- [`docs/PLATFORM_NEUTRAL_ENGINE.md`](PLATFORM_NEUTRAL_ENGINE.md) — engine/adapter layering
|
||||
- [`docs/PERSISTENCE_BOUNDARY.md`](PERSISTENCE_BOUNDARY.md) — the `AssessorRepository` contract and import transaction
|
||||
- [`docs/CANONICAL_MODEL.md`](CANONICAL_MODEL.md) — the normalized data model
|
||||
- [`docs/RECOGNITION_ENGINE.md`](RECOGNITION_ENGINE.md) — profile selection
|
||||
- [`docs/UNIFIED_RESULT_MODEL.md`](UNIFIED_RESULT_MODEL.md) / [`docs/RESULT_CONTRACT.md`](RESULT_CONTRACT.md) — the read model behind the UI
|
||||
- [`docs/SCORING_MODEL.md`](SCORING_MODEL.md) — native-score-first scoring engine
|
||||
|
||||
### 2.2 The app service in detail
|
||||
|
||||
The single `app` container (or local `node dist-server/server.mjs`) does three
|
||||
jobs from one origin (port `4173`):
|
||||
|
||||
1. Serves the built React SPA from `dist/`.
|
||||
2. Serves the versioned JSON API under `/api/v1`.
|
||||
3. Runs background work: isolated import parser workers, GPO inventory/analysis
|
||||
workers, CVE enrichment, AI overview jobs, and regulatory retrieval.
|
||||
|
||||
Same-origin is a deliberate security property: the server sets **no CORS**, and
|
||||
every write requires the `X-Assessor-Request: 1` header with a matching `Origin`.
|
||||
See [`docs/LOCAL_BACKEND.md`](LOCAL_BACKEND.md) and [`SECURITY.md`](../SECURITY.md).
|
||||
|
||||
### 2.3 API surface
|
||||
|
||||
All endpoints live under `/api/v1`. Top-level resource groups:
|
||||
|
||||
`health` · `audit-events` · `db-structure` · `reference-data` · `projects`
|
||||
(with nested `policy-analyses`, `environment-profiles`, `policy-snapshot-aggregations`,
|
||||
`policy-evaluations`, `policy-finding-runs`, `gpo-backups`, `scan-runs`) ·
|
||||
`assessments` · `findings` · `results` · `reviews` · `profiles` · `comparisons` ·
|
||||
`cves` · `import-sessions`.
|
||||
|
||||
Imports are **atomic sessions** (stage → commit → rollback). Errors always carry
|
||||
`code`, `message`, and `requestId`. The contract is in
|
||||
[`src/api/assessor-api-contract.ts`](../src/api/assessor-api-contract.ts); the server
|
||||
in [`server/assessor-http-server.ts`](../server/assessor-http-server.ts).
|
||||
|
||||
---
|
||||
|
||||
## 3. Repository layout
|
||||
|
||||
```
|
||||
server/ Node backend
|
||||
assessor-http-server.ts HTTP router (/api/v1 + static app)
|
||||
index.ts process entry, wiring, env parsing
|
||||
postgres/ repository adapter + migrations/
|
||||
gpo/ Group Policy backup ingestion & analysis
|
||||
cve/ NVD CVE enrichment (opt-in, backend-only)
|
||||
ai/ LiteLLM client + AI overview service
|
||||
regulatory/ ISO corpus, chunking, Qdrant index, RAG runtime, mapping
|
||||
artifacts/ filesystem artifact store
|
||||
src/ Browser app + platform-neutral core
|
||||
importers/ domain/ platform-neutral engine & canonical model
|
||||
profiles/ scanner profiles (see §5)
|
||||
recognition/ analysis/ comparison/ scoring/ evaluation/
|
||||
persistence/ repository contract + LocalAssessorRepository
|
||||
platform/browser/ browser adapters (File, DOM, storage, download)
|
||||
api/ typed API client & contract
|
||||
features/ components/ styles/ React UI
|
||||
docs/ all detailed documentation (see §9)
|
||||
tests/ vitest suites + local-only & design-language checks
|
||||
deploy/nginx/ reverse-proxy vhosts for prod + staging
|
||||
.gitea/workflows/ CI/CD (deploy.yml)
|
||||
Dockerfile multi-stage build for the app image
|
||||
docker-compose.deploy.yml 3-service deployed stack (app+postgres+qdrant)
|
||||
docker-compose.postgres.yml local backing services (postgres+qdrant)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Local development
|
||||
|
||||
### Prerequisites
|
||||
- Node.js 22 (matches the runtime image) and npm
|
||||
- Docker + Docker Compose (for PostgreSQL + Qdrant)
|
||||
|
||||
### First-time setup
|
||||
|
||||
```bash
|
||||
cp .env.postgres.example .env.postgres
|
||||
# edit .env.postgres: set long, local-only ASSESSOR_POSTGRES_PASSWORD and ASSESSOR_QDRANT_API_KEY
|
||||
npm ci
|
||||
npm run db:up # starts the canonical postgres + qdrant (project assessor-2_0)
|
||||
npm run dev # builds/starts backend on 127.0.0.1:4173, then Vite client on 127.0.0.1:5173
|
||||
```
|
||||
|
||||
`npm run dev` checks `/api/v1/health` first, brings up a missing backend, then
|
||||
opens the Vite client (which proxies `/api` to `4173`). The backend binds to
|
||||
`127.0.0.1` only — it is **not** exposed on the network in local mode.
|
||||
|
||||
> The canonical local datastore is fixed: Compose project `assessor-2_0`,
|
||||
> database `assessor`, volume `assessor-2_0_assessor-postgres-data`, on
|
||||
> `127.0.0.1:5432`. Never create an alternative DB/volume/port for this
|
||||
> workspace (see [`AGENTS.md`](../AGENTS.md) → "Canonical local PostgreSQL"). If
|
||||
> port 5432 is taken by a different container, stop and report the conflict.
|
||||
|
||||
### Useful scripts
|
||||
|
||||
| Command | What it does |
|
||||
|------------------------|-----------------------------------------------------------------|
|
||||
| `npm run dev` | Full local stack (backend + Vite client) |
|
||||
| `npm run dev:client` | Vite client only (backend managed separately) |
|
||||
| `npm run build` | Build client (`dist/`) and server (`dist-server/`) |
|
||||
| `npm run start:local` | Run the built server on 4173 using `.env.postgres` |
|
||||
| `npm run serve:local` | `build` then `start:local` |
|
||||
| `npm run db:up` / `db:down` / `db:logs` | Manage local postgres+qdrant |
|
||||
| `npm run rag:prepare-iso` / `rag:ingest` / `rag:evaluate` | Build/populate/evaluate the regulatory index |
|
||||
| `npm test` | vitest + local-only check + design-language check |
|
||||
|
||||
Without `ASSESSOR_DATABASE_URL` the backend runs in an explicitly ephemeral
|
||||
in-memory mode. If the URL is set but the DB is unreachable, startup **fails**
|
||||
(no silent fallback), so a misconfiguration can never produce imports that look
|
||||
saved but are not. Details: [`docs/POSTGRESQL.md`](POSTGRESQL.md).
|
||||
|
||||
---
|
||||
|
||||
## 5. Supported scanner profiles
|
||||
|
||||
Profiles live in [`src/profiles/`](../src/profiles/): `pingcastle`, `purple-knight`,
|
||||
`cis-cat`, `defender-for-iot`, `maester`, `group-policy`, `generic`, and
|
||||
`public-search` (regulatory). Each profile must declare its scoring semantics
|
||||
(`securityScoring`) and extend the generic import pipeline — see
|
||||
[`docs/TOOL_PROFILES.md`](TOOL_PROFILES.md) and the profile rules in
|
||||
[`AGENTS.md`](../AGENTS.md).
|
||||
|
||||
Any change to recognition, parsing, mapping, identity, scoring inputs, or
|
||||
persisted evidence must follow [`docs/REIMPORT_POLICY.md`](REIMPORT_POLICY.md)
|
||||
(add a processing-manifest version and mark affected historical imports for
|
||||
reimport).
|
||||
|
||||
---
|
||||
|
||||
## 6. Deployment
|
||||
|
||||
Deployment is fully driven by the Gitea workflow
|
||||
[`.gitea/workflows/deploy.yml`](../.gitea/workflows/deploy.yml). **You do not run
|
||||
`docker compose` by hand on the server for a normal release** — a push does it.
|
||||
|
||||
### How a release happens
|
||||
1. Push to `main` (→ stable) or `experimental` (→ experimental).
|
||||
2. The workflow loads that environment's complete env-file secret
|
||||
(`PROD_ENV_FILE` / `STAGING_ENV_FILE`) and validates required keys are present.
|
||||
3. Runs `npm ci`, the full `npm test`, and `npm run build`.
|
||||
4. Acquires a per-environment Docker lock (Gitea ignores the `concurrency` key),
|
||||
validates the resolved Compose config, and checks port/project isolation
|
||||
(only `app`/`postgres`/`qdrant` may exist; the target port must not be
|
||||
published by another project).
|
||||
5. **Backs up** the existing environment (custom `pg_dump`, artifact tarball,
|
||||
`SHA256SUMS`) into the external volume `assessor_deployment_backups`.
|
||||
6. `docker compose ... up -d --build --wait` for that environment only.
|
||||
7. Verifies health via `/api/v1/health` inside the container, then releases the lock.
|
||||
|
||||
The two environments never share PostgreSQL, artifacts, or the Qdrant index —
|
||||
Compose scopes all named volumes by project. `restart: unless-stopped` keeps both
|
||||
alive across daemon/host restarts. Feature promotion is a `experimental → main`
|
||||
merge; **databases and artifacts are never copied between environments.**
|
||||
|
||||
### Reverse proxy / TLS
|
||||
nginx terminates TLS and proxies to the VPN ports; vhosts are in
|
||||
[`deploy/nginx/assessor.conf`](../deploy/nginx/assessor.conf). `proxy_set_header
|
||||
Host $host` is mandatory — ASSESSOR rejects writes (403 `cross_origin_rejected`)
|
||||
if the forwarded host does not match the browser origin. `client_max_body_size`
|
||||
is 200m to allow ~100 MiB uploads (base64 overhead).
|
||||
|
||||
### Backups & restore
|
||||
The workflow writes a backup before every update and **never deletes backups**.
|
||||
Restore tests must use an isolated temporary Compose project and synthetic data,
|
||||
verify `SHA256SUMS`, and `pg_restore --exit-on-error` into a fresh DB. Never run
|
||||
`docker compose down --volumes` on a permanent environment. Full procedure:
|
||||
[`docs/DEPLOYMENT.md`](DEPLOYMENT.md) and
|
||||
[`docs/POSTGRESQL.md`](POSTGRESQL.md).
|
||||
|
||||
### Manual deploy (reference only)
|
||||
```bash
|
||||
docker compose --env-file .env -p assessor_prod -f docker-compose.deploy.yml \
|
||||
up -d --build --wait
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Configuration
|
||||
|
||||
Real secret values live only in the git-ignored `.env.postgres` (workstation) or
|
||||
the deployment env-file secrets — never in the repo or in client-side config.
|
||||
[`.env.postgres.example`](../.env.postgres.example) is the annotated template.
|
||||
|
||||
| Variable | Purpose |
|
||||
|----------------------------------|----------------------------------------------------------------|
|
||||
| `ASSESSOR_HOST` / `ASSESSOR_PORT`| Bind address / port (default `127.0.0.1:4173`) |
|
||||
| `ASSESSOR_DATABASE_URL` | PostgreSQL DSN; absent = ephemeral in-memory dev mode |
|
||||
| `ASSESSOR_DATABASE_SSL` | `disable` only for local loopback; TLS required for remote DBs |
|
||||
| `ASSESSOR_POSTGRES_*` | DB name/user/password (compose builds the DSN in deploy) |
|
||||
| `ASSESSOR_ARTIFACT_ROOT` | Artifact store path (must be outside the public web root) |
|
||||
| `ASSESSOR_QDRANT_URL` / `_API_KEY` | Regulatory index endpoint & key |
|
||||
| `ASSESSOR_RAG_EMBEDDING_*` | Approved embedding model for regulatory ingestion/retrieval |
|
||||
| `ASSESSOR_AI_MODE` (+ `ASSESSOR_AI_*`) | Enables the LiteLLM AI overview; unset = outbound AI off |
|
||||
| `NVD_API_KEY` / `ASSESSOR_NVD_TIMEOUT_MS` | Optional CVE enrichment against the fixed NVD endpoint |
|
||||
| `ASSESSOR_PARSER_*` / `ASSESSOR_GPO_*` | Import/GPO worker limits (timeout, memory, concurrency) |
|
||||
|
||||
AI and CVE calls are the only outbound network paths, they are backend-only,
|
||||
opt-in, and off by default. Trust boundaries:
|
||||
[`docs/AI_OVERVIEW_SECURITY.md`](AI_OVERVIEW_SECURITY.md),
|
||||
[`docs/REGULATORY_RAG.md`](REGULATORY_RAG.md),
|
||||
[`docs/IMPORT_SECURITY_LIMITS.md`](IMPORT_SECURITY_LIMITS.md).
|
||||
|
||||
---
|
||||
|
||||
## 8. Data & persistence
|
||||
|
||||
- **PostgreSQL 16** is the system of record: relational keys/foreign keys for
|
||||
identity plus the full versioned canonical object as JSONB (so new scanner
|
||||
fields are captured losslessly). Full-text `tsvector` + GIN indexes back search.
|
||||
- **Migrations** are versioned SQL files in
|
||||
[`server/postgres/migrations/`](../server/postgres/migrations/) (currently through
|
||||
`026_*`). On start the server takes an advisory lock, verifies each migration's
|
||||
version + SHA-256, applies new ones in their own transaction, and refuses to
|
||||
start if an already-applied migration was edited. Never edit an applied
|
||||
migration — add a new numbered file.
|
||||
- **Qdrant** holds the regulatory retrieval index. It is authenticated and
|
||||
**rebuildable** from the versioned corpus — its volume speeds restarts but is
|
||||
not a system-of-record backup.
|
||||
- **Artifacts** (e.g. GPO raw backups) go to a filesystem store on a dedicated
|
||||
volume, outside the public web root.
|
||||
|
||||
Details: [`docs/POSTGRESQL.md`](POSTGRESQL.md),
|
||||
[`docs/REGULATORY_RAG.md`](REGULATORY_RAG.md), [`MIGRATION.md`](../MIGRATION.md).
|
||||
|
||||
---
|
||||
|
||||
## 9. Testing & verification
|
||||
|
||||
```bash
|
||||
npm test # vitest + local-only network check + design-language check
|
||||
npm run build # type-check + build client and server (CI gate)
|
||||
npm run test:postgres # integration tests against a real PostgreSQL 16 (ASSESSOR_TEST_DATABASE_URL)
|
||||
```
|
||||
|
||||
Baseline verification for any relevant change is `npm test` + `npm run build`
|
||||
(both run in CI before deploy). **Testing is privacy-safe:** use synthetic
|
||||
fixtures; never read, log, screenshot, or quote real report content, file names,
|
||||
finding titles, or tenant data. Against a running app, verify only content-free
|
||||
signals (HTTP status, health, anonymized counts, boolean DOM state). See the
|
||||
"Privacy-safe testing" section of [`AGENTS.md`](../AGENTS.md) and
|
||||
[`docs/TEST_BASELINE.md`](TEST_BASELINE.md).
|
||||
|
||||
---
|
||||
|
||||
## 10. Security model (summary)
|
||||
|
||||
- Local/same-origin by default: no CORS, write requests need `X-Assessor-Request`
|
||||
with a matching `Origin`; the backend binds loopback locally.
|
||||
- Imported HTML/XML/Excel/JSON/CSV are **untrusted input** — active content is
|
||||
never executed; HTML is inertized before table extraction; Excel formulas/VBA
|
||||
stay disabled. Parsers run in isolated workers with time/memory limits.
|
||||
- No external uploads, telemetry, or CDN assets. The only outbound paths are the
|
||||
opt-in backend AI (LiteLLM) and CVE enrichment (NVD).
|
||||
- Embedded report data is parsed with a bounded static grammar, never evaluated.
|
||||
|
||||
Full model: [`SECURITY.md`](../SECURITY.md), [`docs/IMPORT_SECURITY_LIMITS.md`](IMPORT_SECURITY_LIMITS.md),
|
||||
[`docs/architecture/gpo-threat-model.md`](architecture/gpo-threat-model.md).
|
||||
|
||||
---
|
||||
|
||||
## 11. Documentation map
|
||||
|
||||
**Start here → this overview.** Then, by topic:
|
||||
|
||||
| Area | Documents |
|
||||
|------|-----------|
|
||||
| Rules for contributors | [`AGENTS.md`](../AGENTS.md) · [`CLAUDE.md`](../CLAUDE.md) |
|
||||
| Plan & history | [`docs/IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md) · [`MIGRATION.md`](../MIGRATION.md) |
|
||||
| Architecture | [`docs/PLATFORM_NEUTRAL_ENGINE.md`](PLATFORM_NEUTRAL_ENGINE.md) · [`docs/PERSISTENCE_BOUNDARY.md`](PERSISTENCE_BOUNDARY.md) · [`docs/architecture/`](architecture/) |
|
||||
| Data model | [`docs/CANONICAL_MODEL.md`](CANONICAL_MODEL.md) · [`docs/UNIFIED_RESULT_MODEL.md`](UNIFIED_RESULT_MODEL.md) · [`docs/RESULT_CONTRACT.md`](RESULT_CONTRACT.md) · [`docs/SCORING_MODEL.md`](SCORING_MODEL.md) |
|
||||
| Import | [`docs/IMPORT_WORKFLOW.md`](IMPORT_WORKFLOW.md) · [`docs/IMPORT_INTERMEDIATE_MODEL.md`](IMPORT_INTERMEDIATE_MODEL.md) · [`docs/RECOGNITION_ENGINE.md`](RECOGNITION_ENGINE.md) · [`docs/TOOL_PROFILES.md`](TOOL_PROFILES.md) · [`docs/REIMPORT_POLICY.md`](REIMPORT_POLICY.md) |
|
||||
| Compare & filter | [`docs/COMPARISON_ENGINE.md`](COMPARISON_ENGINE.md) · [`docs/DYNAMIC_FILTERS.md`](DYNAMIC_FILTERS.md) |
|
||||
| Backend & persistence | [`docs/LOCAL_BACKEND.md`](LOCAL_BACKEND.md) · [`docs/POSTGRESQL.md`](POSTGRESQL.md) |
|
||||
| Deployment | [`docs/DEPLOYMENT.md`](DEPLOYMENT.md) · [`deploy/nginx/assessor.conf`](../deploy/nginx/assessor.conf) |
|
||||
| AI & regulatory | [`docs/AI_OVERVIEW_SECURITY.md`](AI_OVERVIEW_SECURITY.md) · [`docs/LLM_DASHBOARD_PROMPT.md`](LLM_DASHBOARD_PROMPT.md) · [`docs/REGULATORY_RAG.md`](REGULATORY_RAG.md) |
|
||||
| Security & testing | [`SECURITY.md`](../SECURITY.md) · [`docs/IMPORT_SECURITY_LIMITS.md`](IMPORT_SECURITY_LIMITS.md) · [`docs/TEST_BASELINE.md`](TEST_BASELINE.md) · [`docs/USER_FLOW_CHECKS.md`](USER_FLOW_CHECKS.md) |
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
# Regulatory-control RAG with Qdrant
|
||||
|
||||
Status: Phase 13F foundation. The current slice indexes and retrieves regulatory control text and can create privacy-minimized retrieval candidates for scanner findings. It does not map findings, call an LLM for classification, expose a browser API, or persist mapping decisions.
|
||||
Status: Phase 13F one-finding vertical slice. The server prepares the licensed German ISO 27001/27002 corpus, indexes section-safe ISO 27002 semantic chunks plus ISO 27001 references, retrieves and groups candidates, requests a grounded LiteLLM proposal, persists runs and reviewer decisions, and exposes the workflow inline in Work. Batch mapping, sparse retrieval, reranking, and confidence calibration remain out of scope.
|
||||
|
||||
## Boundaries
|
||||
|
||||
- PostgreSQL remains the only system of record for ASSESSOR projects, assessments, scans, findings, results, reviews, and future accepted mappings.
|
||||
- PostgreSQL remains the only system of record for ASSESSOR projects, assessments, scans, findings, results, reviews, mapping runs, and mapping decisions.
|
||||
- Qdrant is a derived, rebuildable search index. Losing or deleting its volume must not lose an ASSESSOR scan, finding, result, or review.
|
||||
- The versioned regulatory corpus JSON is the rebuild source for Qdrant. It must be retained and governed separately from the index.
|
||||
- React does not connect to Qdrant or the embedding provider. Both are backend-only boundaries.
|
||||
- Regulatory retrieval produces candidates, not compliance decisions. A later constrained mapping stage must retain the retrieved chunk IDs, scores, corpus versions, model version, prompt/schema version, and final decision in PostgreSQL.
|
||||
- Regulatory retrieval produces candidates, not compliance decisions. The constrained mapping stage retains retrieved chunk IDs, scores, corpus/source hashes, model, prompt/schema versions, and append-only reviewer decisions in PostgreSQL.
|
||||
|
||||
```text
|
||||
versioned regulatory corpus JSON
|
||||
|
|
@ -17,21 +17,22 @@ versioned regulatory corpus JSON
|
|||
approved multilingual embedding service
|
||||
| dense vectors
|
||||
v
|
||||
Qdrant assessor_regulatory_chunks_v1
|
||||
Qdrant assessor_regulatory_chunks_v2
|
||||
^
|
||||
scanner finding -> public-signal query builder -> same embedding model -> dense candidates
|
||||
-> group top 32 chunks into top 8 controls
|
||||
-> optional future hybrid fusion -> optional reranker
|
||||
-> later constrained mapper -> PostgreSQL audit record
|
||||
-> constrained mapper -> human decision -> PostgreSQL audit record
|
||||
```
|
||||
|
||||
## Collection contract
|
||||
|
||||
The collection is `assessor_regulatory_chunks_v1`. It has two named vector slots:
|
||||
The active collection is `assessor_regulatory_chunks_v2`. It has two named vector slots:
|
||||
|
||||
- `dense`: the configured embedding dimension with cosine distance.
|
||||
- `sparse`: reserved now so a later sparse/BM25-style encoder can be added without changing the retrieval abstractions.
|
||||
|
||||
Qdrant payloads use schema version 1 and contain:
|
||||
Qdrant payloads use schema version 2 and contain:
|
||||
|
||||
| Field | Purpose |
|
||||
|---|---|
|
||||
|
|
@ -44,32 +45,21 @@ Qdrant payloads use schema version 1 and contain:
|
|||
| `title`, `text`, `text_sha256` | Retrieval content and integrity metadata |
|
||||
| `embedding_model` | Prevents querying vectors from a different model space |
|
||||
| `source_uri` | Optional public/source reference |
|
||||
| `annex_a_control_id`, `control_theme` | Shared `5.1`–`8.34` identity, Annex A alias, and theme |
|
||||
| `source_standard`, `source_edition`, `national_edition` | ISO/base and DIN source identity |
|
||||
| `section_kind`, `source_page_start`, `source_page_end` | Section-safe retrieval and page provenance |
|
||||
| `retrieval_role` | `semantic` for ISO 27002 or `reference` for ISO 27001 |
|
||||
| `parser_version`, `source_sha256` | Reproducible parser/source provenance |
|
||||
|
||||
All fields used by filters have Qdrant payload indexes. An existing collection with a different dense dimension, distance, or missing sparse slot fails closed and must be rebuilt from the corpus.
|
||||
|
||||
## Corpus format and ingestion
|
||||
## ISO corpus preparation and ingestion
|
||||
|
||||
The CLI accepts a UTF-8 JSON file of no more than 64 MiB:
|
||||
The server-only PDF.js CLI accepts explicit ISO 27001 and ISO 27002 paths and writes corpus-v2 JSON below the ignored `.assessor-data/regulatory-corpora/` directory:
|
||||
|
||||
```json
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"documents": [
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "synthetic-framework",
|
||||
"corpusVersion": "2026-09-15",
|
||||
"documentId": "synthetic-control-1-de",
|
||||
"framework": "Synthetic Framework",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "CTRL-1",
|
||||
"language": "de",
|
||||
"title": "Synthetic control title",
|
||||
"text": "Synthetic control text used only to demonstrate the contract.",
|
||||
"sourceUri": "https://example.invalid/framework/control-1"
|
||||
}
|
||||
]
|
||||
}
|
||||
```text
|
||||
npm run rag:prepare-iso -- "ISO27001 2024.PDF" "ISO27002 2024.PDF"
|
||||
npm run rag:ingest -- .assessor-data/regulatory-corpora/iso-27001-27002-de.json
|
||||
```
|
||||
|
||||
The checked-in [`tests/fixtures/regulatory-corpus.synthetic.json`](../tests/fixtures/regulatory-corpus.synthetic.json) is an unlicensed synthetic EN/DE smoke corpus. It exists to verify rebuilding and retrieval mechanics only; it is not a regulatory source and must never be presented as compliance guidance.
|
||||
|
|
@ -81,11 +71,11 @@ npm run db:up
|
|||
npm run rag:ingest -- path/to/regulatory-corpus.json
|
||||
```
|
||||
|
||||
Ingestion validates every document, splits text deterministically on paragraph/sentence/word boundaries, embeds all chunks in bounded batches, and only then replaces the indexed chunks for each `(corpusId, documentId)`. Replacement is not a cross-system transaction: if Qdrant fails after deletion, rerun ingestion from the unchanged corpus.
|
||||
The parser accepts encrypted documents only when no password is required and copying is permitted. It rejects scanned/empty, oversized, copy-prohibited, structurally unexpected, incomplete, duplicate, or mismatched catalogs. The build must contain the matching 93 controls with theme counts 37/8/14/34. It retains hashes, editions, control/section metadata, and page provenance without committing licensed text. Ingestion splits only inside a control section, embeds all chunks in bounded batches, and then replaces indexed chunks per `(corpusId, documentId)`. If Qdrant fails after deletion, rerun ingestion from the unchanged corpus JSON.
|
||||
|
||||
## Configuration and trust
|
||||
|
||||
Qdrant 1.19.1 runs in Docker with a persistent named volume, an API key, and outbound usage telemetry disabled. Local REST access is loopback-only; deployment access stays on the Compose network. Remote plain HTTP Qdrant URLs are rejected.
|
||||
Qdrant 1.19.1 runs in Docker with a persistent named volume and outbound usage telemetry disabled. Local REST access is loopback-only; deployment access stays on the Compose network. An API key is optional for an unsecured loopback/internal self-host and required for remote hosts. Remote plain HTTP Qdrant URLs are rejected.
|
||||
|
||||
The embedding adapter uses an OpenAI-compatible `/v1/embeddings` endpoint. Remote endpoints must use HTTPS. Plain HTTP is accepted only for `localhost`, `127.0.0.1`, or `::1`, enabling a same-machine llama.cpp development server without weakening remote transport. Configuration must explicitly provide the model name and its exact native/requested dimensions. Redirects are rejected, response size and time are bounded, and API keys remain in backend environment variables.
|
||||
|
||||
|
|
@ -107,29 +97,44 @@ ASSESSOR_RAG_EMBEDDING_MODEL=bge-m3
|
|||
ASSESSOR_RAG_EMBEDDING_API_KEY=<same-local-secret>
|
||||
ASSESSOR_RAG_EMBEDDING_DIMENSIONS=1024
|
||||
ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS=false
|
||||
ASSESSOR_RAG_ACTIVE_CORPUS_ID=iso-27001-27002-de
|
||||
ASSESSOR_RAG_ACTIVE_CORPUS_VERSION=<generated-corpus-version>
|
||||
```
|
||||
|
||||
For a later LiteLLM comparison with `text-embedding-3-large`, use the approved HTTPS gateway and set `ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS=true` if requesting 1,024-dimensional output. OpenAI supports the `dimensions` parameter for `text-embedding-3` models; llama.cpp currently returns the model's native dimension and should leave this flag false. Changing model identity or dimensions requires rebuilding the derived Qdrant corpus before comparing results.
|
||||
|
||||
Embedding is an external processing boundary. Regulatory text is sent during ingestion. The finding retrieval service can now send only the minimized public-signal query described below; unrestricted report text, paths, tenant identities, reviews, raw evidence, and local provenance IDs remain outside the embedding request. No browser endpoint or automatic mapping workflow activates this path.
|
||||
Embedding is an external processing boundary. Regulatory text is sent during ingestion. The finding retrieval service sends only the privacy-safe check query described below; the result table, paths, tenant identities, reviews, raw evidence, and local provenance IDs remain outside the embedding and completion requests.
|
||||
|
||||
## Finding query privacy boundary
|
||||
|
||||
`buildRegulatoryFindingQuery` accepts the existing `UnifiedResultProjection` but emits only fixed public concepts:
|
||||
`buildRegulatoryFindingQuery` accepts the existing `UnifiedResultProjection` and builds the query from the **shared privacy-safe finding projection** (`src/domain/privacy-safe-finding.ts`, `privacySafeFinding`) — the identical boundary used by the AI overview:
|
||||
|
||||
- exact Maester or PingCastle rule IDs that already exist in the checked-in public catalog;
|
||||
- fixed public technical topic labels recognized locally from title, description, or recommendation; and
|
||||
- a fixed public scanner name for known profiles.
|
||||
- the rule-level **check text** — title, plus description and recommendation when present — which comes from the deduplicated finding *definition* (canonical key `tool:ruleId`) and describes the check, not the tenant; email addresses are redacted;
|
||||
- a catalog Maester/PingCastle rule ID, a fixed public scanner name, and recognized public technical topics, all as optional query **enrichment**.
|
||||
|
||||
The query never copies raw title, description, recommendation, asset, source path, project/tenant identity, review owner, review text, original values, or provenance IDs. Unsupported result kinds and scanner findings without a recognized public rule or topic fail closed as `insufficient-public-context` and do not call the retriever. The local `sourceResultId` is returned only for in-process correlation and is not included in the embedding request.
|
||||
The query never reads the **result table / occurrence data**: affected asset, source locator/path, per-object values, project/tenant identity, review owner, review text, original values, or provenance IDs. Because the query is built from the rule-level check text, **every scanner finding with a check title maps**; only non-scanner records and findings without a check title fail closed as `insufficient-public-context` and do not call the retriever. The local `sourceResultId` is returned only for in-process correlation and is not included in the embedding request.
|
||||
|
||||
This is a technical minimization boundary, not organizational approval to process production findings externally. The service remains backend-only and has no API/UI route. Any production activation still requires approval of the embedding destination, catalog coverage, retention terms, and the precise finding classes allowed to cross that boundary.
|
||||
This is a technical minimization boundary. Activation still depends on the approved embedding/LiteLLM destinations, catalog coverage, retention terms, and the precise finding classes allowed to cross that boundary. Residual note: `privacySafeFinding` redacts emails only; it relies on definitions being rule-generic, so a scanner that folds an identifier (host, IP, path) into a *definition* title/description would carry it into the query. Strengthening that redaction is a shared decision affecting both AI features (see implementation plan).
|
||||
|
||||
## One-finding mapping workflow
|
||||
|
||||
The Work finding reader loads mapping state only after `Map ISO controls` is opened. Mappings are **global**: the finding contributes only its non-personal rule identity and check context, and the mapping is keyed by that identity plus the pipeline versions, so one rule is mapped once and every occurrence in every project reuses it. The API returns HTTP 202 and the UI polls while queued/running. The service retrieves 32 eligible ISO 27002 chunks, groups by maximum dense score, keeps eight controls, attaches up to two semantic passages and the matching ISO 27001 statement, and bounds the LiteLLM payload to 48,000 characters.
|
||||
|
||||
The completion schema permits at most five `primary` or `supporting` proposals, advisory low/medium/high confidence, a bounded rationale, grounded evidence IDs, or an explicit `no_relevant_control`/`insufficient_context` outcome. Unknown controls, foreign evidence IDs, duplicates, malformed JSON, and oversized content fail closed. A reviewer can accept or reject any grounded top-eight candidate with an optional comment; each change appends a **global** decision event keyed by rule identity and control, using optimistic concurrency, shared across every project. Acceptance means relevant control, never compliance.
|
||||
|
||||
Routes:
|
||||
|
||||
- `GET /api/v1/projects/{projectId}/regulatory-mappings/{resultProjectionId}`
|
||||
- `POST /api/v1/projects/{projectId}/regulatory-mapping-runs`
|
||||
- `POST /api/v1/projects/{projectId}/regulatory-mapping-decisions`
|
||||
|
||||
On the consolidated migration line (2026-10-02), the single migration 026 `regulatory-mapping` creates the global tables directly, together with the project-scoped `assessor_regulatory_mapping_batches`; the superseded occurrence-bound tables are never created. The remainder of this paragraph records the original history on the former experimental line, where migration 025 originally added occurrence-bound, project-scoped run and decision tables and migration 026 superseded it: it dropped those tables and created the global `assessor_regulatory_mappings` (keyed by `rule_key` + pipeline versions, `UNIQUE(rule_key, corpus_version, retrieval_version, prompt_version, model)`) and a global, rule-keyed `assessor_regulatory_mapping_decisions`; neither carries `project_id` or occurrence identity, and both survive project/finding deletion. Before 026 was applied, the 025 tables held only regenerable local test data (19 runs, 10 decisions); a verified backup of both tables was taken at `backups/regulatory_mapping_025_backup_20260915-155410.sql` and the data was then intentionally discarded per the user's decision. A live development backend applied 026 through its own migration runner (checksum recorded and re-verified against the source file); no reimport is required because mapping runs are regenerable and no scanner data changed.
|
||||
|
||||
### Local Qdrant integration check
|
||||
|
||||
`tests/server/qdrant-regulatory-index.integration.test.ts` is opt-in and uses deterministic synthetic vectors. With `ASSESSOR_QDRANT_INTEGRATION=1` and the normal Qdrant/model identity environment variables, it verifies the real collection, payload indexes, document replacement, filters, and retrieval without calling an embedding provider. The standard suite skips it so tests do not unexpectedly mutate a developer's derived index.
|
||||
|
||||
On 2026-09-15 this check passed against Qdrant 1.19.1 in WSL: authenticated access returned HTTP 200, unauthenticated access returned HTTP 401, the collection was green, Dense used 1,024-dimensional cosine vectors, the Sparse slot existed, all nine filter payload indexes existed, and one synthetic integration point was retrieved and then removed. PostgreSQL was not accessed or changed.
|
||||
On 2026-09-15 this check passed against Qdrant 1.19.1 in WSL: the collection was green, Dense used 1,024-dimensional cosine vectors, the Sparse slot existed, all 12 filter payload indexes existed, and one synthetic integration point was retrieved and then removed.
|
||||
|
||||
## Dense retrieval evaluation
|
||||
|
||||
|
|
@ -175,7 +180,7 @@ Live smoke result on 2026-09-15 with local llama.cpp build 10472, BGE-M3 FP16, a
|
|||
- `RegulatoryReranker` is optional and receives only the bounded candidate set. Without one, dense ordering is retained.
|
||||
- `RegulatoryRetriever` is the consumer-facing contract. Mapping code should depend on it, not on Qdrant.
|
||||
|
||||
Before enabling automatic finding mapping, complete organizational privacy approval, broaden public-signal coverage only through reviewed catalogs, establish the real dense baseline, and add control-level grouping, hybrid evaluation, reranker evaluation, calibrated confidence thresholds, a constrained output schema, human-review states, and append-only PostgreSQL mapping provenance.
|
||||
Before assessment-wide batch mapping, broaden public-signal coverage only through reviewed catalogs, establish a representative independently labeled EN/DE baseline, and evaluate sparse/hybrid retrieval, reranking, and confidence calibration. The one-finding workflow must remain human-reviewed.
|
||||
|
||||
## Reference basis
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
"dependencies": {
|
||||
"@fontsource-variable/inter": "^5.3.0",
|
||||
"jsdom": "^29.1.1",
|
||||
"pdfjs-dist": "^5.4.296",
|
||||
"pg": "^8.22.0",
|
||||
"react": "19.2.4",
|
||||
"react-dom": "19.2.4",
|
||||
|
|
@ -1021,6 +1022,271 @@
|
|||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz",
|
||||
"integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"workspaces": [
|
||||
"e2e/*"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "0.1.100",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.100",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.100",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.100",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.100",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "0.1.100",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.100"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz",
|
||||
"integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz",
|
||||
"integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz",
|
||||
"integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz",
|
||||
"integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz",
|
||||
"integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz",
|
||||
"integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz",
|
||||
"integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz",
|
||||
"integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "0.1.100",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz",
|
||||
"integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/pluginutils": {
|
||||
"version": "1.0.0-rc.2",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.2.tgz",
|
||||
|
|
@ -2281,6 +2547,18 @@
|
|||
"node": ">= 14.16"
|
||||
}
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "5.4.296",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz",
|
||||
"integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=20.16.0 || >=22.3.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^0.1.80"
|
||||
}
|
||||
},
|
||||
"node_modules/pg": {
|
||||
"version": "8.22.0",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
"db:down": "docker compose --env-file .env.postgres -f docker-compose.postgres.yml down",
|
||||
"db:logs": "docker compose --env-file .env.postgres -f docker-compose.postgres.yml logs -f postgres",
|
||||
"rag:ingest": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-ingest.mjs",
|
||||
"rag:prepare-iso": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-prepare-iso.mjs",
|
||||
"rag:evaluate": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-evaluate.mjs",
|
||||
"test:postgres": "vitest run --config vitest.config.ts tests/postgres/postgres-repository.integration.test.ts && vitest run --config vitest.config.ts tests/postgres/gpo-backup-management.integration.test.ts tests/postgres/policy-analysis.integration.test.ts",
|
||||
"test": "vitest run --config vitest.config.ts && node tests/local-only-check.mjs && node tests/design-language-check.mjs",
|
||||
|
|
@ -25,6 +26,7 @@
|
|||
"dependencies": {
|
||||
"@fontsource-variable/inter": "^5.3.0",
|
||||
"jsdom": "^29.1.1",
|
||||
"pdfjs-dist": "^5.4.296",
|
||||
"pg": "^8.22.0",
|
||||
"react": "19.2.4",
|
||||
"react-dom": "19.2.4",
|
||||
|
|
@ -41,5 +43,8 @@
|
|||
"typescript": "5.9.3",
|
||||
"vite": "^7.3.6",
|
||||
"vitest": "^3.2.7"
|
||||
},
|
||||
"allowScripts": {
|
||||
"esbuild@0.27.7": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ import type { GpoPolicyAnalysisOverviewService } from './gpo/gpo-policy-analysis
|
|||
import type { AiOverviewService } from './ai/ai-overview-service';
|
||||
import { AiOverviewServiceError } from './ai/ai-overview-service';
|
||||
import type { AiOverviewCreateRequest, AiOverviewScope } from '../src/domain/ai-overview';
|
||||
import type { RegulatoryMappingCreateRequest, RegulatoryMappingDecisionCreateRequest } from '../src/domain/regulatory-mapping';
|
||||
import type { RegulatoryMappingService } from './regulatory/regulatory-mapping-service';
|
||||
import { RegulatoryMappingServiceError } from './regulatory/regulatory-mapping-service';
|
||||
import type { GpoAutomaticAnalysisService } from './gpo/gpo-automatic-analysis-service';
|
||||
import type { MicrosoftSecurityBaselineSelectionContext } from '../src/domain/microsoft-security-baseline';
|
||||
import { listProjectGpoBackupStatus } from './gpo/gpo-backup-status-service';
|
||||
|
|
@ -161,6 +164,7 @@ export interface AssessorHttpServerOptions {
|
|||
/** Write-once report archive outside the database; report runs are refused without it. */
|
||||
reportArchive?: FileSystemReportArchive;
|
||||
aiOverviewService?: AiOverviewService;
|
||||
regulatoryMappingService?: RegulatoryMappingService;
|
||||
}
|
||||
|
||||
class HttpError extends Error {
|
||||
|
|
@ -372,6 +376,7 @@ function statusForError(error: unknown): { status: number; code: string; message
|
|||
if (error instanceof RepositoryValidationError) return { status: 400, code: 'repository_validation', message: error.message };
|
||||
if (error instanceof RiskAcceptanceValidationError) return { status: 400, code: 'risk_acceptance_validation', message: error.message };
|
||||
if (error instanceof AiOverviewServiceError) return { status: error.status, code: error.code, message: error.message };
|
||||
if (error instanceof RegulatoryMappingServiceError) return { status: error.status, code: error.code, message: error.message };
|
||||
return { status: 500, code: 'internal_error', message: 'Die Anfrage konnte nicht verarbeitet werden.' };
|
||||
}
|
||||
|
||||
|
|
@ -1056,6 +1061,20 @@ export function createAssessorHttpServer(options: AssessorHttpServerOptions): Se
|
|||
const body = await readJson<AiOverviewCreateRequest>(request, maximumBodyBytes);
|
||||
status = 202; sendJson(response, status, await options.aiOverviewService.start(segments[1], body), requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'regulatory-mappings' && segments.length === 4 && method === 'GET') {
|
||||
if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.');
|
||||
status = 200; sendJson(response, status, await options.regulatoryMappingService.get(segments[1], segments[3]), requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'regulatory-mapping-runs' && segments.length === 3 && method === 'POST') {
|
||||
if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.');
|
||||
const body = await readJson<RegulatoryMappingCreateRequest>(request, maximumBodyBytes);
|
||||
status = 202; sendJson(response, status, await options.regulatoryMappingService.start(segments[1], body), requestId); return;
|
||||
}
|
||||
if (segments[0] === 'projects' && segments[2] === 'regulatory-mapping-decisions' && segments.length === 3 && method === 'POST') {
|
||||
if (!options.regulatoryMappingService) throw new HttpError(503, 'regulatory_mapping_disabled', 'Regulatory mapping is not configured.');
|
||||
const body = await readJson<RegulatoryMappingDecisionCreateRequest>(request, maximumBodyBytes);
|
||||
status = 200; sendJson(response, status, await options.regulatoryMappingService.decide(segments[1], body), requestId); return;
|
||||
}
|
||||
if (segments.length === 1 && segments[0] === 'assessments' && method === 'POST') {
|
||||
const assessment = await readJson<Assessment>(request, maximumBodyBytes);
|
||||
await repository.saveAssessment(assessment);
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ import { FileSystemReportArchive, resolveReportArchiveRoot, type ReportArchiveEn
|
|||
import { createPostgresReportReadRuntime, memoryReportReadRuntime, type ReportReadRuntime } from './reporting/report-read-runtime';
|
||||
import { createLiteLlmClientFromEnvironment } from './ai/litellm-client';
|
||||
import { AiOverviewService } from './ai/ai-overview-service';
|
||||
import { createRegulatoryRagRuntime } from './regulatory/regulatory-rag-runtime';
|
||||
import { RegulatoryMappingService } from './regulatory/regulatory-mapping-service';
|
||||
|
||||
function boundedInteger(value: string | undefined, fallback: number, minimum: number, maximum: number, name: string): number {
|
||||
if (value === undefined || value.trim() === '') return fallback;
|
||||
|
|
@ -75,13 +77,35 @@ async function main(): Promise<void> {
|
|||
reportArchive = undefined;
|
||||
console.warn('Report Engine nicht verfügbar:', error instanceof Error ? error.message : 'Unbekannter Fehler');
|
||||
}
|
||||
const aiClient = createLiteLlmClientFromEnvironment();
|
||||
const aiOverviewService = new AiOverviewService(runtime.repository, {
|
||||
client: createLiteLlmClientFromEnvironment(),
|
||||
client: aiClient,
|
||||
maximumChunkCharacters: boundedInteger(process.env.ASSESSOR_AI_CHUNK_CHARS, 80_000, 10_000, 500_000, 'ASSESSOR_AI_CHUNK_CHARS'),
|
||||
maximumCalls: boundedInteger(process.env.ASSESSOR_AI_MAX_CALLS, 64, 1, 64, 'ASSESSOR_AI_MAX_CALLS'),
|
||||
leaseDurationMs: boundedInteger(process.env.ASSESSOR_AI_JOB_LEASE_MS, 5 * 60_000, 10_000, 30 * 60_000, 'ASSESSOR_AI_JOB_LEASE_MS')
|
||||
});
|
||||
await aiOverviewService.recoverInterruptedRuns();
|
||||
let regulatoryMappingService: RegulatoryMappingService;
|
||||
try {
|
||||
const corpusId = process.env.ASSESSOR_RAG_ACTIVE_CORPUS_ID?.trim();
|
||||
const corpusVersion = process.env.ASSESSOR_RAG_ACTIVE_CORPUS_VERSION?.trim();
|
||||
if (!corpusId || !corpusVersion) throw new Error('No active regulatory corpus is configured.');
|
||||
const rag = createRegulatoryRagRuntime();
|
||||
regulatoryMappingService = new RegulatoryMappingService(runtime.repository, {
|
||||
findingRetrieval: rag.findingRetrieval,
|
||||
index: rag.index,
|
||||
embeddingModel: rag.embeddingModel,
|
||||
client: aiClient,
|
||||
corpusId,
|
||||
corpusVersion,
|
||||
...(!aiClient ? { disabledReason: 'Work LiteLLM is not configured.' } : {})
|
||||
});
|
||||
} catch (error) {
|
||||
regulatoryMappingService = new RegulatoryMappingService(runtime.repository, {
|
||||
disabledReason: error instanceof Error ? error.message : 'Regulatory mapping is unavailable.'
|
||||
});
|
||||
}
|
||||
await regulatoryMappingService.recoverInterruptedRuns();
|
||||
const artifactStore = new FileSystemBackendArtifactStore(artifactRoot);
|
||||
const mitreCatalogService = new MitreCatalogService(join(artifactRoot, 'reference-data', 'mitre-enterprise.json'));
|
||||
const gpoBackupSubmissionService = new GpoBackupSubmissionService(
|
||||
|
|
@ -143,6 +167,7 @@ async function main(): Promise<void> {
|
|||
gpoPolicyFindingService,
|
||||
gpoPolicyAnalysisOverviewService,
|
||||
aiOverviewService,
|
||||
regulatoryMappingService,
|
||||
automaticGpoAnalysis,
|
||||
importExecutor
|
||||
});
|
||||
|
|
@ -158,7 +183,7 @@ async function main(): Promise<void> {
|
|||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
console.log(`${signal}: ASSESSOR wird beendet.`);
|
||||
void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop()]).then(() => {
|
||||
void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop(), aiOverviewService.close(), regulatoryMappingService.close()]).then(() => {
|
||||
server.close(async error => {
|
||||
try {
|
||||
await reportRuntime?.close();
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import baselineDrivenPolicyEvaluation from './migrations/020_baseline_driven_pol
|
|||
import referenceRecognition from './migrations/023_reference_recognition.sql?raw';
|
||||
import resultDbMaterialization from './migrations/024_result_db_materialization.sql?raw';
|
||||
import aiOverviewRuns from './migrations/025_ai_overview_runs.sql?raw';
|
||||
import regulatoryMapping from './migrations/026_regulatory_mapping.sql?raw';
|
||||
|
||||
export interface DatabaseMigration {
|
||||
version: string;
|
||||
|
|
@ -51,5 +52,6 @@ export const databaseMigrations: readonly DatabaseMigration[] = [
|
|||
{ version: '020', name: 'baseline-driven-policy-evaluation', sql: baselineDrivenPolicyEvaluation },
|
||||
{ version: '023', name: 'reference-recognition', sql: referenceRecognition },
|
||||
{ version: '024', name: 'result-db-materialization', sql: resultDbMaterialization },
|
||||
{ version: '025', name: 'ai-overview-runs', sql: aiOverviewRuns }
|
||||
{ version: '025', name: 'ai-overview-runs', sql: aiOverviewRuns },
|
||||
{ version: '026', name: 'regulatory-mapping', sql: regulatoryMapping }
|
||||
];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
-- Regulatory ISO mappings are global, tenant-independent knowledge keyed by rule
|
||||
-- identity plus pipeline versions (see docs/REGULATORY_RAG.md). Batches are
|
||||
-- project-scoped runs over a filtered finding set.
|
||||
|
||||
CREATE TABLE assessor_regulatory_mappings (
|
||||
id text PRIMARY KEY,
|
||||
rule_key text NOT NULL,
|
||||
source_tool text NOT NULL,
|
||||
source_rule_id text,
|
||||
run_status text NOT NULL CHECK (run_status IN ('queued', 'running', 'succeeded', 'failed')),
|
||||
query_hash text NOT NULL CHECK (query_hash ~ '^[a-f0-9]{64}$'),
|
||||
corpus_id text NOT NULL,
|
||||
corpus_version text NOT NULL,
|
||||
embedding_model text NOT NULL,
|
||||
retrieval_version text NOT NULL,
|
||||
prompt_version text NOT NULL,
|
||||
output_schema_version integer NOT NULL,
|
||||
model text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
data jsonb NOT NULL,
|
||||
-- The id is a deterministic digest of (rule_key, corpus/retrieval/prompt versions, model),
|
||||
-- so there is exactly one row per rule per pipeline version tuple.
|
||||
CONSTRAINT assessor_regulatory_mappings_identity_uq
|
||||
UNIQUE (rule_key, corpus_version, retrieval_version, prompt_version, model)
|
||||
);
|
||||
|
||||
CREATE INDEX assessor_regulatory_mappings_rule_idx
|
||||
ON assessor_regulatory_mappings (rule_key, updated_at DESC, id DESC);
|
||||
|
||||
CREATE TABLE assessor_regulatory_mapping_decisions (
|
||||
id text PRIMARY KEY,
|
||||
rule_key text NOT NULL,
|
||||
source_tool text NOT NULL,
|
||||
source_rule_id text,
|
||||
mapping_id text NOT NULL,
|
||||
control_id text NOT NULL,
|
||||
decision text NOT NULL CHECK (decision IN ('accepted', 'rejected')),
|
||||
previous_decision_id text REFERENCES assessor_regulatory_mapping_decisions(id),
|
||||
created_at timestamptz NOT NULL,
|
||||
data jsonb NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX assessor_regulatory_mapping_decisions_rule_control_idx
|
||||
ON assessor_regulatory_mapping_decisions (rule_key, control_id, created_at DESC, id DESC);
|
||||
|
||||
CREATE TABLE assessor_regulatory_mapping_batches (
|
||||
id text PRIMARY KEY,
|
||||
project_id text NOT NULL REFERENCES assessor_projects(id) ON DELETE CASCADE,
|
||||
scope_key text NOT NULL,
|
||||
filter_hash text NOT NULL CHECK (filter_hash ~ '^[a-f0-9]{64}$'),
|
||||
idempotency_key text NOT NULL,
|
||||
run_status text NOT NULL CHECK (run_status IN ('queued', 'running', 'completed', 'failed')),
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
data jsonb NOT NULL,
|
||||
CONSTRAINT assessor_regulatory_mapping_batches_idempotency_uq UNIQUE (project_id, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX assessor_regulatory_mapping_batches_scope_idx
|
||||
ON assessor_regulatory_mapping_batches (project_id, scope_key, created_at DESC, id DESC);
|
||||
|
||||
CREATE UNIQUE INDEX assessor_regulatory_mapping_batches_active_scope_uq
|
||||
ON assessor_regulatory_mapping_batches (project_id, scope_key)
|
||||
WHERE run_status IN ('queued', 'running');
|
||||
|
|
@ -27,6 +27,8 @@ import type { EnvironmentObservation, EnvironmentSnapshot, EnvironmentSnapshotQu
|
|||
import { environmentCoverage } from '../../src/domain/environment-context';
|
||||
import type { AiOverviewRun } from '../../src/domain/ai-overview';
|
||||
import { aiOverviewRunValidationErrors } from '../../src/domain/ai-overview';
|
||||
import type { RegulatoryMappingDecision, RegulatoryMappingRun } from '../../src/domain/regulatory-mapping';
|
||||
import { regulatoryMappingDecisionValidationErrors, regulatoryMappingRunValidationErrors } from '../../src/domain/regulatory-mapping';
|
||||
import { storedPolicyFindingRunValidationErrors } from '../../src/domain/policy-finding';
|
||||
import { latestReferenceRuns, type ReferenceRecognitionRun, type ReferenceCatalogSnapshot } from '../../src/domain/reference-recognition';
|
||||
import { validateReferenceRun } from '../../src/persistence/reference-run-validation';
|
||||
|
|
@ -373,6 +375,8 @@ const DB_STRUCTURE_LAYER_BY_TABLE: Record<string, DbStructureLayerId> = {
|
|||
assessor_asset_edges: 'canonical',
|
||||
assessor_edge_coverage: 'canonical',
|
||||
assessor_finding_definitions: 'result',
|
||||
assessor_regulatory_mappings: 'result',
|
||||
assessor_regulatory_mapping_decisions: 'governance',
|
||||
assessor_ai_overview_runs: 'governance',
|
||||
assessor_finding_occurrences: 'result',
|
||||
assessor_result_categories: 'result',
|
||||
|
|
@ -686,6 +690,79 @@ export class PostgresAssessorRepository implements AssessorRepository {
|
|||
`, [projectId, scopeKey ?? null]));
|
||||
}
|
||||
|
||||
async getRegulatoryMappingRun(id: string): Promise<RegulatoryMappingRun | null> {
|
||||
return dataRow(await this.pool.query<DataRow<RegulatoryMappingRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mappings WHERE id = $1', [id]
|
||||
));
|
||||
}
|
||||
|
||||
async saveRegulatoryMappingRun(run: RegulatoryMappingRun): Promise<RegulatoryMappingRun> {
|
||||
const errors = regulatoryMappingRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.mutate(async client => {
|
||||
await client.query(`
|
||||
INSERT INTO assessor_regulatory_mappings
|
||||
(id, rule_key, source_tool, source_rule_id, run_status, query_hash, corpus_id, corpus_version,
|
||||
embedding_model, retrieval_version, prompt_version, output_schema_version, model, created_at, updated_at, data)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
run_status = excluded.run_status, updated_at = excluded.updated_at, data = excluded.data
|
||||
`, [run.id, run.ruleKey, run.sourceTool, run.sourceRuleId ?? null, run.status, run.queryHash, run.corpusId,
|
||||
run.corpusVersion, run.embeddingModel, run.retrievalVersion, run.promptVersion, run.outputSchemaVersion,
|
||||
run.model, run.createdAt, run.updatedAt, json(run)]);
|
||||
return clone(run);
|
||||
});
|
||||
}
|
||||
|
||||
async listRegulatoryMappingRuns(): Promise<RegulatoryMappingRun[]> {
|
||||
return dataRows(await this.pool.query<DataRow<RegulatoryMappingRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mappings ORDER BY created_at DESC, id DESC'
|
||||
));
|
||||
}
|
||||
|
||||
async createRegulatoryMappingDecision(
|
||||
decision: RegulatoryMappingDecision,
|
||||
expectedLatestDecisionId?: string
|
||||
): Promise<RegulatoryMappingDecision> {
|
||||
const errors = regulatoryMappingDecisionValidationErrors(decision);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
return this.mutate(async client => {
|
||||
const run = dataRow(await client.query<DataRow<RegulatoryMappingRun>>(
|
||||
'SELECT data FROM assessor_regulatory_mappings WHERE id = $1 FOR UPDATE', [decision.mappingId]
|
||||
));
|
||||
if (!run || run.ruleKey !== decision.ruleKey || run.status !== 'succeeded') {
|
||||
throw new RepositoryNotFoundError('Completed regulatory mapping does not exist for this rule.');
|
||||
}
|
||||
if (!run.candidates.some(candidate => candidate.controlId === decision.controlId)) {
|
||||
throw new RepositoryValidationError('Decision control was not a grounded candidate.');
|
||||
}
|
||||
const latest = dataRow(await client.query<DataRow<RegulatoryMappingDecision>>(`
|
||||
SELECT data FROM assessor_regulatory_mapping_decisions
|
||||
WHERE rule_key = $1 AND control_id = $2
|
||||
ORDER BY created_at DESC, id DESC LIMIT 1
|
||||
`, [decision.ruleKey, decision.controlId]));
|
||||
if ((expectedLatestDecisionId ?? undefined) !== (latest?.id ?? undefined)) {
|
||||
throw new RepositoryConflictError('Regulatory mapping decision changed before this update.');
|
||||
}
|
||||
const stored = { ...decision, ...(latest ? { previousDecisionId: latest.id } : {}) };
|
||||
await client.query(`
|
||||
INSERT INTO assessor_regulatory_mapping_decisions
|
||||
(id, rule_key, source_tool, source_rule_id, mapping_id, control_id, decision, previous_decision_id, created_at, data)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb)
|
||||
`, [stored.id, stored.ruleKey, stored.sourceTool, stored.sourceRuleId ?? null, stored.mappingId, stored.controlId,
|
||||
stored.decision, stored.previousDecisionId ?? null, stored.createdAt, json(stored)]);
|
||||
return clone(stored);
|
||||
});
|
||||
}
|
||||
|
||||
async listRegulatoryMappingDecisions(ruleKey: string): Promise<RegulatoryMappingDecision[]> {
|
||||
return dataRows(await this.pool.query<DataRow<RegulatoryMappingDecision>>(`
|
||||
SELECT data FROM assessor_regulatory_mapping_decisions
|
||||
WHERE rule_key = $1
|
||||
ORDER BY created_at DESC, id DESC
|
||||
`, [ruleKey]));
|
||||
}
|
||||
|
||||
async beginImport(start: ImportSessionStart): Promise<ImportSession> {
|
||||
if (!start.sessionId.trim()) throw new RepositoryValidationError('Eine Importsitzung benötigt eine ID.');
|
||||
if (this.activeImportSessions.has(start.sessionId)) throw new RepositoryConflictError(`Importsitzung ${start.sessionId} ist bereits aktiv.`);
|
||||
|
|
@ -2668,7 +2745,12 @@ export class PostgresAssessorRepository implements AssessorRepository {
|
|||
policySnapshotAggregations: await this.listPolicySnapshotAggregations(projectId),
|
||||
policyEvaluations: await this.listPolicyEvaluations(projectId),
|
||||
policyFindingRuns,
|
||||
aiOverviewRuns: (await this.listAiOverviewRuns(projectId)).filter(item => item.status === 'succeeded')
|
||||
aiOverviewRuns: (await this.listAiOverviewRuns(projectId)).filter(item => item.status === 'succeeded'),
|
||||
// Regulatory mappings and decisions are global reference data, not project-scoped.
|
||||
regulatoryMappingRuns: (await this.listRegulatoryMappingRuns()).filter(item => item.status === 'succeeded'),
|
||||
regulatoryMappingDecisions: dataRows(await this.pool.query<DataRow<RegulatoryMappingDecision>>(
|
||||
'SELECT data FROM assessor_regulatory_mapping_decisions ORDER BY created_at, id'
|
||||
))
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,334 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { getDocument, PermissionFlag } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
||||
import type {
|
||||
RegulatoryControlDocument,
|
||||
RegulatoryControlTheme,
|
||||
RegulatoryCorpusV2,
|
||||
RegulatorySectionKind
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
|
||||
export const ISO_PDF_PARSER_VERSION = '1.0.0';
|
||||
const MAXIMUM_PDF_BYTES = 32 * 1024 * 1024;
|
||||
const EXPECTED_THEME_COUNTS: Record<RegulatoryControlTheme, number> = {
|
||||
organizational: 37,
|
||||
people: 8,
|
||||
physical: 14,
|
||||
technological: 34
|
||||
};
|
||||
|
||||
export interface PdfSpan {
|
||||
text: string;
|
||||
page: number;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
interface PdfJsTextItem {
|
||||
str: string;
|
||||
transform: unknown[];
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface PdfLine {
|
||||
text: string;
|
||||
page: number;
|
||||
y: number;
|
||||
spans: PdfSpan[];
|
||||
}
|
||||
|
||||
export interface ParsedPdf {
|
||||
sha256: string;
|
||||
pageCount: number;
|
||||
pages: Array<{ page: number; lines: PdfLine[] }>;
|
||||
}
|
||||
|
||||
export interface ParsedSection {
|
||||
kind: RegulatorySectionKind;
|
||||
text: string;
|
||||
pageStart: number;
|
||||
pageEnd: number;
|
||||
}
|
||||
|
||||
export interface ParsedControl {
|
||||
controlId: string;
|
||||
title: string;
|
||||
theme: RegulatoryControlTheme;
|
||||
sections: ParsedSection[];
|
||||
}
|
||||
|
||||
export interface IsoPdfCorpusOptions {
|
||||
iso27001Path: string;
|
||||
iso27002Path: string;
|
||||
corpusId?: string;
|
||||
corpusVersion?: string;
|
||||
generatedAt?: string;
|
||||
}
|
||||
|
||||
export async function buildIsoPdfCorpus(options: IsoPdfCorpusOptions): Promise<RegulatoryCorpusV2> {
|
||||
const [iso27001, iso27002] = await Promise.all([
|
||||
extractPdf(options.iso27001Path, 'ISO/IEC 27001'),
|
||||
extractPdf(options.iso27002Path, 'ISO/IEC 27002')
|
||||
]);
|
||||
const references = parseIso27001AnnexA(iso27001);
|
||||
const semanticControls = parseIso27002Controls(iso27002);
|
||||
validateIsoControlSets(references, semanticControls);
|
||||
const corpusId = options.corpusId?.trim() || 'iso-27001-27002-de';
|
||||
const combinedHash = sha256(`${iso27001.sha256}\n${iso27002.sha256}`);
|
||||
const corpusVersion = options.corpusVersion?.trim() || `din-2024-de-${combinedHash.slice(0, 12)}`;
|
||||
const common = { corpusId, corpusVersion, language: 'de', parserVersion: ISO_PDF_PARSER_VERSION } as const;
|
||||
const documents: RegulatoryControlDocument[] = [];
|
||||
|
||||
for (const control of semanticControls) {
|
||||
const reference = references.find(item => item.controlId === control.controlId)!;
|
||||
documents.push(documentForSection(common, control, reference.sections[0], {
|
||||
sourceStandard: 'ISO/IEC 27001', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27001:2024-01',
|
||||
sourceSha256: iso27001.sha256, retrievalRole: 'reference'
|
||||
}));
|
||||
for (const section of control.sections) {
|
||||
documents.push(documentForSection(common, control, section, {
|
||||
sourceStandard: 'ISO/IEC 27002', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27002:2024-01',
|
||||
sourceSha256: iso27002.sha256, retrievalRole: 'semantic'
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
corpusId,
|
||||
corpusVersion,
|
||||
language: 'de',
|
||||
parserVersion: ISO_PDF_PARSER_VERSION,
|
||||
generatedAt: options.generatedAt ?? new Date().toISOString(),
|
||||
sources: [
|
||||
{ standard: 'ISO/IEC 27001', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27001:2024-01', sha256: iso27001.sha256, pageCount: iso27001.pageCount },
|
||||
{ standard: 'ISO/IEC 27002', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27002:2024-01', sha256: iso27002.sha256, pageCount: iso27002.pageCount }
|
||||
],
|
||||
documents
|
||||
};
|
||||
}
|
||||
|
||||
function documentForSection(
|
||||
common: { corpusId: string; corpusVersion: string; language: string; parserVersion: string },
|
||||
control: ParsedControl,
|
||||
section: ParsedSection,
|
||||
source: Pick<RegulatoryControlDocument, 'sourceStandard' | 'sourceEdition' | 'nationalEdition' | 'sourceSha256' | 'retrievalRole'>
|
||||
): RegulatoryControlDocument {
|
||||
const sourceToken = source.sourceStandard === 'ISO/IEC 27001' ? '27001' : '27002';
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
...common,
|
||||
documentId: `${common.corpusId}:${control.controlId}:${sourceToken}:${section.kind}`,
|
||||
framework: 'ISO/IEC 27001 Annex A',
|
||||
frameworkVersion: '2022',
|
||||
controlId: control.controlId,
|
||||
annexAControlId: `A.${control.controlId}`,
|
||||
controlTheme: control.theme,
|
||||
title: control.title,
|
||||
text: section.text,
|
||||
sectionKind: section.kind,
|
||||
sourcePageStart: section.pageStart,
|
||||
sourcePageEnd: section.pageEnd,
|
||||
...source
|
||||
};
|
||||
}
|
||||
|
||||
async function extractPdf(path: string, expectedStandard: string): Promise<ParsedPdf> {
|
||||
const file = await stat(path);
|
||||
if (!file.isFile() || file.size < 1 || file.size > MAXIMUM_PDF_BYTES) {
|
||||
throw new Error(`${expectedStandard} PDF must be a non-empty file no larger than 32 MiB.`);
|
||||
}
|
||||
const bytes = await readFile(path);
|
||||
let document;
|
||||
try {
|
||||
document = await getDocument({ data: new Uint8Array(bytes), password: '', isEvalSupported: false, useSystemFonts: true }).promise;
|
||||
} catch {
|
||||
throw new Error(`${expectedStandard} PDF could not be opened without a password.`);
|
||||
}
|
||||
try {
|
||||
const permissions = await document.getPermissions();
|
||||
if (permissions && !permissions.includes(PermissionFlag.COPY)) throw new Error(`${expectedStandard} PDF does not permit text extraction.`);
|
||||
if (document.numPages < 1 || document.numPages > 1_000) throw new Error(`${expectedStandard} PDF page count is outside the supported range.`);
|
||||
const pages: ParsedPdf['pages'] = [];
|
||||
let characterCount = 0;
|
||||
for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {
|
||||
const page = await document.getPage(pageNumber);
|
||||
const viewport = page.getViewport({ scale: 1 });
|
||||
const content = await page.getTextContent({ disableNormalization: false });
|
||||
const spans = content.items.flatMap(item => toSpan(item, pageNumber, viewport.height));
|
||||
const lines = groupLines(spans);
|
||||
characterCount += lines.reduce((sum, line) => sum + line.text.length, 0);
|
||||
pages.push({ page: pageNumber, lines });
|
||||
page.cleanup();
|
||||
}
|
||||
if (characterCount < document.numPages * 100) throw new Error(`${expectedStandard} PDF does not contain a usable text layer.`);
|
||||
return { sha256: createHash('sha256').update(bytes).digest('hex'), pageCount: document.numPages, pages };
|
||||
} finally {
|
||||
await document.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
function toSpan(item: unknown, page: number, pageHeight: number): PdfSpan[] {
|
||||
if (!isTextItem(item)) return [];
|
||||
const [a, b, c, d, x, y] = item.transform.map(Number);
|
||||
if (![a, b, c, d, x, y].every(Number.isFinite) || Math.abs(b) > 0.2 || Math.abs(c) > 0.2) return [];
|
||||
if (y < 70 || y > pageHeight - 55) return [];
|
||||
const text = normalizeInline(item.str);
|
||||
return text ? [{ text, page, x, y, width: Number(item.width) || 0, height: Number(item.height) || Math.abs(d) }] : [];
|
||||
}
|
||||
|
||||
function groupLines(spans: PdfSpan[]): PdfLine[] {
|
||||
const rows: PdfSpan[][] = [];
|
||||
for (const span of spans.slice().sort((left, right) => right.y - left.y || left.x - right.x)) {
|
||||
const row = rows.find(candidate => Math.abs(candidate[0].y - span.y) <= Math.max(1.5, span.height * 0.12));
|
||||
if (row) row.push(span); else rows.push([span]);
|
||||
}
|
||||
return rows.map(row => {
|
||||
const ordered = row.sort((left, right) => left.x - right.x);
|
||||
return { page: ordered[0].page, y: ordered[0].y, spans: ordered, text: joinSpans(ordered) };
|
||||
}).filter(line => line.text);
|
||||
}
|
||||
|
||||
function joinSpans(spans: PdfSpan[]): string {
|
||||
let result = '';
|
||||
let previous: PdfSpan | undefined;
|
||||
for (const span of spans) {
|
||||
const gap = previous ? span.x - (previous.x + previous.width) : 0;
|
||||
const separator = result && !/\s$/.test(result) && !/^\s/.test(span.text) && gap > Math.max(1.2, span.height * 0.12) ? ' ' : '';
|
||||
result += separator + span.text;
|
||||
previous = span;
|
||||
}
|
||||
return normalizeInline(result);
|
||||
}
|
||||
|
||||
export function parseIso27002Controls(pdf: ParsedPdf): ParsedControl[] {
|
||||
const lines = pdf.pages.flatMap(page => page.lines);
|
||||
const starts: number[] = [];
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
if (!/^([5-8]\.\d{1,2})\s+\S/.test(lines[index].text)) continue;
|
||||
const nearby = new Set(lines.slice(index + 1, index + 35).map(line => line.text));
|
||||
if (nearby.has('Maßnahme') && nearby.has('Zweck') && (nearby.has('Anleitung') || nearby.has('Leitfaden'))) starts.push(index);
|
||||
}
|
||||
const uniqueStarts = starts.filter((start, index) => !starts.slice(0, index).some(other => controlId(lines[other].text) === controlId(lines[start].text)));
|
||||
return uniqueStarts.map((start, index) => parseIso27002Block(lines.slice(start, uniqueStarts[index + 1] ?? lines.length)));
|
||||
}
|
||||
|
||||
function parseIso27002Block(block: PdfLine[]): ParsedControl {
|
||||
const match = /^([5-8]\.\d{1,2})\s+(.+)$/.exec(block[0].text);
|
||||
if (!match) throw new Error('ISO/IEC 27002 control heading is malformed.');
|
||||
const labels: Array<[RegulatorySectionKind, readonly string[]]> = [
|
||||
['control', ['Maßnahme']], ['purpose', ['Zweck']], ['guidance', ['Anleitung', 'Leitfaden']], ['other-information', ['Weitere Informationen']]
|
||||
];
|
||||
const positions = labels.map(([kind, aliases]) => ({ kind, index: block.findIndex(line => aliases.includes(line.text)) }));
|
||||
if (positions.some(position => position.index < 0)) throw new Error(`ISO/IEC 27002 control ${match[1]} is missing a required section.`);
|
||||
const attributesAt = block.slice(1, positions[0].index).findIndex(line => /Maßnahmenart/.test(line.text));
|
||||
if (attributesAt < 0) throw new Error(`ISO/IEC 27002 control ${match[1]} is missing its attribute table.`);
|
||||
const titleLines = [match[2], ...block.slice(1, 1 + attributesAt).map(line => line.text)];
|
||||
const attributeLines = block.slice(1 + attributesAt, positions[0].index);
|
||||
const sections: ParsedSection[] = [sectionFromLines('attributes', attributeLines)];
|
||||
positions.forEach((position, positionIndex) => {
|
||||
const next = positions[positionIndex + 1]?.index ?? block.length;
|
||||
sections.push(sectionFromLines(position.kind, block.slice(position.index + 1, next)));
|
||||
});
|
||||
return { controlId: match[1], title: linesToText(titleLines), theme: themeFor(match[1]), sections };
|
||||
}
|
||||
|
||||
export function parseIso27001AnnexA(pdf: ParsedPdf): ParsedControl[] {
|
||||
const startPageIndex = pdf.pages.findIndex(page => page.lines.some(line => /Tabelle A\.1/.test(line.text))
|
||||
&& page.lines.flatMap(line => line.spans).filter(span => span.text === 'Maßnahme').length >= 3);
|
||||
if (startPageIndex < 0) throw new Error('ISO/IEC 27001 Annex A table was not found.');
|
||||
const controls: Array<{ controlId: string; pageStart: number; pageEnd: number; title: string[]; statement: string[] }> = [];
|
||||
let current: (typeof controls)[number] | undefined;
|
||||
for (const page of pdf.pages.slice(startPageIndex)) {
|
||||
const statementX = Math.min(...page.lines.flatMap(line => line.spans)
|
||||
.filter(span => span.text === 'Maßnahme').map(span => span.x));
|
||||
if (!Number.isFinite(statementX)) continue;
|
||||
for (const line of page.lines) {
|
||||
const idSpan = line.spans.find(span => span.x < 125 && /^[5-8]\.\d{1,2}$/.test(span.text));
|
||||
if (idSpan) {
|
||||
current = { controlId: idSpan.text, pageStart: page.page, pageEnd: page.page, title: [], statement: [] };
|
||||
controls.push(current);
|
||||
}
|
||||
if (!current) continue;
|
||||
current.pageEnd = page.page;
|
||||
for (const span of line.spans) {
|
||||
if (span === idSpan || span.text === 'Maßnahme' || /Tabelle A\.1|Informationssicherheitsmaßnahmen/.test(span.text)) continue;
|
||||
if (span.x >= 65 && span.x < statementX) current.title.push(span.text);
|
||||
else if (span.x >= statementX) current.statement.push(span.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
return controls.map(control => ({
|
||||
controlId: control.controlId,
|
||||
title: linesToText(control.title),
|
||||
theme: themeFor(control.controlId),
|
||||
sections: [{
|
||||
kind: 'control', text: linesToText(control.statement),
|
||||
pageStart: control.pageStart, pageEnd: control.pageEnd
|
||||
}]
|
||||
}));
|
||||
}
|
||||
|
||||
function sectionFromLines(kind: RegulatorySectionKind, lines: PdfLine[]): ParsedSection {
|
||||
const meaningful = lines.filter(line => !isFurniture(line.text));
|
||||
const text = linesToText(meaningful.map(line => line.text));
|
||||
if (!text) throw new Error(`ISO/IEC 27002 ${kind} section is empty.`);
|
||||
return { kind, text, pageStart: meaningful[0].page, pageEnd: meaningful.at(-1)!.page };
|
||||
}
|
||||
|
||||
export function validateIsoControlSets(references: ParsedControl[], semantic: ParsedControl[]): void {
|
||||
const referenceIds = references.map(item => item.controlId);
|
||||
const semanticIds = semantic.map(item => item.controlId);
|
||||
for (const [label, ids] of [['ISO/IEC 27001', referenceIds], ['ISO/IEC 27002', semanticIds]] as const) {
|
||||
if (ids.length !== 93 || new Set(ids).size !== ids.length) {
|
||||
throw new Error(`${label} must contain exactly 93 unique controls; parsed ${ids.length} entries and ${new Set(ids).size} unique identifiers.`);
|
||||
}
|
||||
const counts = Object.fromEntries(Object.keys(EXPECTED_THEME_COUNTS).map(theme => [theme, ids.filter(id => themeFor(id) === theme).length]));
|
||||
for (const [theme, expected] of Object.entries(EXPECTED_THEME_COUNTS)) {
|
||||
if (counts[theme] !== expected) throw new Error(`${label} ${theme} control count must be ${expected}.`);
|
||||
}
|
||||
}
|
||||
if (referenceIds.slice().sort().join('\n') !== semanticIds.slice().sort().join('\n')) {
|
||||
throw new Error('ISO/IEC 27001 Annex A and ISO/IEC 27002 control identifiers do not match.');
|
||||
}
|
||||
}
|
||||
|
||||
function themeFor(controlIdValue: string): RegulatoryControlTheme {
|
||||
if (controlIdValue.startsWith('5.')) return 'organizational';
|
||||
if (controlIdValue.startsWith('6.')) return 'people';
|
||||
if (controlIdValue.startsWith('7.')) return 'physical';
|
||||
if (controlIdValue.startsWith('8.')) return 'technological';
|
||||
throw new Error(`Unsupported ISO control identifier ${controlIdValue}.`);
|
||||
}
|
||||
|
||||
function controlId(value: string): string | undefined {
|
||||
return /^([5-8]\.\d{1,2})\s+/.exec(value)?.[1];
|
||||
}
|
||||
|
||||
function linesToText(lines: string[]): string {
|
||||
return lines.map(normalizeInline).filter(Boolean).join('\n')
|
||||
.replace(/(\p{L})-\n(?=\p{Ll})/gu, '$1')
|
||||
.replace(/\n+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function normalizeInline(value: string): string {
|
||||
return value.normalize('NFKC').replace(/\u00ad/g, '').replace(/[\u00a0\u2007\u202f]/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function isFurniture(value: string): boolean {
|
||||
return /^(?:DIN EN ISO\/IEC 2700[12]:|EN ISO\/IEC 2700[12]:|Normen-Download-|\d{1,3})/.test(value);
|
||||
}
|
||||
|
||||
function isTextItem(value: unknown): value is PdfJsTextItem {
|
||||
return Boolean(value) && typeof value === 'object' && typeof (value as PdfJsTextItem).str === 'string'
|
||||
&& Array.isArray((value as PdfJsTextItem).transform) && (value as PdfJsTextItem).transform.length === 6;
|
||||
}
|
||||
|
||||
function sha256(value: string): string {
|
||||
return createHash('sha256').update(value, 'utf8').digest('hex');
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import type {
|
|||
RegulatoryRetrievalFilters
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
|
||||
export const REGULATORY_QDRANT_COLLECTION = 'assessor_regulatory_chunks_v1';
|
||||
export const REGULATORY_QDRANT_COLLECTION = 'assessor_regulatory_chunks_v2';
|
||||
export const REGULATORY_DENSE_VECTOR = 'dense';
|
||||
export const REGULATORY_SPARSE_VECTOR = 'sparse';
|
||||
|
||||
|
|
@ -24,6 +24,7 @@ export interface RegulatoryChunkIndex {
|
|||
ensureCollection(): Promise<void>;
|
||||
replaceDocuments(chunks: IndexedRegulatoryChunk[]): Promise<void>;
|
||||
searchDense(vector: number[], limit: number, filters?: RegulatoryRetrievalFilters): Promise<RegulatoryRetrievalCandidate[]>;
|
||||
list(filters: RegulatoryRetrievalFilters, limit: number): Promise<RegulatoryChunk[]>;
|
||||
}
|
||||
|
||||
export class QdrantClientError extends Error {
|
||||
|
|
@ -35,7 +36,7 @@ export class QdrantClientError extends Error {
|
|||
|
||||
interface QdrantRegulatoryIndexOptions {
|
||||
baseUrl: URL;
|
||||
apiKey: string;
|
||||
apiKey?: string;
|
||||
dimensions: number;
|
||||
embeddingModel: string;
|
||||
timeoutMs: number;
|
||||
|
|
@ -51,7 +52,10 @@ const payloadIndexes = [
|
|||
['framework_version', 'keyword'],
|
||||
['control_id', 'keyword'],
|
||||
['language', 'keyword'],
|
||||
['embedding_model', 'keyword']
|
||||
['embedding_model', 'keyword'],
|
||||
['source_standard', 'keyword'],
|
||||
['section_kind', 'keyword'],
|
||||
['retrieval_role', 'keyword']
|
||||
] as const;
|
||||
|
||||
export class QdrantRegulatoryIndex implements RegulatoryChunkIndex {
|
||||
|
|
@ -129,6 +133,19 @@ export class QdrantRegulatoryIndex implements RegulatoryChunkIndex {
|
|||
return points.map(point => parseCandidate(point));
|
||||
}
|
||||
|
||||
async list(filters: RegulatoryRetrievalFilters, limit: number): Promise<RegulatoryChunk[]> {
|
||||
if (!Number.isInteger(limit) || limit < 1 || limit > 100) throw new Error('Regulatory list limit must be between 1 and 100.');
|
||||
const response = await this.request('POST', `/collections/${REGULATORY_QDRANT_COLLECTION}/points/scroll`, {
|
||||
filter: { must: [keywordCondition('embedding_model', this.options.embeddingModel), ...filterConditions(filters)] },
|
||||
limit,
|
||||
with_payload: true,
|
||||
with_vector: false
|
||||
});
|
||||
const points = object(response.body)?.result?.points;
|
||||
if (!Array.isArray(points)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant returned an invalid scroll response.');
|
||||
return points.map(point => parseChunkPayload(object(point)?.payload));
|
||||
}
|
||||
|
||||
private async request(method: string, path: string, body?: unknown, acceptedStatuses: number[] = [200]): Promise<{ status: number; body: unknown }> {
|
||||
const url = new URL(path, withTrailingSlash(this.options.baseUrl));
|
||||
let response: Response;
|
||||
|
|
@ -140,7 +157,7 @@ export class QdrantRegulatoryIndex implements RegulatoryChunkIndex {
|
|||
headers: {
|
||||
Accept: 'application/json',
|
||||
...(body === undefined ? {} : { 'Content-Type': 'application/json' }),
|
||||
'api-key': this.options.apiKey
|
||||
...(this.options.apiKey ? { 'api-key': this.options.apiKey } : {})
|
||||
},
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) })
|
||||
});
|
||||
|
|
@ -167,7 +184,7 @@ export function createQdrantRegulatoryIndexFromEnvironment(
|
|||
): QdrantRegulatoryIndex {
|
||||
const urlValue = environment.ASSESSOR_QDRANT_URL?.trim();
|
||||
const apiKey = environment.ASSESSOR_QDRANT_API_KEY?.trim();
|
||||
if (!urlValue || !apiKey) throw new Error('Regulatory RAG requires ASSESSOR_QDRANT_URL and ASSESSOR_QDRANT_API_KEY.');
|
||||
if (!urlValue) throw new Error('Regulatory RAG requires ASSESSOR_QDRANT_URL.');
|
||||
let baseUrl: URL;
|
||||
try { baseUrl = new URL(urlValue); } catch { throw new Error('ASSESSOR_QDRANT_URL must be an absolute URL.'); }
|
||||
if (!['http:', 'https:'].includes(baseUrl.protocol) || baseUrl.username || baseUrl.password || baseUrl.search || baseUrl.hash) {
|
||||
|
|
@ -177,6 +194,9 @@ export function createQdrantRegulatoryIndexFromEnvironment(
|
|||
if (baseUrl.protocol !== 'https:' && !['localhost', '127.0.0.1', '[::1]', 'qdrant'].includes(baseUrl.hostname.toLowerCase())) {
|
||||
throw new Error('ASSESSOR_QDRANT_URL must use HTTPS unless it targets loopback or the internal qdrant service.');
|
||||
}
|
||||
if (!apiKey && !['localhost', '127.0.0.1', '[::1]', 'qdrant'].includes(baseUrl.hostname.toLowerCase())) {
|
||||
throw new Error('Remote Qdrant requires ASSESSOR_QDRANT_API_KEY.');
|
||||
}
|
||||
return new QdrantRegulatoryIndex({
|
||||
baseUrl,
|
||||
apiKey,
|
||||
|
|
@ -219,7 +239,18 @@ function chunkPayload(chunk: RegulatoryChunk, embeddingModel: string): Record<st
|
|||
text: chunk.text,
|
||||
text_sha256: chunk.textSha256,
|
||||
embedding_model: embeddingModel,
|
||||
...(chunk.sourceUri ? { source_uri: chunk.sourceUri } : {})
|
||||
...(chunk.sourceUri ? { source_uri: chunk.sourceUri } : {}),
|
||||
...(chunk.annexAControlId ? { annex_a_control_id: chunk.annexAControlId } : {}),
|
||||
...(chunk.controlTheme ? { control_theme: chunk.controlTheme } : {}),
|
||||
...(chunk.sourceStandard ? { source_standard: chunk.sourceStandard } : {}),
|
||||
...(chunk.sourceEdition ? { source_edition: chunk.sourceEdition } : {}),
|
||||
...(chunk.nationalEdition ? { national_edition: chunk.nationalEdition } : {}),
|
||||
...(chunk.sectionKind ? { section_kind: chunk.sectionKind } : {}),
|
||||
...(chunk.sourcePageStart ? { source_page_start: chunk.sourcePageStart } : {}),
|
||||
...(chunk.sourcePageEnd ? { source_page_end: chunk.sourcePageEnd } : {}),
|
||||
...(chunk.retrievalRole ? { retrieval_role: chunk.retrievalRole } : {}),
|
||||
...(chunk.parserVersion ? { parser_version: chunk.parserVersion } : {}),
|
||||
...(chunk.sourceSha256 ? { source_sha256: chunk.sourceSha256 } : {})
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -228,16 +259,20 @@ function parseCandidate(value: unknown): RegulatoryRetrievalCandidate {
|
|||
const payload = object(point?.payload);
|
||||
const score = point?.score;
|
||||
if (!payload || typeof score !== 'number' || !Number.isFinite(score)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant returned an invalid point.');
|
||||
return { score, chunk: parseChunkPayload(payload) };
|
||||
}
|
||||
|
||||
function parseChunkPayload(value: unknown): RegulatoryChunk {
|
||||
const payload = object(value);
|
||||
if (!payload) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant point is missing its payload.');
|
||||
const stringField = (name: string): string => {
|
||||
const field = payload[name];
|
||||
if (typeof field !== 'string' || !field) throw new QdrantClientError('qdrant_invalid_response', `Qdrant point is missing ${name}.`);
|
||||
return field;
|
||||
};
|
||||
if (payload.schema_version !== 1 || !Number.isInteger(payload.chunk_index)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant point has an unsupported regulatory schema.');
|
||||
return {
|
||||
score,
|
||||
chunk: {
|
||||
schemaVersion: 1,
|
||||
if (![1, 2].includes(Number(payload.schema_version)) || !Number.isInteger(payload.chunk_index)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant point has an unsupported regulatory schema.');
|
||||
const chunk: RegulatoryChunk = {
|
||||
schemaVersion: Number(payload.schema_version) as 1 | 2,
|
||||
corpusId: stringField('corpus_id'),
|
||||
corpusVersion: stringField('corpus_version'),
|
||||
documentId: stringField('document_id'),
|
||||
|
|
@ -250,9 +285,20 @@ function parseCandidate(value: unknown): RegulatoryRetrievalCandidate {
|
|||
title: stringField('title'),
|
||||
text: stringField('text'),
|
||||
textSha256: stringField('text_sha256'),
|
||||
...(typeof payload.source_uri === 'string' && payload.source_uri ? { sourceUri: payload.source_uri } : {})
|
||||
}
|
||||
...(typeof payload.source_uri === 'string' && payload.source_uri ? { sourceUri: payload.source_uri } : {}),
|
||||
...(typeof payload.annex_a_control_id === 'string' ? { annexAControlId: payload.annex_a_control_id } : {}),
|
||||
...(optionalEnum(payload.control_theme, ['organizational', 'people', 'physical', 'technological'] as const, 'control_theme') ? { controlTheme: payload.control_theme as RegulatoryChunk['controlTheme'] } : {}),
|
||||
...(optionalEnum(payload.source_standard, ['ISO/IEC 27001', 'ISO/IEC 27002'] as const, 'source_standard') ? { sourceStandard: payload.source_standard as RegulatoryChunk['sourceStandard'] } : {}),
|
||||
...(typeof payload.source_edition === 'string' ? { sourceEdition: payload.source_edition } : {}),
|
||||
...(typeof payload.national_edition === 'string' ? { nationalEdition: payload.national_edition } : {}),
|
||||
...(optionalEnum(payload.section_kind, ['attributes', 'control', 'purpose', 'guidance', 'other-information'] as const, 'section_kind') ? { sectionKind: payload.section_kind as RegulatoryChunk['sectionKind'] } : {}),
|
||||
...(Number.isInteger(payload.source_page_start) ? { sourcePageStart: Number(payload.source_page_start) } : {}),
|
||||
...(Number.isInteger(payload.source_page_end) ? { sourcePageEnd: Number(payload.source_page_end) } : {}),
|
||||
...(optionalEnum(payload.retrieval_role, ['semantic', 'reference'] as const, 'retrieval_role') ? { retrievalRole: payload.retrieval_role as RegulatoryChunk['retrievalRole'] } : {}),
|
||||
...(typeof payload.parser_version === 'string' ? { parserVersion: payload.parser_version } : {}),
|
||||
...(typeof payload.source_sha256 === 'string' ? { sourceSha256: payload.source_sha256 } : {})
|
||||
};
|
||||
return chunk;
|
||||
}
|
||||
|
||||
function filterConditions(filters: RegulatoryRetrievalFilters): Array<Record<string, unknown>> {
|
||||
|
|
@ -262,7 +308,10 @@ function filterConditions(filters: RegulatoryRetrievalFilters): Array<Record<str
|
|||
['framework', filters.framework],
|
||||
['framework_version', filters.frameworkVersion],
|
||||
['control_id', filters.controlId],
|
||||
['language', filters.language?.toLowerCase()]
|
||||
['language', filters.language?.toLowerCase()],
|
||||
['source_standard', filters.sourceStandard],
|
||||
['section_kind', filters.sectionKind],
|
||||
['retrieval_role', filters.retrievalRole]
|
||||
];
|
||||
return values.filter((entry): entry is [string, string] => Boolean(entry[1])).map(([key, value]) => keywordCondition(key, value));
|
||||
}
|
||||
|
|
@ -321,3 +370,9 @@ async function readBoundedText(response: Response, maximumBytes: number): Promis
|
|||
function object(value: unknown): Record<string, any> | undefined {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, any> : undefined;
|
||||
}
|
||||
|
||||
function optionalEnum<const T extends readonly string[]>(value: unknown, allowed: T, field: string): T[number] | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (typeof value !== 'string' || !allowed.includes(value)) throw new QdrantClientError('qdrant_invalid_response', `Qdrant point has invalid ${field}.`);
|
||||
return value as T[number];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ export interface RegulatoryChunkingOptions {
|
|||
}
|
||||
|
||||
export function validateRegulatoryControlDocument(value: unknown): RegulatoryControlDocument {
|
||||
if (!isObject(value) || value.schemaVersion !== 1) throw new Error('Regulatory documents must use schemaVersion 1.');
|
||||
if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion))) throw new Error('Regulatory documents must use schemaVersion 1 or 2.');
|
||||
const required = ['corpusId', 'corpusVersion', 'documentId', 'framework', 'frameworkVersion', 'controlId', 'language', 'title', 'text'] as const;
|
||||
const document = value as Record<string, unknown>;
|
||||
for (const field of required) {
|
||||
|
|
@ -22,8 +22,9 @@ export function validateRegulatoryControlDocument(value: unknown): RegulatoryCon
|
|||
if (document.sourceUri !== undefined && (typeof document.sourceUri !== 'string' || !document.sourceUri.trim())) {
|
||||
throw new Error('Regulatory document sourceUri must be a non-empty string when present.');
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
const schemaVersion = Number(document.schemaVersion) as 1 | 2;
|
||||
const result: RegulatoryControlDocument = {
|
||||
schemaVersion,
|
||||
corpusId: String(document.corpusId).trim(),
|
||||
corpusVersion: String(document.corpusVersion).trim(),
|
||||
documentId: String(document.documentId).trim(),
|
||||
|
|
@ -35,6 +36,8 @@ export function validateRegulatoryControlDocument(value: unknown): RegulatoryCon
|
|||
text: normalizeText(String(document.text)),
|
||||
...(document.sourceUri === undefined ? {} : { sourceUri: String(document.sourceUri).trim() })
|
||||
};
|
||||
if (schemaVersion === 2) Object.assign(result, validateVersionTwoFields(document, result.controlId));
|
||||
return result;
|
||||
}
|
||||
|
||||
export function chunkRegulatoryControl(
|
||||
|
|
@ -63,7 +66,60 @@ export function chunkRegulatoryControl(
|
|||
}
|
||||
|
||||
export function regulatoryChunkEmbeddingText(chunk: RegulatoryChunk): string {
|
||||
return `${chunk.framework} ${chunk.frameworkVersion} | ${chunk.controlId} | ${chunk.title}\n\n${chunk.text}`;
|
||||
const context = [
|
||||
`${chunk.framework} ${chunk.frameworkVersion}`,
|
||||
chunk.annexAControlId ? `${chunk.controlId} (${chunk.annexAControlId})` : chunk.controlId,
|
||||
chunk.controlTheme,
|
||||
chunk.sectionKind,
|
||||
chunk.title
|
||||
].filter(Boolean).join(' | ');
|
||||
return `${context}\n\n${chunk.text}`;
|
||||
}
|
||||
|
||||
function validateVersionTwoFields(document: Record<string, unknown>, controlId: string): Partial<RegulatoryControlDocument> {
|
||||
if (!/^[5-8]\.\d{1,2}$/.test(controlId)) throw new Error('Regulatory schemaVersion 2 controlId must use the canonical 5.x-8.x form.');
|
||||
const annexAControlId = requiredEnumLike(document, 'annexAControlId');
|
||||
if (annexAControlId !== `A.${controlId}`) throw new Error(`Regulatory annexAControlId must be A.${controlId}.`);
|
||||
const controlTheme = requiredEnum(document, 'controlTheme', ['organizational', 'people', 'physical', 'technological'] as const);
|
||||
const sourceStandard = requiredEnum(document, 'sourceStandard', ['ISO/IEC 27001', 'ISO/IEC 27002'] as const);
|
||||
const sectionKind = requiredEnum(document, 'sectionKind', ['attributes', 'control', 'purpose', 'guidance', 'other-information'] as const);
|
||||
const retrievalRole = requiredEnum(document, 'retrievalRole', ['semantic', 'reference'] as const);
|
||||
const sourceEdition = requiredEnumLike(document, 'sourceEdition');
|
||||
const nationalEdition = requiredEnumLike(document, 'nationalEdition');
|
||||
const parserVersion = requiredEnumLike(document, 'parserVersion');
|
||||
const sourceSha256 = requiredEnumLike(document, 'sourceSha256');
|
||||
if (!/^[a-f0-9]{64}$/.test(sourceSha256)) throw new Error('Regulatory sourceSha256 must be a lower-case SHA-256 digest.');
|
||||
const sourcePageStart = requiredPage(document, 'sourcePageStart');
|
||||
const sourcePageEnd = requiredPage(document, 'sourcePageEnd');
|
||||
if (sourcePageEnd < sourcePageStart) throw new Error('Regulatory sourcePageEnd must not precede sourcePageStart.');
|
||||
if (sourceStandard === 'ISO/IEC 27001' && (sectionKind !== 'control' || retrievalRole !== 'reference')) {
|
||||
throw new Error('ISO/IEC 27001 schemaVersion 2 documents must be reference control statements.');
|
||||
}
|
||||
if (sourceStandard === 'ISO/IEC 27002' && retrievalRole !== 'semantic') {
|
||||
throw new Error('ISO/IEC 27002 schemaVersion 2 documents must be semantic evidence.');
|
||||
}
|
||||
return {
|
||||
annexAControlId, controlTheme, sourceStandard, sourceEdition, nationalEdition,
|
||||
sectionKind, sourcePageStart, sourcePageEnd, retrievalRole, parserVersion, sourceSha256
|
||||
};
|
||||
}
|
||||
|
||||
function requiredEnumLike(document: Record<string, unknown>, field: string): string {
|
||||
const value = document[field];
|
||||
if (typeof value !== 'string' || !value.trim()) throw new Error(`Regulatory schemaVersion 2 ${field} must be a non-empty string.`);
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function requiredEnum<const T extends readonly string[]>(document: Record<string, unknown>, field: string, values: T): T[number] {
|
||||
const value = requiredEnumLike(document, field);
|
||||
if (!values.includes(value)) throw new Error(`Regulatory schemaVersion 2 ${field} is not supported.`);
|
||||
return value as T[number];
|
||||
}
|
||||
|
||||
function requiredPage(document: Record<string, unknown>, field: string): number {
|
||||
const value = document[field];
|
||||
if (!Number.isInteger(value) || Number(value) < 1 || Number(value) > 10_000) throw new Error(`Regulatory schemaVersion 2 ${field} must be a positive page number.`);
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
function splitText(text: string, maximumCharacters: number): string[] {
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ async function main(): Promise<void> {
|
|||
}
|
||||
|
||||
function corpusDocuments(value: unknown): RegulatoryControlDocument[] {
|
||||
if (!isObject(value) || value.schemaVersion !== 1 || !Array.isArray(value.documents) || !value.documents.length) {
|
||||
throw new Error('Regulatory corpus must use schemaVersion 1 and contain a non-empty documents array.');
|
||||
if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion)) || !Array.isArray(value.documents) || !value.documents.length) {
|
||||
throw new Error('Regulatory corpus must use schemaVersion 1 or 2 and contain a non-empty documents array.');
|
||||
}
|
||||
if (value.documents.length > 100_000) throw new Error('Regulatory corpus exceeds the 100000 document limit.');
|
||||
return value.documents.map(validateRegulatoryControlDocument);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,331 @@
|
|||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import type { AiCompletionClient } from '../ai/litellm-client';
|
||||
import { LiteLlmClientError } from '../ai/litellm-client';
|
||||
import type { AssessorRepository } from '../../src/persistence/assessor-repository';
|
||||
import { RepositoryNotFoundError, RepositoryValidationError } from '../../src/persistence/repository-errors';
|
||||
import {
|
||||
REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION,
|
||||
REGULATORY_MAPPING_PROMPT_VERSION,
|
||||
REGULATORY_RETRIEVAL_VERSION,
|
||||
groundRegulatoryMappingResult,
|
||||
regulatoryMappingResultValidationErrors,
|
||||
type RegulatoryMappingCandidate,
|
||||
type RegulatoryMappingCreateRequest,
|
||||
type RegulatoryMappingDecision,
|
||||
type RegulatoryMappingDecisionCreateRequest,
|
||||
type RegulatoryMappingEvidence,
|
||||
type RegulatoryMappingResponse,
|
||||
type RegulatoryMappingResult,
|
||||
type RegulatoryMappingRun
|
||||
} from '../../src/domain/regulatory-mapping';
|
||||
import type { RegulatoryRetrievalCandidate } from '../../src/domain/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import { privacySafeFinding } from '../../src/domain/privacy-safe-finding';
|
||||
import type { RegulatoryFindingRetrievalService } from './regulatory-finding-retrieval';
|
||||
import type { RegulatoryChunkIndex } from './qdrant-regulatory-index';
|
||||
import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query';
|
||||
|
||||
const POLL_AFTER_MS = 1_500;
|
||||
const MAXIMUM_PROMPT_CHARACTERS = 48_000;
|
||||
|
||||
export interface RegulatoryMappingServiceOptions {
|
||||
findingRetrieval?: RegulatoryFindingRetrievalService;
|
||||
index?: RegulatoryChunkIndex;
|
||||
client?: AiCompletionClient;
|
||||
corpusId?: string;
|
||||
corpusVersion?: string;
|
||||
embeddingModel?: string;
|
||||
disabledReason?: string;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
export class RegulatoryMappingService {
|
||||
private readonly activeTasks = new Set<Promise<void>>();
|
||||
private readonly now: () => Date;
|
||||
|
||||
constructor(private readonly repository: AssessorRepository, private readonly options: RegulatoryMappingServiceOptions = {}) {
|
||||
this.now = options.now ?? (() => new Date());
|
||||
}
|
||||
|
||||
get generationEnabled(): boolean {
|
||||
return Boolean(this.options.findingRetrieval && this.options.index && this.options.client
|
||||
&& this.options.corpusId && this.options.corpusVersion && this.options.embeddingModel);
|
||||
}
|
||||
|
||||
async recoverInterruptedRuns(): Promise<void> {
|
||||
const completedAt = this.now().toISOString();
|
||||
for (const run of (await this.repository.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) {
|
||||
await this.repository.saveRegulatoryMappingRun({
|
||||
...run,
|
||||
status: 'failed',
|
||||
updatedAt: completedAt,
|
||||
completedAt,
|
||||
errorCode: 'regulatory_mapping_interrupted'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async get(projectId: string, resultProjectionId: string): Promise<RegulatoryMappingResponse> {
|
||||
const finding = await this.finding(projectId, resultProjectionId);
|
||||
const supported = finding.sourceKind === 'scanner-finding' && Boolean(buildRegulatoryFindingQuery(finding));
|
||||
const availability = supported ? await this.availability() : { enabled: false };
|
||||
const enabled = availability.enabled && supported;
|
||||
const ruleKey = supported ? this.ruleKey(finding) : undefined;
|
||||
const run = enabled && ruleKey ? await this.repository.getRegulatoryMappingRun(this.mappingId(ruleKey)) : null;
|
||||
const decisions = ruleKey ? await this.repository.listRegulatoryMappingDecisions(ruleKey) : [];
|
||||
return {
|
||||
version: 1,
|
||||
projectId,
|
||||
resultProjectionId,
|
||||
generationEnabled: enabled,
|
||||
state: run?.status ?? (enabled ? 'empty' : 'unsupported'),
|
||||
...(!supported ? { disabledReason: finding.sourceKind !== 'scanner-finding'
|
||||
? 'Only canonical scanner findings can be mapped in this version.'
|
||||
: 'This finding has insufficient public context for regulatory mapping.' }
|
||||
: !availability.enabled ? { disabledReason: availability.reason ?? this.options.disabledReason ?? 'Regulatory mapping is not configured.' } : {}),
|
||||
...(run && ['queued', 'running'].includes(run.status) ? { pollAfterMs: POLL_AFTER_MS } : {}),
|
||||
...(run ? { run: withoutEvidenceText(run) } : {}),
|
||||
decisions
|
||||
};
|
||||
}
|
||||
|
||||
async start(projectId: string, request: RegulatoryMappingCreateRequest): Promise<RegulatoryMappingResponse> {
|
||||
if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503);
|
||||
const finding = await this.finding(projectId, request.resultProjectionId);
|
||||
if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422);
|
||||
const ruleKey = this.ruleKey(finding);
|
||||
const id = this.mappingId(ruleKey);
|
||||
const existing = await this.repository.getRegulatoryMappingRun(id);
|
||||
if (existing && ['queued', 'running'].includes(existing.status)) return this.get(projectId, request.resultProjectionId);
|
||||
let retrieval;
|
||||
try {
|
||||
retrieval = await this.options.findingRetrieval!.retrieve(finding, 32, {
|
||||
corpusId: this.options.corpusId,
|
||||
corpusVersion: this.options.corpusVersion,
|
||||
sourceStandard: 'ISO/IEC 27002',
|
||||
retrievalRole: 'semantic'
|
||||
});
|
||||
} catch {
|
||||
throw new RegulatoryMappingServiceError('regulatory_retrieval_unavailable', 'Regulatory retrieval is currently unavailable.', 503);
|
||||
}
|
||||
if (retrieval.status !== 'candidates' || !retrieval.preparedQuery) {
|
||||
throw new RegulatoryMappingServiceError('insufficient_public_context', 'This finding has insufficient public context for regulatory mapping.', 422);
|
||||
}
|
||||
const candidates = await this.groupCandidates(retrieval.candidates);
|
||||
if (!candidates.length) throw new RegulatoryMappingServiceError('regulatory_corpus_empty', 'The active regulatory corpus returned no eligible controls.', 503);
|
||||
const timestamp = this.now().toISOString();
|
||||
const queryHash = createHash('sha256').update(JSON.stringify(retrieval.preparedQuery), 'utf8').digest('hex');
|
||||
const proposed: RegulatoryMappingRun = {
|
||||
version: 1,
|
||||
id,
|
||||
ruleKey,
|
||||
sourceTool: finding.sourceTool ?? 'unknown',
|
||||
...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}),
|
||||
status: 'queued',
|
||||
queryHash,
|
||||
preparedQuery: retrieval.preparedQuery,
|
||||
corpusId: this.options.corpusId!,
|
||||
corpusVersion: this.options.corpusVersion!,
|
||||
sourceHashes: [...new Set(candidates.flatMap(candidate => candidate.evidence.map(evidence => evidence.sourceSha256)))].sort(),
|
||||
embeddingModel: this.options.embeddingModel!,
|
||||
retrievalVersion: REGULATORY_RETRIEVAL_VERSION,
|
||||
promptVersion: REGULATORY_MAPPING_PROMPT_VERSION,
|
||||
outputSchemaVersion: REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION,
|
||||
model: this.options.client!.model,
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
candidates
|
||||
};
|
||||
await this.repository.saveRegulatoryMappingRun(proposed);
|
||||
const task = this.process(proposed).finally(() => this.activeTasks.delete(task));
|
||||
this.activeTasks.add(task);
|
||||
return this.get(projectId, request.resultProjectionId);
|
||||
}
|
||||
|
||||
async decide(projectId: string, request: RegulatoryMappingDecisionCreateRequest): Promise<RegulatoryMappingResponse> {
|
||||
if (!this.generationEnabled) throw new RegulatoryMappingServiceError('regulatory_mapping_disabled', this.options.disabledReason ?? 'Regulatory mapping is not configured.', 503);
|
||||
const finding = await this.finding(projectId, request.resultProjectionId);
|
||||
if (finding.sourceKind !== 'scanner-finding') throw new RegulatoryMappingServiceError('unsupported_finding_kind', 'Only canonical scanner findings can be mapped in this version.', 422);
|
||||
const ruleKey = this.ruleKey(finding);
|
||||
const comment = request.comment?.trim();
|
||||
const decision: RegulatoryMappingDecision = {
|
||||
version: 1,
|
||||
id: `regulatory-decision-${randomUUID()}`,
|
||||
ruleKey,
|
||||
sourceTool: finding.sourceTool ?? 'unknown',
|
||||
...(finding.sourceRuleId ? { sourceRuleId: finding.sourceRuleId } : {}),
|
||||
mappingId: this.mappingId(ruleKey),
|
||||
controlId: request.controlId,
|
||||
decision: request.decision,
|
||||
...(comment ? { comment } : {}),
|
||||
createdAt: this.now().toISOString()
|
||||
};
|
||||
await this.repository.createRegulatoryMappingDecision(decision, request.expectedLatestDecisionId);
|
||||
return this.get(projectId, request.resultProjectionId);
|
||||
}
|
||||
|
||||
// The global sharing key: the rule identity `tool:ruleId`, shared across every tenant.
|
||||
// Rules without an id fall back to a stable hash of the non-personal check title so
|
||||
// identical findings still share, while never keying on tenant or occurrence.
|
||||
private ruleKey(finding: UnifiedResultProjection): string {
|
||||
const tool = finding.sourceTool?.trim() || 'unknown';
|
||||
const ruleId = finding.sourceRuleId?.trim();
|
||||
if (ruleId) return `${tool}:${ruleId}`.slice(0, 200);
|
||||
const hash = createHash('sha256').update(privacySafeFinding(finding).title.trim(), 'utf8').digest('hex').slice(0, 24);
|
||||
return `${tool}#title:${hash}`.slice(0, 200);
|
||||
}
|
||||
|
||||
// Deterministic id per (ruleKey, pipeline versions) so one row exists per rule per
|
||||
// version tuple: regeneration reuses it, and a version bump yields a new mapping.
|
||||
private mappingId(ruleKey: string): string {
|
||||
const tuple = [ruleKey, this.options.corpusVersion, REGULATORY_RETRIEVAL_VERSION, REGULATORY_MAPPING_PROMPT_VERSION, this.options.client!.model].join(' ');
|
||||
return `regulatory-mapping-${createHash('sha256').update(tuple, 'utf8').digest('hex').slice(0, 40)}`;
|
||||
}
|
||||
|
||||
async close(): Promise<void> { await Promise.allSettled([...this.activeTasks]); }
|
||||
|
||||
private async availability(): Promise<{ enabled: boolean; reason?: string }> {
|
||||
if (!this.generationEnabled) return { enabled: false, reason: this.options.disabledReason ?? 'Regulatory mapping is not configured.' };
|
||||
try {
|
||||
const sample = await this.options.index!.list({
|
||||
corpusId: this.options.corpusId,
|
||||
corpusVersion: this.options.corpusVersion,
|
||||
sourceStandard: 'ISO/IEC 27002',
|
||||
retrievalRole: 'semantic'
|
||||
}, 1);
|
||||
return sample.length ? { enabled: true } : { enabled: false, reason: 'The active regulatory corpus is not indexed.' };
|
||||
} catch {
|
||||
return { enabled: false, reason: 'Qdrant or the active regulatory corpus is unavailable.' };
|
||||
}
|
||||
}
|
||||
|
||||
private async finding(projectId: string, resultProjectionId: string) {
|
||||
if (!resultProjectionId?.trim()) throw new RepositoryValidationError('A resultProjectionId is required.');
|
||||
const finding = await this.repository.getUnifiedResultById(resultProjectionId);
|
||||
if (!finding || finding.projectId !== projectId) throw new RepositoryNotFoundError('The selected finding does not exist in this project.');
|
||||
return finding;
|
||||
}
|
||||
|
||||
private async groupCandidates(hits: RegulatoryRetrievalCandidate[]): Promise<RegulatoryMappingCandidate[]> {
|
||||
const byControl = new Map<string, RegulatoryRetrievalCandidate[]>();
|
||||
for (const hit of hits) {
|
||||
if (!hit.chunk.annexAControlId || !hit.chunk.sectionKind || !hit.chunk.sourcePageStart || !hit.chunk.sourcePageEnd || !hit.chunk.sourceSha256) continue;
|
||||
const values = byControl.get(hit.chunk.controlId) ?? [];
|
||||
values.push(hit);
|
||||
byControl.set(hit.chunk.controlId, values);
|
||||
}
|
||||
const selected = [...byControl.entries()]
|
||||
.map(([controlId, values]) => ({ controlId, values: values.sort((left, right) => right.score - left.score), score: Math.max(...values.map(value => value.score)) }))
|
||||
.sort((left, right) => right.score - left.score || left.controlId.localeCompare(right.controlId, 'en', { numeric: true }))
|
||||
.slice(0, 8);
|
||||
return Promise.all(selected.map(async selection => {
|
||||
const semantic = selection.values.slice(0, 2).map(value => evidence(value.chunk));
|
||||
const references = await this.options.index!.list({
|
||||
corpusId: this.options.corpusId,
|
||||
corpusVersion: this.options.corpusVersion,
|
||||
controlId: selection.controlId,
|
||||
sourceStandard: 'ISO/IEC 27001',
|
||||
retrievalRole: 'reference'
|
||||
}, 1);
|
||||
if (references.length !== 1 || !references[0].sourceSha256) {
|
||||
throw new RegulatoryMappingServiceError('regulatory_reference_missing', `The active corpus is missing the ISO 27001 reference for ${selection.controlId}.`, 503);
|
||||
}
|
||||
const first = selection.values[0].chunk;
|
||||
return {
|
||||
controlId: selection.controlId,
|
||||
annexAControlId: first.annexAControlId!,
|
||||
title: first.title,
|
||||
score: selection.score,
|
||||
evidence: [...semantic, ...references.map(evidence)]
|
||||
};
|
||||
}));
|
||||
}
|
||||
|
||||
private async process(queued: RegulatoryMappingRun): Promise<void> {
|
||||
const startedAt = this.now().toISOString();
|
||||
const run = await this.repository.saveRegulatoryMappingRun({ ...queued, status: 'running', startedAt, updatedAt: startedAt });
|
||||
try {
|
||||
const prompt = mappingPrompt(run);
|
||||
const completion = await this.options.client!.complete(SYSTEM_PROMPT, prompt);
|
||||
const allowed = new Map(run.candidates.map(candidate => [
|
||||
candidate.controlId, new Set(candidate.evidence.map(item => item.chunkId))
|
||||
]));
|
||||
const grounded = groundRegulatoryMappingResult(completion.value, allowed);
|
||||
const errors = regulatoryMappingResultValidationErrors(grounded, allowed);
|
||||
if (errors.length) throw new RegulatoryMappingServiceError('regulatory_mapping_invalid_output', errors.join(' '), 502);
|
||||
const completedAt = this.now().toISOString();
|
||||
await this.repository.saveRegulatoryMappingRun({
|
||||
...run,
|
||||
status: 'succeeded',
|
||||
result: grounded as RegulatoryMappingResult,
|
||||
...(completion.usage ? { tokenUsage: completion.usage } : {}),
|
||||
updatedAt: completedAt,
|
||||
completedAt
|
||||
});
|
||||
} catch (error) {
|
||||
const completedAt = this.now().toISOString();
|
||||
const failure = safeFailure(error);
|
||||
const detail = safeFailureDetail(error);
|
||||
try {
|
||||
await this.repository.saveRegulatoryMappingRun({
|
||||
...run, status: 'failed', updatedAt: completedAt, completedAt, errorCode: failure,
|
||||
...(detail ? { errorDetail: detail } : {})
|
||||
});
|
||||
} catch { /* Preserve the original safe failure code. */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class RegulatoryMappingServiceError extends Error {
|
||||
constructor(readonly code: string, message: string, readonly status: number) { super(message); this.name = 'RegulatoryMappingServiceError'; }
|
||||
}
|
||||
|
||||
// Response projection: the persisted run keeps the full ISO clause text (the model reads it),
|
||||
// but the copyrighted text is never shipped to the browser — only the citation fields are.
|
||||
function withoutEvidenceText(run: RegulatoryMappingRun): RegulatoryMappingRun {
|
||||
return {
|
||||
...run,
|
||||
candidates: run.candidates.map(candidate => ({
|
||||
...candidate,
|
||||
evidence: candidate.evidence.map(({ text, ...reference }) => reference)
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
function evidence(chunk: RegulatoryRetrievalCandidate['chunk']): RegulatoryMappingEvidence {
|
||||
return {
|
||||
chunkId: chunk.chunkId,
|
||||
textSha256: chunk.textSha256,
|
||||
sourceSha256: chunk.sourceSha256!,
|
||||
sourceStandard: chunk.sourceStandard!,
|
||||
sectionKind: chunk.sectionKind!,
|
||||
pageStart: chunk.sourcePageStart!,
|
||||
pageEnd: chunk.sourcePageEnd!,
|
||||
text: chunk.text
|
||||
};
|
||||
}
|
||||
|
||||
function mappingPrompt(run: RegulatoryMappingRun): string {
|
||||
// The Annex A id is always "A." + controlId, so it is redundant for the model and
|
||||
// sending it invites a reply that uses the Annex A form; expose only the controlId it must return.
|
||||
const candidates = run.candidates.map(({ annexAControlId, ...candidate }) => candidate);
|
||||
const payload = JSON.stringify({ finding: run.preparedQuery, candidates });
|
||||
if (payload.length > MAXIMUM_PROMPT_CHARACTERS) throw new RegulatoryMappingServiceError('regulatory_mapping_prompt_too_large', 'The bounded regulatory prompt exceeded its limit.', 422);
|
||||
return payload;
|
||||
}
|
||||
|
||||
const SYSTEM_PROMPT = `Map one public security finding query to the supplied ISO control candidates. A mapping means the control is topically relevant to the finding, never proof of compliance. Map a candidate when it governs the same security subject as the finding (for example password or authentication management, logging, or access restriction), even when the control text does not state the finding's specific value, threshold, parameter, or configuration; ISO controls are principle-level and rarely restate such specifics. Judge relevance from the control's subject and its supplied evidence, not from whether the exact requirement text appears. Return JSON only with schemaVersion 1, outcome mapped|no_relevant_control|insufficient_context, optional bounded reason, and mappings (maximum five). Each mapping requires controlId, relationship primary|supporting, confidence low|medium|high, a concise rationale under 600 characters, and one to three evidenceChunkIds copied only from that control's supplied evidence. Use no_relevant_control only when no candidate governs the finding's subject, and insufficient_context only when the finding query itself lacks enough meaning to judge. Do not invent controls or evidence.`;
|
||||
|
||||
function safeFailure(error: unknown): string {
|
||||
if (error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError) return error.code;
|
||||
return 'regulatory_mapping_failed';
|
||||
}
|
||||
|
||||
// Our own service and client error messages are developer-authored and structural
|
||||
// (field paths, ISO control numbers) — never finding, tenant, or report content.
|
||||
// Clamp the length and strip any markup-like sequence before it is stored or shown.
|
||||
function safeFailureDetail(error: unknown): string | undefined {
|
||||
if (!(error instanceof RegulatoryMappingServiceError || error instanceof LiteLlmClientError)) return undefined;
|
||||
const detail = error.message.replace(/<\/?[a-z][^>]*>/gi, ' ').replace(/\s+/g, ' ').trim().slice(0, 600);
|
||||
return detail || undefined;
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { buildIsoPdfCorpus } from './iso-pdf-corpus';
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const [iso27001Path, iso27002Path, outputPath, corpusVersion] = process.argv.slice(2);
|
||||
if (!iso27001Path || !iso27002Path || !outputPath) {
|
||||
throw new Error('Usage: npm run rag:prepare-iso -- <iso27001.pdf> <iso27002.pdf> <output.json> [corpus-version]');
|
||||
}
|
||||
const target = resolve(outputPath);
|
||||
const corpus = await buildIsoPdfCorpus({
|
||||
iso27001Path: resolve(iso27001Path),
|
||||
iso27002Path: resolve(iso27002Path),
|
||||
...(corpusVersion?.trim() ? { corpusVersion: corpusVersion.trim() } : {})
|
||||
});
|
||||
await mkdir(dirname(target), { recursive: true });
|
||||
await writeFile(target, `${JSON.stringify(corpus, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' });
|
||||
const controlCount = new Set(corpus.documents.map(document => document.controlId)).size;
|
||||
console.log(`Prepared corpus ${corpus.corpusId}@${corpus.corpusVersion}: ${controlCount} controls, ${corpus.documents.length} section documents.`);
|
||||
}
|
||||
|
||||
void main().catch(error => {
|
||||
console.error(error instanceof Error ? error.message : 'ISO corpus preparation failed.');
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
@ -30,6 +30,8 @@ export function createRegulatoryRagRuntime(
|
|||
ingestion: RegulatoryIngestionService;
|
||||
retrieval: RegulatoryRetrievalPipeline;
|
||||
findingRetrieval: RegulatoryFindingRetrievalService;
|
||||
index: ReturnType<typeof createQdrantRegulatoryIndexFromEnvironment>;
|
||||
embeddingModel: string;
|
||||
} {
|
||||
const embeddings = createEmbeddingClientFromEnvironment(environment);
|
||||
const index = createQdrantRegulatoryIndexFromEnvironment(embeddings.dimensions, embeddings.model, environment);
|
||||
|
|
@ -41,7 +43,9 @@ export function createRegulatoryRagRuntime(
|
|||
embeddingBatchSize: boundedInteger(environment.ASSESSOR_RAG_EMBEDDING_BATCH_SIZE, 32, 1, 128, 'ASSESSOR_RAG_EMBEDDING_BATCH_SIZE')
|
||||
}),
|
||||
retrieval,
|
||||
findingRetrieval: new RegulatoryFindingRetrievalService(retrieval)
|
||||
findingRetrieval: new RegulatoryFindingRetrievalService(retrieval),
|
||||
index,
|
||||
embeddingModel: embeddings.model
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,10 @@ import {
|
|||
type MicrosoftSecurityBaselineAnalysisResponse,
|
||||
type MicrosoftSecurityBaselineAnalysisListResponse,
|
||||
type AiOverviewCreateRequest,
|
||||
type AiOverviewResponse
|
||||
type AiOverviewResponse,
|
||||
type RegulatoryMappingCreateRequest,
|
||||
type RegulatoryMappingDecisionCreateRequest,
|
||||
type RegulatoryMappingResponse
|
||||
} from './assessor-api-contract';
|
||||
|
||||
export class AssessorApiError extends Error {
|
||||
|
|
@ -226,6 +229,22 @@ export class AssessorApiClient {
|
|||
});
|
||||
}
|
||||
|
||||
getRegulatoryMappings(projectId: string, resultProjectionId: string): Promise<RegulatoryMappingResponse> {
|
||||
return this.request(`/projects/${encodeURIComponent(projectId)}/regulatory-mappings/${encodeURIComponent(resultProjectionId)}`);
|
||||
}
|
||||
|
||||
generateRegulatoryMappings(projectId: string, request: RegulatoryMappingCreateRequest): Promise<RegulatoryMappingResponse> {
|
||||
return this.request(`/projects/${encodeURIComponent(projectId)}/regulatory-mapping-runs`, {
|
||||
method: 'POST', body: JSON.stringify(request)
|
||||
});
|
||||
}
|
||||
|
||||
saveRegulatoryMappingDecision(projectId: string, request: RegulatoryMappingDecisionCreateRequest): Promise<RegulatoryMappingResponse> {
|
||||
return this.request(`/projects/${encodeURIComponent(projectId)}/regulatory-mapping-decisions`, {
|
||||
method: 'POST', body: JSON.stringify(request)
|
||||
});
|
||||
}
|
||||
|
||||
listProjectImportFiles(projectId: string): Promise<ImportFileRecord[]> {
|
||||
return this.request(`/projects/${encodeURIComponent(projectId)}/import-files`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,11 @@ import type { PolicyAnalysisOverview } from '../domain/policy-analysis-overview'
|
|||
import type { MitreGraphData } from '../features/mitre-graph/mitre-graph';
|
||||
import type { AssessmentContext } from '../assessment-context/context-engine';
|
||||
export type { AiOverviewCreateRequest, AiOverviewResponse } from '../domain/ai-overview';
|
||||
export type {
|
||||
RegulatoryMappingCreateRequest,
|
||||
RegulatoryMappingDecisionCreateRequest,
|
||||
RegulatoryMappingResponse
|
||||
} from '../domain/regulatory-mapping';
|
||||
export const ASSESSOR_API_PREFIX = '/api/v1';
|
||||
export const ASSESSOR_MUTATION_HEADER = 'X-Assessor-Request';
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import type { UnifiedResultProjection } from './unified-result';
|
||||
|
||||
/**
|
||||
* The single privacy boundary shared by the AI overview and the ISO control mapping.
|
||||
* It projects a canonical finding to its rule-level check fields only — title,
|
||||
* description, recommendation and normalized metadata — and never includes the
|
||||
* result-table / occurrence data (affectedAsset, source locators, per-object values).
|
||||
* Definitions are deduplicated per rule (canonicalKey = tool:ruleId), so this text
|
||||
* describes the check, not the tenant. Email addresses are redacted defensively.
|
||||
*/
|
||||
export interface PrivacySafeFinding {
|
||||
id: string | null;
|
||||
title: string;
|
||||
description?: string;
|
||||
recommendation?: string;
|
||||
severity: string;
|
||||
outcome: string;
|
||||
resultType: string;
|
||||
isIoe: boolean;
|
||||
deviation: boolean;
|
||||
tool: string;
|
||||
}
|
||||
|
||||
export function privacySafeText(value: string): string {
|
||||
return value.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, '[redacted-email]');
|
||||
}
|
||||
|
||||
export function privacySafeCheckText(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return !trimmed || trimmed.toLocaleLowerCase('en-US') === 'none' ? undefined : privacySafeText(trimmed);
|
||||
}
|
||||
|
||||
export function privacySafeFinding(item: UnifiedResultProjection): PrivacySafeFinding {
|
||||
const description = privacySafeCheckText(item.description);
|
||||
const recommendation = privacySafeCheckText(item.recommendation);
|
||||
return {
|
||||
id: item.sourceRuleId ?? null,
|
||||
title: privacySafeText(item.title),
|
||||
...(description ? { description } : {}),
|
||||
...(recommendation ? { recommendation } : {}),
|
||||
severity: item.severityNormalized,
|
||||
outcome: item.outcomeNormalized,
|
||||
resultType: item.resultType,
|
||||
isIoe: item.isIoe,
|
||||
deviation: item.isDeviation,
|
||||
tool: item.sourceTool ?? 'unknown'
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
import type { AiOverviewTokenUsage } from './ai-overview';
|
||||
import type { RegulatoryFindingQuery, RegulatorySectionKind } from './regulatory-rag';
|
||||
|
||||
export const REGULATORY_MAPPING_PROMPT_VERSION = '1.2.0';
|
||||
export const REGULATORY_MAPPING_OUTPUT_SCHEMA_VERSION = 1 as const;
|
||||
export const REGULATORY_RETRIEVAL_VERSION = 'dense-control-grouping-1.1.0';
|
||||
|
||||
export type RegulatoryMappingRunStatus = 'queued' | 'running' | 'succeeded' | 'failed';
|
||||
export type RegulatoryMappingRelationship = 'primary' | 'supporting';
|
||||
export type RegulatoryMappingConfidence = 'low' | 'medium' | 'high';
|
||||
export type RegulatoryMappingDecisionValue = 'accepted' | 'rejected';
|
||||
|
||||
export interface RegulatoryMappingEvidence {
|
||||
chunkId: string;
|
||||
textSha256: string;
|
||||
sourceSha256: string;
|
||||
sourceStandard: 'ISO/IEC 27001' | 'ISO/IEC 27002';
|
||||
sectionKind: RegulatorySectionKind;
|
||||
pageStart: number;
|
||||
pageEnd: number;
|
||||
// Full ISO clause text. Required in the persisted run (the model reads it to reason),
|
||||
// but omitted from the API response projection so the copyrighted text never reaches the browser.
|
||||
text?: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryMappingCandidate {
|
||||
controlId: string;
|
||||
annexAControlId: string;
|
||||
title: string;
|
||||
score: number;
|
||||
evidence: RegulatoryMappingEvidence[];
|
||||
}
|
||||
|
||||
export interface RegulatoryMappingProposal {
|
||||
controlId: string;
|
||||
relationship: RegulatoryMappingRelationship;
|
||||
confidence: RegulatoryMappingConfidence;
|
||||
rationale: string;
|
||||
evidenceChunkIds: string[];
|
||||
}
|
||||
|
||||
export interface RegulatoryMappingResult {
|
||||
schemaVersion: 1;
|
||||
outcome: 'mapped' | 'no_relevant_control' | 'insufficient_context';
|
||||
reason?: string;
|
||||
mappings: RegulatoryMappingProposal[];
|
||||
}
|
||||
|
||||
/**
|
||||
* A global, tenant-independent mapping of one scanner rule to ISO controls. Keyed by
|
||||
* `ruleKey` (the rule identity `tool:ruleId`, or a stable title hash for rules without
|
||||
* an id) plus the pipeline versions, so the same rule maps once and every occurrence in
|
||||
* every project reuses it. Never carries project, tenant, or occurrence identity.
|
||||
*/
|
||||
export interface RegulatoryMappingRun {
|
||||
version: 1;
|
||||
id: string;
|
||||
ruleKey: string;
|
||||
sourceTool: string;
|
||||
sourceRuleId?: string;
|
||||
status: RegulatoryMappingRunStatus;
|
||||
queryHash: string;
|
||||
preparedQuery: RegulatoryFindingQuery;
|
||||
corpusId: string;
|
||||
corpusVersion: string;
|
||||
sourceHashes: string[];
|
||||
embeddingModel: string;
|
||||
retrievalVersion: string;
|
||||
promptVersion: string;
|
||||
outputSchemaVersion: 1;
|
||||
model: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
startedAt?: string;
|
||||
completedAt?: string;
|
||||
tokenUsage?: AiOverviewTokenUsage;
|
||||
errorCode?: string;
|
||||
errorDetail?: string;
|
||||
candidates: RegulatoryMappingCandidate[];
|
||||
result?: RegulatoryMappingResult;
|
||||
}
|
||||
|
||||
/** A global accept/reject decision, keyed by rule identity and control. Shared across all projects. */
|
||||
export interface RegulatoryMappingDecision {
|
||||
version: 1;
|
||||
id: string;
|
||||
ruleKey: string;
|
||||
sourceTool: string;
|
||||
sourceRuleId?: string;
|
||||
mappingId: string;
|
||||
controlId: string;
|
||||
decision: RegulatoryMappingDecisionValue;
|
||||
comment?: string;
|
||||
previousDecisionId?: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryMappingResponse {
|
||||
version: 1;
|
||||
projectId: string;
|
||||
resultProjectionId: string;
|
||||
generationEnabled: boolean;
|
||||
state: 'empty' | RegulatoryMappingRunStatus | 'unsupported';
|
||||
disabledReason?: string;
|
||||
pollAfterMs?: number;
|
||||
run?: RegulatoryMappingRun;
|
||||
decisions: RegulatoryMappingDecision[];
|
||||
}
|
||||
|
||||
export interface RegulatoryMappingCreateRequest {
|
||||
resultProjectionId: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryMappingDecisionCreateRequest {
|
||||
resultProjectionId: string;
|
||||
controlId: string;
|
||||
decision: RegulatoryMappingDecisionValue;
|
||||
comment?: string;
|
||||
expectedLatestDecisionId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trims each mapping's evidence citations to the exact chunk ids supplied for that
|
||||
* control, tolerating a model that miscopies or mis-scopes one of several long
|
||||
* compound chunk ids. This only removes ungrounded ids — it never invents evidence —
|
||||
* and drops a mapping that, for a known control, cited no real evidence at all.
|
||||
* Malformed or unknown-control mappings are left untouched for the validator to report.
|
||||
*/
|
||||
export function groundRegulatoryMappingResult(
|
||||
value: unknown,
|
||||
allowedCandidates: ReadonlyMap<string, ReadonlySet<string>>
|
||||
): unknown {
|
||||
if (!isObject(value) || !Array.isArray(value.mappings)) return value;
|
||||
const mappings: unknown[] = [];
|
||||
for (const mapping of value.mappings) {
|
||||
const allowed = isObject(mapping) && typeof mapping.controlId === 'string' ? allowedCandidates.get(mapping.controlId) : undefined;
|
||||
if (!allowed || !Array.isArray((mapping as Record<string, unknown>).evidenceChunkIds)) { mappings.push(mapping); continue; }
|
||||
const grounded = [...new Set(((mapping as Record<string, unknown>).evidenceChunkIds as unknown[])
|
||||
.filter((id): id is string => typeof id === 'string' && allowed.has(id)))].slice(0, 3);
|
||||
if (grounded.length) mappings.push({ ...(mapping as Record<string, unknown>), evidenceChunkIds: grounded });
|
||||
}
|
||||
return { ...value, mappings };
|
||||
}
|
||||
|
||||
export function regulatoryMappingResultValidationErrors(
|
||||
value: unknown,
|
||||
allowedCandidates?: ReadonlyMap<string, ReadonlySet<string>>
|
||||
): string[] {
|
||||
const errors: string[] = [];
|
||||
if (!isObject(value)) return ['Regulatory mapping output must be a JSON object.'];
|
||||
if (value.schemaVersion !== 1) errors.push('schemaVersion must be 1.');
|
||||
if (!['mapped', 'no_relevant_control', 'insufficient_context'].includes(String(value.outcome))) errors.push('outcome is invalid.');
|
||||
if (!Array.isArray(value.mappings) || value.mappings.length > 5) errors.push('mappings must contain at most five entries.');
|
||||
else {
|
||||
const controls = new Set<string>();
|
||||
value.mappings.forEach((mapping, index) => {
|
||||
if (!isObject(mapping)) { errors.push(`mappings[${index}] must be an object.`); return; }
|
||||
const controlId = boundedString(mapping.controlId, 20, `mappings[${index}].controlId`, errors);
|
||||
enumeration(mapping.relationship, ['primary', 'supporting'], `mappings[${index}].relationship`, errors);
|
||||
enumeration(mapping.confidence, ['low', 'medium', 'high'], `mappings[${index}].confidence`, errors);
|
||||
boundedString(mapping.rationale, 600, `mappings[${index}].rationale`, errors);
|
||||
if (controlId && controls.has(controlId)) errors.push(`mappings[${index}] duplicates control ${controlId}.`);
|
||||
if (controlId) controls.add(controlId);
|
||||
const allowedEvidence = controlId ? allowedCandidates?.get(controlId) : undefined;
|
||||
if (!Array.isArray(mapping.evidenceChunkIds) || mapping.evidenceChunkIds.length < 1 || mapping.evidenceChunkIds.length > 3) {
|
||||
errors.push(`mappings[${index}].evidenceChunkIds must contain one to three entries.`);
|
||||
} else mapping.evidenceChunkIds.forEach((chunkId, evidenceIndex) => {
|
||||
if (typeof chunkId !== 'string' || !allowedEvidence?.has(chunkId)) errors.push(`mappings[${index}].evidenceChunkIds[${evidenceIndex}] is not grounded in the supplied control.`);
|
||||
});
|
||||
if (controlId && allowedCandidates && !allowedCandidates.has(controlId)) errors.push(`mappings[${index}] references an unknown candidate control.`);
|
||||
});
|
||||
}
|
||||
if (value.outcome === 'mapped' && (!Array.isArray(value.mappings) || !value.mappings.length)) errors.push('mapped outcome requires at least one mapping.');
|
||||
if (value.outcome !== 'mapped' && Array.isArray(value.mappings) && value.mappings.length) errors.push('A refusal outcome must not include mappings.');
|
||||
if (value.outcome !== 'mapped') boundedString(value.reason, 600, 'reason', errors);
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function regulatoryMappingRunValidationErrors(run: RegulatoryMappingRun): string[] {
|
||||
const errors: string[] = [];
|
||||
for (const value of [run.id, run.ruleKey, run.sourceTool, run.corpusId, run.corpusVersion, run.embeddingModel, run.model]) {
|
||||
if (!value?.trim()) errors.push('Regulatory mapping run identity is incomplete.');
|
||||
}
|
||||
if (run.ruleKey && run.ruleKey.length > 200) errors.push('Regulatory mapping ruleKey is too long.');
|
||||
if (run.sourceRuleId !== undefined && (!run.sourceRuleId.trim() || run.sourceRuleId.length > 200)) errors.push('Regulatory mapping sourceRuleId is invalid.');
|
||||
if (!['queued', 'running', 'succeeded', 'failed'].includes(run.status)) errors.push('Regulatory mapping run status is invalid.');
|
||||
if (!/^[a-f0-9]{64}$/.test(run.queryHash)) errors.push('Regulatory mapping queryHash must be SHA-256.');
|
||||
if (!run.sourceHashes.length || run.sourceHashes.some(value => !/^[a-f0-9]{64}$/.test(value))) errors.push('Regulatory mapping sourceHashes must contain SHA-256 digests.');
|
||||
if (!Array.isArray(run.candidates) || run.candidates.length < 1 || run.candidates.length > 8) errors.push('Regulatory mapping candidates must contain one to eight controls.');
|
||||
else {
|
||||
const controls = new Set<string>();
|
||||
run.candidates.forEach(candidate => {
|
||||
if (!/^[5-8]\.\d{1,2}$/.test(candidate.controlId) || candidate.annexAControlId !== `A.${candidate.controlId}`) errors.push('Regulatory mapping candidate control identity is invalid.');
|
||||
if (controls.has(candidate.controlId)) errors.push(`Regulatory mapping candidate ${candidate.controlId} is duplicated.`);
|
||||
controls.add(candidate.controlId);
|
||||
if (!candidate.title.trim() || candidate.title.length > 500 || !Number.isFinite(candidate.score)) errors.push(`Regulatory mapping candidate ${candidate.controlId} metadata is invalid.`);
|
||||
if (!candidate.evidence.length || candidate.evidence.length > 3) errors.push(`Regulatory mapping candidate ${candidate.controlId} must contain one to three evidence chunks.`);
|
||||
candidate.evidence.forEach(evidence => {
|
||||
if (!evidence.chunkId.trim() || !/^[a-f0-9]{64}$/.test(evidence.textSha256) || !/^[a-f0-9]{64}$/.test(evidence.sourceSha256)
|
||||
|| !Number.isInteger(evidence.pageStart) || !Number.isInteger(evidence.pageEnd) || evidence.pageStart < 1 || evidence.pageEnd < evidence.pageStart
|
||||
|| typeof evidence.text !== 'string' || !evidence.text.trim() || evidence.text.length > 20_000) errors.push(`Regulatory mapping candidate ${candidate.controlId} evidence is invalid.`);
|
||||
});
|
||||
});
|
||||
}
|
||||
if (run.status === 'succeeded' && !run.result) errors.push('A succeeded regulatory mapping run requires a result.');
|
||||
if (run.result) errors.push(...regulatoryMappingResultValidationErrors(run.result, candidateEvidence(run.candidates)));
|
||||
return [...new Set(errors)];
|
||||
}
|
||||
|
||||
export function regulatoryMappingDecisionValidationErrors(decision: RegulatoryMappingDecision): string[] {
|
||||
const errors: string[] = [];
|
||||
for (const value of [decision.id, decision.ruleKey, decision.sourceTool, decision.mappingId, decision.controlId]) {
|
||||
if (!value?.trim()) errors.push('Regulatory mapping decision identity is incomplete.');
|
||||
}
|
||||
if (decision.sourceRuleId !== undefined && (!decision.sourceRuleId.trim() || decision.sourceRuleId.length > 200)) errors.push('Regulatory mapping decision sourceRuleId is invalid.');
|
||||
if (!['accepted', 'rejected'].includes(decision.decision)) errors.push('Regulatory mapping decision is invalid.');
|
||||
if (decision.comment !== undefined && (decision.comment.length > 1_000 || /<\/?[a-z][^>]*>/i.test(decision.comment))) errors.push('Regulatory mapping decision comment must be plain text with at most 1000 characters.');
|
||||
return [...new Set(errors)];
|
||||
}
|
||||
|
||||
function candidateEvidence(candidates: RegulatoryMappingCandidate[]): Map<string, Set<string>> {
|
||||
return new Map(candidates.map(candidate => [candidate.controlId, new Set(candidate.evidence.map(evidence => evidence.chunkId))]));
|
||||
}
|
||||
|
||||
function boundedString(value: unknown, maximum: number, path: string, errors: string[]): string | undefined {
|
||||
if (typeof value !== 'string' || !value.trim() || value.length > maximum || /<\/?[a-z][^>]*>/i.test(value)) {
|
||||
errors.push(`${path} must be non-empty plain text with at most ${maximum} characters.`);
|
||||
return undefined;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function enumeration(value: unknown, allowed: readonly string[], path: string, errors: string[]): void {
|
||||
if (typeof value !== 'string' || !allowed.includes(value)) errors.push(`${path} is invalid.`);
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is Record<string, any> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
|
@ -1,8 +1,13 @@
|
|||
export const REGULATORY_CHUNK_SCHEMA_VERSION = 1 as const;
|
||||
export const REGULATORY_CHUNK_SCHEMA_VERSION = 2 as const;
|
||||
export const REGULATORY_FINDING_QUERY_SCHEMA_VERSION = 1 as const;
|
||||
|
||||
export type RegulatoryDocumentSchemaVersion = 1 | 2;
|
||||
export type RegulatoryControlTheme = 'organizational' | 'people' | 'physical' | 'technological';
|
||||
export type RegulatorySectionKind = 'attributes' | 'control' | 'purpose' | 'guidance' | 'other-information';
|
||||
export type RegulatoryRetrievalRole = 'semantic' | 'reference';
|
||||
|
||||
export interface RegulatoryControlDocument {
|
||||
schemaVersion: 1;
|
||||
schemaVersion: RegulatoryDocumentSchemaVersion;
|
||||
corpusId: string;
|
||||
corpusVersion: string;
|
||||
documentId: string;
|
||||
|
|
@ -13,10 +18,40 @@ export interface RegulatoryControlDocument {
|
|||
title: string;
|
||||
text: string;
|
||||
sourceUri?: string;
|
||||
annexAControlId?: string;
|
||||
controlTheme?: RegulatoryControlTheme;
|
||||
sourceStandard?: 'ISO/IEC 27001' | 'ISO/IEC 27002';
|
||||
sourceEdition?: string;
|
||||
nationalEdition?: string;
|
||||
sectionKind?: RegulatorySectionKind;
|
||||
sourcePageStart?: number;
|
||||
sourcePageEnd?: number;
|
||||
retrievalRole?: RegulatoryRetrievalRole;
|
||||
parserVersion?: string;
|
||||
sourceSha256?: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryCorpusSource {
|
||||
standard: 'ISO/IEC 27001' | 'ISO/IEC 27002';
|
||||
sourceEdition: string;
|
||||
nationalEdition: string;
|
||||
sha256: string;
|
||||
pageCount: number;
|
||||
}
|
||||
|
||||
export interface RegulatoryCorpusV2 {
|
||||
schemaVersion: 2;
|
||||
corpusId: string;
|
||||
corpusVersion: string;
|
||||
language: string;
|
||||
parserVersion: string;
|
||||
generatedAt: string;
|
||||
sources: RegulatoryCorpusSource[];
|
||||
documents: RegulatoryControlDocument[];
|
||||
}
|
||||
|
||||
export interface RegulatoryChunk {
|
||||
schemaVersion: 1;
|
||||
schemaVersion: RegulatoryDocumentSchemaVersion;
|
||||
corpusId: string;
|
||||
corpusVersion: string;
|
||||
documentId: string;
|
||||
|
|
@ -30,6 +65,17 @@ export interface RegulatoryChunk {
|
|||
text: string;
|
||||
textSha256: string;
|
||||
sourceUri?: string;
|
||||
annexAControlId?: string;
|
||||
controlTheme?: RegulatoryControlTheme;
|
||||
sourceStandard?: 'ISO/IEC 27001' | 'ISO/IEC 27002';
|
||||
sourceEdition?: string;
|
||||
nationalEdition?: string;
|
||||
sectionKind?: RegulatorySectionKind;
|
||||
sourcePageStart?: number;
|
||||
sourcePageEnd?: number;
|
||||
retrievalRole?: RegulatoryRetrievalRole;
|
||||
parserVersion?: string;
|
||||
sourceSha256?: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalFilters {
|
||||
|
|
@ -39,6 +85,9 @@ export interface RegulatoryRetrievalFilters {
|
|||
frameworkVersion?: string;
|
||||
controlId?: string;
|
||||
language?: string;
|
||||
sourceStandard?: string;
|
||||
sectionKind?: RegulatorySectionKind;
|
||||
retrievalRole?: RegulatoryRetrievalRole;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalRequest {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import type { UnifiedResultProjection } from '../domain/unified-result';
|
||||
import { POLICY_CLASSIFICATION_LABELS } from '../domain/policy-status-labels';
|
||||
import { discloseResultDbProjection } from '../domain/result-db-disclosure';
|
||||
import { privacySafeFinding, privacySafeText } from '../domain/privacy-safe-finding';
|
||||
|
||||
const SAFE_ENVIRONMENT_PROPERTIES = new Set([
|
||||
'domain.controller_count',
|
||||
|
|
@ -42,23 +43,7 @@ export function resultsToPrivacySafeExportJson(items: UnifiedResultProjection[])
|
|||
policyStatuses: countBy(policyFindings, policyStatus),
|
||||
environmentStates: countBy(environment, item => item.environmentState ?? 'unknown')
|
||||
},
|
||||
findings: actionable.map(item => {
|
||||
const description = copiedText(item.description);
|
||||
const recommendation = copiedText(item.recommendation);
|
||||
return {
|
||||
id: item.sourceRuleId ?? null,
|
||||
title: privacySafeText(item.title),
|
||||
...(description ? { description } : {}),
|
||||
...(recommendation ? { recommendation } : {}),
|
||||
severity: item.severityNormalized,
|
||||
outcome: item.outcomeNormalized,
|
||||
resultType: item.resultType,
|
||||
isIoe: item.isIoe,
|
||||
deviation: item.isDeviation,
|
||||
tool: item.sourceTool ?? 'unknown',
|
||||
...privacySafePolicy(item)
|
||||
};
|
||||
}),
|
||||
findings: actionable.map(item => ({ ...privacySafeFinding(item), ...privacySafePolicy(item) })),
|
||||
notRunChecks: notRun.map(item => ({
|
||||
id: item.sourceRuleId ?? null,
|
||||
title: privacySafeText(item.title),
|
||||
|
|
@ -127,15 +112,6 @@ function countBy(items: UnifiedResultProjection[], valueOf: (item: UnifiedResult
|
|||
return counts;
|
||||
}
|
||||
|
||||
function copiedText(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return !trimmed || trimmed.toLocaleLowerCase('en-US') === 'none' ? undefined : privacySafeText(trimmed);
|
||||
}
|
||||
|
||||
function privacySafeText(value: string): string {
|
||||
return value.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, '[redacted-email]');
|
||||
}
|
||||
|
||||
function copiedEnvironmentProperty(item: UnifiedResultProjection): string | undefined {
|
||||
if (item.sourceKind !== 'environment-observation' || !item.environmentProperty) return undefined;
|
||||
if (SAFE_ENVIRONMENT_PROPERTIES.has(item.environmentProperty) || item.environmentProperty.endsWith('.row_count')) {
|
||||
|
|
|
|||
|
|
@ -381,6 +381,8 @@ export function DatasetViewer() {
|
|||
onResults={hasResults(selected) ? () => openResults(selected) : undefined}
|
||||
resultsOpen={Boolean(resultColor(selected.id))}
|
||||
resultColor={resultColor(selected.id)}
|
||||
projectId={app.projectId}
|
||||
backendStatus={app.backendStatus}
|
||||
/>
|
||||
</div>
|
||||
) : view === 'table' ? (
|
||||
|
|
|
|||
|
|
@ -30,6 +30,9 @@ import { policyDiagnosticBadges, policyFindingCategories } from '../../profiles/
|
|||
import { findingWebSearch } from '../../profiles/public-search/search';
|
||||
import { policyEvidenceResultSheets, POLICY_CONFLICT_DETAILS, POLICY_DUPLICATE_DETAILS } from '../../profiles/group-policy/conflict-presentation';
|
||||
import { maesterExecutionErrorText, maesterFindingTitle, maesterTagText } from '../../profiles/maester/presentation';
|
||||
import type { BackendStatus } from '../../app/app-state';
|
||||
import { RegulatoryMappingPanel } from './RegulatoryMappingPanel';
|
||||
import { useRegulatoryMapping } from './use-regulatory-mapping';
|
||||
|
||||
interface Props {
|
||||
record: FindingRecord;
|
||||
|
|
@ -52,6 +55,8 @@ interface Props {
|
|||
resultsOpen?: boolean;
|
||||
/** Farbe des geöffneten Result-Fensters; entspricht der Kachelmarkierung. */
|
||||
resultColor?: string;
|
||||
projectId?: string;
|
||||
backendStatus?: BackendStatus;
|
||||
}
|
||||
|
||||
const statuses = ['Open', 'In Review', 'Clarification Needed', 'Actionable', 'Blocked', 'Done', 'Not Relevant'];
|
||||
|
|
@ -383,7 +388,7 @@ export function policyAnalyzerSearchKeys(record: FindingRecord, title: string, t
|
|||
return keys;
|
||||
}
|
||||
|
||||
export function FindingReader({ record, title, review: storedReview, previousPolicyReviews, linkedSheets, resultReferences, search, position, recordCount, toolId, onPrevious, onNext, onReview, onRiskAcceptance, onResults, resultsOpen, resultColor }: Props) {
|
||||
export function FindingReader({ record, title, review: storedReview, previousPolicyReviews, linkedSheets, resultReferences, search, position, recordCount, toolId, onPrevious, onNext, onReview, onRiskAcceptance, onResults, resultsOpen, resultColor, projectId = '', backendStatus = 'unavailable' }: Props) {
|
||||
const resultButtonStyle = resultsOpen && resultColor ? ({ '--result-color': resultColor } as CSSProperties) : undefined;
|
||||
const [openLinkedResultsFor, setOpenLinkedResultsFor] = useState('');
|
||||
const [riskAcceptanceFor, setRiskAcceptanceFor] = useState('');
|
||||
|
|
@ -407,6 +412,7 @@ export function FindingReader({ record, title, review: storedReview, previousPol
|
|||
|| resultReferences.some(reference => /source-table:\d+/i.test(reference));
|
||||
const updateText = (field: keyof Review) => (event: ChangeEvent<HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement>) => onReview({ [field]: event.target.value });
|
||||
const riskState = review.riskAcceptance ? riskAcceptanceState(review.riskAcceptance, record.findingOccurrenceId) : undefined;
|
||||
const regulatoryMapping = useRegulatoryMapping(projectId, record.findingOccurrenceId, backendStatus);
|
||||
|
||||
return (
|
||||
<article className="finding-reader">
|
||||
|
|
@ -450,11 +456,14 @@ export function FindingReader({ record, title, review: storedReview, previousPol
|
|||
<div className="reader-heading-actions">
|
||||
<FindingContextBadges record={record} isPolicySetting={resolvedToolId === 'group-policy'}/>
|
||||
{webSearch && <a className="btn" href={webSearch.url} target="_blank" rel="noopener noreferrer" referrerPolicy="no-referrer" data-search-specificity={webSearch.specificity} title={`${webSearch.specificity === 'topic' ? 'Topic search — ' : ''}${webSearch.query}`}>AI search</a>}
|
||||
<span className="bookmark-label">Bookmark</span>
|
||||
<button className="bookmark-btn" type="button" aria-pressed={review.include} onClick={() => onReview({ include: !review.include })}><BookmarkIcon filled={review.include} /></button>
|
||||
<button className="btn" type="button" aria-expanded={regulatoryMapping.open} disabled={!regulatoryMapping.open && Boolean(regulatoryMapping.disabledReason)} title={regulatoryMapping.disabledReason} onClick={() => regulatoryMapping.setOpen(!regulatoryMapping.open)}>
|
||||
Map ISO controls{regulatoryMapping.acceptedCount ? ` (${regulatoryMapping.acceptedCount})` : ''}
|
||||
</button>
|
||||
{/* Gleiche Kennzeichnung wie auf den Kacheln: ist das Ergebnisfenster
|
||||
zu diesem Fund offen, traegt die Schaltflaeche den Fensterton. */}
|
||||
<button className={`btn results-detail-btn results-btn${resultsOpen ? ' is-open' : ''}`} style={resultButtonStyle} type="button" disabled={!onResults} onClick={onResults}>Results</button>
|
||||
<span className="bookmark-label">Bookmark</span>
|
||||
<button className="bookmark-btn" type="button" aria-pressed={review.include} onClick={() => onReview({ include: !review.include })}><BookmarkIcon filled={review.include} /></button>
|
||||
</div>
|
||||
</div>
|
||||
{riskAcceptanceFor === record.id && <RiskAcceptanceDialog key={record.id} title={title} occurrenceId={record.findingOccurrenceId} value={review.riskAcceptance} onSave={onRiskAcceptance} onClose={() => setRiskAcceptanceFor('')} />}
|
||||
|
|
@ -507,6 +516,8 @@ export function FindingReader({ record, title, review: storedReview, previousPol
|
|||
: <RichTextValue value={field.value} search={search} label={field.label} onResults={onResults} />}
|
||||
</details>)}
|
||||
</section>
|
||||
{regulatoryMapping.open && <RegulatoryMappingPanel response={regulatoryMapping.response} loading={regulatoryMapping.loading} error={regulatoryMapping.error}
|
||||
onStart={() => void regulatoryMapping.start()} onDecision={(candidate, decision, comment) => void regulatoryMapping.decide(candidate, decision, comment)} />}
|
||||
{/* Work tracking is opt-in below the finding so the first screen states the problem. */}
|
||||
<FindingReferences record={record}/>
|
||||
<details className="reader-tracking">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
import { useState } from 'react';
|
||||
import type {
|
||||
RegulatoryMappingCandidate,
|
||||
RegulatoryMappingDecision,
|
||||
RegulatoryMappingProposal,
|
||||
RegulatoryMappingResponse
|
||||
} from '../../domain/regulatory-mapping';
|
||||
|
||||
interface Props {
|
||||
response?: RegulatoryMappingResponse;
|
||||
loading: boolean;
|
||||
error: string;
|
||||
onStart(): void;
|
||||
onDecision(candidate: RegulatoryMappingCandidate, decision: 'accepted' | 'rejected', comment?: string): void;
|
||||
}
|
||||
|
||||
export function RegulatoryMappingPanel({ response, loading, error, onStart, onDecision }: Props) {
|
||||
const run = response?.run;
|
||||
const proposals = run?.result?.mappings ?? [];
|
||||
const proposalIds = new Set(proposals.map(item => item.controlId));
|
||||
const proposedCandidates = proposals.flatMap(proposal => {
|
||||
const candidate = run?.candidates.find(item => item.controlId === proposal.controlId);
|
||||
return candidate ? [{ candidate, proposal }] : [];
|
||||
});
|
||||
const otherCandidates = run?.candidates.filter(item => !proposalIds.has(item.controlId)) ?? [];
|
||||
|
||||
return <section className="regulatory-mapping-panel" aria-label="ISO control mapping">
|
||||
<header><div><h3>ISO control mapping</h3><p>Proposals indicate relevance to this finding, not proof of compliance.</p></div>
|
||||
{response?.generationEnabled && !['queued', 'running'].includes(response.state) && <button className="btn" type="button" disabled={loading} onClick={onStart}>{run ? 'Generate again' : 'Generate proposals'}</button>}
|
||||
</header>
|
||||
{loading && !response && <p role="status">Loading mapping state…</p>}
|
||||
{error && <p className="regulatory-mapping-error" role="alert">{error}</p>}
|
||||
{response && !response.generationEnabled && <p role="status">{response.disabledReason ?? 'Regulatory mapping is unavailable.'}</p>}
|
||||
{response && ['queued', 'running'].includes(response.state) && <p role="status">{response.state === 'queued' ? 'Mapping queued…' : 'Retrieving and evaluating ISO controls…'}</p>}
|
||||
{run?.status === 'failed' && <p className="regulatory-mapping-error" role="status">Mapping stopped safely ({run.errorCode ?? 'unknown error'}). You can start a new run.{run.errorDetail ? <><br/><span className="regulatory-mapping-error-detail">{run.errorDetail}</span></> : null}</p>}
|
||||
{run?.status === 'succeeded' && run.result?.outcome !== 'mapped' && <p role="status">{run.result?.reason ?? (run.result?.outcome === 'insufficient_context' ? 'Insufficient public context.' : 'No relevant control was proposed.')}</p>}
|
||||
{proposedCandidates.length > 0 && <div className="regulatory-control-list">
|
||||
{proposedCandidates.map(item => <ControlCandidate key={item.candidate.controlId} {...item} decisions={response?.decisions ?? []} loading={loading} onDecision={onDecision}/>) }
|
||||
</div>}
|
||||
{otherCandidates.length > 0 && <details className="regulatory-other-candidates"><summary>Other retrieved controls ({otherCandidates.length})</summary>
|
||||
<div className="regulatory-control-list">{otherCandidates.map(candidate => <ControlCandidate key={candidate.controlId} candidate={candidate} decisions={response?.decisions ?? []} loading={loading} onDecision={onDecision}/>)}</div>
|
||||
</details>}
|
||||
</section>;
|
||||
}
|
||||
|
||||
function ControlCandidate({ candidate, proposal, decisions, loading, onDecision }: {
|
||||
candidate: RegulatoryMappingCandidate;
|
||||
proposal?: RegulatoryMappingProposal;
|
||||
decisions: RegulatoryMappingDecision[];
|
||||
loading: boolean;
|
||||
onDecision(candidate: RegulatoryMappingCandidate, decision: 'accepted' | 'rejected', comment?: string): void;
|
||||
}) {
|
||||
const [comment, setComment] = useState('');
|
||||
const current = decisions.find(item => item.controlId === candidate.controlId);
|
||||
return <article className="regulatory-control-candidate">
|
||||
<header><div><h4>{candidate.annexAControlId} — {candidate.title}</h4>
|
||||
<p>{proposal ? <>{proposal.relationship} · {proposal.confidence} confidence</> : 'Retrieved candidate'} · score {candidate.score.toFixed(3)}</p></div>
|
||||
{current && <span className="status-badge">{current.decision}</span>}
|
||||
</header>
|
||||
{proposal?.rationale && <p>{proposal.rationale}</p>}
|
||||
<details><summary>Evidence ({candidate.evidence.length})</summary>
|
||||
<ul className="regulatory-evidence-list">{candidate.evidence.map(item => <li key={item.chunkId}>
|
||||
{item.sourceStandard} · {item.sectionKind} · {item.pageStart === item.pageEnd ? `page ${item.pageStart}` : `pages ${item.pageStart}–${item.pageEnd}`}
|
||||
</li>)}</ul>
|
||||
</details>
|
||||
<label>Review comment (optional)<input value={comment} maxLength={1000} onChange={event => setComment(event.target.value)} /></label>
|
||||
<div className="regulatory-decision-actions">
|
||||
<button className="btn" type="button" disabled={loading} onClick={() => onDecision(candidate, 'accepted', comment)}>Accept</button>
|
||||
<button className="btn" type="button" disabled={loading} onClick={() => onDecision(candidate, 'rejected', comment)}>Reject</button>
|
||||
</div>
|
||||
</article>;
|
||||
}
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import type { BackendStatus } from '../../app/app-state';
|
||||
import { AssessorApiClient } from '../../api/assessor-api-client';
|
||||
import type {
|
||||
RegulatoryMappingCandidate,
|
||||
RegulatoryMappingDecisionValue,
|
||||
RegulatoryMappingResponse
|
||||
} from '../../domain/regulatory-mapping';
|
||||
|
||||
const assessorApi = new AssessorApiClient();
|
||||
|
||||
export function useRegulatoryMapping(
|
||||
projectId: string,
|
||||
occurrenceId: string | undefined,
|
||||
backendStatus: BackendStatus
|
||||
) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const [response, setResponse] = useState<RegulatoryMappingResponse>();
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState('');
|
||||
const resultProjectionId = occurrenceId ? `scanner-finding:${occurrenceId}` : '';
|
||||
|
||||
useEffect(() => {
|
||||
setOpen(false);
|
||||
setResponse(undefined);
|
||||
setLoading(false);
|
||||
setError('');
|
||||
}, [projectId, resultProjectionId]);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
if (!projectId || !resultProjectionId || backendStatus !== 'connected') return;
|
||||
setLoading(true);
|
||||
try {
|
||||
setResponse(await assessorApi.getRegulatoryMappings(projectId, resultProjectionId));
|
||||
setError('');
|
||||
} catch (reason) {
|
||||
setError(reason instanceof Error ? reason.message : 'Regulatory mapping could not be loaded.');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [backendStatus, projectId, resultProjectionId]);
|
||||
|
||||
useEffect(() => { if (open && !response && !loading && !error) void load(); }, [error, load, loading, open, response]);
|
||||
useEffect(() => {
|
||||
if (!open || !response || !['queued', 'running'].includes(response.state)) return;
|
||||
const timeout = window.setTimeout(() => void load(), response.pollAfterMs ?? 1_500);
|
||||
return () => window.clearTimeout(timeout);
|
||||
}, [load, open, response]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (!projectId || !resultProjectionId) return;
|
||||
setLoading(true);
|
||||
try {
|
||||
setResponse(await assessorApi.generateRegulatoryMappings(projectId, { resultProjectionId }));
|
||||
setError('');
|
||||
} catch (reason) {
|
||||
setError(reason instanceof Error ? reason.message : 'Regulatory mapping could not be started.');
|
||||
} finally { setLoading(false); }
|
||||
}, [projectId, resultProjectionId]);
|
||||
|
||||
const decide = useCallback(async (candidate: RegulatoryMappingCandidate, decision: RegulatoryMappingDecisionValue, comment?: string) => {
|
||||
if (!projectId || !resultProjectionId || !response?.run) return;
|
||||
const latest = response.decisions.find(item => item.controlId === candidate.controlId);
|
||||
setLoading(true);
|
||||
try {
|
||||
setResponse(await assessorApi.saveRegulatoryMappingDecision(projectId, {
|
||||
resultProjectionId,
|
||||
controlId: candidate.controlId,
|
||||
decision,
|
||||
...(comment?.trim() ? { comment: comment.trim() } : {}),
|
||||
...(latest ? { expectedLatestDecisionId: latest.id } : {})
|
||||
}));
|
||||
setError('');
|
||||
} catch (reason) {
|
||||
setError(reason instanceof Error ? reason.message : 'The mapping decision could not be saved.');
|
||||
} finally { setLoading(false); }
|
||||
}, [projectId, response, resultProjectionId]);
|
||||
|
||||
const acceptedCount = useMemo(() => {
|
||||
const latest = new Map<string, RegulatoryMappingDecisionValue>();
|
||||
response?.decisions.forEach(item => { if (!latest.has(item.controlId)) latest.set(item.controlId, item.decision); });
|
||||
return [...latest.values()].filter(value => value === 'accepted').length;
|
||||
}, [response]);
|
||||
|
||||
const localDisabledReason = !occurrenceId ? 'This record has no canonical finding occurrence.'
|
||||
: backendStatus !== 'connected' ? 'The backend is not connected.'
|
||||
: response && !response.generationEnabled ? response.disabledReason ?? 'Regulatory mapping is unavailable.' : undefined;
|
||||
|
||||
return {
|
||||
open,
|
||||
setOpen,
|
||||
response,
|
||||
loading,
|
||||
error,
|
||||
start,
|
||||
decide,
|
||||
acceptedCount,
|
||||
disabledReason: localDisabledReason
|
||||
};
|
||||
}
|
||||
|
|
@ -35,6 +35,7 @@ import type {
|
|||
} from '../domain/unified-result';
|
||||
import type { EnvironmentObservation, EnvironmentSnapshot, EnvironmentSnapshotQuery, ScannerReportSection } from '../domain/environment-context';
|
||||
import type { AiOverviewRun } from '../domain/ai-overview';
|
||||
import type { RegulatoryMappingDecision, RegulatoryMappingRun } from '../domain/regulatory-mapping';
|
||||
|
||||
export interface FindingOccurrenceQuery {
|
||||
projectId?: string;
|
||||
|
|
@ -146,6 +147,8 @@ export interface ProjectDataExport {
|
|||
environmentProfiles: EnvironmentProfile[];
|
||||
policySnapshotAggregations: StoredPolicySnapshotAggregation[];
|
||||
aiOverviewRuns: AiOverviewRun[];
|
||||
regulatoryMappingRuns: RegulatoryMappingRun[];
|
||||
regulatoryMappingDecisions: RegulatoryMappingDecision[];
|
||||
}
|
||||
|
||||
export interface ControlledDeleteConfirmation {
|
||||
|
|
@ -220,6 +223,11 @@ export interface AssessorRepository {
|
|||
createAiOverviewRun(run: AiOverviewRun): Promise<AiOverviewRun>;
|
||||
updateAiOverviewRun(run: AiOverviewRun): Promise<AiOverviewRun>;
|
||||
listAiOverviewRuns(projectId: string, scopeKey?: string): Promise<AiOverviewRun[]>;
|
||||
getRegulatoryMappingRun(id: string): Promise<RegulatoryMappingRun | null>;
|
||||
saveRegulatoryMappingRun(run: RegulatoryMappingRun): Promise<RegulatoryMappingRun>;
|
||||
listRegulatoryMappingRuns(): Promise<RegulatoryMappingRun[]>;
|
||||
createRegulatoryMappingDecision(decision: RegulatoryMappingDecision, expectedLatestDecisionId?: string): Promise<RegulatoryMappingDecision>;
|
||||
listRegulatoryMappingDecisions(ruleKey: string): Promise<RegulatoryMappingDecision[]>;
|
||||
|
||||
saveImportArtifact(artifact: ImportArtifact): Promise<void>;
|
||||
saveImportArtifactAndEnqueueJob(artifact: ImportArtifact, job: IngestionJob): Promise<ArtifactJobCommitResult>;
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ import type { EnvironmentObservation, EnvironmentSnapshot, EnvironmentSnapshotQu
|
|||
import { environmentCoverage } from '../domain/environment-context';
|
||||
import type { AiOverviewRun } from '../domain/ai-overview';
|
||||
import { aiOverviewRunValidationErrors } from '../domain/ai-overview';
|
||||
import type { RegulatoryMappingDecision, RegulatoryMappingRun } from '../domain/regulatory-mapping';
|
||||
import { regulatoryMappingDecisionValidationErrors, regulatoryMappingRunValidationErrors } from '../domain/regulatory-mapping';
|
||||
import { storedPolicyFindingRunValidationErrors } from '../domain/policy-finding';
|
||||
import {
|
||||
environmentProfileValidationErrors,
|
||||
|
|
@ -103,6 +105,8 @@ interface LocalRepositoryState {
|
|||
findingDefinitions: FindingDefinition[];
|
||||
findingOccurrences: FindingOccurrence[];
|
||||
aiOverviewRuns: AiOverviewRun[];
|
||||
regulatoryMappingRuns: RegulatoryMappingRun[];
|
||||
regulatoryMappingDecisions: RegulatoryMappingDecision[];
|
||||
evidence: EvidenceRecord[];
|
||||
reviews: FindingReview[];
|
||||
diagnostics: ImportDiagnostic[];
|
||||
|
|
@ -138,6 +142,8 @@ const emptyState = (): LocalRepositoryState => ({
|
|||
findingDefinitions: [],
|
||||
findingOccurrences: [],
|
||||
aiOverviewRuns: [],
|
||||
regulatoryMappingRuns: [],
|
||||
regulatoryMappingDecisions: [],
|
||||
evidence: [],
|
||||
reviews: [],
|
||||
diagnostics: [],
|
||||
|
|
@ -426,6 +432,49 @@ export class LocalAssessorRepository implements AssessorRepository {
|
|||
.sort((left, right) => right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id));
|
||||
}
|
||||
|
||||
async getRegulatoryMappingRun(id: string): Promise<RegulatoryMappingRun | null> {
|
||||
const run = this.state.regulatoryMappingRuns.find(item => item.id === id);
|
||||
return run ? clone(run) : null;
|
||||
}
|
||||
|
||||
async saveRegulatoryMappingRun(run: RegulatoryMappingRun): Promise<RegulatoryMappingRun> {
|
||||
const errors = regulatoryMappingRunValidationErrors(run);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
const next = clone(this.state);
|
||||
const index = next.regulatoryMappingRuns.findIndex(item => item.id === run.id);
|
||||
if (index >= 0) next.regulatoryMappingRuns[index] = clone(run);
|
||||
else next.regulatoryMappingRuns.push(clone(run));
|
||||
this.replaceState(next);
|
||||
return clone(run);
|
||||
}
|
||||
|
||||
async listRegulatoryMappingRuns(): Promise<RegulatoryMappingRun[]> {
|
||||
return this.state.regulatoryMappingRuns.map(clone)
|
||||
.sort((left, right) => right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id));
|
||||
}
|
||||
|
||||
async createRegulatoryMappingDecision(decision: RegulatoryMappingDecision, expectedLatestDecisionId?: string): Promise<RegulatoryMappingDecision> {
|
||||
const errors = regulatoryMappingDecisionValidationErrors(decision);
|
||||
if (errors.length) throw new RepositoryValidationError(errors.join(' '));
|
||||
const run = this.state.regulatoryMappingRuns.find(item => item.id === decision.mappingId);
|
||||
if (!run || run.ruleKey !== decision.ruleKey || run.status !== 'succeeded') {
|
||||
throw new RepositoryNotFoundError('Completed regulatory mapping does not exist for this rule.');
|
||||
}
|
||||
if (!run.candidates.some(candidate => candidate.controlId === decision.controlId)) throw new RepositoryValidationError('Decision control was not a grounded candidate.');
|
||||
const latest = this.state.regulatoryMappingDecisions.filter(item => item.ruleKey === decision.ruleKey && item.controlId === decision.controlId)
|
||||
.sort((left, right) => right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id))[0];
|
||||
if ((expectedLatestDecisionId ?? undefined) !== (latest?.id ?? undefined)) throw new RepositoryConflictError('Regulatory mapping decision changed before this update.');
|
||||
const next = clone(this.state);
|
||||
next.regulatoryMappingDecisions.push(clone({ ...decision, ...(latest ? { previousDecisionId: latest.id } : {}) }));
|
||||
this.replaceState(next);
|
||||
return clone(next.regulatoryMappingDecisions.at(-1)!);
|
||||
}
|
||||
|
||||
async listRegulatoryMappingDecisions(ruleKey: string): Promise<RegulatoryMappingDecision[]> {
|
||||
return this.state.regulatoryMappingDecisions.filter(item => item.ruleKey === ruleKey)
|
||||
.map(clone).sort((left, right) => right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id));
|
||||
}
|
||||
|
||||
async beginImport(start: ImportSessionStart): Promise<ImportSession> {
|
||||
if (!start.sessionId.trim()) throw new RepositoryValidationError('Eine Importsitzung benötigt eine ID.');
|
||||
if (this.activeImportSessions.has(start.sessionId)) throw new RepositoryConflictError(`Importsitzung ${start.sessionId} ist bereits aktiv.`);
|
||||
|
|
@ -1538,7 +1587,10 @@ export class LocalAssessorRepository implements AssessorRepository {
|
|||
policySnapshotAggregations: this.state.policySnapshotAggregations.filter(item => item.run.projectId === projectId).map(clone),
|
||||
policyEvaluations: this.state.policyEvaluations.filter(item => item.run.projectId === projectId).map(clone),
|
||||
policyFindingRuns: policyFindingRuns.map(clone),
|
||||
aiOverviewRuns: this.state.aiOverviewRuns.filter(item => item.projectId === projectId && item.status === 'succeeded').map(clone)
|
||||
aiOverviewRuns: this.state.aiOverviewRuns.filter(item => item.projectId === projectId && item.status === 'succeeded').map(clone),
|
||||
// Regulatory mappings and decisions are global reference data, not project-scoped.
|
||||
regulatoryMappingRuns: this.state.regulatoryMappingRuns.filter(item => item.status === 'succeeded').map(clone),
|
||||
regulatoryMappingDecisions: this.state.regulatoryMappingDecisions.map(clone)
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -1601,6 +1653,7 @@ export class LocalAssessorRepository implements AssessorRepository {
|
|||
next.gpoPolicyAnalyses = next.gpoPolicyAnalyses.filter(item => !processingRunIds.has(item.run.sourceProcessingRunId));
|
||||
next.artifactDeletionRequests = next.artifactDeletionRequests.filter(item => !artifactIds.has(item.artifactId));
|
||||
next.aiOverviewRuns = next.aiOverviewRuns.filter(item => !item.scanRunIds.includes(scanRunId));
|
||||
// Regulatory mappings and decisions are global and survive scan-run removal.
|
||||
const remainingHashes = new Set(next.importFiles.map(item => item.sha256).filter(Boolean) as string[]);
|
||||
next.importFingerprints = next.importFingerprints.filter(item => !removedHashes.has(item.sha256) || remainingHashes.has(item.sha256));
|
||||
this.replaceState(next);
|
||||
|
|
@ -1668,6 +1721,7 @@ export class LocalAssessorRepository implements AssessorRepository {
|
|||
next.policyFindingRuns = next.policyFindingRuns.filter(item => !evaluationRunIds.has(item.run.evaluationRunId));
|
||||
next.artifactDeletionRequests = next.artifactDeletionRequests.filter(item => !artifactIds.has(item.artifactId));
|
||||
next.aiOverviewRuns = next.aiOverviewRuns.filter(item => !item.assessmentIds.includes(assessmentId));
|
||||
// Regulatory mappings and decisions are global and survive assessment removal.
|
||||
const remainingDefinitionIds = new Set([
|
||||
...next.findingOccurrences.map(item => item.definitionId),
|
||||
...next.policyFindingRuns.flatMap(item => item.occurrences.map(occurrence => occurrence.definitionId))
|
||||
|
|
@ -1724,6 +1778,7 @@ export class LocalAssessorRepository implements AssessorRepository {
|
|||
next.policySnapshotAggregations = next.policySnapshotAggregations.filter(item => item.run.projectId !== projectId);
|
||||
next.policyEvaluations = next.policyEvaluations.filter(item => item.run.projectId !== projectId);
|
||||
next.aiOverviewRuns = next.aiOverviewRuns.filter(item => item.projectId !== projectId);
|
||||
// Regulatory mappings and decisions are global and survive project deletion.
|
||||
this.replaceState(next);
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
import type { RegulatoryFindingQuery } from '../../domain/regulatory-rag';
|
||||
import { REGULATORY_FINDING_QUERY_SCHEMA_VERSION } from '../../domain/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../domain/unified-result';
|
||||
import { privacySafeFinding } from '../../domain/privacy-safe-finding';
|
||||
import catalog from './catalog.json';
|
||||
import { publicSecurityTopics } from './public-security-topics';
|
||||
|
||||
const MAXIMUM_TITLE_CHARACTERS = 500;
|
||||
const MAXIMUM_DETAIL_CHARACTERS = 1_200;
|
||||
const MAXIMUM_RECOMMENDATION_CHARACTERS = 600;
|
||||
|
||||
const publicTools: Record<string, string> = {
|
||||
'cis-cat': 'CIS-CAT / CIS Benchmarks',
|
||||
maester: 'Maester',
|
||||
|
|
@ -17,30 +22,38 @@ const publicRuleIds: Record<string, readonly string[]> = {
|
|||
};
|
||||
|
||||
/**
|
||||
* Builds an embedding query only from catalog-owned identifiers and fixed public
|
||||
* concepts. Raw finding prose is inspected locally for topic recognition and is
|
||||
* never copied into the returned query.
|
||||
* Builds an embedding query from the finding's rule-level check text — its title,
|
||||
* description and recommendation projected through the shared privacy boundary
|
||||
* ({@link privacySafeFinding}). That text describes the check, not the tenant: the
|
||||
* result-table / occurrence data (affectedAsset, per-object values) is never read,
|
||||
* and email addresses are redacted. Every scanner finding with a check title maps;
|
||||
* catalog rule IDs and fixed public topics only sharpen the query when present.
|
||||
*/
|
||||
export function buildRegulatoryFindingQuery(finding: UnifiedResultProjection): RegulatoryFindingQuery | undefined {
|
||||
if (finding.sourceKind !== 'scanner-finding') return undefined;
|
||||
const safe = privacySafeFinding(finding);
|
||||
const title = safe.title.trim().slice(0, MAXIMUM_TITLE_CHARACTERS);
|
||||
if (!title) return undefined;
|
||||
const toolId = finding.sourceTool?.trim().toLowerCase() ?? '';
|
||||
const publicTool = publicTools[toolId];
|
||||
const publicRuleId = recognizedRuleId(toolId, finding.sourceRuleId);
|
||||
const evidence = [finding.title, finding.description, finding.recommendation].filter(Boolean).join('\n').slice(0, 20_000);
|
||||
const topics = publicSecurityTopics(evidence);
|
||||
if (!publicRuleId && !topics.length) return undefined;
|
||||
const specificity = publicRuleId ? 'rule' : 'topic';
|
||||
const detail = safe.description?.slice(0, MAXIMUM_DETAIL_CHARACTERS);
|
||||
const recommendation = safe.recommendation?.slice(0, MAXIMUM_RECOMMENDATION_CHARACTERS);
|
||||
const topics = publicSecurityTopics([title, detail, recommendation].filter(Boolean).join('\n'));
|
||||
const query = [
|
||||
'Retrieve regulatory controls whose normative requirements directly address this security finding.',
|
||||
publicTool && `Public assessment tool: ${publicTool}.`,
|
||||
publicRuleId && `Public rule ID: ${publicRuleId}.`,
|
||||
`Security check: ${title}.`,
|
||||
detail && `Check detail: ${detail}.`,
|
||||
recommendation && `Recommended action: ${recommendation}.`,
|
||||
topics.length && `Public technical topics: ${topics.join('; ')}.`,
|
||||
'Prefer controls with direct requirement text over merely related guidance. Do not infer customer scope, assets, configuration values, or compliance status.'
|
||||
].filter(Boolean).join(' ');
|
||||
return {
|
||||
schemaVersion: REGULATORY_FINDING_QUERY_SCHEMA_VERSION,
|
||||
query,
|
||||
specificity,
|
||||
specificity: publicRuleId ? 'rule' : 'topic',
|
||||
...(publicTool ? { publicTool } : {}),
|
||||
...(publicRuleId ? { publicRuleId } : {}),
|
||||
publicTopics: topics
|
||||
|
|
|
|||
|
|
@ -361,6 +361,22 @@ mark, .search-highlight {
|
|||
.reader-summary p { margin: 0; color: var(--text); font-size: var(--ui-reading-size); line-height: 1.55; white-space: pre-wrap; }
|
||||
.review-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 14px; padding: 20px; border: 1px solid var(--line); background: var(--panel); }
|
||||
.review-grid textarea { min-height: 96px; resize: vertical; }
|
||||
.regulatory-mapping-panel { margin-top: 14px; padding: 20px; border: 1px solid var(--line); background: var(--panel); }
|
||||
.regulatory-mapping-panel > header, .regulatory-control-candidate > header { display: flex; align-items: flex-start; justify-content: space-between; gap: 16px; }
|
||||
.regulatory-mapping-panel h3, .regulatory-mapping-panel h4, .regulatory-mapping-panel p { margin-top: 0; }
|
||||
.regulatory-mapping-panel > header p, .regulatory-control-candidate > header p { color: var(--text-muted); font-size: var(--ui-small-size); }
|
||||
.regulatory-control-list { display: grid; gap: 12px; margin-top: 16px; }
|
||||
.regulatory-control-candidate { padding: 16px; background: var(--tile-surface); }
|
||||
.regulatory-control-candidate > p { font-size: 16px; line-height: 1.55; }
|
||||
.regulatory-control-candidate > label { display: grid; gap: 7px; margin-top: 14px; color: var(--heading); font-size: var(--ui-label-size); font-weight: var(--fw-semibold); }
|
||||
.regulatory-control-candidate input { width: 100%; min-height: 39px; padding: 8px 11px; border: var(--border-hair) solid var(--line-strong); background: var(--surface-inset); color: var(--text); }
|
||||
.regulatory-decision-actions { display: flex; gap: 10px; margin-top: 10px; }
|
||||
.regulatory-other-candidates { margin-top: 16px; }
|
||||
.regulatory-other-candidates > summary, .regulatory-control-candidate > details > summary { cursor: pointer; font-size: var(--ui-control-size); font-weight: var(--fw-semibold); }
|
||||
.regulatory-evidence-list { list-style: none; display: grid; gap: 6px; margin-top: 12px; padding: 0; }
|
||||
.regulatory-evidence-list > li { color: var(--text-muted); font-size: var(--ui-small-size); }
|
||||
.regulatory-mapping-error { color: var(--danger); }
|
||||
.regulatory-mapping-error-detail { color: var(--text-muted); font-size: var(--ui-small-size); }
|
||||
.review-markers { display: flex; flex-wrap: wrap; gap: 10px 18px; grid-column: 1 / -1; margin: 0; padding: 12px; border: 1px solid var(--line); }
|
||||
.review-markers label { color: var(--text); font-size: var(--ui-control-size); }
|
||||
.reader-tracking { margin-top: 14px; padding: 20px 18px; border: 1px solid var(--line); background: var(--panel); }
|
||||
|
|
|
|||
|
|
@ -89,6 +89,30 @@ describe('AssessorApiClient', () => {
|
|||
expect(fetcher.mock.calls[1][0]).toBe('/api/v1/projects/project%2Fone/ai-overview');
|
||||
expect(new Headers(fetcher.mock.calls[1][1]?.headers).get('X-Assessor-Request')).toBe('1');
|
||||
});
|
||||
it('keeps regulatory retrieval and decisions behind project-bound same-origin APIs', async () => {
|
||||
const fetcher = vi.fn<typeof fetch>().mockResolvedValue(new Response(JSON.stringify({
|
||||
version: 1, projectId: 'project/one', resultProjectionId: 'scanner-finding:occurrence/one',
|
||||
generationEnabled: true, state: 'empty', decisions: []
|
||||
}), { status: 200, headers: { 'Content-Type': 'application/json' } }));
|
||||
const client = new AssessorApiClient({ fetcher });
|
||||
|
||||
await client.getRegulatoryMappings('project/one', 'scanner-finding:occurrence/one');
|
||||
await client.generateRegulatoryMappings('project/one', {
|
||||
resultProjectionId: 'scanner-finding:occurrence/one'
|
||||
});
|
||||
await client.saveRegulatoryMappingDecision('project/one', {
|
||||
resultProjectionId: 'scanner-finding:occurrence/one', controlId: '5.1', decision: 'accepted'
|
||||
});
|
||||
|
||||
expect(fetcher.mock.calls.map(call => call[0])).toEqual([
|
||||
'/api/v1/projects/project%2Fone/regulatory-mappings/scanner-finding%3Aoccurrence%2Fone',
|
||||
'/api/v1/projects/project%2Fone/regulatory-mapping-runs',
|
||||
'/api/v1/projects/project%2Fone/regulatory-mapping-decisions'
|
||||
]);
|
||||
expect(new Headers(fetcher.mock.calls[0][1]?.headers).has('X-Assessor-Request')).toBe(false);
|
||||
expect(new Headers(fetcher.mock.calls[1][1]?.headers).get('X-Assessor-Request')).toBe('1');
|
||||
expect(new Headers(fetcher.mock.calls[2][1]?.headers).get('X-Assessor-Request')).toBe('1');
|
||||
});
|
||||
it('submits GPO backups only through the project-bound mutation endpoint', async () => {
|
||||
const fetcher = vi.fn<typeof fetch>().mockResolvedValue(new Response(JSON.stringify({
|
||||
version: 1, artifactId: 'artifact-1', jobId: 'job-1', sha256: 'a'.repeat(64), sizeBytes: 3,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { groundRegulatoryMappingResult, regulatoryMappingResultValidationErrors } from '../../src/domain/regulatory-mapping';
|
||||
|
||||
const grounded = new Map([['5.1', new Set(['semantic-5.1', 'reference-5.1'])]]);
|
||||
|
||||
describe('regulatory mapping output validation', () => {
|
||||
it('accepts a grounded proposal and rejects unknown controls, duplicate controls, and foreign evidence', () => {
|
||||
const valid = {
|
||||
schemaVersion: 1, outcome: 'mapped', mappings: [{
|
||||
controlId: '5.1', relationship: 'primary', confidence: 'high', rationale: 'Synthetic rationale.',
|
||||
evidenceChunkIds: ['semantic-5.1', 'reference-5.1']
|
||||
}]
|
||||
};
|
||||
expect(regulatoryMappingResultValidationErrors(valid, grounded)).toEqual([]);
|
||||
expect(regulatoryMappingResultValidationErrors({ ...valid, mappings: [
|
||||
...valid.mappings, { ...valid.mappings[0], evidenceChunkIds: ['other'] }
|
||||
] }, grounded)).toEqual(expect.arrayContaining([
|
||||
expect.stringContaining('duplicates control 5.1'),
|
||||
expect.stringContaining('not grounded')
|
||||
]));
|
||||
expect(regulatoryMappingResultValidationErrors({ ...valid, mappings: [{ ...valid.mappings[0], controlId: '9.9' }] }, grounded))
|
||||
.toEqual(expect.arrayContaining([expect.stringContaining('unknown candidate control')]));
|
||||
});
|
||||
|
||||
it('trims ungrounded evidence ids, drops fully ungrounded known-control mappings, and leaves unknown controls for validation', () => {
|
||||
const result = groundRegulatoryMappingResult({
|
||||
schemaVersion: 1, outcome: 'mapped', mappings: [
|
||||
{ controlId: '5.1', relationship: 'primary', confidence: 'high', rationale: 'One good id kept.', evidenceChunkIds: ['semantic-5.1', 'foreign'] },
|
||||
{ controlId: '5.1', relationship: 'supporting', confidence: 'low', rationale: 'No real evidence.', evidenceChunkIds: ['foreign', 'other'] }
|
||||
]
|
||||
}, grounded) as { mappings: Array<{ evidenceChunkIds: string[] }> };
|
||||
expect(result.mappings).toHaveLength(1);
|
||||
expect(result.mappings[0].evidenceChunkIds).toEqual(['semantic-5.1']);
|
||||
|
||||
const unknown = groundRegulatoryMappingResult({
|
||||
schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: '9.9', relationship: 'primary', confidence: 'high', rationale: 'x', evidenceChunkIds: ['semantic-5.1'] }]
|
||||
}, grounded) as { mappings: unknown[] };
|
||||
expect(unknown.mappings).toHaveLength(1);
|
||||
expect(regulatoryMappingResultValidationErrors(unknown, grounded)).toEqual(expect.arrayContaining([expect.stringContaining('unknown candidate control')]));
|
||||
});
|
||||
|
||||
it('requires refusal outcomes to contain no mappings and a bounded reason', () => {
|
||||
expect(regulatoryMappingResultValidationErrors({
|
||||
schemaVersion: 1, outcome: 'no_relevant_control', reason: 'No direct relationship.', mappings: []
|
||||
}, grounded)).toEqual([]);
|
||||
expect(regulatoryMappingResultValidationErrors({
|
||||
schemaVersion: 1, outcome: 'insufficient_context', reason: 'Unknown.', mappings: [{
|
||||
controlId: '5.1', relationship: 'supporting', confidence: 'low', rationale: 'Synthetic.', evidenceChunkIds: ['semantic-5.1']
|
||||
}]
|
||||
}, grounded)).toContain('A refusal outcome must not include mappings.');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import { renderToStaticMarkup } from 'react-dom/server';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { RegulatoryMappingPanel } from '../../src/features/finding-reader/RegulatoryMappingPanel';
|
||||
import type { RegulatoryMappingResponse } from '../../src/domain/regulatory-mapping';
|
||||
|
||||
const response: RegulatoryMappingResponse = {
|
||||
version: 1, projectId: 'project-1', resultProjectionId: 'scanner-finding:occurrence-1', generationEnabled: true,
|
||||
state: 'succeeded', decisions: [], run: {
|
||||
version: 1, id: 'regulatory-mapping-abc', ruleKey: 'purple-knight:A-SmbSigning', sourceTool: 'purple-knight', sourceRuleId: 'A-SmbSigning',
|
||||
status: 'succeeded', queryHash: 'a'.repeat(64),
|
||||
preparedQuery: { schemaVersion: 1, query: 'Synthetic public query.', specificity: 'topic', publicTopics: ['Synthetic topic'] },
|
||||
corpusId: 'iso-test', corpusVersion: 'v2', sourceHashes: ['b'.repeat(64)], embeddingModel: 'bge-m3',
|
||||
retrievalVersion: 'test', promptVersion: 'test', outputSchemaVersion: 1, model: 'work-model',
|
||||
createdAt: '2026-09-15T10:00:00.000Z', updatedAt: '2026-09-15T10:00:00.000Z',
|
||||
candidates: [{
|
||||
controlId: '5.1', annexAControlId: 'A.5.1', title: 'Synthetic title', score: 0.91,
|
||||
evidence: [{ chunkId: 'chunk-1', textSha256: 'c'.repeat(64), sourceSha256: 'b'.repeat(64), sourceStandard: 'ISO/IEC 27002',
|
||||
sectionKind: 'purpose', pageStart: 12, pageEnd: 13, text: 'Synthetic evidence text.' }]
|
||||
}, {
|
||||
controlId: '5.2', annexAControlId: 'A.5.2', title: 'Other synthetic title', score: 0.72,
|
||||
evidence: [{ chunkId: 'chunk-2', textSha256: 'd'.repeat(64), sourceSha256: 'b'.repeat(64), sourceStandard: 'ISO/IEC 27001',
|
||||
sectionKind: 'control', pageStart: 14, pageEnd: 14, text: 'Other synthetic evidence.' }]
|
||||
}],
|
||||
result: { schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: '5.1', relationship: 'primary', confidence: 'high',
|
||||
rationale: 'Synthetic rationale.', evidenceChunkIds: ['chunk-1'] }] }
|
||||
}
|
||||
};
|
||||
|
||||
describe('regulatory mapping panel', () => {
|
||||
it('renders proposed controls before collapsed other candidates with page-referenced evidence and review actions', () => {
|
||||
const html = renderToStaticMarkup(<RegulatoryMappingPanel response={response} loading={false} error="" onStart={() => {}} onDecision={() => {}}/>);
|
||||
expect(html.indexOf('A.5.1 — Synthetic title')).toBeLessThan(html.indexOf('Other retrieved controls'));
|
||||
expect(html).not.toContain('5.1 · A.5.1');
|
||||
expect(html).toContain('primary · high confidence');
|
||||
expect(html).toContain('pages 12–13');
|
||||
expect(html).toContain('ISO/IEC 27002 · purpose');
|
||||
expect(html).not.toContain('Synthetic evidence text.');
|
||||
expect(html).toContain('Accept');
|
||||
expect(html).toContain('Reject');
|
||||
expect(html).toContain('not proof of compliance');
|
||||
});
|
||||
|
||||
it('surfaces the structural failure detail alongside the safe error code', () => {
|
||||
const failed: RegulatoryMappingResponse = {
|
||||
...response, state: 'failed',
|
||||
run: { ...response.run!, status: 'failed', result: undefined,
|
||||
errorCode: 'regulatory_mapping_invalid_output', errorDetail: 'mappings[0].evidenceChunkIds[0] is not grounded in the supplied control.' }
|
||||
};
|
||||
const html = renderToStaticMarkup(<RegulatoryMappingPanel response={failed} loading={false} error="" onStart={() => {}} onDecision={() => {}}/>);
|
||||
expect(html).toContain('regulatory_mapping_invalid_output');
|
||||
expect(html).toContain('is not grounded in the supplied control.');
|
||||
});
|
||||
|
||||
it('shows a concrete disabled reason without exposing a generation action', () => {
|
||||
const html = renderToStaticMarkup(<RegulatoryMappingPanel response={{ ...response, generationEnabled: false, state: 'unsupported', disabledReason: 'Qdrant is unavailable.', run: undefined }} loading={false} error="" onStart={() => {}} onDecision={() => {}}/>);
|
||||
expect(html).toContain('Qdrant is unavailable.');
|
||||
expect(html).not.toContain('Generate proposals');
|
||||
});
|
||||
});
|
||||
|
|
@ -44,9 +44,12 @@ assert.doesNotMatch(indexHtml, /<(?:script|link)\b[^>]+(?:src|href)=["']https?:/
|
|||
const packageJson = JSON.parse(read('package.json'));
|
||||
assert.deepEqual(
|
||||
Object.keys(packageJson.dependencies).sort(),
|
||||
['@fontsource-variable/inter', 'jsdom', 'pg', 'react', 'react-dom', 'saxes', 'xlsx'],
|
||||
['@fontsource-variable/inter', 'jsdom', 'pdfjs-dist', 'pg', 'react', 'react-dom', 'saxes', 'xlsx'],
|
||||
'production dependencies changed; review same-origin and local-processing guarantees before accepting new packages'
|
||||
);
|
||||
for (const relativePath of sourceFiles('src')) {
|
||||
assert.doesNotMatch(read(relativePath), /from\s+['"]pdfjs-dist/, `${relativePath} must not load the server-only PDF parser in the browser application`);
|
||||
}
|
||||
|
||||
const excelImporter = read(join('src', 'importers', 'excel-importer.ts'));
|
||||
assert.match(excelImporter, /cellFormula:\s*false/, 'Excel formulas must remain disabled');
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const sql = readFileSync(new URL('../../server/postgres/migrations/026_regulatory_mapping.sql', import.meta.url), 'utf8');
|
||||
const [globalSql, batchSql] = sql.split('CREATE TABLE assessor_regulatory_mapping_batches');
|
||||
|
||||
describe('global regulatory mapping persistence schema', () => {
|
||||
it('creates global, rule-keyed mapping tables beside the project-scoped batch table', () => {
|
||||
|
||||
// Mappings are global: keyed by rule identity plus pipeline versions, never by project or occurrence.
|
||||
expect(sql).toContain('CREATE TABLE assessor_regulatory_mappings');
|
||||
expect(globalSql).not.toContain('project_id');
|
||||
expect(sql).not.toContain('source_result_id');
|
||||
expect(sql).not.toContain('DROP TABLE');
|
||||
expect(batchSql).toContain('project_id text NOT NULL REFERENCES assessor_projects(id) ON DELETE CASCADE');
|
||||
expect(sql).toContain('rule_key text NOT NULL');
|
||||
expect(sql).toContain('UNIQUE (rule_key, corpus_version, retrieval_version, prompt_version, model)');
|
||||
|
||||
// Decisions are global, append-only, keyed by rule identity and control.
|
||||
expect(sql).toContain('CREATE TABLE assessor_regulatory_mapping_decisions');
|
||||
expect(sql).toContain('previous_decision_id text REFERENCES assessor_regulatory_mapping_decisions(id)');
|
||||
expect(sql).toContain("decision text NOT NULL CHECK (decision IN ('accepted', 'rejected'))");
|
||||
expect(sql).toMatch(/assessor_regulatory_mapping_decisions_rule_control_idx[\s\S]*\(rule_key, control_id/);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
parseIso27002Controls,
|
||||
validateIsoControlSets,
|
||||
type ParsedControl,
|
||||
type ParsedPdf,
|
||||
type PdfLine
|
||||
} from '../../server/regulatory/iso-pdf-corpus';
|
||||
|
||||
const line = (text: string, page: number): PdfLine => ({ text, page, y: 500, spans: [{ text, page, x: 100, y: 500, width: text.length * 5, height: 10 }] });
|
||||
|
||||
function controls(): ParsedControl[] {
|
||||
const counts = [[5, 37], [6, 8], [7, 14], [8, 34]];
|
||||
return counts.flatMap(([theme, count]) => Array.from({ length: count }, (_, index) => ({
|
||||
controlId: `${theme}.${index + 1}`,
|
||||
title: `Synthetic ${theme}.${index + 1}`,
|
||||
theme: theme === 5 ? 'organizational' as const : theme === 6 ? 'people' as const : theme === 7 ? 'physical' as const : 'technological' as const,
|
||||
sections: [{ kind: 'control' as const, text: 'Synthetic statement.', pageStart: 1, pageEnd: 1 }]
|
||||
})));
|
||||
}
|
||||
|
||||
describe('ISO PDF corpus structure parser', () => {
|
||||
it('keeps sections inside one control, preserves umlauts, and spans multi-page guidance', () => {
|
||||
const pdf: ParsedPdf = { sha256: 'a'.repeat(64), pageCount: 2, pages: [
|
||||
{ page: 1, lines: [
|
||||
line('5.1 Informationssicherheitsrichtlinien', 1), line('Maßnahmenart', 1), line('präventiv', 1),
|
||||
line('Maßnahme', 1), line('Synthetische Maßnahme.', 1), line('Zweck', 1), line('Synthetischer Zweck.', 1),
|
||||
line('Anleitung', 1), line('Mehrseitige Anleitung mit Umlauten äöü.', 1)
|
||||
] },
|
||||
{ page: 2, lines: [line('DIN EN ISO/IEC 27002:2024-01', 2), line('Fortsetzung der Anleitung.', 2),
|
||||
line('Weitere Informationen', 2), line('Synthetische Referenz.', 2)] }
|
||||
] };
|
||||
|
||||
const parsed = parseIso27002Controls(pdf);
|
||||
expect(parsed).toHaveLength(1);
|
||||
expect(parsed[0].sections.map(section => section.kind)).toEqual(['attributes', 'control', 'purpose', 'guidance', 'other-information']);
|
||||
expect(parsed[0].sections.find(section => section.kind === 'guidance')).toMatchObject({ pageStart: 1, pageEnd: 2 });
|
||||
expect(parsed[0].sections.find(section => section.kind === 'guidance')?.text).toContain('äöü');
|
||||
expect(parsed[0].sections.some(section => section.text.includes('DIN EN'))).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects incomplete, duplicate, and mismatched 93-control catalogs', () => {
|
||||
const reference = controls();
|
||||
const semantic = controls();
|
||||
expect(() => validateIsoControlSets(reference, semantic)).not.toThrow();
|
||||
expect(() => validateIsoControlSets(reference.slice(1), semantic)).toThrow(/exactly 93/);
|
||||
expect(() => validateIsoControlSets([...reference.slice(0, -1), reference[0]], semantic)).toThrow(/unique controls/);
|
||||
expect(() => validateIsoControlSets(reference, [...semantic.slice(0, -1), { ...semantic.at(-1)!, controlId: '8.35' }])).toThrow(/identifiers do not match/);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { createQdrantRegulatoryIndexFromEnvironment } from '../../server/regulatory/qdrant-regulatory-index';
|
||||
import { createQdrantRegulatoryIndexFromEnvironment, REGULATORY_QDRANT_COLLECTION } from '../../server/regulatory/qdrant-regulatory-index';
|
||||
import { DenseRegulatoryCandidateRetriever, RegulatoryIngestionService } from '../../server/regulatory/regulatory-rag-service';
|
||||
import type { RegulatoryControlDocument } from '../../src/domain/regulatory-rag';
|
||||
import type { TextEmbeddingClient } from '../../server/regulatory/embedding-client';
|
||||
|
|
@ -54,10 +54,10 @@ describeQdrant('Qdrant regulatory index integration', () => {
|
|||
async function deleteSyntheticCorpus(corpusId: string): Promise<void> {
|
||||
const baseUrl = process.env.ASSESSOR_QDRANT_URL;
|
||||
const apiKey = process.env.ASSESSOR_QDRANT_API_KEY;
|
||||
if (!baseUrl || !apiKey) return;
|
||||
const response = await fetch(new URL('/collections/assessor_regulatory_chunks_v1/points/delete?wait=true', baseUrl), {
|
||||
if (!baseUrl) return;
|
||||
const response = await fetch(new URL(`/collections/${REGULATORY_QDRANT_COLLECTION}/points/delete?wait=true`, baseUrl), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'api-key': apiKey },
|
||||
headers: { 'Content-Type': 'application/json', ...(apiKey ? { 'api-key': apiKey } : {}) },
|
||||
body: JSON.stringify({ filter: { must: [{ key: 'corpus_id', match: { value: corpusId } }] } })
|
||||
});
|
||||
if (!response.ok) throw new Error(`Synthetic Qdrant integration cleanup failed with HTTP ${response.status}.`);
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ describe('Qdrant regulatory index', () => {
|
|||
|
||||
const bodies = fetcher.mock.calls.map(call => call[1]?.body && JSON.parse(String(call[1]?.body))).filter(Boolean);
|
||||
expect(bodies[0]).toMatchObject({ vectors: { dense: { size: 2, distance: 'Cosine' } }, sparse_vectors: { sparse: {} } });
|
||||
expect(bodies.filter(body => body.field_name)).toHaveLength(9);
|
||||
expect(bodies.filter(body => body.field_name)).toHaveLength(12);
|
||||
expect(fetcher.mock.calls.every(call => new Headers(call[1]?.headers).get('api-key') === 'secret')).toBe(true);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ describe('regulatory chunker', () => {
|
|||
});
|
||||
|
||||
it('validates the version, required values and language before indexing', () => {
|
||||
expect(() => validateRegulatoryControlDocument({ ...document, schemaVersion: 2 })).toThrow(/schemaVersion 1/);
|
||||
expect(() => validateRegulatoryControlDocument({ ...document, schemaVersion: 3 })).toThrow(/schemaVersion 1 or 2/);
|
||||
expect(() => validateRegulatoryControlDocument({ ...document, controlId: ' ' })).toThrow(/controlId/);
|
||||
expect(() => validateRegulatoryControlDocument({ ...document, language: 'not_a_tag' })).toThrow(/BCP 47/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,18 +4,23 @@ import type { RegulatoryRetrievalCandidate, RegulatoryRetrievalRequest } from '.
|
|||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query';
|
||||
|
||||
const privateMarkers = ['Tenant Acme', '10.23.45.67', 'secret.example', 'C:\\Customer\\report.xml', 'analyst@acme.invalid'];
|
||||
// Result-table / occurrence data and raw addresses that must never reach the query.
|
||||
const resultTableAsset = 'DC01.customer.local';
|
||||
const reporterEmail = 'ops@acme.invalid';
|
||||
const ownerEmail = 'analyst@acme.invalid';
|
||||
|
||||
function finding(overrides: Partial<UnifiedResultProjection> = {}): UnifiedResultProjection {
|
||||
return {
|
||||
resultProjectionId: 'projection-private', sourceKind: 'scanner-finding', sourceResultId: 'source-private',
|
||||
projectId: 'tenant-private', assessmentId: 'assessment-private', collectionRunId: 'run-private',
|
||||
sourceTool: 'purple-knight', title: `${privateMarkers[0]} has SMB signing disabled on ${privateMarkers[1]}`,
|
||||
description: `Host ${privateMarkers[2]} loaded ${privateMarkers[3]}`,
|
||||
recommendation: `Contact ${privateMarkers[4]}`, affectedAsset: privateMarkers[2],
|
||||
resultProjectionId: 'projection-1', sourceKind: 'scanner-finding', sourceResultId: 'source-1',
|
||||
projectId: 'tenant-1', assessmentId: 'assessment-1', collectionRunId: 'run-1',
|
||||
sourceTool: 'purple-knight',
|
||||
title: 'SMB signing is not required on domain controllers',
|
||||
description: `The SMB server permits unsigned sessions, allowing relay attacks. Reported by ${reporterEmail}.`,
|
||||
recommendation: 'Require SMB signing through Group Policy on all domain controllers.',
|
||||
affectedAsset: resultTableAsset,
|
||||
severityOriginal: 'high', severityNormalized: 'high', outcomeOriginal: 'fail', outcomeNormalized: 'fail',
|
||||
technicalExecutionState: 'completed', resultType: 'compliance_check', isIoe: false, isDeviation: true,
|
||||
lifecycleState: 'new', mappingConfidence: 1, reviewStatus: 'open', reviewOwner: privateMarkers[4],
|
||||
lifecycleState: 'new', mappingConfidence: 1, reviewStatus: 'open', reviewOwner: ownerEmail,
|
||||
importedAt: '2026-09-15T10:00:00.000Z', createdAt: '2026-09-15T10:00:00.000Z', frameworkReferences: [], categories: [],
|
||||
...overrides
|
||||
};
|
||||
|
|
@ -32,7 +37,7 @@ const candidate: RegulatoryRetrievalCandidate = {
|
|||
};
|
||||
|
||||
describe('regulatory finding retrieval boundary', () => {
|
||||
it('emits only fixed public concepts and never copies private finding data', () => {
|
||||
it('builds the query from the non-personal check text and never from the result table', () => {
|
||||
const query = buildRegulatoryFindingQuery(finding());
|
||||
|
||||
expect(query).toMatchObject({
|
||||
|
|
@ -41,49 +46,60 @@ describe('regulatory finding retrieval boundary', () => {
|
|||
publicTool: 'Semperis Purple Knight',
|
||||
publicTopics: ['SMB signing requirements']
|
||||
});
|
||||
for (const marker of privateMarkers) expect(query?.query).not.toContain(marker);
|
||||
expect(query?.query).not.toContain('disabled');
|
||||
// The rule-level check text is the intended source.
|
||||
expect(query?.query).toContain('SMB signing is not required on domain controllers');
|
||||
expect(query?.query).toContain('unsigned sessions');
|
||||
// Result-table / occurrence data and raw email addresses never appear.
|
||||
expect(query?.query).not.toContain(resultTableAsset);
|
||||
expect(query?.query).not.toContain(reporterEmail);
|
||||
expect(query?.query).not.toContain(ownerEmail);
|
||||
expect(query?.query).toContain('[redacted-email]');
|
||||
});
|
||||
|
||||
it('allows only exact catalog-owned rule IDs', () => {
|
||||
it('maps every scanner finding, sharpening with a catalog rule ID when available', () => {
|
||||
expect(buildRegulatoryFindingQuery(finding({
|
||||
sourceTool: 'pingcastle', sourceRuleId: 'a-krbtgt', title: privateMarkers[0], description: undefined, recommendation: undefined
|
||||
sourceTool: 'pingcastle', sourceRuleId: 'a-krbtgt'
|
||||
}))).toMatchObject({ specificity: 'rule', publicRuleId: 'A-Krbtgt' });
|
||||
|
||||
expect(buildRegulatoryFindingQuery(finding({
|
||||
sourceTool: 'pingcastle', sourceRuleId: 'A-Customer-Private', title: privateMarkers[0], description: undefined, recommendation: undefined
|
||||
}))).toBeUndefined();
|
||||
// Unknown tool, non-catalog rule ID, no topic match: still mapped from the check title.
|
||||
const generic = buildRegulatoryFindingQuery(finding({
|
||||
sourceTool: 'customer-private-scanner', sourceRuleId: 'A-Custom',
|
||||
title: 'Weak service account configuration', description: undefined, recommendation: undefined
|
||||
}));
|
||||
expect(generic).toMatchObject({ specificity: 'topic' });
|
||||
expect(generic?.query).toContain('Weak service account configuration');
|
||||
expect(generic?.publicRuleId).toBeUndefined();
|
||||
});
|
||||
|
||||
it('fails closed for non-scanner records or findings without a public signal', () => {
|
||||
it('fails closed only for non-scanner records or findings without a check title', () => {
|
||||
expect(buildRegulatoryFindingQuery(finding({ sourceKind: 'environment-observation' }))).toBeUndefined();
|
||||
expect(buildRegulatoryFindingQuery(finding({
|
||||
sourceTool: 'customer-private-scanner', title: privateMarkers[0], description: undefined, recommendation: undefined
|
||||
}))).toBeUndefined();
|
||||
expect(buildRegulatoryFindingQuery(finding({ title: ' ' }))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('passes only the prepared public query and explicit filters to the retriever', async () => {
|
||||
it('passes only the prepared query and explicit filters to the retriever', async () => {
|
||||
const retrieve = vi.fn(async (_request: RegulatoryRetrievalRequest) => [candidate]);
|
||||
const service = new RegulatoryFindingRetrievalService({ retrieve });
|
||||
|
||||
const result = await service.retrieve(finding(), 5, { framework: 'Synthetic Framework' });
|
||||
|
||||
expect(result).toMatchObject({ sourceResultId: 'source-private', status: 'candidates', candidates: [candidate] });
|
||||
expect(result).toMatchObject({ sourceResultId: 'source-1', status: 'candidates', candidates: [candidate] });
|
||||
expect(retrieve).toHaveBeenCalledOnce();
|
||||
expect(retrieve.mock.calls[0][0]).toEqual({
|
||||
query: result.preparedQuery?.query,
|
||||
limit: 5,
|
||||
filters: { framework: 'Synthetic Framework' }
|
||||
});
|
||||
for (const marker of privateMarkers) expect(JSON.stringify(retrieve.mock.calls[0][0])).not.toContain(marker);
|
||||
for (const marker of [resultTableAsset, reporterEmail, ownerEmail]) {
|
||||
expect(JSON.stringify(retrieve.mock.calls[0][0])).not.toContain(marker);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not call retrieval when public context is insufficient', async () => {
|
||||
it('does not call retrieval for a finding without a check title', async () => {
|
||||
const retrieve = vi.fn();
|
||||
const service = new RegulatoryFindingRetrievalService({ retrieve });
|
||||
|
||||
await expect(service.retrieve(finding({ title: privateMarkers[0], description: undefined, recommendation: undefined }), 5))
|
||||
.resolves.toEqual({ sourceResultId: 'source-private', status: 'insufficient-public-context', candidates: [] });
|
||||
await expect(service.retrieve(finding({ title: '' }), 5))
|
||||
.resolves.toEqual({ sourceResultId: 'source-1', status: 'insufficient-public-context', candidates: [] });
|
||||
expect(retrieve).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,138 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { RegulatoryMappingService } from '../../server/regulatory/regulatory-mapping-service';
|
||||
import { RegulatoryFindingRetrievalService } from '../../server/regulatory/regulatory-finding-retrieval';
|
||||
import type { RegulatoryMappingDecision, RegulatoryMappingRun } from '../../src/domain/regulatory-mapping';
|
||||
import type { RegulatoryChunk, RegulatoryRetrievalRequest } from '../../src/domain/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import type { AssessorRepository } from '../../src/persistence/assessor-repository';
|
||||
|
||||
const stamp = '2026-09-15T10:00:00.000Z';
|
||||
// Result-table / occurrence data that must never reach the model or the query.
|
||||
const resultTableAsset = 'DC01.customer.local';
|
||||
|
||||
function finding(overrides: Partial<UnifiedResultProjection> = {}): UnifiedResultProjection {
|
||||
return {
|
||||
resultProjectionId: 'scanner-finding:occurrence-1', sourceKind: 'scanner-finding', sourceResultId: 'occurrence-1',
|
||||
projectId: 'project-1', assessmentId: 'assessment-1', collectionRunId: 'run-1', scanRunId: 'run-1',
|
||||
sourceTool: 'purple-knight', sourceRuleId: 'A-SmbSigning', title: 'SMB signing is not required on domain controllers',
|
||||
description: 'The SMB server permits unsigned sessions, allowing relay attacks.', affectedAsset: resultTableAsset,
|
||||
severityOriginal: 'high', severityNormalized: 'high', outcomeOriginal: 'fail', outcomeNormalized: 'fail',
|
||||
technicalExecutionState: 'completed', resultType: 'compliance_check', isIoe: false, isDeviation: true,
|
||||
lifecycleState: 'new', mappingConfidence: 1, reviewStatus: 'unreviewed', importedAt: stamp, createdAt: stamp, frameworkReferences: [], categories: [],
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function chunk(sourceStandard: 'ISO/IEC 27001' | 'ISO/IEC 27002'): RegulatoryChunk {
|
||||
const reference = sourceStandard === 'ISO/IEC 27001';
|
||||
return {
|
||||
schemaVersion: 2, corpusId: 'iso-test', corpusVersion: 'v2', documentId: reference ? '27001-5.1' : '27002-5.1-purpose',
|
||||
framework: 'ISO/IEC 27002', frameworkVersion: '2022', controlId: '5.1', annexAControlId: 'A.5.1',
|
||||
controlTheme: 'organizational', sourceStandard, sourceEdition: '2022', nationalEdition: 'DIN 2024',
|
||||
sectionKind: reference ? 'control' : 'purpose', sourcePageStart: 1, sourcePageEnd: 1,
|
||||
retrievalRole: reference ? 'reference' : 'semantic', parserVersion: 'test', sourceSha256: reference ? '1'.repeat(64) : '2'.repeat(64),
|
||||
chunkId: reference ? 'reference-5.1' : 'semantic-5.1', chunkIndex: 0, language: 'de', title: 'Synthetic control',
|
||||
text: reference ? 'Synthetic normative statement.' : 'Synthetic semantic evidence.', textSha256: 'a'.repeat(64)
|
||||
};
|
||||
}
|
||||
|
||||
/** In-memory repository keyed like the global (rule-keyed) schema. */
|
||||
function mockRepository(findings: UnifiedResultProjection[]) {
|
||||
const byProjection = new Map(findings.map(item => [item.resultProjectionId, item]));
|
||||
const runs = new Map<string, RegulatoryMappingRun>();
|
||||
const decisions: RegulatoryMappingDecision[] = [];
|
||||
return {
|
||||
getUnifiedResultById: vi.fn(async (id: string) => byProjection.get(id) ?? null),
|
||||
getRegulatoryMappingRun: vi.fn(async (id: string) => runs.get(id) ?? null),
|
||||
saveRegulatoryMappingRun: vi.fn(async (run: RegulatoryMappingRun) => { runs.set(run.id, run); return run; }),
|
||||
listRegulatoryMappingRuns: vi.fn(async () => [...runs.values()]),
|
||||
createRegulatoryMappingDecision: vi.fn(async (decision: RegulatoryMappingDecision, expected?: string) => {
|
||||
const run = runs.get(decision.mappingId);
|
||||
if (!run || run.ruleKey !== decision.ruleKey || run.status !== 'succeeded') throw new Error('missing mapping');
|
||||
if (!run.candidates.some(c => c.controlId === decision.controlId)) throw new Error('ungrounded control');
|
||||
const latest = decisions.filter(d => d.ruleKey === decision.ruleKey && d.controlId === decision.controlId)
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id))[0];
|
||||
if ((expected ?? undefined) !== (latest?.id ?? undefined)) throw new Error('conflict');
|
||||
const stored = { ...decision, ...(latest ? { previousDecisionId: latest.id } : {}) };
|
||||
decisions.push(stored);
|
||||
return stored;
|
||||
}),
|
||||
listRegulatoryMappingDecisions: vi.fn(async (ruleKey: string) => decisions.filter(d => d.ruleKey === ruleKey)
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id)))
|
||||
} as unknown as AssessorRepository;
|
||||
}
|
||||
|
||||
function service(repository: AssessorRepository, complete: (system: string, input: string) => Promise<unknown>) {
|
||||
return new RegulatoryMappingService(repository, {
|
||||
findingRetrieval: new RegulatoryFindingRetrievalService({ retrieve: vi.fn(async (_r: RegulatoryRetrievalRequest) => [{ score: 0.9, chunk: chunk('ISO/IEC 27002') }]) }),
|
||||
index: { ensureCollection: vi.fn(), replaceDocuments: vi.fn(), searchDense: vi.fn(), list: vi.fn(async () => [chunk('ISO/IEC 27001')]) },
|
||||
client: { model: 'work-model', complete: complete as never }, corpusId: 'iso-test', corpusVersion: 'v2', embeddingModel: 'bge-m3',
|
||||
now: () => new Date(stamp)
|
||||
});
|
||||
}
|
||||
|
||||
describe('regulatory mapping service', () => {
|
||||
it('shares one global mapping and decision across projects and sends no result-table data', async () => {
|
||||
const findingB = finding({ resultProjectionId: 'scanner-finding:occurrence-2', sourceResultId: 'occurrence-2', projectId: 'project-2', affectedAsset: 'DC02.other.local' });
|
||||
const repository = mockRepository([finding(), findingB]);
|
||||
const complete = vi.fn(async (_system: string, _input: string) => ({ value: {
|
||||
schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: '5.1', relationship: 'primary', confidence: 'high',
|
||||
rationale: 'Synthetic rationale.', evidenceChunkIds: ['semantic-5.1', 'reference-5.1'] }]
|
||||
}, usage: { totalTokens: 12 } }));
|
||||
const svc = service(repository, complete);
|
||||
|
||||
await svc.start('project-1', { resultProjectionId: 'scanner-finding:occurrence-1' });
|
||||
await svc.close();
|
||||
|
||||
// The same rule mapped once in project-1 is visible to project-2 (global sharing).
|
||||
const shared = await svc.get('project-2', 'scanner-finding:occurrence-2');
|
||||
expect(shared).toMatchObject({ state: 'succeeded', run: { ruleKey: 'purple-knight:A-SmbSigning', sourceTool: 'purple-knight', embeddingModel: 'bge-m3', tokenUsage: { totalTokens: 12 } } });
|
||||
expect(shared.run?.result?.mappings[0].controlId).toBe('5.1');
|
||||
expect(complete).toHaveBeenCalledOnce();
|
||||
|
||||
// The copyrighted ISO clause text is stripped from the response but still reaches the model.
|
||||
expect(shared.run?.candidates.every(c => c.evidence.every(item => item.text === undefined))).toBe(true);
|
||||
expect(complete.mock.calls[0][0]).toContain('governs the same security subject');
|
||||
expect(complete.mock.calls[0][1]).toContain('Synthetic semantic evidence.');
|
||||
expect(complete.mock.calls[0][1]).toContain('SMB signing is not required');
|
||||
expect(complete.mock.calls[0][1]).not.toContain(resultTableAsset);
|
||||
expect(complete.mock.calls[0][1]).not.toContain('DC02.other.local');
|
||||
expect(complete.mock.calls[0][1]).not.toContain('A.5.1');
|
||||
|
||||
// A decision made in project-1 is visible to project-2 (global decisions).
|
||||
await svc.decide('project-1', { resultProjectionId: 'scanner-finding:occurrence-1', controlId: '5.1', decision: 'accepted' });
|
||||
const afterDecision = await svc.get('project-2', 'scanner-finding:occurrence-2');
|
||||
expect(afterDecision.decisions.map(d => ({ controlId: d.controlId, decision: d.decision }))).toEqual([{ controlId: '5.1', decision: 'accepted' }]);
|
||||
});
|
||||
|
||||
it('drops one ungrounded evidence id but keeps the otherwise-valid mapping', async () => {
|
||||
const { svc } = await runWith({
|
||||
schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: '5.1', relationship: 'primary', confidence: 'high',
|
||||
rationale: 'Synthetic rationale.', evidenceChunkIds: ['semantic-5.1', 'not-a-supplied-chunk'] }]
|
||||
});
|
||||
const completed = await svc.get('project-1', 'scanner-finding:occurrence-1');
|
||||
expect(completed.run?.status).toBe('succeeded');
|
||||
expect(completed.run?.result?.mappings[0].evidenceChunkIds).toEqual(['semantic-5.1']);
|
||||
expect(completed.run?.errorCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it('fails safely with a structural detail when the model maps to an unknown control', async () => {
|
||||
const { svc } = await runWith({
|
||||
schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: 'A.5.1', relationship: 'primary', confidence: 'high',
|
||||
rationale: 'Synthetic rationale.', evidenceChunkIds: ['semantic-5.1'] }]
|
||||
});
|
||||
const completed = await svc.get('project-1', 'scanner-finding:occurrence-1');
|
||||
expect(completed.run?.status).toBe('failed');
|
||||
expect(completed.run?.errorCode).toBe('regulatory_mapping_invalid_output');
|
||||
expect(completed.run?.errorDetail).toContain('unknown candidate control');
|
||||
expect(completed.run?.errorDetail).not.toContain(resultTableAsset);
|
||||
});
|
||||
});
|
||||
|
||||
async function runWith(value: unknown): Promise<{ svc: RegulatoryMappingService }> {
|
||||
const repository = mockRepository([finding()]);
|
||||
const svc = service(repository, vi.fn(async () => ({ value })));
|
||||
await svc.start('project-1', { resultProjectionId: 'scanner-finding:occurrence-1' });
|
||||
await svc.close();
|
||||
return { svc };
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
import { randomUUID } from 'node:crypto';
|
||||
import { Pool } from 'pg';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createLiteLlmClientFromEnvironment } from '../../server/ai/litellm-client';
|
||||
import { runDatabaseMigrations } from '../../server/postgres/migration-runner';
|
||||
import { PostgresAssessorRepository } from '../../server/postgres/postgres-assessor-repository';
|
||||
import { createRegulatoryRagRuntime } from '../../server/regulatory/regulatory-rag-runtime';
|
||||
import { RegulatoryMappingService } from '../../server/regulatory/regulatory-mapping-service';
|
||||
import type { Assessment, AssessmentProject, ScanRun } from '../../src/domain/canonical-model';
|
||||
import { RepositoryConflictError } from '../../src/persistence/repository-errors';
|
||||
|
||||
const enabled = process.env.ASSESSOR_REGULATORY_MAPPING_INTEGRATION === '1' && Boolean(process.env.ASSESSOR_TEST_DATABASE_URL);
|
||||
const describeIntegration = enabled ? describe : describe.skip;
|
||||
|
||||
describeIntegration('regulatory mapping end-to-end integration', () => {
|
||||
it('maps one synthetic finding through BGE-M3, Qdrant, LiteLLM, and PostgreSQL', async () => {
|
||||
const stamp = new Date().toISOString();
|
||||
const suffix = randomUUID();
|
||||
const project: AssessmentProject = { id: `synthetic-regulatory-${suffix}`, name: 'Synthetic regulatory integration', createdAt: stamp, updatedAt: stamp, attributes: {} };
|
||||
const assessment: Assessment = { id: `synthetic-assessment-${suffix}`, projectId: project.id, name: 'Synthetic', createdAt: stamp, updatedAt: stamp };
|
||||
const scanRun: ScanRun = { id: `synthetic-run-${suffix}`, assessmentId: assessment.id, sequence: 1, label: 'Synthetic', tool: { id: 'purple-knight', name: 'Purple Knight', version: 'synthetic' }, importedAt: stamp };
|
||||
const occurrenceId = `synthetic-occurrence-${suffix}`;
|
||||
const pool = new Pool({ connectionString: process.env.ASSESSOR_TEST_DATABASE_URL, max: 2, connectionTimeoutMillis: 5_000 });
|
||||
await runDatabaseMigrations(pool);
|
||||
const repository = await PostgresAssessorRepository.create(pool);
|
||||
const rag = createRegulatoryRagRuntime();
|
||||
const client = createLiteLlmClientFromEnvironment();
|
||||
if (!client) throw new Error('Approved LiteLLM configuration is required.');
|
||||
const service = new RegulatoryMappingService(repository, {
|
||||
findingRetrieval: rag.findingRetrieval, index: rag.index, embeddingModel: rag.embeddingModel, client,
|
||||
corpusId: process.env.ASSESSOR_RAG_ACTIVE_CORPUS_ID,
|
||||
corpusVersion: process.env.ASSESSOR_RAG_ACTIVE_CORPUS_VERSION
|
||||
});
|
||||
let deleted = false;
|
||||
try {
|
||||
await repository.saveProject(project);
|
||||
await repository.saveAssessment(assessment);
|
||||
const session = await repository.beginImport({ sessionId: `synthetic-session-${suffix}`, projectId: project.id, assessmentId: assessment.id, scanRun });
|
||||
await session.stage({
|
||||
importFiles: [{ id: `synthetic-file-${suffix}`, scanRunId: scanRun.id, fileName: 'synthetic.csv', format: 'CSV', sizeBytes: 100,
|
||||
importedAt: stamp, detectedTool: scanRun.tool, detection: { level: 'high', score: 1, reasons: ['synthetic'] },
|
||||
processedWith: { parserId: 'csv', parserVersion: 'synthetic', profileId: 'purple-knight', profileVersion: 'synthetic', mappingVersion: 'synthetic' },
|
||||
sourceArtifact: { kind: 'not-retained', reason: 'Synthetic integration fixture' } }],
|
||||
sourceTables: [{ id: `synthetic-table-${suffix}`, importFileId: `synthetic-file-${suffix}`, name: 'Synthetic', index: 0, rowCount: 1, columnCount: 2, attributes: {} }],
|
||||
findingDefinitions: [{ id: `synthetic-definition-${suffix}`, canonicalKey: `purple-knight:synthetic-${suffix}`, title: 'SMB signing disabled',
|
||||
description: 'Synthetic public-topic fixture', category: 'Synthetic', frameworkReferences: [], createdAt: stamp, updatedAt: stamp }],
|
||||
findingOccurrences: [{ id: occurrenceId, definitionId: `synthetic-definition-${suffix}`, scanRunId: scanRun.id,
|
||||
importFileId: `synthetic-file-${suffix}`, sourceTableId: `synthetic-table-${suffix}`,
|
||||
sourceIdentity: { tool: scanRun.tool, originalRuleId: 'SYNTHETIC-SMB', originalTitle: 'SMB signing disabled' },
|
||||
sourcePosition: { rowIndex: 1 }, originalFields: [{ key: 'title', label: 'Title', value: 'SMB signing disabled', columnIndex: 0 }],
|
||||
fieldMappings: [], severity: { originalValue: 'high', normalizedValue: 'high', provenance: { decisionSource: 'source-native',
|
||||
confidence: { level: 'high', score: 1, reasons: ['synthetic'] }, mappingVersion: 'synthetic' } },
|
||||
status: { originalValue: 'Open', normalizedValue: 'open', provenance: { decisionSource: 'source-native',
|
||||
confidence: { level: 'high', score: 1, reasons: ['synthetic'] }, mappingVersion: 'synthetic' } } }]
|
||||
});
|
||||
await session.commit();
|
||||
|
||||
const resultProjectionId = `scanner-finding:${occurrenceId}`;
|
||||
await service.start(project.id, { resultProjectionId });
|
||||
await service.close();
|
||||
const result = await service.get(project.id, resultProjectionId);
|
||||
expect(result.run?.status, result.run?.errorCode).toBe('succeeded');
|
||||
expect(result.run?.candidates).toHaveLength(8);
|
||||
expect(result.run?.candidates.every(candidate => candidate.evidence.some(item => item.sourceStandard === 'ISO/IEC 27001'))).toBe(true);
|
||||
expect(result.run?.result?.mappings.length).toBeLessThanOrEqual(5);
|
||||
expect(result.run?.result?.mappings.every(mapping => {
|
||||
const candidate = result.run?.candidates.find(item => item.controlId === mapping.controlId);
|
||||
return Boolean(candidate && mapping.evidenceChunkIds.every(id => candidate.evidence.some(item => item.chunkId === id)));
|
||||
})).toBe(true);
|
||||
const candidate = result.run!.candidates[0];
|
||||
const ruleKey = result.run!.ruleKey;
|
||||
const reviewed = await service.decide(project.id, {
|
||||
resultProjectionId, controlId: candidate.controlId, decision: 'accepted', comment: 'Synthetic review.'
|
||||
});
|
||||
expect(reviewed.decisions[0]).toMatchObject({ controlId: candidate.controlId, decision: 'accepted' });
|
||||
// Omitting expectedLatestDecisionId conflicts with the existing global decision for this rule+control.
|
||||
await expect(repository.createRegulatoryMappingDecision({
|
||||
version: 1, id: `synthetic-conflict-${suffix}`, ruleKey, sourceTool: 'purple-knight', sourceRuleId: 'SYNTHETIC-SMB',
|
||||
mappingId: result.run!.id, controlId: candidate.controlId, decision: 'rejected', createdAt: new Date().toISOString()
|
||||
})).rejects.toBeInstanceOf(RepositoryConflictError);
|
||||
await repository.deleteProject(project.id, { projectId: project.id, cascade: true });
|
||||
deleted = true;
|
||||
// Regulatory mappings and decisions are global: they survive project deletion.
|
||||
await expect(repository.getRegulatoryMappingRun(result.run!.id)).resolves.not.toBeNull();
|
||||
expect((await repository.listRegulatoryMappingDecisions(ruleKey)).length).toBeGreaterThan(0);
|
||||
} finally {
|
||||
await service.close();
|
||||
if (!deleted) await repository.deleteProject(project.id, { projectId: project.id, cascade: true }).catch(() => false);
|
||||
await pool.end();
|
||||
}
|
||||
}, 120_000);
|
||||
});
|
||||
|
|
@ -26,7 +26,7 @@ describe('regulatory RAG services', () => {
|
|||
};
|
||||
const ensureCollection = vi.fn(async () => {});
|
||||
const replaceDocuments = vi.fn(async (_chunks: IndexedRegulatoryChunk[]) => {});
|
||||
const index: RegulatoryChunkIndex = { ensureCollection, replaceDocuments, searchDense: vi.fn() };
|
||||
const index: RegulatoryChunkIndex = { ensureCollection, replaceDocuments, searchDense: vi.fn(), list: vi.fn() };
|
||||
const service = new RegulatoryIngestionService(embeddings, index, { maximumChunkCharacters: 220, embeddingBatchSize: 2 });
|
||||
|
||||
const result = await service.ingest([document]);
|
||||
|
|
@ -48,7 +48,7 @@ describe('regulatory RAG services', () => {
|
|||
const searchDense = vi.fn(async () => candidates);
|
||||
const dense = new DenseRegulatoryCandidateRetriever(
|
||||
{ model: 'multilingual-model', dimensions: 2, embed },
|
||||
{ ensureCollection: vi.fn(), replaceDocuments: vi.fn(), searchDense }
|
||||
{ ensureCollection: vi.fn(), replaceDocuments: vi.fn(), searchDense, list: vi.fn() }
|
||||
);
|
||||
const rerank = vi.fn(async (_query: string, values: RegulatoryRetrievalCandidate[]) => values.slice().reverse());
|
||||
const retrieval = new RegulatoryRetrievalPipeline(dense, { rerank }, 4);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ export default defineConfig({
|
|||
'import-worker': 'server/import-worker.ts',
|
||||
'gpo-inventory-worker': 'server/gpo/gpo-inventory-worker.ts',
|
||||
'regulatory-ingest': 'server/regulatory/regulatory-ingest.ts',
|
||||
'regulatory-prepare-iso': 'server/regulatory/regulatory-prepare-iso.ts',
|
||||
'regulatory-evaluate': 'server/regulatory/regulatory-evaluate.ts'
|
||||
},
|
||||
output: {
|
||||
|
|
|
|||
Loading…
Reference in New Issue