From b60e9dc8eb7d1e629fb63e637f14a5e56c940062 Mon Sep 17 00:00:00 2001 From: Binh Date: Wed, 23 Sep 2026 11:01:07 +0200 Subject: [PATCH] feat: add ISO regulatory corpus PDF uploads --- .env.postgres.example | 2 + .gitignore | 2 +- SECURITY.md | 28 +++ docker-compose.deploy.yml | 6 + docs/DEPLOYMENT.md | 22 ++ docs/IMPLEMENTATION_PLAN.md | 8 + docs/REGULATORY_RAG.md | 15 ++ server/assessor-http-server.ts | 38 +++- server/index.ts | 25 +- server/regulatory/iso-pdf-worker-client.ts | 24 ++ server/regulatory/iso-pdf-worker.ts | 7 + server/regulatory/qdrant-regulatory-index.ts | 23 +- .../regulatory-corpus-upload-service.ts | 215 ++++++++++++++++++ .../regulatory/regulatory-mapping-service.ts | 14 +- server/regulatory/regulatory-rag-service.ts | 4 +- src/api/assessor-api-client.ts | 16 ++ src/domain/regulatory-corpus-management.ts | 30 +++ src/features/project-import/DataHub.tsx | 2 + .../project-import/IsoCorpusLibrary.tsx | 74 ++++++ src/styles/datahub-redesign.css | 11 + tests/api/assessor-api-client.test.ts | 14 ++ .../iso-corpus-upload-interaction.test.tsx | 72 ++++++ tests/server/qdrant-regulatory-index.test.ts | 25 ++ .../regulatory-corpus-upload-http.test.ts | 190 ++++++++++++++++ vite.server.config.ts | 1 + 25 files changed, 856 insertions(+), 12 deletions(-) create mode 100644 server/regulatory/iso-pdf-worker-client.ts create mode 100644 server/regulatory/iso-pdf-worker.ts create mode 100644 server/regulatory/regulatory-corpus-upload-service.ts create mode 100644 src/domain/regulatory-corpus-management.ts create mode 100644 src/features/project-import/IsoCorpusLibrary.tsx create mode 100644 tests/features/iso-corpus-upload-interaction.test.tsx create mode 100644 tests/server/regulatory-corpus-upload-http.test.ts diff --git a/.env.postgres.example b/.env.postgres.example index c35b306..7aa0fd8 100644 --- a/.env.postgres.example +++ b/.env.postgres.example @@ -26,6 +26,8 @@ ASSESSOR_QDRANT_API_KEY=replace-with-a-long-random-local-qdrant-key # ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS=60000 # ASSESSOR_RAG_CHUNK_CHARACTERS=1600 # ASSESSOR_RAG_EMBEDDING_BATCH_SIZE=32 +# Operator-only ISO PDF upload in Data Hub. Use an independent random secret of at least 32 characters. +# ASSESSOR_ISO_UPLOAD_TOKEN=replace-with-a-long-random-upload-token # The mapping API is enabled only for this explicit, already-ingested corpus. # ASSESSOR_RAG_ACTIVE_CORPUS_ID=iso-27001-27002-de # ASSESSOR_RAG_ACTIVE_CORPUS_VERSION=din-2024-de-example diff --git a/.gitignore b/.gitignore index 6554b9c..fd683d0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ src-tauri/gen/ vite.config.js vite.config.d.ts *.png -artifacts/objects/ +artifacts/ Mock-Defender/ outputs/ evaluation/ diff --git a/SECURITY.md b/SECURITY.md index b1a6566..cac36a5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,5 +1,33 @@ # Sicherheitsmodell +## ISO-Korpus-Upload und Embedding (Phase 13D) + +Die beiden lizenzierten DIN-ISO-PDFs werden nur durch eine bewusste Aktion im +Data Hub an die Same-Origin-API uebertragen. Der Upload erfordert zusaetzlich +einen serverseitig konfigurierten, mindestens 32 Zeichen langen Operator-Token; +der Backendvergleich erfolgt zeitkonstant vor dem Lesen des Anfrageinhalts. +Der Token wird nicht in Browser-Storage, URLs, Antworten oder Logs abgelegt. +Er ist eine Uebergangsberechtigung fuer diese eine globale Aktion und ersetzt +keine Nutzerverwaltung oder allgemeine Autorisierung. + +Jedes PDF ist auf 32 MiB begrenzt. Die bestehende serverseitige PDF.js-Pruefung +verlangt kopierbaren Text, passende Struktur und dieselben 93 Kontrollen. +Das Parsing laeuft in einem begrenzten Worker. Die Original-PDFs liegen nur +waehrend des Jobs in einem privaten Verzeichnis und werden danach entfernt. +Die daraus erzeugte versionierte JSON-Korpusquelle liegt dauerhaft im privaten +Artefakt-Volume, damit der abgeleitete Qdrant-Index wiederhergestellt werden +kann. Es gibt keinen API-Endpunkt zum Lesen der PDFs oder des Kontrolltexts; +die Status-API zeigt nur Version, Zaehlwerte und Quellhashes. Server- und +Volume-Administratoren koennen die Quelle technisch weiterhin lesen. + +Beim Embedding geht der ISO-Kontrolltext nur vom Backend an den konfigurierten, +fuer diesen lizenzierten Inhalt freigegebenen Embedding-Endpunkt. Bei +fehlgeschlagener Verarbeitung bleibt die bisher aktivierte Version erhalten. +Die neue Version wird erst nach vollstaendigem Indexlauf aktiviert. Qdrant +bleibt ableitbar; PostgreSQL-Findings und Review-Entscheidungen werden durch +den Upload nicht veraendert. Phase-14-Authentifizierung und Rollen bleiben +offen. + ## Accepted Risk: dokumentierte externe Entscheidung Ergaenzung 2026-09-10: Der lokale Accepted-Risk-Dialog speichert begruendete, diff --git a/docker-compose.deploy.yml b/docker-compose.deploy.yml index db3a0d3..5e80cab 100644 --- a/docker-compose.deploy.yml +++ b/docker-compose.deploy.yml @@ -29,6 +29,12 @@ services: ASSESSOR_RAG_EMBEDDING_MODEL: ${ASSESSOR_RAG_EMBEDDING_MODEL:-} ASSESSOR_RAG_EMBEDDING_API_KEY: ${ASSESSOR_RAG_EMBEDDING_API_KEY:-} ASSESSOR_RAG_EMBEDDING_DIMENSIONS: ${ASSESSOR_RAG_EMBEDDING_DIMENSIONS:-} + ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS: ${ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS:-} + ASSESSOR_AI_MODE: ${ASSESSOR_AI_MODE:-} + ASSESSOR_AI_BASE_URL: ${ASSESSOR_AI_BASE_URL:-} + ASSESSOR_AI_MODEL: ${ASSESSOR_AI_MODEL:-} + ASSESSOR_AI_API_KEY: ${ASSESSOR_AI_API_KEY:-} + ASSESSOR_ISO_UPLOAD_TOKEN: ${ASSESSOR_ISO_UPLOAD_TOKEN:-} ports: - "${ASSESSOR_BIND:?set ASSESSOR_BIND}:${ASSESSOR_PUBLISH_PORT:?set ASSESSOR_PUBLISH_PORT}:4173" volumes: diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 1c9e3df..a84ca9d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -60,6 +60,28 @@ production use their existing complete environment-file secrets. See [`AI_OVERVIEW_SECURITY.md`](AI_OVERVIEW_SECURITY.md) for the transmitted-data and trust-boundary contract. +To upload the licensed DIN ISO 27001 and 27002 PDFs from Data Hub, add the +following values to the environment's `STAGING_ENV_FILE` or `PROD_ENV_FILE`: + +```text +ASSESSOR_QDRANT_API_KEY= +ASSESSOR_RAG_EMBEDDING_BASE_URL=https://approved-embedding-gateway.example/v1 +ASSESSOR_RAG_EMBEDDING_MODEL= +ASSESSOR_RAG_EMBEDDING_API_KEY= +ASSESSOR_RAG_EMBEDDING_DIMENSIONS=1024 +ASSESSOR_ISO_UPLOAD_TOKEN= +``` + +The operator enters the upload token in Data Hub; it is checked by the backend +before either PDF is read and is not saved in browser storage. Keep the +LiteLLM settings above as well to enable mapping after indexing. The upload +stores the generated corpus privately in the environment's artifact volume, +deletes temporary original PDFs after parsing, and activates a new version +only after successful embedding. No server shell or manual corpus-version +environment edit is needed for this workflow. Use an embedding destination +approved for the licensed ISO text. The token protects only this upload action; +it does not add general user authentication. + ## Mandatory server inventory Before the first push-triggered deployment, connect to the target server over diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 8ebfd35..867f01e 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -2,6 +2,8 @@ Status: lebendes Projektdokument +ISO-Korpus-Upload 2026-09-23 (Phase 13D, Nutzerauftrag): Data Hub nimmt die passenden DIN-27001-/27002-PDFs gemeinsam ueber eine token-geschuetzte Same-Origin-API an. Der vorhandene PDF-Parser laeuft in einem begrenzten Worker; der private versionierte JSON-Korpus bleibt als Wiederaufbauquelle im Artefakt-Volume, temporaere Original-PDFs werden nach dem Parsing und nach unterbrochenen Starts entfernt. Ein Hintergrundjob zeigt Parse-/Embedding-/Aktivierungsstatus und Chunkfortschritt; die Statusabfrage erkennt einen unvollstaendigen oder unerreichbaren Index. Qdrant-Ersatz ist nun auf die Korpusversion begrenzt; erst nach vollstaendigem Ingest wechselt die aktive Mappingversion, bei einer neuen fehlgeschlagenen Version bleibt die bisherige aktiv. Ein identischer Upload kann einen verlorenen Index anhand der erwarteten Chunkzahl neu aufbauen. Der Operator-Token ist eine Uebergangsberechtigung bis Phase 14 und schuetzt ausschliesslich diese globale Aktion. Der Nutzer hat Speicherung der privaten Korpusquelle und den konfigurierten Embedding-Endpunkt fuer diesen lizenzierten Inhalt bestaetigt. Vor der letzten Statuspraezisierung bestand der Standardlauf mit 1.584 Tests und 16 uebersprungenen opt-in Integrationen; danach bestanden die vier fokussierten HTTP-/UI-/Client-/Indexdateien mit 29 Tests sowie beide Typechecks. Beide Builds und Local-only-/Designchecks bestanden vor der Statuspraezisierung. Keine PostgreSQL-Migration oder Scanner-Reimport; die echte PDF-/Provider-/Qdrant-Abnahme auf dem Zielserver bleibt offen. Phase 13D bleibt in Arbeit. + Review remediation 2026-09-23 (Phase 7 and Phase 13D, verified): Defender device-risk projection now preserves the exact source-occurrence/device relationship when separate occurrences in one run share a CVE; a synthetic repository acceptance test rejects a device absent from the selected occurrence. Saved regulatory mappings remain readable when the retrieval index is unavailable. A failed AI Overview regeneration reports the failed attempt while displaying the last successful overview. Existing risk decisions, scanner data, scoring, mapping decisions and database schema are unchanged; no reimport or migration is required. The remaining historical risk is that a previously recorded device-finding acceptance from a same-CVE multi-occurrence run may contain incorrect captured provenance and requires review before reliance. Focused tests pass 22/22 plus repository acceptance 16/16; the full standard suite passes 1,576 tests with 16 opt-in tests skipped, both production builds and typechecks pass, and local-only/design checks pass. Phases 7 and 13D remain in progress. Defender-device-scoped risk acceptance 2026-09-21 (Phases 7 and 9-11 extension with Phase 13 validation, explicit user request): Existing whole-finding acceptance remains unchanged. Work can record one CVE for one exact Defender device; Defender Device view can record one immutable snapshot of every detailed actionable CVE in the selected active scan-run scope. The shared platform-neutral projection uses the existing Defender identity model, rejects unresolved or ambiguous identities, retains canonical definition/CVE and exact occurrence provenance, and carries coverage forward only for the same project, exact device key and captured definition. Independent whole-finding, device-finding and device-snapshot decisions overlap without revoking each other. Migration 028, repository/API parity, project export/load, optimistic revision checks and the Accepted Risks register are implemented. The canonical PostgreSQL datastore was backed up and verified before migration; 028 is recorded and a synthetic PostgreSQL accept/conflict/revoke/export/cascade test passed with cleanup. Verification: 287 files and 1,567 tests passed with six files and 16 opt-in tests skipped; client/server TypeScript checks, local-only and design-language checks, both production builds and the dedicated canonical PostgreSQL integration passed. Phase 14 authentication, authorization, signing, evidence verification and retention remain explicitly unclaimed. @@ -1895,6 +1897,7 @@ Umgesetzter Grundstand vom 2026-09-15: - Der produktiv gebaute End-to-End-Pfad ist mit lokalem llama.cpp/BGE-M3 FP16 live gelaufen: sechs Dokumente/sechs Chunks; drei Cross-Language-Smoke-Cases bei K=3 jeweils Rang 1 und damit HitRate, Recall und MRR 1,0. Der Mini-Korpus ist nur ein Wiring-Smoke-Test, kein belastbarer Modellvergleich. - Der Grundstand ist in `docs/REGULATORY_RAG.md` dokumentiert und mit synthetischen Vertrags-, Client-, Ingestion-, Retrieval- und Deploymenttests abgesichert. - Der PDF.js-Importer erzeugt aus den explizit uebergebenen ISO-27001-/27002-Pfaden einen lokalen Korpus-v2 mit Quellhashes, Ausgaben, 93 geteilten Kontrollidentitaeten, Abschnittsarten und Seitenbereichen. Nur ISO-27002-Abschnitte werden semantisch gerankt; die entsprechende ISO-27001-Aussage wird als autoritative Referenzevidenz angehaengt. +- Data Hub bietet fuer genau diese beiden PDFs einen token-geschuetzten Upload mit begrenztem Parserworker und asynchronem Embeddingstatus. Die private Korpusquelle ueberlebt Serverneustarts, die Original-PDFs nicht; die Aktivierung erfolgt nach erfolgreichem Indexaufbau ohne manuellen CLI- oder Umgebungsversionsschritt. - Ein manueller Work-Vertikalschnitt startet pro kanonischem Scanner-Finding einen idempotenten asynchronen Lauf. Eine breite generische Suche reserviert fuenf Controls; die optionale Topic-Query liefert eine breite sowie getrennte `control`-, `purpose`- und `guidance`-Suche, deren Control-Rankings per Reciprocal Rank Fusion die restlichen sieben Plaetze fuellen. Ohne Topic-Treffer wird dieselbe generische Query verwendet. LiteLLM erhaelt hoechstens zwoelf Controls; maximal fuenf Vorschlaege werden gegen exakt gelieferte Evidenz validiert und Accept/Reject-Ereignisse append-only in PostgreSQL gespeichert. - Result DB startet auf ausdrueckliche Aktion einen projektgebundenen Batch fuer exakt denselben vollstaendigen normalisierten Filter-/Timeslot-Scope wie Export JSON. Der Dienst paginiert serverseitig unabhaengig von der Anzeige, dedupliziert Scanner-Vorkommen global nach `ruleKey`, verwendet erfolgreiche und aktive Mappings wieder, versucht fehlgeschlagene Regeln einmal erneut und begrenzt Einzel- plus Batchpipelines gemeinsam auf zwei. Migration 027 speichert nur Scope, Filterhash, Idempotenz, Status und Aggregatfortschritt; Entscheidungen bleiben manuell im bestehenden Work-Panel. @@ -2030,6 +2033,7 @@ Festlegung 2026-09-06 (Phase 13B/13C, ausdruecklicher Nutzerwunsch): Vor Rueckfr Festlegung vom 2026-09-06 (Phase 11, lokaler Persistenzbetrieb): Der vorhandene Docker-Compose-Bestand `assessor-2_0` mit der Datenbank `assessor`, dem Container `assessor-2_0-postgres-1` und dem persistenten Volume `assessor-2_0_assessor-postgres-data` ist der einzige kanonische lokale ASSESSOR-Datenbestand fuer diesen Workspace. Lokale Starts verwenden ausschliesslich `127.0.0.1:5432`. Die Compose-Konfiguration bindet dieses Volume deshalb namentlich als `external`; bei fehlendem Bestand oder Portkonflikt muss der Start fehlschlagen, statt eine weitere ASSESSOR-Datenbank oder einen Ersatzport anzulegen. Diese ausdrueckliche Nutzerfestlegung ersetzt die bisherige implizite, verzeichnisabhaengige Compose-Projektidentitaet. Der interne Browser-/Backend-/PostgreSQL-Zielaufbau bleibt unveraendert. | Datum | Entscheidung | Begründung | +| 2026-09-23 | Der globale ISO-Korpus wird bis zur Phase-14-Nutzerverwaltung nur mit einem separaten Operator-Upload-Token aus Data Hub erneuert. Die privaten versionierten JSON-Korpusquellen bleiben im Artefakt-Volume, Original-PDFs nur temporaer; der freigegebene Embedding-Endpunkt darf den lizenzierten Kontrolltext erhalten. | Der Nutzer hat Token, private Wiederaufbauquelle und Embedding-Ziel ausdruecklich bestaetigt. Eine fuer alle Projekte wirksame Korpusaktivierung darf nicht allein von der allgemeinen VPN-/App-Erreichbarkeit abhangen. | | 2026-09-21 | Defender scoped risk acceptance keeps the one-finding-per-CVE projection. A device-finding decision binds one canonical CVE definition to one exact stable device key; a device-snapshot decision is a server-derived immutable set of all detailed actionable CVEs in the selected active scan-run scope. Coverage is the union of independent valid decisions and carries forward only inside the same project for the same exact device key and captured definition. | This preserves existing whole-finding behavior and source provenance while allowing the two requested device-level workflows. Server-derived membership prevents a client from omitting current CVEs; immutable keys and fail-closed unresolved/ambiguous identities prevent accidental cross-device or cross-import carry-forward. | | 2026-09-15 | Nicht-personenbezogene Umgebungsaggregate (Konten-, Inventar-, Passwortrichtlinien- und Konfigurationszaehlungen bzw. -booleans) ueberschreiten die Result-DB-Offenlegungsgrenze und den AI-Datenvertrag ausschliesslich ueber exakte Property-Allowlist-Regeln; identifizierende Namen und alle Zeitstempel bleiben ausgeschlossen. Die Werte werden aus bereits extrahierten Rohbeobachtungen abgeleitet und unter stabilen Property-Namen statt fragiler `tabelle.N`-Schluessel emittiert. | Client Info zeigte diese Aggregate bereits aus dem Roh-Snapshot, waehrend Result DB und AI sie fail-closed verwarfen. Stabile Ableitungen erlauben eine explizite, pruefbare Offenlegung ohne die fail-closed Grenze zu einer generischen Freigabe zu oeffnen; der Nutzer hat den Umfang (Konten inkl. Aufschluesselung, Inventar, Richtlinie, Konfiguration) und den Ausschluss von Namen/Zeitstempeln ausdruecklich bestimmt. | | 2026-09-14 | AI Overview verwendet ausschliesslich das per Backend konfigurierte, unternehmensverwaltete LiteLLM ueber HTTPS; ein lokales Modell oder Browserzugriff auf Gateway/Key ist nicht vorgesehen. Der vollstaendige aktive Timeslot wird ueber den bestehenden Privacy-Safe-JSON-Vertrag uebertragen, grosse Eingaben werden ohne Record-Trunkierung hierarchisch zusammengefuehrt, und nur validierte Endergebnisse werden je Timeslot persistiert. | Der Nutzer hat Work-LiteLLM samt URL, Modellalias und API-Key bereit. Die bestehende Exportgrenze verhindert einen zweiten, weiter gefassten AI-Datenvertrag; serverseitige Secrets, explizite Ausloesung, lokale Validierung und append-only Historie halten Netz-, Kosten-, Datenschutz- und Auditgrenzen nachvollziehbar. | @@ -2346,6 +2350,8 @@ Diese Entscheidungen werden erst getroffen, wenn die jeweilige Phase ansteht ode ## 11. Risiken +ISO-Frontend-Upload 2026-09-23: Der Token ist keine persoenliche Identitaet, Rolle oder allgemeine Sitzungsberechtigung; ein kompromittierter Token erlaubt bis zur Rotation eine globale Korpusneuerstellung. Der serverseitige Worker, Groessenlimits, private Quelle, keine Download-API und die Aktivierung nach erfolgreichem Ingest begrenzen die technische Wirkung. Der Embedding-Gateway erhaelt lizenzierten Kontrolltext und unterliegt seiner freigegebenen Aufbewahrungspraxis. Ein Prozessabbruch markiert den Job nach Neustart als fehlgeschlagen und entfernt temporaere PDFs; ein teilweise geschriebener neuer Qdrant-Index bleibt abgeleitet und kann per erneutem Frontend-Upload repariert werden. Der Betrieb mit mehreren gleichzeitig schreibenden App-Instanzen ist durch diesen Dateistatus nicht abgesichert; der aktuelle Compose-Stack startet genau eine. Reale DIN-PDFs, Gateway und Ziel-Qdrant wurden in dieser Implementierung nicht beruehrt. + Regulatory RAG 2026-09-15: Regulatorische Texte koennen urheber-, lizenz- oder versionsgebunden sein; ein technisch lesbarer Korpus ist deshalb noch kein freigegebener Produktkorpus. Lokale llama.cpp-Embeddings bleiben auf Loopback, waehrend minimierte Querytexte bei einem spaeteren Work-LiteLLM-Vergleich die ASSESSOR-Prozessgrenze verlassen. Gegenmassnahmen sind ein autorisierter versionierter Korpus, explizite Modell-/Dimensionsbindung, Backend-only Secrets, Remote-HTTPS, begrenzte Requests und der implementierte fail-closed Public-Signal-Query-Builder; organisatorische Datenschutzfreigabe bleibt fuer externe Ziele erforderlich. Dense Similarity ist nur Kandidatenranking und weder Kontrollzuordnung noch Confidence; vor automatischer Nutzung sind reale sprachuebergreifende Evaluation, Hybrid-/Reranker-Vergleich, Schwellenkalibrierung und Human Review erforderlich. Qdrant-Dokumentersatz ist nicht transaktional mit einem Embeddingdienst; Fehler koennen einen unvollstaendigen abgeleiteten Index hinterlassen, der aus dem unveraenderten Korpus neu aufgebaut werden muss. Das Volume ist Betriebsbeschleunigung, keine fachliche Sicherung. Der autorisiert geloeschte alte Qdrant ist entfernt und 1.19.1 belegt kanonisch Port 6333; das alte Volume bleibt vorsorglich erhalten. Der lokale BGE-M3-End-to-End-Smoke-Test war mit drei klar getrennten Themen perfekt, ist statistisch und fachlich aber nicht repraesentativ. Ein kuratierter lizenzierter Korpus, schwierigere Negativ-/Mehrfachkontrollfaelle, unabhaengige Labels und der LiteLLM-Vergleich bleiben offen. Retrieval-Gap-Benchmark 2026-09-17, korrigiert 2026-09-18: Die zehn PingCastle-Faelle und 17 Sollbeziehungen stammten aus derselben manuellen Validierungsrunde, die die Luecken identifiziert hatte; Recall@12 1,0 auf diesem Satz belegte nur die konkrete Reparatur und war wegen Auswahl- und Konzeptkurationsbias kein unabhaengiger Gesamtqualitaetsnachweis. Auf ausdrueckliche Nutzerentscheidung wird dieser fachlich veraenderliche Satz nicht als feste Regression ausgefuehrt. Sollzuordnungen bleiben Human-Review-/Evaluationsdaten; die verpflichtende Suite prueft ausschliesslich regulationsunabhaengige Retrievalmechanik. @@ -2664,6 +2670,8 @@ Festgelegtes Betriebsrisiko vom 2026-09-06: Verzeichnisabhaengige Compose-Projek ## 12. Änderungsprotokoll +2026-09-23 (Phase 13D): Token-geschuetzten ISO-Doppel-PDF-Upload, privaten Korpusstand, begrenzten Parserworker, Embeddingfortschritt, Indexvollstaendigkeitsanzeige, versionsgebundene Qdrant-Ersetzung und Aktivierung nach erfolgreichem Indexlauf implementiert. Synthetische HTTP-, UI-, Client- und Indexregressionen ergaenzt; Standardlauf vor letzter Statuspraezisierung 1.584 bestanden/16 opt-in uebersprungen, danach fokussiert 29/29 und beide Typechecks. Beide Builds sowie Local-only-/Designpruefung vor der Statuspraezisierung erfolgreich. Echte Lizenzdateien und Zielserver bleiben fuer eine gesonderte Abnahme unberuehrt. Keine PostgreSQL-Migration und kein Scanner-Reimport. + Verified 2026-09-23 (Phase 7 and Phase 13D review fixes): Scoped Defender acceptance now rejects a device not present in the selected same-CVE source occurrence while retaining cross-occurrence identity conflict detection. Saved ISO mappings remain available during Qdrant outages. AI Overview preserves and labels the last successful result after a failed regeneration. Synthetic domain, repository-command, service and rendered-panel regressions pass; standard suite 1,576 passed/16 opt-in skipped, builds/typechecks and local-only/design checks pass. No database or scanner-data mutation. Existing device-finding decisions from the affected source pattern need provenance review; no historical decision was silently rewritten. Verifiziert 2026-09-15 (Phase 13D, llama.cpp/BGE-M3): Lokalen Server Health/Modellmetadaten geprueft, sechs rein synthetische EN/DE-Dokumente ueber den produktiv gebauten Embedding-/Ingestionpfad als sechs 1.024-dimensionale Chunks indexiert und drei Cross-Language-Cases ausgewertet. HitRate@3, Recall@3 und MRR jeweils 1,0; alle Ziele Rang 1. Ergebnis als Wiring-Smoke-Test, nicht als Produktionsqualitaet dokumentiert. Qdrant enthaelt sechs synthetische Punkte; PostgreSQL/Realdaten unberuehrt. 1.285 Standardtests bestanden, acht PostgreSQL- und ein Qdrant-Integrationstest uebersprungen; Local-only-/Designcheck und beide Builds erfolgreich. diff --git a/docs/REGULATORY_RAG.md b/docs/REGULATORY_RAG.md index 0c19d33..eedca17 100644 --- a/docs/REGULATORY_RAG.md +++ b/docs/REGULATORY_RAG.md @@ -58,6 +58,21 @@ All fields used by filters have Qdrant payload indexes. An existing collection w ## ISO corpus preparation and ingestion +Data Hub offers a token-protected upload for the matching DIN EN ISO/IEC +27001:2024-01 and 27002:2024-01 PDFs. The same server parser validates both +files, a bounded background job embeds the resulting corpus, and the new +version becomes active only after Qdrant ingestion succeeds. Status, chunk +progress, and incomplete or unreachable index state are visible in Data Hub. +The generated versioned JSON is retained in +the private artifact volume for rebuilding; temporary original PDFs are +removed after parsing. The API never serves licensed text or PDFs to the +browser. A previous active corpus stays selected if parsing or embedding +fails. The corpus upload token is a temporary operator control until Phase 14 +user management provides roles; it does not authorize other application +operations. + +The CLI remains available for local preparation and disaster recovery: + The server-only PDF.js CLI accepts explicit ISO 27001 and ISO 27002 paths and writes corpus-v2 JSON below the ignored `.assessor-data/regulatory-corpora/` directory: ```text diff --git a/server/assessor-http-server.ts b/server/assessor-http-server.ts index a829085..4b73d3e 100644 --- a/server/assessor-http-server.ts +++ b/server/assessor-http-server.ts @@ -1,6 +1,6 @@ import { RiskAcceptanceValidationError, type RiskAcceptanceCommand } from '../src/domain/risk-acceptance'; import type { ScopedRiskAcceptanceWrite } from '../src/domain/scoped-risk-acceptance'; -import { createHash } from 'node:crypto'; +import { createHash, timingSafeEqual } from 'node:crypto'; import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; import { readFile, stat } from 'node:fs/promises'; import { extname, resolve, sep } from 'node:path'; @@ -40,6 +40,7 @@ import type { } from '../src/domain/regulatory-mapping'; import type { RegulatoryMappingService } from './regulatory/regulatory-mapping-service'; import { RegulatoryMappingServiceError } from './regulatory/regulatory-mapping-service'; +import { RegulatoryCorpusUploadError, type RegulatoryCorpusUploadService } from './regulatory/regulatory-corpus-upload-service'; import type { GpoAutomaticAnalysisService } from './gpo/gpo-automatic-analysis-service'; import type { MicrosoftSecurityBaselineSelectionContext } from '../src/domain/microsoft-security-baseline'; import { listProjectGpoBackupStatus } from './gpo/gpo-backup-status-service'; @@ -153,6 +154,8 @@ export interface AssessorHttpServerOptions { cveEnrichmentService?: CveEnrichmentService; aiOverviewService?: AiOverviewService; regulatoryMappingService?: RegulatoryMappingService; + regulatoryCorpusUploadService?: RegulatoryCorpusUploadService; + isoUploadToken?: string; } class HttpError extends Error { @@ -211,6 +214,15 @@ function verifyMutationRequest(request: IncomingMessage): void { } } +function verifyIsoUploadToken(request: IncomingMessage, expected: string | undefined): void { + if (!expected || expected.length < 32) throw new HttpError(503, 'iso_upload_token_unconfigured', 'An ISO upload token of at least 32 characters must be configured.'); + const supplied = request.headers['x-assessor-iso-upload-token']; + const candidate = typeof supplied === 'string' ? supplied : ''; + const expectedHash = createHash('sha256').update(expected).digest(); + const candidateHash = createHash('sha256').update(candidate).digest(); + if (!candidate || !timingSafeEqual(expectedHash, candidateHash)) throw new HttpError(403, 'invalid_iso_upload_token', 'The ISO upload token is invalid.'); +} + function routeSegments(pathname: string): string[] { return pathname.slice(ASSESSOR_API_PREFIX.length).split('/').filter(Boolean).map(segment => decodeURIComponent(segment)); } @@ -373,6 +385,7 @@ function statusForError(error: unknown): { status: number; code: string; message } if (error instanceof AiOverviewServiceError) return { status: error.status, code: error.code, message: error.message }; if (error instanceof RegulatoryMappingServiceError) return { status: error.status, code: error.code, message: error.message }; + if (error instanceof RegulatoryCorpusUploadError) return { status: error.status, code: error.code, message: error.message }; if (error instanceof RiskAcceptanceValidationError) return { status: 400, code: 'risk_acceptance_validation', message: error.message }; return { status: 500, code: 'internal_error', message: 'Die Anfrage konnte nicht verarbeitet werden.' }; } @@ -685,6 +698,29 @@ export function createAssessorHttpServer(options: AssessorHttpServerOptions): Se if (!options.microsoftSecurityBaselineService) throw new HttpError(503, 'baseline_catalog_unavailable', 'The Microsoft baseline catalog is not configured.'); status = 200; sendJson(response, status, { version: 1, records: await options.microsoftSecurityBaselineService.list() }, requestId); return; } + if (segments.join('/') === 'reference-data/iso-corpus' && method === 'GET') { + if (!options.regulatoryCorpusUploadService) throw new HttpError(503, 'iso_corpus_unavailable', 'ISO corpus management is not configured.'); + const corpusStatus = await options.regulatoryCorpusUploadService.liveStatus(); + const tokenConfigured = Boolean(options.isoUploadToken && options.isoUploadToken.length >= 32); + status = 200; sendJson(response, status, tokenConfigured ? corpusStatus : { + ...corpusStatus, configured: false, disabledReason: 'An ISO upload token of at least 32 characters must be configured.' + }, requestId); return; + } + if (segments.join('/') === 'reference-data/iso-corpus' && method === 'POST') { + if (!options.regulatoryCorpusUploadService) throw new HttpError(503, 'iso_corpus_unavailable', 'ISO corpus management is not configured.'); + verifyIsoUploadToken(request, options.isoUploadToken); + const body = await readJson<{ iso27001: { sizeBytes: number; bytesBase64: string }; iso27002: { sizeBytes: number; bytesBase64: string } }>(request, maximumBodyBytes); + const decode = (file: { sizeBytes: number; bytesBase64: string } | undefined): Buffer => { + if (!file || !Number.isSafeInteger(file.sizeBytes) || file.sizeBytes < 1 || file.sizeBytes > 32 * 1024 * 1024 || !isValidImportBase64(file.bytesBase64)) { + throw new HttpError(400, 'invalid_iso_pdf', 'Both ISO PDFs require a valid size and base64 payload of at most 32 MiB each.'); + } + const bytes = Buffer.from(file.bytesBase64, 'base64'); + if (bytes.length !== file.sizeBytes) throw new HttpError(400, 'file_size_mismatch', 'ISO PDF metadata and actual file size do not match.'); + return bytes; + }; + const result = await options.regulatoryCorpusUploadService.submit(decode(body?.iso27001), decode(body?.iso27002)); + status = 202; sendJson(response, status, result, requestId); return; + } if (segments.join('/') === 'reference-data/microsoft-security-baselines' && method === 'POST') { if (!options.microsoftSecurityBaselineService) throw new HttpError(503, 'baseline_catalog_unavailable', 'The Microsoft baseline catalog is not configured.'); const body = await readJson<{ fileName: string; sizeBytes: number; bytesBase64: string }>(request, maximumBodyBytes); diff --git a/server/index.ts b/server/index.ts index c5330ff..d786230 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto'; -import { resolve } from 'node:path'; +import { join, resolve } from 'node:path'; import { defaultImportResourcePolicy } from '../src/importers/resource-policy'; import { createAssessorHttpServer } from './assessor-http-server'; import { NodeImportWorkerExecutor } from './import-worker-client'; @@ -23,6 +23,7 @@ import { createLiteLlmClientFromEnvironment } from './ai/litellm-client'; import { AiOverviewService } from './ai/ai-overview-service'; import { createRegulatoryRagRuntime } from './regulatory/regulatory-rag-runtime'; import { RegulatoryMappingService } from './regulatory/regulatory-mapping-service'; +import { RegulatoryCorpusUploadService } from './regulatory/regulatory-corpus-upload-service'; function boundedInteger(value: string | undefined, fallback: number, minimum: number, maximum: number, name: string): number { if (value === undefined || value.trim() === '') return fallback; @@ -72,11 +73,12 @@ async function main(): Promise { }); await aiOverviewService.recoverInterruptedRuns(); let regulatoryMappingService: RegulatoryMappingService; + let rag: ReturnType | undefined; + let regulatoryDisabledReason: string | undefined; try { const corpusId = process.env.ASSESSOR_RAG_ACTIVE_CORPUS_ID?.trim(); const corpusVersion = process.env.ASSESSOR_RAG_ACTIVE_CORPUS_VERSION?.trim(); - if (!corpusId || !corpusVersion) throw new Error('No active regulatory corpus is configured.'); - const rag = createRegulatoryRagRuntime(); + rag = createRegulatoryRagRuntime(); regulatoryMappingService = new RegulatoryMappingService(runtime.repository, { findingRetrieval: rag.findingRetrieval, index: rag.index, @@ -87,11 +89,22 @@ async function main(): Promise { ...(!aiClient ? { disabledReason: 'Work LiteLLM is not configured.' } : {}) }); } catch (error) { + regulatoryDisabledReason = error instanceof Error ? error.message : 'Regulatory mapping is unavailable.'; regulatoryMappingService = new RegulatoryMappingService(runtime.repository, { - disabledReason: error instanceof Error ? error.message : 'Regulatory mapping is unavailable.' + disabledReason: regulatoryDisabledReason }); } await regulatoryMappingService.recoverInterruptedRuns(); + const regulatoryCorpusUploadService = new RegulatoryCorpusUploadService({ + root: join(artifactRoot, 'regulatory-corpora'), + ingestion: rag?.ingestion, + countIndexed: rag ? (corpusId, corpusVersion) => rag!.index.count({ corpusId, corpusVersion }) : undefined, + pruneIndexed: rag ? (corpusId, corpusVersion) => rag!.index.deleteOtherCorpusVersions(corpusId, corpusVersion) : undefined, + mapping: regulatoryMappingService, + embeddingModel: rag?.embeddingModel, + disabledReason: regulatoryDisabledReason + }); + await regulatoryCorpusUploadService.initialize(); const artifactStore = new FileSystemBackendArtifactStore(artifactRoot); const gpoBackupSubmissionService = new GpoBackupSubmissionService( runtime.repository, @@ -155,6 +168,8 @@ async function main(): Promise { cveEnrichmentService, aiOverviewService, regulatoryMappingService, + regulatoryCorpusUploadService, + isoUploadToken: process.env.ASSESSOR_ISO_UPLOAD_TOKEN?.trim(), automaticGpoAnalysis, importExecutor }); @@ -170,7 +185,7 @@ async function main(): Promise { if (shuttingDown) return; shuttingDown = true; console.log(`${signal}: ASSESSOR wird beendet.`); - void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop(), aiOverviewService.close(), regulatoryMappingService.close()]).then(() => { + void Promise.all([gpoJobRunner.stop(), gpoArtifactReconciliationRunner.stop(), aiOverviewService.close(), regulatoryCorpusUploadService.stop()]).then(() => regulatoryMappingService.close()).then(() => { server.close(async error => { try { await runtime.close(); diff --git a/server/regulatory/iso-pdf-worker-client.ts b/server/regulatory/iso-pdf-worker-client.ts new file mode 100644 index 0000000..3f341ab --- /dev/null +++ b/server/regulatory/iso-pdf-worker-client.ts @@ -0,0 +1,24 @@ +import { Worker } from 'node:worker_threads'; +import type { RegulatoryCorpusV2 } from '../../src/domain/regulatory-rag'; +import type { IsoPdfCorpusOptions } from './iso-pdf-corpus'; + +export function prepareIsoCorpusInWorker(options: IsoPdfCorpusOptions, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(new URL('./iso-pdf-worker.mjs', import.meta.url), { + workerData: options, + resourceLimits: { maxOldGenerationSizeMb: 768, maxYoungGenerationSizeMb: 128 } + }); + const timer = setTimeout(() => { void worker.terminate(); reject(new Error('ISO PDF parsing exceeded five minutes.')); }, 5 * 60_000); + const abort = () => { clearTimeout(timer); void worker.terminate(); reject(signal!.reason); }; + if (signal?.aborted) abort(); else signal?.addEventListener('abort', abort, { once: true }); + const finish = (error?: Error, corpus?: RegulatoryCorpusV2) => { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + if (error) reject(error); else if (corpus) resolve(corpus); + }; + worker.once('message', (message: { corpus?: RegulatoryCorpusV2; error?: string }) => finish( + message.error ? new Error(message.error) : undefined, message.corpus)); + worker.once('error', error => finish(error)); + worker.once('exit', code => { if (code !== 0) finish(new Error(`ISO PDF parser exited with code ${code}.`)); }); + }); +} diff --git a/server/regulatory/iso-pdf-worker.ts b/server/regulatory/iso-pdf-worker.ts new file mode 100644 index 0000000..05e69f0 --- /dev/null +++ b/server/regulatory/iso-pdf-worker.ts @@ -0,0 +1,7 @@ +import { parentPort, workerData } from 'node:worker_threads'; +import { buildIsoPdfCorpus, type IsoPdfCorpusOptions } from './iso-pdf-corpus'; + +void buildIsoPdfCorpus(workerData as IsoPdfCorpusOptions).then( + corpus => parentPort?.postMessage({ corpus }), + error => parentPort?.postMessage({ error: error instanceof Error ? error.message : 'ISO PDF parsing failed.' }) +); diff --git a/server/regulatory/qdrant-regulatory-index.ts b/server/regulatory/qdrant-regulatory-index.ts index d735684..a091cba 100644 --- a/server/regulatory/qdrant-regulatory-index.ts +++ b/server/regulatory/qdrant-regulatory-index.ts @@ -102,11 +102,12 @@ export class QdrantRegulatoryIndex implements RegulatoryChunkIndex { validateEmbedding(entry.embedding, this.options.dimensions); if (entry.sparseVector) validateSparseVector(entry.sparseVector); } - const documents = new Map(); - for (const { chunk } of chunks) documents.set(`${chunk.corpusId}\0${chunk.documentId}`, { corpusId: chunk.corpusId, documentId: chunk.documentId }); + const documents = new Map(); + for (const { chunk } of chunks) documents.set(`${chunk.corpusId}\0${chunk.corpusVersion}\0${chunk.documentId}`, + { corpusId: chunk.corpusId, corpusVersion: chunk.corpusVersion, documentId: chunk.documentId }); for (const document of documents.values()) { await this.request('POST', `/collections/${REGULATORY_QDRANT_COLLECTION}/points/delete?wait=true`, { - filter: { must: [keywordCondition('corpus_id', document.corpusId), keywordCondition('document_id', document.documentId)] } + filter: { must: [keywordCondition('corpus_id', document.corpusId), keywordCondition('corpus_version', document.corpusVersion), keywordCondition('document_id', document.documentId)] } }); } for (let offset = 0; offset < chunks.length; offset += 64) { @@ -181,6 +182,22 @@ export class QdrantRegulatoryIndex implements RegulatoryChunkIndex { return points.map(point => parseChunkPayload(object(point)?.payload)); } + async count(filters: RegulatoryRetrievalFilters): Promise { + const response = await this.request('POST', `/collections/${REGULATORY_QDRANT_COLLECTION}/points/count`, { + filter: { must: [keywordCondition('embedding_model', this.options.embeddingModel), ...filterConditions(filters)] }, + exact: true + }); + const count = object(response.body)?.result?.count; + if (!Number.isSafeInteger(count) || count < 0) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant returned an invalid point count.'); + return count; + } + + async deleteOtherCorpusVersions(corpusId: string, keepCorpusVersion: string): Promise { + await this.request('POST', `/collections/${REGULATORY_QDRANT_COLLECTION}/points/delete?wait=true`, { + filter: { must: [keywordCondition('corpus_id', corpusId)], must_not: [keywordCondition('corpus_version', keepCorpusVersion)] } + }); + } + private async request(method: string, path: string, body?: unknown, acceptedStatuses: number[] = [200]): Promise<{ status: number; body: unknown }> { const url = new URL(path, withTrailingSlash(this.options.baseUrl)); let response: Response; diff --git a/server/regulatory/regulatory-corpus-upload-service.ts b/server/regulatory/regulatory-corpus-upload-service.ts new file mode 100644 index 0000000..5a8e8e5 --- /dev/null +++ b/server/regulatory/regulatory-corpus-upload-service.ts @@ -0,0 +1,215 @@ +import { randomUUID } from 'node:crypto'; +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'; +import { join, resolve, sep } from 'node:path'; +import type { RegulatoryCorpusV2 } from '../../src/domain/regulatory-rag'; +import type { RegulatoryCorpusJob, RegulatoryCorpusStatus, RegulatoryCorpusSummary } from '../../src/domain/regulatory-corpus-management'; +import type { IsoPdfCorpusOptions } from './iso-pdf-corpus'; +import { prepareIsoCorpusInWorker } from './iso-pdf-worker-client'; +import type { RegulatoryIngestionService } from './regulatory-rag-service'; +import type { RegulatoryMappingService } from './regulatory-mapping-service'; + +const MAXIMUM_PDF_BYTES = 32 * 1024 * 1024; +const MAXIMUM_CORPUS_BYTES = 64 * 1024 * 1024; + +export class RegulatoryCorpusUploadError extends Error { + constructor(readonly code: string, message: string, readonly status = 400) { + super(message); + } +} + +export interface RegulatoryCorpusUploadOptions { + root: string; + ingestion?: RegulatoryIngestionService; + mapping: RegulatoryMappingService; + embeddingModel?: string; + disabledReason?: string; + countIndexed?: (corpusId: string, corpusVersion: string) => Promise; + pruneIndexed?: (corpusId: string, keepCorpusVersion: string) => Promise; + prepare?: (options: IsoPdfCorpusOptions, signal?: AbortSignal) => Promise; + now?: () => Date; +} + +export class RegulatoryCorpusUploadService { + private active: RegulatoryCorpusSummary | null = null; + private job: RegulatoryCorpusJob | null = null; + private task?: Promise; + private abort?: AbortController; + private readonly now: () => Date; + private readonly prepare: NonNullable; + + constructor(private readonly options: RegulatoryCorpusUploadOptions) { + this.now = options.now ?? (() => new Date()); + this.prepare = options.prepare ?? prepareIsoCorpusInWorker; + } + + async initialize(): Promise { + await mkdir(this.options.root, { recursive: true, mode: 0o700 }); + const root = resolve(this.options.root); + for (const entry of await readdir(root, { withFileTypes: true })) { + if (!entry.isDirectory() || !/^\.upload-[a-f0-9-]{36}$/.test(entry.name)) continue; + const temporary = resolve(root, entry.name); + if (!temporary.startsWith(root + sep)) throw new Error('ISO temporary path escaped its private storage root.'); + await rm(temporary, { recursive: true, force: true }); + } + this.active = await this.readState('active.json'); + this.job = await this.readState('job.json'); + if (this.job?.status === 'running') { + this.job = { ...this.job, status: 'failed', completedAt: this.now().toISOString(), error: 'Indexing was interrupted. Upload both PDFs again.' }; + await this.writeState('job.json', this.job); + } + if (this.active && await this.corpusExists(this.active.corpusVersion)) { + if (this.active.embeddingModel === this.options.embeddingModel) { + await this.options.mapping.activateCorpus(this.active.corpusId, this.active.corpusVersion); + } else if (this.options.ingestion) { + this.job = { id: randomUUID(), status: 'failed', phase: 'complete', startedAt: this.now().toISOString(), + completedAt: this.now().toISOString(), error: 'The embedding model changed. Upload both PDFs to rebuild the index.' }; + } + } else if (this.active) { + this.active = null; + this.job = { id: randomUUID(), status: 'failed', phase: 'complete', startedAt: this.now().toISOString(), + completedAt: this.now().toISOString(), error: 'The active corpus source is missing. Upload both PDFs again.' }; + } + } + + status(): RegulatoryCorpusStatus { + return { + version: 1, + configured: Boolean(this.options.ingestion), + ...(!this.options.ingestion ? { disabledReason: this.options.disabledReason ?? 'Embedding or Qdrant is not configured.' } : {}), + active: this.active, + job: this.job + }; + } + + async liveStatus(): Promise { + const status = this.status(); + if (!this.active || !this.options.countIndexed) return status; + try { + const count = await this.options.countIndexed(this.active.corpusId, this.active.corpusVersion); + return { ...status, indexStatus: count === this.active.chunkCount ? 'ready' : 'incomplete' }; + } catch { return { ...status, indexStatus: 'unavailable' }; } + } + + async submit(iso27001: Buffer, iso27002: Buffer): Promise { + if (!this.options.ingestion) throw new RegulatoryCorpusUploadError('regulatory_ingestion_unavailable', this.options.disabledReason ?? 'Embedding or Qdrant is not configured.', 503); + if (this.task) throw new RegulatoryCorpusUploadError('regulatory_ingestion_busy', 'A corpus upload is already running.', 409); + for (const [label, bytes] of [['ISO/IEC 27001', iso27001], ['ISO/IEC 27002', iso27002]] as const) { + if (!Buffer.isBuffer(bytes) || bytes.length < 1 || bytes.length > MAXIMUM_PDF_BYTES || !bytes.subarray(0, 1024).includes('%PDF-')) { + throw new RegulatoryCorpusUploadError('invalid_iso_pdf', `${label} must be a PDF no larger than 32 MiB.`); + } + } + this.job = { id: randomUUID(), status: 'running', phase: 'validating', startedAt: this.now().toISOString() }; + const abort = new AbortController(); + this.abort = abort; + // Claim the task slot before the first await so overlapping submissions receive 409. + const persisted = this.writeState('job.json', this.job); + this.task = persisted.then(() => this.run(iso27001, iso27002, abort.signal), () => undefined) + .finally(() => { this.task = undefined; }); + await persisted; + return this.status(); + } + + async close(): Promise { await this.task; } + + async stop(): Promise { + this.abort?.abort(new Error('Indexing was interrupted by a server shutdown. Upload both PDFs again.')); + await this.task; + } + + private async run(iso27001: Buffer, iso27002: Buffer, signal: AbortSignal): Promise { + const temporaryRoot = join(this.options.root, `.upload-${this.job!.id}`); + try { + await mkdir(temporaryRoot, { mode: 0o700 }); + const iso27001Path = join(temporaryRoot, '27001.pdf'); + const iso27002Path = join(temporaryRoot, '27002.pdf'); + await Promise.all([writeFile(iso27001Path, iso27001, { mode: 0o600 }), writeFile(iso27002Path, iso27002, { mode: 0o600 })]); + const corpus = await this.prepare({ iso27001Path, iso27002Path }, signal); + signal.throwIfAborted(); + try { await rm(temporaryRoot, { recursive: true, force: true }); } + catch { throw new Error('Temporary ISO PDFs could not be removed after parsing.'); } + if (!/^[a-z0-9][a-z0-9._-]{0,100}$/.test(corpus.corpusVersion)) throw new Error('The generated corpus version is invalid.'); + const existing = this.active; + if (existing?.corpusId === corpus.corpusId && existing.corpusVersion === corpus.corpusVersion + && existing.embeddingModel === this.options.embeddingModel && existing.parserVersion === corpus.parserVersion + && this.options.countIndexed) { + const count = await this.options.countIndexed(corpus.corpusId, corpus.corpusVersion).catch(() => 0); + if (count === existing.chunkCount) { + await this.finish(); + return; + } + } + const serialized = JSON.stringify(corpus); + if (Buffer.byteLength(serialized) > MAXIMUM_CORPUS_BYTES) throw new Error('The generated corpus exceeds 64 MiB.'); + await this.writeState(`${corpus.corpusVersion}.json`, corpus); + await this.updatePhase('embedding'); + const result = await this.options.ingestion!.ingest(corpus.documents, async (completedChunks, totalChunks) => { + signal.throwIfAborted(); + this.job = { ...this.job!, completedChunks, totalChunks }; + await this.writeState('job.json', this.job); + }); + await this.updatePhase('activating'); + const active: RegulatoryCorpusSummary = { + corpusId: corpus.corpusId, corpusVersion: corpus.corpusVersion, + embeddingModel: this.options.embeddingModel!, parserVersion: corpus.parserVersion, + documentCount: result.documentCount, chunkCount: result.chunkCount, + activatedAt: this.now().toISOString(), + sources: corpus.sources.map(source => ({ standard: source.standard, sha256: source.sha256, pageCount: source.pageCount })) + }; + await this.writeState('active.json', active); + await this.options.mapping.activateCorpus(corpus.corpusId, corpus.corpusVersion); + this.active = active; + // Superseded versions are no longer retrievable; a failed prune is retried by the next activation. + await this.options.pruneIndexed?.(corpus.corpusId, corpus.corpusVersion).catch(() => undefined); + await this.finish(); + } catch (error) { + this.job = { ...this.job!, status: 'failed', completedAt: this.now().toISOString(), + error: error instanceof Error ? error.message.slice(0, 300) : 'Corpus ingestion failed.' }; + await this.writeState('job.json', this.job).catch(() => undefined); + } finally { + iso27001.fill(0); + iso27002.fill(0); + const root = resolve(this.options.root); + const temporary = resolve(temporaryRoot); + if (!temporary.startsWith(root + sep)) throw new Error('ISO temporary path escaped its private storage root.'); + try { await rm(temporary, { recursive: true, force: true }); } + catch { + this.job = { ...this.job!, status: 'failed', completedAt: this.now().toISOString(), + error: 'Temporary ISO PDFs could not be removed from private storage. Restart the server after fixing storage access.' }; + await this.writeState('job.json', this.job).catch(() => undefined); + } + } + } + + private async updatePhase(phase: RegulatoryCorpusJob['phase']): Promise { + this.job = { ...this.job!, phase }; + await this.writeState('job.json', this.job); + } + + private async finish(): Promise { + this.job = { ...this.job!, status: 'succeeded', phase: 'complete', completedAt: this.now().toISOString() }; + await this.writeState('job.json', this.job); + } + + private async corpusExists(version: string): Promise { + if (!/^[a-z0-9][a-z0-9._-]{0,100}$/.test(version)) return false; + try { return (await stat(join(this.options.root, `${version}.json`))).isFile(); } + catch { return false; } + } + + private async readState(name: string): Promise { + try { return JSON.parse(await readFile(join(this.options.root, name), 'utf8')) as T; } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } + } + + private async writeState(name: string, value: RegulatoryCorpusV2 | RegulatoryCorpusSummary | RegulatoryCorpusJob): Promise { + const path = join(this.options.root, name); + const temporary = `${path}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify(value), { mode: 0o600, flag: 'wx' }); + await rename(temporary, path); + } finally { await rm(temporary, { force: true }); } + } +} diff --git a/server/regulatory/regulatory-mapping-service.ts b/server/regulatory/regulatory-mapping-service.ts index 035c2cb..63961f2 100644 --- a/server/regulatory/regulatory-mapping-service.ts +++ b/server/regulatory/regulatory-mapping-service.ts @@ -62,6 +62,7 @@ export class RegulatoryMappingService { resolve: (status: 'succeeded' | 'failed') => void; }> = []; private activeWorkerCount = 0; + private activationPending = false; constructor(private readonly repository: AssessorRepository, private readonly options: RegulatoryMappingServiceOptions = {}) { this.now = options.now ?? (() => new Date()); @@ -72,10 +73,21 @@ export class RegulatoryMappingService { } get generationEnabled(): boolean { - return Boolean(this.options.findingRetrieval && this.options.index && this.options.client + return !this.activationPending && Boolean(this.options.findingRetrieval && this.options.index && this.options.client && this.options.corpusId && this.options.corpusVersion && this.options.embeddingModel); } + async activateCorpus(corpusId: string, corpusVersion: string): Promise { + this.activationPending = true; + try { + await this.close(); + this.options.corpusId = corpusId; + this.options.corpusVersion = corpusVersion; + } finally { + this.activationPending = false; + } + } + async recoverInterruptedRuns(): Promise { const completedAt = this.now().toISOString(); for (const run of (await this.repository.listRegulatoryMappingRuns()).filter(item => ['queued', 'running'].includes(item.status))) { diff --git a/server/regulatory/regulatory-rag-service.ts b/server/regulatory/regulatory-rag-service.ts index b92f7c8..cf2255a 100644 --- a/server/regulatory/regulatory-rag-service.ts +++ b/server/regulatory/regulatory-rag-service.ts @@ -104,7 +104,7 @@ export class RegulatoryIngestionService { } } - async ingest(documents: RegulatoryControlDocument[]): Promise { + async ingest(documents: RegulatoryControlDocument[], onProgress?: (completedChunks: number, totalChunks: number) => Promise | void): Promise { if (!documents.length) throw new Error('At least one regulatory control document is required.'); const documentKeys = new Set(); const chunks = documents.flatMap(input => { @@ -115,9 +115,11 @@ export class RegulatoryIngestionService { return chunkRegulatoryControl(document, { maximumCharacters: this.maximumChunkCharacters }); }); const vectors: number[][] = []; + await onProgress?.(0, chunks.length); for (let offset = 0; offset < chunks.length; offset += this.embeddingBatchSize) { const batch = chunks.slice(offset, offset + this.embeddingBatchSize).map(regulatoryChunkEmbeddingText); vectors.push(...await this.embeddings.embed(batch)); + await onProgress?.(Math.min(offset + batch.length, chunks.length), chunks.length); } if (vectors.length !== chunks.length) throw new Error('Embedding provider returned an unexpected vector count.'); const sparseVectors = buildRegulatorySparseVectors(chunks); diff --git a/src/api/assessor-api-client.ts b/src/api/assessor-api-client.ts index 6924e63..827782a 100644 --- a/src/api/assessor-api-client.ts +++ b/src/api/assessor-api-client.ts @@ -12,6 +12,7 @@ import type { ProjectDataExport } from '../persistence/assessor-repository'; import type { EnvironmentSnapshot, EnvironmentSnapshotQuery } from '../domain/environment-context'; import type { ImportedDataset } from '../domain/dataset'; import type { MicrosoftSecurityBaselineSelectionContext } from '../domain/microsoft-security-baseline'; +import type { RegulatoryCorpusStatus } from '../domain/regulatory-corpus-management'; import type { ImportBatchPreview, ImportBatchSelection } from '../domain/import-workflow'; import { ASSESSOR_API_PREFIX, @@ -391,6 +392,21 @@ export class AssessorApiClient { return this.request('/reference-data/microsoft-security-baselines'); } + getIsoCorpusStatus(): Promise { + return this.request('/reference-data/iso-corpus'); + } + + async submitIsoCorpus(iso27001: File, iso27002: File, token: string): Promise { + const [first, second] = await Promise.all([iso27001.arrayBuffer(), iso27002.arrayBuffer()]); + return this.request('/reference-data/iso-corpus', { + method: 'POST', headers: { 'X-Assessor-Iso-Upload-Token': token }, + body: JSON.stringify({ + iso27001: { sizeBytes: iso27001.size, bytesBase64: bytesToBase64(new Uint8Array(first)) }, + iso27002: { sizeBytes: iso27002.size, bytesBase64: bytesToBase64(new Uint8Array(second)) } + }) + }); + } + async submitMicrosoftSecurityBaseline(file: File): Promise { const bytes = new Uint8Array(await file.arrayBuffer()); return this.request('/reference-data/microsoft-security-baselines', { method: 'POST', body: JSON.stringify({ fileName: file.name, sizeBytes: file.size, bytesBase64: bytesToBase64(bytes) }) }); diff --git a/src/domain/regulatory-corpus-management.ts b/src/domain/regulatory-corpus-management.ts new file mode 100644 index 0000000..2299e0a --- /dev/null +++ b/src/domain/regulatory-corpus-management.ts @@ -0,0 +1,30 @@ +export interface RegulatoryCorpusSummary { + corpusId: string; + corpusVersion: string; + embeddingModel: string; + parserVersion: string; + documentCount: number; + chunkCount: number; + activatedAt: string; + sources: Array<{ standard: string; sha256: string; pageCount: number }>; +} + +export interface RegulatoryCorpusJob { + id: string; + status: 'running' | 'succeeded' | 'failed'; + phase: 'validating' | 'embedding' | 'activating' | 'complete'; + startedAt: string; + completedAt?: string; + completedChunks?: number; + totalChunks?: number; + error?: string; +} + +export interface RegulatoryCorpusStatus { + version: 1; + configured: boolean; + disabledReason?: string; + active: RegulatoryCorpusSummary | null; + indexStatus?: 'ready' | 'incomplete' | 'unavailable'; + job: RegulatoryCorpusJob | null; +} diff --git a/src/features/project-import/DataHub.tsx b/src/features/project-import/DataHub.tsx index bed6a94..d3a5aa9 100644 --- a/src/features/project-import/DataHub.tsx +++ b/src/features/project-import/DataHub.tsx @@ -14,6 +14,7 @@ import { PolicyAnalysisPanel } from './PolicyAnalysisPanel'; import { PanelLoadingState } from '../../components/PanelLoadingState/PanelLoadingState'; import { BackendConnectionDialog } from '../../components/BackendConnectionDialog/BackendConnectionDialog'; import { MicrosoftBaselineLibrary } from './MicrosoftBaselineLibrary'; +import { IsoCorpusLibrary } from './IsoCorpusLibrary'; import { MitreMatrixLibrary } from './MitreMatrixLibrary'; import { clearStoredMitreMatrix, loadStoredMitreMatrix, saveStoredMitreMatrix, validateMitreMatrixData } from '../mitre-graph/mitre-graph'; import { TriangleIcon } from '../../components/TriangleIcon'; @@ -881,6 +882,7 @@ export function DataHub({ activeProjectId, onProjectSelected, onProjectDataChang />
+ {SHOW_MITRE_MATRIX_LIBRARY && (null); + const [iso27001, setIso27001] = useState(null); + const [iso27002, setIso27002] = useState(null); + const [token, setToken] = useState(''); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const load = useCallback(async () => { + try { setStatus(await api.getIsoCorpusStatus()); setError(null); } + catch (reason) { setError(reason instanceof Error ? reason.message : 'ISO corpus status could not be loaded.'); } + }, []); + useEffect(() => { void load(); }, [load]); + useEffect(() => { + if (status?.job?.status !== 'running') return; + const interval = setInterval(() => { void load(); }, 2_000); + return () => clearInterval(interval); + }, [load, status?.job?.status]); + + const submit = async (event: FormEvent) => { + event.preventDefault(); + const form = event.currentTarget as HTMLFormElement; + if (!iso27001 || !iso27002 || !token) return; + if (iso27001.size > MAXIMUM_PDF_BYTES || iso27002.size > MAXIMUM_PDF_BYTES) { + setError('Each PDF must be no larger than 32 MiB.'); return; + } + setSubmitting(true); setError(null); + try { + setStatus(await api.submitIsoCorpus(iso27001, iso27002, token)); + setIso27001(null); setIso27002(null); setToken(''); + form.reset(); + } catch (reason) { setError(reason instanceof Error ? reason.message : 'ISO corpus upload failed.'); } + finally { setSubmitting(false); } + }; + + const running = status?.job?.status === 'running'; + return
+ + {status?.active ? '1' : '0'}ISO Control Corpus + {running ? status.job?.phase === 'embedding' && status.job.totalChunks + ? `Embedding ${status.job.completedChunks ?? 0} / ${status.job.totalChunks}` + : `Processing: ${status.job?.phase}` : status?.active + ? status.indexStatus === 'incomplete' ? 'Rebuild required' : status.indexStatus === 'unavailable' ? 'Index unavailable' : 'Ready' + : 'Not indexed'} + +
+ {status?.active &&
+ Active version {status.active.corpusVersion} + Controls and sections {status.active.documentCount.toLocaleString('en-US')} · {status.active.chunkCount.toLocaleString('en-US')} indexed chunks + Embedding model {status.active.embeddingModel} + {status.active.sources.map(source => {source.standard} {source.pageCount} pages · SHA-256 {source.sha256.slice(0, 12)}…)} +
} + {status?.indexStatus === 'incomplete' &&
The index is incomplete. Upload the same PDFs to rebuild it.
} + {status?.job?.status === 'failed' &&
{status.job.error}
} + {status?.job?.status === 'succeeded' &&
ISO corpus indexed and active.
} + {!status?.configured && status?.disabledReason &&
{status.disabledReason}
} +
void submit(event)}> + + + + +
+

The server keeps the generated corpus privately for rebuilding the index. Original PDFs are removed after parsing.

+ {error &&
{error}
} +
+
; +} diff --git a/src/styles/datahub-redesign.css b/src/styles/datahub-redesign.css index ded009b..13cdd1c 100644 --- a/src/styles/datahub-redesign.css +++ b/src/styles/datahub-redesign.css @@ -113,6 +113,15 @@ .datahub-reference-row > .mitre-library { grid-area: 1 / 1 / 2 / -1; } .datahub-reference-row > .baseline-catalog { grid-area: 2 / 1 / 3 / -1; margin-top: 0; } .datahub-reference-row.is-mitre-hidden > .baseline-catalog { grid-area: 1 / 1 / 2 / -1; } +.datahub-reference-row > .iso-corpus-catalog { grid-area: 3 / 1 / 4 / -1; } +.datahub-reference-row.is-mitre-hidden > .iso-corpus-catalog { grid-area: 2 / 1 / 3 / -1; } +.iso-corpus-content { display: grid; gap: var(--space-4); padding: var(--space-5); } +.iso-corpus-summary { display: grid; gap: var(--space-2); color: var(--text-primary); font-size: var(--type-table-body-size); } +.iso-corpus-summary strong { font-weight: var(--fw-semibold); } +.iso-corpus-form { display: flex; flex-wrap: wrap; align-items: end; gap: var(--space-4); } +.iso-corpus-form label { display: grid; flex: 1 1 var(--tile-content-width); gap: var(--space-2); color: var(--text-primary); font-size: var(--type-table-body-size); } +.iso-corpus-form .btn { flex: 0 0 auto; } +.iso-corpus-note { margin: 0; color: var(--text-secondary); font-size: var(--type-table-body-size); } .datahub-reference-row > .datahub-rack-module { margin-bottom: 0; } /* Der importierte Baseline-Katalog verwendet dieselbe aufklappbare Kopf- und Tabellenform wie die Scan-History im Mandantendetail. */ @@ -131,6 +140,8 @@ .datahub-reference-row > .mitre-library { grid-area: 1 / 1; } .datahub-reference-row > .baseline-catalog { grid-area: 2 / 1; } .datahub-reference-row.is-mitre-hidden > .baseline-catalog { grid-area: 1 / 1; } + .datahub-reference-row > .iso-corpus-catalog { grid-area: 3 / 1; } + .datahub-reference-row.is-mitre-hidden > .iso-corpus-catalog { grid-area: 2 / 1; } .tenant-timeslot-toolbar, .tenant-timeslot-create { grid-template-columns: 1fr; } .datahub-rack-header { grid-template-columns: 20px 10px minmax(0, 1fr) auto; } .datahub-rack-actions { justify-content: stretch; } diff --git a/tests/api/assessor-api-client.test.ts b/tests/api/assessor-api-client.test.ts index dc48a59..6dc331b 100644 --- a/tests/api/assessor-api-client.test.ts +++ b/tests/api/assessor-api-client.test.ts @@ -4,6 +4,20 @@ import { AssessorApiClient } from '../../src/api/assessor-api-client'; const timestamp = '2026-09-06T10:00:00.000Z'; describe('AssessorApiClient', () => { + it('sends both ISO PDFs and the operator token only to the same-origin upload endpoint', async () => { + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify({ version: 1, configured: true, active: null, job: null }), { status: 202 })); + const client = new AssessorApiClient({ fetcher }); + const file = (value: string) => ({ size: value.length, arrayBuffer: async () => new TextEncoder().encode(value).buffer }) as File; + await client.submitIsoCorpus(file('%PDF-first'), file('%PDF-second'), 'synthetic-token'); + const [url, request] = fetcher.mock.calls[0]; + expect(url).toBe('/api/v1/reference-data/iso-corpus'); + expect(new Headers(request?.headers).get('X-Assessor-Iso-Upload-Token')).toBe('synthetic-token'); + expect(new Headers(request?.headers).get('X-Assessor-Request')).toBe('1'); + expect(JSON.parse(String(request?.body))).toMatchObject({ + iso27001: { bytesBase64: btoa('%PDF-first') }, + iso27002: { bytesBase64: btoa('%PDF-second') } + }); + }); it('uses the versioned same-origin API and adds the mutation header only to writes', async () => { const fetcher = vi.fn() .mockResolvedValueOnce(new Response(JSON.stringify({ diff --git a/tests/features/iso-corpus-upload-interaction.test.tsx b/tests/features/iso-corpus-upload-interaction.test.tsx new file mode 100644 index 0000000..02b6448 --- /dev/null +++ b/tests/features/iso-corpus-upload-interaction.test.tsx @@ -0,0 +1,72 @@ +// @vitest-environment jsdom + +import { act } from 'react'; +import { createRoot } from 'react-dom/client'; +import { afterEach, expect, it, vi } from 'vitest'; +import { AssessorApiClient } from '../../src/api/assessor-api-client'; +import type { RegulatoryCorpusStatus } from '../../src/domain/regulatory-corpus-management'; +import { IsoCorpusLibrary } from '../../src/features/project-import/IsoCorpusLibrary'; + +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); + +it('uploads both selected ISO PDFs with the entered operator token and shows embedding progress', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); + const initial: RegulatoryCorpusStatus = { version: 1, configured: true, active: null, job: null }; + vi.spyOn(AssessorApiClient.prototype, 'getIsoCorpusStatus').mockResolvedValue(initial); + const submit = vi.spyOn(AssessorApiClient.prototype, 'submitIsoCorpus').mockResolvedValue({ + ...initial, job: { id: 'synthetic-job', status: 'running', phase: 'embedding', startedAt: '2026-09-23T00:00:00Z', + completedChunks: 2, totalChunks: 8 } + }); + const container = document.createElement('div'); + document.body.append(container); + const root = createRoot(container); + + try { + await act(async () => root.render()); + const fileInputs = container.querySelectorAll('input[type="file"]'); + const first = new File(['%PDF-synthetic-27001'], '27001.pdf', { type: 'application/pdf' }); + const second = new File(['%PDF-synthetic-27002'], '27002.pdf', { type: 'application/pdf' }); + await act(async () => { + Object.defineProperty(fileInputs[0], 'files', { configurable: true, value: [first] }); + fileInputs[0].dispatchEvent(new Event('change', { bubbles: true })); + Object.defineProperty(fileInputs[1], 'files', { configurable: true, value: [second] }); + fileInputs[1].dispatchEvent(new Event('change', { bubbles: true })); + const token = container.querySelector('input[type="password"]')!; + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!.call(token, 'synthetic-secret'); + token.dispatchEvent(new Event('input', { bubbles: true })); + }); + await act(async () => container.querySelector('form')!.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }))); + expect(submit).toHaveBeenCalledWith(first, second, 'synthetic-secret'); + expect(container.textContent).toContain('Embedding 2 / 8'); + expect(container.querySelector('input[type="password"]')!.value).toBe(''); + } finally { + await act(async () => root.unmount()); + container.remove(); + } +}); + +it('clears a transient status polling error once a later poll succeeds', async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true); + vi.useFakeTimers(); + const job = { id: 'synthetic-job', status: 'running' as const, phase: 'embedding' as const, startedAt: '2026-09-23T00:00:00Z' }; + vi.spyOn(AssessorApiClient.prototype, 'getIsoCorpusStatus') + .mockResolvedValueOnce({ version: 1, configured: true, active: null, job }) + .mockRejectedValueOnce(new Error('Synthetic transient failure.')) + .mockResolvedValue({ version: 1, configured: true, active: null, job: { ...job, status: 'succeeded', phase: 'complete' } }); + const container = document.createElement('div'); + document.body.append(container); + const root = createRoot(container); + + try { + await act(async () => root.render()); + await act(async () => { await vi.advanceTimersByTimeAsync(2_000); }); + expect(container.textContent).toContain('Synthetic transient failure.'); + await act(async () => { await vi.advanceTimersByTimeAsync(2_000); }); + expect(container.textContent).not.toContain('Synthetic transient failure.'); + expect(container.textContent).toContain('ISO corpus indexed and active.'); + } finally { + await act(async () => root.unmount()); + container.remove(); + vi.useRealTimers(); + } +}); diff --git a/tests/server/qdrant-regulatory-index.test.ts b/tests/server/qdrant-regulatory-index.test.ts index 9d6783c..ac71935 100644 --- a/tests/server/qdrant-regulatory-index.test.ts +++ b/tests/server/qdrant-regulatory-index.test.ts @@ -71,6 +71,7 @@ describe('Qdrant regulatory index', () => { expect(requests[0].body.filter.must).toEqual([ { key: 'corpus_id', match: { value: 'iso-27001' } }, + { key: 'corpus_version', match: { value: '2022-de-1' } }, { key: 'document_id', match: { value: 'a-5-1-de' } } ]); expect(requests[1].body.points[0]).toMatchObject({ vector: { @@ -99,4 +100,28 @@ describe('Qdrant regulatory index', () => { ASSESSOR_QDRANT_URL: 'http://qdrant.example', ASSESSOR_QDRANT_API_KEY: 'secret' })).toThrow(/must use HTTPS/); }); + + it('deletes every other version of the corpus while keeping the active version', async () => { + const fetcher = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => json({ result: {} })); + const index = new QdrantRegulatoryIndex({ baseUrl: new URL('http://127.0.0.1:6333'), apiKey: 'secret', + dimensions: 2, embeddingModel: 'multilingual-model', timeoutMs: 1_000, fetcher }); + await index.deleteOtherCorpusVersions('iso-27001', '2022-de-2'); + expect(String(fetcher.mock.calls[0][0])).toContain('/points/delete?wait=true'); + expect(JSON.parse(String(fetcher.mock.calls[0][1]?.body))).toEqual({ filter: { + must: [{ key: 'corpus_id', match: { value: 'iso-27001' } }], + must_not: [{ key: 'corpus_version', match: { value: '2022-de-2' } }] + } }); + }); + + it('counts only points in the requested corpus version and embedding model', async () => { + const fetcher = vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => json({ result: { count: 2 } })); + const index = new QdrantRegulatoryIndex({ baseUrl: new URL('http://127.0.0.1:6333'), apiKey: 'secret', + dimensions: 2, embeddingModel: 'multilingual-model', timeoutMs: 1_000, fetcher }); + await expect(index.count({ corpusId: 'iso-27001', corpusVersion: '2022-de-1' })).resolves.toBe(2); + expect(JSON.parse(String(fetcher.mock.calls[0][1]?.body))).toMatchObject({ exact: true, filter: { must: [ + { key: 'embedding_model', match: { value: 'multilingual-model' } }, + { key: 'corpus_id', match: { value: 'iso-27001' } }, + { key: 'corpus_version', match: { value: '2022-de-1' } } + ] } }); + }); }); diff --git a/tests/server/regulatory-corpus-upload-http.test.ts b/tests/server/regulatory-corpus-upload-http.test.ts new file mode 100644 index 0000000..e712960 --- /dev/null +++ b/tests/server/regulatory-corpus-upload-http.test.ts @@ -0,0 +1,190 @@ +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createAssessorHttpServer } from '../../server/assessor-http-server'; +import { RegulatoryCorpusUploadService } from '../../server/regulatory/regulatory-corpus-upload-service'; +import type { RegulatoryIngestionService } from '../../server/regulatory/regulatory-rag-service'; +import type { RegulatoryMappingService } from '../../server/regulatory/regulatory-mapping-service'; +import type { RegulatoryCorpusV2 } from '../../src/domain/regulatory-rag'; +import { LocalAssessorRepository } from '../../src/persistence/local-assessor-repository'; + +const token = 'synthetic-operator-token-at-least-32-characters'; +const pdf = Buffer.from('%PDF-synthetic-only'); +const path = '/api/v1/reference-data/iso-corpus'; + +describe('operator ISO corpus upload from Data Hub', () => { + let directory: string; + let server: Server; + let baseUrl: string; + let service: RegulatoryCorpusUploadService; + let corpusVersion: string; + let failEmbedding: boolean; + let indexedCount: number; + let prepareGate: Promise | undefined; + const pruneIndexed = vi.fn(async () => undefined); + const activateCorpus = vi.fn(async () => undefined); + const ingest = vi.fn(async () => { + if (failEmbedding) throw new Error('Synthetic embedding failure.'); + indexedCount = 2; + return { documentCount: 1, chunkCount: 2, embeddingModel: 'synthetic-model' }; + }); + + beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'assessor-iso-upload-')); + await writeFile(join(directory, 'index.html'), '
synthetic app
'); + corpusVersion = 'synthetic-v1'; + failEmbedding = false; + indexedCount = 0; + prepareGate = undefined; + pruneIndexed.mockClear(); + activateCorpus.mockClear(); + ingest.mockClear(); + service = makeService(); + await service.initialize(); + server = createAssessorHttpServer({ + repository: new LocalAssessorRepository(), staticRoot: directory, + regulatoryCorpusUploadService: service, isoUploadToken: token + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterEach(async () => { + await service.close(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + await rm(directory, { recursive: true, force: true }); + }); + + function makeService(): RegulatoryCorpusUploadService { + return new RegulatoryCorpusUploadService({ + root: join(directory, 'artifacts', 'regulatory-corpora'), + embeddingModel: 'synthetic-model', + countIndexed: async () => indexedCount, + ingestion: { ingest } as unknown as RegulatoryIngestionService, + mapping: { activateCorpus } as unknown as RegulatoryMappingService, + pruneIndexed, + prepare: async () => { await prepareGate; return ({ + schemaVersion: 2, corpusId: 'synthetic-iso', corpusVersion, language: 'de', + parserVersion: 'synthetic', generatedAt: '2026-09-23T00:00:00.000Z', + sources: [ + { standard: 'ISO/IEC 27001', sourceEdition: '2022', nationalEdition: 'synthetic', sha256: 'a'.repeat(64), pageCount: 2 }, + { standard: 'ISO/IEC 27002', sourceEdition: '2022', nationalEdition: 'synthetic', sha256: 'b'.repeat(64), pageCount: 3 } + ], + documents: [{ schemaVersion: 2, corpusId: 'synthetic-iso', corpusVersion, + documentId: 'synthetic-control', framework: 'Synthetic', frameworkVersion: '1', + controlId: '5.1', language: 'de', title: 'Synthetic control', text: 'Private synthetic control text.' }] + } satisfies RegulatoryCorpusV2); } + }); + } + + function upload(uploadToken?: string): Promise { + const file = { sizeBytes: pdf.length, bytesBase64: pdf.toString('base64') }; + return fetch(`${baseUrl}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Assessor-Request': '1', Origin: baseUrl, + ...(uploadToken ? { 'X-Assessor-Iso-Upload-Token': uploadToken } : {}) }, + body: JSON.stringify({ iso27001: file, iso27002: file }) + }); + } + + it('requires the operator token and never returns the licensed corpus text', async () => { + expect((await upload()).status).toBe(403); + expect((await upload('wrong-token')).status).toBe(403); + expect(ingest).not.toHaveBeenCalled(); + + const accepted = await upload(token); + expect(accepted.status).toBe(202); + expect((await accepted.json()).job.status).toBe('running'); + await service.close(); + + const status = await fetch(`${baseUrl}${path}`); + expect(status.status).toBe(200); + const body = await status.json(); + expect(body.active).toMatchObject({ corpusVersion: 'synthetic-v1', documentCount: 1, chunkCount: 2 }); + expect(body.indexStatus).toBe('ready'); + expect(body.job.status).toBe('succeeded'); + expect(JSON.stringify(body)).not.toContain('Private synthetic control text'); + expect(JSON.stringify(body)).not.toContain(token); + expect((await readFile(join(directory, 'artifacts', 'regulatory-corpora', 'synthetic-v1.json'), 'utf8'))) + .toContain('Private synthetic control text'); + expect(activateCorpus).toHaveBeenCalledWith('synthetic-iso', 'synthetic-v1'); + }); + + it('keeps the previous version active after a failed replacement and reloads it after restart', async () => { + await upload(token); + await service.close(); + corpusVersion = 'synthetic-v2'; + failEmbedding = true; + expect((await upload(token)).status).toBe(202); + await service.close(); + expect(service.status().active?.corpusVersion).toBe('synthetic-v1'); + expect(service.status().job?.status).toBe('failed'); + expect(activateCorpus).toHaveBeenCalledTimes(1); + + const restarted = makeService(); + await restarted.initialize(); + expect(restarted.status().active?.corpusVersion).toBe('synthetic-v1'); + expect(activateCorpus).toHaveBeenCalledTimes(2); + }); + + it('reuses a complete active index and rebuilds it from the same upload when Qdrant points are missing', async () => { + await upload(token); + await service.close(); + expect(ingest).toHaveBeenCalledTimes(1); + indexedCount = 2; + await upload(token); + await service.close(); + expect(ingest).toHaveBeenCalledTimes(1); + indexedCount = 0; + expect((await (await fetch(`${baseUrl}${path}`)).json()).indexStatus).toBe('incomplete'); + await upload(token); + await service.close(); + expect(ingest).toHaveBeenCalledTimes(2); + }); + + it('rejects an overlapping second upload with 409 instead of starting a parallel job', async () => { + const [first, second] = await Promise.all([upload(token), upload(token)]); + expect([first.status, second.status].sort()).toEqual([202, 409]); + await service.close(); + expect(ingest).toHaveBeenCalledTimes(1); + expect(service.status().job?.status).toBe('succeeded'); + }); + + it('prunes superseded corpus versions only after a successful activation', async () => { + await upload(token); + await service.close(); + expect(pruneIndexed).toHaveBeenCalledWith('synthetic-iso', 'synthetic-v1'); + corpusVersion = 'synthetic-v2'; + failEmbedding = true; + await upload(token); + await service.close(); + expect(pruneIndexed).toHaveBeenCalledTimes(1); + }); + + it('stops a running upload on server shutdown without embedding or activating it', async () => { + let release!: () => void; + prepareGate = new Promise(resolve => { release = resolve; }); + expect((await upload(token)).status).toBe(202); + const stopped = service.stop(); + release(); + await stopped; + expect(ingest).not.toHaveBeenCalled(); + expect(activateCorpus).not.toHaveBeenCalled(); + expect(service.status().job).toMatchObject({ status: 'failed', error: expect.stringContaining('server shutdown') }); + }); + + it('removes original PDFs left by an interrupted process before restoring corpus status', async () => { + const temporary = join(directory, 'artifacts', 'regulatory-corpora', '.upload-12345678-1234-1234-1234-123456789abc'); + await mkdir(temporary); + await writeFile(join(temporary, '27001.pdf'), pdf); + const restarted = makeService(); + await restarted.initialize(); + await expect(stat(temporary)).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); diff --git a/vite.server.config.ts b/vite.server.config.ts index 14cf469..5a0c289 100644 --- a/vite.server.config.ts +++ b/vite.server.config.ts @@ -15,6 +15,7 @@ export default defineConfig({ 'gpo-inventory-worker': 'server/gpo/gpo-inventory-worker.ts', 'regulatory-ingest': 'server/regulatory/regulatory-ingest.ts', 'regulatory-prepare-iso': 'server/regulatory/regulatory-prepare-iso.ts', + 'iso-pdf-worker': 'server/regulatory/iso-pdf-worker.ts', 'regulatory-evaluate': 'server/regulatory/regulatory-evaluate.ts' }, output: {