feat(regulatory-rag): add Qdrant-backed regulatory RAG retrieval and deployment wiring
This commit is contained in:
parent
d9ebdd22fd
commit
7877c00c4e
|
|
@ -10,6 +10,23 @@ ASSESSOR_GPO_JOB_LEASE_MS=60000
|
|||
ASSESSOR_GPO_JOB_RETRY_MS=5000
|
||||
ASSESSOR_GPO_JOB_POLL_MS=1000
|
||||
|
||||
# Qdrant is a rebuildable regulatory retrieval index. This key is local/deployment secret material.
|
||||
ASSESSOR_QDRANT_URL=http://127.0.0.1:6333
|
||||
ASSESSOR_QDRANT_PORT=6333
|
||||
ASSESSOR_QDRANT_API_KEY=replace-with-a-long-random-local-qdrant-key
|
||||
|
||||
# Required only for regulatory ingestion/retrieval. Use an approved multilingual embedding model.
|
||||
# ASSESSOR_RAG_EMBEDDING_BASE_URL=https://work-litellm.example/v1
|
||||
# ASSESSOR_RAG_EMBEDDING_MODEL=approved-multilingual-embedding-model
|
||||
# ASSESSOR_RAG_EMBEDDING_API_KEY=replace-with-work-litellm-key
|
||||
# ASSESSOR_RAG_EMBEDDING_DIMENSIONS=1024
|
||||
# Set true only when the provider supports OpenAI's dimensions request (for example text-embedding-3-large).
|
||||
# Leave false/unset for llama.cpp, which returns the model's native dimension.
|
||||
# ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS=false
|
||||
# ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS=60000
|
||||
# ASSESSOR_RAG_CHUNK_CHARACTERS=1600
|
||||
# ASSESSOR_RAG_EMBEDDING_BATCH_SIZE=32
|
||||
|
||||
# Optional work LiteLLM AI Overview. Leave ASSESSOR_AI_MODE unset to keep outbound AI disabled.
|
||||
# Store real values only in the ignored .env.postgres or deployment environment secret.
|
||||
# ASSESSOR_AI_MODE=litellm
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@
|
|||
# experimental -> experimental (10.0.0.154:4174)
|
||||
#
|
||||
# The existing HTTPS production deployment is not managed by this workflow.
|
||||
# Both new environments use separate Compose projects, databases and artifact
|
||||
# volumes. Their complete environment files come from different Gitea secrets.
|
||||
# Both new environments use separate Compose projects, databases, artifact
|
||||
# volumes and rebuildable Qdrant indexes. Their complete environment files come
|
||||
# from different Gitea secrets.
|
||||
name: Deploy
|
||||
|
||||
on:
|
||||
|
|
@ -33,7 +34,7 @@ jobs:
|
|||
fi
|
||||
printf '%s\n' "$ASSESSOR_ENV_FILE" > .env
|
||||
chmod 600 .env
|
||||
for key in ASSESSOR_POSTGRES_DB ASSESSOR_POSTGRES_USER ASSESSOR_POSTGRES_PASSWORD; do
|
||||
for key in ASSESSOR_POSTGRES_DB ASSESSOR_POSTGRES_USER ASSESSOR_POSTGRES_PASSWORD ASSESSOR_QDRANT_API_KEY; do
|
||||
if ! grep -q "^${key}=." .env; then
|
||||
echo "::error::PROD_ENV_FILE does not contain a value for ${key}."
|
||||
exit 1
|
||||
|
|
@ -57,7 +58,7 @@ jobs:
|
|||
fi
|
||||
printf '%s\n' "$ASSESSOR_ENV_FILE" > .env
|
||||
chmod 600 .env
|
||||
for key in ASSESSOR_POSTGRES_DB ASSESSOR_POSTGRES_USER ASSESSOR_POSTGRES_PASSWORD; do
|
||||
for key in ASSESSOR_POSTGRES_DB ASSESSOR_POSTGRES_USER ASSESSOR_POSTGRES_PASSWORD ASSESSOR_QDRANT_API_KEY; do
|
||||
if ! grep -q "^${key}=." .env; then
|
||||
echo "::error::STAGING_ENV_FILE does not contain a value for ${key}."
|
||||
exit 1
|
||||
|
|
@ -118,7 +119,7 @@ jobs:
|
|||
services="$(docker ps -a --filter "label=com.docker.compose.project=$ASSESSOR_STACK" --format '{{.Label "com.docker.compose.service"}}' | sort -u)"
|
||||
for service in $services; do
|
||||
case "$service" in
|
||||
app|postgres) ;;
|
||||
app|postgres|qdrant) ;;
|
||||
*)
|
||||
echo "::error::Compose project $ASSESSOR_STACK contains an unexpected service."
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@
|
|||
# The workflow always supplies a distinct project name and VPN-only host port:
|
||||
# assessor_prod -> 10.0.0.154:4173
|
||||
# assessor_experimental -> 10.0.0.154:4174
|
||||
# Compose scopes both named volumes by project, so the environments never share
|
||||
# PostgreSQL or artifacts. PostgreSQL is intentionally not published to the host.
|
||||
# Compose scopes all named volumes by project, so the environments never share
|
||||
# PostgreSQL, artifacts, or the rebuildable Qdrant index. PostgreSQL and Qdrant
|
||||
# are intentionally not published to the host.
|
||||
name: ${ASSESSOR_STACK:-assessor_prod}
|
||||
|
||||
services:
|
||||
|
|
@ -23,6 +24,12 @@ services:
|
|||
ASSESSOR_POSTGRES_USER: ${ASSESSOR_POSTGRES_USER:?set ASSESSOR_POSTGRES_USER}
|
||||
ASSESSOR_POSTGRES_PASSWORD: ${ASSESSOR_POSTGRES_PASSWORD:?set ASSESSOR_POSTGRES_PASSWORD}
|
||||
ASSESSOR_DATABASE_SSL: disable
|
||||
ASSESSOR_QDRANT_URL: http://qdrant:6333
|
||||
ASSESSOR_QDRANT_API_KEY: ${ASSESSOR_QDRANT_API_KEY:?set ASSESSOR_QDRANT_API_KEY}
|
||||
ASSESSOR_RAG_EMBEDDING_BASE_URL: ${ASSESSOR_RAG_EMBEDDING_BASE_URL:-}
|
||||
ASSESSOR_RAG_EMBEDDING_MODEL: ${ASSESSOR_RAG_EMBEDDING_MODEL:-}
|
||||
ASSESSOR_RAG_EMBEDDING_API_KEY: ${ASSESSOR_RAG_EMBEDDING_API_KEY:-}
|
||||
ASSESSOR_RAG_EMBEDDING_DIMENSIONS: ${ASSESSOR_RAG_EMBEDDING_DIMENSIONS:-}
|
||||
ports:
|
||||
- "${ASSESSOR_BIND:?set ASSESSOR_BIND}:${ASSESSOR_PUBLISH_PORT:?set ASSESSOR_PUBLISH_PORT}:4173"
|
||||
volumes:
|
||||
|
|
@ -53,7 +60,17 @@ services:
|
|||
volumes:
|
||||
- assessor-postgres-data:/var/lib/postgresql/data
|
||||
|
||||
qdrant:
|
||||
image: qdrant/qdrant:v1.19.1
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
QDRANT__SERVICE__API_KEY: ${ASSESSOR_QDRANT_API_KEY:?set ASSESSOR_QDRANT_API_KEY}
|
||||
QDRANT__TELEMETRY_DISABLED: "true"
|
||||
volumes:
|
||||
- assessor-qdrant-data:/qdrant/storage
|
||||
|
||||
volumes:
|
||||
assessor-artifacts:
|
||||
assessor-report-archive:
|
||||
assessor-postgres-data:
|
||||
assessor-qdrant-data:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,20 @@ services:
|
|||
volumes:
|
||||
- assessor-postgres-data:/var/lib/postgresql/data
|
||||
|
||||
qdrant:
|
||||
image: qdrant/qdrant:v1.19.1
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
QDRANT__SERVICE__API_KEY: ${ASSESSOR_QDRANT_API_KEY:?ASSESSOR_QDRANT_API_KEY must be set}
|
||||
QDRANT__TELEMETRY_DISABLED: "true"
|
||||
ports:
|
||||
- "127.0.0.1:${ASSESSOR_QDRANT_PORT:-6333}:6333"
|
||||
volumes:
|
||||
- assessor-qdrant-data:/qdrant/storage
|
||||
|
||||
volumes:
|
||||
assessor-postgres-data:
|
||||
name: assessor-2_0_assessor-postgres-data
|
||||
external: true
|
||||
assessor-qdrant-data:
|
||||
name: assessor-2_0_assessor-qdrant-data
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ Also identify the existing reverse proxy and firewall policy without changing
|
|||
them. Confirm that `10.0.0.154` is a host address, port 4174 is unused, the
|
||||
runner uses this Docker daemon, and `assessor_experimental` does not belong to
|
||||
an unrelated stack. Port 4173 and project `assessor_prod` are intentionally
|
||||
adopted as stable; only its expected `app` and `postgres` services may exist.
|
||||
adopted as stable; only its expected `app`, `postgres`, and `qdrant` services may exist.
|
||||
Any other conflict blocks deployment and is never resolved by stopping or
|
||||
deleting the existing resource.
|
||||
|
||||
|
|
@ -83,14 +83,16 @@ dependencies, runs the complete test suite and production build, validates the
|
|||
resolved Compose configuration, checks port/project isolation, backs up an
|
||||
existing target environment, then builds and starts only that environment.
|
||||
|
||||
Compose project scoping produces independent PostgreSQL, artifact and report-archive volumes.
|
||||
Compose project scoping produces independent PostgreSQL, artifact, report-archive and Qdrant volumes.
|
||||
The Report Engine (2.10.0) archives every report run write-once under
|
||||
`ASSESSOR_REPORT_ARCHIVE_DIR` (`/var/lib/assessor/report-archive` in Compose). The archive
|
||||
holds complete technical customer data, is deleted together with its client and must be
|
||||
included in backup and retention decisions like the artifact volume. The Report Engine
|
||||
reads PostgreSQL through its own pool with `default_transaction_read_only=on`; it runs
|
||||
no migrations and writes nothing to the database.
|
||||
PostgreSQL has no host-published port. `restart: unless-stopped` preserves both
|
||||
PostgreSQL and Qdrant have no host-published ports. Qdrant is an
|
||||
authenticated, rebuildable regulatory index; its volume improves restart time
|
||||
but is not a system-of-record backup. `restart: unless-stopped` preserves both
|
||||
environments across Docker daemon and server restarts.
|
||||
|
||||
Feature promotion is a merge from `experimental` to `main`. Stable is rebuilt
|
||||
|
|
|
|||
|
|
@ -1047,6 +1047,16 @@ Referenzerkennung 2026-09-26 (Phase 13D, erster Umfang umgesetzt und verifiziert
|
|||
|
||||
Status: lebendes Projektdokument
|
||||
|
||||
Regulatory-RAG-Llama-Livebaseline 2026-09-15 (Phase 13F, Nutzerstart): Der lokal gestartete llama.cpp-Server Build 10472 meldet Health 200 und bedient BGE-M3 FP16 ueber den OpenAI-kompatiblen Endpoint. Die produktiv gebauten CLIs indexierten den eingecheckten ausschliesslich synthetischen EN/DE-Korpus mit sechs Dokumenten als sechs 1.024-dimensionale Chunks in den kanonischen Qdrant 1.19.1. Drei ausdruecklich sprachuebergreifende Smoke-Cases erreichten bei K=3 HitRate 1,0, Recall 1,0 und MRR 1,0; alle erwarteten Kontrollen standen auf Rang 1. Das belegt End-to-End-Verkabelung und grundlegende Sprachueberquerung, wegen nur drei klar getrennten Kontrollthemen aber keine produktive Retrievalqualitaet. Qdrant enthaelt sechs synthetische Punkte; PostgreSQL und Realdaten wurden nicht beruehrt. 1.285 Standardtests bestanden, acht optionale PostgreSQL- und ein optionaler Qdrant-Test uebersprungen; Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich (bestehender Bundlehinweis). Ein repraesentativer autorisierter Korpus und unabhaengig gelabelter EN/DE-Satz bleiben fuer die reale Baseline erforderlich.
|
||||
|
||||
Regulatory-RAG-Local-Embeddings 2026-09-15 (Phase 13F, Nutzerentscheidung): Die autorisierte Legacy-Collection `regulation_catalog_v1` wurde nach exakter Ein-Collection-Pruefung geloescht; der leere alte Qdrant-1.12.4-Container wurde entfernt, sein bisheriges Volume jedoch bewahrt. Der authentifizierte `assessor-2_0-qdrant-1` mit Qdrant 1.19.1 laeuft nun am kanonischen `127.0.0.1:6333`; Auth 401/200, gruene Collection und null Punkte sind bestaetigt. Fuer die lokale Dense-Entwicklung ist BGE-M3 mit llama.cpp festgelegt: multilingual, native 1.024 Dimensionen und damit passend zur Collection. Der Embeddingadapter erlaubt HTTP nur fuer Loopback, verlangt weiterhin einen Bearer-Key und lehnt Remote-HTTP ab. Ein neuer optionaler Request-Dimensionsschalter bleibt fuer einen spaeteren LiteLLM-Vergleich mit `text-embedding-3-large` standardmaessig aus, weil llama.cpp die native Modelldimension liefert. Das installierte llama.cpp Build 10472 ist vorhanden; ein BGE-M3-GGUF beziehungsweise dessen freigegebener Download und Serverstart fehlen noch. 1.285 Standardtests bestanden; acht optionale PostgreSQL- und ein optionaler Qdrant-Test uebersprungen, Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich (bestehender Bundlehinweis). Keine PostgreSQL-Mutation, keine Realdateninspektion und kein Reimport.
|
||||
|
||||
Regulatory-RAG-WSL-Liveabnahme 2026-09-15 (Phase 13F, Nutzerkorrektur): Docker Engine 29.7.2 und Compose 5.5.0 sind in WSL2/Ubuntu erreichbar. Der kanonische Container `assessor-2_0-postgres-1` ist gesund und bindet unveraendert `127.0.0.1:5432` mit dem kanonischen externen Volume; PostgreSQL wurde nicht gelesen oder mutiert. Da ein veralteter, unauthentifizierter Qdrant 1.12.4 aus dem separaten Compose-Projekt `assessor` Port 6333 und das nichtleere Collection-Label `regulation_catalog_v1` belegt, wurde er nicht gestoppt oder veraendert. Der neue authentifizierte `assessor-2_0-qdrant-1` mit Qdrant 1.19.1 wurde fuer die Abnahme auf Loopback-Port 6334 und dem getrennten rebuildbaren Volume gestartet. Ohne Key antwortet er 401, mit Key 200. Ein opt-in Produktioncode-Integrationstest erstellte die gruene Collection `assessor_regulatory_chunks_v1` mit Dense 1.024/Cosine, Sparse-Slot, neun Payloadindizes und einem synthetischen Punkt und pruefte Replace, Filter und Retrieval erfolgreich; sein reservierter synthetischer Korpus wurde danach geloescht und die Collection enthaelt wieder null Punkte. Der reale Ingestion-CLI-Lauf stoppte vor Provideraufruf und Qdrantmutation, weil die konfigurierte loopback-HTTP-Embedding-URL die verpflichtende HTTPS-Pruefung nicht besteht und lokal nicht erreichbar ist. 1.284 Standardtests bestanden; acht optionale PostgreSQL- und ein optionaler Qdrant-Test wurden im Standardlauf uebersprungen, der Qdrant-Test separat 1/1 bestanden. Local-only-/Designpruefung und beide Produktionsbuilds erfolgreich (bestehender Bundlehinweis). Kein Realdatenzugriff, keine PostgreSQL-Mutation und kein Reimport.
|
||||
|
||||
Regulatory-RAG-Finding-Retrieval 2026-09-15 (Phase 13F, ausdruecklicher Nutzerauftrag): Die bestehende `UnifiedResultProjection` ist ueber eine backendinterne fail-closed Grenze an den `RegulatoryRetriever` angebunden. Nur katalogeigene oeffentliche Maester-/PingCastle-Regel-IDs, fest definierte oeffentliche Technikthemen und bekannte oeffentliche Toolnamen gelangen in die Embeddingquery; Rohtexte werden ausschliesslich lokal zur Themenklassifikation gelesen, waehrend Titel, Beschreibung, Empfehlung, Assets, Pfade, Mandanten-/Projekt-/Review- und Provenienzdaten nicht kopiert werden. Ohne oeffentliches Signal findet kein Retrieval statt. Ein begrenzter versionierter EN/DE-Evaluationsvertrag und `rag:evaluate` messen HitRate@K, Recall@K und MRR auf Kontrollebene und geben nur Aggregatmetriken aus. 1.283 synthetische Tests bestanden, acht optionale PostgreSQL-Integrationstests uebersprungen; Local-only-/Designpruefung und Client-/Server-Produktionsbuild erfolgreich (bestehender Bundlehinweis). Kein API-/UI-Pfad, keine Datenbankmutation, keine automatische Mappingbehauptung und kein Reimport. Organisatorische Datenschutzfreigabe, autorisierter Korpus und reale Dense-Baseline bleiben offen.
|
||||
|
||||
Regulatory-RAG-Grundlage 2026-09-15 (Phase 13F, ausdruecklicher Nutzerauftrag): PostgreSQL bleibt alleinige fachliche Quelle fuer Mandanten, Assessments, Scans, Findings, Ergebnisse und Reviews. Qdrant 1.19.1 ist als authentifizierter, rebuildbarer regulatorischer Suchindex in lokale und Deployment-Compose-Stacks aufgenommen. Ein versionierter plattformneutraler Chunkvertrag, deterministische Kontrolltextsegmentierung, ein abgesicherter OpenAI-kompatibler Adapter fuer ein explizit konfiguriertes mehrsprachiges Embeddingmodell, Qdrant-Collection-/Payload-Indexverwaltung, dichte gefilterte Suche, ein CLI-Ingestionpfad und getrennte Candidate-/Reranker-Schnittstellen bilden den minimalen Vertikalschnitt. Die Sparse-Vektorposition ist fuer spaetere Hybridabfrage reserviert, wird aber noch nicht als implementierte BM25-/Sparse-Suche ausgegeben. Finding-Query-Builder, automatische Zuordnung, LLM-Klassifikation, Confidence-Kalibrierung, Nutzeroberflaeche und persistierte Mappingprovenienz bleiben offen. 1.275 synthetische Tests bestanden, acht optionale PostgreSQL-Integrationstests uebersprungen; Local-only-/Designpruefung und Client-/Server-Produktionsbuild erfolgreich (bestehender Bundlehinweis). Docker ist auf diesem Host nicht installiert; deshalb keine Compose-Livevalidierung, kein Qdrant-Start und kein echter Embedding-/Korpuslauf. Keine Scanner-, PostgreSQL- oder Realdatenmutation und kein Reimport.
|
||||
|
||||
Work-LiteLLM-AI-Overview 2026-09-14 (Phasen 7/13C mit Phase 11, ausdruecklicher Nutzerauftrag): Client Info besitzt eine eigene `AI Overview`-Registerkarte fuer normale, gemischte und reine Defender-Timeslots. Eine explizite Generate-/Regenerate-Aktion liest den gesamten aktiven Timeslot ohne Source- oder Darstellungsfilter, verwendet unveraendert den versionierten `resultsToPrivacySafeExportJson`-Vertrag, bildet daraus einen kanonisch sortierten SHA-256-Input und uebergibt ausschliesslich diesen datensparsamen Inhalt serverseitig an das konfigurierte Work-LiteLLM. HTTPS-Pflicht, fest konfigurierte Ziel-URL, Redirectablehnung, Bearer-Secret nur im Backend, Timeout, Antwortgroesse, objektgrenzenbasierte hierarchische Chunks, maximal 64 Aufrufe und lokale Schema-/Quell-ID-Pruefung begrenzen die externe Verarbeitung. Migration 024 und beide Repository-Implementierungen speichern append-only Runs mit eindeutiger Timeslot-Idempotenz, Lifecycle, Lease, Versions-/Hash-/Modellmetadaten und ausschliesslich dem validierten Endergebnis; Browser, Export und Logs erhalten weder API-Key noch Providerantworten oder Zwischenprompts. Gespeicherte Ergebnisse bleiben bei deaktivierter Generierung lesbar und werden bei Input-, Modell-, Prompt- oder Schemadrift als veraltet markiert. AI-Ausgaben sind rein beratend und veraendern keine Scores, Findings, Reviews, Imports oder Accepted-Risk-Daten. 1.262 synthetische Tests bestanden, acht optionale PostgreSQL-Integrationstests uebersprungen; Local-only-/Designpruefung und Client-/Server-Produktionsbuild erfolgreich (bestehender Bundlehinweis). Die lokale Secretdatei enthaelt Modus, HTTPS-Work-URL, Modell und Key; ein minimaler synthetischer Workstation-Request erreichte den Gateway, wurde jedoch mit HTTP 401 abgewiesen, ohne den Providerbody auszulesen oder auszugeben. Docker ist auf diesem Ausfuehrungshost nicht verfuegbar; daher wurden weder kanonisches PostgreSQL-Backup noch Anwendung von Migration 024 oder PostgreSQL-Integration ausgefuehrt. Credentialfreigabe, erfolgreicher synthetischer Workstation-/Server-Liveaufruf und Migration bleiben Betriebsabnahme. Keine Realdateninspektion und kein Reimport erforderlich; Phasenstatus unveraendert.
|
||||
|
||||
2026-09-29 - Phase 7 / produktiver Management Report mit Heatmap und Ergebnisbalken (2.9.1, umgesetzt und fokussiert verifiziert): Die bestehende Client-Info-Aktion `Management Report` erzeugt fuer den gewaehlten Kunden und Timeslot jetzt den faktenbasierten HTML-Bericht aus der gemeinsamen Reportprojektion. Der alte Mixed-Method-Score, Donut, Domain-Spine, Priority-Finding-/Scannerlisten, Konvergenz, Massnahmen und Source Overview entfallen nur aus Report A; Report B bleibt technisch. Der Bericht zeigt erfasste DC, bewertete und abweichende Ergebnisse, eine Heatmap des Abweichungsanteils und die zusaetzlich vom Nutzer gezeigte Pass-/Abweichungs-Grafik mit Pass Rate und Stueckzahlen je Security Domain. Unbewertete Bereiche erhalten keine Prozentzahl; GPO-Ergebniszeilen und Policy-Vergleiche bleiben getrennte Zaehleinheiten. Keine LLM-, Internet-, Zeitfortschreibungs- oder Business-Risikoableitung. Der separate Dummy-HTML-Entwurf und sein Download-ZIP enthalten dieselbe zweite Grafik; die Produktionszahlen sind dynamisch, der Dummy-Entwurf bleibt ein gespeicherter Beispielstand. `docs/REPORT_STRATEGY.md` haelt die explizite Abloesung des frueheren Report-A-Inhaltsvertrags fest. Entscheidung: Patchversion 2.9.1 fuer die rueckwaertskompatible Export-Darstellung; keine Projektions-, API-, Persistenz-, Import- oder Datenbankaenderung. Verifiziert: 13 fokussierte Renderer-, Client-Info-Klickpfad- und UI/API-Versionsvertragstests, beide Produktionsbuilds, Design- und Local-only-Pruefung sowie `git diff --check`. Risiko/Abnahmegrenze: Die Security-Domain-Zuordnung bleibt die bestehende heuristische Zuordnung; die Grafik ist keine Risiko- oder Schutzgradbewertung. Eine visuelle Browserabnahme des produktiv heruntergeladenen HTML steht aus; der Phasenstatus wird nicht als neu abgeschlossen markiert. Nachpruefung 2026-09-30 vor dem Release-Commit: Vollsuite 1645 bestanden, 14 uebersprungen, 7 Fehler in 6 Dateien; dieselben 7 Fehler treten auf dem vorherigen Commit ohne 2.9.1 identisch auf und sind damit bestehend, nicht durch 2.9.1 verursacht. Client- und Serverproduktionsbuild bestanden.
|
||||
|
|
@ -1668,6 +1678,7 @@ Die heutige Tauri-Anwendung bleibt zunächst eine lokale Laufzeitumgebung. Das
|
|||
| 13C | Scanner Context und Environment Inventory | in Arbeit |
|
||||
| 13D | Referenzerkennung und katalogbasierte Einordnung (ATT&CK/D3FEND erster Umfang) | in Arbeit |
|
||||
| 13E | Kontext-Engine und abschaltbarer Kontext-Layer (GPO-Pilot, Ausgabe auch ueber Phase-7-Reports); Engine und API umgesetzt, UI-Layer offen | in Arbeit |
|
||||
| 13F | Regulatorische Kontrollsuche und RAG-Grundlage mit Qdrant | in Arbeit |
|
||||
| 14 | Mehrbenutzer- und Firmenbetrieb | offen |
|
||||
|
||||
## 7. Phasen im Detail
|
||||
|
|
@ -3119,6 +3130,46 @@ Abnahme erster Umfang (Pilot-Cluster `Credential Relay Exposure` mit etwa 8 bis
|
|||
|
||||
Nutzerentscheidung 2026-09-30, hebt den Ausschluss der Scannerquellen auf: Der Kontext gilt fuer die Findings aller Scans. Umsetzung in zwei Schritten: (1) Abschnitt `Context` bei jedem Finding, (2) Scanner-Beobachtungen ueber exakt katalogisierte Rule-IDs; danach weitere Cluster. Weiter offen: weitere Cluster, Verlaufsbetrachtung ueber den bestehenden Timeslot-Vergleich (keine zweite Drift-Logik), Darstellung im Management Report.
|
||||
|
||||
### Phase 13F - Regulatorische Kontrollsuche und RAG-Grundlage
|
||||
|
||||
Status: in Arbeit
|
||||
|
||||
Ziele:
|
||||
|
||||
- Regulatorische Kontrolltexte versioniert und reproduzierbar fuer mehrsprachige semantische Suche aufbereiten.
|
||||
- Qdrant ausschliesslich als rebuildbaren Retrievalindex verwenden; PostgreSQL bleibt fachliche Quelle fuer Scans, Findings, Ergebnisse, Reviews und spaetere Mappingentscheidungen.
|
||||
- Retrieval, Embedding, Chunking, spaetere Hybridfusion, Reranking und Mapping als getrennte Backendgrenzen halten.
|
||||
|
||||
Umgesetzter Grundstand vom 2026-09-15:
|
||||
|
||||
- Lokaler und Deployment-Compose-Stack enthalten das gepinnte Qdrant-Image 1.19.1, einen API-Key, deaktivierte ausgehende Nutzungstelemetrie und ein getrenntes persistentes Volume. Lokal ist nur die REST-Schnittstelle auf Loopback veroeffentlicht; im Deployment bleibt Qdrant intern.
|
||||
- `RegulatoryControlDocument` und `RegulatoryChunk` definieren einen versionierten, framework-/versions-/control-/chunk-/sprachgebundenen Vertrag. Chunk- und Inhaltsidentitaeten sind deterministisch; Originalkorpus und Korpusversion bleiben die Wiederaufbauquelle.
|
||||
- Die Collection `assessor_regulatory_chunks_v1` verwendet benannte Dense-/Sparse-Vektoren, Cosine fuer Dense und Payloadindizes fuer alle Filterfelder. Sparse ist nur die vorbereitete Schema- und Schnittstellenposition, noch kein aktiver Retriever.
|
||||
- Ein abgesicherter OpenAI-kompatibler Embeddingadapter verlangt fuer Remoteziele HTTPS, erlaubt HTTP nur auf Loopback fuer den lokalen llama.cpp-Testbetrieb und bindet ein ausdruecklich freigegebenes mehrsprachiges Modell an dessen exakte Dimension. Redirects, unbeschraenkte Antworten und Dimensionsdrift werden abgewiesen. Ein opt-in `dimensions`-Request unterstuetzt spaetere OpenAI-/LiteLLM-Vergleiche, bleibt fuer llama.cpp aus.
|
||||
- CLI-Ingestion validiert und segmentiert einen begrenzten JSON-Korpus, bettet Chunks in Batches ein und ersetzt dokumentgebundene Qdrant-Punkte. Die Queryseite verwendet dasselbe Modell und kann Korpus, Framework, Version, Control und Sprache exakt filtern.
|
||||
- `RegulatoryRetriever`, `RegulatoryCandidateRetriever` und `RegulatoryReranker` trennen Verbraucher, Candidate Retrieval und optionale Nachsortierung. Spaetere Hybridfusion kann die Candidate-Stufe ersetzen, ohne Mappingcode an Qdrant zu binden.
|
||||
- Der Finding-Query-Builder akzeptiert die bestehende `UnifiedResultProjection`, gibt aber ausschliesslich katalogeigene Maester-/PingCastle-Regel-IDs, fest definierte oeffentliche Technikthemen und bekannte oeffentliche Toolnamen weiter. Rohtexte dienen nur lokaler Themenklassifikation; Titel, Beschreibung, Empfehlung, Asset, Pfade, Mandanten-/Projekt- und Reviewdaten werden nicht kopiert. Ohne oeffentliches Signal bleibt der Aufruf fail-closed und erreicht den Retriever nicht.
|
||||
- Ein begrenzter versionierter Evaluationsvertrag und die CLI `rag:evaluate` messen HitRate@K, Recall@K und MRR auf Kontrollebene. Query- und erwartete Kontrollsprache sind getrennt deklariert; die CLI gibt nur Aggregatmetriken aus. Synthetische Unit-Rankings pruefen den Vertrag, gelten aber nicht als Qualitaetsnachweis des realen Modells.
|
||||
- Ein eingecheckter synthetischer EN/DE-Smoke-Korpus und drei sprachuebergreifende Labelcases sichern den ausfuehrbaren Vertrag. Der opt-in WSL-Qdrant-Integrationstest prueft mit deterministischen synthetischen Vektoren den realen 1.19.1-Collection-/Index-/Replace-/Filter-/Retrievalpfad; er bewertet ausdruecklich keine Embeddingqualitaet.
|
||||
- Der produktiv gebaute End-to-End-Pfad ist mit lokalem llama.cpp/BGE-M3 FP16 live gelaufen: sechs Dokumente/sechs Chunks; drei Cross-Language-Smoke-Cases bei K=3 jeweils Rang 1 und damit HitRate, Recall und MRR 1,0. Der Mini-Korpus ist nur ein Wiring-Smoke-Test, kein belastbarer Modellvergleich.
|
||||
- Der Grundstand ist in `docs/REGULATORY_RAG.md` dokumentiert und mit synthetischen Vertrags-, Client-, Ingestion-, Retrieval- und Deploymenttests abgesichert.
|
||||
|
||||
Offen bis zur Phasenabnahme:
|
||||
|
||||
- autorisierter und lizenzrechtlich geklaerter regulatorischer Quellkorpus samt Update-/Freigabeprozess
|
||||
- organisatorische Datenschutzfreigabe fuer den Embeddingzielpfad und kuratierte Erweiterung der oeffentlichen Signal-/Regelkataloge
|
||||
- repraesentativer freigegebener EN/DE-Evaluationssatz und dokumentierte reale Dense-Baseline gegen den autorisierten Korpus
|
||||
- aktive Sparse-/BM25-Erzeugung, Hybridfusion, Kontrollgruppierung und evaluierter Reranker
|
||||
- constrained Mappingmodell, Confidence-Kalibrierung, Human Review und append-only PostgreSQL-Provenienz
|
||||
- API/UI, Mandanten-/Rollenabgrenzung, Betriebsmonitoring sowie praktischer Rebuild- und Lasttest
|
||||
|
||||
Abnahmekriterien:
|
||||
|
||||
- Ein freigegebener synthetischer Referenzkorpus kann vollstaendig neu aufgebaut und sprachuebergreifend mit dokumentierten Qualitaetsmetriken abgefragt werden.
|
||||
- Qdrant-Ausfall oder Rebuild veraendert keine fachlichen PostgreSQL-Daten.
|
||||
- Jede gespeicherte Zuordnung referenziert Korpus-/Control-/Chunk-, Embedding-, Retrieval-, Reranker-, Prompt-/Schema- und Entscheidungsprovenienz.
|
||||
- Unzureichende oder widerspruechliche Evidenz bleibt reviewpflichtig; Retrievalscore allein wird nie als fachliche Confidence ausgegeben.
|
||||
|
||||
## 8. Aktuell empfohlene Umsetzungsetappe
|
||||
|
||||
2026-09-26 - Phase 13D / Phase 13C Darstellung (2.8.3, umgesetzt und verifiziert): Catalog references startet bei aktuellem Client eingeklappt; gruen vor Ueberschrift und aktuellem/erfolgreichem Status, orange und automatisch offen bei Handlungsbedarf oder Fehler. Pruefung, Verarbeitung und leere Clients bleiben neutral. Katalogversionen bleiben eingeklappt sichtbar, Erfolg nach Update bleibt offen, manuelles Umschalten funktioniert per Maus und Tastatur. Timeslots-Zeile mit gleichem Abstand oben/unten und gemeinsam zentrierten Controls. Purple-Knight-Timeslots benennen Active Directory, Entra ID oder kombinierten Scope aus bestehenden kanonischen Namen bzw. exakt lauf-/dateigebundenen gespeicherten Metadaten/Umgebungsbeobachtungen. Ohne Scopebeleg bleibt der Originalname; keine Vermutung aus Dateinamen oder fremden Scans, keine Persistenz-/Importer-/Scoringaenderung und keine neue Reimportpflicht. 36 gezielte Interaktions-/Layout-/Versionsvertragstests, beide Produktionsbuilds, Local-only-/Designcheck und synthetische Dark-/Light-Sicht- und Tastaturpruefung bestanden. App und Backend gesund auf 2.8.3 mit kanonischer PostgreSQL. Patchversion fuer zusammengehoerige UI-Korrekturen; Phase 13D bleibt in Arbeit. Die bekannte Gesamtsuite-Baseline mit sechs fachfremden Fehlern wurde nicht erneut ausgefuehrt.
|
||||
|
|
@ -3137,7 +3188,8 @@ Die naechste Umsetzung fuehrt Phase 13B auf dem konsolidierten Import-, Policy-A
|
|||
|
||||
1. Phase 13A ist abgeschlossen: Erkennung, technische Dokumenttypen, Source Extractors, Mappingprofile, Finding-Identität sowie allgemeine Artefakt-, Processing- und Jobgrenzen sind konsolidiert und die Scannerpersistenz verwendet den Candidate-Pfad.
|
||||
2. Phase 13B ist in Arbeit: Der sichere Ingestion-Unterbau, technische Policy-Identitaeten, versionierte ADMX-/Baselineanalysen, deterministisches Reprocessing, Environment Profiles, konservative Snapshotaggregation, typisierte Evaluation, fail-closed Finding Factory, Unified-Result-Read-Projektion, projektgebundene Review-/Orchestrierungsoberflaechen und die aktuelle snapshotbezogene Ergebnisdeduplizierung sind umgesetzt. Als naechste vertikale Etappe folgen fachlich kuratierte Policy-Regeln und vollstaendige versionierte Windows-Inbox-ADMX-/ADML-Kataloge; danach weitere Preference-CSEs und Exportdetails. Ohne RSoP/GP Results bleibt die effektive Anwendung weiterhin ausdruecklich nicht verifiziert.
|
||||
3. Erst nach verifizierter Phase 13B den kontrollierten Firmenbetrieb aus Phase 14 mit SSO, Rollen, HTTPS, Monitoring, Backup/Restore und Aufbewahrung fortsetzen.
|
||||
3. Phase 13F besitzt die rebuildbare Qdrant-, Chunking-, Embedding-, Ingestion- und Dense-Retrieval-Grundlage sowie einen fail-closed Public-Signal-Finding-Query-Builder und einen ausfuehrbaren Evaluationsvertrag. Als naechste Etappe folgen der freigegebene Korpus, organisatorische Datenschutzfreigabe, ein unabhaengig gelabelter EN/DE-Evaluationssatz, die reale Dense-Baseline und danach der persistierte Review-/Mappingvertrag. Sparse/Hybrid und Reranking werden erst nach messbarer Dense-Baseline aktiviert.
|
||||
4. Erst nach verifizierter Phase 13B den kontrollierten Firmenbetrieb aus Phase 14 mit SSO, Rollen, HTTPS, Monitoring, Backup/Restore und Aufbewahrung fortsetzen.
|
||||
|
||||
Die technische Reihenfolge ist bis einschliesslich Phase 13A umgesetzt. Phase 13B baut auf dem verifizierten sicheren Ingestion-Unterbau auf und reicht inzwischen produktiv von der begrenzten Parser-/Workerverarbeitung ueber unveraenderliche Analyse-, Profil-, Aggregations- und Evaluationslaeufe bis zur streng evidenzgebundenen Finding Factory und ihrer gemeinsamen Unified-Result-Leseprojektion. Das Backend leitet projektgebundene Aktionsbereitschaft aus aktiven Revisionen und exakt passenden Referenzdaten ab; der Datahub fuehrt Environment-Profile-Anlage, Reprocessing, Aggregation, Evaluation und Finding Factory ueber die autoritativen idempotenten Services aus. Der produktive Regelkatalog bleibt bis fachlicher Kuratierung leer, und die vorhandenen Microsoft-Baselines bleiben ohne vollstaendige Windows-Inbox-ADMX-/ADML-Kataloge partial; dadurch erzeugt dieser reale Bestand weiterhin nur Review und keine Findings.
|
||||
|
||||
|
|
@ -3199,6 +3251,10 @@ Festlegung 2026-09-14 (GPO-Varianten, Nutzerfreigabe): Unbekannte Zielrolle verh
|
|||
|
||||
Festlegung 2026-09-11 (Phase-8-Planergaenzung): Regeldefinition, objekt-/scopebezogene Vergleichsidentitaet und zeitgebundene Beobachtung bleiben getrennt. Abwesenheit ohne belastbare Gegenpruefung ist keine Behebung. Scorevergleich und Behebungsfreigabe erhalten unterschiedliche fachliche Bedingungen; teilweise neue Evidenz ersetzt nicht pauschal alle alten Verpflichtungen. A und B bleiben die ausdrueckliche Messbasis, aeltere Historie dient getrennt der Verpflichtungs-/Lifecycleverfolgung. Die Festlegung praezisiert das Quellenmodell vom 2026-09-06 und ersetzt fuer die kuenftige Implementierung die historische Regel `abwesend = resolved`; keine historischen Daten oder Ergebnisse werden umgeschrieben. Details und Grenzen: [TIMESLOT_COMPARISON_PLAN.md](TIMESLOT_COMPARISON_PLAN.md).
|
||||
|
||||
Festlegung 2026-09-15 (Phase 13F, Finding-Retrievalgrenze): Nur Scanner-Findings mit einem exakt katalogisierten oeffentlichen Maester-/PingCastle-Regel-ID oder einem lokal erkannten fest definierten oeffentlichen Technikthema duerfen eine Embeddingquery erzeugen. Freitext, kundenspezifische Tool-/Regel-IDs, Assets, Pfade, Werte, Mandanten-/Projekt-/Review- und Provenienzdaten werden nie kopiert; ohne oeffentliches Signal bleibt Retrieval fail-closed. Der lokale Source-Result-Bezug bleibt ausserhalb des Embeddingrequests. Diese technische Minimierung ersetzt keine organisatorische Datenschutzfreigabe. Evaluationsmetriken werden auf Kontrollebene ermittelt und Retrievalscores bleiben Kandidatenranking, keine fachliche Confidence.
|
||||
|
||||
Festlegung 2026-09-15 (Phase 13F, ausdruecklicher Nutzerauftrag): Qdrant ist der rebuildbare Suchindex fuer versionierte regulatorische Chunks und niemals fachliche Quelle fuer Scannergebnisse oder Mappingentscheidungen. PostgreSQL bleibt System of Record fuer Scans, Findings, Ergebnisse, Reviews und spaetere append-only Mappingprovenienz. Dense und Sparse sind benannte Vektorraeume; zunaechst wird nur mehrsprachiges Dense Retrieval aktiviert. Candidate Retrieval und optionaler Reranker bleiben getrennte Schnittstellen. Ein Modellwechsel darf alte Vektoren nicht still weiterverwenden; Modellname und Dimension werden explizit gebunden, Abfragen filtern auf das aktive Modell, und ein Korpus-Rebuild bleibt Pflicht. Der Embeddingdienst ist eine eigene externe Vertrauensgrenze; Findinginhalte duerfen erst nach einer getrennten Datenschutzfreigabe als minimierte Query uebertragen werden.
|
||||
|
||||
Festlegung 2026-09-10 (Purple-Knight-Begleitdateien): Ein nativer Score darf aus einer separaten Summary auf die Findings desselben Scanlaufs uebertragen werden, wenn genau ein nativer Traeger und eine eindeutige Ergebnispopulation denselben Werkzeug- und Verarbeitungskontrakt teilen. Datei- und Vorkommensherkunft bleiben gemeinsam erhalten; Mehrdeutigkeit verhindert die Verbindung.
|
||||
|
||||
Festlegung 2026-09-10 (weitere Scanner): Native Gesamtwerte werden bei jeder Profilerweiterung zuerst geprueft und deklarativ mit ihrer Bedeutung erfasst. Die Scoredefinition ist Pflicht, die Berechnung gemeinsam; ihre Version gehoert zur Vergleichskompatibilitaet. Extraktion und Originalherkunft bleiben im generischen Importpfad. AGENTS.md und docs/SCORING_MODEL.md halten die Erweiterungsregel fest.
|
||||
|
|
@ -3617,6 +3673,8 @@ Timeslotvergleich 2026-09-11: Der bisherige Zweilaufadapter kann Quellenidentita
|
|||
|
||||
Accepted Risk 2026-09-10: Deklarierte Namen und Nachweisreferenzen sind keine authentifizierten Signaturen oder geprueften Berechtigungen. Bestehende Projektloeschung und Datenbankadministration begrenzen den Schutz der JSONB-Historie. Es gibt noch keinen unabhaengigen Archivierungs-/Legal-Hold-Vertrag und keinen Fristenbenachrichtigungsdienst. Diese Grenzen sind im Dialog und in `SECURITY.md` dokumentiert; der Ausbau bleibt Teil von Phase 14. Synthetische Repository-/SQL-Transaktionstests ersetzen keine freigegebene PostgreSQL-Integrationsabnahme.
|
||||
|
||||
Regulatory RAG 2026-09-15: Regulatorische Texte koennen urheber-, lizenz- oder versionsgebunden sein; ein technisch lesbarer Korpus ist deshalb noch kein freigegebener Produktkorpus. Lokale llama.cpp-Embeddings bleiben auf Loopback, waehrend minimierte Querytexte bei einem spaeteren Work-LiteLLM-Vergleich die ASSESSOR-Prozessgrenze verlassen. Gegenmassnahmen sind ein autorisierter versionierter Korpus, explizite Modell-/Dimensionsbindung, Backend-only Secrets, Remote-HTTPS, begrenzte Requests und der implementierte fail-closed Public-Signal-Query-Builder; organisatorische Datenschutzfreigabe bleibt fuer externe Ziele erforderlich. Dense Similarity ist nur Kandidatenranking und weder Kontrollzuordnung noch Confidence; vor automatischer Nutzung sind reale sprachuebergreifende Evaluation, Hybrid-/Reranker-Vergleich, Schwellenkalibrierung und Human Review erforderlich. Qdrant-Dokumentersatz ist nicht transaktional mit einem Embeddingdienst; Fehler koennen einen unvollstaendigen abgeleiteten Index hinterlassen, der aus dem unveraenderten Korpus neu aufgebaut werden muss. Das Volume ist Betriebsbeschleunigung, keine fachliche Sicherung. Der autorisiert geloeschte alte Qdrant ist entfernt und 1.19.1 belegt kanonisch Port 6333; das alte Volume bleibt vorsorglich erhalten. Der lokale BGE-M3-End-to-End-Smoke-Test war mit drei klar getrennten Themen perfekt, ist statistisch und fachlich aber nicht repraesentativ. Ein kuratierter lizenzierter Korpus, schwierigere Negativ-/Mehrfachkontrollfaelle, unabhaengige Labels und der LiteLLM-Vergleich bleiben offen.
|
||||
|
||||
Work-LiteLLM 2026-09-14: Der Unternehmensgateway ist eine ausdruecklich freigegebene externe Vertrauensgrenze und kann Requests gemaess eigener Betriebsrichtlinie routen oder protokollieren; ASSESSOR kann diese Aufbewahrung nicht kontrollieren. Die API sendet deshalb nur den vorhandenen Privacy-Safe-Export nach explizitem Nutzerklick, nie Rohdateien, Reviews, Quelltabellen oder Storage Keys. Gatewayausfall, Modellwechsel oder ungueltige JSON-Ausgabe lassen den bisherigen erfolgreichen Run unveraendert und erfordern eine bewusste neue Generierung. Der synthetische Workstation-Connectivitycheck erreichte die konfigurierte URL, erhielt aber HTTP 401; Keyaktivierung, Berechtigung oder Gatewaykonfiguration muessen betrieblich geklaert werden, ohne Secrets oder Providerbody offenzulegen. Migration 024 ist in diesem Lauf mangels Docker weder auf den kanonischen lokalen Datastore angewandt noch per PostgreSQL-Integration verifiziert; vor Anwendung bleiben verifiziertes Backup und erfolgreicher synthetischer Workstation-/Server-Liveaufruf verpflichtend.
|
||||
|
||||
Purple-Knight-Begleitdateien 2026-09-10: Alte Importe koennen den nativen Wert noch nicht als Kontextbeobachtung enthalten. Dann kann die neue Leseprojektion ihn nicht nachtraeglich aus Rohtext erfinden und die betroffenen Summary-/Findings-Dateien muessen gemeinsam neu importiert werden. Mehrere unterschiedliche Ergebnisberichte in demselben Scanlauf werden bewusst nicht automatisch mit einem einzelnen Summarywert verbunden.
|
||||
|
|
@ -4112,6 +4170,16 @@ Verifiziert 2026-09-11 (Phase 7, Client-Info-Risikoregister): Zweigeteilte Accep
|
|||
|
||||
Verifiziert 2026-09-10 (Phase 7/9-11/13, Accepted Risk): Findingheader-Button, vollstaendiger Entscheidungsdialog, Entwurf/Annahme/Widerruf und versionierte Review-JSONB-Ablage ueber geschuetzte API implementiert. Best-Practice-Ausgangspunkt gemaess Nutzerantwort, getrennte Freigabedokumentation und verbindliche Fristen; keine behauptete Signaturpruefung. Reimport erhaelt Originalentscheidungen mit erneuter Pruefung fuer den Ersatzbefund. 1.214 Tests, Local-only-/Designchecks, beide Builds und synthetische Browserpruefung erfolgreich; acht optionale DB-Integrationstests uebersprungen. Frontend/Backend HTTP 200. Keine Migration oder Realdatenmutation; Firmenfreigaben und unabhaengige Aufbewahrung bleiben in Phase 14 offen.
|
||||
|
||||
Verifiziert 2026-09-15 (Phase 13F, llama.cpp/BGE-M3): Lokalen Server Health/Modellmetadaten geprueft, sechs rein synthetische EN/DE-Dokumente ueber den produktiv gebauten Embedding-/Ingestionpfad als sechs 1.024-dimensionale Chunks indexiert und drei Cross-Language-Cases ausgewertet. HitRate@3, Recall@3 und MRR jeweils 1,0; alle Ziele Rang 1. Ergebnis als Wiring-Smoke-Test, nicht als Produktionsqualitaet dokumentiert. Qdrant enthaelt sechs synthetische Punkte; PostgreSQL/Realdaten unberuehrt. 1.285 Standardtests bestanden, acht PostgreSQL- und ein Qdrant-Integrationstest uebersprungen; Local-only-/Designcheck und beide Builds erfolgreich.
|
||||
|
||||
Implementiert 2026-09-15 (Phase 13F, lokales Embedding): Autorisierte Legacy-Collection geloescht, alten leeren Qdrant-1.12.4-Container entfernt und dessen Volume bewahrt; authentifizierten 1.19.1-Dienst auf kanonischen Loopback-Port 6333 verschoben und Auth/Collectionstatus erneut bestaetigt. BGE-M3/1.024 als lokale llama.cpp-Dense-Baseline festgelegt. Embeddingadapter um strikt loopbackbegrenztes HTTP und optionalen `dimensions`-Request fuer spaeteres `text-embedding-3-large` via HTTPS-LiteLLM erweitert; llama.cpp bleibt auf nativer Dimension. 1.285 Standardtests bestanden, acht PostgreSQL- und ein Qdrant-Integrationstest uebersprungen; Local-only-/Designcheck und beide Builds erfolgreich. Kein PostgreSQL- oder Realdatenzugriff, kein Reimport. BGE-M3-GGUF/Serverstart und echte Modellmetriken bleiben offen.
|
||||
|
||||
Verifiziert 2026-09-15 (Phase 13F, WSL-Qdrant): Docker in WSL erkannt, kanonisches PostgreSQL gesund und unberuehrt bestaetigt. Alten nichtleeren Qdrant 1.12.4 auf 6333 bewahrt; neuen authentifizierten Qdrant 1.19.1 im kanonischen Compose-Projekt auf temporaerem Loopback-Port 6334 gestartet. Auth 401/200, gruene Collection, Dense 1.024/Cosine, Sparse-Slot, neun Payloadindizes sowie synthetisches Replace/Filter/Retrieval live verifiziert; Testkorpus danach geloescht, null Punkte verblieben. Synthetischen EN/DE-Smoke-Korpus, drei Cross-Language-Labelcases und opt-in Integrationstest hinzugefuegt. Reale Ingestion vor Versand gestoppt, weil die konfigurierte lokale HTTP-Embedding-URL unzulassig und nicht erreichbar ist. 1.284 Standardtests bestanden, acht PostgreSQL- und ein Qdrant-Integrationstest uebersprungen; Qdrant separat 1/1 bestanden, Local-only-/Designpruefung und beide Builds erfolgreich. Keine PostgreSQL-Mutation, keine Realdateninspektion und kein Reimport.
|
||||
|
||||
Implementiert 2026-09-15 (Phase 13F, Finding-Retrievalgrenze): Bestehende Unified Results ueber einen fail-closed Public-Signal-Query-Builder backendintern an den Regulatory Retriever angebunden; katalogeigene Maester-/PingCastle-Regel-IDs und fest definierte Technikthemen verhindern die Kopie von Rohbericht-, Asset-, Pfad-, Mandanten-, Review- und Provenienzdaten in Embeddingrequests. Versionierten begrenzten EN/DE-Evaluationsvertrag und Aggregat-CLI fuer HitRate@K, Recall@K und MRR hinzugefuegt. 1.283 synthetische Tests bestanden, acht optionale PostgreSQL-Integrationstests uebersprungen; Local-only-/Designpruefung und Client-/Serverbuild erfolgreich. Kein API-/UI-Pfad, keine Datenbankmutation, keine Mappingbehauptung und kein Reimport; Datenschutzfreigabe, Korpus und reale Dense-Baseline bleiben offen.
|
||||
|
||||
Implementiert 2026-09-15 (Phase 13F): Qdrant-Compose-Service, versionierter regulatorischer Dokument-/Chunkvertrag, deterministischer Chunker, OpenAI-kompatibler mehrsprachiger Embeddingadapter, Collection-/Payload-Indexverwaltung, dokumentweiser Replace-Ingestion, gefiltertes Dense Retrieval sowie Candidate-/Reranker-Erweiterungsgrenzen und CLI-Ingestion hinzugefuegt. 1.275 synthetische Tests bestanden, acht optionale PostgreSQL-Integrationstests uebersprungen; Local-only-/Designpruefung und Client-/Serverbuild erfolgreich. PostgreSQL-, Scanner- und UI-Vertraege bleiben unveraendert; keine Mappingbehauptung und kein Reimport. Docker fehlt auf dem Host, daher keine Compose-/Qdrant-/Embedding-Liveabnahme. Betriebs- und Sicherheitsgrenzen in `docs/REGULATORY_RAG.md` dokumentiert.
|
||||
|
||||
Verifiziert 2026-09-10 (Phase 7/13C, Purple-Knight-Scorezuordnung): Separate native Summary und eindeutige Findings desselben Scanlaufs werden sicher verbunden. Regression 65 native/96 Passed/19 Deviations ergibt 65 Native score in der Dashboardprojektion. 1.199 Tests, Sicherheits-/Designchecks und beide Produktionsbuilds erfolgreich. Keine Datenmutation oder Realdateninspektion.
|
||||
|
||||
Verifiziert 2026-09-10 (Phase 6/13C/7, weitere Scanner): Native-first-Regel in verpflichtende Profildefinitionen und gemeinsamen Katalogpfad ueberfuehrt. Zehn neue Regressionen; 1.198 Tests, Sicherheits-/Designchecks und beide Produktionsbuilds erfolgreich. Keine Migration, Reimportmutation oder Backendneustart.
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ umgehen.
|
|||
## Lokale Datenbank starten
|
||||
|
||||
1. `.env.postgres.example` nach `.env.postgres` kopieren.
|
||||
2. Ein langes, nur lokal verwendetes Passwort eintragen.
|
||||
2. Lange, nur lokal verwendete PostgreSQL- und Qdrant-Schluessel eintragen.
|
||||
3. `npm run db:up` ausführen.
|
||||
4.
|
||||
pm run start:local oder
|
||||
|
|
@ -40,6 +40,11 @@ auf dem Entwicklungsrechner vorgesehen. Für entfernte Datenbankziele ist TLS
|
|||
standardmäßig erforderlich. Die spätere Firmenkonfiguration benötigt eine
|
||||
vertrauenswürdige interne Zertifikatskette.
|
||||
|
||||
`npm run db:up` startet im kanonischen Compose-Projekt auch Qdrant fuer die
|
||||
optionale regulatorische Suche. Das aendert die PostgreSQL-Zustaendigkeit nicht:
|
||||
Qdrant ist ein aus dem versionierten Korpus wiederaufbaubarer Index. Konfiguration
|
||||
und Ingestion stehen in `docs/REGULATORY_RAG.md`.
|
||||
|
||||
## Migrationen
|
||||
|
||||
Migrationen liegen versioniert unter `server/postgres/migrations`. Beim Start
|
||||
|
|
|
|||
|
|
@ -0,0 +1,189 @@
|
|||
# Regulatory-control RAG with Qdrant
|
||||
|
||||
Status: Phase 13F foundation. The current slice indexes and retrieves regulatory control text and can create privacy-minimized retrieval candidates for scanner findings. It does not map findings, call an LLM for classification, expose a browser API, or persist mapping decisions.
|
||||
|
||||
## Boundaries
|
||||
|
||||
- PostgreSQL remains the only system of record for ASSESSOR projects, assessments, scans, findings, results, reviews, and future accepted mappings.
|
||||
- Qdrant is a derived, rebuildable search index. Losing or deleting its volume must not lose an ASSESSOR scan, finding, result, or review.
|
||||
- The versioned regulatory corpus JSON is the rebuild source for Qdrant. It must be retained and governed separately from the index.
|
||||
- React does not connect to Qdrant or the embedding provider. Both are backend-only boundaries.
|
||||
- Regulatory retrieval produces candidates, not compliance decisions. A later constrained mapping stage must retain the retrieved chunk IDs, scores, corpus versions, model version, prompt/schema version, and final decision in PostgreSQL.
|
||||
|
||||
```text
|
||||
versioned regulatory corpus JSON
|
||||
| validate and chunk
|
||||
v
|
||||
approved multilingual embedding service
|
||||
| dense vectors
|
||||
v
|
||||
Qdrant assessor_regulatory_chunks_v1
|
||||
^
|
||||
scanner finding -> public-signal query builder -> same embedding model -> dense candidates
|
||||
-> optional future hybrid fusion -> optional reranker
|
||||
-> later constrained mapper -> PostgreSQL audit record
|
||||
```
|
||||
|
||||
## Collection contract
|
||||
|
||||
The collection is `assessor_regulatory_chunks_v1`. It has two named vector slots:
|
||||
|
||||
- `dense`: the configured embedding dimension with cosine distance.
|
||||
- `sparse`: reserved now so a later sparse/BM25-style encoder can be added without changing the retrieval abstractions.
|
||||
|
||||
Qdrant payloads use schema version 1 and contain:
|
||||
|
||||
| Field | Purpose |
|
||||
|---|---|
|
||||
| `corpus_id`, `corpus_version` | Versioned rebuild source |
|
||||
| `document_id` | Stable control-language document identity |
|
||||
| `framework`, `framework_version` | Regulatory framework scope |
|
||||
| `control_id` | Native control identifier |
|
||||
| `chunk_id`, `chunk_index` | Deterministic chunk identity and order |
|
||||
| `language` | Lower-case BCP 47 language tag |
|
||||
| `title`, `text`, `text_sha256` | Retrieval content and integrity metadata |
|
||||
| `embedding_model` | Prevents querying vectors from a different model space |
|
||||
| `source_uri` | Optional public/source reference |
|
||||
|
||||
All fields used by filters have Qdrant payload indexes. An existing collection with a different dense dimension, distance, or missing sparse slot fails closed and must be rebuilt from the corpus.
|
||||
|
||||
## Corpus format and ingestion
|
||||
|
||||
The CLI accepts a UTF-8 JSON file of no more than 64 MiB:
|
||||
|
||||
```json
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"documents": [
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "synthetic-framework",
|
||||
"corpusVersion": "2026-09-15",
|
||||
"documentId": "synthetic-control-1-de",
|
||||
"framework": "Synthetic Framework",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "CTRL-1",
|
||||
"language": "de",
|
||||
"title": "Synthetic control title",
|
||||
"text": "Synthetic control text used only to demonstrate the contract.",
|
||||
"sourceUri": "https://example.invalid/framework/control-1"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The checked-in [`tests/fixtures/regulatory-corpus.synthetic.json`](../tests/fixtures/regulatory-corpus.synthetic.json) is an unlicensed synthetic EN/DE smoke corpus. It exists to verify rebuilding and retrieval mechanics only; it is not a regulatory source and must never be presented as compliance guidance.
|
||||
|
||||
Configure `.env.postgres` from the commented variables in `.env.postgres.example`, then run:
|
||||
|
||||
```text
|
||||
npm run db:up
|
||||
npm run rag:ingest -- path/to/regulatory-corpus.json
|
||||
```
|
||||
|
||||
Ingestion validates every document, splits text deterministically on paragraph/sentence/word boundaries, embeds all chunks in bounded batches, and only then replaces the indexed chunks for each `(corpusId, documentId)`. Replacement is not a cross-system transaction: if Qdrant fails after deletion, rerun ingestion from the unchanged corpus.
|
||||
|
||||
## Configuration and trust
|
||||
|
||||
Qdrant 1.19.1 runs in Docker with a persistent named volume, an API key, and outbound usage telemetry disabled. Local REST access is loopback-only; deployment access stays on the Compose network. Remote plain HTTP Qdrant URLs are rejected.
|
||||
|
||||
The embedding adapter uses an OpenAI-compatible `/v1/embeddings` endpoint. Remote endpoints must use HTTPS. Plain HTTP is accepted only for `localhost`, `127.0.0.1`, or `::1`, enabling a same-machine llama.cpp development server without weakening remote transport. Configuration must explicitly provide the model name and its exact native/requested dimensions. Redirects are rejected, response size and time are bounded, and API keys remain in backend environment variables.
|
||||
|
||||
### Recommended local development model
|
||||
|
||||
Use BGE-M3 for the first local dense baseline. Its published model contract is multilingual, 1,024-dimensional, and supports up to 8,192 tokens. Those dimensions match the current Qdrant collection. llama.cpp supports dedicated embedding mode, model/default or explicit pooling, bearer API keys, and the OpenAI-compatible `/v1/embeddings` endpoint.
|
||||
|
||||
Example command, using a locally obtained compatible BGE-M3 GGUF file:
|
||||
|
||||
```text
|
||||
llama-server.exe -m path/to/bge-m3.gguf --embedding --pooling cls --host 127.0.0.1 --port 8080 --api-key <local-secret> -c 8192
|
||||
```
|
||||
|
||||
Use matching ASSESSOR settings:
|
||||
|
||||
```text
|
||||
ASSESSOR_RAG_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
|
||||
ASSESSOR_RAG_EMBEDDING_MODEL=bge-m3
|
||||
ASSESSOR_RAG_EMBEDDING_API_KEY=<same-local-secret>
|
||||
ASSESSOR_RAG_EMBEDDING_DIMENSIONS=1024
|
||||
ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS=false
|
||||
```
|
||||
|
||||
For a later LiteLLM comparison with `text-embedding-3-large`, use the approved HTTPS gateway and set `ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS=true` if requesting 1,024-dimensional output. OpenAI supports the `dimensions` parameter for `text-embedding-3` models; llama.cpp currently returns the model's native dimension and should leave this flag false. Changing model identity or dimensions requires rebuilding the derived Qdrant corpus before comparing results.
|
||||
|
||||
Embedding is an external processing boundary. Regulatory text is sent during ingestion. The finding retrieval service can now send only the minimized public-signal query described below; unrestricted report text, paths, tenant identities, reviews, raw evidence, and local provenance IDs remain outside the embedding request. No browser endpoint or automatic mapping workflow activates this path.
|
||||
|
||||
## Finding query privacy boundary
|
||||
|
||||
`buildRegulatoryFindingQuery` accepts the existing `UnifiedResultProjection` but emits only fixed public concepts:
|
||||
|
||||
- exact Maester or PingCastle rule IDs that already exist in the checked-in public catalog;
|
||||
- fixed public technical topic labels recognized locally from title, description, or recommendation; and
|
||||
- a fixed public scanner name for known profiles.
|
||||
|
||||
The query never copies raw title, description, recommendation, asset, source path, project/tenant identity, review owner, review text, original values, or provenance IDs. Unsupported result kinds and scanner findings without a recognized public rule or topic fail closed as `insufficient-public-context` and do not call the retriever. The local `sourceResultId` is returned only for in-process correlation and is not included in the embedding request.
|
||||
|
||||
This is a technical minimization boundary, not organizational approval to process production findings externally. The service remains backend-only and has no API/UI route. Any production activation still requires approval of the embedding destination, catalog coverage, retention terms, and the precise finding classes allowed to cross that boundary.
|
||||
|
||||
### Local Qdrant integration check
|
||||
|
||||
`tests/server/qdrant-regulatory-index.integration.test.ts` is opt-in and uses deterministic synthetic vectors. With `ASSESSOR_QDRANT_INTEGRATION=1` and the normal Qdrant/model identity environment variables, it verifies the real collection, payload indexes, document replacement, filters, and retrieval without calling an embedding provider. The standard suite skips it so tests do not unexpectedly mutate a developer's derived index.
|
||||
|
||||
On 2026-09-15 this check passed against Qdrant 1.19.1 in WSL: authenticated access returned HTTP 200, unauthenticated access returned HTTP 401, the collection was green, Dense used 1,024-dimensional cosine vectors, the Sparse slot existed, all nine filter payload indexes existed, and one synthetic integration point was retrieved and then removed. PostgreSQL was not accessed or changed.
|
||||
|
||||
## Dense retrieval evaluation
|
||||
|
||||
The evaluation CLI runs the configured dense retriever against a bounded, versioned labeled case file and reports control-level HitRate@K, Recall@K, and mean reciprocal rank. Queries can declare a language different from the expected control language so English-to-German and German-to-English cases remain explicit.
|
||||
|
||||
```json
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"cases": [
|
||||
{
|
||||
"id": "synthetic-en-to-de-mfa",
|
||||
"query": "Require multifactor authentication",
|
||||
"queryLanguage": "en",
|
||||
"filters": { "corpusId": "synthetic-framework" },
|
||||
"expectedControls": [
|
||||
{
|
||||
"framework": "Synthetic Framework",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "CTRL-MFA",
|
||||
"language": "de"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Run it after ingestion with:
|
||||
|
||||
```text
|
||||
npm run rag:evaluate -- path/to/evaluation-cases.json 10
|
||||
```
|
||||
|
||||
The CLI prints aggregate metrics only; it does not print queries or retrieved control text. Unit tests verify the metric contract with synthetic rankings, but those values are not evidence of real embedding quality. A meaningful baseline requires an approved representative corpus, independently labeled cases, and the configured multilingual model running against Qdrant.
|
||||
|
||||
The checked-in [`tests/fixtures/regulatory-evaluation.synthetic.json`](../tests/fixtures/regulatory-evaluation.synthetic.json) provides three explicitly cross-language smoke cases for the synthetic corpus.
|
||||
|
||||
Live smoke result on 2026-09-15 with local llama.cpp build 10472, BGE-M3 FP16, and Qdrant 1.19.1: six EN/DE documents produced six 1,024-dimensional chunks. All three cross-language cases returned their expected control at rank 1, giving HitRate@3 `1.0`, Recall@3 `1.0`, and MRR `1.0`. This verifies wiring and basic language crossing only; three controls are far too small and distinct to estimate production retrieval quality.
|
||||
|
||||
## Extension seams
|
||||
|
||||
- `RegulatoryCandidateRetriever` is the replaceable candidate stage. The current implementation is dense Qdrant retrieval; a later hybrid implementation can combine dense and sparse/BM25 candidates here.
|
||||
- `RegulatoryReranker` is optional and receives only the bounded candidate set. Without one, dense ordering is retained.
|
||||
- `RegulatoryRetriever` is the consumer-facing contract. Mapping code should depend on it, not on Qdrant.
|
||||
|
||||
Before enabling automatic finding mapping, complete organizational privacy approval, broaden public-signal coverage only through reviewed catalogs, establish the real dense baseline, and add control-level grouping, hybrid evaluation, reranker evaluation, calibrated confidence thresholds, a constrained output schema, human-review states, and append-only PostgreSQL mapping provenance.
|
||||
|
||||
## Reference basis
|
||||
|
||||
- Qdrant collections and named vectors: https://qdrant.tech/documentation/manage-data/collections/
|
||||
- Qdrant payload indexing and filtering: https://qdrant.tech/documentation/manage-data/indexing/
|
||||
- Qdrant hybrid and multi-stage Query API: https://qdrant.tech/documentation/search/hybrid-queries/
|
||||
- Self-hosted Qdrant security: https://qdrant.tech/documentation/tutorials-operations/secure-qdrant/
|
||||
- LiteLLM OpenAI-compatible endpoints: https://docs.litellm.ai/
|
||||
- llama.cpp server embeddings and authentication: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md
|
||||
- BGE-M3 model contract: https://huggingface.co/BAAI/bge-m3
|
||||
- OpenAI embedding dimensions parameter: https://developers.openai.com/api/reference/resources/embeddings/methods/create
|
||||
|
|
@ -15,6 +15,8 @@
|
|||
"db:up": "docker compose --env-file .env.postgres -f docker-compose.postgres.yml up -d",
|
||||
"db:down": "docker compose --env-file .env.postgres -f docker-compose.postgres.yml down",
|
||||
"db:logs": "docker compose --env-file .env.postgres -f docker-compose.postgres.yml logs -f postgres",
|
||||
"rag:ingest": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-ingest.mjs",
|
||||
"rag:evaluate": "npm run build:server && node --env-file-if-exists=.env.postgres dist-server/regulatory-evaluate.mjs",
|
||||
"test:postgres": "vitest run --config vitest.config.ts tests/postgres/postgres-repository.integration.test.ts && vitest run --config vitest.config.ts tests/postgres/gpo-backup-management.integration.test.ts tests/postgres/policy-analysis.integration.test.ts",
|
||||
"test:imports": "node scripts/check-import-contracts.mjs",
|
||||
"test": "npm run test:imports && vitest run --config vitest.config.ts && node tests/local-only-check.mjs && node tests/design-language-check.mjs",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,183 @@
|
|||
export interface TextEmbeddingClient {
|
||||
readonly model: string;
|
||||
readonly dimensions: number;
|
||||
embed(texts: string[]): Promise<number[][]>;
|
||||
}
|
||||
|
||||
export interface EmbeddingEnvironment {
|
||||
ASSESSOR_RAG_EMBEDDING_BASE_URL?: string;
|
||||
ASSESSOR_RAG_EMBEDDING_MODEL?: string;
|
||||
ASSESSOR_RAG_EMBEDDING_API_KEY?: string;
|
||||
ASSESSOR_RAG_EMBEDDING_DIMENSIONS?: string;
|
||||
ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS?: string;
|
||||
ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS?: string;
|
||||
}
|
||||
|
||||
export class EmbeddingClientError extends Error {
|
||||
constructor(readonly code: string, message: string) {
|
||||
super(message);
|
||||
this.name = 'EmbeddingClientError';
|
||||
}
|
||||
}
|
||||
|
||||
interface OpenAiCompatibleEmbeddingClientOptions {
|
||||
endpoint: URL;
|
||||
model: string;
|
||||
apiKey: string;
|
||||
dimensions: number;
|
||||
timeoutMs: number;
|
||||
requestDimensions?: boolean;
|
||||
fetcher?: typeof fetch;
|
||||
}
|
||||
|
||||
export class OpenAiCompatibleEmbeddingClient implements TextEmbeddingClient {
|
||||
readonly model: string;
|
||||
readonly dimensions: number;
|
||||
private readonly endpoint: URL;
|
||||
private readonly apiKey: string;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly requestDimensions: boolean;
|
||||
private readonly fetcher: typeof fetch;
|
||||
|
||||
constructor(options: OpenAiCompatibleEmbeddingClientOptions) {
|
||||
this.endpoint = new URL(options.endpoint.href);
|
||||
this.model = options.model;
|
||||
this.apiKey = options.apiKey;
|
||||
this.dimensions = options.dimensions;
|
||||
this.timeoutMs = options.timeoutMs;
|
||||
this.requestDimensions = options.requestDimensions ?? false;
|
||||
this.fetcher = options.fetcher ?? globalThis.fetch.bind(globalThis);
|
||||
}
|
||||
|
||||
async embed(texts: string[]): Promise<number[][]> {
|
||||
if (!texts.length || texts.some(text => typeof text !== 'string' || !text.trim())) {
|
||||
throw new EmbeddingClientError('invalid_embedding_input', 'At least one non-empty embedding input is required.');
|
||||
}
|
||||
let response: Response;
|
||||
try {
|
||||
response = await this.fetcher(this.endpoint, {
|
||||
method: 'POST',
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${this.apiKey}`
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
input: texts,
|
||||
encoding_format: 'float',
|
||||
...(this.requestDimensions ? { dimensions: this.dimensions } : {})
|
||||
})
|
||||
});
|
||||
} catch {
|
||||
throw new EmbeddingClientError('embedding_unavailable', 'The configured embedding service could not be reached.');
|
||||
}
|
||||
if (!response.ok) throw new EmbeddingClientError('embedding_http_error', `The embedding service returned HTTP ${response.status}.`);
|
||||
const body = await readBoundedJson(response, 32 * 1024 * 1024);
|
||||
if (!isObject(body) || !Array.isArray(body.data) || body.data.length !== texts.length) {
|
||||
throw new EmbeddingClientError('embedding_invalid_response', 'The embedding service returned an invalid result count.');
|
||||
}
|
||||
const ordered = body.data.map((entry, responseIndex) => ({ entry, responseIndex })).sort((left, right) => {
|
||||
const leftIndex = isObject(left.entry) && Number.isInteger(left.entry.index) ? Number(left.entry.index) : left.responseIndex;
|
||||
const rightIndex = isObject(right.entry) && Number.isInteger(right.entry.index) ? Number(right.entry.index) : right.responseIndex;
|
||||
return leftIndex - rightIndex;
|
||||
});
|
||||
return ordered.map(({ entry }) => {
|
||||
const vector = isObject(entry) && Array.isArray(entry.embedding) ? entry.embedding : undefined;
|
||||
if (!vector || vector.length !== this.dimensions || vector.some(value => typeof value !== 'number' || !Number.isFinite(value))) {
|
||||
throw new EmbeddingClientError('embedding_invalid_dimensions', `The embedding service must return ${this.dimensions} finite dimensions per input.`);
|
||||
}
|
||||
return vector as number[];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function createEmbeddingClientFromEnvironment(
|
||||
environment: EmbeddingEnvironment = {
|
||||
ASSESSOR_RAG_EMBEDDING_BASE_URL: process.env.ASSESSOR_RAG_EMBEDDING_BASE_URL,
|
||||
ASSESSOR_RAG_EMBEDDING_MODEL: process.env.ASSESSOR_RAG_EMBEDDING_MODEL,
|
||||
ASSESSOR_RAG_EMBEDDING_API_KEY: process.env.ASSESSOR_RAG_EMBEDDING_API_KEY,
|
||||
ASSESSOR_RAG_EMBEDDING_DIMENSIONS: process.env.ASSESSOR_RAG_EMBEDDING_DIMENSIONS,
|
||||
ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS: process.env.ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS,
|
||||
ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS: process.env.ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS
|
||||
},
|
||||
fetcher?: typeof fetch
|
||||
): OpenAiCompatibleEmbeddingClient {
|
||||
const baseUrl = environment.ASSESSOR_RAG_EMBEDDING_BASE_URL?.trim();
|
||||
const model = environment.ASSESSOR_RAG_EMBEDDING_MODEL?.trim();
|
||||
const apiKey = environment.ASSESSOR_RAG_EMBEDDING_API_KEY?.trim();
|
||||
if (!baseUrl || !model || !apiKey) {
|
||||
throw new Error('Regulatory RAG requires ASSESSOR_RAG_EMBEDDING_BASE_URL, ASSESSOR_RAG_EMBEDDING_MODEL, and ASSESSOR_RAG_EMBEDDING_API_KEY.');
|
||||
}
|
||||
return new OpenAiCompatibleEmbeddingClient({
|
||||
endpoint: embeddingEndpoint(baseUrl),
|
||||
model,
|
||||
apiKey,
|
||||
dimensions: boundedInteger(environment.ASSESSOR_RAG_EMBEDDING_DIMENSIONS, undefined, 64, 16_384, 'ASSESSOR_RAG_EMBEDDING_DIMENSIONS'),
|
||||
requestDimensions: booleanFlag(environment.ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS, 'ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS'),
|
||||
timeoutMs: boundedInteger(environment.ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS, 60_000, 1_000, 300_000, 'ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS'),
|
||||
fetcher
|
||||
});
|
||||
}
|
||||
|
||||
export function embeddingEndpoint(baseUrl: string): URL {
|
||||
let url: URL;
|
||||
try { url = new URL(baseUrl); } catch { throw new Error('ASSESSOR_RAG_EMBEDDING_BASE_URL must be an absolute HTTP(S) URL.'); }
|
||||
const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname.toLowerCase());
|
||||
if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) {
|
||||
throw new Error('ASSESSOR_RAG_EMBEDDING_BASE_URL must use HTTPS unless it targets loopback.');
|
||||
}
|
||||
if (url.username || url.password || url.search || url.hash) {
|
||||
throw new Error('ASSESSOR_RAG_EMBEDDING_BASE_URL must not contain credentials, query parameters, or a fragment.');
|
||||
}
|
||||
const path = url.pathname.replace(/\/+$/, '');
|
||||
url.pathname = path.endsWith('/embeddings') ? path : path ? `${path}/embeddings` : '/v1/embeddings';
|
||||
return url;
|
||||
}
|
||||
|
||||
function booleanFlag(value: string | undefined, name: string): boolean {
|
||||
if (!value?.trim()) return false;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new Error(`${name} must be true or false.`);
|
||||
}
|
||||
|
||||
function boundedInteger(value: string | undefined, fallback: number | undefined, minimum: number, maximum: number, name: string): number {
|
||||
if (!value?.trim()) {
|
||||
if (fallback !== undefined) return fallback;
|
||||
throw new Error(`${name} is required.`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) throw new Error(`${name} must be an integer between ${minimum} and ${maximum}.`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function readBoundedJson(response: Response, maximumBytes: number): Promise<unknown> {
|
||||
const declared = Number(response.headers.get('content-length') ?? 0);
|
||||
if (declared > maximumBytes) throw new EmbeddingClientError('embedding_response_too_large', 'The embedding response exceeded the size limit.');
|
||||
if (!response.body) throw new EmbeddingClientError('embedding_invalid_response', 'The embedding service returned an empty response.');
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.byteLength;
|
||||
if (size > maximumBytes) {
|
||||
await reader.cancel();
|
||||
throw new EmbeddingClientError('embedding_response_too_large', 'The embedding response exceeded the size limit.');
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const merged = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) { merged.set(chunk, offset); offset += chunk.byteLength; }
|
||||
const text = new TextDecoder().decode(merged);
|
||||
try { return JSON.parse(text); } catch { throw new EmbeddingClientError('embedding_invalid_response', 'The embedding service returned invalid JSON.'); }
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is Record<string, any> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
|
@ -0,0 +1,323 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import type {
|
||||
RegulatoryChunk,
|
||||
RegulatoryRetrievalCandidate,
|
||||
RegulatoryRetrievalFilters
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
|
||||
export const REGULATORY_QDRANT_COLLECTION = 'assessor_regulatory_chunks_v1';
|
||||
export const REGULATORY_DENSE_VECTOR = 'dense';
|
||||
export const REGULATORY_SPARSE_VECTOR = 'sparse';
|
||||
|
||||
export interface QdrantEnvironment {
|
||||
ASSESSOR_QDRANT_URL?: string;
|
||||
ASSESSOR_QDRANT_API_KEY?: string;
|
||||
ASSESSOR_QDRANT_TIMEOUT_MS?: string;
|
||||
}
|
||||
|
||||
export interface IndexedRegulatoryChunk {
|
||||
chunk: RegulatoryChunk;
|
||||
embedding: number[];
|
||||
}
|
||||
|
||||
export interface RegulatoryChunkIndex {
|
||||
ensureCollection(): Promise<void>;
|
||||
replaceDocuments(chunks: IndexedRegulatoryChunk[]): Promise<void>;
|
||||
searchDense(vector: number[], limit: number, filters?: RegulatoryRetrievalFilters): Promise<RegulatoryRetrievalCandidate[]>;
|
||||
}
|
||||
|
||||
export class QdrantClientError extends Error {
|
||||
constructor(readonly code: string, message: string) {
|
||||
super(message);
|
||||
this.name = 'QdrantClientError';
|
||||
}
|
||||
}
|
||||
|
||||
interface QdrantRegulatoryIndexOptions {
|
||||
baseUrl: URL;
|
||||
apiKey: string;
|
||||
dimensions: number;
|
||||
embeddingModel: string;
|
||||
timeoutMs: number;
|
||||
fetcher?: typeof fetch;
|
||||
}
|
||||
|
||||
const payloadIndexes = [
|
||||
['schema_version', 'integer'],
|
||||
['corpus_id', 'keyword'],
|
||||
['corpus_version', 'keyword'],
|
||||
['document_id', 'keyword'],
|
||||
['framework', 'keyword'],
|
||||
['framework_version', 'keyword'],
|
||||
['control_id', 'keyword'],
|
||||
['language', 'keyword'],
|
||||
['embedding_model', 'keyword']
|
||||
] as const;
|
||||
|
||||
export class QdrantRegulatoryIndex implements RegulatoryChunkIndex {
|
||||
private readonly fetcher: typeof fetch;
|
||||
|
||||
constructor(private readonly options: QdrantRegulatoryIndexOptions) {
|
||||
this.fetcher = options.fetcher ?? globalThis.fetch.bind(globalThis);
|
||||
}
|
||||
|
||||
async ensureCollection(): Promise<void> {
|
||||
const existing = await this.request('GET', `/collections/${REGULATORY_QDRANT_COLLECTION}`, undefined, [200, 404]);
|
||||
if (existing.status === 404) {
|
||||
await this.request('PUT', `/collections/${REGULATORY_QDRANT_COLLECTION}`, {
|
||||
vectors: { [REGULATORY_DENSE_VECTOR]: { size: this.options.dimensions, distance: 'Cosine' } },
|
||||
sparse_vectors: { [REGULATORY_SPARSE_VECTOR]: {} },
|
||||
on_disk_payload: true
|
||||
});
|
||||
} else {
|
||||
validateCollection(existing.body, this.options.dimensions);
|
||||
}
|
||||
const current = existing.status === 404
|
||||
? await this.request('GET', `/collections/${REGULATORY_QDRANT_COLLECTION}`)
|
||||
: existing;
|
||||
const indexedFields = payloadSchemaFields(current.body);
|
||||
for (const [fieldName, fieldSchema] of payloadIndexes) {
|
||||
if (indexedFields.has(fieldName)) continue;
|
||||
await this.request('PUT', `/collections/${REGULATORY_QDRANT_COLLECTION}/index?wait=true`, {
|
||||
field_name: fieldName,
|
||||
field_schema: fieldSchema
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async replaceDocuments(chunks: IndexedRegulatoryChunk[]): Promise<void> {
|
||||
if (!chunks.length) return;
|
||||
for (const entry of chunks) validateEmbedding(entry.embedding, this.options.dimensions);
|
||||
const documents = new Map<string, { corpusId: string; documentId: string }>();
|
||||
for (const { chunk } of chunks) documents.set(`${chunk.corpusId}\0${chunk.documentId}`, { corpusId: chunk.corpusId, documentId: chunk.documentId });
|
||||
for (const document of documents.values()) {
|
||||
await this.request('POST', `/collections/${REGULATORY_QDRANT_COLLECTION}/points/delete?wait=true`, {
|
||||
filter: { must: [keywordCondition('corpus_id', document.corpusId), keywordCondition('document_id', document.documentId)] }
|
||||
});
|
||||
}
|
||||
for (let offset = 0; offset < chunks.length; offset += 64) {
|
||||
const batch = chunks.slice(offset, offset + 64);
|
||||
await this.request('PUT', `/collections/${REGULATORY_QDRANT_COLLECTION}/points?wait=true`, {
|
||||
points: batch.map(({ chunk, embedding }) => ({
|
||||
id: pointId(chunk.chunkId),
|
||||
vector: { [REGULATORY_DENSE_VECTOR]: embedding },
|
||||
payload: chunkPayload(chunk, this.options.embeddingModel)
|
||||
}))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async searchDense(
|
||||
vector: number[],
|
||||
limit: number,
|
||||
filters: RegulatoryRetrievalFilters = {}
|
||||
): Promise<RegulatoryRetrievalCandidate[]> {
|
||||
validateEmbedding(vector, this.options.dimensions);
|
||||
if (!Number.isInteger(limit) || limit < 1 || limit > 100) throw new Error('Regulatory retrieval limit must be between 1 and 100.');
|
||||
const must = [keywordCondition('embedding_model', this.options.embeddingModel), ...filterConditions(filters)];
|
||||
const response = await this.request('POST', `/collections/${REGULATORY_QDRANT_COLLECTION}/points/query`, {
|
||||
query: vector,
|
||||
using: REGULATORY_DENSE_VECTOR,
|
||||
filter: { must },
|
||||
limit,
|
||||
with_payload: true,
|
||||
with_vector: false
|
||||
});
|
||||
const result = object(response.body)?.result;
|
||||
const points = Array.isArray(object(result)?.points) ? object(result)!.points as unknown[] : Array.isArray(result) ? result : undefined;
|
||||
if (!points) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant returned an invalid query response.');
|
||||
return points.map(point => parseCandidate(point));
|
||||
}
|
||||
|
||||
private async request(method: string, path: string, body?: unknown, acceptedStatuses: number[] = [200]): Promise<{ status: number; body: unknown }> {
|
||||
const url = new URL(path, withTrailingSlash(this.options.baseUrl));
|
||||
let response: Response;
|
||||
try {
|
||||
response = await this.fetcher(url, {
|
||||
method,
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(this.options.timeoutMs),
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
...(body === undefined ? {} : { 'Content-Type': 'application/json' }),
|
||||
'api-key': this.options.apiKey
|
||||
},
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) })
|
||||
});
|
||||
} catch {
|
||||
throw new QdrantClientError('qdrant_unavailable', 'Qdrant could not be reached.');
|
||||
}
|
||||
if (!acceptedStatuses.includes(response.status)) throw new QdrantClientError('qdrant_http_error', `Qdrant returned HTTP ${response.status}.`);
|
||||
if (response.status === 404) return { status: 404, body: undefined };
|
||||
const text = await readBoundedText(response, 16 * 1024 * 1024);
|
||||
try { return { status: response.status, body: text ? JSON.parse(text) : undefined }; }
|
||||
catch { throw new QdrantClientError('qdrant_invalid_response', 'Qdrant returned invalid JSON.'); }
|
||||
}
|
||||
}
|
||||
|
||||
export function createQdrantRegulatoryIndexFromEnvironment(
|
||||
dimensions: number,
|
||||
embeddingModel: string,
|
||||
environment: QdrantEnvironment = {
|
||||
ASSESSOR_QDRANT_URL: process.env.ASSESSOR_QDRANT_URL,
|
||||
ASSESSOR_QDRANT_API_KEY: process.env.ASSESSOR_QDRANT_API_KEY,
|
||||
ASSESSOR_QDRANT_TIMEOUT_MS: process.env.ASSESSOR_QDRANT_TIMEOUT_MS
|
||||
},
|
||||
fetcher?: typeof fetch
|
||||
): QdrantRegulatoryIndex {
|
||||
const urlValue = environment.ASSESSOR_QDRANT_URL?.trim();
|
||||
const apiKey = environment.ASSESSOR_QDRANT_API_KEY?.trim();
|
||||
if (!urlValue || !apiKey) throw new Error('Regulatory RAG requires ASSESSOR_QDRANT_URL and ASSESSOR_QDRANT_API_KEY.');
|
||||
let baseUrl: URL;
|
||||
try { baseUrl = new URL(urlValue); } catch { throw new Error('ASSESSOR_QDRANT_URL must be an absolute URL.'); }
|
||||
if (!['http:', 'https:'].includes(baseUrl.protocol) || baseUrl.username || baseUrl.password || baseUrl.search || baseUrl.hash) {
|
||||
throw new Error('ASSESSOR_QDRANT_URL must be an HTTP(S) URL without credentials, query parameters, or a fragment.');
|
||||
}
|
||||
if (baseUrl.pathname !== '/' && baseUrl.pathname !== '') throw new Error('ASSESSOR_QDRANT_URL must not contain a path.');
|
||||
if (baseUrl.protocol !== 'https:' && !['localhost', '127.0.0.1', '[::1]', 'qdrant'].includes(baseUrl.hostname.toLowerCase())) {
|
||||
throw new Error('ASSESSOR_QDRANT_URL must use HTTPS unless it targets loopback or the internal qdrant service.');
|
||||
}
|
||||
return new QdrantRegulatoryIndex({
|
||||
baseUrl,
|
||||
apiKey,
|
||||
dimensions,
|
||||
embeddingModel,
|
||||
timeoutMs: boundedInteger(environment.ASSESSOR_QDRANT_TIMEOUT_MS, 15_000, 1_000, 120_000, 'ASSESSOR_QDRANT_TIMEOUT_MS'),
|
||||
fetcher
|
||||
});
|
||||
}
|
||||
|
||||
function validateCollection(value: unknown, dimensions: number): void {
|
||||
const result = object(object(value)?.result);
|
||||
const params = object(object(object(result?.config)?.params));
|
||||
const vectors = object(params?.vectors);
|
||||
const dense = object(vectors?.[REGULATORY_DENSE_VECTOR]);
|
||||
const sparse = object(params?.sparse_vectors);
|
||||
if (dense?.size !== dimensions || String(dense.distance).toLowerCase() !== 'cosine' || !object(sparse?.[REGULATORY_SPARSE_VECTOR])) {
|
||||
throw new QdrantClientError('qdrant_collection_incompatible', `Qdrant collection ${REGULATORY_QDRANT_COLLECTION} is incompatible and must be rebuilt.`);
|
||||
}
|
||||
}
|
||||
|
||||
function payloadSchemaFields(value: unknown): Set<string> {
|
||||
const payloadSchema = object(object(value)?.result)?.payload_schema;
|
||||
return new Set(Object.keys(object(payloadSchema) ?? {}));
|
||||
}
|
||||
|
||||
function chunkPayload(chunk: RegulatoryChunk, embeddingModel: string): Record<string, unknown> {
|
||||
return {
|
||||
schema_version: chunk.schemaVersion,
|
||||
corpus_id: chunk.corpusId,
|
||||
corpus_version: chunk.corpusVersion,
|
||||
document_id: chunk.documentId,
|
||||
framework: chunk.framework,
|
||||
framework_version: chunk.frameworkVersion,
|
||||
control_id: chunk.controlId,
|
||||
chunk_id: chunk.chunkId,
|
||||
chunk_index: chunk.chunkIndex,
|
||||
language: chunk.language,
|
||||
title: chunk.title,
|
||||
text: chunk.text,
|
||||
text_sha256: chunk.textSha256,
|
||||
embedding_model: embeddingModel,
|
||||
...(chunk.sourceUri ? { source_uri: chunk.sourceUri } : {})
|
||||
};
|
||||
}
|
||||
|
||||
function parseCandidate(value: unknown): RegulatoryRetrievalCandidate {
|
||||
const point = object(value);
|
||||
const payload = object(point?.payload);
|
||||
const score = point?.score;
|
||||
if (!payload || typeof score !== 'number' || !Number.isFinite(score)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant returned an invalid point.');
|
||||
const stringField = (name: string): string => {
|
||||
const field = payload[name];
|
||||
if (typeof field !== 'string' || !field) throw new QdrantClientError('qdrant_invalid_response', `Qdrant point is missing ${name}.`);
|
||||
return field;
|
||||
};
|
||||
if (payload.schema_version !== 1 || !Number.isInteger(payload.chunk_index)) throw new QdrantClientError('qdrant_invalid_response', 'Qdrant point has an unsupported regulatory schema.');
|
||||
return {
|
||||
score,
|
||||
chunk: {
|
||||
schemaVersion: 1,
|
||||
corpusId: stringField('corpus_id'),
|
||||
corpusVersion: stringField('corpus_version'),
|
||||
documentId: stringField('document_id'),
|
||||
framework: stringField('framework'),
|
||||
frameworkVersion: stringField('framework_version'),
|
||||
controlId: stringField('control_id'),
|
||||
chunkId: stringField('chunk_id'),
|
||||
chunkIndex: Number(payload.chunk_index),
|
||||
language: stringField('language'),
|
||||
title: stringField('title'),
|
||||
text: stringField('text'),
|
||||
textSha256: stringField('text_sha256'),
|
||||
...(typeof payload.source_uri === 'string' && payload.source_uri ? { sourceUri: payload.source_uri } : {})
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function filterConditions(filters: RegulatoryRetrievalFilters): Array<Record<string, unknown>> {
|
||||
const values: Array<[string, string | undefined]> = [
|
||||
['corpus_id', filters.corpusId],
|
||||
['corpus_version', filters.corpusVersion],
|
||||
['framework', filters.framework],
|
||||
['framework_version', filters.frameworkVersion],
|
||||
['control_id', filters.controlId],
|
||||
['language', filters.language?.toLowerCase()]
|
||||
];
|
||||
return values.filter((entry): entry is [string, string] => Boolean(entry[1])).map(([key, value]) => keywordCondition(key, value));
|
||||
}
|
||||
|
||||
function keywordCondition(key: string, value: string): Record<string, unknown> {
|
||||
return { key, match: { value } };
|
||||
}
|
||||
|
||||
function pointId(chunkId: string): string {
|
||||
const hash = createHash('sha256').update(chunkId, 'utf8').digest('hex').slice(0, 32);
|
||||
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20)}`;
|
||||
}
|
||||
|
||||
function validateEmbedding(value: number[], dimensions: number): void {
|
||||
if (!Array.isArray(value) || value.length !== dimensions || value.some(entry => typeof entry !== 'number' || !Number.isFinite(entry))) {
|
||||
throw new Error(`Regulatory embeddings must contain ${dimensions} finite values.`);
|
||||
}
|
||||
}
|
||||
|
||||
function boundedInteger(value: string | undefined, fallback: number, minimum: number, maximum: number, name: string): number {
|
||||
if (!value?.trim()) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) throw new Error(`${name} must be an integer between ${minimum} and ${maximum}.`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function withTrailingSlash(url: URL): URL {
|
||||
const copy = new URL(url.href);
|
||||
if (!copy.pathname.endsWith('/')) copy.pathname += '/';
|
||||
return copy;
|
||||
}
|
||||
|
||||
async function readBoundedText(response: Response, maximumBytes: number): Promise<string> {
|
||||
const declared = Number(response.headers.get('content-length') ?? 0);
|
||||
if (declared > maximumBytes) throw new QdrantClientError('qdrant_response_too_large', 'Qdrant response exceeded the size limit.');
|
||||
if (!response.body) return '';
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.byteLength;
|
||||
if (size > maximumBytes) {
|
||||
await reader.cancel();
|
||||
throw new QdrantClientError('qdrant_response_too_large', 'Qdrant response exceeded the size limit.');
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const merged = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) { merged.set(chunk, offset); offset += chunk.byteLength; }
|
||||
return new TextDecoder().decode(merged);
|
||||
}
|
||||
|
||||
function object(value: unknown): Record<string, any> | undefined {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, any> : undefined;
|
||||
}
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
REGULATORY_CHUNK_SCHEMA_VERSION,
|
||||
type RegulatoryChunk,
|
||||
type RegulatoryControlDocument
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
|
||||
export interface RegulatoryChunkingOptions {
|
||||
maximumCharacters: number;
|
||||
}
|
||||
|
||||
export function validateRegulatoryControlDocument(value: unknown): RegulatoryControlDocument {
|
||||
if (!isObject(value) || value.schemaVersion !== 1) throw new Error('Regulatory documents must use schemaVersion 1.');
|
||||
const required = ['corpusId', 'corpusVersion', 'documentId', 'framework', 'frameworkVersion', 'controlId', 'language', 'title', 'text'] as const;
|
||||
const document = value as Record<string, unknown>;
|
||||
for (const field of required) {
|
||||
if (typeof document[field] !== 'string' || !document[field].trim()) throw new Error(`Regulatory document ${field} must be a non-empty string.`);
|
||||
}
|
||||
if (!/^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$/.test(String(document.language))) {
|
||||
throw new Error('Regulatory document language must be a BCP 47 language tag.');
|
||||
}
|
||||
if (document.sourceUri !== undefined && (typeof document.sourceUri !== 'string' || !document.sourceUri.trim())) {
|
||||
throw new Error('Regulatory document sourceUri must be a non-empty string when present.');
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
corpusId: String(document.corpusId).trim(),
|
||||
corpusVersion: String(document.corpusVersion).trim(),
|
||||
documentId: String(document.documentId).trim(),
|
||||
framework: String(document.framework).trim(),
|
||||
frameworkVersion: String(document.frameworkVersion).trim(),
|
||||
controlId: String(document.controlId).trim(),
|
||||
language: String(document.language).trim().toLowerCase(),
|
||||
title: String(document.title).trim(),
|
||||
text: normalizeText(String(document.text)),
|
||||
...(document.sourceUri === undefined ? {} : { sourceUri: String(document.sourceUri).trim() })
|
||||
};
|
||||
}
|
||||
|
||||
export function chunkRegulatoryControl(
|
||||
input: RegulatoryControlDocument,
|
||||
options: RegulatoryChunkingOptions
|
||||
): RegulatoryChunk[] {
|
||||
const document = validateRegulatoryControlDocument(input);
|
||||
if (!Number.isInteger(options.maximumCharacters) || options.maximumCharacters < 200 || options.maximumCharacters > 20_000) {
|
||||
throw new Error('maximumCharacters must be an integer between 200 and 20000.');
|
||||
}
|
||||
const parts = splitText(document.text, options.maximumCharacters);
|
||||
return parts.map((text, chunkIndex) => {
|
||||
const textSha256 = sha256(text);
|
||||
const identity = sha256([
|
||||
String(REGULATORY_CHUNK_SCHEMA_VERSION), document.corpusId, document.corpusVersion,
|
||||
document.documentId, String(chunkIndex), textSha256
|
||||
].join('\n'));
|
||||
return {
|
||||
...document,
|
||||
chunkId: `${document.documentId}:${chunkIndex}:${identity.slice(0, 12)}`,
|
||||
chunkIndex,
|
||||
text,
|
||||
textSha256
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function regulatoryChunkEmbeddingText(chunk: RegulatoryChunk): string {
|
||||
return `${chunk.framework} ${chunk.frameworkVersion} | ${chunk.controlId} | ${chunk.title}\n\n${chunk.text}`;
|
||||
}
|
||||
|
||||
function splitText(text: string, maximumCharacters: number): string[] {
|
||||
const paragraphs = text.split(/\n{2,}/).map(value => value.trim()).filter(Boolean);
|
||||
const units = paragraphs.flatMap(paragraph => splitLongUnit(paragraph, maximumCharacters));
|
||||
const chunks: string[] = [];
|
||||
let current = '';
|
||||
for (const unit of units) {
|
||||
const candidate = current ? `${current}\n\n${unit}` : unit;
|
||||
if (candidate.length <= maximumCharacters) {
|
||||
current = candidate;
|
||||
continue;
|
||||
}
|
||||
if (current) chunks.push(current);
|
||||
current = unit;
|
||||
}
|
||||
if (current) chunks.push(current);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function splitLongUnit(value: string, maximumCharacters: number): string[] {
|
||||
if (value.length <= maximumCharacters) return [value];
|
||||
const result: string[] = [];
|
||||
let remaining = value;
|
||||
while (remaining.length > maximumCharacters) {
|
||||
const boundary = Math.max(
|
||||
remaining.lastIndexOf('. ', maximumCharacters),
|
||||
remaining.lastIndexOf('; ', maximumCharacters),
|
||||
remaining.lastIndexOf(' ', maximumCharacters)
|
||||
);
|
||||
const splitAt = boundary >= Math.floor(maximumCharacters * 0.6) ? boundary + 1 : maximumCharacters;
|
||||
result.push(remaining.slice(0, splitAt).trim());
|
||||
remaining = remaining.slice(splitAt).trim();
|
||||
}
|
||||
if (remaining) result.push(remaining);
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeText(value: string): string {
|
||||
return value.replace(/\r\n?/g, '\n').replace(/[ \t]+\n/g, '\n').trim();
|
||||
}
|
||||
|
||||
function sha256(value: string): string {
|
||||
return createHash('sha256').update(value, 'utf8').digest('hex');
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import { createRegulatoryRagRuntime } from './regulatory-rag-runtime';
|
||||
import {
|
||||
evaluateRegulatoryRetriever,
|
||||
regulatoryRetrievalEvaluationCases
|
||||
} from './regulatory-retrieval-evaluation';
|
||||
|
||||
const MAXIMUM_EVALUATION_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const inputPath = process.argv[2];
|
||||
const limit = Number(process.argv[3] ?? 10);
|
||||
if (!inputPath) throw new Error('Usage: npm run rag:evaluate -- <evaluation-cases.json> [limit]');
|
||||
const resolvedPath = resolve(inputPath);
|
||||
const inputStat = await stat(resolvedPath);
|
||||
if (!inputStat.isFile() || inputStat.size > MAXIMUM_EVALUATION_BYTES) {
|
||||
throw new Error('Regulatory retrieval evaluation input must be a file no larger than 16 MiB.');
|
||||
}
|
||||
const bytes = await readFile(resolvedPath);
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('Regulatory retrieval evaluation input is not valid JSON.'); }
|
||||
const cases = regulatoryRetrievalEvaluationCases(value);
|
||||
const report = await evaluateRegulatoryRetriever(createRegulatoryRagRuntime().retrieval, cases, limit);
|
||||
console.log(JSON.stringify({
|
||||
caseCount: report.caseCount,
|
||||
limit: report.limit,
|
||||
hitRateAtK: report.hitRateAtK,
|
||||
recallAtK: report.recallAtK,
|
||||
meanReciprocalRank: report.meanReciprocalRank
|
||||
}));
|
||||
}
|
||||
|
||||
void main().catch(error => {
|
||||
console.error(error instanceof Error ? error.message : 'Regulatory retrieval evaluation failed.');
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import type {
|
||||
RegulatoryFindingQuery,
|
||||
RegulatoryRetrievalCandidate,
|
||||
RegulatoryRetrievalFilters,
|
||||
RegulatoryRetriever
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query';
|
||||
|
||||
export interface RegulatoryFindingRetrievalResult {
|
||||
sourceResultId: string;
|
||||
status: 'candidates' | 'insufficient-public-context';
|
||||
preparedQuery?: RegulatoryFindingQuery;
|
||||
candidates: RegulatoryRetrievalCandidate[];
|
||||
}
|
||||
|
||||
export class RegulatoryFindingRetrievalService {
|
||||
constructor(private readonly retriever: RegulatoryRetriever) {}
|
||||
|
||||
async retrieve(
|
||||
finding: UnifiedResultProjection,
|
||||
limit: number,
|
||||
filters?: RegulatoryRetrievalFilters
|
||||
): Promise<RegulatoryFindingRetrievalResult> {
|
||||
const preparedQuery = buildRegulatoryFindingQuery(finding);
|
||||
if (!preparedQuery) {
|
||||
return { sourceResultId: finding.sourceResultId, status: 'insufficient-public-context', candidates: [] };
|
||||
}
|
||||
const candidates = await this.retriever.retrieve({ query: preparedQuery.query, limit, filters });
|
||||
return { sourceResultId: finding.sourceResultId, status: 'candidates', preparedQuery, candidates };
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import type { RegulatoryControlDocument } from '../../src/domain/regulatory-rag';
|
||||
import { validateRegulatoryControlDocument } from './regulatory-chunker';
|
||||
import { createRegulatoryRagRuntime } from './regulatory-rag-runtime';
|
||||
|
||||
const MAXIMUM_CORPUS_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const inputPath = process.argv[2];
|
||||
if (!inputPath) throw new Error('Usage: npm run rag:ingest -- <regulatory-corpus.json>');
|
||||
const resolvedPath = resolve(inputPath);
|
||||
const inputStat = await stat(resolvedPath);
|
||||
if (!inputStat.isFile() || inputStat.size > MAXIMUM_CORPUS_BYTES) throw new Error('Regulatory corpus must be a file no larger than 64 MiB.');
|
||||
const bytes = await readFile(resolvedPath);
|
||||
let value: unknown;
|
||||
try { value = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('Regulatory corpus is not valid JSON.'); }
|
||||
const documents = corpusDocuments(value);
|
||||
const result = await createRegulatoryRagRuntime().ingestion.ingest(documents);
|
||||
console.log(`Indexed ${result.documentCount} regulatory documents as ${result.chunkCount} chunks with ${result.embeddingModel}.`);
|
||||
}
|
||||
|
||||
function corpusDocuments(value: unknown): RegulatoryControlDocument[] {
|
||||
if (!isObject(value) || value.schemaVersion !== 1 || !Array.isArray(value.documents) || !value.documents.length) {
|
||||
throw new Error('Regulatory corpus must use schemaVersion 1 and contain a non-empty documents array.');
|
||||
}
|
||||
if (value.documents.length > 100_000) throw new Error('Regulatory corpus exceeds the 100000 document limit.');
|
||||
return value.documents.map(validateRegulatoryControlDocument);
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
||||
void main().catch(error => {
|
||||
console.error(error instanceof Error ? error.message : 'Regulatory ingestion failed.');
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import { createEmbeddingClientFromEnvironment, type EmbeddingEnvironment } from './embedding-client';
|
||||
import { createQdrantRegulatoryIndexFromEnvironment, type QdrantEnvironment } from './qdrant-regulatory-index';
|
||||
import { RegulatoryFindingRetrievalService } from './regulatory-finding-retrieval';
|
||||
import {
|
||||
DenseRegulatoryCandidateRetriever,
|
||||
RegulatoryIngestionService,
|
||||
RegulatoryRetrievalPipeline
|
||||
} from './regulatory-rag-service';
|
||||
|
||||
export interface RegulatoryRagEnvironment extends EmbeddingEnvironment, QdrantEnvironment {
|
||||
ASSESSOR_RAG_CHUNK_CHARACTERS?: string;
|
||||
ASSESSOR_RAG_EMBEDDING_BATCH_SIZE?: string;
|
||||
}
|
||||
|
||||
export function createRegulatoryRagRuntime(
|
||||
environment: RegulatoryRagEnvironment = {
|
||||
ASSESSOR_QDRANT_URL: process.env.ASSESSOR_QDRANT_URL,
|
||||
ASSESSOR_QDRANT_API_KEY: process.env.ASSESSOR_QDRANT_API_KEY,
|
||||
ASSESSOR_QDRANT_TIMEOUT_MS: process.env.ASSESSOR_QDRANT_TIMEOUT_MS,
|
||||
ASSESSOR_RAG_EMBEDDING_BASE_URL: process.env.ASSESSOR_RAG_EMBEDDING_BASE_URL,
|
||||
ASSESSOR_RAG_EMBEDDING_MODEL: process.env.ASSESSOR_RAG_EMBEDDING_MODEL,
|
||||
ASSESSOR_RAG_EMBEDDING_API_KEY: process.env.ASSESSOR_RAG_EMBEDDING_API_KEY,
|
||||
ASSESSOR_RAG_EMBEDDING_DIMENSIONS: process.env.ASSESSOR_RAG_EMBEDDING_DIMENSIONS,
|
||||
ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS: process.env.ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS,
|
||||
ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS: process.env.ASSESSOR_RAG_EMBEDDING_TIMEOUT_MS,
|
||||
ASSESSOR_RAG_CHUNK_CHARACTERS: process.env.ASSESSOR_RAG_CHUNK_CHARACTERS,
|
||||
ASSESSOR_RAG_EMBEDDING_BATCH_SIZE: process.env.ASSESSOR_RAG_EMBEDDING_BATCH_SIZE
|
||||
}
|
||||
): {
|
||||
ingestion: RegulatoryIngestionService;
|
||||
retrieval: RegulatoryRetrievalPipeline;
|
||||
findingRetrieval: RegulatoryFindingRetrievalService;
|
||||
} {
|
||||
const embeddings = createEmbeddingClientFromEnvironment(environment);
|
||||
const index = createQdrantRegulatoryIndexFromEnvironment(embeddings.dimensions, embeddings.model, environment);
|
||||
const dense = new DenseRegulatoryCandidateRetriever(embeddings, index);
|
||||
const retrieval = new RegulatoryRetrievalPipeline(dense);
|
||||
return {
|
||||
ingestion: new RegulatoryIngestionService(embeddings, index, {
|
||||
maximumChunkCharacters: boundedInteger(environment.ASSESSOR_RAG_CHUNK_CHARACTERS, 1_600, 200, 20_000, 'ASSESSOR_RAG_CHUNK_CHARACTERS'),
|
||||
embeddingBatchSize: boundedInteger(environment.ASSESSOR_RAG_EMBEDDING_BATCH_SIZE, 32, 1, 128, 'ASSESSOR_RAG_EMBEDDING_BATCH_SIZE')
|
||||
}),
|
||||
retrieval,
|
||||
findingRetrieval: new RegulatoryFindingRetrievalService(retrieval)
|
||||
};
|
||||
}
|
||||
|
||||
function boundedInteger(value: string | undefined, fallback: number, minimum: number, maximum: number, name: string): number {
|
||||
if (!value?.trim()) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) throw new Error(`${name} must be an integer between ${minimum} and ${maximum}.`);
|
||||
return parsed;
|
||||
}
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
import type {
|
||||
RegulatoryControlDocument,
|
||||
RegulatoryReranker,
|
||||
RegulatoryRetrievalCandidate,
|
||||
RegulatoryRetrievalRequest,
|
||||
RegulatoryRetriever
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
import type { TextEmbeddingClient } from './embedding-client';
|
||||
import { chunkRegulatoryControl, regulatoryChunkEmbeddingText, validateRegulatoryControlDocument } from './regulatory-chunker';
|
||||
import type { RegulatoryChunkIndex } from './qdrant-regulatory-index';
|
||||
|
||||
export interface RegulatoryCandidateRetriever {
|
||||
retrieveCandidates(request: RegulatoryRetrievalRequest): Promise<RegulatoryRetrievalCandidate[]>;
|
||||
}
|
||||
|
||||
export class DenseRegulatoryCandidateRetriever implements RegulatoryCandidateRetriever {
|
||||
constructor(
|
||||
private readonly embeddings: TextEmbeddingClient,
|
||||
private readonly index: RegulatoryChunkIndex
|
||||
) {}
|
||||
|
||||
async retrieveCandidates(request: RegulatoryRetrievalRequest): Promise<RegulatoryRetrievalCandidate[]> {
|
||||
const [vector] = await this.embeddings.embed([validatedQuery(request.query)]);
|
||||
return this.index.searchDense(vector, validatedLimit(request.limit), request.filters);
|
||||
}
|
||||
}
|
||||
|
||||
export class RegulatoryRetrievalPipeline implements RegulatoryRetriever {
|
||||
constructor(
|
||||
private readonly candidates: RegulatoryCandidateRetriever,
|
||||
private readonly reranker?: RegulatoryReranker,
|
||||
private readonly rerankCandidateMultiplier = 4
|
||||
) {
|
||||
if (!Number.isInteger(rerankCandidateMultiplier) || rerankCandidateMultiplier < 1 || rerankCandidateMultiplier > 10) {
|
||||
throw new Error('rerankCandidateMultiplier must be an integer between 1 and 10.');
|
||||
}
|
||||
}
|
||||
|
||||
async retrieve(request: RegulatoryRetrievalRequest): Promise<RegulatoryRetrievalCandidate[]> {
|
||||
const limit = validatedLimit(request.limit);
|
||||
const query = validatedQuery(request.query);
|
||||
const candidateLimit = this.reranker ? Math.min(100, limit * this.rerankCandidateMultiplier) : limit;
|
||||
const candidates = await this.candidates.retrieveCandidates({ ...request, query, limit: candidateLimit });
|
||||
if (!this.reranker) return candidates.slice(0, limit);
|
||||
return (await this.reranker.rerank(query, candidates)).slice(0, limit);
|
||||
}
|
||||
}
|
||||
|
||||
export interface RegulatoryIngestionOptions {
|
||||
maximumChunkCharacters?: number;
|
||||
embeddingBatchSize?: number;
|
||||
}
|
||||
|
||||
export interface RegulatoryIngestionResult {
|
||||
documentCount: number;
|
||||
chunkCount: number;
|
||||
embeddingModel: string;
|
||||
}
|
||||
|
||||
export class RegulatoryIngestionService {
|
||||
private readonly maximumChunkCharacters: number;
|
||||
private readonly embeddingBatchSize: number;
|
||||
|
||||
constructor(
|
||||
private readonly embeddings: TextEmbeddingClient,
|
||||
private readonly index: RegulatoryChunkIndex,
|
||||
options: RegulatoryIngestionOptions = {}
|
||||
) {
|
||||
this.maximumChunkCharacters = options.maximumChunkCharacters ?? 1_600;
|
||||
this.embeddingBatchSize = options.embeddingBatchSize ?? 32;
|
||||
if (!Number.isInteger(this.embeddingBatchSize) || this.embeddingBatchSize < 1 || this.embeddingBatchSize > 128) {
|
||||
throw new Error('embeddingBatchSize must be an integer between 1 and 128.');
|
||||
}
|
||||
}
|
||||
|
||||
async ingest(documents: RegulatoryControlDocument[]): Promise<RegulatoryIngestionResult> {
|
||||
if (!documents.length) throw new Error('At least one regulatory control document is required.');
|
||||
const documentKeys = new Set<string>();
|
||||
const chunks = documents.flatMap(input => {
|
||||
const document = validateRegulatoryControlDocument(input);
|
||||
const key = `${document.corpusId}\0${document.documentId}`;
|
||||
if (documentKeys.has(key)) throw new Error(`Duplicate regulatory document identity: ${document.documentId}.`);
|
||||
documentKeys.add(key);
|
||||
return chunkRegulatoryControl(document, { maximumCharacters: this.maximumChunkCharacters });
|
||||
});
|
||||
const vectors: number[][] = [];
|
||||
for (let offset = 0; offset < chunks.length; offset += this.embeddingBatchSize) {
|
||||
const batch = chunks.slice(offset, offset + this.embeddingBatchSize).map(regulatoryChunkEmbeddingText);
|
||||
vectors.push(...await this.embeddings.embed(batch));
|
||||
}
|
||||
if (vectors.length !== chunks.length) throw new Error('Embedding provider returned an unexpected vector count.');
|
||||
await this.index.ensureCollection();
|
||||
await this.index.replaceDocuments(chunks.map((chunk, index) => ({ chunk, embedding: vectors[index] })));
|
||||
return { documentCount: documents.length, chunkCount: chunks.length, embeddingModel: this.embeddings.model };
|
||||
}
|
||||
}
|
||||
|
||||
function validatedQuery(value: string): string {
|
||||
if (typeof value !== 'string' || !value.trim()) throw new Error('Regulatory retrieval query must be non-empty.');
|
||||
if (value.length > 20_000) throw new Error('Regulatory retrieval query exceeds 20000 characters.');
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function validatedLimit(value: number): number {
|
||||
if (!Number.isInteger(value) || value < 1 || value > 100) throw new Error('Regulatory retrieval limit must be between 1 and 100.');
|
||||
return value;
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
import type {
|
||||
RegulatoryRetrievalCandidate,
|
||||
RegulatoryRetrievalFilters,
|
||||
RegulatoryRetriever
|
||||
} from '../../src/domain/regulatory-rag';
|
||||
|
||||
export interface RegulatoryExpectedControl {
|
||||
framework: string;
|
||||
frameworkVersion?: string;
|
||||
controlId: string;
|
||||
language?: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalEvaluationCase {
|
||||
id: string;
|
||||
query: string;
|
||||
queryLanguage: string;
|
||||
filters?: RegulatoryRetrievalFilters;
|
||||
expectedControls: RegulatoryExpectedControl[];
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalEvaluationCaseResult {
|
||||
id: string;
|
||||
queryLanguage: string;
|
||||
expectedCount: number;
|
||||
matchedCount: number;
|
||||
firstRelevantRank?: number;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalEvaluationReport {
|
||||
caseCount: number;
|
||||
limit: number;
|
||||
hitRateAtK: number;
|
||||
recallAtK: number;
|
||||
meanReciprocalRank: number;
|
||||
cases: RegulatoryRetrievalEvaluationCaseResult[];
|
||||
}
|
||||
|
||||
export function regulatoryRetrievalEvaluationCases(value: unknown): RegulatoryRetrievalEvaluationCase[] {
|
||||
if (!isObject(value) || value.schemaVersion !== 1 || !Array.isArray(value.cases)) {
|
||||
throw new Error('Regulatory retrieval evaluation input must use schemaVersion 1 and contain a cases array.');
|
||||
}
|
||||
if (!value.cases.length || value.cases.length > 1_000) {
|
||||
throw new Error('Regulatory retrieval evaluation requires between 1 and 1000 cases.');
|
||||
}
|
||||
const cases = value.cases as RegulatoryRetrievalEvaluationCase[];
|
||||
const ids = new Set<string>();
|
||||
for (const testCase of cases) {
|
||||
validateCase(testCase);
|
||||
if (ids.has(testCase.id)) throw new Error(`Duplicate regulatory retrieval evaluation case id: ${testCase.id}.`);
|
||||
ids.add(testCase.id);
|
||||
}
|
||||
return cases;
|
||||
}
|
||||
|
||||
/** Computes control-level metrics while leaving chunk ranking unchanged. */
|
||||
export async function evaluateRegulatoryRetriever(
|
||||
retriever: RegulatoryRetriever,
|
||||
cases: RegulatoryRetrievalEvaluationCase[],
|
||||
limit: number
|
||||
): Promise<RegulatoryRetrievalEvaluationReport> {
|
||||
if (!cases.length || cases.length > 1_000) throw new Error('Regulatory retrieval evaluation requires between 1 and 1000 cases.');
|
||||
if (!Number.isInteger(limit) || limit < 1 || limit > 100) throw new Error('Regulatory retrieval evaluation limit must be between 1 and 100.');
|
||||
const results: RegulatoryRetrievalEvaluationCaseResult[] = [];
|
||||
let matchedTotal = 0;
|
||||
let expectedTotal = 0;
|
||||
let hitTotal = 0;
|
||||
let reciprocalRankTotal = 0;
|
||||
for (const testCase of cases) {
|
||||
validateCase(testCase);
|
||||
const candidates = await retriever.retrieve({ query: testCase.query, limit, filters: testCase.filters });
|
||||
const result = evaluateCase(testCase, candidates.slice(0, limit));
|
||||
results.push(result);
|
||||
matchedTotal += result.matchedCount;
|
||||
expectedTotal += result.expectedCount;
|
||||
if (result.firstRelevantRank !== undefined) {
|
||||
hitTotal += 1;
|
||||
reciprocalRankTotal += 1 / result.firstRelevantRank;
|
||||
}
|
||||
}
|
||||
return {
|
||||
caseCount: cases.length,
|
||||
limit,
|
||||
hitRateAtK: hitTotal / cases.length,
|
||||
recallAtK: matchedTotal / expectedTotal,
|
||||
meanReciprocalRank: reciprocalRankTotal / cases.length,
|
||||
cases: results
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateCase(
|
||||
testCase: RegulatoryRetrievalEvaluationCase,
|
||||
candidates: RegulatoryRetrievalCandidate[]
|
||||
): RegulatoryRetrievalEvaluationCaseResult {
|
||||
const expected = new Map(testCase.expectedControls.map(control => [controlKey(control), control]));
|
||||
const matched = new Set<string>();
|
||||
let firstRelevantRank: number | undefined;
|
||||
candidates.forEach((candidate, index) => {
|
||||
for (const [key, control] of expected) {
|
||||
if (!matchesControl(candidate, control)) continue;
|
||||
matched.add(key);
|
||||
firstRelevantRank ??= index + 1;
|
||||
}
|
||||
});
|
||||
return {
|
||||
id: testCase.id,
|
||||
queryLanguage: testCase.queryLanguage.toLowerCase(),
|
||||
expectedCount: expected.size,
|
||||
matchedCount: matched.size,
|
||||
...(firstRelevantRank === undefined ? {} : { firstRelevantRank })
|
||||
};
|
||||
}
|
||||
|
||||
function matchesControl(candidate: RegulatoryRetrievalCandidate, expected: RegulatoryExpectedControl): boolean {
|
||||
return fold(candidate.chunk.framework) === fold(expected.framework)
|
||||
&& (!expected.frameworkVersion || fold(candidate.chunk.frameworkVersion) === fold(expected.frameworkVersion))
|
||||
&& fold(candidate.chunk.controlId) === fold(expected.controlId)
|
||||
&& (!expected.language || fold(candidate.chunk.language) === fold(expected.language));
|
||||
}
|
||||
|
||||
function controlKey(control: RegulatoryExpectedControl): string {
|
||||
return [control.framework, control.frameworkVersion ?? '', control.controlId, control.language ?? ''].map(fold).join('\0');
|
||||
}
|
||||
|
||||
function validateCase(testCase: RegulatoryRetrievalEvaluationCase): void {
|
||||
if (!isObject(testCase)
|
||||
|| !boundedString(testCase.id, 200)
|
||||
|| !boundedString(testCase.query, 20_000)
|
||||
|| !boundedString(testCase.queryLanguage, 35)
|
||||
|| !Array.isArray(testCase.expectedControls)
|
||||
|| !testCase.expectedControls.length
|
||||
|| testCase.expectedControls.length > 100) {
|
||||
throw new Error('Every regulatory retrieval evaluation case requires an id, query, queryLanguage, and expected controls.');
|
||||
}
|
||||
if (testCase.filters !== undefined && (!isObject(testCase.filters)
|
||||
|| Object.values(testCase.filters).some(value => value !== undefined && !boundedString(value, 500)))) {
|
||||
throw new Error(`Regulatory retrieval evaluation case ${testCase.id} has invalid filters.`);
|
||||
}
|
||||
if (testCase.expectedControls.some(control => !isObject(control)
|
||||
|| !boundedString(control.framework, 500)
|
||||
|| !boundedString(control.controlId, 500)
|
||||
|| (control.frameworkVersion !== undefined && !boundedString(control.frameworkVersion, 500))
|
||||
|| (control.language !== undefined && !boundedString(control.language, 35)))) {
|
||||
throw new Error(`Regulatory retrieval evaluation case ${testCase.id} has invalid expected controls.`);
|
||||
}
|
||||
}
|
||||
|
||||
const fold = (value: string): string => value.trim().toLowerCase();
|
||||
const boundedString = (value: unknown, maximum: number): value is string => typeof value === 'string' && Boolean(value.trim()) && value.length <= maximum;
|
||||
const isObject = (value: unknown): value is Record<string, unknown> => Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
export const REGULATORY_CHUNK_SCHEMA_VERSION = 1 as const;
|
||||
export const REGULATORY_FINDING_QUERY_SCHEMA_VERSION = 1 as const;
|
||||
|
||||
export interface RegulatoryControlDocument {
|
||||
schemaVersion: 1;
|
||||
corpusId: string;
|
||||
corpusVersion: string;
|
||||
documentId: string;
|
||||
framework: string;
|
||||
frameworkVersion: string;
|
||||
controlId: string;
|
||||
language: string;
|
||||
title: string;
|
||||
text: string;
|
||||
sourceUri?: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryChunk {
|
||||
schemaVersion: 1;
|
||||
corpusId: string;
|
||||
corpusVersion: string;
|
||||
documentId: string;
|
||||
framework: string;
|
||||
frameworkVersion: string;
|
||||
controlId: string;
|
||||
chunkId: string;
|
||||
chunkIndex: number;
|
||||
language: string;
|
||||
title: string;
|
||||
text: string;
|
||||
textSha256: string;
|
||||
sourceUri?: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalFilters {
|
||||
corpusId?: string;
|
||||
corpusVersion?: string;
|
||||
framework?: string;
|
||||
frameworkVersion?: string;
|
||||
controlId?: string;
|
||||
language?: string;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalRequest {
|
||||
query: string;
|
||||
limit: number;
|
||||
filters?: RegulatoryRetrievalFilters;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetrievalCandidate {
|
||||
score: number;
|
||||
chunk: RegulatoryChunk;
|
||||
}
|
||||
|
||||
export interface RegulatoryRetriever {
|
||||
retrieve(request: RegulatoryRetrievalRequest): Promise<RegulatoryRetrievalCandidate[]>;
|
||||
}
|
||||
|
||||
export interface RegulatoryFindingQuery {
|
||||
schemaVersion: 1;
|
||||
query: string;
|
||||
specificity: 'rule' | 'topic';
|
||||
publicTool?: string;
|
||||
publicRuleId?: string;
|
||||
publicTopics: string[];
|
||||
}
|
||||
|
||||
export interface RegulatoryReranker {
|
||||
rerank(query: string, candidates: RegulatoryRetrievalCandidate[]): Promise<RegulatoryRetrievalCandidate[]>;
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
// Fixed public concepts only. Matching may inspect report prose locally, but it
|
||||
// never returns that prose to a caller.
|
||||
const topics: Array<[RegExp, string]> = [
|
||||
[/\bAdminSDHolder\b/i, 'Active Directory AdminSDHolder ACL inheritance'],
|
||||
[/\bDCSync\b|\bDC Sync\b/i, 'Active Directory replication permissions / DCSync'],
|
||||
[/\bDCShadow\b/i, 'Active Directory DCShadow rogue replication'],
|
||||
[/\bSIDHistory\b/i, 'Active Directory SIDHistory permissions'],
|
||||
[/\bresource.based constrained delegation\b|\bRBCD\b/i, 'Kerberos resource-based constrained delegation'],
|
||||
[/\bunconstrained delegation\b/i, 'Kerberos unconstrained delegation'],
|
||||
[/\bconstrained delegation\b/i, 'Kerberos constrained delegation'],
|
||||
[/\bkrbtgt\b/i, 'Kerberos krbtgt account protection'],
|
||||
[/\bSMB\s*signing\b/i, 'SMB signing requirements'], [/\bSMBv?1\b/i, 'SMB version 1 removal'],
|
||||
[/\bLDAP\s*signing\b/i, 'LDAP signing requirements'], [/\bchannel binding\b/i, 'LDAP channel binding'],
|
||||
[/\bNTLM\b/i, 'NTLM authentication restrictions'], [/\bWDigest\b/i, 'WDigest credential caching'],
|
||||
[/\bLAPS\b/i, 'Windows LAPS password management'], [/\bBitLocker\b/i, 'BitLocker encryption'],
|
||||
[/\bCredential Guard\b/i, 'Windows Defender Credential Guard'],
|
||||
[/\bTLS\b/i, 'TLS protocol and cipher configuration'], [/\bcertificate templates?\b/i, 'Active Directory Certificate Services template security'],
|
||||
[/\bConditional Access\b/i, 'Microsoft Entra Conditional Access'], [/\bMFA\b|\bmultifactor authentication\b/i, 'Multifactor authentication'],
|
||||
[/\blegacy authentication\b/i, 'Legacy authentication blocking'], [/\bbreak.glass\b|\bemergency access\b/i, 'Emergency access account protection'],
|
||||
[/\bPIM\b|\bPrivileged Identity Management\b/i, 'Entra Privileged Identity Management'],
|
||||
[/\bIntune\b/i, 'Microsoft Intune device configuration'], [/\bExchange Online\b/i, 'Exchange Online security'],
|
||||
[/\bDefender\b/i, 'Microsoft Defender protection'], [/\bAppLocker\b/i, 'AppLocker application control'],
|
||||
[/\baudit logs?.*(?:retention|maintained)|(?:retention|maintained).*audit logs?/i, 'Microsoft Purview audit log retention'],
|
||||
[/\bPowerShell\b/i, 'PowerShell security logging'], [/\bWindows audit\b|\baudit policy\b/i, 'Windows audit policy'],
|
||||
[/\bpassword\b/i, 'Password policy'], [/\bfirewall\b/i, 'Firewall security policy'],
|
||||
[/\bOpenSSH\b|\bsshd\b/i, 'OpenSSH server hardening'], [/\bsudo\b/i, 'Linux sudo privilege configuration'],
|
||||
[/\bauditd\b/i, 'Linux auditd configuration'], [/\bSELinux\b/i, 'SELinux mandatory access control'],
|
||||
[/\bAppArmor\b/i, 'AppArmor mandatory access control']
|
||||
];
|
||||
|
||||
export function publicSecurityTopics(evidence: string, maximum = 4): string[] {
|
||||
if (!Number.isInteger(maximum) || maximum < 1) return [];
|
||||
return topics.filter(([pattern]) => pattern.test(evidence)).map(([, publicText]) => publicText).slice(0, maximum);
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
import type { RegulatoryFindingQuery } from '../../domain/regulatory-rag';
|
||||
import { REGULATORY_FINDING_QUERY_SCHEMA_VERSION } from '../../domain/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../domain/unified-result';
|
||||
import catalog from './catalog.json';
|
||||
import { publicSecurityTopics } from './public-security-topics';
|
||||
|
||||
const publicTools: Record<string, string> = {
|
||||
'cis-cat': 'CIS-CAT / CIS Benchmarks',
|
||||
maester: 'Maester',
|
||||
pingcastle: 'PingCastle',
|
||||
'purple-knight': 'Semperis Purple Knight'
|
||||
};
|
||||
|
||||
const publicRuleIds: Record<string, readonly string[]> = {
|
||||
maester: catalog.maesterIds,
|
||||
pingcastle: catalog.pingcastleIds
|
||||
};
|
||||
|
||||
/**
|
||||
* Builds an embedding query only from catalog-owned identifiers and fixed public
|
||||
* concepts. Raw finding prose is inspected locally for topic recognition and is
|
||||
* never copied into the returned query.
|
||||
*/
|
||||
export function buildRegulatoryFindingQuery(finding: UnifiedResultProjection): RegulatoryFindingQuery | undefined {
|
||||
if (finding.sourceKind !== 'scanner-finding') return undefined;
|
||||
const toolId = finding.sourceTool?.trim().toLowerCase() ?? '';
|
||||
const publicTool = publicTools[toolId];
|
||||
const publicRuleId = recognizedRuleId(toolId, finding.sourceRuleId);
|
||||
const evidence = [finding.title, finding.description, finding.recommendation].filter(Boolean).join('\n').slice(0, 20_000);
|
||||
const topics = publicSecurityTopics(evidence);
|
||||
if (!publicRuleId && !topics.length) return undefined;
|
||||
const specificity = publicRuleId ? 'rule' : 'topic';
|
||||
const query = [
|
||||
'Retrieve regulatory controls whose normative requirements directly address this security finding.',
|
||||
publicTool && `Public assessment tool: ${publicTool}.`,
|
||||
publicRuleId && `Public rule ID: ${publicRuleId}.`,
|
||||
topics.length && `Public technical topics: ${topics.join('; ')}.`,
|
||||
'Prefer controls with direct requirement text over merely related guidance. Do not infer customer scope, assets, configuration values, or compliance status.'
|
||||
].filter(Boolean).join(' ');
|
||||
return {
|
||||
schemaVersion: REGULATORY_FINDING_QUERY_SCHEMA_VERSION,
|
||||
query,
|
||||
specificity,
|
||||
...(publicTool ? { publicTool } : {}),
|
||||
...(publicRuleId ? { publicRuleId } : {}),
|
||||
publicTopics: topics
|
||||
};
|
||||
}
|
||||
|
||||
function recognizedRuleId(toolId: string, value: string | undefined): string | undefined {
|
||||
const candidate = value?.trim();
|
||||
if (!candidate) return undefined;
|
||||
return publicRuleIds[toolId]?.find(id => id.toLowerCase() === candidate.toLowerCase());
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@ import { toolIdForRecord } from '../../importers/status-normalization';
|
|||
import { policyValueName } from '../group-policy/presentation';
|
||||
import { policyPublicSearchDetails } from '../group-policy/web-search';
|
||||
import catalog from './catalog.json';
|
||||
import { publicSecurityTopics } from './public-security-topics';
|
||||
|
||||
export interface PublicSearch { query: string; url: string; specificity: 'setting' | 'rule' | 'topic' }
|
||||
const tools: Record<string, string> = { 'group-policy': 'Microsoft Group Policy', 'cis-cat': 'CIS-CAT / CIS Benchmarks',
|
||||
|
|
@ -15,35 +16,6 @@ const nativeSettings: Record<string, string> = {
|
|||
LockoutBadCount: 'Account lockout threshold', LockoutDuration: 'Account lockout duration', ResetLockoutCount: 'Account lockout counter reset',
|
||||
};
|
||||
|
||||
// Fixed public concepts only. Matching never copies report prose into a query.
|
||||
const topics: Array<[RegExp, string]> = [
|
||||
[/\bAdminSDHolder\b/i, 'Active Directory AdminSDHolder ACL inheritance'],
|
||||
[/\bDCSync\b|\bDC Sync\b/i, 'Active Directory replication permissions / DCSync'],
|
||||
[/\bDCShadow\b/i, 'Active Directory DCShadow rogue replication'],
|
||||
[/\bSIDHistory\b/i, 'Active Directory SIDHistory permissions'],
|
||||
[/\bresource.based constrained delegation\b|\bRBCD\b/i, 'Kerberos resource-based constrained delegation'],
|
||||
[/\bunconstrained delegation\b/i, 'Kerberos unconstrained delegation'],
|
||||
[/\bconstrained delegation\b/i, 'Kerberos constrained delegation'],
|
||||
[/\bkrbtgt\b/i, 'Kerberos krbtgt account protection'],
|
||||
[/\bSMB\s*signing\b/i, 'SMB signing requirements'], [/\bSMBv?1\b/i, 'SMB version 1 removal'],
|
||||
[/\bLDAP\s*signing\b/i, 'LDAP signing requirements'], [/\bchannel binding\b/i, 'LDAP channel binding'],
|
||||
[/\bNTLM\b/i, 'NTLM authentication restrictions'], [/\bWDigest\b/i, 'WDigest credential caching'],
|
||||
[/\bLAPS\b/i, 'Windows LAPS password management'], [/\bBitLocker\b/i, 'BitLocker encryption'],
|
||||
[/\bCredential Guard\b/i, 'Windows Defender Credential Guard'],
|
||||
[/\bTLS\b/i, 'TLS protocol and cipher configuration'], [/\bcertificate templates?\b/i, 'Active Directory Certificate Services template security'],
|
||||
[/\bConditional Access\b/i, 'Microsoft Entra Conditional Access'], [/\bMFA\b|\bmultifactor authentication\b/i, 'Multifactor authentication'],
|
||||
[/\blegacy authentication\b/i, 'Legacy authentication blocking'], [/\bbreak.glass\b|\bemergency access\b/i, 'Emergency access account protection'],
|
||||
[/\bPIM\b|\bPrivileged Identity Management\b/i, 'Entra Privileged Identity Management'],
|
||||
[/\bIntune\b/i, 'Microsoft Intune device configuration'], [/\bExchange Online\b/i, 'Exchange Online security'],
|
||||
[/\bDefender\b/i, 'Microsoft Defender protection'], [/\bAppLocker\b/i, 'AppLocker application control'],
|
||||
[/\baudit logs?.*(?:retention|maintained)|(?:retention|maintained).*audit logs?/i, 'Microsoft Purview audit log retention'],
|
||||
[/\bPowerShell\b/i, 'PowerShell security logging'], [/\bWindows audit\b|\baudit policy\b/i, 'Windows audit policy'],
|
||||
[/\bpassword\b/i, 'Password policy'], [/\bfirewall\b/i, 'Firewall security policy'],
|
||||
[/\bOpenSSH\b|\bsshd\b/i, 'OpenSSH server hardening'], [/\bsudo\b/i, 'Linux sudo privilege configuration'],
|
||||
[/\bauditd\b/i, 'Linux auditd configuration'], [/\bSELinux\b/i, 'SELinux mandatory access control'],
|
||||
[/\bAppArmor\b/i, 'AppArmor mandatory access control'],
|
||||
];
|
||||
|
||||
function field(record: FindingRecord, labels: string[]): string {
|
||||
const values = record.fields.filter(f => labels.includes(f.label.trim().toLowerCase()));
|
||||
return values.length === 1 && values[0].value.length <= 16000 ? values[0].value.trim() : '';
|
||||
|
|
@ -115,7 +87,7 @@ export function findingWebSearch(record: FindingRecord, toolId?: string): Public
|
|||
const rule = ruleContext(record, tool);
|
||||
const policy = tool === 'group-policy' ? policyPublicSearchDetails(record) : undefined;
|
||||
const evidence = [record.title, field(record, ['description']), field(record, ['rationale'])].join('\n').slice(0, 20000);
|
||||
const subjects = topics.filter(([pattern]) => pattern.test(evidence)).map(([, publicText]) => publicText).slice(0, 4);
|
||||
const subjects = publicSecurityTopics(evidence);
|
||||
const scenario = ['Fail', 'Failed', 'Pass', 'Passed', 'Not configured', 'Conflicts', 'Duplicates'].find(s => fold(s) === fold(field(record, ['status'])));
|
||||
const productSource = field(record, ['baseline', 'benchmark', 'benchmark title', 'benchmark name']);
|
||||
const products = ['Windows Server 2016', 'Windows Server 2019', 'Windows Server 2022', 'Windows Server 2025', 'Windows 10', 'Windows 11', 'Microsoft 365', 'Microsoft Edge', 'Office 2016', 'Ubuntu', 'Debian', 'Red Hat Enterprise Linux']
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ describe('deployment workflow', () => {
|
|||
|
||||
it('validates all database settings without printing secret values', () => {
|
||||
expect(workflow.match(/if \[ -z "\$ASSESSOR_ENV_FILE" \]/g)).toHaveLength(2);
|
||||
expect(workflow.match(/for key in ASSESSOR_POSTGRES_DB ASSESSOR_POSTGRES_USER ASSESSOR_POSTGRES_PASSWORD/g)).toHaveLength(2);
|
||||
expect(workflow.match(/for key in ASSESSOR_POSTGRES_DB ASSESSOR_POSTGRES_USER ASSESSOR_POSTGRES_PASSWORD ASSESSOR_QDRANT_API_KEY/g)).toHaveLength(2);
|
||||
expect(workflow).toContain('secret values were not printed.');
|
||||
expect(workflow).not.toMatch(/echo "\$ASSESSOR_ENV_FILE"/);
|
||||
expect(workflow).not.toContain('printf \'%s\\n\' "${{ secrets.');
|
||||
|
|
@ -63,6 +63,7 @@ describe('deployment Compose stack', () => {
|
|||
expect(compose).toContain('${ASSESSOR_POSTGRES_DB:?set ASSESSOR_POSTGRES_DB}');
|
||||
expect(compose).toContain('${ASSESSOR_POSTGRES_USER:?set ASSESSOR_POSTGRES_USER}');
|
||||
expect(compose).toContain('${ASSESSOR_POSTGRES_PASSWORD:?set ASSESSOR_POSTGRES_PASSWORD}');
|
||||
expect(compose).toContain('${ASSESSOR_QDRANT_API_KEY:?set ASSESSOR_QDRANT_API_KEY}');
|
||||
});
|
||||
|
||||
it('keeps PostgreSQL internal and scopes persistent volumes through Compose', () => {
|
||||
|
|
@ -70,6 +71,10 @@ describe('deployment Compose stack', () => {
|
|||
expect(postgresSection).not.toMatch(/\n ports:/);
|
||||
expect(compose).toContain('assessor-artifacts:/var/lib/assessor/artifacts');
|
||||
expect(compose).toContain('assessor-postgres-data:/var/lib/postgresql/data');
|
||||
expect(compose.match(/restart: unless-stopped/g)).toHaveLength(2);
|
||||
expect(compose.match(/restart: unless-stopped/g)).toHaveLength(3);
|
||||
expect(compose).toContain('qdrant/qdrant:v1.19.1');
|
||||
expect(compose).toContain('QDRANT__TELEMETRY_DISABLED: "true"');
|
||||
expect(compose).toContain('assessor-qdrant-data:/qdrant/storage');
|
||||
expect(compose.slice(compose.indexOf(' qdrant:'))).not.toMatch(/\n ports:/);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const localCompose = readFileSync('docker-compose.postgres.yml', 'utf8');
|
||||
const deploymentCompose = readFileSync('docker-compose.deploy.yml', 'utf8');
|
||||
const packageJson = JSON.parse(readFileSync('package.json', 'utf8')) as { scripts: Record<string, string> };
|
||||
|
||||
describe('regulatory Qdrant Compose boundary', () => {
|
||||
it('adds an authenticated telemetry-free loopback service without changing canonical PostgreSQL', () => {
|
||||
expect(localCompose).toContain('name: assessor-2_0');
|
||||
expect(localCompose).toContain('name: assessor-2_0_assessor-postgres-data');
|
||||
expect(localCompose).toContain('external: true');
|
||||
expect(localCompose).toContain('qdrant/qdrant:v1.19.1');
|
||||
expect(localCompose).toContain('QDRANT__SERVICE__API_KEY: ${ASSESSOR_QDRANT_API_KEY:?ASSESSOR_QDRANT_API_KEY must be set}');
|
||||
expect(localCompose).toContain('QDRANT__TELEMETRY_DISABLED: "true"');
|
||||
expect(localCompose).toContain('127.0.0.1:${ASSESSOR_QDRANT_PORT:-6333}:6333');
|
||||
expect(localCompose).toContain('name: assessor-2_0_assessor-qdrant-data');
|
||||
});
|
||||
|
||||
it('keeps deployment Qdrant internal and exposes a corpus ingestion command', () => {
|
||||
const qdrant = deploymentCompose.slice(deploymentCompose.indexOf(' qdrant:'));
|
||||
expect(qdrant).not.toMatch(/\n ports:/);
|
||||
expect(qdrant).toContain('assessor-qdrant-data:/qdrant/storage');
|
||||
expect(packageJson.scripts['rag:ingest']).toContain('--env-file-if-exists=.env.postgres');
|
||||
expect(packageJson.scripts['rag:ingest']).toContain('dist-server/regulatory-ingest.mjs');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
{
|
||||
"schemaVersion": 1,
|
||||
"documents": [
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"documentId": "synthetic-mfa-en",
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-MFA",
|
||||
"language": "en",
|
||||
"title": "Multifactor authentication",
|
||||
"text": "Privileged and remote access must use multifactor authentication. Authentication factors must be independently verifiable.",
|
||||
"sourceUri": "https://example.invalid/assessor-synthetic-controls/SYN-MFA"
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"documentId": "synthetic-mfa-de",
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-MFA",
|
||||
"language": "de",
|
||||
"title": "Mehrfaktorauthentifizierung",
|
||||
"text": "Privilegierte Zugriffe und Fernzugriffe muessen eine Mehrfaktorauthentifizierung verwenden. Die Faktoren muessen unabhaengig pruefbar sein.",
|
||||
"sourceUri": "https://example.invalid/assessor-synthetic-controls/SYN-MFA"
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"documentId": "synthetic-password-en",
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-PASSWORD",
|
||||
"language": "en",
|
||||
"title": "Password protection",
|
||||
"text": "Password policies must prevent weak and reused passwords and must protect stored authentication secrets.",
|
||||
"sourceUri": "https://example.invalid/assessor-synthetic-controls/SYN-PASSWORD"
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"documentId": "synthetic-password-de",
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-PASSWORD",
|
||||
"language": "de",
|
||||
"title": "Passwortschutz",
|
||||
"text": "Passwortrichtlinien muessen schwache und wiederverwendete Passwoerter verhindern und gespeicherte Anmeldegeheimnisse schuetzen.",
|
||||
"sourceUri": "https://example.invalid/assessor-synthetic-controls/SYN-PASSWORD"
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"documentId": "synthetic-logging-en",
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-LOGGING",
|
||||
"language": "en",
|
||||
"title": "Security event logging",
|
||||
"text": "Security-relevant events must be logged, protected against alteration, and reviewed for suspicious activity.",
|
||||
"sourceUri": "https://example.invalid/assessor-synthetic-controls/SYN-LOGGING"
|
||||
},
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"documentId": "synthetic-logging-de",
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-LOGGING",
|
||||
"language": "de",
|
||||
"title": "Protokollierung von Sicherheitsereignissen",
|
||||
"text": "Sicherheitsrelevante Ereignisse muessen protokolliert, gegen Veraenderung geschuetzt und auf verdaechtige Aktivitaeten geprueft werden.",
|
||||
"sourceUri": "https://example.invalid/assessor-synthetic-controls/SYN-LOGGING"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
{
|
||||
"schemaVersion": 1,
|
||||
"cases": [
|
||||
{
|
||||
"id": "synthetic-en-to-de-mfa",
|
||||
"query": "Require a second independent authentication factor for privileged remote access",
|
||||
"queryLanguage": "en",
|
||||
"filters": {
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"language": "de"
|
||||
},
|
||||
"expectedControls": [
|
||||
{
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-MFA",
|
||||
"language": "de"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "synthetic-de-to-en-password",
|
||||
"query": "Schwache und wiederverwendete Passwoerter verhindern",
|
||||
"queryLanguage": "de",
|
||||
"filters": {
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"language": "en"
|
||||
},
|
||||
"expectedControls": [
|
||||
{
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-PASSWORD",
|
||||
"language": "en"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "synthetic-en-to-de-logging",
|
||||
"query": "Protect audit logs from tampering and review suspicious security events",
|
||||
"queryLanguage": "en",
|
||||
"filters": {
|
||||
"corpusId": "assessor-synthetic-regulatory",
|
||||
"corpusVersion": "1",
|
||||
"language": "de"
|
||||
},
|
||||
"expectedControls": [
|
||||
{
|
||||
"framework": "ASSESSOR Synthetic Controls",
|
||||
"frameworkVersion": "1",
|
||||
"controlId": "SYN-LOGGING",
|
||||
"language": "de"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
createEmbeddingClientFromEnvironment,
|
||||
embeddingEndpoint,
|
||||
EmbeddingClientError,
|
||||
OpenAiCompatibleEmbeddingClient
|
||||
} from '../../server/regulatory/embedding-client';
|
||||
|
||||
describe('regulatory embedding client', () => {
|
||||
it('allows loopback HTTP, requires remote HTTPS, and requires explicit dimensions', () => {
|
||||
expect(embeddingEndpoint('https://litellm.work.example/v1').href).toBe('https://litellm.work.example/v1/embeddings');
|
||||
expect(embeddingEndpoint('http://127.0.0.1:8080/v1').href).toBe('http://127.0.0.1:8080/v1/embeddings');
|
||||
expect(() => embeddingEndpoint('http://litellm.work.example/v1')).toThrow(/HTTPS/);
|
||||
expect(() => createEmbeddingClientFromEnvironment({
|
||||
ASSESSOR_RAG_EMBEDDING_BASE_URL: 'https://litellm.work.example/v1',
|
||||
ASSESSOR_RAG_EMBEDDING_MODEL: 'multilingual-model',
|
||||
ASSESSOR_RAG_EMBEDDING_API_KEY: 'secret'
|
||||
})).toThrow(/DIMENSIONS is required/);
|
||||
});
|
||||
|
||||
it('requests a shortened dimension only when explicitly configured', async () => {
|
||||
const bodies: unknown[] = [];
|
||||
const fetcher = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
bodies.push(JSON.parse(String(init?.body)));
|
||||
return new Response(JSON.stringify({ data: [{ index: 0, embedding: Array(64).fill(0) }] }), { status: 200 });
|
||||
});
|
||||
const client = createEmbeddingClientFromEnvironment({
|
||||
ASSESSOR_RAG_EMBEDDING_BASE_URL: 'https://litellm.work.example/v1',
|
||||
ASSESSOR_RAG_EMBEDDING_MODEL: 'text-embedding-3-large',
|
||||
ASSESSOR_RAG_EMBEDDING_API_KEY: 'secret',
|
||||
ASSESSOR_RAG_EMBEDDING_DIMENSIONS: '64',
|
||||
ASSESSOR_RAG_EMBEDDING_REQUEST_DIMENSIONS: 'true'
|
||||
}, fetcher);
|
||||
|
||||
await client.embed(['Synthetic text']);
|
||||
|
||||
expect(bodies).toEqual([{ model: 'text-embedding-3-large', input: ['Synthetic text'], encoding_format: 'float', dimensions: 64 }]);
|
||||
});
|
||||
|
||||
it('returns embeddings in provider index order without exposing the API key', async () => {
|
||||
const fetcher = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
expect(init?.redirect).toBe('error');
|
||||
expect(new Headers(init?.headers).get('authorization')).toBe('Bearer secret');
|
||||
expect(JSON.parse(String(init?.body))).toEqual({ model: 'multilingual-model', input: ['Deutsch', 'English'], encoding_format: 'float' });
|
||||
return new Response(JSON.stringify({ data: [
|
||||
{ index: 1, embedding: [0, 1] },
|
||||
{ index: 0, embedding: [1, 0] }
|
||||
] }), { status: 200 });
|
||||
});
|
||||
const client = new OpenAiCompatibleEmbeddingClient({
|
||||
endpoint: new URL('https://litellm.work.example/v1/embeddings'),
|
||||
model: 'multilingual-model', apiKey: 'secret', dimensions: 2, timeoutMs: 1_000, fetcher
|
||||
});
|
||||
|
||||
await expect(client.embed(['Deutsch', 'English'])).resolves.toEqual([[1, 0], [0, 1]]);
|
||||
expect(fetcher).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('rejects incompatible vector dimensions', async () => {
|
||||
const client = new OpenAiCompatibleEmbeddingClient({
|
||||
endpoint: new URL('https://litellm.work.example/v1/embeddings'),
|
||||
model: 'multilingual-model', apiKey: 'secret', dimensions: 3, timeoutMs: 1_000,
|
||||
fetcher: async () => new Response(JSON.stringify({ data: [{ index: 0, embedding: [1, 0] }] }), { status: 200 })
|
||||
});
|
||||
await expect(client.embed(['query'])).rejects.toMatchObject({ code: 'embedding_invalid_dimensions' } satisfies Partial<EmbeddingClientError>);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { createQdrantRegulatoryIndexFromEnvironment } from '../../server/regulatory/qdrant-regulatory-index';
|
||||
import { DenseRegulatoryCandidateRetriever, RegulatoryIngestionService } from '../../server/regulatory/regulatory-rag-service';
|
||||
import type { RegulatoryControlDocument } from '../../src/domain/regulatory-rag';
|
||||
import type { TextEmbeddingClient } from '../../server/regulatory/embedding-client';
|
||||
|
||||
const runIntegration = process.env.ASSESSOR_QDRANT_INTEGRATION === '1';
|
||||
const describeQdrant = runIntegration ? describe : describe.skip;
|
||||
|
||||
describeQdrant('Qdrant regulatory index integration', () => {
|
||||
it('creates, replaces, filters, and retrieves a synthetic control', async () => {
|
||||
const dimensions = Number(process.env.ASSESSOR_RAG_EMBEDDING_DIMENSIONS);
|
||||
const model = process.env.ASSESSOR_RAG_EMBEDDING_MODEL ?? '';
|
||||
if (!Number.isInteger(dimensions) || dimensions < 64 || !model) throw new Error('Configured embedding dimensions and model are required.');
|
||||
const vector = Array.from({ length: dimensions }, (_, index) => index === 0 ? 1 : 0);
|
||||
const embeddings: TextEmbeddingClient = { model, dimensions, embed: async texts => texts.map(() => vector) };
|
||||
const index = createQdrantRegulatoryIndexFromEnvironment(dimensions, model);
|
||||
const ingestion = new RegulatoryIngestionService(embeddings, index);
|
||||
const document: RegulatoryControlDocument = {
|
||||
schemaVersion: 1,
|
||||
corpusId: 'assessor-qdrant-integration',
|
||||
corpusVersion: '1',
|
||||
documentId: 'synthetic-live-control-de',
|
||||
framework: 'ASSESSOR Synthetic Controls',
|
||||
frameworkVersion: '1',
|
||||
controlId: 'SYN-LIVE',
|
||||
language: 'de',
|
||||
title: 'Synthetische Livekontrolle',
|
||||
text: 'Dieser synthetische Text prueft ausschliesslich den lokalen rebuildbaren Qdrant-Index.'
|
||||
};
|
||||
|
||||
try {
|
||||
await expect(ingestion.ingest([document])).resolves.toMatchObject({ documentCount: 1, chunkCount: 1, embeddingModel: model });
|
||||
const retrieval = new DenseRegulatoryCandidateRetriever(embeddings, index);
|
||||
const candidates = await retrieval.retrieveCandidates({
|
||||
query: 'synthetic local index check',
|
||||
limit: 3,
|
||||
filters: { corpusId: document.corpusId, framework: document.framework, language: 'de' }
|
||||
});
|
||||
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0]).toMatchObject({ chunk: {
|
||||
corpusId: document.corpusId,
|
||||
documentId: document.documentId,
|
||||
controlId: document.controlId,
|
||||
language: 'de'
|
||||
} });
|
||||
} finally {
|
||||
await deleteSyntheticCorpus(document.corpusId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
async function deleteSyntheticCorpus(corpusId: string): Promise<void> {
|
||||
const baseUrl = process.env.ASSESSOR_QDRANT_URL;
|
||||
const apiKey = process.env.ASSESSOR_QDRANT_API_KEY;
|
||||
if (!baseUrl || !apiKey) return;
|
||||
const response = await fetch(new URL('/collections/assessor_regulatory_chunks_v1/points/delete?wait=true', baseUrl), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'api-key': apiKey },
|
||||
body: JSON.stringify({ filter: { must: [{ key: 'corpus_id', match: { value: corpusId } }] } })
|
||||
});
|
||||
if (!response.ok) throw new Error(`Synthetic Qdrant integration cleanup failed with HTTP ${response.status}.`);
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { RegulatoryChunk } from '../../src/domain/regulatory-rag';
|
||||
import {
|
||||
createQdrantRegulatoryIndexFromEnvironment,
|
||||
QdrantClientError,
|
||||
QdrantRegulatoryIndex,
|
||||
REGULATORY_QDRANT_COLLECTION
|
||||
} from '../../server/regulatory/qdrant-regulatory-index';
|
||||
|
||||
const chunk: RegulatoryChunk = {
|
||||
schemaVersion: 1,
|
||||
corpusId: 'iso-27001', corpusVersion: '2022-de-1', documentId: 'a-5-1-de',
|
||||
framework: 'ISO/IEC 27001', frameworkVersion: '2022', controlId: 'A.5.1',
|
||||
chunkId: 'a-5-1-de:0:abcdef', chunkIndex: 0, language: 'de', title: 'Policies',
|
||||
text: 'Synthetic regulatory text.', textSha256: 'a'.repeat(64)
|
||||
};
|
||||
|
||||
function json(value: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(value), { status, headers: { 'Content-Type': 'application/json' } });
|
||||
}
|
||||
|
||||
describe('Qdrant regulatory index', () => {
|
||||
it('creates the versioned dense/sparse collection and payload indexes', async () => {
|
||||
const fetcher = vi.fn(async (urlValue: string | URL | Request, init?: RequestInit) => {
|
||||
const url = String(urlValue);
|
||||
if (init?.method === 'GET' && url.endsWith(`/collections/${REGULATORY_QDRANT_COLLECTION}`)) {
|
||||
const getCount = fetcher.mock.calls.filter(call => call[1]?.method === 'GET').length;
|
||||
if (getCount === 1) return json({}, 404);
|
||||
return json({ result: { config: { params: {
|
||||
vectors: { dense: { size: 2, distance: 'Cosine' } }, sparse_vectors: { sparse: {} }
|
||||
} }, payload_schema: {} } });
|
||||
}
|
||||
return json({ result: true, status: 'ok' });
|
||||
});
|
||||
const index = new QdrantRegulatoryIndex({
|
||||
baseUrl: new URL('http://127.0.0.1:6333'), apiKey: 'secret', dimensions: 2,
|
||||
embeddingModel: 'multilingual-model', timeoutMs: 1_000, fetcher
|
||||
});
|
||||
|
||||
await index.ensureCollection();
|
||||
|
||||
const bodies = fetcher.mock.calls.map(call => call[1]?.body && JSON.parse(String(call[1]?.body))).filter(Boolean);
|
||||
expect(bodies[0]).toMatchObject({ vectors: { dense: { size: 2, distance: 'Cosine' } }, sparse_vectors: { sparse: {} } });
|
||||
expect(bodies.filter(body => body.field_name)).toHaveLength(9);
|
||||
expect(fetcher.mock.calls.every(call => new Headers(call[1]?.headers).get('api-key') === 'secret')).toBe(true);
|
||||
});
|
||||
|
||||
it('replaces document points and performs filtered dense retrieval', async () => {
|
||||
const requests: Array<{ url: string; body?: any }> = [];
|
||||
const fetcher = vi.fn(async (urlValue: string | URL | Request, init?: RequestInit) => {
|
||||
const body = init?.body ? JSON.parse(String(init.body)) : undefined;
|
||||
requests.push({ url: String(urlValue), body });
|
||||
if (String(urlValue).endsWith('/points/query')) return json({ result: { points: [{ score: 0.91, payload: {
|
||||
schema_version: 1, corpus_id: chunk.corpusId, corpus_version: chunk.corpusVersion,
|
||||
document_id: chunk.documentId, framework: chunk.framework, framework_version: chunk.frameworkVersion,
|
||||
control_id: chunk.controlId, chunk_id: chunk.chunkId, chunk_index: chunk.chunkIndex,
|
||||
language: chunk.language, title: chunk.title, text: chunk.text, text_sha256: chunk.textSha256
|
||||
} }] } });
|
||||
return json({ result: true, status: 'ok' });
|
||||
});
|
||||
const index = new QdrantRegulatoryIndex({
|
||||
baseUrl: new URL('http://127.0.0.1:6333'), apiKey: 'secret', dimensions: 2,
|
||||
embeddingModel: 'multilingual-model', timeoutMs: 1_000, fetcher
|
||||
});
|
||||
|
||||
await index.replaceDocuments([{ chunk, embedding: [0.2, 0.8] }]);
|
||||
await expect(index.searchDense([0.4, 0.6], 5, { framework: 'ISO/IEC 27001', language: 'DE' }))
|
||||
.resolves.toEqual([{ score: 0.91, chunk }]);
|
||||
|
||||
expect(requests[0].body.filter.must).toEqual([
|
||||
{ key: 'corpus_id', match: { value: 'iso-27001' } },
|
||||
{ key: 'document_id', match: { value: 'a-5-1-de' } }
|
||||
]);
|
||||
expect(requests[1].body.points[0]).toMatchObject({ vector: { dense: [0.2, 0.8] }, payload: {
|
||||
framework: 'ISO/IEC 27001', control_id: 'A.5.1', language: 'de', embedding_model: 'multilingual-model'
|
||||
} });
|
||||
expect(requests[2].body.filter.must).toEqual(expect.arrayContaining([
|
||||
{ key: 'embedding_model', match: { value: 'multilingual-model' } },
|
||||
{ key: 'framework', match: { value: 'ISO/IEC 27001' } },
|
||||
{ key: 'language', match: { value: 'de' } }
|
||||
]));
|
||||
});
|
||||
|
||||
it('fails closed for an incompatible collection and unsafe remote HTTP', async () => {
|
||||
const index = new QdrantRegulatoryIndex({
|
||||
baseUrl: new URL('http://127.0.0.1:6333'), apiKey: 'secret', dimensions: 2,
|
||||
embeddingModel: 'model', timeoutMs: 1_000,
|
||||
fetcher: async () => json({ result: { config: { params: {
|
||||
vectors: { dense: { size: 3, distance: 'Cosine' } }, sparse_vectors: { sparse: {} }
|
||||
} }, payload_schema: {} } })
|
||||
});
|
||||
await expect(index.ensureCollection()).rejects.toMatchObject({ code: 'qdrant_collection_incompatible' } satisfies Partial<QdrantClientError>);
|
||||
expect(() => createQdrantRegulatoryIndexFromEnvironment(2, 'model', {
|
||||
ASSESSOR_QDRANT_URL: 'http://qdrant.example', ASSESSOR_QDRANT_API_KEY: 'secret'
|
||||
})).toThrow(/must use HTTPS/);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type { RegulatoryControlDocument } from '../../src/domain/regulatory-rag';
|
||||
import { chunkRegulatoryControl, regulatoryChunkEmbeddingText, validateRegulatoryControlDocument } from '../../server/regulatory/regulatory-chunker';
|
||||
|
||||
const document: RegulatoryControlDocument = {
|
||||
schemaVersion: 1,
|
||||
corpusId: 'iso-27001',
|
||||
corpusVersion: '2022-de-1',
|
||||
documentId: 'iso-27001-2022-a.5.1-de',
|
||||
framework: 'ISO/IEC 27001',
|
||||
frameworkVersion: '2022',
|
||||
controlId: 'A.5.1',
|
||||
language: 'de',
|
||||
title: 'Informationssicherheitsrichtlinien',
|
||||
text: `${'Erster kontrollierter Absatz. '.repeat(8)}\r\n\r\n${'Second controlled paragraph. '.repeat(8)}`,
|
||||
sourceUri: 'https://example.invalid/iso-27001/a-5-1'
|
||||
};
|
||||
|
||||
describe('regulatory chunker', () => {
|
||||
it('creates deterministic bounded chunks with complete metadata', () => {
|
||||
const first = chunkRegulatoryControl(document, { maximumCharacters: 220 });
|
||||
const second = chunkRegulatoryControl(document, { maximumCharacters: 220 });
|
||||
|
||||
expect(first).toEqual(second);
|
||||
expect(first.length).toBeGreaterThan(1);
|
||||
expect(first.every(chunk => chunk.text.length <= 220)).toBe(true);
|
||||
expect(first.map(chunk => chunk.chunkIndex)).toEqual(first.map((_, index) => index));
|
||||
expect(first.every(chunk => chunk.framework === 'ISO/IEC 27001' && chunk.language === 'de')).toBe(true);
|
||||
expect(first.every(chunk => /^[a-f0-9]{64}$/.test(chunk.textSha256))).toBe(true);
|
||||
expect(regulatoryChunkEmbeddingText(first[0])).toContain('ISO/IEC 27001 2022 | A.5.1');
|
||||
});
|
||||
|
||||
it('validates the version, required values and language before indexing', () => {
|
||||
expect(() => validateRegulatoryControlDocument({ ...document, schemaVersion: 2 })).toThrow(/schemaVersion 1/);
|
||||
expect(() => validateRegulatoryControlDocument({ ...document, controlId: ' ' })).toThrow(/controlId/);
|
||||
expect(() => validateRegulatoryControlDocument({ ...document, language: 'not_a_tag' })).toThrow(/BCP 47/);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { RegulatoryFindingRetrievalService } from '../../server/regulatory/regulatory-finding-retrieval';
|
||||
import type { RegulatoryRetrievalCandidate, RegulatoryRetrievalRequest } from '../../src/domain/regulatory-rag';
|
||||
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
||||
import { buildRegulatoryFindingQuery } from '../../src/profiles/public-search/regulatory-query';
|
||||
|
||||
const privateMarkers = ['Tenant Acme', '10.23.45.67', 'secret.example', 'C:\\Customer\\report.xml', 'analyst@acme.invalid'];
|
||||
|
||||
function finding(overrides: Partial<UnifiedResultProjection> = {}): UnifiedResultProjection {
|
||||
return {
|
||||
resultProjectionId: 'projection-private', sourceKind: 'scanner-finding', sourceResultId: 'source-private',
|
||||
projectId: 'tenant-private', assessmentId: 'assessment-private', collectionRunId: 'run-private',
|
||||
sourceTool: 'purple-knight', title: `${privateMarkers[0]} has SMB signing disabled on ${privateMarkers[1]}`,
|
||||
description: `Host ${privateMarkers[2]} loaded ${privateMarkers[3]}`,
|
||||
recommendation: `Contact ${privateMarkers[4]}`, affectedAsset: privateMarkers[2],
|
||||
severityOriginal: 'high', severityNormalized: 'high', outcomeOriginal: 'fail', outcomeNormalized: 'fail',
|
||||
technicalExecutionState: 'completed', resultType: 'compliance_check', isIoe: false, isDeviation: true,
|
||||
lifecycleState: 'new', mappingConfidence: 1, reviewStatus: 'open', reviewOwner: privateMarkers[4],
|
||||
importedAt: '2026-09-15T10:00:00.000Z', createdAt: '2026-09-15T10:00:00.000Z', frameworkReferences: [], categories: [],
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
const candidate: RegulatoryRetrievalCandidate = {
|
||||
score: 0.8,
|
||||
chunk: {
|
||||
schemaVersion: 1, corpusId: 'synthetic', corpusVersion: '1', documentId: 'control-de',
|
||||
framework: 'Synthetic Framework', frameworkVersion: '1', controlId: 'CTRL-1', chunkId: 'chunk-1',
|
||||
chunkIndex: 0, language: 'de', title: 'Synthetic control', text: 'Synthetic regulatory text.',
|
||||
textSha256: 'a'.repeat(64)
|
||||
}
|
||||
};
|
||||
|
||||
describe('regulatory finding retrieval boundary', () => {
|
||||
it('emits only fixed public concepts and never copies private finding data', () => {
|
||||
const query = buildRegulatoryFindingQuery(finding());
|
||||
|
||||
expect(query).toMatchObject({
|
||||
schemaVersion: 1,
|
||||
specificity: 'topic',
|
||||
publicTool: 'Semperis Purple Knight',
|
||||
publicTopics: ['SMB signing requirements']
|
||||
});
|
||||
for (const marker of privateMarkers) expect(query?.query).not.toContain(marker);
|
||||
expect(query?.query).not.toContain('disabled');
|
||||
});
|
||||
|
||||
it('allows only exact catalog-owned rule IDs', () => {
|
||||
expect(buildRegulatoryFindingQuery(finding({
|
||||
sourceTool: 'pingcastle', sourceRuleId: 'a-krbtgt', title: privateMarkers[0], description: undefined, recommendation: undefined
|
||||
}))).toMatchObject({ specificity: 'rule', publicRuleId: 'A-Krbtgt' });
|
||||
|
||||
expect(buildRegulatoryFindingQuery(finding({
|
||||
sourceTool: 'pingcastle', sourceRuleId: 'A-Customer-Private', title: privateMarkers[0], description: undefined, recommendation: undefined
|
||||
}))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('fails closed for non-scanner records or findings without a public signal', () => {
|
||||
expect(buildRegulatoryFindingQuery(finding({ sourceKind: 'environment-observation' }))).toBeUndefined();
|
||||
expect(buildRegulatoryFindingQuery(finding({
|
||||
sourceTool: 'customer-private-scanner', title: privateMarkers[0], description: undefined, recommendation: undefined
|
||||
}))).toBeUndefined();
|
||||
});
|
||||
|
||||
it('passes only the prepared public query and explicit filters to the retriever', async () => {
|
||||
const retrieve = vi.fn(async (_request: RegulatoryRetrievalRequest) => [candidate]);
|
||||
const service = new RegulatoryFindingRetrievalService({ retrieve });
|
||||
|
||||
const result = await service.retrieve(finding(), 5, { framework: 'Synthetic Framework' });
|
||||
|
||||
expect(result).toMatchObject({ sourceResultId: 'source-private', status: 'candidates', candidates: [candidate] });
|
||||
expect(retrieve).toHaveBeenCalledOnce();
|
||||
expect(retrieve.mock.calls[0][0]).toEqual({
|
||||
query: result.preparedQuery?.query,
|
||||
limit: 5,
|
||||
filters: { framework: 'Synthetic Framework' }
|
||||
});
|
||||
for (const marker of privateMarkers) expect(JSON.stringify(retrieve.mock.calls[0][0])).not.toContain(marker);
|
||||
});
|
||||
|
||||
it('does not call retrieval when public context is insufficient', async () => {
|
||||
const retrieve = vi.fn();
|
||||
const service = new RegulatoryFindingRetrievalService({ retrieve });
|
||||
|
||||
await expect(service.retrieve(finding({ title: privateMarkers[0], description: undefined, recommendation: undefined }), 5))
|
||||
.resolves.toEqual({ sourceResultId: 'source-private', status: 'insufficient-public-context', candidates: [] });
|
||||
expect(retrieve).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { RegulatoryControlDocument, RegulatoryRetrievalCandidate } from '../../src/domain/regulatory-rag';
|
||||
import type { TextEmbeddingClient } from '../../server/regulatory/embedding-client';
|
||||
import type { IndexedRegulatoryChunk, RegulatoryChunkIndex } from '../../server/regulatory/qdrant-regulatory-index';
|
||||
import {
|
||||
DenseRegulatoryCandidateRetriever,
|
||||
RegulatoryIngestionService,
|
||||
RegulatoryRetrievalPipeline
|
||||
} from '../../server/regulatory/regulatory-rag-service';
|
||||
|
||||
const document: RegulatoryControlDocument = {
|
||||
schemaVersion: 1, corpusId: 'nis2', corpusVersion: '2024-1', documentId: 'nis2-21-2-a-de',
|
||||
framework: 'NIS2', frameworkVersion: '2022/2555', controlId: '21(2)(a)', language: 'de',
|
||||
title: 'Risikoanalyse', text: 'Synthetischer Kontrolltext. '.repeat(30)
|
||||
};
|
||||
|
||||
describe('regulatory RAG services', () => {
|
||||
it('embeds every chunk before replacing documents in the derived index', async () => {
|
||||
const embeddingBatches: string[][] = [];
|
||||
const embeddings: TextEmbeddingClient = {
|
||||
model: 'multilingual-model', dimensions: 2,
|
||||
embed: async texts => {
|
||||
embeddingBatches.push(texts);
|
||||
return texts.map((_, index) => [index, 1]);
|
||||
}
|
||||
};
|
||||
const ensureCollection = vi.fn(async () => {});
|
||||
const replaceDocuments = vi.fn(async (_chunks: IndexedRegulatoryChunk[]) => {});
|
||||
const index: RegulatoryChunkIndex = { ensureCollection, replaceDocuments, searchDense: vi.fn() };
|
||||
const service = new RegulatoryIngestionService(embeddings, index, { maximumChunkCharacters: 220, embeddingBatchSize: 2 });
|
||||
|
||||
const result = await service.ingest([document]);
|
||||
|
||||
expect(result.documentCount).toBe(1);
|
||||
expect(result.chunkCount).toBeGreaterThan(1);
|
||||
expect(embeddingBatches.every(batch => batch.length <= 2)).toBe(true);
|
||||
expect(ensureCollection).toHaveBeenCalledOnce();
|
||||
expect(replaceDocuments).toHaveBeenCalledOnce();
|
||||
expect(replaceDocuments.mock.calls[0][0]).toHaveLength(result.chunkCount);
|
||||
});
|
||||
|
||||
it('supports multilingual dense candidates and an optional later reranker stage', async () => {
|
||||
const embed = vi.fn(async () => [[0.5, 0.5]]);
|
||||
const candidates: RegulatoryRetrievalCandidate[] = Array.from({ length: 8 }, (_, index) => ({
|
||||
score: 1 - index / 10,
|
||||
chunk: { ...document, chunkId: `chunk-${index}`, chunkIndex: index, textSha256: String(index).padStart(64, '0') }
|
||||
}));
|
||||
const searchDense = vi.fn(async () => candidates);
|
||||
const dense = new DenseRegulatoryCandidateRetriever(
|
||||
{ model: 'multilingual-model', dimensions: 2, embed },
|
||||
{ ensureCollection: vi.fn(), replaceDocuments: vi.fn(), searchDense }
|
||||
);
|
||||
const rerank = vi.fn(async (_query: string, values: RegulatoryRetrievalCandidate[]) => values.slice().reverse());
|
||||
const retrieval = new RegulatoryRetrievalPipeline(dense, { rerank }, 4);
|
||||
|
||||
const result = await retrieval.retrieve({ query: 'How is the German control relevant?', limit: 2, filters: { language: 'de' } });
|
||||
|
||||
expect(embed).toHaveBeenCalledWith(['How is the German control relevant?']);
|
||||
expect(searchDense).toHaveBeenCalledWith([0.5, 0.5], 8, { language: 'de' });
|
||||
expect(rerank).toHaveBeenCalledOnce();
|
||||
expect(result.map(item => item.chunk.chunkId)).toEqual(['chunk-7', 'chunk-6']);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
evaluateRegulatoryRetriever,
|
||||
regulatoryRetrievalEvaluationCases
|
||||
} from '../../server/regulatory/regulatory-retrieval-evaluation';
|
||||
import type { RegulatoryRetrievalCandidate } from '../../src/domain/regulatory-rag';
|
||||
|
||||
function candidate(controlId: string, language: string, chunkIndex: number): RegulatoryRetrievalCandidate {
|
||||
return {
|
||||
score: 1 - chunkIndex / 10,
|
||||
chunk: {
|
||||
schemaVersion: 1, corpusId: 'synthetic', corpusVersion: '1', documentId: `${controlId}-${language}`,
|
||||
framework: 'Synthetic Framework', frameworkVersion: '1', controlId, chunkId: `${controlId}-${language}-${chunkIndex}`,
|
||||
chunkIndex, language, title: 'Synthetic control', text: 'Synthetic regulatory text.', textSha256: String(chunkIndex).padStart(64, '0')
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
describe('regulatory retrieval evaluation', () => {
|
||||
it('reports control-level cross-language recall, hit rate, and reciprocal rank', async () => {
|
||||
const results: Record<string, RegulatoryRetrievalCandidate[]> = {
|
||||
'Require multifactor authentication': [candidate('CTRL-X', 'de', 0), candidate('CTRL-MFA', 'de', 1), candidate('CTRL-MFA', 'de', 2)],
|
||||
'Passwortrichtlinie durchsetzen': [candidate('CTRL-PASSWORD', 'en', 0)],
|
||||
'Require secure audit logging': [candidate('CTRL-X', 'de', 0)]
|
||||
};
|
||||
const retriever = { retrieve: async ({ query }: { query: string }) => results[query] };
|
||||
|
||||
const report = await evaluateRegulatoryRetriever(retriever, [
|
||||
{
|
||||
id: 'en-to-de', query: 'Require multifactor authentication', queryLanguage: 'en',
|
||||
expectedControls: [{ framework: 'Synthetic Framework', frameworkVersion: '1', controlId: 'CTRL-MFA', language: 'de' }]
|
||||
},
|
||||
{
|
||||
id: 'de-to-en', query: 'Passwortrichtlinie durchsetzen', queryLanguage: 'de',
|
||||
expectedControls: [{ framework: 'Synthetic Framework', controlId: 'CTRL-PASSWORD', language: 'en' }]
|
||||
},
|
||||
{
|
||||
id: 'miss', query: 'Require secure audit logging', queryLanguage: 'en',
|
||||
expectedControls: [{ framework: 'Synthetic Framework', controlId: 'CTRL-AUDIT' }]
|
||||
}
|
||||
], 3);
|
||||
|
||||
expect(report).toEqual({
|
||||
caseCount: 3, limit: 3, hitRateAtK: 2 / 3, recallAtK: 2 / 3, meanReciprocalRank: 0.5,
|
||||
cases: [
|
||||
{ id: 'en-to-de', queryLanguage: 'en', expectedCount: 1, matchedCount: 1, firstRelevantRank: 2 },
|
||||
{ id: 'de-to-en', queryLanguage: 'de', expectedCount: 1, matchedCount: 1, firstRelevantRank: 1 },
|
||||
{ id: 'miss', queryLanguage: 'en', expectedCount: 1, matchedCount: 0 }
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects incomplete evaluation cases', async () => {
|
||||
await expect(evaluateRegulatoryRetriever({ retrieve: async () => [] }, [{
|
||||
id: '', query: 'query', queryLanguage: 'en', expectedControls: []
|
||||
}], 5)).rejects.toThrow('requires an id');
|
||||
});
|
||||
|
||||
it('validates the bounded versioned CLI input contract', () => {
|
||||
expect(regulatoryRetrievalEvaluationCases({
|
||||
schemaVersion: 1,
|
||||
cases: [{
|
||||
id: 'en-to-de', query: 'Require multifactor authentication', queryLanguage: 'en',
|
||||
filters: { framework: 'Synthetic Framework' },
|
||||
expectedControls: [{ framework: 'Synthetic Framework', controlId: 'CTRL-MFA', language: 'de' }]
|
||||
}]
|
||||
})).toHaveLength(1);
|
||||
expect(() => regulatoryRetrievalEvaluationCases({ schemaVersion: 1, cases: [
|
||||
{ id: 'duplicate', query: 'one', queryLanguage: 'en', expectedControls: [{ framework: 'F', controlId: '1' }] },
|
||||
{ id: 'duplicate', query: 'two', queryLanguage: 'de', expectedControls: [{ framework: 'F', controlId: '2' }] }
|
||||
] })).toThrow('Duplicate');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { validateRegulatoryControlDocument } from '../../server/regulatory/regulatory-chunker';
|
||||
import { regulatoryRetrievalEvaluationCases } from '../../server/regulatory/regulatory-retrieval-evaluation';
|
||||
import corpus from '../fixtures/regulatory-corpus.synthetic.json';
|
||||
import evaluation from '../fixtures/regulatory-evaluation.synthetic.json';
|
||||
|
||||
describe('synthetic regulatory reference fixtures', () => {
|
||||
it('keeps a versioned bilingual corpus and matching cross-language cases', () => {
|
||||
const documents = corpus.documents.map(validateRegulatoryControlDocument);
|
||||
const cases = regulatoryRetrievalEvaluationCases(evaluation);
|
||||
|
||||
expect(documents).toHaveLength(6);
|
||||
expect(new Set(documents.map(document => document.language))).toEqual(new Set(['de', 'en']));
|
||||
expect(cases).toHaveLength(3);
|
||||
expect(cases.every(testCase => testCase.expectedControls[0].language !== testCase.queryLanguage)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -12,7 +12,9 @@ export default defineConfig({
|
|||
input: {
|
||||
server: 'server/index.ts',
|
||||
'import-worker': 'server/import-worker.ts',
|
||||
'gpo-inventory-worker': 'server/gpo/gpo-inventory-worker.ts'
|
||||
'gpo-inventory-worker': 'server/gpo/gpo-inventory-worker.ts',
|
||||
'regulatory-ingest': 'server/regulatory/regulatory-ingest.ts',
|
||||
'regulatory-evaluate': 'server/regulatory/regulatory-evaluate.ts'
|
||||
},
|
||||
output: {
|
||||
entryFileNames: '[name].mjs'
|
||||
|
|
|
|||
Loading…
Reference in New Issue