128 lines
6.9 KiB
TypeScript
128 lines
6.9 KiB
TypeScript
import type { RegulatoryControlDocument } from '../../src/compliance/regulatory-rag';
|
|
import { BSI_CORPUS_ID, BSI_SOURCE_STANDARD } from '../../src/compliance/regulatory-mapping';
|
|
import { bsiRequirementText, type BsiKernelCatalog } from '../../src/compliance/bsi-grundschutz';
|
|
import type { BsiSearchIndexStatus } from '../../src/compliance/bsi-catalog-management';
|
|
import type { BsiCatalogs } from './bsi-snapshots';
|
|
import type { RegulatoryIngestionService } from './regulatory-rag-service';
|
|
|
|
export interface BsiCorpusOptions {
|
|
catalogs: () => Promise<BsiCatalogs>;
|
|
ingestion?: RegulatoryIngestionService;
|
|
countIndexed?: (corpusId: string, corpusVersion: string) => Promise<number>;
|
|
pruneIndexed?: (corpusId: string, keepCorpusVersion: string) => Promise<void>;
|
|
/** Chunks of every catalog state, so the Data Hub can tell an outdated index from a missing one. */
|
|
countCorpus?: (corpusId: string) => Promise<number>;
|
|
}
|
|
|
|
/** The corpus version of a Kernel catalog: its date and the start of its source hash, so every catalog state indexes once. */
|
|
export function bsiCorpusVersion(catalog: Pick<BsiKernelCatalog, 'version' | 'sourceSha256'>): string {
|
|
return `kernel-${catalog.version.slice(0, 10)}-${catalog.sourceSha256.slice(0, 12)}`;
|
|
}
|
|
|
|
/**
|
|
* One retrieval document per Kernel requirement: its statement and guidance with the parameters filled in, in the
|
|
* published German. The BSI text may be indexed and shown under CC BY-SA 4.0, unlike the ISO text.
|
|
*/
|
|
export function bsiCorpusDocuments(catalog: BsiKernelCatalog): RegulatoryControlDocument[] {
|
|
const corpusVersion = bsiCorpusVersion(catalog);
|
|
return catalog.requirements.map(requirement => ({
|
|
schemaVersion: 1,
|
|
corpusId: BSI_CORPUS_ID,
|
|
corpusVersion,
|
|
documentId: `bsi-kernel:${requirement.id}`,
|
|
framework: BSI_SOURCE_STANDARD,
|
|
frameworkVersion: catalog.version,
|
|
controlId: requirement.id,
|
|
language: 'de',
|
|
title: requirement.title,
|
|
text: [bsiRequirementText(requirement.statement, requirement.params),
|
|
requirement.guidance ? bsiRequirementText(requirement.guidance, requirement.params) : ''].filter(Boolean).join('\n\n'),
|
|
sourceUri: catalog.sourceUrl
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* The BSI Kernel corpus in the shared Qdrant/BGE-M3 path under its own corpus id (Phase 13F, BSI stage 2). It is
|
|
* built from the active catalog from the Data Hub BSI tile or, when nobody did, when a BSI mapping first needs it,
|
|
* once per catalog state; older catalog states are pruned afterwards. Concurrent callers share one build.
|
|
*/
|
|
export class BsiCorpus {
|
|
private building?: { corpusVersion: string; task: Promise<string>; done: boolean; progress?: { completed: number; total: number } };
|
|
private failure?: { corpusVersion: string; error: string };
|
|
private builtAt?: { corpusVersion: string; at: string };
|
|
|
|
constructor(private readonly options: BsiCorpusOptions) {}
|
|
|
|
get configured(): boolean {
|
|
return Boolean(this.options.ingestion && this.options.countIndexed);
|
|
}
|
|
|
|
async corpusVersion(): Promise<string> {
|
|
return bsiCorpusVersion((await this.options.catalogs()).catalog);
|
|
}
|
|
|
|
/** The indexed corpus version of the active catalog, building it first when it is missing. */
|
|
async ensure(): Promise<string> {
|
|
if (!this.configured) throw new Error('Embedding or Qdrant is not configured.');
|
|
const { catalog } = await this.options.catalogs();
|
|
const corpusVersion = bsiCorpusVersion(catalog);
|
|
if (this.building?.corpusVersion !== corpusVersion) this.track(catalog, corpusVersion, false);
|
|
return this.building!.task;
|
|
}
|
|
|
|
/**
|
|
* Starts building the index of the active catalog from the Data Hub (user decision 2026-10-09) without waiting for
|
|
* it; `force` embeds every requirement again even when the index is complete. A running build is not started twice.
|
|
*/
|
|
async startBuild(force: boolean): Promise<BsiSearchIndexStatus> {
|
|
if (!this.configured) return this.status();
|
|
const { catalog } = await this.options.catalogs();
|
|
const corpusVersion = bsiCorpusVersion(catalog);
|
|
if (!(this.building?.corpusVersion === corpusVersion && !this.building.done)) this.track(catalog, corpusVersion, force).catch(() => undefined);
|
|
return this.status();
|
|
}
|
|
|
|
async status(): Promise<BsiSearchIndexStatus> {
|
|
const { catalog } = await this.options.catalogs();
|
|
const corpusVersion = bsiCorpusVersion(catalog);
|
|
const base = { version: 1 as const, catalogVersion: catalog.version, corpusVersion, requirementCount: catalog.requirements.length };
|
|
if (!this.configured) return { ...base, state: 'unavailable', indexedChunks: 0, disabledReason: 'Embedding or Qdrant is not configured.' };
|
|
const indexedChunks = await this.options.countIndexed!(BSI_CORPUS_ID, corpusVersion).catch(() => 0);
|
|
const building = this.building?.corpusVersion === corpusVersion ? this.building : undefined;
|
|
if (building && !building.done) return { ...base, state: 'building', indexedChunks, ...(building.progress ? { progress: { ...building.progress } } : {}) };
|
|
if (this.failure?.corpusVersion === corpusVersion) return { ...base, state: 'failed', indexedChunks, error: this.failure.error };
|
|
if (indexedChunks >= catalog.requirements.length) {
|
|
return { ...base, state: 'built', indexedChunks, ...(this.builtAt?.corpusVersion === corpusVersion ? { builtAt: this.builtAt.at } : {}) };
|
|
}
|
|
const anyVersion = await (this.options.countCorpus?.(BSI_CORPUS_ID) ?? Promise.resolve(0)).catch(() => 0);
|
|
return { ...base, state: anyVersion > indexedChunks ? 'outdated' : 'not-built', indexedChunks };
|
|
}
|
|
|
|
private track(catalog: BsiKernelCatalog, corpusVersion: string, force: boolean): Promise<string> {
|
|
const entry: NonNullable<BsiCorpus['building']> = { corpusVersion, task: Promise.resolve(corpusVersion), done: false };
|
|
entry.task = this.build(catalog, corpusVersion, force, (completed, total) => { entry.progress = { completed, total }; });
|
|
this.building = entry;
|
|
this.failure = undefined;
|
|
entry.task.then(() => {
|
|
entry.done = true;
|
|
this.builtAt = { corpusVersion, at: new Date().toISOString() };
|
|
}, (error: unknown) => {
|
|
// A failed build is retried by the next caller and reported in the Data Hub until then.
|
|
entry.done = true;
|
|
if (this.building === entry) this.building = undefined;
|
|
this.failure = { corpusVersion, error: error instanceof Error ? error.message.slice(0, 300) : 'The BSI search index could not be built.' };
|
|
});
|
|
return entry.task;
|
|
}
|
|
|
|
private async build(catalog: BsiKernelCatalog, corpusVersion: string, force: boolean,
|
|
onProgress: (completed: number, total: number) => void): Promise<string> {
|
|
const indexed = await this.options.countIndexed!(BSI_CORPUS_ID, corpusVersion).catch(() => 0);
|
|
if (force || indexed < catalog.requirements.length) {
|
|
await this.options.ingestion!.ingest(bsiCorpusDocuments(catalog), onProgress);
|
|
}
|
|
await this.options.pruneIndexed?.(BSI_CORPUS_ID, corpusVersion).catch(() => undefined);
|
|
return corpusVersion;
|
|
}
|
|
}
|