import { readFile, stat } from 'node:fs/promises'; import { resolve } from 'node:path'; import type { RegulatoryControlDocument } from '../../src/compliance/regulatory-rag'; import { validateRegulatoryControlDocument } from './regulatory-chunker'; import { createRegulatoryRagRuntime } from './regulatory-rag-runtime'; const MAXIMUM_CORPUS_BYTES = 64 * 1024 * 1024; async function main(): Promise { const inputPath = process.argv[2]; if (!inputPath) throw new Error('Usage: npm run rag:ingest -- '); const resolvedPath = resolve(inputPath); const inputStat = await stat(resolvedPath); if (!inputStat.isFile() || inputStat.size > MAXIMUM_CORPUS_BYTES) throw new Error('Regulatory corpus must be a file no larger than 64 MiB.'); const bytes = await readFile(resolvedPath); let value: unknown; try { value = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('Regulatory corpus is not valid JSON.'); } const documents = corpusDocuments(value); const result = await createRegulatoryRagRuntime().ingestion.ingest(documents); console.log(`Indexed ${result.documentCount} regulatory documents as ${result.chunkCount} chunks with ${result.embeddingModel}.`); } function corpusDocuments(value: unknown): RegulatoryControlDocument[] { if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion)) || !Array.isArray(value.documents) || !value.documents.length) { throw new Error('Regulatory corpus must use schemaVersion 1 or 2 and contain a non-empty documents array.'); } if (value.documents.length > 100_000) throw new Error('Regulatory corpus exceeds the 100000 document limit.'); return value.documents.map(validateRegulatoryControlDocument); } function isObject(value: unknown): value is Record { return Boolean(value) && typeof value === 'object' && !Array.isArray(value); } void main().catch(error => { console.error(error instanceof Error ? error.message : 'Regulatory ingestion failed.'); process.exitCode = 1; });