39 lines
2.0 KiB
TypeScript
39 lines
2.0 KiB
TypeScript
import { readFile, stat } from 'node:fs/promises';
|
|
import { resolve } from 'node:path';
|
|
import type { RegulatoryControlDocument } from '../../src/domain/regulatory-rag';
|
|
import { validateRegulatoryControlDocument } from './regulatory-chunker';
|
|
import { createRegulatoryRagRuntime } from './regulatory-rag-runtime';
|
|
|
|
const MAXIMUM_CORPUS_BYTES = 64 * 1024 * 1024;
|
|
|
|
async function main(): Promise<void> {
|
|
const inputPath = process.argv[2];
|
|
if (!inputPath) throw new Error('Usage: npm run rag:ingest -- <regulatory-corpus.json>');
|
|
const resolvedPath = resolve(inputPath);
|
|
const inputStat = await stat(resolvedPath);
|
|
if (!inputStat.isFile() || inputStat.size > MAXIMUM_CORPUS_BYTES) throw new Error('Regulatory corpus must be a file no larger than 64 MiB.');
|
|
const bytes = await readFile(resolvedPath);
|
|
let value: unknown;
|
|
try { value = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('Regulatory corpus is not valid JSON.'); }
|
|
const documents = corpusDocuments(value);
|
|
const result = await createRegulatoryRagRuntime().ingestion.ingest(documents);
|
|
console.log(`Indexed ${result.documentCount} regulatory documents as ${result.chunkCount} chunks with ${result.embeddingModel}.`);
|
|
}
|
|
|
|
function corpusDocuments(value: unknown): RegulatoryControlDocument[] {
|
|
if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion)) || !Array.isArray(value.documents) || !value.documents.length) {
|
|
throw new Error('Regulatory corpus must use schemaVersion 1 or 2 and contain a non-empty documents array.');
|
|
}
|
|
if (value.documents.length > 100_000) throw new Error('Regulatory corpus exceeds the 100000 document limit.');
|
|
return value.documents.map(validateRegulatoryControlDocument);
|
|
}
|
|
|
|
function isObject(value: unknown): value is Record<string, unknown> {
|
|
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
}
|
|
|
|
void main().catch(error => {
|
|
console.error(error instanceof Error ? error.message : 'Regulatory ingestion failed.');
|
|
process.exitCode = 1;
|
|
});
|