38 lines
1.5 KiB
TypeScript
38 lines
1.5 KiB
TypeScript
import { readFile, stat } from 'node:fs/promises';
|
|
import { resolve } from 'node:path';
|
|
import { createRegulatoryRagRuntime } from './regulatory-rag-runtime';
|
|
import {
|
|
evaluateRegulatoryRetriever,
|
|
regulatoryRetrievalEvaluationCases
|
|
} from './regulatory-retrieval-evaluation';
|
|
|
|
const MAXIMUM_EVALUATION_BYTES = 16 * 1024 * 1024;
|
|
|
|
async function main(): Promise<void> {
|
|
const inputPath = process.argv[2];
|
|
const limit = Number(process.argv[3] ?? 10);
|
|
if (!inputPath) throw new Error('Usage: npm run rag:evaluate -- <evaluation-cases.json> [limit]');
|
|
const resolvedPath = resolve(inputPath);
|
|
const inputStat = await stat(resolvedPath);
|
|
if (!inputStat.isFile() || inputStat.size > MAXIMUM_EVALUATION_BYTES) {
|
|
throw new Error('Regulatory retrieval evaluation input must be a file no larger than 16 MiB.');
|
|
}
|
|
const bytes = await readFile(resolvedPath);
|
|
let value: unknown;
|
|
try { value = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('Regulatory retrieval evaluation input is not valid JSON.'); }
|
|
const cases = regulatoryRetrievalEvaluationCases(value);
|
|
const report = await evaluateRegulatoryRetriever(createRegulatoryRagRuntime().retrieval, cases, limit);
|
|
console.log(JSON.stringify({
|
|
caseCount: report.caseCount,
|
|
limit: report.limit,
|
|
hitRateAtK: report.hitRateAtK,
|
|
recallAtK: report.recallAtK,
|
|
meanReciprocalRank: report.meanReciprocalRank
|
|
}));
|
|
}
|
|
|
|
void main().catch(error => {
|
|
console.error(error instanceof Error ? error.message : 'Regulatory retrieval evaluation failed.');
|
|
process.exitCode = 1;
|
|
});
|