74 lines
3.6 KiB
TypeScript
74 lines
3.6 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
import {
|
|
evaluateRegulatoryRetriever,
|
|
regulatoryRetrievalEvaluationCases
|
|
} from '../../server/compliance/regulatory-retrieval-evaluation';
|
|
import type { RegulatoryRetrievalCandidate } from '../../src/compliance/regulatory-rag';
|
|
|
|
function candidate(controlId: string, language: string, chunkIndex: number): RegulatoryRetrievalCandidate {
|
|
return {
|
|
score: 1 - chunkIndex / 10,
|
|
chunk: {
|
|
schemaVersion: 1, corpusId: 'synthetic', corpusVersion: '1', documentId: `${controlId}-${language}`,
|
|
framework: 'Synthetic Framework', frameworkVersion: '1', controlId, chunkId: `${controlId}-${language}-${chunkIndex}`,
|
|
chunkIndex, language, title: 'Synthetic control', text: 'Synthetic regulatory text.', textSha256: String(chunkIndex).padStart(64, '0')
|
|
}
|
|
};
|
|
}
|
|
|
|
describe('regulatory retrieval evaluation', () => {
|
|
it('reports control-level cross-language recall, hit rate, and reciprocal rank', async () => {
|
|
const results: Record<string, RegulatoryRetrievalCandidate[]> = {
|
|
'Require multifactor authentication': [candidate('CTRL-X', 'de', 0), candidate('CTRL-MFA', 'de', 1), candidate('CTRL-MFA', 'de', 2)],
|
|
'Passwortrichtlinie durchsetzen': [candidate('CTRL-PASSWORD', 'en', 0)],
|
|
'Require secure audit logging': [candidate('CTRL-X', 'de', 0)]
|
|
};
|
|
const retriever = { retrieve: async ({ query }: { query: string }) => results[query] };
|
|
|
|
const report = await evaluateRegulatoryRetriever(retriever, [
|
|
{
|
|
id: 'en-to-de', query: 'Require multifactor authentication', queryLanguage: 'en',
|
|
expectedControls: [{ framework: 'Synthetic Framework', frameworkVersion: '1', controlId: 'CTRL-MFA', language: 'de' }]
|
|
},
|
|
{
|
|
id: 'de-to-en', query: 'Passwortrichtlinie durchsetzen', queryLanguage: 'de',
|
|
expectedControls: [{ framework: 'Synthetic Framework', controlId: 'CTRL-PASSWORD', language: 'en' }]
|
|
},
|
|
{
|
|
id: 'miss', query: 'Require secure audit logging', queryLanguage: 'en',
|
|
expectedControls: [{ framework: 'Synthetic Framework', controlId: 'CTRL-AUDIT' }]
|
|
}
|
|
], 3);
|
|
|
|
expect(report).toEqual({
|
|
caseCount: 3, limit: 3, hitRateAtK: 2 / 3, recallAtK: 2 / 3, meanReciprocalRank: 0.5,
|
|
cases: [
|
|
{ id: 'en-to-de', queryLanguage: 'en', expectedCount: 1, matchedCount: 1, firstRelevantRank: 2 },
|
|
{ id: 'de-to-en', queryLanguage: 'de', expectedCount: 1, matchedCount: 1, firstRelevantRank: 1 },
|
|
{ id: 'miss', queryLanguage: 'en', expectedCount: 1, matchedCount: 0 }
|
|
]
|
|
});
|
|
});
|
|
|
|
it('rejects incomplete evaluation cases', async () => {
|
|
await expect(evaluateRegulatoryRetriever({ retrieve: async () => [] }, [{
|
|
id: '', query: 'query', queryLanguage: 'en', expectedControls: []
|
|
}], 5)).rejects.toThrow('requires an id');
|
|
});
|
|
|
|
it('validates the bounded versioned CLI input contract', () => {
|
|
expect(regulatoryRetrievalEvaluationCases({
|
|
schemaVersion: 1,
|
|
cases: [{
|
|
id: 'en-to-de', query: 'Require multifactor authentication', queryLanguage: 'en',
|
|
filters: { framework: 'Synthetic Framework' },
|
|
expectedControls: [{ framework: 'Synthetic Framework', controlId: 'CTRL-MFA', language: 'de' }]
|
|
}]
|
|
})).toHaveLength(1);
|
|
expect(() => regulatoryRetrievalEvaluationCases({ schemaVersion: 1, cases: [
|
|
{ id: 'duplicate', query: 'one', queryLanguage: 'en', expectedControls: [{ framework: 'F', controlId: '1' }] },
|
|
{ id: 'duplicate', query: 'two', queryLanguage: 'de', expectedControls: [{ framework: 'F', controlId: '2' }] }
|
|
] })).toThrow('Duplicate');
|
|
});
|
|
});
|