Assessor_2.0/server/compliance/regulatory-retrieval-evaluation.ts

import type {
  RegulatoryRetrievalCandidate,
  RegulatoryRetrievalFilters,
  RegulatoryRetriever
} from '../../src/compliance/regulatory-rag';

export interface RegulatoryExpectedControl {
  framework: string;
  frameworkVersion?: string;
  controlId: string;
  language?: string;
}

export interface RegulatoryRetrievalEvaluationCase {
  id: string;
  query: string;
  queryLanguage: string;
  filters?: RegulatoryRetrievalFilters;
  expectedControls: RegulatoryExpectedControl[];
}

export interface RegulatoryRetrievalEvaluationCaseResult {
  id: string;
  queryLanguage: string;
  expectedCount: number;
  matchedCount: number;
  firstRelevantRank?: number;
}

export interface RegulatoryRetrievalEvaluationReport {
  caseCount: number;
  limit: number;
  hitRateAtK: number;
  recallAtK: number;
  meanReciprocalRank: number;
  cases: RegulatoryRetrievalEvaluationCaseResult[];
}

export function regulatoryRetrievalEvaluationCases(value: unknown): RegulatoryRetrievalEvaluationCase[] {
  if (!isObject(value) || value.schemaVersion !== 1 || !Array.isArray(value.cases)) {
    throw new Error('Regulatory retrieval evaluation input must use schemaVersion 1 and contain a cases array.');
  }
  if (!value.cases.length || value.cases.length > 1_000) {
    throw new Error('Regulatory retrieval evaluation requires between 1 and 1000 cases.');
  }
  const cases = value.cases as RegulatoryRetrievalEvaluationCase[];
  const ids = new Set<string>();
  for (const testCase of cases) {
    validateCase(testCase);
    if (ids.has(testCase.id)) throw new Error(`Duplicate regulatory retrieval evaluation case id: ${testCase.id}.`);
    ids.add(testCase.id);
  }
  return cases;
}

/** Computes control-level metrics while leaving chunk ranking unchanged. */
export async function evaluateRegulatoryRetriever(
  retriever: RegulatoryRetriever,
  cases: RegulatoryRetrievalEvaluationCase[],
  limit: number
): Promise<RegulatoryRetrievalEvaluationReport> {
  if (!cases.length || cases.length > 1_000) throw new Error('Regulatory retrieval evaluation requires between 1 and 1000 cases.');
  if (!Number.isInteger(limit) || limit < 1 || limit > 100) throw new Error('Regulatory retrieval evaluation limit must be between 1 and 100.');
  const results: RegulatoryRetrievalEvaluationCaseResult[] = [];
  let matchedTotal = 0;
  let expectedTotal = 0;
  let hitTotal = 0;
  let reciprocalRankTotal = 0;
  for (const testCase of cases) {
    validateCase(testCase);
    const candidates = await retriever.retrieve({ query: testCase.query, limit, filters: testCase.filters });
    const result = evaluateCase(testCase, candidates.slice(0, limit));
    results.push(result);
    matchedTotal += result.matchedCount;
    expectedTotal += result.expectedCount;
    if (result.firstRelevantRank !== undefined) {
      hitTotal += 1;
      reciprocalRankTotal += 1 / result.firstRelevantRank;
    }
  }
  return {
    caseCount: cases.length,
    limit,
    hitRateAtK: hitTotal / cases.length,
    recallAtK: matchedTotal / expectedTotal,
    meanReciprocalRank: reciprocalRankTotal / cases.length,
    cases: results
  };
}

function evaluateCase(
  testCase: RegulatoryRetrievalEvaluationCase,
  candidates: RegulatoryRetrievalCandidate[]
): RegulatoryRetrievalEvaluationCaseResult {
  const expected = new Map(testCase.expectedControls.map(control => [controlKey(control), control]));
  const matched = new Set<string>();
  let firstRelevantRank: number | undefined;
  candidates.forEach((candidate, index) => {
    for (const [key, control] of expected) {
      if (!matchesControl(candidate, control)) continue;
      matched.add(key);
      firstRelevantRank ??= index + 1;
    }
  });
  return {
    id: testCase.id,
    queryLanguage: testCase.queryLanguage.toLowerCase(),
    expectedCount: expected.size,
    matchedCount: matched.size,
    ...(firstRelevantRank === undefined ? {} : { firstRelevantRank })
  };
}

function matchesControl(candidate: RegulatoryRetrievalCandidate, expected: RegulatoryExpectedControl): boolean {
  return fold(candidate.chunk.framework) === fold(expected.framework)
    && (!expected.frameworkVersion || fold(candidate.chunk.frameworkVersion) === fold(expected.frameworkVersion))
    && fold(candidate.chunk.controlId) === fold(expected.controlId)
    && (!expected.language || fold(candidate.chunk.language) === fold(expected.language));
}

function controlKey(control: RegulatoryExpectedControl): string {
  return [control.framework, control.frameworkVersion ?? '', control.controlId, control.language ?? ''].map(fold).join('\0');
}

function validateCase(testCase: RegulatoryRetrievalEvaluationCase): void {
  if (!isObject(testCase)
    || !boundedString(testCase.id, 200)
    || !boundedString(testCase.query, 20_000)
    || !boundedString(testCase.queryLanguage, 35)
    || !Array.isArray(testCase.expectedControls)
    || !testCase.expectedControls.length
    || testCase.expectedControls.length > 100) {
    throw new Error('Every regulatory retrieval evaluation case requires an id, query, queryLanguage, and expected controls.');
  }
  if (testCase.filters !== undefined && (!isObject(testCase.filters)
    || Object.values(testCase.filters).some(value => value !== undefined && !boundedString(value, 500)))) {
    throw new Error(`Regulatory retrieval evaluation case ${testCase.id} has invalid filters.`);
  }
  if (testCase.expectedControls.some(control => !isObject(control)
    || !boundedString(control.framework, 500)
    || !boundedString(control.controlId, 500)
    || (control.frameworkVersion !== undefined && !boundedString(control.frameworkVersion, 500))
    || (control.language !== undefined && !boundedString(control.language, 35)))) {
    throw new Error(`Regulatory retrieval evaluation case ${testCase.id} has invalid expected controls.`);
  }
}

const fold = (value: string): string => value.trim().toLowerCase();
const boundedString = (value: unknown, maximum: number): value is string => typeof value === 'string' && Boolean(value.trim()) && value.length <= maximum;
const isObject = (value: unknown): value is Record<string, unknown> => Boolean(value) && typeof value === 'object' && !Array.isArray(value);