595 lines
30 KiB
JavaScript
595 lines
30 KiB
JavaScript
import fs from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { createRequire } from 'node:module';
|
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
import pg from 'pg';
|
|
|
|
const { Client } = pg;
|
|
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
|
const projectRoot = path.resolve(scriptDir, '..');
|
|
const defaultBaselinePath = path.join(projectRoot, 'evaluation', 'regulatory-mapping', 'manual-baseline.json');
|
|
const retrievalVersionSourcePath = path.join(projectRoot, 'src', 'domain', 'regulatory-mapping.ts');
|
|
|
|
const colors = {
|
|
navy: '#17365D', paleBlue: '#D9EAF7', blue: '#DDEBF7', white: '#FFFFFF',
|
|
text: '#1F2937', muted: '#5B6573', line: '#D0D7DE', input: '#FFF2CC',
|
|
green: '#E2F0D9', greenText: '#2F6B2F', amber: '#FFF2CC', amberText: '#8A5A00',
|
|
red: '#F4CCCC', redText: '#9C0006', gray: '#E7E6E6'
|
|
};
|
|
const font = 'Arial';
|
|
|
|
function usage() {
|
|
return `Usage: npm run rag:compare-mappings -- [options]
|
|
|
|
Reads regulatory mapping runs from the canonical local PostgreSQL database and
|
|
creates an Excel comparison against the versioned manual evaluation baseline.
|
|
|
|
Options:
|
|
--current-version <version> Version to evaluate. Defaults to the source constant.
|
|
--previous-version <version> Comparison version. Defaults to the latest common prior version.
|
|
--baseline <path> Baseline JSON. Defaults to evaluation/regulatory-mapping/manual-baseline.json.
|
|
--output <path> Output XLSX path. Defaults to outputs/regulatory-mapping/.
|
|
--help Show this help.
|
|
`;
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const options = {};
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const argument = argv[index];
|
|
if (argument === '--help') options.help = true;
|
|
else if (['--current-version', '--previous-version', '--baseline', '--output'].includes(argument)) {
|
|
const value = argv[index + 1];
|
|
if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value.`);
|
|
options[argument.slice(2).replace(/-([a-z])/g, (_, letter) => letter.toUpperCase())] = value;
|
|
index += 1;
|
|
} else throw new Error(`Unknown option: ${argument}`);
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function controlId(value) {
|
|
return String(value ?? '').trim().replace(/^A\./i, '');
|
|
}
|
|
|
|
function displayControl(value) {
|
|
const normalized = controlId(value);
|
|
return normalized ? `A.${normalized}` : 'None';
|
|
}
|
|
|
|
function displayControls(values) {
|
|
return values.length ? values.map(displayControl).join(', ') : 'None';
|
|
}
|
|
|
|
function validateBaseline(value) {
|
|
if (!value || value.schemaVersion !== 1 || !Array.isArray(value.scanners) || !value.scanners.length) {
|
|
throw new Error('The mapping baseline must use schemaVersion 1 and contain scanners.');
|
|
}
|
|
const tools = new Set();
|
|
for (const scanner of value.scanners) {
|
|
if (!scanner.sourceTool || !scanner.label || !Array.isArray(scanner.findings) || !scanner.findings.length) {
|
|
throw new Error('Every baseline scanner requires sourceTool, label and findings.');
|
|
}
|
|
if (tools.has(scanner.sourceTool)) throw new Error(`Duplicate baseline scanner ${scanner.sourceTool}.`);
|
|
tools.add(scanner.sourceTool);
|
|
const rules = new Set();
|
|
for (const finding of scanner.findings) {
|
|
if (!finding.sourceRuleId || rules.has(finding.sourceRuleId)) {
|
|
throw new Error(`Missing or duplicate rule in ${scanner.sourceTool}.`);
|
|
}
|
|
rules.add(finding.sourceRuleId);
|
|
if (finding.primary !== null && finding.primary !== undefined && !/^\d+\.\d+$/.test(controlId(finding.primary))) {
|
|
throw new Error(`Invalid primary control for ${scanner.sourceTool}:${finding.sourceRuleId}.`);
|
|
}
|
|
if (!Array.isArray(finding.supporting)) throw new Error(`Supporting controls must be an array for ${scanner.sourceTool}:${finding.sourceRuleId}.`);
|
|
const relationships = [finding.primary, ...finding.supporting].filter(Boolean).map(controlId);
|
|
if (relationships.some(id => !/^\d+\.\d+$/.test(id)) || new Set(relationships).size !== relationships.length) {
|
|
throw new Error(`Invalid or duplicate controls for ${scanner.sourceTool}:${finding.sourceRuleId}.`);
|
|
}
|
|
finding.primary = finding.primary ? controlId(finding.primary) : null;
|
|
finding.supporting = finding.supporting.map(controlId);
|
|
}
|
|
}
|
|
return value;
|
|
}
|
|
|
|
async function configuredCurrentVersion() {
|
|
const source = await fs.readFile(retrievalVersionSourcePath, 'utf8');
|
|
const match = source.match(/REGULATORY_RETRIEVAL_VERSION\s*=\s*'([^']+)'/);
|
|
if (!match) throw new Error('Could not read REGULATORY_RETRIEVAL_VERSION from the source.');
|
|
return match[1];
|
|
}
|
|
|
|
function loopbackHost(host) {
|
|
return ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(host.toLowerCase());
|
|
}
|
|
|
|
function databaseOptions() {
|
|
const explicit = process.env.ASSESSOR_DATABASE_URL?.trim();
|
|
if (explicit) {
|
|
const url = new URL(explicit);
|
|
if (!['postgres:', 'postgresql:'].includes(url.protocol) || !loopbackHost(url.hostname)) {
|
|
throw new Error('The comparison script only connects to local PostgreSQL.');
|
|
}
|
|
return { connectionString: explicit };
|
|
}
|
|
const host = process.env.ASSESSOR_POSTGRES_HOST?.trim() || '127.0.0.1';
|
|
if (!loopbackHost(host)) throw new Error('The comparison script only connects to local PostgreSQL.');
|
|
const password = process.env.ASSESSOR_POSTGRES_PASSWORD?.trim();
|
|
if (!password) throw new Error('ASSESSOR_POSTGRES_PASSWORD is required. Run through npm so .env.postgres is loaded.');
|
|
return {
|
|
host,
|
|
port: Number(process.env.ASSESSOR_POSTGRES_PORT || 5432),
|
|
database: process.env.ASSESSOR_POSTGRES_DB || 'assessor',
|
|
user: process.env.ASSESSOR_POSTGRES_USER || 'assessor',
|
|
password
|
|
};
|
|
}
|
|
|
|
async function findPreviousVersion(client, sourceTools, currentVersion) {
|
|
const result = await client.query(`
|
|
SELECT retrieval_version, MAX(created_at) AS latest
|
|
FROM assessor_regulatory_mappings
|
|
WHERE source_tool = ANY($1::text[]) AND retrieval_version <> $2
|
|
GROUP BY retrieval_version
|
|
HAVING COUNT(DISTINCT source_tool) = $3
|
|
ORDER BY latest DESC, retrieval_version DESC
|
|
LIMIT 1
|
|
`, [sourceTools, currentVersion, sourceTools.length]);
|
|
if (!result.rowCount) throw new Error('No prior retrieval version covers every baseline scanner. Pass --previous-version explicitly.');
|
|
return result.rows[0].retrieval_version;
|
|
}
|
|
|
|
async function loadRuns(client, sourceTools, versions) {
|
|
const result = await client.query(`
|
|
SELECT source_tool, source_rule_id, retrieval_version, prompt_version, run_status,
|
|
corpus_version, created_at, updated_at, data
|
|
FROM assessor_regulatory_mappings
|
|
WHERE source_tool = ANY($1::text[]) AND retrieval_version = ANY($2::text[])
|
|
ORDER BY updated_at, id
|
|
`, [sourceTools, versions]);
|
|
const runs = new Map();
|
|
for (const row of result.rows) {
|
|
runs.set(`${row.source_tool}|${row.retrieval_version}|${row.source_rule_id}`, row);
|
|
}
|
|
return { rows: result.rows, runs };
|
|
}
|
|
|
|
function expectedRoles(finding) {
|
|
return new Map([
|
|
...(finding.primary ? [[finding.primary, 'primary']] : []),
|
|
...finding.supporting.map(id => [id, 'supporting'])
|
|
]);
|
|
}
|
|
|
|
function evaluateRun(run, finding) {
|
|
const candidates = new Map();
|
|
for (const [index, candidate] of (run?.data?.candidates ?? []).entries()) {
|
|
const id = controlId(candidate.controlId ?? candidate.annexAControlId);
|
|
if (id && !candidates.has(id)) candidates.set(id, { rank: index + 1, score: candidate.score });
|
|
}
|
|
const selected = new Map();
|
|
if (run?.run_status === 'succeeded') {
|
|
for (const mapping of run?.data?.result?.mappings ?? []) {
|
|
const id = controlId(mapping.controlId);
|
|
if (id && ['primary', 'supporting'].includes(mapping.relationship)) {
|
|
selected.set(id, { role: mapping.relationship, confidence: mapping.confidence ?? '' });
|
|
}
|
|
}
|
|
}
|
|
const expected = expectedRoles(finding);
|
|
const expectedIds = [...expected.keys()];
|
|
const retrieved = expectedIds.filter(id => candidates.has(id));
|
|
const selectedExpected = expectedIds.filter(id => selected.has(id));
|
|
const exact = expectedIds.filter(id => selected.get(id)?.role === expected.get(id));
|
|
const retrievalGaps = expectedIds.filter(id => !candidates.has(id));
|
|
const selectionGaps = expectedIds.filter(id => candidates.has(id) && !selected.has(id));
|
|
const roleMismatches = selectedExpected.filter(id => selected.get(id)?.role !== expected.get(id));
|
|
const extras = [...selected.keys()].filter(id => !expected.has(id));
|
|
const primary = [...selected.entries()].find(([, value]) => value.role === 'primary')?.[0] ?? null;
|
|
const supporting = [...selected.entries()].filter(([, value]) => value.role === 'supporting').map(([id]) => id);
|
|
const status = run?.run_status ?? 'missing';
|
|
let verdict;
|
|
if (!run) verdict = 'Missing run';
|
|
else if (status !== 'succeeded') verdict = 'Failed';
|
|
else if (!retrievalGaps.length && !selectionGaps.length && !roleMismatches.length && !extras.length) verdict = 'Exact';
|
|
else if (!retrievalGaps.length && !selectionGaps.length && !roleMismatches.length) verdict = 'Exact + extras';
|
|
else if (!retrievalGaps.length && !selectionGaps.length && roleMismatches.length) verdict = extras.length ? 'Role mismatch + extras' : 'Role mismatch';
|
|
else verdict = extras.length ? 'Incomplete + extras' : 'Incomplete';
|
|
return {
|
|
run, status, candidates, selected, retrieved, selectedExpected, exact,
|
|
retrievalGaps, selectionGaps, roleMismatches, extras, primary, supporting, verdict
|
|
};
|
|
}
|
|
|
|
function changeFor(previous, current) {
|
|
const statusRank = value => value.status === 'succeeded' ? 2 : value.status === 'failed' ? 1 : 0;
|
|
const statusDelta = statusRank(current) - statusRank(previous);
|
|
if (statusDelta > 0) return 'Improved';
|
|
if (statusDelta < 0) return 'Regressed';
|
|
const deltas = [
|
|
current.retrieved.length - previous.retrieved.length,
|
|
current.selectedExpected.length - previous.selectedExpected.length,
|
|
current.exact.length - previous.exact.length,
|
|
previous.extras.length - current.extras.length
|
|
];
|
|
const improved = deltas.some(value => value > 0);
|
|
const regressed = deltas.some(value => value < 0);
|
|
return improved && regressed ? 'Mixed' : improved ? 'Improved' : regressed ? 'Regressed' : 'Unchanged';
|
|
}
|
|
|
|
function relationshipComparison(evaluation, control, manualRole) {
|
|
if (!evaluation.run) return 'Missing run';
|
|
if (!manualRole) return evaluation.selected.has(control) ? 'Extra mapping' : '—';
|
|
if (!evaluation.candidates.has(control)) return 'Retrieval gap';
|
|
if (evaluation.status !== 'succeeded') return 'Run failed after retrieval';
|
|
const selected = evaluation.selected.get(control);
|
|
if (!selected) return 'Selection gap';
|
|
return selected.role === manualRole ? 'Match' : 'Role mismatch';
|
|
}
|
|
|
|
function buildComparison(baseline, runs, previousVersion, currentVersion) {
|
|
const findings = [];
|
|
for (const scanner of baseline.scanners) {
|
|
for (const finding of scanner.findings) {
|
|
const previousRun = runs.get(`${scanner.sourceTool}|${previousVersion}|${finding.sourceRuleId}`);
|
|
const currentRun = runs.get(`${scanner.sourceTool}|${currentVersion}|${finding.sourceRuleId}`);
|
|
const previous = evaluateRun(previousRun, finding);
|
|
const current = evaluateRun(currentRun, finding);
|
|
findings.push({ scanner, finding, previous, current, change: changeFor(previous, current) });
|
|
}
|
|
}
|
|
return findings;
|
|
}
|
|
|
|
function aggregate(items, key) {
|
|
const totals = {
|
|
findings: items.length, succeeded: 0, primaryExpected: 0, primaryRetrieved: 0, primaryExact: 0,
|
|
supportingExpected: 0, supportingRetrieved: 0, supportingExact: 0,
|
|
expected: 0, retrieved: 0, selected: 0, exact: 0, extras: 0
|
|
};
|
|
for (const item of items) {
|
|
const evaluation = item[key];
|
|
const primaryExpected = item.finding.primary ? 1 : 0;
|
|
totals.succeeded += Number(evaluation.status === 'succeeded');
|
|
totals.primaryExpected += primaryExpected;
|
|
totals.primaryRetrieved += Number(Boolean(item.finding.primary && evaluation.candidates.has(item.finding.primary)));
|
|
totals.primaryExact += Number(Boolean(item.finding.primary && evaluation.selected.get(item.finding.primary)?.role === 'primary'));
|
|
totals.supportingExpected += item.finding.supporting.length;
|
|
totals.supportingRetrieved += item.finding.supporting.filter(id => evaluation.candidates.has(id)).length;
|
|
totals.supportingExact += item.finding.supporting.filter(id => evaluation.selected.get(id)?.role === 'supporting').length;
|
|
totals.expected += primaryExpected + item.finding.supporting.length;
|
|
totals.retrieved += evaluation.retrieved.length;
|
|
totals.selected += evaluation.selectedExpected.length;
|
|
totals.exact += evaluation.exact.length;
|
|
totals.extras += evaluation.extras.length;
|
|
}
|
|
return totals;
|
|
}
|
|
|
|
function versionTag(value) {
|
|
const match = value.match(/(\d+\.\d+\.\d+)$/);
|
|
return match ? `v${match[1]}` : value.replace(/[^a-z0-9.-]+/gi, '_');
|
|
}
|
|
|
|
async function loadArtifactTool() {
|
|
const candidates = [
|
|
process.env.ASSESSOR_ARTIFACT_TOOL_NODE_MODULES,
|
|
path.join(os.homedir(), '.cache', 'codex-runtimes', 'codex-primary-runtime', 'dependencies', 'node', 'node_modules')
|
|
].filter(Boolean);
|
|
for (const nodeModules of candidates) {
|
|
try {
|
|
await fs.access(path.join(nodeModules, '@oai', 'artifact-tool', 'package.json'));
|
|
const runtimeRequire = createRequire(path.join(nodeModules, '_assessor-artifact-resolver.cjs'));
|
|
return await import(pathToFileURL(runtimeRequire.resolve('@oai/artifact-tool')).href);
|
|
} catch {
|
|
// Try the next configured runtime.
|
|
}
|
|
}
|
|
throw new Error('Could not locate @oai/artifact-tool. Set ASSESSOR_ARTIFACT_TOOL_NODE_MODULES to the bundled node_modules directory.');
|
|
}
|
|
|
|
function styleHeader(range) {
|
|
range.format = {
|
|
fill: colors.navy,
|
|
font: { name: font, size: 10, bold: true, color: colors.white },
|
|
horizontalAlignment: 'center', verticalAlignment: 'center', wrapText: true,
|
|
borders: { insideVertical: { style: 'thin', color: colors.white }, bottom: { style: 'thin', color: colors.navy } }
|
|
};
|
|
range.format.rowHeight = 30;
|
|
}
|
|
|
|
function titleBlock(sheet, title, subtitle, lastColumn) {
|
|
sheet.getRange('A2').values = [[title]];
|
|
sheet.getRange('A2').format.font = { name: font, size: 14, bold: true, color: colors.navy };
|
|
sheet.getRange('A3').values = [[subtitle]];
|
|
sheet.getRange(`A3:${lastColumn}3`).format.font = { name: font, size: 10, italic: true, color: colors.muted };
|
|
sheet.getRange(`A4:${lastColumn}4`).format.borders = { bottom: { style: 'thin', color: colors.navy } };
|
|
}
|
|
|
|
function applyBase(sheet, range) {
|
|
sheet.getRange(range).format.font = { name: font, size: 10, color: colors.text };
|
|
sheet.getRange(range).format.verticalAlignment = 'center';
|
|
}
|
|
|
|
function applyWidths(sheet, widths, rowCount) {
|
|
widths.forEach((width, index) => {
|
|
sheet.getRangeByIndexes(0, index, Math.max(rowCount, 1), 1).format.columnWidth = width;
|
|
});
|
|
}
|
|
|
|
async function buildWorkbook(artifactTool, comparison, metadata) {
|
|
const { SpreadsheetFile, Workbook } = artifactTool;
|
|
const workbook = Workbook.create();
|
|
const summary = workbook.worksheets.add('Summary');
|
|
const findingSheet = workbook.worksheets.add('Finding comparison');
|
|
const relationshipSheet = workbook.worksheets.add('Relationship detail');
|
|
const method = workbook.worksheets.add('Method');
|
|
for (const sheet of [summary, findingSheet, relationshipSheet, method]) sheet.showGridLines = false;
|
|
summary.tabColor = colors.navy;
|
|
findingSheet.tabColor = '#5B9BD5';
|
|
relationshipSheet.tabColor = '#ED7D31';
|
|
method.tabColor = '#7F8C8D';
|
|
|
|
const groups = [
|
|
...metadata.baseline.scanners.map(scanner => ({ label: scanner.label, items: comparison.filter(item => item.scanner.sourceTool === scanner.sourceTool) })),
|
|
{ label: 'All scanners', items: comparison }
|
|
];
|
|
const summaryRows = [];
|
|
const metrics = [
|
|
['Runs succeeded', 'findings', 'succeeded'],
|
|
['Primary controls retrieved', 'primaryExpected', 'primaryRetrieved'],
|
|
['Primary controls exact role', 'primaryExpected', 'primaryExact'],
|
|
['Supporting controls retrieved', 'supportingExpected', 'supportingRetrieved'],
|
|
['Supporting controls exact role', 'supportingExpected', 'supportingExact'],
|
|
['All expected relationships retrieved', 'expected', 'retrieved'],
|
|
['All expected relationships selected', 'expected', 'selected'],
|
|
['All expected relationships exact role', 'expected', 'exact'],
|
|
['Extra selected relationships', null, 'extras']
|
|
];
|
|
for (const group of groups) {
|
|
const previous = aggregate(group.items, 'previous');
|
|
const current = aggregate(group.items, 'current');
|
|
for (const [label, denominatorKey, valueKey] of metrics) {
|
|
const expected = denominatorKey ? previous[denominatorKey] : null;
|
|
summaryRows.push([
|
|
group.label, label, expected, previous[valueKey], expected ? previous[valueKey] / expected : null,
|
|
current[valueKey], expected ? current[valueKey] / expected : null, current[valueKey] - previous[valueKey]
|
|
]);
|
|
}
|
|
}
|
|
titleBlock(summary, 'ISO mapping comparison', `${metadata.previousVersion} compared with ${metadata.currentVersion}.`, 'N');
|
|
summary.getRange('A6:H6').values = [[
|
|
'Scanner', 'Metric', 'Expected', 'Previous', 'Previous rate', 'Current', 'Current rate', 'Delta'
|
|
]];
|
|
styleHeader(summary.getRange('A6:H6'));
|
|
summary.getRange('A7').write(summaryRows);
|
|
const summaryEnd = 6 + summaryRows.length;
|
|
summary.getRange(`E7:E${summaryEnd}`).format.numberFormat = '0.0%';
|
|
summary.getRange(`G7:G${summaryEnd}`).format.numberFormat = '0.0%';
|
|
summary.getRange(`A7:H${summaryEnd}`).format.rowHeight = 22;
|
|
summary.getRange(`A7:H${summaryEnd}`).format.wrapText = true;
|
|
summary.tables.add(`A6:H${summaryEnd}`, true, 'MappingMetricComparison').style = 'TableStyleMedium2';
|
|
|
|
summary.getRange('J6:N6').values = [['Scanner', 'Improved', 'Regressed', 'Mixed', 'Unchanged']];
|
|
styleHeader(summary.getRange('J6:N6'));
|
|
const changeRows = groups.map(group => [
|
|
group.label,
|
|
group.items.filter(item => item.change === 'Improved').length,
|
|
group.items.filter(item => item.change === 'Regressed').length,
|
|
group.items.filter(item => item.change === 'Mixed').length,
|
|
group.items.filter(item => item.change === 'Unchanged').length
|
|
]);
|
|
summary.getRange('J7').write(changeRows);
|
|
summary.tables.add(`J6:N${6 + changeRows.length}`, true, 'MappingFindingChanges').style = 'TableStyleMedium2';
|
|
applyBase(summary, `A1:N${summaryEnd}`);
|
|
styleHeader(summary.getRange('A6:H6'));
|
|
styleHeader(summary.getRange('J6:N6'));
|
|
applyWidths(summary, [18, 42, 12, 12, 14, 12, 14, 12, 3, 18, 12, 12, 12, 12], summaryEnd);
|
|
|
|
const findingRows = comparison.map(item => [
|
|
item.scanner.label,
|
|
item.finding.sourceRuleId,
|
|
displayControl(item.finding.primary),
|
|
displayControls(item.finding.supporting),
|
|
item.previous.status,
|
|
displayControl(item.previous.primary),
|
|
displayControls(item.previous.supporting),
|
|
item.current.status,
|
|
displayControl(item.current.primary),
|
|
displayControls(item.current.supporting),
|
|
item.previous.verdict,
|
|
item.current.verdict,
|
|
item.change,
|
|
displayControls(item.previous.retrievalGaps),
|
|
displayControls(item.current.retrievalGaps),
|
|
displayControls(item.previous.selectionGaps),
|
|
displayControls(item.current.selectionGaps),
|
|
displayControls(item.previous.roleMismatches),
|
|
displayControls(item.current.roleMismatches),
|
|
displayControls(item.previous.extras),
|
|
displayControls(item.current.extras),
|
|
[...item.current.candidates.keys()].map((id, index) => `${index + 1}. ${displayControl(id)}`).join(', '),
|
|
'',
|
|
''
|
|
]);
|
|
titleBlock(findingSheet, 'Finding comparison', 'One row per baseline rule. Yellow columns are reserved for human review.', 'X');
|
|
const findingHeaders = [
|
|
'Scanner', 'Rule ID', 'Manual primary', 'Manual supporting', 'Previous status', 'Previous primary',
|
|
'Previous supporting', 'Current status', 'Current primary', 'Current supporting', 'Previous verdict',
|
|
'Current verdict', 'Change', 'Previous retrieval gaps', 'Current retrieval gaps',
|
|
'Previous selection gaps', 'Current selection gaps', 'Previous role mismatches', 'Current role mismatches',
|
|
'Previous extras', 'Current extras', 'Current 12-control shortlist', 'Your decision', 'Your notes'
|
|
];
|
|
findingSheet.getRange('A6:X6').values = [findingHeaders];
|
|
styleHeader(findingSheet.getRange('A6:X6'));
|
|
findingSheet.getRange('A7').write(findingRows);
|
|
const findingEnd = 6 + findingRows.length;
|
|
applyBase(findingSheet, `A1:X${findingEnd}`);
|
|
styleHeader(findingSheet.getRange('A6:X6'));
|
|
findingSheet.getRange(`A7:X${findingEnd}`).format.wrapText = true;
|
|
findingSheet.getRange(`A7:X${findingEnd}`).format.verticalAlignment = 'top';
|
|
findingSheet.getRange(`A7:X${findingEnd}`).format.rowHeight = 48;
|
|
findingSheet.getRange(`W7:X${findingEnd}`).format.fill = colors.input;
|
|
findingSheet.getRange(`W7:W${findingEnd}`).dataValidation = { rule: { type: 'list', values: ['Accept', 'Reject', 'Needs discussion'] } };
|
|
const changeRange = findingSheet.getRange(`M7:M${findingEnd}`);
|
|
changeRange.conditionalFormats.add('containsText', { text: 'Improved', format: { fill: colors.green, font: { color: colors.greenText, bold: true } } });
|
|
changeRange.conditionalFormats.add('containsText', { text: 'Regressed', format: { fill: colors.red, font: { color: colors.redText, bold: true } } });
|
|
changeRange.conditionalFormats.add('containsText', { text: 'Mixed', format: { fill: colors.amber, font: { color: colors.amberText, bold: true } } });
|
|
findingSheet.tables.add(`A6:X${findingEnd}`, true, 'MappingFindingComparison').style = 'TableStyleMedium2';
|
|
findingSheet.freezePanes.freezeRows(6);
|
|
findingSheet.freezePanes.freezeColumns(2);
|
|
applyWidths(findingSheet, [16, 18, 18, 30, 16, 18, 30, 16, 18, 30, 22, 22, 14, 28, 28, 28, 28, 28, 28, 28, 28, 72, 20, 40], findingEnd);
|
|
|
|
const relationshipRows = [];
|
|
for (const item of comparison) {
|
|
const manual = expectedRoles(item.finding);
|
|
const controls = [...manual.keys()];
|
|
const extras = [...new Set([...item.previous.selected.keys(), ...item.current.selected.keys()])]
|
|
.filter(id => !manual.has(id))
|
|
.sort((left, right) => left.localeCompare(right, 'en', { numeric: true }));
|
|
for (const id of [...controls, ...extras]) {
|
|
const manualRole = manual.get(id) ?? '';
|
|
const previousSelected = item.previous.selected.get(id);
|
|
const currentSelected = item.current.selected.get(id);
|
|
relationshipRows.push([
|
|
item.scanner.label,
|
|
item.finding.sourceRuleId,
|
|
displayControl(id),
|
|
manualRole || '—',
|
|
previousSelected?.role ?? '—',
|
|
currentSelected?.role ?? '—',
|
|
item.previous.candidates.get(id)?.rank ?? null,
|
|
item.current.candidates.get(id)?.rank ?? null,
|
|
relationshipComparison(item.previous, id, manualRole),
|
|
relationshipComparison(item.current, id, manualRole),
|
|
currentSelected?.confidence ?? '—',
|
|
item.change
|
|
]);
|
|
}
|
|
}
|
|
titleBlock(relationshipSheet, 'Relationship detail', 'Manual, previous and current roles with shortlist availability.', 'L');
|
|
relationshipSheet.getRange('A6:L6').values = [[
|
|
'Scanner', 'Rule ID', 'Control', 'Manual role', 'Previous role', 'Current role',
|
|
'Previous candidate rank', 'Current candidate rank', 'Previous comparison', 'Current comparison',
|
|
'Current confidence', 'Finding change'
|
|
]];
|
|
styleHeader(relationshipSheet.getRange('A6:L6'));
|
|
relationshipSheet.getRange('A7').write(relationshipRows);
|
|
const relationshipEnd = 6 + relationshipRows.length;
|
|
applyBase(relationshipSheet, `A1:L${relationshipEnd}`);
|
|
styleHeader(relationshipSheet.getRange('A6:L6'));
|
|
relationshipSheet.getRange(`A7:L${relationshipEnd}`).format.rowHeight = 24;
|
|
relationshipSheet.getRange(`A7:L${relationshipEnd}`).format.wrapText = true;
|
|
relationshipSheet.tables.add(`A6:L${relationshipEnd}`, true, 'MappingRelationshipDetail').style = 'TableStyleMedium2';
|
|
relationshipSheet.freezePanes.freezeRows(6);
|
|
relationshipSheet.freezePanes.freezeColumns(2);
|
|
applyWidths(relationshipSheet, [16, 18, 14, 16, 16, 16, 18, 18, 26, 26, 18, 16], relationshipEnd);
|
|
|
|
titleBlock(method, 'Method', 'Inputs, version selection and interpretation.', 'H');
|
|
method.getRange('A6:B6').values = [['Item', 'Value']];
|
|
styleHeader(method.getRange('A6:B6'));
|
|
const promptVersions = version => [...new Set(metadata.rows.filter(row => row.retrieval_version === version).map(row => row.prompt_version))].sort().join(', ');
|
|
const corpusVersions = version => [...new Set(metadata.rows.filter(row => row.retrieval_version === version).map(row => row.corpus_version))].sort().join(', ');
|
|
const methodRows = [
|
|
['Generated', new Date(metadata.generatedAt)],
|
|
['Manual baseline', metadata.baseline.baselineId],
|
|
['Previous retrieval version', metadata.previousVersion],
|
|
['Previous prompt version(s)', promptVersions(metadata.previousVersion)],
|
|
['Previous corpus version(s)', corpusVersions(metadata.previousVersion)],
|
|
['Current retrieval version', metadata.currentVersion],
|
|
['Current prompt version(s)', promptVersions(metadata.currentVersion)],
|
|
['Current corpus version(s)', corpusVersions(metadata.currentVersion)],
|
|
['Baseline findings', comparison.length],
|
|
['Baseline relationships', comparison.reduce((total, item) => total + Number(Boolean(item.finding.primary)) + item.finding.supporting.length, 0)],
|
|
['Comparison rule', 'Improved/regressed requires one-sided metric movement. Opposing recall, role or extra-control changes are Mixed.'],
|
|
['Retrieval interpretation', 'Candidate ranks and RRF ordering are comparison signals, not probabilities or confidence percentages.'],
|
|
['Governance', 'This workbook supports human accept/reject review. The manual baseline is not a CI assertion or product rule.'],
|
|
['Privacy', 'The baseline stores scanner rule IDs and control roles only. Finding text and tenant values are excluded from this workbook.']
|
|
];
|
|
method.getRange('A7').write(methodRows);
|
|
const methodEnd = 6 + methodRows.length;
|
|
applyBase(method, `A1:H${methodEnd}`);
|
|
styleHeader(method.getRange('A6:B6'));
|
|
method.getRange(`A7:B${methodEnd}`).format.wrapText = true;
|
|
method.getRange(`A7:B${methodEnd}`).format.rowHeight = 32;
|
|
method.getRange('B7').format.numberFormat = 'yyyy-mm-dd hh:mm';
|
|
method.tables.add(`A6:B${methodEnd}`, true, 'MappingComparisonMethod').style = 'TableStyleMedium2';
|
|
applyWidths(method, [30, 100, 14, 14, 14, 14, 14, 14], methodEnd);
|
|
|
|
workbook.recalculate();
|
|
return { workbook, SpreadsheetFile };
|
|
}
|
|
|
|
async function main() {
|
|
const options = parseArgs(process.argv.slice(2));
|
|
if (options.help) {
|
|
console.log(usage());
|
|
return;
|
|
}
|
|
const baselinePath = path.resolve(projectRoot, options.baseline ?? defaultBaselinePath);
|
|
const baseline = validateBaseline(JSON.parse(await fs.readFile(baselinePath, 'utf8')));
|
|
const currentVersion = options.currentVersion ?? await configuredCurrentVersion();
|
|
const client = new Client(databaseOptions());
|
|
await client.connect();
|
|
try {
|
|
const sourceTools = baseline.scanners.map(scanner => scanner.sourceTool);
|
|
const previousVersion = options.previousVersion ?? await findPreviousVersion(client, sourceTools, currentVersion);
|
|
if (previousVersion === currentVersion) throw new Error('Previous and current retrieval versions must differ.');
|
|
const loaded = await loadRuns(client, sourceTools, [previousVersion, currentVersion]);
|
|
const missingCurrent = baseline.scanners.flatMap(scanner => scanner.findings
|
|
.filter(finding => !loaded.runs.has(`${scanner.sourceTool}|${currentVersion}|${finding.sourceRuleId}`))
|
|
.map(finding => `${scanner.label}:${finding.sourceRuleId}`));
|
|
if (missingCurrent.length) {
|
|
const counts = baseline.scanners.map(scanner => {
|
|
const missing = scanner.findings.filter(finding => !loaded.runs.has(`${scanner.sourceTool}|${currentVersion}|${finding.sourceRuleId}`)).length;
|
|
return `${scanner.label} ${missing}/${scanner.findings.length}`;
|
|
}).join(', ');
|
|
throw new Error(`Current version ${currentVersion} is incomplete (${counts}). Run the mappings before comparing.`);
|
|
}
|
|
const comparison = buildComparison(baseline, loaded.runs, previousVersion, currentVersion);
|
|
const generatedAt = new Date().toISOString();
|
|
const defaultOutput = path.join(
|
|
projectRoot,
|
|
'outputs',
|
|
'regulatory-mapping',
|
|
`ISO_Mapping_${versionTag(currentVersion)}_vs_${versionTag(previousVersion)}_Comparison.xlsx`
|
|
);
|
|
const outputPath = path.resolve(projectRoot, options.output ?? defaultOutput);
|
|
const artifactTool = await loadArtifactTool();
|
|
const built = await buildWorkbook(artifactTool, comparison, {
|
|
baseline, previousVersion, currentVersion, generatedAt, rows: loaded.rows
|
|
});
|
|
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
const output = await built.SpreadsheetFile.exportXlsx(built.workbook);
|
|
await output.save(outputPath);
|
|
await fs.rm(`${outputPath}.inspect.ndjson`, { force: true });
|
|
console.log(`Created ${path.relative(projectRoot, outputPath)}.`);
|
|
for (const scanner of baseline.scanners) {
|
|
const items = comparison.filter(item => item.scanner.sourceTool === scanner.sourceTool);
|
|
const previous = aggregate(items, 'previous');
|
|
const current = aggregate(items, 'current');
|
|
console.log(`${scanner.label}: retrieved ${previous.retrieved}/${previous.expected} -> ${current.retrieved}/${current.expected}; exact roles ${previous.exact}/${previous.expected} -> ${current.exact}/${current.expected}; extras ${previous.extras} -> ${current.extras}.`);
|
|
}
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
|
if (isMain) main().catch(error => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exitCode = 1;
|
|
});
|
|
|
|
export {
|
|
aggregate,
|
|
buildComparison,
|
|
changeFor,
|
|
controlId,
|
|
evaluateRun,
|
|
parseArgs,
|
|
validateBaseline
|
|
};
|