335 lines
14 KiB
TypeScript
335 lines
14 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { readFile, stat } from 'node:fs/promises';
|
|
import { getDocument, PermissionFlag } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
import type {
|
|
RegulatoryControlDocument,
|
|
RegulatoryControlTheme,
|
|
RegulatoryCorpusV2,
|
|
RegulatorySectionKind
|
|
} from '../../src/compliance/regulatory-rag';
|
|
|
|
export const ISO_PDF_PARSER_VERSION = '1.0.0';
|
|
const MAXIMUM_PDF_BYTES = 32 * 1024 * 1024;
|
|
const EXPECTED_THEME_COUNTS: Record<RegulatoryControlTheme, number> = {
|
|
organizational: 37,
|
|
people: 8,
|
|
physical: 14,
|
|
technological: 34
|
|
};
|
|
|
|
export interface PdfSpan {
|
|
text: string;
|
|
page: number;
|
|
x: number;
|
|
y: number;
|
|
width: number;
|
|
height: number;
|
|
}
|
|
|
|
interface PdfJsTextItem {
|
|
str: string;
|
|
transform: unknown[];
|
|
width: number;
|
|
height: number;
|
|
}
|
|
|
|
export interface PdfLine {
|
|
text: string;
|
|
page: number;
|
|
y: number;
|
|
spans: PdfSpan[];
|
|
}
|
|
|
|
export interface ParsedPdf {
|
|
sha256: string;
|
|
pageCount: number;
|
|
pages: Array<{ page: number; lines: PdfLine[] }>;
|
|
}
|
|
|
|
export interface ParsedSection {
|
|
kind: RegulatorySectionKind;
|
|
text: string;
|
|
pageStart: number;
|
|
pageEnd: number;
|
|
}
|
|
|
|
export interface ParsedControl {
|
|
controlId: string;
|
|
title: string;
|
|
theme: RegulatoryControlTheme;
|
|
sections: ParsedSection[];
|
|
}
|
|
|
|
export interface IsoPdfCorpusOptions {
|
|
iso27001Path: string;
|
|
iso27002Path: string;
|
|
corpusId?: string;
|
|
corpusVersion?: string;
|
|
generatedAt?: string;
|
|
}
|
|
|
|
export async function buildIsoPdfCorpus(options: IsoPdfCorpusOptions): Promise<RegulatoryCorpusV2> {
|
|
const [iso27001, iso27002] = await Promise.all([
|
|
extractPdf(options.iso27001Path, 'ISO/IEC 27001'),
|
|
extractPdf(options.iso27002Path, 'ISO/IEC 27002')
|
|
]);
|
|
const references = parseIso27001AnnexA(iso27001);
|
|
const semanticControls = parseIso27002Controls(iso27002);
|
|
validateIsoControlSets(references, semanticControls);
|
|
const corpusId = options.corpusId?.trim() || 'iso-27001-27002-de';
|
|
const combinedHash = sha256(`${iso27001.sha256}\n${iso27002.sha256}`);
|
|
const corpusVersion = options.corpusVersion?.trim() || `din-2024-de-${combinedHash.slice(0, 12)}`;
|
|
const common = { corpusId, corpusVersion, language: 'de', parserVersion: ISO_PDF_PARSER_VERSION } as const;
|
|
const documents: RegulatoryControlDocument[] = [];
|
|
|
|
for (const control of semanticControls) {
|
|
const reference = references.find(item => item.controlId === control.controlId)!;
|
|
documents.push(documentForSection(common, control, reference.sections[0], {
|
|
sourceStandard: 'ISO/IEC 27001', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27001:2024-01',
|
|
sourceSha256: iso27001.sha256, retrievalRole: 'reference'
|
|
}));
|
|
for (const section of control.sections) {
|
|
documents.push(documentForSection(common, control, section, {
|
|
sourceStandard: 'ISO/IEC 27002', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27002:2024-01',
|
|
sourceSha256: iso27002.sha256, retrievalRole: 'semantic'
|
|
}));
|
|
}
|
|
}
|
|
|
|
return {
|
|
schemaVersion: 2,
|
|
corpusId,
|
|
corpusVersion,
|
|
language: 'de',
|
|
parserVersion: ISO_PDF_PARSER_VERSION,
|
|
generatedAt: options.generatedAt ?? new Date().toISOString(),
|
|
sources: [
|
|
{ standard: 'ISO/IEC 27001', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27001:2024-01', sha256: iso27001.sha256, pageCount: iso27001.pageCount },
|
|
{ standard: 'ISO/IEC 27002', sourceEdition: '2022', nationalEdition: 'DIN EN ISO/IEC 27002:2024-01', sha256: iso27002.sha256, pageCount: iso27002.pageCount }
|
|
],
|
|
documents
|
|
};
|
|
}
|
|
|
|
function documentForSection(
|
|
common: { corpusId: string; corpusVersion: string; language: string; parserVersion: string },
|
|
control: ParsedControl,
|
|
section: ParsedSection,
|
|
source: Pick<RegulatoryControlDocument, 'sourceStandard' | 'sourceEdition' | 'nationalEdition' | 'sourceSha256' | 'retrievalRole'>
|
|
): RegulatoryControlDocument {
|
|
const sourceToken = source.sourceStandard === 'ISO/IEC 27001' ? '27001' : '27002';
|
|
return {
|
|
schemaVersion: 2,
|
|
...common,
|
|
documentId: `${common.corpusId}:${control.controlId}:${sourceToken}:${section.kind}`,
|
|
framework: 'ISO/IEC 27001 Annex A',
|
|
frameworkVersion: '2022',
|
|
controlId: control.controlId,
|
|
annexAControlId: `A.${control.controlId}`,
|
|
controlTheme: control.theme,
|
|
title: control.title,
|
|
text: section.text,
|
|
sectionKind: section.kind,
|
|
sourcePageStart: section.pageStart,
|
|
sourcePageEnd: section.pageEnd,
|
|
...source
|
|
};
|
|
}
|
|
|
|
async function extractPdf(path: string, expectedStandard: string): Promise<ParsedPdf> {
|
|
const file = await stat(path);
|
|
if (!file.isFile() || file.size < 1 || file.size > MAXIMUM_PDF_BYTES) {
|
|
throw new Error(`${expectedStandard} PDF must be a non-empty file no larger than 32 MiB.`);
|
|
}
|
|
const bytes = await readFile(path);
|
|
let document;
|
|
try {
|
|
document = await getDocument({ data: new Uint8Array(bytes), password: '', isEvalSupported: false, useSystemFonts: true }).promise;
|
|
} catch {
|
|
throw new Error(`${expectedStandard} PDF could not be opened without a password.`);
|
|
}
|
|
try {
|
|
const permissions = await document.getPermissions();
|
|
if (permissions && !permissions.includes(PermissionFlag.COPY)) throw new Error(`${expectedStandard} PDF does not permit text extraction.`);
|
|
if (document.numPages < 1 || document.numPages > 1_000) throw new Error(`${expectedStandard} PDF page count is outside the supported range.`);
|
|
const pages: ParsedPdf['pages'] = [];
|
|
let characterCount = 0;
|
|
for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {
|
|
const page = await document.getPage(pageNumber);
|
|
const viewport = page.getViewport({ scale: 1 });
|
|
const content = await page.getTextContent({ disableNormalization: false });
|
|
const spans = content.items.flatMap(item => toSpan(item, pageNumber, viewport.height));
|
|
const lines = groupLines(spans);
|
|
characterCount += lines.reduce((sum, line) => sum + line.text.length, 0);
|
|
pages.push({ page: pageNumber, lines });
|
|
page.cleanup();
|
|
}
|
|
if (characterCount < document.numPages * 100) throw new Error(`${expectedStandard} PDF does not contain a usable text layer.`);
|
|
return { sha256: createHash('sha256').update(bytes).digest('hex'), pageCount: document.numPages, pages };
|
|
} finally {
|
|
await document.destroy();
|
|
}
|
|
}
|
|
|
|
function toSpan(item: unknown, page: number, pageHeight: number): PdfSpan[] {
|
|
if (!isTextItem(item)) return [];
|
|
const [a, b, c, d, x, y] = item.transform.map(Number);
|
|
if (![a, b, c, d, x, y].every(Number.isFinite) || Math.abs(b) > 0.2 || Math.abs(c) > 0.2) return [];
|
|
if (y < 70 || y > pageHeight - 55) return [];
|
|
const text = normalizeInline(item.str);
|
|
return text ? [{ text, page, x, y, width: Number(item.width) || 0, height: Number(item.height) || Math.abs(d) }] : [];
|
|
}
|
|
|
|
function groupLines(spans: PdfSpan[]): PdfLine[] {
|
|
const rows: PdfSpan[][] = [];
|
|
for (const span of spans.slice().sort((left, right) => right.y - left.y || left.x - right.x)) {
|
|
const row = rows.find(candidate => Math.abs(candidate[0].y - span.y) <= Math.max(1.5, span.height * 0.12));
|
|
if (row) row.push(span); else rows.push([span]);
|
|
}
|
|
return rows.map(row => {
|
|
const ordered = row.sort((left, right) => left.x - right.x);
|
|
return { page: ordered[0].page, y: ordered[0].y, spans: ordered, text: joinSpans(ordered) };
|
|
}).filter(line => line.text);
|
|
}
|
|
|
|
function joinSpans(spans: PdfSpan[]): string {
|
|
let result = '';
|
|
let previous: PdfSpan | undefined;
|
|
for (const span of spans) {
|
|
const gap = previous ? span.x - (previous.x + previous.width) : 0;
|
|
const separator = result && !/\s$/.test(result) && !/^\s/.test(span.text) && gap > Math.max(1.2, span.height * 0.12) ? ' ' : '';
|
|
result += separator + span.text;
|
|
previous = span;
|
|
}
|
|
return normalizeInline(result);
|
|
}
|
|
|
|
export function parseIso27002Controls(pdf: ParsedPdf): ParsedControl[] {
|
|
const lines = pdf.pages.flatMap(page => page.lines);
|
|
const starts: number[] = [];
|
|
for (let index = 0; index < lines.length; index += 1) {
|
|
if (!/^([5-8]\.\d{1,2})\s+\S/.test(lines[index].text)) continue;
|
|
const nearby = new Set(lines.slice(index + 1, index + 35).map(line => line.text));
|
|
if (nearby.has('Maßnahme') && nearby.has('Zweck') && (nearby.has('Anleitung') || nearby.has('Leitfaden'))) starts.push(index);
|
|
}
|
|
const uniqueStarts = starts.filter((start, index) => !starts.slice(0, index).some(other => controlId(lines[other].text) === controlId(lines[start].text)));
|
|
return uniqueStarts.map((start, index) => parseIso27002Block(lines.slice(start, uniqueStarts[index + 1] ?? lines.length)));
|
|
}
|
|
|
|
function parseIso27002Block(block: PdfLine[]): ParsedControl {
|
|
const match = /^([5-8]\.\d{1,2})\s+(.+)$/.exec(block[0].text);
|
|
if (!match) throw new Error('ISO/IEC 27002 control heading is malformed.');
|
|
const labels: Array<[RegulatorySectionKind, readonly string[]]> = [
|
|
['control', ['Maßnahme']], ['purpose', ['Zweck']], ['guidance', ['Anleitung', 'Leitfaden']], ['other-information', ['Weitere Informationen']]
|
|
];
|
|
const positions = labels.map(([kind, aliases]) => ({ kind, index: block.findIndex(line => aliases.includes(line.text)) }));
|
|
if (positions.some(position => position.index < 0)) throw new Error(`ISO/IEC 27002 control ${match[1]} is missing a required section.`);
|
|
const attributesAt = block.slice(1, positions[0].index).findIndex(line => /Maßnahmenart/.test(line.text));
|
|
if (attributesAt < 0) throw new Error(`ISO/IEC 27002 control ${match[1]} is missing its attribute table.`);
|
|
const titleLines = [match[2], ...block.slice(1, 1 + attributesAt).map(line => line.text)];
|
|
const attributeLines = block.slice(1 + attributesAt, positions[0].index);
|
|
const sections: ParsedSection[] = [sectionFromLines('attributes', attributeLines)];
|
|
positions.forEach((position, positionIndex) => {
|
|
const next = positions[positionIndex + 1]?.index ?? block.length;
|
|
sections.push(sectionFromLines(position.kind, block.slice(position.index + 1, next)));
|
|
});
|
|
return { controlId: match[1], title: linesToText(titleLines), theme: themeFor(match[1]), sections };
|
|
}
|
|
|
|
export function parseIso27001AnnexA(pdf: ParsedPdf): ParsedControl[] {
|
|
const startPageIndex = pdf.pages.findIndex(page => page.lines.some(line => /Tabelle A\.1/.test(line.text))
|
|
&& page.lines.flatMap(line => line.spans).filter(span => span.text === 'Maßnahme').length >= 3);
|
|
if (startPageIndex < 0) throw new Error('ISO/IEC 27001 Annex A table was not found.');
|
|
const controls: Array<{ controlId: string; pageStart: number; pageEnd: number; title: string[]; statement: string[] }> = [];
|
|
let current: (typeof controls)[number] | undefined;
|
|
for (const page of pdf.pages.slice(startPageIndex)) {
|
|
const statementX = Math.min(...page.lines.flatMap(line => line.spans)
|
|
.filter(span => span.text === 'Maßnahme').map(span => span.x));
|
|
if (!Number.isFinite(statementX)) continue;
|
|
for (const line of page.lines) {
|
|
const idSpan = line.spans.find(span => span.x < 125 && /^[5-8]\.\d{1,2}$/.test(span.text));
|
|
if (idSpan) {
|
|
current = { controlId: idSpan.text, pageStart: page.page, pageEnd: page.page, title: [], statement: [] };
|
|
controls.push(current);
|
|
}
|
|
if (!current) continue;
|
|
current.pageEnd = page.page;
|
|
for (const span of line.spans) {
|
|
if (span === idSpan || span.text === 'Maßnahme' || /Tabelle A\.1|Informationssicherheitsmaßnahmen/.test(span.text)) continue;
|
|
if (span.x >= 65 && span.x < statementX) current.title.push(span.text);
|
|
else if (span.x >= statementX) current.statement.push(span.text);
|
|
}
|
|
}
|
|
}
|
|
return controls.map(control => ({
|
|
controlId: control.controlId,
|
|
title: linesToText(control.title),
|
|
theme: themeFor(control.controlId),
|
|
sections: [{
|
|
kind: 'control', text: linesToText(control.statement),
|
|
pageStart: control.pageStart, pageEnd: control.pageEnd
|
|
}]
|
|
}));
|
|
}
|
|
|
|
function sectionFromLines(kind: RegulatorySectionKind, lines: PdfLine[]): ParsedSection {
|
|
const meaningful = lines.filter(line => !isFurniture(line.text));
|
|
const text = linesToText(meaningful.map(line => line.text));
|
|
if (!text) throw new Error(`ISO/IEC 27002 ${kind} section is empty.`);
|
|
return { kind, text, pageStart: meaningful[0].page, pageEnd: meaningful.at(-1)!.page };
|
|
}
|
|
|
|
export function validateIsoControlSets(references: ParsedControl[], semantic: ParsedControl[]): void {
|
|
const referenceIds = references.map(item => item.controlId);
|
|
const semanticIds = semantic.map(item => item.controlId);
|
|
for (const [label, ids] of [['ISO/IEC 27001', referenceIds], ['ISO/IEC 27002', semanticIds]] as const) {
|
|
if (ids.length !== 93 || new Set(ids).size !== ids.length) {
|
|
throw new Error(`${label} must contain exactly 93 unique controls; parsed ${ids.length} entries and ${new Set(ids).size} unique identifiers.`);
|
|
}
|
|
const counts = Object.fromEntries(Object.keys(EXPECTED_THEME_COUNTS).map(theme => [theme, ids.filter(id => themeFor(id) === theme).length]));
|
|
for (const [theme, expected] of Object.entries(EXPECTED_THEME_COUNTS)) {
|
|
if (counts[theme] !== expected) throw new Error(`${label} ${theme} control count must be ${expected}.`);
|
|
}
|
|
}
|
|
if (referenceIds.slice().sort().join('\n') !== semanticIds.slice().sort().join('\n')) {
|
|
throw new Error('ISO/IEC 27001 Annex A and ISO/IEC 27002 control identifiers do not match.');
|
|
}
|
|
}
|
|
|
|
function themeFor(controlIdValue: string): RegulatoryControlTheme {
|
|
if (controlIdValue.startsWith('5.')) return 'organizational';
|
|
if (controlIdValue.startsWith('6.')) return 'people';
|
|
if (controlIdValue.startsWith('7.')) return 'physical';
|
|
if (controlIdValue.startsWith('8.')) return 'technological';
|
|
throw new Error(`Unsupported ISO control identifier ${controlIdValue}.`);
|
|
}
|
|
|
|
function controlId(value: string): string | undefined {
|
|
return /^([5-8]\.\d{1,2})\s+/.exec(value)?.[1];
|
|
}
|
|
|
|
function linesToText(lines: string[]): string {
|
|
return lines.map(normalizeInline).filter(Boolean).join('\n')
|
|
.replace(/(\p{L})-\n(?=\p{Ll})/gu, '$1')
|
|
.replace(/\n+/g, ' ')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
function normalizeInline(value: string): string {
|
|
return value.normalize('NFKC').replace(/\u00ad/g, '').replace(/[\u00a0\u2007\u202f]/g, ' ').replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function isFurniture(value: string): boolean {
|
|
return /^(?:DIN EN ISO\/IEC 2700[12]:|EN ISO\/IEC 2700[12]:|Normen-Download-|\d{1,3})/.test(value);
|
|
}
|
|
|
|
function isTextItem(value: unknown): value is PdfJsTextItem {
|
|
return Boolean(value) && typeof value === 'object' && typeof (value as PdfJsTextItem).str === 'string'
|
|
&& Array.isArray((value as PdfJsTextItem).transform) && (value as PdfJsTextItem).transform.length === 6;
|
|
}
|
|
|
|
function sha256(value: string): string {
|
|
return createHash('sha256').update(value, 'utf8').digest('hex');
|
|
}
|