80 lines
3.0 KiB
TypeScript
80 lines
3.0 KiB
TypeScript
import { createRequire } from 'node:module';
|
|
import type {
|
|
DecodedHtmlDocument,
|
|
DecodedXmlDocument,
|
|
DecodedXmlRow,
|
|
ImportRuntimeAdapters,
|
|
MarkupDocumentAdapter,
|
|
TextDecoderAdapter
|
|
} from '../src/importers/runtime-adapters';
|
|
import {
|
|
decodeHtmlDocument,
|
|
decodeHtmlLexically,
|
|
isHtmlStackOverflow,
|
|
shouldUseBoundedHtmlExtraction
|
|
} from '../src/importers/html-extraction';
|
|
|
|
// A worker gets a fresh module graph for every source. Load the fixed local
|
|
// DOM dependency only when markup actually needs it, not for CSV/JSON/Excel.
|
|
const requireMarkup = createRequire(import.meta.url);
|
|
function markupConstructor(): typeof import('jsdom').JSDOM {
|
|
return (requireMarkup('jsdom') as typeof import('jsdom')).JSDOM;
|
|
}
|
|
|
|
function walkXml(node: Element, path: string, rows: DecodedXmlRow[]): void {
|
|
const attributes = [...node.attributes].map(attribute => `${attribute.name}="${attribute.value}"`).join('; ');
|
|
const attributeValues = Object.fromEntries([...node.attributes].map(attribute => [attribute.name, attribute.value]));
|
|
const text = [...node.childNodes]
|
|
.filter(child => child.nodeType === 3 || child.nodeType === 4)
|
|
.map(child => child.textContent?.trim() ?? '')
|
|
.filter(Boolean)
|
|
.join(' ');
|
|
rows.push({ path, attributes, attributeValues, text });
|
|
[...node.children].forEach((child, index) => walkXml(child, `${path}/${child.nodeName}[${index + 1}]`, rows));
|
|
}
|
|
|
|
export const nodeMarkupDocumentAdapter: MarkupDocumentAdapter = {
|
|
decodeHtml(text: string): DecodedHtmlDocument {
|
|
if (shouldUseBoundedHtmlExtraction(text)) return decodeHtmlLexically(text);
|
|
try {
|
|
const JSDOM = markupConstructor();
|
|
const documentNode = new JSDOM(text, { contentType: 'text/html' }).window.document;
|
|
return decodeHtmlDocument(documentNode);
|
|
} catch (error) {
|
|
if (isHtmlStackOverflow(error)) return decodeHtmlLexically(text);
|
|
throw error;
|
|
}
|
|
},
|
|
|
|
decodeXml(text: string): DecodedXmlDocument {
|
|
if (/<!DOCTYPE\b|<!ENTITY\b|\bSYSTEM\s+["']|\bPUBLIC\s+["']/i.test(text)) {
|
|
return { valid: false, message: 'XML-Dokumenttyp-, Entitäts- und externe Referenzdeklarationen sind nicht zulässig. xml-doctype-rejected-by-adapter' };
|
|
}
|
|
try {
|
|
const JSDOM = markupConstructor();
|
|
const documentNode = new JSDOM(text, { contentType: 'application/xml' }).window.document;
|
|
const root = documentNode.documentElement;
|
|
const rows: DecodedXmlRow[] = [];
|
|
walkXml(root, root.nodeName, rows);
|
|
return { valid: true, rootName: root.nodeName, rows };
|
|
} catch {
|
|
return { valid: false, message: 'Die XML-Struktur ist nicht wohlgeformt.' };
|
|
}
|
|
}
|
|
};
|
|
|
|
export const nodeTextDecoderAdapter: TextDecoderAdapter = {
|
|
decode(bytes) {
|
|
try {
|
|
return { text: new TextDecoder('utf-8', { fatal: true }).decode(bytes), encoding: 'utf-8' };
|
|
} catch {
|
|
return { text: new TextDecoder('windows-1252').decode(bytes), encoding: 'windows-1252' };
|
|
}
|
|
}
|
|
};
|
|
|
|
export const nodeImportRuntimeAdapters: ImportRuntimeAdapters = {
|
|
textDecoder: nodeTextDecoderAdapter,
|
|
markup: nodeMarkupDocumentAdapter
|
|
};
|