252 lines
10 KiB
TypeScript
252 lines
10 KiB
TypeScript
import { detectHeaderRow, uniqueHeaders } from '../analysis/header-detector';
|
|
import type { JsonValue } from '../domain/canonical-model';
|
|
import type { ImportSourceMetadata } from '../domain/import-source';
|
|
import { importSourceExtension } from '../domain/import-source';
|
|
import type {
|
|
DecodedSourceView,
|
|
DiscardedImportContent,
|
|
ImportCell,
|
|
ImportDiagnostic,
|
|
ImportDocument,
|
|
ImportFileMetadata,
|
|
ImportFormat,
|
|
ImportProcessorReference,
|
|
ImportProvenance,
|
|
ImportSourcePosition,
|
|
ImportTransformationRecord,
|
|
ImportTable,
|
|
ImportValueType
|
|
} from '../domain/import-intermediate';
|
|
import type { ImportedDataset } from '../domain/dataset';
|
|
import { stableSheetId } from '../shared/identifiers';
|
|
import { clean, normalizeNumericValue, stringifyCell } from '../shared/text';
|
|
import { makeSheet } from './sheet-factory';
|
|
|
|
export function toImportJsonValue(value: unknown): JsonValue {
|
|
if (value === undefined || value === null) return null;
|
|
if (value instanceof Date) return value.toISOString();
|
|
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') return value;
|
|
if (Array.isArray(value)) return value.map(toImportJsonValue);
|
|
if (typeof value === 'object') {
|
|
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, toImportJsonValue(item)]));
|
|
}
|
|
return String(value);
|
|
}
|
|
|
|
function detectValueType(rawValue: unknown, decodedValue: string): ImportValueType {
|
|
if (rawValue === undefined || rawValue === null || decodedValue === '') return 'empty';
|
|
if (rawValue instanceof Date || /^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2})?/.test(decodedValue)) return 'date';
|
|
if (typeof rawValue === 'number' || normalizeNumericValue(decodedValue) !== null) return 'number';
|
|
if (typeof rawValue === 'boolean' || /^(?:true|false)$/i.test(decodedValue)) return 'boolean';
|
|
if (typeof rawValue === 'object') return 'json';
|
|
return 'string';
|
|
}
|
|
|
|
export function fileMetadata(source: ImportSourceMetadata): ImportFileMetadata {
|
|
return {
|
|
...source,
|
|
extension: importSourceExtension(source)
|
|
};
|
|
}
|
|
function formatAdapter(format: ImportFormat): ImportProcessorReference {
|
|
return {
|
|
kind: 'format-adapter',
|
|
id: `${format.toLocaleLowerCase('en-US')}-format-adapter`,
|
|
version: format === 'HTML' ? '1.1.1' : format === 'XML' ? '1.1.0' : '1.0.0'
|
|
};
|
|
}
|
|
|
|
|
|
function provenance(
|
|
fileName: string,
|
|
processor: ImportProcessorReference,
|
|
source: Omit<ImportProvenance['source'], 'fileName'> = {},
|
|
transformations: ImportTransformationRecord[] = []
|
|
): ImportProvenance {
|
|
return {
|
|
version: 1,
|
|
source: { fileName, ...source },
|
|
processor,
|
|
transformations: [{ id: processor.id, version: processor.version, kind: 'decode', lossless: true }, ...transformations]
|
|
};
|
|
}
|
|
|
|
type AdditionalDecodedView =
|
|
| { kind: 'object'; id: string; name: string; rootKind: 'object' | 'array' | 'primitive'; value: JsonValue; attributes?: Record<string, JsonValue>; source?: Omit<ImportProvenance['source'], 'fileName'> }
|
|
| { kind: 'hierarchy'; id: string; name: string; nodes: Array<{ path: string; name: string; value?: JsonValue; attributes?: Record<string, JsonValue>; source?: Omit<ImportProvenance['source'], 'fileName'> }>; attributes?: Record<string, JsonValue>; source?: Omit<ImportProvenance['source'], 'fileName'> }
|
|
| { kind: 'text'; id: string; name: string; text: string; attributes?: Record<string, JsonValue>; source?: Omit<ImportProvenance['source'], 'fileName'> }
|
|
| { kind: 'container'; id: string; name: string; entries: Array<{ path: string; sizeBytes?: number; mediaType?: string }>; attributes?: Record<string, JsonValue>; source?: Omit<ImportProvenance['source'], 'fileName'> };
|
|
|
|
interface CreateImportTableOptions {
|
|
sourceFile: string;
|
|
name: string;
|
|
index: number;
|
|
decodedRows: unknown[][];
|
|
rawRows?: unknown[][];
|
|
rawText?: string;
|
|
attributes?: Record<string, JsonValue>;
|
|
positionForCell?: (rowIndex: number, columnIndex: number) => Partial<ImportSourcePosition>;
|
|
/** Zeilen, die der Bericht selbst als ausgewertet markiert (siehe `ImportRow.highlighted`). */
|
|
highlightedRows?: boolean[];
|
|
}
|
|
|
|
export function createImportTable(options: CreateImportTableOptions): ImportTable {
|
|
const rawRows = options.rawRows ?? options.decodedRows;
|
|
const width = options.decodedRows.reduce((maximum, row) => Math.max(maximum, row.length),
|
|
rawRows.reduce((maximum, row) => Math.max(maximum, row.length), 0));
|
|
const detectedHeaderIndex = options.decodedRows.length ? detectHeaderRow(options.decodedRows) : -1;
|
|
const headerCandidates = detectedHeaderIndex >= 0
|
|
? [{
|
|
rowIndex: detectedHeaderIndex,
|
|
headers: uniqueHeaders(options.decodedRows[detectedHeaderIndex] ?? []),
|
|
confidence: 0.7,
|
|
reason: 'Strukturell stärkste Kopfzeile der ersten zwölf Zeilen.'
|
|
}]
|
|
: [];
|
|
const rows = options.decodedRows.map((decodedRow, rowIndex) => {
|
|
const rawRow = rawRows[rowIndex] ?? [];
|
|
const cells: ImportCell[] = Array.from({ length: Math.max(decodedRow.length, rawRow.length) }, (_, columnIndex) => {
|
|
const rawValue = rawRow[columnIndex];
|
|
const decodedValue = stringifyCell(decodedRow[columnIndex]);
|
|
return {
|
|
rawValue: toImportJsonValue(rawValue),
|
|
decodedValue,
|
|
detectedType: detectValueType(rawValue, decodedValue),
|
|
source: {
|
|
fileName: options.sourceFile,
|
|
tableIndex: options.index,
|
|
rowIndex,
|
|
columnIndex,
|
|
...options.positionForCell?.(rowIndex, columnIndex)
|
|
}
|
|
};
|
|
});
|
|
return {
|
|
index: rowIndex,
|
|
cells,
|
|
source: { fileName: options.sourceFile, tableIndex: options.index, rowIndex },
|
|
highlighted: options.highlightedRows?.[rowIndex]
|
|
};
|
|
});
|
|
return {
|
|
id: stableSheetId(options.sourceFile, options.name, options.index),
|
|
name: options.name,
|
|
index: options.index,
|
|
columns: Array.from({ length: width }, (_, columnIndex) => ({
|
|
index: columnIndex,
|
|
headerCandidates: headerCandidates.map(candidate => ({
|
|
rowIndex: candidate.rowIndex,
|
|
value: candidate.headers[columnIndex] ?? '',
|
|
confidence: candidate.confidence
|
|
})).filter(candidate => clean(candidate.value) !== '')
|
|
})),
|
|
rows,
|
|
rawText: options.rawText ?? options.decodedRows.map(row => row.map(stringifyCell).join(';')).join('\n'),
|
|
headerCandidates,
|
|
attributes: options.attributes ?? {}
|
|
};
|
|
}
|
|
|
|
interface CreateImportDocumentOptions {
|
|
file: ImportSourceMetadata;
|
|
format: ImportFormat;
|
|
rawText: string;
|
|
tables: ImportTable[];
|
|
diagnostics?: ImportDiagnostic[];
|
|
discardedContent?: DiscardedImportContent[];
|
|
additionalViews?: AdditionalDecodedView[];
|
|
transformations?: ImportTransformationRecord[];
|
|
}
|
|
|
|
export function createImportDocument(options: CreateImportDocumentOptions): ImportDocument {
|
|
const processor = formatAdapter(options.format);
|
|
const diagnostics = (options.diagnostics ?? []).map(diagnostic => ({
|
|
...diagnostic,
|
|
stage: diagnostic.stage ?? 'structure-decoding' as const,
|
|
provenance: diagnostic.provenance ?? provenance(options.file.name, processor)
|
|
}));
|
|
const discardedContent = options.discardedContent ?? [];
|
|
const status = diagnostics.some(item => item.severity === 'error')
|
|
? 'failed'
|
|
: diagnostics.some(item => item.severity === 'warning') || discardedContent.length
|
|
? 'partial'
|
|
: 'complete';
|
|
const commonTransformations = options.transformations ?? [];
|
|
const tableViews: DecodedSourceView[] = options.tables.map(table => ({
|
|
kind: 'table',
|
|
id: `view-${table.id}`,
|
|
name: table.name,
|
|
tableId: table.id,
|
|
source: { fileName: options.file.name, tableIndex: table.index },
|
|
provenance: provenance(options.file.name, processor, { tableIndex: table.index }, [
|
|
...commonTransformations,
|
|
...(table.attributes.compatibilityProjection === true ? [{
|
|
id: 'tabular-compatibility-projection', version: '1.0.0', kind: 'flatten-for-compatibility' as const, lossless: false
|
|
}] : [])
|
|
]),
|
|
attributes: table.attributes
|
|
}));
|
|
const additionalViews: DecodedSourceView[] = (options.additionalViews ?? []).map(view => {
|
|
const source = { fileName: options.file.name, ...view.source };
|
|
const viewProvenance = provenance(options.file.name, processor, view.source, commonTransformations);
|
|
if (view.kind === 'hierarchy') return {
|
|
...view,
|
|
source,
|
|
provenance: viewProvenance,
|
|
attributes: view.attributes ?? {},
|
|
nodes: view.nodes.map(node => ({
|
|
...node,
|
|
attributes: node.attributes ?? {},
|
|
source: { fileName: options.file.name, objectPath: node.path, ...node.source }
|
|
}))
|
|
};
|
|
if (view.kind === 'container') return {
|
|
...view,
|
|
source,
|
|
provenance: viewProvenance,
|
|
attributes: view.attributes ?? {},
|
|
entries: view.entries.map(entry => ({ ...entry, source: { fileName: options.file.name, containerEntryPath: entry.path } }))
|
|
};
|
|
return { ...view, source, provenance: viewProvenance, attributes: view.attributes ?? {} };
|
|
});
|
|
return {
|
|
version: 1,
|
|
status,
|
|
format: options.format,
|
|
file: fileMetadata(options.file),
|
|
rawText: options.rawText,
|
|
tables: options.tables,
|
|
diagnostics,
|
|
discardedContent,
|
|
views: [...tableViews, ...additionalViews],
|
|
processing: [
|
|
{ stage: 'format-decoding', status: status === 'failed' ? 'failed' : 'complete', processor, diagnosticCodes: [] },
|
|
{ stage: 'structure-decoding', status, processor, diagnosticCodes: diagnostics.map(item => item.code) }
|
|
]
|
|
};
|
|
}
|
|
|
|
export function importDocumentToDataset(document: ImportDocument): Omit<ImportedDataset, 'profileId' | 'scanTool'> {
|
|
return {
|
|
fileName: document.file.name,
|
|
fileSize: document.file.sizeBytes,
|
|
lastModified: document.file.lastModified,
|
|
format: document.format,
|
|
rawText: document.rawText,
|
|
sheets: document.tables.map(table => makeSheet(
|
|
table.name,
|
|
table.rows.map(row => row.cells.map(cell => cell.decodedValue)),
|
|
table.rawText,
|
|
document.file.name,
|
|
{
|
|
id: table.id,
|
|
sourceTableIndex: table.index,
|
|
workbookIndex: typeof table.attributes.workbookIndex === 'number' ? table.attributes.workbookIndex : undefined,
|
|
hidden: typeof table.attributes.hidden === 'number' ? table.attributes.hidden : undefined
|
|
},
|
|
table.rows.map(row => row.highlighted ?? false)
|
|
)),
|
|
intermediate: document
|
|
};
|
|
}
|