Assessor_2.0/src/importers/engine.ts

import type { ImportedDataset } from '../domain/dataset';
import type { ImportFormat, ImportFormatDetectionResult, ImportDiagnostic } from '../domain/import-intermediate';
import type { ImportProfile } from '../domain/import-profile';
import type { ImportSource } from '../domain/import-source';
import { importSourceExtension } from '../domain/import-source';
import { recognizeImportDocument, selectImportProfile } from '../recognition/recognition-engine';
import { ImportError } from '../shared/errors';
import { importCsv } from './csv-importer';
import { importExcel } from './excel-importer';
import { detectImportFormat, inspectImportBinary } from './format-detection';
import { importHtml } from './html-importer';
import { importJson } from './json-importer';
import { importXml } from './xml-importer';
import { mapFindingCandidates } from './candidate-mapping';
import { makeSheet } from './sheet-factory';
import type { ImportRuntimeAdapters } from './runtime-adapters';
import { extractSourceDocument } from './source-extractor-registry';
import { rowsToCsv } from '../exports/csv-export';
import { cellText } from '../shared/text';
import {
  resolveImportResourcePolicy, validateDecodedTextResources, validateImportDocumentResources,
  validateImportSourceResources, type ImportResourcePolicy
} from './resource-policy';

export interface ImportOptions {
  delimiter?: string;
  profiles: ImportProfile[];
  resourcePolicy?: Partial<ImportResourcePolicy>;
}

function formatResourceKey(format: ImportFormat): string {
  if (format === 'Excel') return 'xlsx';
  return format.toLocaleLowerCase('en-US');
}

function detectionDiagnostics(detection: ImportFormatDetectionResult): ImportDiagnostic[] {
  const diagnostics: ImportDiagnostic[] = [];
  if (detection.conflicts.length) diagnostics.push({
    severity: 'warning',
    code: 'format-evidence-conflict',
    stage: 'format-decoding',
    message: `Inhalt oder Dateisignatur wurde als ${detection.selected} erkannt; abweichende Endungs- oder Medientyphinweise wurden nicht als Parserentscheidung übernommen.`,
    details: {
      selectedFormat: detection.selected,
      conflicts: detection.conflicts.map(item => ({ signal: item.signal, format: item.format ?? null }))
    }
  });
  if (detection.requiresParserValidation) diagnostics.push({
    severity: 'info',
    code: 'format-parser-validation',
    stage: 'format-decoding',
    message: `Die Formaterkennung für ${detection.selected} beruhte auf schwachen Hinweisen und wurde durch den erfolgreichen sicheren Parserlauf validiert.`
  });
  return diagnostics;
}

function attachFormatDetection(
  imported: Omit<ImportedDataset, 'profileId' | 'scanTool'>,
  detection: ImportFormatDetectionResult,
  decodedEncoding?: string
): Omit<ImportedDataset, 'profileId' | 'scanTool'> {
  if (!imported.intermediate) return imported;
  const addedDiagnostics = detectionDiagnostics(detection);
  const diagnostics = [...imported.intermediate.diagnostics, ...addedDiagnostics];
  const status = imported.intermediate.status === 'failed'
    ? 'failed'
    : imported.intermediate.status === 'partial' || addedDiagnostics.some(item => item.severity === 'warning')
      ? 'partial'
      : 'complete';
  const processing = imported.intermediate.processing.map(item => item.stage === 'format-decoding'
    ? { ...item, status: 'complete' as const, diagnosticCodes: addedDiagnostics.map(diagnostic => diagnostic.code) }
    : item);
  return {
    ...imported,
    intermediate: {
      ...imported.intermediate,
      status,
      file: { ...imported.intermediate.file, encoding: decodedEncoding ?? imported.intermediate.file.encoding },
      diagnostics,
      processing,
      formatDetection: detection
    }
  };
}

function formatError(code: string, message: string, details?: Record<string, string>): ImportError {
  return new ImportError(message, [{ severity: 'error', code, message, stage: 'format-decoding', details }]);
}

export async function importSource(
  source: ImportSource,
  options: ImportOptions,
  adapters: ImportRuntimeAdapters
): Promise<ImportedDataset> {
  const extension = importSourceExtension(source.metadata);
  const resourcePolicy = resolveImportResourcePolicy(options.resourcePolicy);
  validateImportSourceResources(source, resourcePolicy);
  let imported: Omit<ImportedDataset, 'profileId' | 'scanTool'>;
  let detection: ImportFormatDetectionResult;
  let decodedEncoding: string | undefined;
  try {
    const binaryInspection = inspectImportBinary(source);
    const binaryDetection = detectImportFormat(source);
    if (binaryInspection.containerKind === 'zip') {
      throw formatError('unsupported-container', 'ZIP-Container ohne erkannte Excel-Workbookstruktur werden noch nicht importiert.', { containerKind: 'zip' });
    }
    if (binaryInspection.binary && binaryInspection.signedFormat !== 'Excel') {
      throw formatError('unsupported-binary-format', 'Die Binärdatei besitzt keine unterstützte und validierte Dateisignatur.');
    }
    if (binaryDetection?.selected === 'Excel' && binaryDetection.structurallyValidated) {
      detection = binaryDetection;
      imported = await importExcel(source);
    } else {
      const decoded = adapters.textDecoder.decode(source.bytes);
      decodedEncoding = decoded.encoding;
      const textDetection = detectImportFormat(source, decoded.text);
      if (!textDetection) throw formatError('unsupported-import-format', 'Das Dateiformat konnte nicht sicher einem unterstützten Format zugeordnet werden.');
      if (textDetection.binary) throw formatError('unsupported-binary-format', 'Die Binärdatei besitzt keine unterstützte und validierte Dateisignatur.');
      if (textDetection.selected === 'Excel') {
        throw formatError('excel-signature-mismatch', 'Eine als Excel deklarierte Datei besitzt keine gültige Excel-Dateisignatur.');
      }
      detection = textDetection;
      validateDecodedTextResources(decoded.text, formatResourceKey(detection.selected), resourcePolicy);
      if (detection.selected === 'JSON') imported = importJson(source.metadata, decoded.text);
      else if (detection.selected === 'XML') imported = importXml(source.metadata, decoded.text, adapters.markup);
      else if (detection.selected === 'HTML') imported = importHtml(source.metadata, decoded.text, adapters.markup);
      else imported = importCsv(source.metadata, decoded.text, options.delimiter);
    }
  } catch (error) {
    if (error instanceof ImportError) throw error;
    const parserMessage = error instanceof Error ? error.message : 'Unbekannter Parserfehler';
    throw new ImportError(`Die Datei „${source.metadata.name}“ konnte nicht verarbeitet werden.`, [{
      severity: 'error',
      code: 'file-parse-error',
      message: parserMessage,
      details: { extension }
    }], { cause: error });
  }
  imported = attachFormatDetection(imported, detection, decodedEncoding);
  if (imported.intermediate) validateImportDocumentResources(imported.intermediate, resourcePolicy);
  if (!options.profiles.length) {
    throw new ImportError('Für den Import ist mindestens ein Profil erforderlich.', [{
      severity: 'error',
      code: 'missing-import-profile',
      message: 'Die Importengine wurde ohne verfügbares Importprofil aufgerufen.'
    }]);
  }
  if (imported.intermediate) {
    const extraction = extractSourceDocument(imported.intermediate);
    const supportingTableIndexBase = Math.max(-1, ...imported.intermediate.tables.map(table => table.index)) + 1;
    const supportingSheets = (extraction.supportingTables ?? []).map((table, index) => {
      const rows = [table.headers, ...table.rows.map(row => row.map(cellText))];
      return makeSheet(table.name, rows, rowsToCsv(rows), source.metadata.name, {
        sourceFile: source.metadata.name,
        sourceSheet: table.name,
        sourceTableIndex: supportingTableIndexBase + index
      });
    });
    imported = {
      ...imported,
      sheets: [...imported.sheets, ...supportingSheets],
      extraction,
      intermediate: {
        ...imported.intermediate,
        diagnostics: [...imported.intermediate.diagnostics, ...extraction.diagnostics],
        processing: [...imported.intermediate.processing, extraction.processing]
      }
    };
  }
  const hasTableContent = imported.sheets.some(sheet => sheet.rows.some(row => row.some(cell => cell.trim())));
  const hasExtractedContent = Boolean(imported.extraction && (
    imported.extraction.candidates.length
    || imported.extraction.reportSections?.length
    || imported.extraction.environmentObservations?.length
  ));
  if (!hasTableContent && !hasExtractedContent) {
    throw new ImportError('Die Datei enthält keine auswertbaren Tabellen.', [{
      severity: 'error',
      code: 'empty-import',
      message: 'Der Parser und die registrierten Quellenextraktoren haben keine auswertbaren Inhalte geliefert.'
    }]);
  }
  const recognition = imported.intermediate ? recognizeImportDocument(imported.intermediate) : undefined;
  const profile = recognition
    ? selectImportProfile(recognition, options.profiles)
    : options.profiles.find(candidate => candidate.definition.fallback) ?? options.profiles[0];
  const mapping = imported.extraction
    ? mapFindingCandidates(imported.extraction, profile, recognition)
    : undefined;
  if (mapping && imported.intermediate) {
    imported = {
      ...imported,
      mapping,
      intermediate: {
        ...imported.intermediate,
        diagnostics: [...imported.intermediate.diagnostics, ...mapping.diagnostics],
        processing: [...imported.intermediate.processing, mapping.processing]
      }
    };
  }
  const sheets = profile.prioritizeSheets(imported.sheets);
  return {
    ...imported,
    sheets,
    profileId: profile.id,
    profileVersion: profile.version,
    scanTool: profile.name,
    recognition,
    mapping
  };
}