Assessor_2.0/tests/importers/intermediate-model.test.ts

// @vitest-environment jsdom

import { describe, expect, it } from 'vitest';
import * as XLSX from 'xlsx';
import { parseCsvDocument } from '../../src/importers/csv-importer';
import { parseExcelDocument } from '../../src/importers/excel-importer';
import { parseHtmlDocument } from '../../src/importers/html-importer';
import { parseJsonDocument } from '../../src/importers/json-importer';
import { createImportTable } from '../../src/importers/intermediate';
import { importXml, parseXmlDocument } from '../../src/importers/xml-importer';
import { browserMarkupDocumentAdapter } from '../../src/platform/browser/import-runtime-adapters';
import { ImportError } from '../../src/shared/errors';
import { binarySource, sourceMetadata } from './import-source-fixtures';

describe('universal import intermediate model', () => {
  it('represents CSV, HTML and Excel tables with the same downstream structure', async () => {
    const csvText = 'Name;Severity;Weight\nSynthetic finding;High;7';
    const csv = parseCsvDocument(sourceMetadata('findings.csv', csvText.length, 'text/csv'), csvText);

    const htmlText = '<table><caption>Findings</caption><tr><th>Name</th><th>Severity</th><th>Weight</th></tr><tr><td>Synthetic finding</td><td>High</td><td>7</td></tr></table>';
    const html = parseHtmlDocument(
      sourceMetadata('findings.html', htmlText.length, 'text/html'),
      htmlText,
      browserMarkupDocumentAdapter
    );

    const workbook = XLSX.utils.book_new();
    XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet([
      ['Name', 'Severity', 'Weight'],
      ['Synthetic finding', 'High', 7]
    ]), 'Findings');
    const excelBytes = XLSX.write(workbook, { type: 'array', bookType: 'xlsx' });
    const excel = await parseExcelDocument(binarySource('findings.xlsx', new Uint8Array(excelBytes)));

    for (const document of [csv, html, excel]) {
      expect(document.version).toBe(1);
      expect(document.status).toBe('complete');
      expect(document.tables).toHaveLength(1);
      expect(document.tables[0].columns).toHaveLength(3);
      expect(document.tables[0].headerCandidates[0]).toMatchObject({
        rowIndex: 0,
        headers: ['Name', 'Severity', 'Weight']
      });
      expect(document.tables[0].rows[1].cells.map(cell => cell.decodedValue)).toEqual([
        'Synthetic finding',
        'High',
        '7'
      ]);
      expect(document.tables[0].rows[1].cells[2].detectedType).toBe('number');
      expect(document.tables[0].rows[1].cells[2].source).toMatchObject({
        fileName: document.file.name,
        tableIndex: document.tables[0].index,
        rowIndex: 1,
        columnIndex: 2
      });
    }

    expect(csv.tables[0].rows[1].cells[2].rawValue).toBe('7');
    expect(excel.tables[0].rows[1].cells[2].rawValue).toBe(7);
    expect(excel.tables[0].rows[1].cells[2].source.cellReference).toBe('C2');
    expect(html.tables[0].rows[1].cells[2].source.elementSelector).toContain('table:nth-of-type(1)');
    for (const document of [csv, html, excel]) {
      expect(document.views[0]).toMatchObject({
        kind: 'table',
        tableId: document.tables[0].id,
        source: { fileName: document.file.name, tableIndex: document.tables[0].index },
        provenance: { version: 1, processor: { kind: 'format-adapter', version: document.format === 'HTML' ? '1.1.1' : '1.0.0' } }
      });
      expect(document.processing.map(item => item.stage)).toEqual(['format-decoding', 'structure-decoding']);
      expect(document.processing[0]).toMatchObject({ stage: 'format-decoding', status: 'complete', diagnosticCodes: [] });
      expect(document.processing[1]).toMatchObject({ stage: 'structure-decoding', status: document.status });
    }
  });

  it('preserves native object, hierarchy and HTML text views beside compatibility tables', () => {
    const jsonText = JSON.stringify({ metadata: { tenant: 'synthetic' }, findings: [{ id: 'SYN-1', enabled: true }] });
    const json = parseJsonDocument(sourceMetadata('nested.json', jsonText.length, 'application/json'), jsonText);
    const objectView = json.views.find(view => view.kind === 'object');
    expect(objectView).toMatchObject({
      kind: 'object',
      rootKind: 'object',
      value: { metadata: { tenant: 'synthetic' }, findings: [{ id: 'SYN-1', enabled: true }] },
      source: { fileName: 'nested.json' }
    });
    expect(json.views.find(view => view.kind === 'table')?.provenance.transformations).toContainEqual(
      expect.objectContaining({ kind: 'flatten-for-compatibility', lossless: false })
    );

    const xmlText = '<report><finding id="SYN-1"><title>Synthetic</title></finding></report>';
    const xml = parseXmlDocument(sourceMetadata('nested.xml', xmlText.length, 'application/xml'), xmlText, browserMarkupDocumentAdapter);
    const hierarchyView = xml.views.find(view => view.kind === 'hierarchy');
    expect(hierarchyView).toMatchObject({ kind: 'hierarchy', name: 'report', source: { fileName: 'nested.xml' } });
    if (hierarchyView?.kind !== 'hierarchy') throw new Error('Expected hierarchy view.');
    expect(hierarchyView.nodes).toContainEqual(expect.objectContaining({
      path: 'report/finding[1]/title[1]',
      name: 'title',
      value: 'Synthetic',
      source: { fileName: 'nested.xml', objectPath: 'report/finding[1]/title[1]' }
    }));

    const htmlText = '<p>Synthetic introduction</p><table><tr><th>Name</th></tr><tr><td>Finding</td></tr></table>';
    const html = parseHtmlDocument(sourceMetadata('mixed.html', htmlText.length, 'text/html'), htmlText, browserMarkupDocumentAdapter);
    expect(html.views).toContainEqual(expect.objectContaining({ kind: 'table' }));
    expect(html.views).toContainEqual(expect.objectContaining({ kind: 'text', text: expect.stringContaining('Synthetic introduction') }));
  });

  it('returns recoverable parser warnings and discarded content structurally', async () => {
    const csvText = 'Name;Evidence\nFinding;"not closed';
    const csv = parseCsvDocument(sourceMetadata('partial.csv', csvText.length), csvText);
    expect(csv.status).toBe('partial');
    expect(csv.diagnostics).toContainEqual(expect.objectContaining({ code: 'csv-unclosed-quote', severity: 'warning' }));

    const htmlText = '<table><tr><th>Name</th></tr><tr><td><script>evil()</script>Safe</td></tr></table>';
    const html = parseHtmlDocument(
      sourceMetadata('untrusted.html', htmlText.length),
      htmlText,
      browserMarkupDocumentAdapter
    );
    expect(html.status).toBe('partial');
    expect(html.discardedContent).toContainEqual(expect.objectContaining({ kind: 'active-html-content', count: 1 }));

    const workbook = XLSX.utils.book_new();
    XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet([]), 'Empty');
    const excelBytes = XLSX.write(workbook, { type: 'array', bookType: 'xlsx' });
    const excel = await parseExcelDocument(binarySource('empty.xlsx', new Uint8Array(excelBytes)));
    expect(excel.status).toBe('partial');
    expect(excel.discardedContent).toContainEqual(expect.objectContaining({ kind: 'empty-sheet' }));
  });

  it('attaches structured diagnostics to controlled parse errors', () => {
    const text = '<report><finding></report>';
    const source = sourceMetadata('broken.xml', text.length, 'application/xml');

    try {
      importXml(source, text, browserMarkupDocumentAdapter);
      throw new Error('Expected XML import to fail.');
    } catch (error) {
      expect(error).toBeInstanceOf(ImportError);
      expect((error as ImportError).diagnostics).toContainEqual(expect.objectContaining({
        code: 'xml-parse-error',
        severity: 'error'
      }));
    }
  });
  it('builds a supported six-figure row model without spreading rows onto the call stack', () => {
    const rows = Array.from({ length: 140_000 }, () => [] as string[]);
    const table = createImportTable({
      sourceFile: 'large.html',
      name: 'Large table',
      index: 0,
      decodedRows: rows
    });

    expect(table.rows).toHaveLength(140_000);
    expect(table.columns).toHaveLength(0);
  });
});