Assessor_2.0/server/regulatory/regulatory-chunker.ts

import { createHash } from 'node:crypto';
import {
  REGULATORY_CHUNK_SCHEMA_VERSION,
  type RegulatoryChunk,
  type RegulatoryControlDocument
} from '../../src/domain/regulatory-rag';

export interface RegulatoryChunkingOptions {
  maximumCharacters: number;
}

export function validateRegulatoryControlDocument(value: unknown): RegulatoryControlDocument {
  if (!isObject(value) || value.schemaVersion !== 1) throw new Error('Regulatory documents must use schemaVersion 1.');
  const required = ['corpusId', 'corpusVersion', 'documentId', 'framework', 'frameworkVersion', 'controlId', 'language', 'title', 'text'] as const;
  const document = value as Record<string, unknown>;
  for (const field of required) {
    if (typeof document[field] !== 'string' || !document[field].trim()) throw new Error(`Regulatory document ${field} must be a non-empty string.`);
  }
  if (!/^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$/.test(String(document.language))) {
    throw new Error('Regulatory document language must be a BCP 47 language tag.');
  }
  if (document.sourceUri !== undefined && (typeof document.sourceUri !== 'string' || !document.sourceUri.trim())) {
    throw new Error('Regulatory document sourceUri must be a non-empty string when present.');
  }
  return {
    schemaVersion: 1,
    corpusId: String(document.corpusId).trim(),
    corpusVersion: String(document.corpusVersion).trim(),
    documentId: String(document.documentId).trim(),
    framework: String(document.framework).trim(),
    frameworkVersion: String(document.frameworkVersion).trim(),
    controlId: String(document.controlId).trim(),
    language: String(document.language).trim().toLowerCase(),
    title: String(document.title).trim(),
    text: normalizeText(String(document.text)),
    ...(document.sourceUri === undefined ? {} : { sourceUri: String(document.sourceUri).trim() })
  };
}

export function chunkRegulatoryControl(
  input: RegulatoryControlDocument,
  options: RegulatoryChunkingOptions
): RegulatoryChunk[] {
  const document = validateRegulatoryControlDocument(input);
  if (!Number.isInteger(options.maximumCharacters) || options.maximumCharacters < 200 || options.maximumCharacters > 20_000) {
    throw new Error('maximumCharacters must be an integer between 200 and 20000.');
  }
  const parts = splitText(document.text, options.maximumCharacters);
  return parts.map((text, chunkIndex) => {
    const textSha256 = sha256(text);
    const identity = sha256([
      String(REGULATORY_CHUNK_SCHEMA_VERSION), document.corpusId, document.corpusVersion,
      document.documentId, String(chunkIndex), textSha256
    ].join('\n'));
    return {
      ...document,
      chunkId: `${document.documentId}:${chunkIndex}:${identity.slice(0, 12)}`,
      chunkIndex,
      text,
      textSha256
    };
  });
}

export function regulatoryChunkEmbeddingText(chunk: RegulatoryChunk): string {
  return `${chunk.framework} ${chunk.frameworkVersion} | ${chunk.controlId} | ${chunk.title}\n\n${chunk.text}`;
}

function splitText(text: string, maximumCharacters: number): string[] {
  const paragraphs = text.split(/\n{2,}/).map(value => value.trim()).filter(Boolean);
  const units = paragraphs.flatMap(paragraph => splitLongUnit(paragraph, maximumCharacters));
  const chunks: string[] = [];
  let current = '';
  for (const unit of units) {
    const candidate = current ? `${current}\n\n${unit}` : unit;
    if (candidate.length <= maximumCharacters) {
      current = candidate;
      continue;
    }
    if (current) chunks.push(current);
    current = unit;
  }
  if (current) chunks.push(current);
  return chunks;
}

function splitLongUnit(value: string, maximumCharacters: number): string[] {
  if (value.length <= maximumCharacters) return [value];
  const result: string[] = [];
  let remaining = value;
  while (remaining.length > maximumCharacters) {
    const boundary = Math.max(
      remaining.lastIndexOf('. ', maximumCharacters),
      remaining.lastIndexOf('; ', maximumCharacters),
      remaining.lastIndexOf(' ', maximumCharacters)
    );
    const splitAt = boundary >= Math.floor(maximumCharacters * 0.6) ? boundary + 1 : maximumCharacters;
    result.push(remaining.slice(0, splitAt).trim());
    remaining = remaining.slice(splitAt).trim();
  }
  if (remaining) result.push(remaining);
  return result;
}

function normalizeText(value: string): string {
  return value.replace(/\r\n?/g, '\n').replace(/[ \t]+\n/g, '\n').trim();
}

function sha256(value: string): string {
  return createHash('sha256').update(value, 'utf8').digest('hex');
}

function isObject(value: unknown): value is Record<string, unknown> {
  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
}