Assessor_2.0/server/regulatory/regulatory-chunker.ts

import { createHash } from 'node:crypto';
import {
  REGULATORY_CHUNK_SCHEMA_VERSION,
  type RegulatoryChunk,
  type RegulatoryControlDocument
} from '../../src/domain/regulatory-rag';

export interface RegulatoryChunkingOptions {
  maximumCharacters: number;
}

export function validateRegulatoryControlDocument(value: unknown): RegulatoryControlDocument {
  if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion))) throw new Error('Regulatory documents must use schemaVersion 1 or 2.');
  const required = ['corpusId', 'corpusVersion', 'documentId', 'framework', 'frameworkVersion', 'controlId', 'language', 'title', 'text'] as const;
  const document = value as Record<string, unknown>;
  for (const field of required) {
    if (typeof document[field] !== 'string' || !document[field].trim()) throw new Error(`Regulatory document ${field} must be a non-empty string.`);
  }
  if (!/^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$/.test(String(document.language))) {
    throw new Error('Regulatory document language must be a BCP 47 language tag.');
  }
  if (document.sourceUri !== undefined && (typeof document.sourceUri !== 'string' || !document.sourceUri.trim())) {
    throw new Error('Regulatory document sourceUri must be a non-empty string when present.');
  }
  const schemaVersion = Number(document.schemaVersion) as 1 | 2;
  const result: RegulatoryControlDocument = {
    schemaVersion,
    corpusId: String(document.corpusId).trim(),
    corpusVersion: String(document.corpusVersion).trim(),
    documentId: String(document.documentId).trim(),
    framework: String(document.framework).trim(),
    frameworkVersion: String(document.frameworkVersion).trim(),
    controlId: String(document.controlId).trim(),
    language: String(document.language).trim().toLowerCase(),
    title: String(document.title).trim(),
    text: normalizeText(String(document.text)),
    ...(document.sourceUri === undefined ? {} : { sourceUri: String(document.sourceUri).trim() })
  };
  if (schemaVersion === 2) Object.assign(result, validateVersionTwoFields(document, result.controlId));
  return result;
}

export function chunkRegulatoryControl(
  input: RegulatoryControlDocument,
  options: RegulatoryChunkingOptions
): RegulatoryChunk[] {
  const document = validateRegulatoryControlDocument(input);
  if (!Number.isInteger(options.maximumCharacters) || options.maximumCharacters < 200 || options.maximumCharacters > 20_000) {
    throw new Error('maximumCharacters must be an integer between 200 and 20000.');
  }
  const parts = splitText(document.text, options.maximumCharacters);
  return parts.map((text, chunkIndex) => {
    const textSha256 = sha256(text);
    const identity = sha256([
      String(REGULATORY_CHUNK_SCHEMA_VERSION), document.corpusId, document.corpusVersion,
      document.documentId, String(chunkIndex), textSha256
    ].join('\n'));
    return {
      ...document,
      chunkId: `${document.documentId}:${chunkIndex}:${identity.slice(0, 12)}`,
      chunkIndex,
      text,
      textSha256
    };
  });
}

export function regulatoryChunkEmbeddingText(chunk: RegulatoryChunk): string {
  const context = [
    `${chunk.framework} ${chunk.frameworkVersion}`,
    chunk.annexAControlId ? `${chunk.controlId} (${chunk.annexAControlId})` : chunk.controlId,
    chunk.controlTheme,
    chunk.sectionKind,
    chunk.title
  ].filter(Boolean).join(' | ');
  return `${context}\n\n${chunk.text}`;
}

function validateVersionTwoFields(document: Record<string, unknown>, controlId: string): Partial<RegulatoryControlDocument> {
  if (!/^[5-8]\.\d{1,2}$/.test(controlId)) throw new Error('Regulatory schemaVersion 2 controlId must use the canonical 5.x-8.x form.');
  const annexAControlId = requiredEnumLike(document, 'annexAControlId');
  if (annexAControlId !== `A.${controlId}`) throw new Error(`Regulatory annexAControlId must be A.${controlId}.`);
  const controlTheme = requiredEnum(document, 'controlTheme', ['organizational', 'people', 'physical', 'technological'] as const);
  const sourceStandard = requiredEnum(document, 'sourceStandard', ['ISO/IEC 27001', 'ISO/IEC 27002'] as const);
  const sectionKind = requiredEnum(document, 'sectionKind', ['attributes', 'control', 'purpose', 'guidance', 'other-information'] as const);
  const retrievalRole = requiredEnum(document, 'retrievalRole', ['semantic', 'reference'] as const);
  const sourceEdition = requiredEnumLike(document, 'sourceEdition');
  const nationalEdition = requiredEnumLike(document, 'nationalEdition');
  const parserVersion = requiredEnumLike(document, 'parserVersion');
  const sourceSha256 = requiredEnumLike(document, 'sourceSha256');
  if (!/^[a-f0-9]{64}$/.test(sourceSha256)) throw new Error('Regulatory sourceSha256 must be a lower-case SHA-256 digest.');
  const sourcePageStart = requiredPage(document, 'sourcePageStart');
  const sourcePageEnd = requiredPage(document, 'sourcePageEnd');
  if (sourcePageEnd < sourcePageStart) throw new Error('Regulatory sourcePageEnd must not precede sourcePageStart.');
  if (sourceStandard === 'ISO/IEC 27001' && (sectionKind !== 'control' || retrievalRole !== 'reference')) {
    throw new Error('ISO/IEC 27001 schemaVersion 2 documents must be reference control statements.');
  }
  if (sourceStandard === 'ISO/IEC 27002' && retrievalRole !== 'semantic') {
    throw new Error('ISO/IEC 27002 schemaVersion 2 documents must be semantic evidence.');
  }
  return {
    annexAControlId, controlTheme, sourceStandard, sourceEdition, nationalEdition,
    sectionKind, sourcePageStart, sourcePageEnd, retrievalRole, parserVersion, sourceSha256
  };
}

function requiredEnumLike(document: Record<string, unknown>, field: string): string {
  const value = document[field];
  if (typeof value !== 'string' || !value.trim()) throw new Error(`Regulatory schemaVersion 2 ${field} must be a non-empty string.`);
  return value.trim();
}

function requiredEnum<const T extends readonly string[]>(document: Record<string, unknown>, field: string, values: T): T[number] {
  const value = requiredEnumLike(document, field);
  if (!values.includes(value)) throw new Error(`Regulatory schemaVersion 2 ${field} is not supported.`);
  return value as T[number];
}

function requiredPage(document: Record<string, unknown>, field: string): number {
  const value = document[field];
  if (!Number.isInteger(value) || Number(value) < 1 || Number(value) > 10_000) throw new Error(`Regulatory schemaVersion 2 ${field} must be a positive page number.`);
  return Number(value);
}

function splitText(text: string, maximumCharacters: number): string[] {
  const paragraphs = text.split(/\n{2,}/).map(value => value.trim()).filter(Boolean);
  const units = paragraphs.flatMap(paragraph => splitLongUnit(paragraph, maximumCharacters));
  const chunks: string[] = [];
  let current = '';
  for (const unit of units) {
    const candidate = current ? `${current}\n\n${unit}` : unit;
    if (candidate.length <= maximumCharacters) {
      current = candidate;
      continue;
    }
    if (current) chunks.push(current);
    current = unit;
  }
  if (current) chunks.push(current);
  return chunks;
}

function splitLongUnit(value: string, maximumCharacters: number): string[] {
  if (value.length <= maximumCharacters) return [value];
  const result: string[] = [];
  let remaining = value;
  while (remaining.length > maximumCharacters) {
    const boundary = Math.max(
      remaining.lastIndexOf('. ', maximumCharacters),
      remaining.lastIndexOf('; ', maximumCharacters),
      remaining.lastIndexOf(' ', maximumCharacters)
    );
    const splitAt = boundary >= Math.floor(maximumCharacters * 0.6) ? boundary + 1 : maximumCharacters;
    result.push(remaining.slice(0, splitAt).trim());
    remaining = remaining.slice(splitAt).trim();
  }
  if (remaining) result.push(remaining);
  return result;
}

function normalizeText(value: string): string {
  return value.replace(/\r\n?/g, '\n').replace(/[ \t]+\n/g, '\n').trim();
}

function sha256(value: string): string {
  return createHash('sha256').update(value, 'utf8').digest('hex');
}

function isObject(value: unknown): value is Record<string, unknown> {
  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
}