import { createHash } from 'node:crypto'; import { REGULATORY_CHUNK_SCHEMA_VERSION, type RegulatoryChunk, type RegulatoryControlDocument } from '../../src/domain/regulatory-rag'; export interface RegulatoryChunkingOptions { maximumCharacters: number; } export function validateRegulatoryControlDocument(value: unknown): RegulatoryControlDocument { if (!isObject(value) || ![1, 2].includes(Number(value.schemaVersion))) throw new Error('Regulatory documents must use schemaVersion 1 or 2.'); const required = ['corpusId', 'corpusVersion', 'documentId', 'framework', 'frameworkVersion', 'controlId', 'language', 'title', 'text'] as const; const document = value as Record; for (const field of required) { if (typeof document[field] !== 'string' || !document[field].trim()) throw new Error(`Regulatory document ${field} must be a non-empty string.`); } if (!/^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$/.test(String(document.language))) { throw new Error('Regulatory document language must be a BCP 47 language tag.'); } if (document.sourceUri !== undefined && (typeof document.sourceUri !== 'string' || !document.sourceUri.trim())) { throw new Error('Regulatory document sourceUri must be a non-empty string when present.'); } const schemaVersion = Number(document.schemaVersion) as 1 | 2; const result: RegulatoryControlDocument = { schemaVersion, corpusId: String(document.corpusId).trim(), corpusVersion: String(document.corpusVersion).trim(), documentId: String(document.documentId).trim(), framework: String(document.framework).trim(), frameworkVersion: String(document.frameworkVersion).trim(), controlId: String(document.controlId).trim(), language: String(document.language).trim().toLowerCase(), title: String(document.title).trim(), text: normalizeText(String(document.text)), ...(document.sourceUri === undefined ? {} : { sourceUri: String(document.sourceUri).trim() }) }; if (schemaVersion === 2) Object.assign(result, validateVersionTwoFields(document, result.controlId)); return result; } export function chunkRegulatoryControl( input: RegulatoryControlDocument, options: RegulatoryChunkingOptions ): RegulatoryChunk[] { const document = validateRegulatoryControlDocument(input); if (!Number.isInteger(options.maximumCharacters) || options.maximumCharacters < 200 || options.maximumCharacters > 20_000) { throw new Error('maximumCharacters must be an integer between 200 and 20000.'); } const parts = splitText(document.text, options.maximumCharacters); return parts.map((text, chunkIndex) => { const textSha256 = sha256(text); const identity = sha256([ String(REGULATORY_CHUNK_SCHEMA_VERSION), document.corpusId, document.corpusVersion, document.documentId, String(chunkIndex), textSha256 ].join('\n')); return { ...document, chunkId: `${document.documentId}:${chunkIndex}:${identity.slice(0, 12)}`, chunkIndex, text, textSha256 }; }); } export function regulatoryChunkEmbeddingText(chunk: RegulatoryChunk): string { const context = [ `${chunk.framework} ${chunk.frameworkVersion}`, chunk.annexAControlId ? `${chunk.controlId} (${chunk.annexAControlId})` : chunk.controlId, chunk.controlTheme, chunk.sectionKind, chunk.title ].filter(Boolean).join(' | '); return `${context}\n\n${chunk.text}`; } function validateVersionTwoFields(document: Record, controlId: string): Partial { if (!/^[5-8]\.\d{1,2}$/.test(controlId)) throw new Error('Regulatory schemaVersion 2 controlId must use the canonical 5.x-8.x form.'); const annexAControlId = requiredEnumLike(document, 'annexAControlId'); if (annexAControlId !== `A.${controlId}`) throw new Error(`Regulatory annexAControlId must be A.${controlId}.`); const controlTheme = requiredEnum(document, 'controlTheme', ['organizational', 'people', 'physical', 'technological'] as const); const sourceStandard = requiredEnum(document, 'sourceStandard', ['ISO/IEC 27001', 'ISO/IEC 27002'] as const); const sectionKind = requiredEnum(document, 'sectionKind', ['attributes', 'control', 'purpose', 'guidance', 'other-information'] as const); const retrievalRole = requiredEnum(document, 'retrievalRole', ['semantic', 'reference'] as const); const sourceEdition = requiredEnumLike(document, 'sourceEdition'); const nationalEdition = requiredEnumLike(document, 'nationalEdition'); const parserVersion = requiredEnumLike(document, 'parserVersion'); const sourceSha256 = requiredEnumLike(document, 'sourceSha256'); if (!/^[a-f0-9]{64}$/.test(sourceSha256)) throw new Error('Regulatory sourceSha256 must be a lower-case SHA-256 digest.'); const sourcePageStart = requiredPage(document, 'sourcePageStart'); const sourcePageEnd = requiredPage(document, 'sourcePageEnd'); if (sourcePageEnd < sourcePageStart) throw new Error('Regulatory sourcePageEnd must not precede sourcePageStart.'); if (sourceStandard === 'ISO/IEC 27001' && (sectionKind !== 'control' || retrievalRole !== 'reference')) { throw new Error('ISO/IEC 27001 schemaVersion 2 documents must be reference control statements.'); } if (sourceStandard === 'ISO/IEC 27002' && retrievalRole !== 'semantic') { throw new Error('ISO/IEC 27002 schemaVersion 2 documents must be semantic evidence.'); } return { annexAControlId, controlTheme, sourceStandard, sourceEdition, nationalEdition, sectionKind, sourcePageStart, sourcePageEnd, retrievalRole, parserVersion, sourceSha256 }; } function requiredEnumLike(document: Record, field: string): string { const value = document[field]; if (typeof value !== 'string' || !value.trim()) throw new Error(`Regulatory schemaVersion 2 ${field} must be a non-empty string.`); return value.trim(); } function requiredEnum(document: Record, field: string, values: T): T[number] { const value = requiredEnumLike(document, field); if (!values.includes(value)) throw new Error(`Regulatory schemaVersion 2 ${field} is not supported.`); return value as T[number]; } function requiredPage(document: Record, field: string): number { const value = document[field]; if (!Number.isInteger(value) || Number(value) < 1 || Number(value) > 10_000) throw new Error(`Regulatory schemaVersion 2 ${field} must be a positive page number.`); return Number(value); } function splitText(text: string, maximumCharacters: number): string[] { const paragraphs = text.split(/\n{2,}/).map(value => value.trim()).filter(Boolean); const units = paragraphs.flatMap(paragraph => splitLongUnit(paragraph, maximumCharacters)); const chunks: string[] = []; let current = ''; for (const unit of units) { const candidate = current ? `${current}\n\n${unit}` : unit; if (candidate.length <= maximumCharacters) { current = candidate; continue; } if (current) chunks.push(current); current = unit; } if (current) chunks.push(current); return chunks; } function splitLongUnit(value: string, maximumCharacters: number): string[] { if (value.length <= maximumCharacters) return [value]; const result: string[] = []; let remaining = value; while (remaining.length > maximumCharacters) { const boundary = Math.max( remaining.lastIndexOf('. ', maximumCharacters), remaining.lastIndexOf('; ', maximumCharacters), remaining.lastIndexOf(' ', maximumCharacters) ); const splitAt = boundary >= Math.floor(maximumCharacters * 0.6) ? boundary + 1 : maximumCharacters; result.push(remaining.slice(0, splitAt).trim()); remaining = remaining.slice(splitAt).trim(); } if (remaining) result.push(remaining); return result; } function normalizeText(value: string): string { return value.replace(/\r\n?/g, '\n').replace(/[ \t]+\n/g, '\n').trim(); } function sha256(value: string): string { return createHash('sha256').update(value, 'utf8').digest('hex'); } function isObject(value: unknown): value is Record { return Boolean(value) && typeof value === 'object' && !Array.isArray(value); }