100 lines
8.0 KiB
TypeScript
100 lines
8.0 KiB
TypeScript
import { describe, expect, it, vi } from 'vitest';
|
|
import { RegulatoryMappingService } from '../../server/compliance/regulatory-mapping-service';
|
|
import { RegulatoryFindingRetrievalService } from '../../server/compliance/regulatory-finding-retrieval';
|
|
import { LocalComplianceStore } from '../../src/compliance/local-compliance-store';
|
|
import type { ComplianceReadPort } from '../../src/compliance/compliance-store';
|
|
import { REGULATORY_MAPPING_PROMPT_VERSION } from '../../src/compliance/regulatory-mapping';
|
|
import type { RegulatoryChunk, RegulatoryRetrievalFilters } from '../../src/compliance/regulatory-rag';
|
|
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
|
|
|
// Prompt 1.3.0 (user decision 2026-10-09): the model reads each ISO candidate's full ISO/IEC 27002 control text in
|
|
// document order, not only the passages retrieval matched. Synthetic corpus text only.
|
|
const stamp = '2026-10-09T10:00:00.000Z';
|
|
type Section = NonNullable<RegulatoryChunk['sectionKind']>;
|
|
|
|
function chunk(sourceStandard: 'ISO/IEC 27001' | 'ISO/IEC 27002', section: Section, index: number, length = 100): RegulatoryChunk {
|
|
const reference = sourceStandard === 'ISO/IEC 27001';
|
|
const documentId = reference ? '27001-8.2' : `27002-8.2-${section}`;
|
|
return { schemaVersion: 2, corpusId: 'iso-test', corpusVersion: 'v2', documentId, framework: 'ISO/IEC 27002', frameworkVersion: '2022',
|
|
controlId: '8.2', annexAControlId: 'A.8.2', controlTheme: 'technological', sourceStandard, sourceEdition: '2022', nationalEdition: 'DIN 2024',
|
|
sectionKind: section, sourcePageStart: 1, sourcePageEnd: 1, retrievalRole: reference ? 'reference' : 'semantic', parserVersion: 'test',
|
|
sourceSha256: '1'.repeat(64), chunkId: `${documentId}:${index}`, chunkIndex: index, language: 'de', title: 'Synthetic privileged access',
|
|
text: `${section} ${index} `.padEnd(length, 'x'), textSha256: 'a'.repeat(64) };
|
|
}
|
|
|
|
const finding: UnifiedResultProjection = {
|
|
resultProjectionId: 'scanner-finding:dummy', sourceKind: 'scanner-finding', sourceResultId: 'dummy', projectId: 'project-1', assessmentId: 'assessment-1',
|
|
collectionRunId: 'run-1', scanRunId: 'run-1', sourceTool: 'purple-knight', sourceRuleId: 'SYN-1', title: 'Synthetic privileged access check',
|
|
description: 'A synthetic privileged access setting is weaker than recommended.', severityOriginal: 'high', severityNormalized: 'high',
|
|
outcomeOriginal: 'fail', outcomeNormalized: 'fail', technicalExecutionState: 'completed', resultType: 'compliance_check', isIoe: false, isDeviation: true,
|
|
lifecycleState: 'new', mappingConfidence: 1, reviewStatus: 'unreviewed', importedAt: stamp, createdAt: stamp, frameworkReferences: [], categories: []
|
|
};
|
|
|
|
const read: ComplianceReadPort = {
|
|
getProject: async () => ({ id: 'project-1', name: 'Synthetic' }) as never, listProjects: async () => [] as never,
|
|
listAssessments: async () => [] as never, listProjectScanRuns: async () => [] as never,
|
|
queryUnifiedResults: async () => ({ total: 0, items: [] }) as never, getUnifiedResultById: async () => finding
|
|
};
|
|
|
|
async function promptFor(corpus: RegulatoryChunk[], matched: RegulatoryChunk[]) {
|
|
const complete = vi.fn(async (_system: string, input: string) => ({ value: { schemaVersion: 1, outcome: 'mapped', mappings: [{ controlId: '8.2',
|
|
relationship: 'primary', confidence: 'high', rationale: 'Synthetic rationale.', evidenceChunkIds: [JSON.parse(input).candidates[0].evidence[0].chunkId] }] } }));
|
|
const list = vi.fn(async (filters: RegulatoryRetrievalFilters) => corpus.filter(item => item.sourceStandard === filters.sourceStandard));
|
|
const svc = new RegulatoryMappingService(read, new LocalComplianceStore(), {
|
|
findingRetrieval: new RegulatoryFindingRetrievalService({ retrieve: vi.fn(async () => matched.map(item => ({ score: 0.9, chunk: item }))) }),
|
|
index: { ensureCollection: vi.fn(), replaceDocuments: vi.fn(), searchDense: vi.fn(), list } as never,
|
|
client: { model: 'work-model', complete: complete as never }, corpusId: 'iso-test', corpusVersion: 'v2', embeddingModel: 'bge-m3', now: () => new Date(stamp)
|
|
});
|
|
await svc.start('project-1', { resultProjectionId: finding.resultProjectionId });
|
|
await svc.close();
|
|
const response = await svc.get('project-1', finding.resultProjectionId);
|
|
return { prompt: JSON.parse(complete.mock.calls[0][1]), system: complete.mock.calls[0][0], response };
|
|
}
|
|
|
|
describe('ISO full control text (prompt 1.3.0)', () => {
|
|
it('sends the 27001 statement and the whole 27002 control, purpose and guidance in document order, not only the matched passage', async () => {
|
|
const guidance = [chunk('ISO/IEC 27002', 'guidance', 0), chunk('ISO/IEC 27002', 'guidance', 1), chunk('ISO/IEC 27002', 'guidance', 2)];
|
|
const corpus = [chunk('ISO/IEC 27001', 'control', 0), ...guidance, chunk('ISO/IEC 27002', 'purpose', 0), chunk('ISO/IEC 27002', 'control', 0),
|
|
chunk('ISO/IEC 27002', 'other-information', 0), chunk('ISO/IEC 27002', 'attributes', 0)];
|
|
const { prompt, system, response } = await promptFor(corpus, [guidance[2]]);
|
|
expect(REGULATORY_MAPPING_PROMPT_VERSION).toBe('1.3.0');
|
|
expect(system).toContain('read the whole text before judging');
|
|
const evidence = prompt.candidates[0].evidence as Array<{ chunkId: string; sourceStandard: string }>;
|
|
// Attributes are tags, not normative text, and are only sent when retrieval matched them.
|
|
expect(evidence.map(item => item.chunkId)).toEqual(['27001-8.2:0', '27002-8.2-control:0', '27002-8.2-purpose:0',
|
|
'27002-8.2-guidance:0', '27002-8.2-guidance:1', '27002-8.2-guidance:2', '27002-8.2-other-information:0']);
|
|
expect(response).toMatchObject({ state: 'succeeded', run: { promptVersion: '1.3.0' } });
|
|
// The browser still receives citations only, never the licensed text.
|
|
expect(JSON.stringify(response)).not.toContain('guidance 1 ');
|
|
});
|
|
|
|
it('caps a long control, keeps every matched passage and adds other information only while it fits', async () => {
|
|
const guidance = Array.from({ length: 12 }, (_, index) => chunk('ISO/IEC 27002', 'guidance', index, 1_500));
|
|
const corpus = [chunk('ISO/IEC 27001', 'control', 0), chunk('ISO/IEC 27002', 'control', 0), ...guidance, chunk('ISO/IEC 27002', 'other-information', 0, 1_500)];
|
|
const { prompt } = await promptFor(corpus, [guidance[11]]);
|
|
const evidence = (prompt.candidates[0].evidence as Array<{ chunkId: string; sourceStandard: string; text: string }>)
|
|
.filter(item => item.sourceStandard === 'ISO/IEC 27002');
|
|
const ids = evidence.map(item => item.chunkId);
|
|
expect(ids).toContain('27002-8.2-guidance:11');
|
|
expect(ids).not.toContain('27002-8.2-other-information:0');
|
|
expect(evidence.reduce((total, item) => total + item.text.length, 0)).toBeLessThanOrEqual(8_000);
|
|
// Document order is kept even for the matched passage that was added first.
|
|
expect(ids.slice(0, 2)).toEqual(['27002-8.2-control:0', '27002-8.2-guidance:0']);
|
|
expect(ids.at(-1)).toBe('27002-8.2-guidance:11');
|
|
});
|
|
|
|
// Regression 2026-10-09: the hybrid search matched many long passages of one control, which were all kept beyond the
|
|
// cap, and a Purple Knight mapping failed with regulatory_mapping_prompt_too_large.
|
|
it('counts matched passages toward the cap, keeping the best one first, so many matches cannot overflow the prompt', async () => {
|
|
const guidance = Array.from({ length: 12 }, (_, index) => chunk('ISO/IEC 27002', 'guidance', index, 1_500));
|
|
const corpus = [chunk('ISO/IEC 27001', 'control', 0), chunk('ISO/IEC 27002', 'control', 0), ...guidance];
|
|
const { prompt, response } = await promptFor(corpus, [...guidance].reverse());
|
|
const evidence = (prompt.candidates[0].evidence as Array<{ chunkId: string; sourceStandard: string; text: string }>)
|
|
.filter(item => item.sourceStandard === 'ISO/IEC 27002');
|
|
expect(response.run?.status).toBe('succeeded');
|
|
expect(evidence.reduce((total, item) => total + item.text.length, 0)).toBeLessThanOrEqual(8_000);
|
|
// The best-ranked match (guidance 11, first in the hybrid ranking) is always kept.
|
|
expect(evidence.map(item => item.chunkId)).toContain('27002-8.2-guidance:11');
|
|
});
|
|
});
|