195 lines
13 KiB
TypeScript
195 lines
13 KiB
TypeScript
import { describe, expect, it, vi } from 'vitest';
|
|
import { AiOverviewService } from '../../server/compliance/ai-overview-service';
|
|
import { LocalAssessorRepository } from '../../src/persistence/local-assessor-repository';
|
|
import { LocalComplianceStore } from '../../src/compliance/local-compliance-store';
|
|
import type { UnifiedResultProjection } from '../../src/domain/unified-result';
|
|
import { validAiOverviewResult } from './ai-overview-fixtures';
|
|
|
|
const stamp = '2026-09-14T08:00:00.000Z';
|
|
|
|
function syntheticResult(): UnifiedResultProjection {
|
|
return {
|
|
resultProjectionId: 'projection-1', sourceKind: 'scanner-finding', sourceResultId: 'occurrence-1',
|
|
projectId: 'project-1', assessmentId: 'assessment-1', collectionRunId: 'run-1', scanRunId: 'run-1',
|
|
sourceTool: 'Synthetic Scanner', sourceRuleId: 'RULE-1', title: 'Synthetic finding', description: 'Synthetic description',
|
|
recommendation: 'Synthetic recommendation', severityOriginal: 'high', severityNormalized: 'high',
|
|
outcomeOriginal: 'fail', outcomeNormalized: 'fail', technicalExecutionState: 'completed', resultType: 'compliance_check',
|
|
isIoe: false, isDeviation: true, lifecycleState: 'new', mappingConfidence: 1,
|
|
reviewStatus: 'unreviewed', importedAt: stamp, createdAt: stamp, frameworkReferences: [], categories: []
|
|
};
|
|
}
|
|
|
|
describe('AI overview service', () => {
|
|
it('uses the complete timeslot query, persists a successful run, and honors idempotency', async () => {
|
|
const repository = new LocalAssessorRepository();
|
|
const store = new LocalComplianceStore();
|
|
await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp });
|
|
await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp });
|
|
const query = vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] });
|
|
const serviceResult = validAiOverviewResult();
|
|
serviceResult.coverageGaps = [];
|
|
serviceResult.environmentOverview.factsUsed = [];
|
|
const complete = vi.fn(async () => ({ value: serviceResult, usage: { totalTokens: 30 } }));
|
|
const service = new AiOverviewService(repository, store, {
|
|
client: { model: 'work-model', complete }, now: () => new Date(stamp)
|
|
});
|
|
|
|
const request = { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-1234' };
|
|
const started = await service.start('project-1', request);
|
|
expect(started.state).toBe('queued');
|
|
await service.close();
|
|
|
|
const completed = await service.get('project-1', request);
|
|
expect(completed).toMatchObject({ state: 'succeeded', stale: false, run: { model: 'work-model', callCount: 1, tokenUsage: { totalTokens: 30 } } });
|
|
expect(query).toHaveBeenCalledWith({
|
|
audience: 'result-db', projectId: 'project-1', assessmentIds: ['assessment-1'], scanRunIds: [], offset: 0, limit: 500
|
|
});
|
|
// The runs live in the engine store and reach the project export beside the Core export, never inside it.
|
|
expect((await store.exportProject('project-1')).aiOverviewRuns).toHaveLength(1);
|
|
expect('aiOverviewRuns' in await repository.exportProject('project-1')).toBe(false);
|
|
|
|
await service.start('project-1', request);
|
|
await service.close();
|
|
expect(complete).toHaveBeenCalledTimes(1);
|
|
expect(await store.listAiOverviewRuns('project-1')).toHaveLength(1);
|
|
});
|
|
|
|
it('keeps the previous success when a later regeneration fails', async () => {
|
|
const repository = new LocalAssessorRepository();
|
|
const store = new LocalComplianceStore();
|
|
await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp });
|
|
await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp });
|
|
vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] });
|
|
const serviceResult = validAiOverviewResult();
|
|
serviceResult.coverageGaps = [];
|
|
serviceResult.environmentOverview.factsUsed = [];
|
|
const complete = vi.fn()
|
|
.mockResolvedValueOnce({ value: serviceResult })
|
|
.mockResolvedValueOnce({ value: { schemaVersion: 1 } });
|
|
let currentTime = stamp;
|
|
const service = new AiOverviewService(repository, store, { client: { model: 'work-model', complete }, now: () => new Date(currentTime) });
|
|
|
|
await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-good' });
|
|
await service.close();
|
|
currentTime = '2026-09-14T09:00:00.000Z';
|
|
await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-fail' });
|
|
await service.close();
|
|
|
|
const runs = await store.listAiOverviewRuns('project-1');
|
|
expect(runs.map(run => run.status).sort()).toEqual(['failed', 'succeeded']);
|
|
expect(runs.find(run => run.status === 'succeeded')?.result?.executiveSummary).toBe('Synthetic executive summary.');
|
|
expect(runs.find(run => run.status === 'failed')?.errorCode).toBe('ai_output_invalid');
|
|
const response = await service.get('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [] });
|
|
expect(response).toMatchObject({
|
|
state: 'failed', run: { errorCode: 'ai_output_invalid' },
|
|
previousSuccessfulRun: { result: { executiveSummary: 'Synthetic executive summary.' } }
|
|
});
|
|
});
|
|
|
|
it('drops references the model invents for findings without a public ID instead of failing', async () => {
|
|
const repository = new LocalAssessorRepository();
|
|
const store = new LocalComplianceStore();
|
|
await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp });
|
|
await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp });
|
|
// A GPO setting reaches the model with a null ID: the Result DB withholds its technical identity.
|
|
const gpoSetting: UnifiedResultProjection = {
|
|
...syntheticResult(), resultProjectionId: 'projection-gpo', sourceKind: 'policy-evaluation', sourceResultId: 'evaluation-1',
|
|
sourceTool: 'GPOs', sourceRuleId: `gpo-policy-identity-${'a'.repeat(32)}`, title: 'Synthetic GPO setting'
|
|
};
|
|
vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 2, items: [syntheticResult(), gpoSetting] });
|
|
const modelOutput = validAiOverviewResult();
|
|
modelOutput.coverageGaps = [{ ...modelOutput.coverageGaps[0], evidenceIds: ['INVENTED-GAP'] }];
|
|
modelOutput.environmentOverview.factsUsed = [];
|
|
modelOutput.themes[2].findingIds = ['Synthetic GPO setting', 'GPO-1', 'GPO-2'];
|
|
modelOutput.priorities[0].findingIds = ['RULE-1', 'GPO-1'];
|
|
modelOutput.quickWins[0].findingIds = ['GPO-2'];
|
|
const service = new AiOverviewService(repository, store, {
|
|
client: { model: 'work-model', complete: vi.fn(async () => ({ value: modelOutput })) }, now: () => new Date(stamp)
|
|
});
|
|
|
|
await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-gpo' });
|
|
await service.close();
|
|
|
|
const [run] = await store.listAiOverviewRuns('project-1');
|
|
expect(run.status).toBe('succeeded');
|
|
expect(run.result?.themes.map(theme => theme.findingIds)).toEqual([['RULE-1'], ['RULE-1'], []]);
|
|
expect(run.result?.priorities[0].findingIds).toEqual(['RULE-1']);
|
|
expect(run.result?.quickWins[0].findingIds).toEqual([]);
|
|
expect(run.result?.coverageGaps[0].evidenceIds).toEqual([]);
|
|
});
|
|
|
|
it('accepts a cited CIS rule number that looks like an IPv4 address and stores the run', async () => {
|
|
const repository = new LocalAssessorRepository();
|
|
const store = new LocalComplianceStore();
|
|
await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp });
|
|
await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp });
|
|
const cisRule: UnifiedResultProjection = { ...syntheticResult(), sourceTool: 'cis-cat', sourceRuleId: '18.10.42.11', title: '18.10.42.11 (L1) Synthetic CIS rule' };
|
|
vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [cisRule] });
|
|
const modelOutput = validAiOverviewResult();
|
|
modelOutput.coverageGaps = [];
|
|
modelOutput.environmentOverview.factsUsed = [];
|
|
modelOutput.themes.forEach(theme => { theme.findingIds = ['18.10.42.11']; });
|
|
modelOutput.priorities[0].findingIds = ['18.10.42.11'];
|
|
modelOutput.quickWins[0].findingIds = ['18.10.42.11'];
|
|
modelOutput.themes[0].summary = 'CIS 18.10.42.11 is not met.';
|
|
const service = new AiOverviewService(repository, store, {
|
|
client: { model: 'work-model', complete: vi.fn(async () => ({ value: modelOutput })) }, now: () => new Date(stamp)
|
|
});
|
|
|
|
await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-cis' });
|
|
await service.close();
|
|
|
|
const [run] = await store.listAiOverviewRuns('project-1');
|
|
expect(run).toMatchObject({ status: 'succeeded', result: { themes: [{ findingIds: ['18.10.42.11'] }, {}, {}] } });
|
|
});
|
|
|
|
it('marks the previous success stale while a regeneration is queued', async () => {
|
|
const repository = new LocalAssessorRepository();
|
|
const store = new LocalComplianceStore();
|
|
await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp });
|
|
await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp });
|
|
vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] });
|
|
const serviceResult = validAiOverviewResult();
|
|
serviceResult.coverageGaps = [];
|
|
serviceResult.environmentOverview.factsUsed = [];
|
|
const complete = vi.fn().mockResolvedValueOnce({ value: serviceResult });
|
|
const service = new AiOverviewService(repository, store, { client: { model: 'work-model', complete }, now: () => new Date(stamp) });
|
|
await service.start('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [], idempotencyKey: 'request-good' });
|
|
await service.close();
|
|
const [saved] = await store.listAiOverviewRuns('project-1');
|
|
await store.createAiOverviewRun({
|
|
...saved, id: 'ai-overview-queued', idempotencyKey: 'request-queued', status: 'queued', inputHash: 'b'.repeat(64),
|
|
result: undefined, completedAt: undefined, createdAt: '2026-09-14T09:00:00.000Z', updatedAt: '2026-09-14T09:00:00.000Z', callCount: 0
|
|
});
|
|
vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [{ ...syntheticResult(), title: 'Changed synthetic finding' }] });
|
|
|
|
const response = await service.get('project-1', { assessmentIds: ['assessment-1'], scanRunIds: [] });
|
|
expect(response).toMatchObject({ state: 'queued', stale: true, previousSuccessfulRun: { id: saved.id } });
|
|
});
|
|
|
|
it('removes the overview of a deleted timeslot from the engine store', async () => {
|
|
const repository = new LocalAssessorRepository();
|
|
const store = new LocalComplianceStore();
|
|
await repository.saveProject({ id: 'project-1', name: 'Synthetic', description: '', attributes: {}, createdAt: stamp, updatedAt: stamp });
|
|
await repository.saveAssessment({ id: 'assessment-1', projectId: 'project-1', name: 'Synthetic timeslot', createdAt: stamp, updatedAt: stamp });
|
|
await repository.saveAssessment({ id: 'assessment-2', projectId: 'project-1', name: 'Second timeslot', createdAt: stamp, updatedAt: stamp });
|
|
vi.spyOn(repository, 'queryUnifiedResults').mockResolvedValue({ total: 1, items: [syntheticResult()] });
|
|
const serviceResult = validAiOverviewResult();
|
|
serviceResult.coverageGaps = [];
|
|
serviceResult.environmentOverview.factsUsed = [];
|
|
const service = new AiOverviewService(repository, store, {
|
|
client: { model: 'work-model', complete: vi.fn(async () => ({ value: serviceResult })) }, now: () => new Date(stamp)
|
|
});
|
|
for (const assessmentId of ['assessment-1', 'assessment-2']) {
|
|
await service.start('project-1', { assessmentIds: [assessmentId], scanRunIds: [], idempotencyKey: `request-${assessmentId}` });
|
|
await service.close();
|
|
}
|
|
expect(await store.listAiOverviewRuns('project-1')).toHaveLength(2);
|
|
|
|
// The Core deletes the timeslot without knowing the engine; the engine prunes its own derived run.
|
|
await repository.deleteAssessment('project-1', 'assessment-1', { projectId: 'project-1', assessmentId: 'assessment-1', cascade: true });
|
|
await service.get('project-1', { assessmentIds: ['assessment-2'], scanRunIds: [] });
|
|
expect((await store.listAiOverviewRuns('project-1')).map(run => run.assessmentIds)).toEqual([['assessment-2']]);
|
|
});
|
|
});
|