366 lines
22 KiB
TypeScript
366 lines
22 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
import type { CanonicalAssessmentModel, FindingOccurrence, JsonValue } from '../../src/domain/canonical-model';
|
|
import type { EnvironmentObservation } from '../../src/domain/environment-context';
|
|
import { compareSecurityScores, defaultSecurityScoringPolicy, securitySourceScores, weightedSecurityScore } from '../../src/domain/security-scoring';
|
|
import { canonicalSecurityScores, securityScoreNumber } from '../../src/scoring/canonical-security-score';
|
|
import { canonicalModelsToDataset } from '../../src/features/project-import/canonical-model-to-dataset';
|
|
import { dashboardSecuritySources } from '../../src/features/dashboard/dashboard-security-score';
|
|
import { loadCanonicalProjectWorkspace, canonicalModelForScanRuns } from '../../src/features/project-import/load-canonical-project-workspace';
|
|
import { LocalAssessorRepository } from '../../src/persistence/local-assessor-repository';
|
|
import type { ToolProfileDefinition } from '../../src/domain/tool-profile-definition';
|
|
import { genericDefinition } from '../../src/profiles/generic/generic-definition';
|
|
import { toolProfileDefinitions } from '../../src/profiles/profile-definition-catalog';
|
|
|
|
const time = '2026-09-01T12:00:00.1234567Z';
|
|
const provenance = { decisionSource: 'source-native' as const, confidence: { level: 'high' as const, score: 1, reasons: ['Synthetic'] }, mappingVersion: '1' };
|
|
|
|
function fixture(id = 'a', toolId = 'purple-knight', results = ['passed', 'passed', 'failed']): CanonicalAssessmentModel {
|
|
const tool = { id: toolId, name: toolId, version: '1' };
|
|
return {
|
|
schemaVersion: 1,
|
|
project: { id: 'synthetic-project', name: 'Synthetic', attributes: {}, createdAt: time, updatedAt: time },
|
|
assessment: { id, projectId: 'synthetic-project', name: id, createdAt: time, updatedAt: time, scoringPolicy: defaultSecurityScoringPolicy() },
|
|
scanRuns: [{ id: `run-${id}`, assessmentId: id, sequence: 1, label: 'Synthetic scan', tool, importedAt: time }],
|
|
importFiles: [{ id: `file-${id}`, scanRunId: `run-${id}`, fileName: 'synthetic.html', format: 'HTML', sizeBytes: 1, importedAt: time,
|
|
detectedTool: tool, detection: provenance.confidence,
|
|
processedWith: { parserId: 'html', parserVersion: '1', profileId: toolId, profileVersion: '1', mappingVersion: '1' },
|
|
sourceArtifact: { kind: 'not-retained', reason: 'synthetic' } }],
|
|
sourceTables: [], findingDefinitions: [], evidence: [], reviews: [], diagnostics: [], environmentObservations: [],
|
|
findingOccurrences: results.map((status, index): FindingOccurrence => ({
|
|
id: `${id}-${index}`, scanRunId: `run-${id}`, importFileId: `file-${id}`, definitionId: `rule-${index}`,
|
|
sourceIdentity: { tool, originalRuleId: `rule-${index}`, canonicalRuleId: `rule-${index}`, originalTitle: `Synthetic rule ${index}` },
|
|
sourcePosition: { rowIndex: index }, fieldMappings: [],
|
|
originalFields: [{ key: 'name', label: 'Name', value: `Synthetic rule ${index}` }, { key: 'status', label: 'Status', value: status }],
|
|
severity: { originalValue: 'high', normalizedValue: 'high', provenance },
|
|
status: { originalValue: status, normalizedValue: status as FindingOccurrence['status']['normalizedValue'], provenance }
|
|
}))
|
|
};
|
|
}
|
|
|
|
function observation(model: CanonicalAssessmentModel, property: string, value: JsonValue): EnvironmentObservation {
|
|
const item: EnvironmentObservation = {
|
|
version: 1, id: `${model.importFiles[0].id}-${property}-${model.environmentObservations?.length}`,
|
|
scanRunId: model.scanRuns[0].id, importFileId: model.importFiles[0].id, tool: model.scanRuns[0].tool,
|
|
subject: { type: 'synthetic-assessment' }, property, label: property, state: 'observed',
|
|
originalValue: value, normalizedValue: value, valueType: 'string', mappingVersion: '1', source: { objectPath: property }, attributes: {}
|
|
};
|
|
model.environmentObservations!.push(item);
|
|
return item;
|
|
}
|
|
|
|
function identity(model: CanonicalAssessmentModel, timestamp = time) {
|
|
observation(model, model.scanRuns[0].tool.id === 'maester' ? 'entra.tenant_id' : 'forest.name', 'synthetic.example');
|
|
observation(model, 'report.generated_at', timestamp);
|
|
}
|
|
|
|
function futureProfile(direction: 'higher-is-better' | 'lower-is-better' = 'higher-is-better'): ToolProfileDefinition {
|
|
return {
|
|
...genericDefinition, id: 'synthetic-future-scanner', name: 'Synthetic future scanner', fallback: false,
|
|
securityScoring: {
|
|
version: '1.0.0',
|
|
native: { kind: 'mapped', properties: ['synthetic.report.overall'], scope: 'overall', scale: '0-100', direction },
|
|
targets: ['synthetic.target'], resultPopulation: 'complete'
|
|
}
|
|
};
|
|
}
|
|
|
|
describe('native-first scoring for future scanner profiles', () => {
|
|
it.each([
|
|
['higher-is-better', 27, 27, 'native'],
|
|
['lower-is-better', 27, 73, 'inverted-native'],
|
|
['higher-is-better', 0, 0, 'native'],
|
|
['lower-is-better', 100, 0, 'inverted-native']
|
|
] as const)('uses a declared %s native score of %s before the pass rate', (direction, native, expected, method) => {
|
|
const profile = futureProfile(direction);
|
|
const model = fixture('future', profile.id);
|
|
observation(model, 'synthetic.report.overall', native);
|
|
const before = JSON.stringify(model);
|
|
expect(canonicalSecurityScores([model], [profile])[0]).toMatchObject({ score: expected, nativeValue: native, method, passed: 2, deviations: 1 });
|
|
expect(JSON.stringify(model)).toBe(before);
|
|
});
|
|
|
|
it('carries a newly registered native score through the actual dataset projection', () => {
|
|
const profile = futureProfile();
|
|
const model = fixture('future', profile.id);
|
|
observation(model, 'synthetic.report.overall', 27);
|
|
toolProfileDefinitions.push(profile);
|
|
try {
|
|
const dataset = canonicalModelsToDataset([model])!;
|
|
expect(dataset.securityScoring?.scans[0]).toMatchObject({ score: 27, nativeValue: 27, nativeProperty: 'synthetic.report.overall', method: 'native' });
|
|
expect(weightedSecurityScore(securitySourceScores(dataset.securityScoring!.scans))).toBe(27);
|
|
} finally {
|
|
toolProfileDefinitions.splice(toolProfileDefinitions.indexOf(profile), 1);
|
|
}
|
|
});
|
|
|
|
it('falls back when the native score is missing, invalid, conflicting or belongs to another run', () => {
|
|
const profile = futureProfile();
|
|
for (const variant of ['missing', 'invalid', 'conflict', 'foreign-run']) {
|
|
const model = fixture(variant, profile.id);
|
|
observation(model, 'synthetic.category.score', 100);
|
|
observation(model, 'score', 100);
|
|
if (variant !== 'missing') {
|
|
const native = observation(model, 'synthetic.report.overall', variant === 'invalid' ? 101 : 27);
|
|
if (variant === 'conflict') observation(model, 'synthetic.report.overall', 90);
|
|
if (variant === 'foreign-run') native.scanRunId = 'unrelated-run';
|
|
}
|
|
const [score] = canonicalSecurityScores([model], [profile]);
|
|
expect(score.method).toBe('pass-rate');
|
|
expect(score.score).toBeCloseTo(200 / 3);
|
|
if (variant === 'invalid' || variant === 'conflict') expect(score.issues).toContain('Native score unavailable or conflicting');
|
|
}
|
|
});
|
|
|
|
it('does not infer native semantics for an unregistered scanner', () => {
|
|
const model = fixture('future', 'unregistered-scanner');
|
|
observation(model, 'synthetic.report.overall', 27);
|
|
observation(model, 'score', 99);
|
|
const [score] = canonicalSecurityScores([model]);
|
|
expect(score.method).toBe('pass-rate');
|
|
expect(score.issues).toContain('Result population not verified');
|
|
});
|
|
|
|
it('uses a native score from a findings-only scanner but does not invent a pass denominator', () => {
|
|
const profile = futureProfile();
|
|
profile.securityScoring.resultPopulation = 'findings-only';
|
|
const model = fixture('future', profile.id, ['failed']);
|
|
expect(canonicalSecurityScores([model], [profile])[0]).toMatchObject({ score: undefined, method: 'unavailable' });
|
|
observation(model, 'synthetic.report.overall', 80);
|
|
expect(canonicalSecurityScores([model], [profile])[0]).toMatchObject({ score: 80, method: 'native' });
|
|
});
|
|
|
|
it('includes the profile scoring version in comparison compatibility', () => {
|
|
const model = fixture('future', 'synthetic-future-scanner');
|
|
const profile = futureProfile();
|
|
const [before] = canonicalSecurityScores([model], [profile]);
|
|
profile.securityScoring.version = '2.0.0';
|
|
const [after] = canonicalSecurityScores([model], [profile]);
|
|
expect(before.processingKey).not.toBe(after.processingKey);
|
|
});
|
|
});
|
|
|
|
describe('versioned hybrid security scoring', () => {
|
|
it('uses Purple Knight 65 exactly once instead of its pass rate', () => {
|
|
const model = fixture();
|
|
observation(model, 'purple_knight.ad.security_posture_score', 65);
|
|
identity(model);
|
|
const before = JSON.stringify(model);
|
|
const [score] = canonicalSecurityScores([model]);
|
|
expect(score).toMatchObject({ score: 65, nativeValue: 65, method: 'native', passed: 2, deviations: 1, observedAt: time });
|
|
expect(weightedSecurityScore(securitySourceScores([score]))).toBe(65);
|
|
expect(JSON.stringify(model)).toBe(before);
|
|
});
|
|
|
|
it('joins a separate Purple Knight native summary to the findings from the exact same scan run', () => {
|
|
const model = fixture('combined', 'purple-knight', Array.from({ length: 96 }, () => 'passed').concat(Array.from({ length: 19 }, () => 'failed')));
|
|
const findingsFile = model.importFiles[0];
|
|
const summaryFile = { ...findingsFile, id: 'file-summary', fileName: 'Assessment_summary.csv' };
|
|
model.importFiles.push(summaryFile);
|
|
const native = observation(model, 'purple_knight.ad.security_posture_score', 65);
|
|
native.importFileId = summaryFile.id;
|
|
const forest = observation(model, 'forest.name', 'synthetic.example');
|
|
forest.importFileId = summaryFile.id;
|
|
const generated = observation(model, 'report.generated_at', time);
|
|
generated.importFileId = summaryFile.id;
|
|
|
|
const [score] = canonicalSecurityScores([model]);
|
|
expect(score).toMatchObject({
|
|
score: 65, nativeValue: 65, method: 'native', passed: 96, deviations: 19, assessed: 115
|
|
});
|
|
expect(score.importFileIds).toEqual(expect.arrayContaining(['file-summary', 'file-combined']));
|
|
const dataset = canonicalModelsToDataset([model])!;
|
|
const [source] = dashboardSecuritySources([{
|
|
tool: { id: 'purple-knight', label: 'Purple Knight', sourceValues: ['Purple Knight'], count: 115 },
|
|
records: dataset.sheets[0].analysis.records, passed: 96, deviations: 19, assessed: 115, score: 100 * 96 / 115
|
|
}], dataset.securityScoring);
|
|
expect(source).toMatchObject({ score: 65, deviations: 19, rating: { methods: ['native'] } });
|
|
});
|
|
|
|
it('calculates a scoreless scan and excludes irrelevant/error/not-run outcomes', () => {
|
|
const model = fixture('a', 'maester', ['passed', 'failed', 'not-applicable', 'error', 'not-run']);
|
|
expect(canonicalSecurityScores([model])[0]).toMatchObject({ score: 50, method: 'pass-rate', assessed: 2, eligible: 4 });
|
|
});
|
|
|
|
it.each([null, '', ' ', false, true, -1, 101, 'unreadable'])('never turns invalid score %s into a zero', value => {
|
|
expect(securityScoreNumber(value)).toBeUndefined();
|
|
const model = fixture();
|
|
observation(model, 'purple_knight.ad.security_posture_score', value);
|
|
const [score] = canonicalSecurityScores([model]);
|
|
expect(score.score).toBeCloseTo(200 / 3);
|
|
expect(score.issues).toContain('Native score unavailable or conflicting');
|
|
});
|
|
|
|
it('preserves legitimate native zero and full marks', () => {
|
|
expect(securityScoreNumber(0)).toBe(0);
|
|
expect(securityScoreNumber('100%')).toBe(100);
|
|
});
|
|
|
|
it('inverts the bounded PingCastle risk but never derives a rate from findings only', () => {
|
|
const model = fixture('a', 'pingcastle', ['failed']);
|
|
expect(canonicalSecurityScores([model])[0].score).toBeUndefined();
|
|
observation(model, 'risk.domain', 82);
|
|
expect(canonicalSecurityScores([model])[0]).toMatchObject({ score: 18, nativeValue: 82, method: 'inverted-native' });
|
|
});
|
|
|
|
it('binds native scores to both the exact file and run, even with stray context', () => {
|
|
const model = fixture();
|
|
const foreign = observation(model, 'purple_knight.ad.security_posture_score', 99);
|
|
foreign.scanRunId = 'unrelated-run';
|
|
expect(canonicalSecurityScores([model])[0].method).toBe('pass-rate');
|
|
foreign.scanRunId = model.scanRuns[0].id;
|
|
foreign.importFileId = 'unrelated-file';
|
|
expect(canonicalSecurityScores([model])[0].method).toBe('pass-rate');
|
|
});
|
|
|
|
it('does not pick the first of conflicting native observations', () => {
|
|
const model = fixture();
|
|
observation(model, 'purple_knight.ad.security_posture_score', 65);
|
|
observation(model, 'purple_knight.ad.security_posture_score', 95);
|
|
expect(canonicalSecurityScores([model])[0]).toMatchObject({ method: 'pass-rate', nativeValue: undefined });
|
|
});
|
|
|
|
it('deduplicates verified same-scan formats and retains their provenance', () => {
|
|
const html = fixture('a');
|
|
const xml = fixture('b');
|
|
xml.assessment.id = html.assessment.id;
|
|
observation(html, 'purple_knight.ad.security_posture_score', 65);
|
|
observation(xml, 'purple_knight.ad.security_posture_score', 65);
|
|
identity(html); identity(xml);
|
|
const scores = canonicalSecurityScores([html, xml]);
|
|
expect(scores).toHaveLength(1);
|
|
expect(scores[0]).toMatchObject({ score: 65, assessed: 3, importFileIds: ['file-a', 'file-b'] });
|
|
});
|
|
|
|
it('exposes contradictory representations instead of selecting the better score', () => {
|
|
const a = fixture('a'); const b = fixture('b'); b.assessment.id = a.assessment.id;
|
|
observation(a, 'purple_knight.ad.security_posture_score', 65); identity(a);
|
|
observation(b, 'purple_knight.ad.security_posture_score', 95); identity(b);
|
|
expect(canonicalSecurityScores([a, b])[0].score).toBeUndefined();
|
|
});
|
|
|
|
it('chooses latest source scan time, not later import time', () => {
|
|
const old = fixture('old'); const current = fixture('current'); current.assessment.id = old.assessment.id;
|
|
observation(old, 'purple_knight.ad.security_posture_score', 20); identity(old);
|
|
old.scanRuns[0].importedAt = '2026-12-31T12:00:00Z';
|
|
observation(current, 'purple_knight.ad.security_posture_score', 80); identity(current, '2026-09-02T12:00:00Z');
|
|
expect(canonicalSecurityScores([old, current]).map(score => score.score)).toEqual([80]);
|
|
});
|
|
|
|
it('keeps tool weight independent of the number of reports and retains fractional precision', () => {
|
|
const [a] = canonicalSecurityScores([fixture('a')]);
|
|
const [b] = canonicalSecurityScores([fixture('b', 'maester')]);
|
|
const values = securitySourceScores([{ ...a, score: 65 }, { ...a, id: 'other-scope', score: 75 }, { ...b, score: 100 / 3 }]);
|
|
expect(weightedSecurityScore(values)).toBeCloseTo((70 + 100 / 3) / 2, 12);
|
|
});
|
|
|
|
it('has no numeric result for a report without evaluated checks or a native score', () => {
|
|
expect(canonicalSecurityScores([fixture('a', 'maester', ['error', 'not-run'])])[0].score).toBeUndefined();
|
|
});
|
|
|
|
it('preserves exact identities through projection and replaces the dashboard fallback', () => {
|
|
const model = fixture(); identity(model); observation(model, 'purple_knight.ad.security_posture_score', 65);
|
|
const dataset = canonicalModelsToDataset([model])!;
|
|
const records = dataset.sheets[0].analysis.records;
|
|
expect(records[0]).toMatchObject({ importFileId: 'file-a', scanRunId: 'run-a' });
|
|
const [result] = dashboardSecuritySources([{
|
|
tool: { id: 'purpleknight', label: 'PurpleKnight', sourceValues: ['pk'], count: 3 },
|
|
records, passed: 2, deviations: 1, assessed: 3, score: 200 / 3
|
|
}], dataset.securityScoring);
|
|
expect(result.score).toBe(65);
|
|
});
|
|
|
|
it('compares only unchanged scopes and retains missing-source obligations', () => {
|
|
const pk = fixture('old-pk'); identity(pk); observation(pk, 'purple_knight.ad.security_posture_score', 20);
|
|
const old = fixture('old-m', 'maester'); identity(old);
|
|
const current = fixture('current', 'maester', ['passed', 'passed', 'passed']); identity(current, '2026-09-03T12:00:00Z');
|
|
const comparison = compareSecurityScores(canonicalSecurityScores([pk, old]), canonicalSecurityScores([current]));
|
|
expect(comparison.scopeChanged).toBe(true);
|
|
expect(comparison.comparable).toBe(1);
|
|
expect(comparison.change).toBeCloseTo(100 / 3);
|
|
expect(comparison.awaitingRevalidation).toMatchObject([{ occurrenceIds: ['old-pk-2'], observedAt: time }]);
|
|
});
|
|
|
|
it('does not present a change of score method or processing version as progress', () => {
|
|
const model = fixture(); identity(model);
|
|
const [a] = canonicalSecurityScores([model]);
|
|
expect(compareSecurityScores([a], [{ ...a, method: 'native', score: 90 }]).change).toBeUndefined();
|
|
expect(compareSecurityScores([a], [{ ...a, processingKey: 'different' }]).change).toBeUndefined();
|
|
const shorter = fixture('shorter', 'purple-knight', ['passed', 'passed']); identity(shorter, '2026-09-03T12:00:00Z');
|
|
expect(compareSecurityScores([a], canonicalSecurityScores([shorter])).change).toBeUndefined();
|
|
});
|
|
|
|
it('keeps a score-only native report visible without inventing finding rows', () => {
|
|
const model = fixture('summary', 'purple-knight', []);
|
|
observation(model, 'purple_knight.ad.security_posture_score', 65); identity(model);
|
|
const dataset = canonicalModelsToDataset([model])!;
|
|
expect(dataset.sheets[0].analysis.records).toHaveLength(0);
|
|
expect(dashboardSecuritySources([], dataset.securityScoring, true)[0]).toMatchObject({ score: 65, records: [] });
|
|
expect(dashboardSecuritySources([], dataset.securityScoring, false)).toEqual([]);
|
|
});
|
|
|
|
it('handles exact duplicate files even when report-time metadata is unavailable', () => {
|
|
const a = fixture('a'); const b = fixture('b'); b.assessment.id = a.assessment.id;
|
|
a.importFiles[0].sha256 = b.importFiles[0].sha256 = 'a'.repeat(64);
|
|
expect(canonicalSecurityScores([a, b])).toHaveLength(1);
|
|
});
|
|
|
|
it('rejects conflicting results even when source status totals agree', () => {
|
|
const a = fixture('a'); const b = fixture('b', 'purple-knight', ['passed', 'failed', 'passed']);
|
|
b.assessment.id = a.assessment.id; identity(a); identity(b);
|
|
expect(canonicalSecurityScores([a, b])[0].score).toBeUndefined();
|
|
});
|
|
|
|
it('does not reuse an AD native score as the overall score of a hybrid report', () => {
|
|
const model = fixture();
|
|
observation(model, 'purple_knight.ad.security_posture_score', 65);
|
|
observation(model, 'purple_knight.entra.security_posture_score', 90);
|
|
expect(canonicalSecurityScores([model])[0]).toMatchObject({ method: 'pass-rate', nativeValue: undefined });
|
|
});
|
|
|
|
it('reads the bounded native PingCastle XML score through its exact mapped root path', () => {
|
|
const model = fixture('a', 'pingcastle', ['failed']);
|
|
const risk = observation(model, 'pingcastle.xml.healthcheckdata_0_globalscore_0', 45);
|
|
risk.attributes.xmlPath = '/HealthcheckData[0]/GlobalScore[0]';
|
|
const [score] = canonicalSecurityScores([model]);
|
|
expect(score).toMatchObject({ score: 55, nativeProperty: risk.property, method: 'inverted-native' });
|
|
});
|
|
|
|
it('keeps only selected-run environment evidence in workspace projections', () => {
|
|
const a = fixture('a'); observation(a, 'purple_knight.ad.security_posture_score', 65);
|
|
const b = fixture('b'); observation(b, 'purple_knight.ad.security_posture_score', 95);
|
|
const merged = { ...a, scanRuns: [...a.scanRuns, ...b.scanRuns], importFiles: [...a.importFiles, ...b.importFiles],
|
|
environmentObservations: [...a.environmentObservations!, ...b.environmentObservations!] };
|
|
expect(canonicalModelForScanRuns(merged, ['run-a'])?.environmentObservations?.map(item => item.scanRunId)).toEqual(['run-a']);
|
|
});
|
|
|
|
it('persists rule version and weights with the assessment through the existing repository contract', async () => {
|
|
const repository = new LocalAssessorRepository();
|
|
const model = fixture();
|
|
model.assessment.scoringPolicy!.weights['purple-knight'] = 2;
|
|
await repository.saveProject(model.project);
|
|
await repository.saveAssessment(model.assessment);
|
|
expect((await repository.listAssessments(model.project.id))[0].scoringPolicy).toEqual(model.assessment.scoringPolicy);
|
|
expect(weightedSecurityScore([{ score: 65, weight: 2 }, { score: 95, weight: 1 }])).toBe(75);
|
|
const policy = defaultSecurityScoringPolicy(); policy.weights['group-policy'] = 2;
|
|
expect(dashboardSecuritySources([{
|
|
tool: { id: 'gpos', label: 'GPOs', sourceValues: [], count: 2 }, records: [], passed: 1, deviations: 1, assessed: 2, score: 50
|
|
}], { policy, scans: [] })[0]).toMatchObject({ score: 50, weight: 2 });
|
|
});
|
|
|
|
it('retains missing-source obligations across successive timeslots without mixing historic findings into the workspace', async () => {
|
|
const old = fixture('old'); identity(old); old.assessment.effectiveAt = '2026-09-01T12:00:00Z';
|
|
const middle = fixture('middle', 'maester'); identity(middle, '2026-09-02T12:00:00Z'); middle.assessment.effectiveAt = '2026-09-02T12:00:00Z';
|
|
const current = fixture('current', 'maester'); identity(current, '2026-09-03T12:00:00Z'); current.assessment.effectiveAt = '2026-09-03T12:00:00Z';
|
|
const models = [old, middle, current];
|
|
const reader = {
|
|
listProjectAssessments: async () => models.map(model => model.assessment),
|
|
loadAssessmentModel: async (id: string) => models.find(model => model.assessment.id === id)!
|
|
};
|
|
const workspace = await loadCanonicalProjectWorkspace(reader, current.project.id, ['run-current'], ['current']);
|
|
expect(workspace.models).toHaveLength(1);
|
|
const context = workspace.dataset!.securityScoring!;
|
|
expect(context.scans.map(scan => scan.id)).toEqual(['file-current']);
|
|
expect(compareSecurityScores(context.baseline!, context.scans).awaitingRevalidation).toMatchObject([{ occurrenceIds: ['old-2'] }]);
|
|
});
|
|
});
|