Assessor_2.0/src/importers/pingcastle-source-extractor.ts

1284 lines
62 KiB
TypeScript

import type {
DecodedHierarchyNode,
DecodedHierarchyView,
DecodedSourceDocument,
ImportDiagnostic,
SourceLocator
} from '../domain/import-intermediate';
import type {
EnvironmentObservationCandidate,
FindingCandidate,
ScannerReportSectionCandidate,
SourceExtractor,
SourceFieldCandidate
} from '../domain/source-extraction';
import type { ImportTable } from '../domain/import-intermediate';
import { findHtmlTableSpans, markupText, sanitizeHtmlText } from './html-extraction';
import { protectHtmlAttributeMarkup } from './html-attribute-markup';
import { htmlAttributeValue as attributeValue } from './html-attributes';
import { indexHtmlElements, type HtmlElementIndex } from './html-element-index';
import { acceptPingCastleRuleId, pingCastleHtmlRuleCoverage } from './pingcastle-rule-validation';
import { derivePingCastleEnvironmentAggregates } from './pingcastle-environment-aggregates';
const maximumRuleCandidates = 5_000;
const maximumFieldMarkupCharacters = 1_000_000;
interface HtmlLabel {
index: number;
label: string;
value: string;
}
interface ResolvedAnchorSection {
sectionId: string;
sectionLabel: string;
text: string;
prose: string;
headers: string[];
rows: string[][];
}
interface DetailsExtras {
embedded?: string[][];
linked?: ResolvedAnchorSection[];
}
interface InternalReference {
index: number;
targetId: string;
relation: 'anchor' | 'collapse' | 'modal';
label: string;
}
interface ElementSpan {
start: number;
bodyStart: number;
bodyEnd: number;
end: number;
}
interface NamedSectionSpan extends ElementSpan {
sectionKey: string;
parentSectionKey?: string;
}
interface LinkResolutionIssue {
code: 'ambiguous' | 'depth-limit' | 'target-limit' | 'table-limit' | 'unresolved';
targetId: string;
}
interface XmlRuleGroup {
path: string;
root?: DecodedHierarchyNode;
direct: Map<string, DecodedHierarchyNode>;
details: DecodedHierarchyNode[];
}
interface PingCastleScores {
global?: { value: string; source: SourceLocator };
staleObjects?: { value: string; source: SourceLocator };
privilegedAccounts?: { value: string; source: SourceLocator };
trusts?: { value: string; source: SourceLocator };
anomalies?: { value: string; source: SourceLocator };
}
function safeId(value: string): string {
return value.replace(/[^a-zA-Z0-9_-]/g, '-');
}
function normalizedLabel(value: string): string {
return value.replace(/\s+/g, ' ').replace(/:$/, '').trim().toLocaleLowerCase('en-US');
}
function sourceField(
key: string,
label: string,
value: string,
source: SourceLocator
): SourceFieldCandidate {
return { key, label, originalValue: value, decodedValue: value, source };
}
// Traegt die vollstaendig aufgeloesten Detailquellen strukturiert (Header/Zeilen)
// zusaetzlich zum Fliesstext im Details-Feld, damit die Work-Ansicht daraus
// eigene Ergebnis-Tabellenblaetter fuer den Results-Popup bauen kann, ohne den
// Text erneut parsen zu muessen. Eine Regel kann mehrere Quellen tragen (die
// direkt eingebettete Beweistabelle UND ein Ankerziel), daher eine Liste.
// decodedValue traegt zur Anzeige nur die kurzen Abschnittsnamen.
function linkedSectionField(sections: ResolvedAnchorSection[], source: SourceLocator): SourceFieldCandidate {
return {
key: 'linked-section',
label: 'Verwandte Daten',
originalValue: {
sections: sections.map(section => ({
sectionId: section.sectionId,
sectionLabel: section.sectionLabel,
prose: section.prose,
headers: section.headers,
rows: section.rows
}))
},
decodedValue: sections.map(section => section.sectionLabel).join(', '),
source
};
}
function paragraphMatch(markup: string, start: number): { value: string; rawValue: string; end: number } {
const boundedEnd = Math.min(markup.length, start + maximumFieldMarkupCharacters);
const fragment = markup.slice(start, boundedEnd);
const paragraph = /^\s*<p\b[^>]*>([\s\S]*?)<\/p\s*>/i.exec(fragment);
if (paragraph) return {
value: markupText(paragraph[1] ?? ''),
rawValue: paragraph[1] ?? '',
end: start + paragraph.index + paragraph[0].length
};
const inlineRemainder = /^\s*([^<]{1,10000})\s*<\/p\s*>/i.exec(fragment);
if (inlineRemainder) return {
value: markupText(inlineRemainder[1] ?? ''),
rawValue: inlineRemainder[1] ?? '',
end: start + inlineRemainder.index + inlineRemainder[0].length
};
return { value: '', rawValue: '', end: start };
}
function tableRowsFromMarkup(tableBody: string): string[][] {
const rows: string[][] = [];
const rowPattern = /<tr\b[^>]*>([\s\S]*?)<\/tr\s*>/gi;
let rowMatch: RegExpExecArray | null;
while ((rowMatch = rowPattern.exec(tableBody))) {
const cells: string[] = [];
const cellPattern = /<t[hd]\b[^>]*>([\s\S]*?)<\/t[hd]\s*>/gi;
let cellMatch: RegExpExecArray | null;
while ((cellMatch = cellPattern.exec(rowMatch[1] ?? ''))) {
cells.push(markupText(cellMatch[1] ?? ''));
}
if (cells.some(Boolean)) rows.push(cells);
}
return rows;
}
// PingCastle rendert nach "Details:" häufig direkt die Beweistabelle (z. B. betroffene
// DNs). Wird nur unmittelbar (bis auf umschließende <div>) folgend erfasst,
// um keine spätere, unabhängige Tabelle fälschlich anzuhängen.
function embeddedTableRowsAfter(markup: string, start: number): string[][] {
const boundedEnd = Math.min(markup.length, start + maximumFieldMarkupCharacters);
const fragment = markup.slice(start, boundedEnd);
const directTable = /^(?:\s|<div\b[^>]*>|<\/div\s*>)*<table\b[^>]*>([\s\S]*?)<\/table\s*>/i.exec(fragment);
if (!directTable) return [];
return tableRowsFromMarkup(directTable[1] ?? '');
}
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
const maximumAnchorSectionCharacters = 200_000;
const maximumLinkedSectionDepth = 8;
const maximumLinkedSectionTargets = 128;
const maximumLinkedSectionTables = 256;
function allSectionTables(markup: string): string[][][] {
const tables: string[][][] = [];
const tablePattern = /<table\b[^>]*>([\s\S]*?)<\/table\s*>/gi;
let match: RegExpExecArray | null;
while ((match = tablePattern.exec(markup))) {
const rows = tableRowsFromMarkup(match[1] ?? '');
if (rows.length) tables.push(rows);
}
return tables;
}
function proseText(markup: string): string {
const withoutTables = markup.replace(/<table\b[^>]*>[\s\S]*?<\/table\s*>/gi, ' ');
return markupText(withoutTables);
}
// PingCastles "Details:" verweist bei vielen Regeln nicht auf eine eingebettete
// Tabelle, sondern per internem Anker (<a href="#id">Label</a>) auf einen
// eigenstaendigen Datenabschnitt (z. B. "LAPS", "Admin Groups", "Domain
// controllers") an anderer Stelle im selben Dokument. Der Abschnitt ist per
// "<!-- SubSection Name --><a name=\"id\"></a> ... <!-- SubSection Name end -->
// <Inhalt> <!-- SubSection Naechster Name -->" strukturell begrenzt. Ohne
// Aufloesung bliebe nur der einleitende Verweissatz sichtbar, die eigentliche
// Information ("bis zum Endpunkt der Verlinkung") ginge verloren.
const subSectionEndMarker = '<i data-subsection-end="1"></i>';
const subSectionStartMarker = '<i data-subsection-start="1"></i>';
function subsectionMarkup(markup: string, anchorId: string, index: HtmlElementIndex): string | null {
const anchor = index.namedAnchors.get(anchorId);
if (!anchor) return null;
const closing = /^\s*<\/a\s*>/i.exec(markup.slice(anchor.bodyStart, anchor.bodyStart + 256));
const afterAnchor = anchor.bodyStart + (closing?.[0].length ?? 0);
const boundedEnd = Math.min(markup.length, afterAnchor + maximumAnchorSectionCharacters);
const fragment = markup.slice(afterAnchor, boundedEnd);
const ownEndMarkerIndex = fragment.indexOf(subSectionEndMarker);
const contentStart = ownEndMarkerIndex >= 0 ? ownEndMarkerIndex + subSectionEndMarker.length : 0;
const rest = fragment.slice(contentStart);
const nextStart = rest.indexOf(subSectionStartMarker);
const nextEnd = rest.indexOf(subSectionEndMarker);
const nextSectionIndex = [nextStart, nextEnd].filter(index => index >= 0).sort((left, right) => left - right)[0] ?? -1;
return nextSectionIndex >= 0 ? rest.slice(0, nextSectionIndex) : rest;
}
function elementSpanById(markup: string, targetId: string, index: HtmlElementIndex): ElementSpan | null {
const opening = index.byId.get(targetId);
if (!opening) return null;
const { tagName, start, bodyStart, selfClosing } = opening;
if (selfClosing) return { start, bodyStart, bodyEnd: bodyStart, end: bodyStart };
const tagPattern = new RegExp('<' + escapeRegExp(tagName) + '\\b[^>]*>|<\\/' + escapeRegExp(tagName) + '\\s*>', 'gi');
tagPattern.lastIndex = bodyStart;
let depth = 1;
let tag: RegExpExecArray | null;
while ((tag = tagPattern.exec(markup))) {
if (/^<\//.test(tag[0])) depth -= 1;
else if (!/\/\s*>$/.test(tag[0])) depth += 1;
if (depth === 0) return { start, bodyStart, bodyEnd: tag.index, end: tagPattern.lastIndex };
}
return null;
}
function referenceLabelAt(markup: string, markerEnd: number, fallback: string): string {
const bounded = markup.slice(markerEnd, Math.min(markup.length, markerEnd + 4_096));
const closing = /<\/(?:a|button)\s*>/i.exec(bounded);
const label = markupText(closing ? bounded.slice(0, closing.index) : bounded.slice(0, 256));
return label || fallback;
}
function internalReferences(markup: string): InternalReference[] {
const references: InternalReference[] = [];
const seen = new Set<string>();
const pattern = /<i\s+data-pingcastle-target="([a-zA-Z0-9_.:-]+)"\s+data-pingcastle-relation="(anchor|collapse|modal)"\s*>\s*<\/i\s*>/gi;
let match: RegExpExecArray | null;
while ((match = pattern.exec(markup))) {
const targetId = match[1];
const relation = match[2].toLocaleLowerCase('en-US') as InternalReference['relation'];
const identity = relation + '|' + targetId;
if (seen.has(identity)) continue;
seen.add(identity);
references.push({
index: match.index,
targetId,
relation,
label: referenceLabelAt(markup, pattern.lastIndex, targetId)
});
}
return references;
}
function targetMarkup(markup: string, targetId: string, index: HtmlElementIndex): { markup: string; label: string } | null {
const subsection = subsectionMarkup(markup, targetId, index);
if (subsection !== null) return { markup: subsection, label: targetId };
const span = elementSpanById(markup, targetId, index);
if (!span) return null;
const body = markup.slice(span.bodyStart, span.bodyEnd);
const title = /<h[1-6]\b[^>]*class\s*=\s*(?:"[^"]*\bmodal-title\b[^"]*"|'[^']*\bmodal-title\b[^']*')[^>]*>([\s\S]*?)<\/h[1-6]\s*>/i.exec(body);
return { markup: body, label: markupText(title?.[1] ?? '') || targetId };
}
function maskRanges(value: string, ranges: ElementSpan[]): string {
if (!ranges.length) return value;
const ordered = ranges
.map(range => ({ start: Math.max(0, range.start), end: Math.min(value.length, range.end) }))
.filter(range => range.end > range.start)
.sort((left, right) => left.start - right.start);
let output = '';
let cursor = 0;
ordered.forEach(range => {
if (range.end <= cursor) return;
const start = Math.max(cursor, range.start);
output += value.slice(cursor, start) + ' '.repeat(range.end - start);
cursor = range.end;
});
return output + value.slice(cursor);
}
function resolveLinkedSections(
markup: string,
roots: InternalReference[],
elementIndex: HtmlElementIndex
): { sections: ResolvedAnchorSection[]; text: string; issues: LinkResolutionIssue[] } {
const sections: ResolvedAnchorSection[] = [];
const textBlocks: string[] = [];
const visited = new Set<string>();
const issues: LinkResolutionIssue[] = [];
const issueKeys = new Set<string>();
const addIssue = (code: LinkResolutionIssue['code'], targetId: string) => {
const key = code + '|' + targetId;
if (issueKeys.has(key)) return;
issueKeys.add(key);
issues.push({ code, targetId });
};
const visit = (reference: InternalReference, parentLabel: string, depth: number) => {
if (depth > maximumLinkedSectionDepth) {
addIssue('depth-limit', reference.targetId);
return;
}
if (visited.size >= maximumLinkedSectionTargets) {
addIssue('target-limit', reference.targetId);
return;
}
if (visited.has(reference.targetId)) return;
visited.add(reference.targetId);
const definitionCount = elementIndex.definitions.get(reference.targetId)?.count ?? 0;
if (definitionCount === 0) {
addIssue('unresolved', reference.targetId);
return;
}
if (definitionCount > 1) {
addIssue('ambiguous', reference.targetId);
return;
}
const target = targetMarkup(markup, reference.targetId, elementIndex);
if (!target) {
addIssue('unresolved', reference.targetId);
return;
}
const label = target.label === reference.targetId ? reference.label : target.label;
const sectionLabel = parentLabel ? parentLabel + ' → ' + label : label;
const allReferences = internalReferences(target.markup);
const targetIndex = indexHtmlElements(target.markup);
const referencedSpans = allReferences
.map(child => ({ child, span: elementSpanById(target.markup, child.targetId, targetIndex) }))
.filter((entry): entry is { child: InternalReference; span: ElementSpan } => Boolean(entry.span));
const directReferences = allReferences.filter(candidate => !referencedSpans.some(entry =>
entry.child.targetId !== candidate.targetId
&& candidate.index >= entry.span.start
&& candidate.index < entry.span.end
));
// Zielcontainer koennen als Geschwister im Subsection-Markup liegen,
// obwohl ihr Verweis erst innerhalb eines anderen Zielcontainers steht.
// Auch diese Definitionen gehoeren nicht zum lokalen Elterninhalt; die
// Rekursion erreicht sie ueber den tatsaechlichen Verweispfad.
const childSpans = referencedSpans.map(entry => entry.span);
const ownMarkup = maskRanges(target.markup, childSpans);
const prose = proseText(ownMarkup).trim();
const tables = allSectionTables(ownMarkup);
const tablesText = tables.map(rows => rows.map(row => row.join(' | ')).join('\n')).join('\n\n');
const text = [prose, tablesText].filter(Boolean).join('\n\n');
if (text) textBlocks.push(text);
if (tables.length) {
const remainingTables = maximumLinkedSectionTables - sections.length;
if (tables.length > remainingTables) addIssue('table-limit', reference.targetId);
tables.slice(0, Math.max(0, remainingTables)).forEach((table, index) => {
const hasMultipleTables = tables.length > 1;
sections.push({
sectionId: hasMultipleTables ? reference.targetId + '--table-' + (index + 1) : reference.targetId,
sectionLabel: hasMultipleTables ? sectionLabel + ' · Tabelle ' + (index + 1) : sectionLabel,
text: table.map(row => row.join(' | ')).join('\n'),
prose,
headers: table[0] ?? [],
rows: table.slice(1)
});
});
} else if (prose) {
sections.push({
sectionId: reference.targetId,
sectionLabel,
text: prose,
prose,
headers: ['Inhalt'],
rows: [[prose]]
});
}
// PingCastle kann einen Link frueher anzeigen als den zugehoerigen Modal-
// oder Collapse-Container. Fuer den vollstaendigen Detailspfad gilt die
// Reihenfolge der tatsaechlichen Zielabschnitte im Bericht, nicht die
// zufaellige Position ihrer Ausloeser.
directReferences
.sort((left, right) => (elementIndex.definitions.get(left.targetId)?.start ?? Number.MAX_SAFE_INTEGER)
- (elementIndex.definitions.get(right.targetId)?.start ?? Number.MAX_SAFE_INTEGER)
|| left.index - right.index)
.forEach(child => visit(child, sectionLabel, depth + 1));
};
roots.forEach(root => visit(root, '', 0));
return { sections, text: textBlocks.join('\n\n'), issues };
}
function htmlLabels(markup: string): {
labels: HtmlLabel[];
detailsExtras: Map<number, DetailsExtras>;
linkIssues: LinkResolutionIssue[];
} {
const labels: HtmlLabel[] = [];
const detailsExtras = new Map<number, DetailsExtras>();
const linkIssues: LinkResolutionIssue[] = [];
const elementIndex = indexHtmlElements(markup);
const pattern = /<strong\b[^>]*>([\s\S]{0,2048}?)<\/strong\s*>/gi;
let match: RegExpExecArray | null;
while ((match = pattern.exec(markup)) && labels.length < maximumRuleCandidates * 16) {
const label = normalizedLabel(markupText(match[1] ?? ''));
if (!label) continue;
const paragraph = paragraphMatch(markup, pattern.lastIndex);
let value = paragraph.value;
if (label === 'details') {
// Eine Regel kann beides tragen: die direkt eingebettete Beweistabelle
// UND einen Ankerverweis auf einen weiteren Datenabschnitt (z. B.
// A-DC-Spooler: betroffene DC-Liste inline plus "Domain controllers").
// Beide Quellen werden erfasst; keine ersetzt die andere.
const embedded = embeddedTableRowsAfter(markup, paragraph.end);
if (embedded.length) {
const embeddedText = embedded.map(row => row.join(' | ')).join('\n');
value = value ? `${value}\n${embeddedText}` : embeddedText;
}
const resolved = resolveLinkedSections(markup, internalReferences(paragraph.rawValue), elementIndex);
linkIssues.push(...resolved.issues);
if (resolved.text) value = value ? `${value}\n${resolved.text}` : resolved.text;
if (embedded.length > 1 || resolved.sections.length) {
detailsExtras.set(match.index, {
embedded: embedded.length > 1 ? embedded : undefined,
linked: resolved.sections.length ? resolved.sections : undefined
});
}
}
labels.push({ index: match.index, label, value });
}
return { labels, detailsExtras, linkIssues };
}
function precedingHeading(markup: string, start: number, tag: 'h2' | 'h3', className?: string): string {
const lowerBound = Math.max(0, start - 250_000);
const fragment = markup.slice(lowerBound, start);
const pattern = new RegExp('<' + tag + '\\b([^>]*)>([\\s\\S]*?)<\\/' + tag + '\\s*>', 'gi');
let selected = '';
let match: RegExpExecArray | null;
while ((match = pattern.exec(fragment))) {
if (className) {
const classMatch = /class\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(match[1] ?? '');
const classes = classMatch?.[1] ?? classMatch?.[2] ?? '';
if (!new RegExp('(?:^|\\s)' + className + '(?:\\s|$)', 'i').test(classes)) continue;
}
selected = markupText(match[2] ?? '');
}
return selected;
}
// PingCastle benennt einen Fund im Card-Header (identisch zur Rationale der
// XML-Exportvariante, z. B. "The spooler service is remotely accessible from
// 1 DC"), waehrend die h3-Ueberschrift im Regelkoerper die generische
// Pruefbeschreibung des Regelkatalogs traegt ("Ensure that ..."). Der Titel
// folgt dem Card-Header, damit HTML- und XML-Importe denselben Fundnamen
// erzeugen; die Pruefbeschreibung bleibt als eigenes Feld erhalten. floor
// begrenzt die Suche auf den Bereich nach der vorherigen Regel, damit ein
// regelloser Block nie den Header einer fremden Regel erbt.
function precedingCardTitle(markup: string, start: number, floor: number): string {
const lowerBound = Math.max(0, floor, start - 4_000);
const fragment = markup.slice(lowerBound, start);
const pattern = /<span\s+class\s*=\s*["'][^"']*\bcard-title\b[^"']*["'][^>]*>([\s\S]*?)<\/span\s*>/gi;
let selected = '';
let match: RegExpExecArray | null;
while ((match = pattern.exec(fragment))) selected = markupText(match[1] ?? '');
return selected;
}
// Die "Points:"-Beschriftung im Regelkoerper enthaelt bei vielen Regeln die
// statische Punkteschema-Beschreibung (z. B. "50 points if the occurence is
// greater than or equals than 50 then 45 ... then 15 points if present"), nicht
// die tatsaechlich vergebene Punktzahl. Diese steht separat im vorangehenden
// Card-Header als "+ N Point(s)" beziehungsweise als "Informative rule" bei
// Regeln mit 0 Punkten. Ohne diesen Wert bleibt score/severity unbestimmt statt
// erfunden.
function precedingAwardedPoints(markup: string, start: number): string {
const lowerBound = Math.max(0, start - 4_000);
const fragment = markup.slice(lowerBound, start);
const awarded = [...fragment.matchAll(/\+\s*([\d.]+)\s*Point\(s\)/gi)];
const informative = [...fragment.matchAll(/\bInformative rule\b/gi)];
const lastAwarded = awarded[awarded.length - 1];
const lastInformative = informative[informative.length - 1];
if (!lastAwarded && !lastInformative) return '';
if (!lastInformative || (lastAwarded && lastAwarded.index! > lastInformative.index!)) return lastAwarded![1];
return '0';
}
// sanitizeHtmlText entfernt href als potenziell gefaehrliches Attribut - richtig
// fuer echte Navigationsziele, wuerde hier aber auch harmlose interne
// Dokumentanker (#id) unlesbar machen, bevor "Details:" sie zum Aufloesen
// braucht. Nur ein eng begrenztes #<sicheres-Zeichenset>-Fragment wird deshalb
// vor der Sanitisierung in ein inertes data-Attribut ueberfuehrt; jeder andere
// href-Wert (externe URLs, javascript:, data: etc.) bleibt unveraendert der
// regulaeren Entfernung unterworfen.
function preserveInternalAnchorHrefs(rawText: string): string {
return rawText.replace(/\bhref\s*=\s*"#([a-zA-Z0-9_.:-]+)"/gi, 'data-anchor-target="$1"');
}
// Interne PingCastle-Navigation wird als inerte Markierung vor der allgemeinen
// HTML-Bereinigung gesichert. Das ist insbesondere fuer Collapse-Schaltflaechen
// notwendig: <button> wird vom Sanitizer vollstaendig entfernt, sein strikt
// lokales data-bs-target darf als reine Strukturinformation aber erhalten
// bleiben. Es werden weder externe Ziele noch Quellattribute durchgereicht.
function preserveInternalReferences(rawText: string): string {
return rawText.replace(/<([a-zA-Z][\w:-]*)\b([^>]*)>/g, (tag, _tagName: string, attributes: string) => {
const toggle = attributeValue(attributes, 'data-bs-toggle').toLocaleLowerCase('en-US');
const candidates = [
attributeValue(attributes, 'href'),
attributeValue(attributes, 'data-bs-target'),
attributeValue(attributes, 'aria-controls')
];
const targets = new Set<string>();
candidates.forEach(candidate => {
const target = /^#?([a-zA-Z0-9_.:-]+)$/.exec(candidate.trim())?.[1];
if (target) targets.add(target);
});
if (!targets.size) return tag;
const relation: InternalReference['relation'] = toggle === 'modal'
? 'modal'
: toggle === 'collapse'
? 'collapse'
: 'anchor';
return tag + [...targets].map(target =>
`<i data-pingcastle-target="${target}" data-pingcastle-relation="${relation}"></i>`
).join('');
});
}
// sanitizeHtmlText entfernt saemtliche HTML-Kommentare - richtig gegen
// versteckten aktiven Inhalt, wuerde aber auch die "<!-- SubSection Name -->"
// / "<!-- SubSection Name end -->"-Grenzmarker loeschen, auf denen
// resolveAnchorSection basiert. Nur dieses eine, PingCastle-eigene
// Kommentarmuster wird deshalb vor der Sanitisierung in inerte, attributlose
// Marker-Tags ueberfuehrt; jeder andere Kommentar wird weiterhin regulaer
// entfernt.
function preserveSubSectionMarkers(rawText: string): string {
return rawText.replace(/<!--\s*SubSection\b[\s\S]*?-->/gi, match =>
/\bend\s*-->$/i.test(match) ? subSectionEndMarker : subSectionStartMarker
);
}
function htmlCandidates(document: DecodedSourceDocument): {
candidates: FindingCandidate[];
diagnostics: ImportDiagnostic[];
reportSections: ScannerReportSectionCandidate[];
environmentObservations: EnvironmentObservationCandidate[];
} {
const preprocessed = preserveSubSectionMarkers(preserveInternalAnchorHrefs(preserveInternalReferences(protectHtmlAttributeMarkup(document.rawText))));
const markup = sanitizeHtmlText(preprocessed).text;
const reportText = markupText(markup.slice(0, Math.min(markup.length, 500_000)));
const scoreValue = (pattern: RegExp) => pattern.exec(reportText)?.[1] ?? '';
const reportScores: PingCastleScores = {
global: { value: scoreValue(/Domain Risk Level\s*:\s*(\d+)\s*\/\s*100/i), source: { fileName: document.file.name, objectPath: 'html/risk-score' } },
staleObjects: { value: scoreValue(/Stale Objects?\s*:\s*(\d+)\s*\/\s*100/i), source: { fileName: document.file.name, objectPath: 'html/risk-score/stale-objects' } },
privilegedAccounts: { value: scoreValue(/Privileged Accounts\s*:\s*(\d+)\s*\/\s*100/i), source: { fileName: document.file.name, objectPath: 'html/risk-score/privileged-accounts' } },
trusts: { value: scoreValue(/Trusts\s*:\s*(\d+)\s*\/\s*100/i), source: { fileName: document.file.name, objectPath: 'html/risk-score/trusts' } },
anomalies: { value: scoreValue(/Anomalies\s*:\s*(\d+)\s*\/\s*100/i), source: { fileName: document.file.name, objectPath: 'html/risk-score/anomalies' } }
};
const { labels, detailsExtras, linkIssues } = htmlLabels(markup);
const ruleLabels = labels.filter(item => item.label === 'rule id');
const candidates: FindingCandidate[] = [];
const ruleDiagnostics: ImportDiagnostic[] = [];
const seenRuleIds = new Set<string>();
for (let ruleIndex = 0; ruleIndex < ruleLabels.length && candidates.length < maximumRuleCandidates; ruleIndex += 1) {
const rule = ruleLabels[ruleIndex];
const ruleId = rule.value.trim();
if (!acceptPingCastleRuleId(ruleId, seenRuleIds, { fileName: document.file.name, objectPath: `html/rule-blocks/${ruleIndex}` }, ruleDiagnostics)) continue;
const nextRuleIndex = ruleLabels[ruleIndex + 1]?.index ?? markup.length;
const blockLabels = labels.filter(item => item.index >= rule.index && item.index < nextRuleIndex);
const firstValue = (...names: string[]) =>
blockLabels.find(item => names.includes(item.label) && item.value)?.value ?? '';
const checkTitle = precedingHeading(markup, rule.index, 'h3');
const cardTitle = precedingCardTitle(markup, rule.index, ruleLabels[ruleIndex - 1]?.index ?? 0);
const title = cardTitle || checkTitle || ruleId;
const rawCategory = precedingHeading(markup, rule.index, 'h2', 'indicator-header');
const category = rawCategory.replace(/\s+rule details\b[\s\S]*$/i, '').replace(/\s*\[\s*\d+\s+rules?\s+matched[\s\S]*$/i, '').trim();
const locator: SourceLocator = { fileName: document.file.name, objectPath: 'html/rules/' + ruleId };
const fields: SourceFieldCandidate[] = [
sourceField('risk-id', 'Rule ID', ruleId, locator),
sourceField('title', 'Title', title, locator),
sourceField('status', 'Status', 'Matched', locator)
];
const values: Array<[string, string, string]> = [
['description', 'Description', firstValue('description')],
['technical-explanation', 'Technical explanation', firstValue('technical explanation')],
['solution', 'Advised solution', firstValue('advised solution')],
['points-scale', 'Points Scale', firstValue('points')],
['points', 'Points', precedingAwardedPoints(markup, rule.index)],
['documentation', 'Documentation', firstValue('documentation')],
['details', 'Details', firstValue('details')]
];
if (checkTitle && checkTitle !== title) values.push(['check-title', 'Check title', checkTitle]);
if (category) values.push(['category', 'Category', category]);
values
.filter(([, , value]) => value)
.forEach(([key, label, value]) => fields.push(sourceField(key, label, value, locator)));
const detailsLabel = blockLabels.find(item => item.label === 'details' && item.value);
const extras = detailsLabel ? detailsExtras.get(detailsLabel.index) : undefined;
const sections: ResolvedAnchorSection[] = [];
if (extras?.embedded) {
sections.push({
sectionId: 'rule-details-' + safeId(ruleId),
sectionLabel: 'Details',
text: '',
prose: '',
headers: extras.embedded[0] ?? [],
rows: extras.embedded.slice(1)
});
}
if (extras?.linked) sections.push(...extras.linked);
if (sections.length) fields.push(linkedSectionField(sections, locator));
appendScoreFields(fields, category, reportScores);
candidates.push({
version: 1,
id: 'candidate-pingcastle-html-' + safeId(ruleId) + '-' + candidates.length,
source: locator,
fields
});
}
const diagnostics: ImportDiagnostic[] = linkIssues.map(issue => ({
severity: 'warning',
code: 'pingcastle-linked-section-' + issue.code,
message: issue.code === 'unresolved'
? 'Ein interner PingCastle-Detailverweis konnte nicht aufgelöst werden.'
: issue.code === 'ambiguous'
? 'Ein interner PingCastle-Detailverweis besitzt mehrere gleichnamige Ziele und wurde nicht automatisch aufgelöst.'
: 'Die rekursive PingCastle-Detailauflösung wurde durch ein Ressourcenlimit begrenzt.',
stage: 'extraction',
source: { fileName: document.file.name, objectPath: 'html/linked-sections/' + issue.targetId },
details: { targetId: issue.targetId, reason: issue.code }
}));
if (ruleLabels.length > maximumRuleCandidates) diagnostics.push({
severity: 'warning',
code: 'pingcastle-rule-limit',
message: 'Der PingCastle-Bericht enthält mehr als ' + maximumRuleCandidates + ' Regelblöcke; die Extraktion wurde begrenzt.',
stage: 'extraction'
});
const sections = reportSections(document);
const coverage = pingCastleHtmlRuleCoverage(document.file.name, markup, candidates);
// Coverage is source metadata, not another set of findings or environment assets.
if (coverage.section) sections.push({ ...coverage.section, ordinal: sections.length });
return {
candidates,
diagnostics: [...diagnostics, ...ruleDiagnostics, ...coverage.diagnostics],
reportSections: sections,
environmentObservations: contextObservations(document, sections.filter(section => section.sectionKey !== 'pingcastle-rule-coverage'), reportScores)
};
}
const excludedFindingSections = new Set([
'sectionStaleObjects',
'sectionPrivilegedAccounts',
'sectionTrusts',
'sectionAnomaliesanalysis'
]);
function sectionSpans(markup: string, includeExcluded = false, elementIndex = indexHtmlElements(markup)): NamedSectionSpan[] {
const candidates: NamedSectionSpan[] = [];
const pattern = /<div\b([^>]*)>/gi;
let match: RegExpExecArray | null;
const seen = new Set<string>();
while ((match = pattern.exec(markup))) {
const sectionKey = attributeValue(match[1] ?? '', 'id');
if (!/^section[a-zA-Z0-9_.:-]+$/i.test(sectionKey) || seen.has(sectionKey)) continue;
const span = elementSpanById(markup, sectionKey, elementIndex);
if (!span) continue;
seen.add(sectionKey);
candidates.push({ ...span, sectionKey });
}
candidates.sort((left, right) => left.start - right.start || right.end - left.end);
candidates.forEach(section => {
const parent = candidates
.filter(candidate => candidate.sectionKey !== section.sectionKey
&& candidate.bodyStart <= section.start && candidate.end >= section.end)
.sort((left, right) => (left.end - left.start) - (right.end - right.start))[0];
if (parent) section.parentSectionKey = parent.sectionKey;
});
const excluded = new Set<string>();
candidates.forEach(section => {
if (excludedFindingSections.has(section.sectionKey)
|| (section.parentSectionKey && excluded.has(section.parentSectionKey))) excluded.add(section.sectionKey);
});
return includeExcluded ? candidates : candidates.filter(section => !excluded.has(section.sectionKey));
}
function tableMatrix(table: ImportTable): { headers: string[]; rows: string[][] } {
const header = table.headerCandidates[0];
const headers = header?.headers ?? table.columns.map(column => column.headerCandidates[0]?.value ?? `Spalte ${column.index + 1}`);
const rows = table.rows
.filter(row => row.index !== header?.rowIndex)
.map(row => row.cells.map(cell => cell.decodedValue));
return { headers, rows };
}
function sectionState(prose: string, tables: ScannerReportSectionCandidate['tables']): ScannerReportSectionCandidate['state'] {
if (/reserved for customers who have purchased a license/i.test(prose)) return 'license-restricted';
if (/access denied/i.test(prose) && !tables.some(table => table.rows.length)) return 'access-denied';
if (tables.some(table => table.rows.length)) return 'observed';
if (/\bno\s+[^.\n]{0,160}\s+(?:has|have|were|was)\s+been\s+found\b/i.test(prose)
|| /\bnumber of [^:\n]{0,100}:\s*0\b/i.test(prose)) return 'explicitly-absent';
if (prose.trim()) return 'observed';
if (tables.length) return 'empty';
return 'empty';
}
function reportSections(document: DecodedSourceDocument): ScannerReportSectionCandidate[] {
const markup = sanitizeHtmlText(document.rawText).text;
const elementIndex = indexHtmlElements(markup);
const sections = sectionSpans(markup, false, elementIndex);
const allSections = sectionSpans(markup, true, elementIndex);
const tableSpans = findHtmlTableSpans(markup);
const rootTableIndexes = tableSpans
.map((span, index) => ({ span, index }))
.filter(({ span }) => !allSections.some(section => span.start >= section.start && span.end <= section.end))
.map(item => item.index)
.filter(index => document.tables[index]);
const rootTables = rootTableIndexes.map(index => {
const table = document.tables[index];
const matrix = tableMatrix(table);
return { name: table.name, sourceTableIndex: table.index, headers: matrix.headers, rows: matrix.rows };
});
const root: ScannerReportSectionCandidate = {
version: 1, sectionKey: 'report-overview', label: 'Report overview', ordinal: 0,
state: rootTables.some(table => table.rows.length) ? 'observed' : 'empty', tables: rootTables,
source: { fileName: document.file.name, objectPath: 'html/report-overview' },
attributes: { tableCount: rootTables.length }
};
return [root, ...sections.map((section, ordinal): ScannerReportSectionCandidate => {
const directChildren = sections.filter(candidate => candidate.parentSectionKey === section.sectionKey);
const ownMarkup = maskRanges(markup.slice(section.bodyStart, section.bodyEnd), directChildren.map(child => ({
start: child.start - section.bodyStart,
bodyStart: child.bodyStart - section.bodyStart,
bodyEnd: child.bodyEnd - section.bodyStart,
end: child.end - section.bodyStart
})));
const label = markupText(/<h[1-6]\b[^>]*>([\s\S]*?)<\/h[1-6]\s*>/i.exec(ownMarkup)?.[1] ?? '') || section.sectionKey;
const prose = proseText(ownMarkup.replace(/<h[1-6]\b[^>]*>[\s\S]*?<\/h[1-6]\s*>/gi, ' ')).trim();
const tableIndexes = tableSpans
.map((span, index) => ({ span, index }))
.filter(({ span }) => span.start >= section.bodyStart && span.end <= section.bodyEnd
&& !directChildren.some(child => span.start >= child.start && span.end <= child.end))
.map(item => item.index)
.filter(index => document.tables[index]);
const tables = tableIndexes.map(index => {
const table = document.tables[index];
const matrix = tableMatrix(table);
return { name: table.name, sourceTableIndex: table.index, headers: matrix.headers, rows: matrix.rows };
});
return {
version: 1,
sectionKey: section.sectionKey,
parentSectionKey: section.parentSectionKey,
label,
ordinal: ordinal + 1,
state: sectionState(prose, tables),
prose: prose || undefined,
tables,
source: { fileName: document.file.name, objectPath: `html/sections/${section.sectionKey}` },
attributes: { tableCount: tables.length }
};
})];
}
function propertyKey(value: string): string {
return value.toLocaleLowerCase('en-US').replace(/[^a-z0-9]+/g, '.').replace(/^\.|\.$/g, '') || 'value';
}
function semanticProperty(sectionKey: string, tableName: string, label: string): string | undefined {
const normalized = normalizedLabel(label);
const domainProperties: Record<string, string> = {
domain: 'domain.fqdn',
'netbios name': 'domain.netbios_name',
'domain functional level': 'domain.functional_level',
'forest functional level': 'forest.functional_level',
'creation date': 'domain.created_at',
'dc count': 'domain.controller_count',
'number of dc': 'domain.controller_count',
'schema version': 'forest.schema_version',
'recycle bin enabled': 'domain.recycle_bin_enabled'
};
if (/domaininformation/i.test(sectionKey) || /domain\s*information|directory/i.test(tableName)) {
if (domainProperties[normalized]) return domainProperties[normalized];
}
const properties: Record<string, string> = {
'operating system': 'operating_system.name',
'nb os': 'operating_system.count',
'trust partner': 'trust.partner',
'trust direction': 'trust.direction',
'trust type': 'trust.type',
'password policy': 'password_policy.name',
'minimum password length': 'password_policy.minimum_length',
'password history': 'password_policy.history_length',
'maximum password age': 'password_policy.maximum_age',
'certificate authority': 'pki.certificate_authority.name',
'certificate template': 'pki.certificate_template.name'
};
return properties[normalized];
}
function subjectKey(value: string): string {
return value.toLocaleLowerCase('en-US').replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'value';
}
function subsectionContext(document: DecodedSourceDocument): {
tableLabels: Map<number, string>;
headings: Array<{ sectionKey: string; label: string }>;
} {
const markup = sanitizeHtmlText(document.rawText).text;
const sections = sectionSpans(markup, true);
const headings: Array<{ start: number; label: string }> = [];
const pattern = /<h2\b[^>]*>([\s\S]*?)<\/h2\s*>/gi;
let match: RegExpExecArray | null;
while ((match = pattern.exec(markup))) {
const label = markupText(match[1] ?? '').trim();
if (label) headings.push({ start: match.index, label });
}
const tableLabels = new Map<number, string>();
findHtmlTableSpans(markup).forEach((table, tableOrdinal) => {
const containingSections = sections
.filter(section => table.start >= section.bodyStart && table.end <= section.bodyEnd)
.sort((left, right) => (right.end - right.start) - (left.end - left.start));
const rootSection = containingSections[0];
if (!rootSection) return;
const heading = headings
.filter(candidate => candidate.start >= rootSection.bodyStart && candidate.start < table.start)
.at(-1);
const sourceTableIndex = document.tables[tableOrdinal]?.index;
if (heading && sourceTableIndex !== undefined) tableLabels.set(sourceTableIndex, heading.label);
});
return {
tableLabels,
headings: headings.flatMap(heading => {
const section = sections
.filter(candidate => heading.start >= candidate.bodyStart && heading.start < candidate.bodyEnd)
.sort((left, right) => (left.end - left.start) - (right.end - right.start))[0];
return section ? [{ sectionKey: section.sectionKey, label: heading.label }] : [];
})
};
}
function semanticSubjectBase(sectionKey: string): string | undefined {
return /domaininformation/i.test(sectionKey) ? 'domain-information'
: /computerinformation|computer/i.test(sectionKey) ? 'computer-information'
: /user/i.test(sectionKey) ? 'user-information'
: /admingroups/i.test(sectionKey) ? 'admin-groups'
: /controlpathsanalysis/i.test(sectionKey) ? 'control-paths-analysis'
: /infrastructure/i.test(sectionKey) ? 'infrastructure'
: /anomalies/i.test(sectionKey) ? 'anomalies'
: /passwordpolicies/i.test(sectionKey) ? 'password-policies'
: /^sectiongpo$/i.test(sectionKey) ? 'gpo'
: /trust/i.test(sectionKey) ? 'trusts-details'
: /pki|certificate/i.test(sectionKey) ? 'pki'
: undefined;
}
function semanticSubject(sectionKey: string, subsectionLabel: string | undefined, tableName: string, headers: string[]): string {
const normalizedHeaders = new Set(headers.map(normalizedLabel));
if (/neverexpiresuser/i.test(sectionKey)) return 'user-information.password-age-distribution';
if (/enabledtrusteddelegationcomputer/i.test(sectionKey)) return 'computer-information.account-analysis';
if (/userinformation/i.test(sectionKey) && normalizedHeaders.has('nb user accounts')) {
return 'user-information.account-analysis';
}
if (/userinformation/i.test(sectionKey) && /^account\s+list$/i.test(tableName)) {
return 'user-information.honey-pot';
}
const base = semanticSubjectBase(sectionKey);
if (!base) return 'pingcastle-table-row';
return subsectionLabel ? `${base}.${subjectKey(subsectionLabel)}` : base;
}
function typedValue(value: string): { normalizedValue: string | number | boolean; valueType: EnvironmentObservationCandidate['valueType'] } {
const trimmed = value.trim();
if (/^(?:true|yes)$/i.test(trimmed)) return { normalizedValue: true, valueType: 'boolean' };
if (/^(?:false|no)$/i.test(trimmed)) return { normalizedValue: false, valueType: 'boolean' };
if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) return { normalizedValue: Number(trimmed), valueType: 'number' };
if (/^\d{4}-\d{2}-\d{2}(?:[ T].*)?Z?$/i.test(trimmed)) return { normalizedValue: trimmed, valueType: 'datetime' };
return { normalizedValue: trimmed, valueType: 'string' };
}
function addObservation(
output: EnvironmentObservationCandidate[],
input: Omit<EnvironmentObservationCandidate, 'version' | 'attributes'> & { attributes?: EnvironmentObservationCandidate['attributes'] }
) {
output.push({ version: 1, attributes: {}, ...input });
}
function contextObservations(
document: DecodedSourceDocument,
sections: ScannerReportSectionCandidate[],
scores: PingCastleScores
): EnvironmentObservationCandidate[] {
const observations: EnvironmentObservationCandidate[] = [];
const reportText = markupText(sanitizeHtmlText(document.rawText).text);
const subsections = subsectionContext(document);
const reportMetadata = /Date:\s*(\d{4}-\d{2}-\d{2})\s*-\s*Engine version:\s*([^\s<]+)/i.exec(reportText);
if (reportMetadata) {
addObservation(observations, {
subject: { type: 'scanner-report', identity: document.file.name }, property: 'report.generated_at', label: 'Report generation date',
originalValue: reportMetadata[1], normalizedValue: reportMetadata[1], valueType: 'datetime', state: 'observed', observedAt: reportMetadata[1],
source: { fileName: document.file.name, objectPath: 'html/report-metadata/date' }
});
addObservation(observations, {
subject: { type: 'scanner-report', identity: document.file.name }, property: 'report.engine_version', label: 'PingCastle engine version',
originalValue: reportMetadata[2], normalizedValue: reportMetadata[2], valueType: 'string', state: 'observed',
source: { fileName: document.file.name, objectPath: 'html/report-metadata/engine-version' }
});
}
([
['risk.domain', 'Domain Risk Level', scores.global],
['risk.stale_objects', 'Stale Objects score', scores.staleObjects],
['risk.privileged_accounts', 'Privileged Accounts score', scores.privilegedAccounts],
['risk.trusts', 'Trusts score', scores.trusts],
['risk.anomalies', 'Anomalies score', scores.anomalies]
] as const).forEach(([property, label, score]) => {
if (!score?.value) return;
addObservation(observations, {
subject: { type: 'active-directory-environment' }, property, label, originalValue: score.value,
normalizedValue: Number(score.value), valueType: 'number', state: 'observed', source: score.source, attributes: { maximum: 100 }
});
});
sections.forEach(section => {
section.tables.forEach((table, tableOrdinal) => {
const subsectionLabel = table.sourceTableIndex === undefined ? undefined : subsections.tableLabels.get(table.sourceTableIndex);
const tableSubjectType = semanticSubject(section.sectionKey, subsectionLabel, table.name, table.headers);
const tableSubject = tableSubjectType === 'pingcastle-table-row'
? { type: 'report-section', identity: section.sectionKey, label: section.label }
: { type: tableSubjectType, identity: `${section.sectionKey}:${subjectKey(subsectionLabel ?? table.name)}`, label: subsectionLabel ?? table.name };
addObservation(observations, {
subject: tableSubject,
property: `inventory.${propertyKey(section.sectionKey)}.${propertyKey(table.name)}.row_count`,
label: `${table.name} row count`, originalValue: table.rows.length, normalizedValue: table.rows.length,
valueType: 'number', state: table.rows.length ? 'observed' : (section.state === 'observed' ? 'empty' : section.state),
sectionKey: section.sectionKey,
source: { fileName: document.file.name, tableIndex: table.sourceTableIndex, objectPath: `html/sections/${section.sectionKey}/tables/${tableOrdinal}/row-count` }
});
if (!table.rows.length) {
addObservation(observations, {
subject: tableSubject,
property: `section.${propertyKey(section.sectionKey)}.table.${tableOrdinal}.presence`, label: table.name,
originalValue: null, valueType: 'object', state: section.state === 'observed' ? 'empty' : section.state,
sectionKey: section.sectionKey,
source: { fileName: document.file.name, tableIndex: table.sourceTableIndex, objectPath: `html/sections/${section.sectionKey}/tables/${tableOrdinal}` }
});
return;
}
table.rows.forEach((row, rowIndex) => {
const stringRow = row.map(value => typeof value === 'string' ? value : String(value ?? ''));
const subjectIdentity = stringRow.find(value => value.trim()) || `${section.sectionKey}:row-${rowIndex + 1}`;
stringRow.forEach((value, columnIndex) => {
const label = table.headers[columnIndex] || `Column ${columnIndex + 1}`;
if (!value.trim()) return;
const typed = typedValue(value);
const stableProperty = semanticProperty(section.sectionKey, table.name, label);
addObservation(observations, {
subject: { type: tableSubjectType, identity: subjectIdentity, label: subjectIdentity },
property: stableProperty ?? `pingcastle.${propertyKey(section.sectionKey)}.${propertyKey(table.name)}.${propertyKey(label)}`,
label, originalValue: value, ...typed, state: 'observed', sectionKey: section.sectionKey,
source: { fileName: document.file.name, tableIndex: table.sourceTableIndex, rowIndex: rowIndex + 1, columnIndex },
attributes: { tableName: table.name, ...(subsectionLabel ? { subsectionLabel } : {}) }
});
});
});
});
});
const tableSubsections = new Set([...subsections.tableLabels.entries()].map(([sourceTableIndex, label]) => {
const section = sections.find(candidate => candidate.tables.some(table => table.sourceTableIndex === sourceTableIndex));
return section ? `${section.sectionKey}\u0000${label}` : '';
}));
const factSubsections = new Set([
'sectionDomainInformation\u0000Entra ID Configuration',
'sectionAnomalies\u0000Backup',
'sectionAnomalies\u0000LAPS',
'sectionAnomalies\u0000Windows Event Forwarding (WEF)',
'sectionAnomalies\u0000krbtgt (Used for Golden ticket attacks)'
]);
subsections.headings.forEach(({ sectionKey, label }) => {
const key = `${sectionKey}\u0000${label}`;
if (tableSubsections.has(key) || factSubsections.has(key)) return;
if (!sections.some(section => section.sectionKey === sectionKey)) return;
const base = semanticSubjectBase(sectionKey);
if (!base) return;
const type = `${base}.${subjectKey(label)}`;
addObservation(observations, {
subject: { type, identity: `${sectionKey}:${subjectKey(label)}`, label },
property: `section.${propertyKey(sectionKey)}.subsection.${propertyKey(label)}.presence`, label,
originalValue: null, valueType: 'object', state: 'observed', sectionKey,
source: { fileName: document.file.name, objectPath: `html/sections/${sectionKey}/subsections/${subjectKey(label)}` },
attributes: { subsectionLabel: label }
});
});
const proseFacts: Array<[RegExp, string, string, EnvironmentObservationCandidate['valueType']]> = [
[/Last backup date:\s*([^\s]+(?:\s+[^\s]+)?)/i, 'backup.last_detected_at', 'Last detected AD backup', 'datetime'],
[/Legacy LAPS installation date:\s*(Never|\d{4}-\d{2}-\d{2}(?:[ T][0-9:.+-]+Z?)?)/i, 'laps.legacy.installation_date', 'Legacy LAPS installation date', 'datetime'],
[/Windows LAPS installation date:\s*(Never|\d{4}-\d{2}-\d{2}(?:[ T][0-9:.+-]+Z?)?)/i, 'laps.windows.installation_date', 'Windows LAPS installation date', 'datetime'],
[/Number of WEF configuration found:\s*(\d+)/i, 'logging.wef.configuration_count', 'WEF configuration count', 'number'],
[/Kerberos password last changed:\s*([^\s]+(?:\s+[^\s]+)?)/i, 'kerberos.krbtgt.password_last_changed', 'krbtgt password last changed', 'datetime'],
[/Kerberos password last changed:[\s\S]{0,200}?version:\s*(\d+)/i, 'kerberos.krbtgt.version', 'krbtgt version', 'number']
];
proseFacts.forEach(([pattern, property, label, declaredType]) => {
const value = pattern.exec(reportText)?.[1]?.trim();
if (!value) return;
const absent = /^never$/i.test(value);
const typed = declaredType === 'number' ? { normalizedValue: Number(value), valueType: declaredType } : { normalizedValue: value, valueType: declaredType };
const factSubject: [string, string?] = property === 'backup.last_detected_at' ? ['anomalies.backup', 'sectionAnomalies']
: property.startsWith('laps.') ? ['anomalies.laps', 'sectionAnomalies']
: property === 'logging.wef.configuration_count' ? ['anomalies.windows-event-forwarding-wef', 'sectionAnomalies']
: property.startsWith('kerberos.krbtgt.') ? ['anomalies.krbtgt-used-for-golden-ticket-attacks', 'sectionAnomalies']
: ['active-directory-environment', undefined];
addObservation(observations, {
subject: { type: factSubject[0] }, property, label, originalValue: value,
...typed, state: absent ? 'explicitly-absent' : 'observed', sectionKey: factSubject[1],
source: { fileName: document.file.name, objectPath: `html/facts/${property}` }
});
});
if (/No Entra ID configuration has been found in this domain/i.test(reportText)) {
addObservation(observations, {
subject: { type: 'domain-information.entra-id-configuration' }, property: 'entra.configuration_detected', label: 'Entra ID configuration detected',
originalValue: false, normalizedValue: false, valueType: 'boolean', state: 'explicitly-absent', sectionKey: 'sectionDomainInformation',
source: { fileName: document.file.name, objectPath: 'html/facts/entra-configuration' }
});
}
observations.push(...derivePingCastleEnvironmentAggregates(observations));
return observations;
}
function hierarchyView(document: DecodedSourceDocument): DecodedHierarchyView | undefined {
return document.views.find((view): view is DecodedHierarchyView => view.kind === 'hierarchy');
}
function xmlRuleGroups(document: DecodedSourceDocument): XmlRuleGroup[] {
const groups = new Map<string, XmlRuleGroup>();
hierarchyView(document)?.nodes.forEach(node => {
const match = /^(.*\/RiskRules\[\d+\]\/HealthcheckRiskRule\[\d+\])(?:\/([^/]+)\[\d+\])?/i.exec(node.path);
if (!match) return;
const path = match[1];
const fieldName = match[2];
const group = groups.get(path) ?? {
path,
direct: new Map<string, DecodedHierarchyNode>(),
details: []
};
if (!fieldName) {
group.root = node;
} else if (!node.path.slice(path.length + 1).includes('/')) {
group.direct.set(fieldName.toLocaleLowerCase('en-US'), node);
} else if (/\/(?:Details|ExtraDetails)\[\d+\]\//i.test(node.path) && nodeValue(node)) {
group.details.push(node);
}
groups.set(path, group);
});
return [...groups.values()];
}
function nodeValue(node: DecodedHierarchyNode | undefined): string {
return typeof node?.value === 'string'
? node.value.trim()
: node?.value == null
? ''
: String(node.value);
}
function directHealthcheckNode(document: DecodedSourceDocument, name: string): DecodedHierarchyNode | undefined {
return hierarchyView(document)?.nodes.find(node =>
new RegExp('^HealthcheckData/' + name + '\\[\\d+\\]$', 'i').test(node.path)
);
}
function xmlReportScores(document: DecodedSourceDocument): PingCastleScores {
const score = (name: string) => {
const node = directHealthcheckNode(document, name);
const value = nodeValue(node);
return node && value ? { value, source: node.source } : undefined;
};
return {
global: score('GlobalScore'),
staleObjects: score('StaleObjectsScore'),
privilegedAccounts: score('PrivilegiedGroupScore'),
trusts: score('TrustScore'),
anomalies: score('AnomalyScore')
};
}
function categoryScore(category: string, scores: PingCastleScores) {
const normalized = category.replace(/[^a-z]/gi, '').toLocaleLowerCase('en-US');
if (normalized === 'staleobjects') return scores.staleObjects;
if (normalized === 'privilegedaccounts' || normalized === 'privilegiedaccounts') return scores.privilegedAccounts;
if (normalized === 'trusts') return scores.trusts;
if (normalized === 'anomalies') return scores.anomalies;
return undefined;
}
function appendScoreFields(fields: SourceFieldCandidate[], category: string, scores: PingCastleScores) {
if (scores.global?.value) {
fields.push(sourceField('global-score', 'Domain Risk Level', scores.global.value, scores.global.source));
}
const categoryValue = categoryScore(category, scores);
if (categoryValue?.value) {
fields.push(sourceField('category-score', 'Category Score', categoryValue.value, categoryValue.source));
}
}
function detailValues(nodes: DecodedHierarchyNode[]): string[] {
const plain = nodes
.filter(node => /\/Details\[\d+\]\//i.test(node.path))
.map(nodeValue)
.filter(Boolean);
const extraNodes = nodes.filter(node => /\/ExtraDetails\[\d+\]\//i.test(node.path));
const itemPaths = [...new Set(extraNodes.map(node =>
/^(.*\/DetailItemBase\[\d+\])/i.exec(node.path)?.[1]
).filter((value): value is string => Boolean(value)))];
const extra = itemPaths.map(path => {
const members = extraNodes.filter(node => node.path.startsWith(path + '/'));
const name = nodeValue(members.find(node => /\/Name\[\d+\]$/i.test(node.path)));
const values = members
.filter(node => /\/(?:Value\[\d+\]|Values\[\d+\]\/string\[\d+\])$/i.test(node.path))
.map(nodeValue)
.filter(Boolean);
return name && values.length ? name + ': ' + values.join(', ') : '';
}).filter(Boolean);
return [...plain, ...extra];
}
function xmlCandidates(document: DecodedSourceDocument): {
candidates: FindingCandidate[];
diagnostics: ImportDiagnostic[];
reportSections: ScannerReportSectionCandidate[];
environmentObservations: EnvironmentObservationCandidate[];
} {
const groups = xmlRuleGroups(document);
const reportScores = xmlReportScores(document);
const candidates: FindingCandidate[] = [];
const seenRuleIds = new Set<string>();
const ruleDiagnostics: ImportDiagnostic[] = [];
for (const group of groups) {
if (candidates.length >= maximumRuleCandidates) break;
const riskIdNode = group.direct.get('riskid');
const ruleId = nodeValue(riskIdNode);
if (!acceptPingCastleRuleId(ruleId, seenRuleIds, riskIdNode?.source ?? { fileName: document.file.name, objectPath: group.path }, ruleDiagnostics)) continue;
const rationaleNode = group.direct.get('rationale');
const noticeNode = group.direct.get('notice');
const source = riskIdNode?.source
?? group.root?.source
?? { fileName: document.file.name, objectPath: group.path };
const fields: SourceFieldCandidate[] = [
sourceField('risk-id', 'Rule ID', ruleId, source),
sourceField('rationale', 'Title', nodeValue(rationaleNode) || nodeValue(noticeNode) || ruleId, rationaleNode?.source ?? source),
sourceField('status', 'Status', 'Matched', source)
];
const directFields: Array<[string, string, string]> = [
['category', 'Category', 'Category'],
['model', 'Model', 'Model'],
['points', 'Points', 'Points'],
['notice', 'Evidence', 'Notice'],
['noticetooltip', 'Evidence', 'NoticeTooltip']
];
directFields.forEach(([key, label, originalName]) => {
const node = group.direct.get(key);
const value = nodeValue(node);
if (value) fields.push(sourceField(originalName, label, value, node?.source ?? source));
});
const details = detailValues(group.details);
if (details.length) {
fields.push(sourceField('Details', 'Details', details.join('\n'), group.details[0]?.source ?? source));
}
appendScoreFields(fields, nodeValue(group.direct.get('category')), reportScores);
candidates.push({
version: 1,
id: 'candidate-pingcastle-xml-' + safeId(ruleId) + '-' + candidates.length,
source,
fields
});
}
const diagnostics: ImportDiagnostic[] = [...ruleDiagnostics, {
severity: 'warning',
code: 'pingcastle-xml-results-unavailable',
message: 'PingCastle-XML wird als eigenständiger Import verarbeitet. Results sind nur aus dem zugehörigen HTML-Bericht verfügbar.',
stage: 'extraction',
source: { fileName: document.file.name, objectPath: 'HealthcheckData/RiskRules' }
}];
if (groups.length > maximumRuleCandidates) diagnostics.push({
severity: 'warning',
code: 'pingcastle-rule-limit',
message: 'Der PingCastle-Bericht enthält mehr als ' + maximumRuleCandidates + ' RiskRules; die Extraktion wurde begrenzt.',
stage: 'extraction'
});
const contextNodes = hierarchyView(document)?.nodes
.filter(node => !/\/RiskRules\[\d+\](?:\/|$)/i.test(node.path)) ?? [];
const parentPaths = new Set(contextNodes.map(node => node.path.slice(0, node.path.lastIndexOf('/'))));
const environmentObservations: EnvironmentObservationCandidate[] = [];
for (const node of contextNodes) {
const value = nodeValue(node);
const xmlAttributes = node.attributes.xmlAttributes;
const attributeEntries = xmlAttributes && typeof xmlAttributes === 'object' && !Array.isArray(xmlAttributes)
? Object.entries(xmlAttributes).filter((entry): entry is [string, string] => typeof entry[1] === 'string') : [];
const base = {
version: 1 as const, subject: { type: 'pingcastle-xml-node', identity: node.path },
property: `pingcastle.xml.${propertyKey(node.path)}`,
label: node.name, sectionKey: 'HealthcheckData', source: node.source,
attributes: { xmlPath: node.path }
};
if (value || (!parentPaths.has(node.path) && !attributeEntries.length && !node.attributes.serialized)) {
environmentObservations.push({ ...base, originalValue: value || null,
...(value ? typedValue(value) : { valueType: 'object' as const }), state: value ? 'observed' : 'empty' });
}
for (const [name, attributeValue] of attributeEntries) {
environmentObservations.push({ ...base, property: `${base.property}.attribute.${propertyKey(name)}`, label: `${node.name} @${name}`,
originalValue: attributeValue, ...typedValue(attributeValue), state: attributeValue.length ? 'observed' : 'empty',
source: { ...node.source, objectPath: `${node.path}/@${name}` }, attributes: { ...base.attributes, attributeName: name } });
}
if (!xmlAttributes && typeof node.attributes.serialized === 'string' && node.attributes.serialized) {
// Compatibility with older/custom adapters: retain the serialization; never guess its ambiguous quoting.
environmentObservations.push({ ...base, property: `${base.property}.serialized.attributes`, label: `${node.name} attributes`,
originalValue: node.attributes.serialized, valueType: 'string', state: 'unsupported' });
diagnostics.push({ severity: 'warning', code: 'pingcastle-xml-attributes-unstructured', stage: 'extraction', source: node.source,
message: 'XML-Attribute liegen nur als Anzeigezeichenkette vor. Der Originalwert bleibt erhalten, eine Attributnormalisierung erfolgt nicht.' });
}
}
const section: ScannerReportSectionCandidate = {
version: 1,
sectionKey: 'HealthcheckData',
label: 'PingCastle HealthcheckData',
ordinal: 0,
state: environmentObservations.some(item => item.state === 'observed') ? 'observed' : 'empty',
tables: [{
name: 'HealthcheckData context',
headers: ['Path', 'Value'],
rows: environmentObservations.map(observation => [observation.source.objectPath ?? '', observation.originalValue])
}],
source: { fileName: document.file.name, objectPath: 'HealthcheckData' },
attributes: { representation: 'xml-hierarchy' }
};
return { candidates, diagnostics, reportSections: [section], environmentObservations };
}
function isHealthcheckXml(document: DecodedSourceDocument): boolean {
return document.format === 'XML' && document.tables.some(table =>
String(table.attributes.rootElement ?? '').toLocaleLowerCase('en-US') === 'healthcheckdata'
);
}
export const pingCastleSourceExtractor: SourceExtractor = {
id: 'pingcastle-healthcheck-source-extractor',
version: '2.2.1',
detect(document) {
if (isHealthcheckXml(document)) {
const riskRuleCount = xmlRuleGroups(document).length;
return {
score: riskRuleCount ? 1 : 0.9,
evidence: [
'XML-Wurzelelement HealthcheckData erkannt.',
...(riskRuleCount ? [riskRuleCount + ' native PingCastle-RiskRule-Struktur(en) erkannt.'] : [])
]
};
}
if (document.format !== 'HTML' || !/ping[\s_-]*castle/i.test(document.rawText.slice(0, 500_000))) {
return { score: 0, evidence: [] };
}
const textView = document.views.find(view => view.kind === 'text');
const safeText = textView?.kind === 'text' ? textView.text : '';
const hasRuleStructure = /\bRule\s*ID\s*:/i.test(safeText);
const hasHealthcheckStructure = /rules?\s+matched|healthcheck/i.test(safeText.slice(0, 2_000_000));
return {
score: hasRuleStructure ? 1 : hasHealthcheckStructure ? 0.85 : 0.7,
evidence: [
'PingCastle-Marker im HTML-Bericht erkannt.',
...(hasRuleStructure ? ['Native PingCastle-Regelblöcke mit Rule ID erkannt.'] : [])
]
};
},
extract(document) {
const extracted = isHealthcheckXml(document) ? xmlCandidates(document) : htmlCandidates(document);
return { version: 1, ...extracted };
}
};