Assessor_2.0/src/domain/reference-recognition.ts

import type { JsonValue, OriginalField, SourcePosition } from './canonical-model';

/** Independent derivation: these references never become FindingDefinition identity keys. */
export const REFERENCE_RECOGNIZER_VERSION = '1.0.0';
export type ReferenceFramework = 'mitre-attack' | 'mitre-d3fend';
export interface ReferenceCatalogEntry {
  id: string;
  title: string;
  kind: 'technique' | 'tactic';
  state: 'active' | 'deprecated' | 'revoked';
  url: string;
  tactics: string[];
}
export interface ReferenceCatalog {
  schemaVersion: 1;
  framework: ReferenceFramework;
  version: string;
  sourceUrl: string;
  sourceSha256: string;
  license: string;
  entries: ReferenceCatalogEntry[];
}
export interface ReferenceCatalogSnapshot extends ReferenceCatalog {
  /** SHA-256 of canonicalJson(catalog), including every recognition input and the notice. */
  sha256: string;
}
export interface RecognizedReference {
  framework: ReferenceFramework;
  id: string;
  title?: string;
  kind?: 'technique' | 'tactic';
  url?: string;
  tactics: string[];
  status: 'active' | 'deprecated' | 'revoked' | 'not-in-catalog';
  evidence: 'declared' | 'mentioned';
  matchedBy: 'id' | 'exact-title';
  catalogVersion: string;
  catalogSha256: string;
  source: {
    fieldKey: string;
    fieldLabel: string;
    fieldPath?: string;
    /** JSON pointer within value; rawValue is authoritative when present. */
    valuePath: string;
    representation: 'rawValue' | 'value';
    start: number;
    end: number;
    offsetUnit: 'utf16';
    text: string;
  };
}
export interface ReferenceOccurrenceResult {
  occurrenceId: string;
  importFileId: string;
  sourceFileSha256?: string;
  sourcePosition: SourcePosition;
  references: RecognizedReference[];
}
export interface ReferenceRecognitionRun {
  schemaVersion: 1;
  id: string;
  projectId: string;
  assessmentId: string;
  scanRunId: string;
  recognizerVersion: string;
  inputSha256: string;
  catalogHashes: string[];
  createdAt: string;
  results: ReferenceOccurrenceResult[];
}
export interface FindingReferenceRecognition {
  runId: string;
  recognizerVersion: string;
  createdAt: string;
  references: RecognizedReference[];
}

/** Header semantics are framework-specific, never scanner-specific. */
export function declaredReferenceFramework(label: string): ReferenceFramework | undefined {
  const value = label.trim().toLowerCase().replace(/[_-]+/g, ' ');
  if (/\b(?:d3fend|mitre\s*(?:d3fend|defen[cs]e))\b/.test(value)) return 'mitre-d3fend';
  if (/\b(?:att&?ck|mitre\s*(?:att&?ck|attack))\b/.test(value)) return 'mitre-attack';
  return undefined;
}

function* strings(value: JsonValue, path = ''): Generator<{ text: string; path: string }> {
  if (typeof value === 'string') yield { text: value, path };
  else if (Array.isArray(value)) {
    for (let i = 0; i < value.length; i++) yield* strings(value[i], `${path}/${i}`);
  } else if (value && typeof value === 'object') {
    for (const [key, item] of Object.entries(value)) yield* strings(item, `${path}/${key.replace(/~/g, '~0').replace(/\//g, '~1')}`);
  }
}
const normalizeTitle = (value: string) => value.trim().replace(/\s+/g, ' ').toLowerCase();

/** Build indexes once per run, then process every source field offline. */
export function createReferenceRecognizer(catalogs: readonly ReferenceCatalogSnapshot[]) {
  const indexes = catalogs.map(catalog => {
    const titles = new Map<string, ReferenceCatalogEntry[]>();
    for (const entry of catalog.entries) {
      const key = normalizeTitle(entry.title);
      titles.set(key, [...(titles.get(key) ?? []), entry]);
    }
    return { catalog, ids: new Map(catalog.entries.map(entry => [entry.id.toUpperCase(), entry])), titles };
  });
  return (fields: readonly OriginalField[]): RecognizedReference[] => {
    const results: RecognizedReference[] = [];
    for (const field of fields) {
      const declared = declaredReferenceFramework(field.label);
      const representation = typeof field.rawValue === 'string' ? 'rawValue' : 'value';
      for (const { text, path } of strings(representation === 'rawValue' ? field.rawValue! : field.value)) {
        for (const { catalog, ids, titles } of indexes) {
          const evidence = declared === catalog.framework ? 'declared' : 'mentioned';
          const occupied: Array<[number, number]> = [];
          const add = (id: string, start: number, end: number, matchedBy: 'id' | 'exact-title', entry?: ReferenceCatalogEntry) => {
            occupied.push([start, end]);
            results.push({ framework: catalog.framework, id: entry?.id ?? id,
              ...(entry ? { title: entry.title, kind: entry.kind, url: entry.url } : {}),
              tactics: entry?.tactics ?? [], status: entry?.state ?? 'not-in-catalog', evidence, matchedBy,
              catalogVersion: catalog.version, catalogSha256: catalog.sha256,
              source: { fieldKey: field.key, fieldLabel: field.label, ...(field.path ? { fieldPath: field.path } : {}),
                valuePath: path, representation, start, end, offsetUnit: 'utf16', text: text.slice(start, end) } });
          };
          // Unicode-aware boundaries prevent partial IDs, embedded host names and longer dotted numbers.
          const pattern = catalog.framework === 'mitre-attack'
            ? /(?<![\p{L}\p{N}_.-])(?:TA\d{4}|T\d{4}(?:\.\d{3})?)(?![\p{L}\p{N}_]|[.-][\p{L}\p{N}])/giu
            : /(?<![\p{L}\p{N}_-])D3-[A-Z][A-Z0-9]*(?![\p{L}\p{N}_-])/giu;
          for (const match of text.matchAll(pattern)) {
            const id = match[0].toUpperCase();
            add(id, match.index!, match.index! + match[0].length, 'id', ids.get(id));
          }
          if (evidence !== 'declared') continue;
          // Only whole, explicitly delimited catalog titles; no fuzzy matches or prose inference.
          for (const fragment of text.matchAll(/[^;,|/\r\n]+/g)) {
            const pieces = [...fragment[0].matchAll(/[^\u2013\u2014]+/g)];
            const candidates = [{ text: fragment[0], start: fragment.index! },
              ...pieces.map(piece => ({ text: piece[0], start: fragment.index! + piece.index! }))];
            const separator = fragment[0].lastIndexOf(' - ');
            if (separator >= 0) candidates.push({ text: fragment[0].slice(separator + 3), start: fragment.index! + separator + 3 });
            for (const candidate of candidates) {
              const trimmed = candidate.text.trim();
              const start = candidate.start + candidate.text.indexOf(trimmed);
              const end = start + trimmed.length;
              if (!trimmed || occupied.some(([a, b]) => start < b && end > a)) continue;
              const entries = titles.get(normalizeTitle(trimmed));
              if (entries?.length === 1) add(entries[0].id, start, end, 'exact-title', entries[0]);
            }
          }
        }
      }
    }
    return results;
  };
}

export function latestReferenceRuns(runs: readonly ReferenceRecognitionRun[]): ReferenceRecognitionRun[] {
  const latest = new Map<string, ReferenceRecognitionRun>();
  for (const run of runs) {
    const previous = latest.get(run.scanRunId);
    if (!previous || `${run.createdAt}\0${run.id}` > `${previous.createdAt}\0${previous.id}`) latest.set(run.scanRunId, run);
  }
  return [...latest.values()];
}