import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FileSystemBackendArtifactStore } from '../../server/artifacts/backend-artifact-store'; import { GpoArtifactReconciler } from '../../server/gpo/gpo-artifact-reconciler'; import { GpoBackupJobProcessor } from '../../server/gpo/gpo-backup-job-processor'; import { GpoBackupManagementService } from '../../server/gpo/gpo-backup-management-service'; import { GpoBackupSubmissionService } from '../../server/gpo/gpo-backup-submission-service'; import { listProjectGpoBackupStatus } from '../../server/gpo/gpo-backup-status-service'; import { directGpoBackupWorkerExecutor, directGpoInventoryExecutor } from '../../server/gpo/gpo-inventory-executor'; import { LocalAssessorRepository } from '../../src/persistence/local-assessor-repository'; import { syntheticGpoBackupZip } from './gpo-synthetic-fixture'; const timestamp = '2026-09-17T10:00:00.000Z'; const project = { id: 'recovery-project', name: 'Synthetic recovery', createdAt: timestamp, updatedAt: timestamp, attributes: {} }; describe('missing GPO source recovery', () => { let directory: string; let repository: LocalAssessorRepository; let store: FileSystemBackendArtifactStore; let service: GpoBackupSubmissionService; const request = () => ({ projectId: project.id, fileName: 'synthetic.zip', bytes: syntheticGpoBackupZip(), receivedAt: timestamp }); const status = () => listProjectGpoBackupStatus(repository, project.id, timestamp, store); const processor = () => new GpoBackupJobProcessor(repository, store, directGpoBackupWorkerExecutor, { maximumArtifactBytes: 1024 * 1024, now: () => new Date(timestamp) }); async function removeSource() { const artifact = (await repository.listImportArtifacts(project.id))[0]; if (artifact.storage.kind !== 'backend-controlled') throw new Error('Synthetic storage required'); await store.remove(artifact.storage.storageKey); return artifact; } beforeEach(async () => { directory = await mkdtemp(join(tmpdir(), 'assessor-source-recovery-')); store = new FileSystemBackendArtifactStore(directory); repository = new LocalAssessorRepository(); await repository.saveProject(project); service = new GpoBackupSubmissionService(repository, store, directGpoInventoryExecutor, 1024 * 1024); }); afterEach(async () => { vi.restoreAllMocks(); await rm(directory, { recursive: true, force: true }); }); it('shows a missing source before the first attempt and stops an already queued job after one failure', async () => { const submitted = await service.submit(request()); await removeSource(); const execute = vi.spyOn(directGpoBackupWorkerExecutor, 'executeBackup'); await expect(status()).resolves.toMatchObject({ records: [{ state: 'upload-required', attempts: 0, jobStatus: 'queued' }] }); await expect(processor().processNext('worker')).resolves.toMatchObject({ status: 'dead-letter', errorCode: 'artifact-store-object-missing' }); await expect(processor().processNext('worker')).resolves.toBeNull(); await expect(status()).resolves.toMatchObject({ records: [{ state: 'upload-required', attempts: 1, maxAttempts: 3 }] }); expect(execute).not.toHaveBeenCalled(); const revision = repository.revision; await expect(new GpoBackupManagementService(repository, store).retry(project.id, submitted.artifactId)) .rejects.toMatchObject({ diagnostics: [{ code: 'artifact-store-object-missing' }] }); expect(repository.revision).toBe(revision); await expect(service.submit({ ...request(), restoreArtifactId: submitted.artifactId, fileName: 'renamed-original.zip' })) .resolves.toMatchObject({ artifactId: submitted.artifactId, sourceRestored: true, deduplicated: true }); await expect(repository.listIngestionJobs(project.id)).resolves.toMatchObject([{ status: 'queued', attempts: 0 }]); await expect(processor().processNext('worker')).resolves.toMatchObject({ status: 'succeeded' }); await expect(status()).resolves.toMatchObject({ records: [{ state: 'partial' }] }); }); it('restores a completed source through ordinary deduplication without replacing analyses or source identity', async () => { await service.submit(request()); await processor().processNext('worker'); const artifact = await removeSource(); const analyses = await repository.listGpoPolicyAnalyses(project.id); const runs = await repository.listProcessingRuns(artifact.id); const revision = repository.revision; await expect(status()).resolves.toMatchObject({ records: [{ state: 'upload-required', processingStatus: 'partial', gpoCount: 1 }] }); await expect(new GpoBackupManagementService(repository, store).reprocess(project.id, artifact.id, runs[0].id)) .rejects.toMatchObject({ diagnostics: [{ code: 'artifact-store-object-missing' }] }); expect(repository.revision).toBe(revision); await expect(service.submit(request())).resolves.toMatchObject({ artifactId: artifact.id, sourceRestored: true }); // No repository mutation is needed just to restore retained source bytes. expect(repository.revision).toBe(revision); await expect(status()).resolves.toMatchObject({ records: [{ state: 'partial' }] }); await expect(repository.listImportArtifacts(project.id)).resolves.toEqual([artifact]); await expect(repository.listProcessingRuns(artifact.id)).resolves.toEqual(runs); await expect(repository.listGpoPolicyAnalyses(project.id)).resolves.toEqual(analyses); }); it('never repairs a missing initial job or starts a new analyzer generation while its source is absent', async () => { await service.submit(request()); await processor().processNext('worker'); const artifact = await removeSource(); // Independent synthetic repository models a legacy artifact without any job. const legacy = new LocalAssessorRepository(); await legacy.saveProject(project); await legacy.saveImportArtifact(artifact); const result = await new GpoArtifactReconciler(legacy, store, { now: () => new Date(timestamp) }).run(); expect(result).toMatchObject({ repairedMissingJobs: 0, unavailableSources: 1 }); await expect(legacy.listIngestionJobs(project.id)).resolves.toEqual([]); // Retained earlier analyzer generation also must not enqueue reprocessing. const analyses = await repository.listGpoPolicyAnalyses(project.id); vi.spyOn(repository, 'listGpoPolicyAnalyses').mockResolvedValue(analyses.map(item => ({ ...item, run: { ...item.run, analyzer: { ...item.run.analyzer, version: 'gpo-backup-policy-analysis:2.2.3' } } }))); expect(await new GpoArtifactReconciler(repository, store, { now: () => new Date(timestamp) }).run()) .toMatchObject({ repairedMissingJobs: 0, unavailableSources: 1 }); await expect(repository.listIngestionJobs(project.id)).resolves.toHaveLength(1); await service.submit(request()); await expect(repository.listIngestionJobs(project.id)).resolves.toHaveLength(2); }); it('rejects wrong bytes, foreign projects and wrong timeslots before storage or history changes', async () => { const submitted = await service.submit(request()); const artifact = await removeSource(); const original = request(); const altered = original.bytes.slice(); altered[0] ^= 1; await expect(service.submit({ ...original, restoreArtifactId: submitted.artifactId, bytes: altered })).rejects.toThrow('does not match'); await repository.saveProject({ ...project, id: 'other-project' }); await expect(service.submit({ ...original, projectId: 'other-project', restoreArtifactId: submitted.artifactId })).rejects.toThrow('not present'); await repository.saveAssessment({ id: 'other-slot', projectId: project.id, name: 'Synthetic slot', effectiveAt: timestamp, createdAt: timestamp, updatedAt: timestamp }); await expect(service.submit({ ...original, assessmentId: 'other-slot', restoreArtifactId: submitted.artifactId })).rejects.toThrow('does not match'); await expect(repository.listImportArtifacts(project.id)).resolves.toEqual([artifact]); await expect(status()).resolves.toMatchObject({ records: [{ state: 'upload-required', attempts: 0 }] }); }); it('keeps inaccessible storage distinct from missing bytes and does not propose destructive recovery', async () => { const submitted = await service.submit(request()); vi.spyOn(store, 'availability').mockResolvedValue('unavailable'); const restore = vi.spyOn(store, 'restore'); await expect(status()).resolves.toMatchObject({ records: [{ state: 'storage-unavailable' }] }); await expect(service.submit({ ...request(), restoreArtifactId: submitted.artifactId })) .rejects.toMatchObject({ diagnostics: [{ code: 'artifact-store-read-failed' }] }); expect(restore).not.toHaveBeenCalled(); }); });