297 lines
14 KiB
TypeScript
297 lines
14 KiB
TypeScript
import { createHash, randomUUID } from 'node:crypto';
|
|
import { createReadStream } from 'node:fs';
|
|
import { link, lstat, mkdir, open, readFile, readdir, stat, unlink } from 'node:fs/promises';
|
|
import { isAbsolute, join, relative, resolve } from 'node:path';
|
|
import { ImportError } from '../../src/shared/errors';
|
|
|
|
export interface StoredArtifactReceipt {
|
|
version: 1;
|
|
storageKey: string;
|
|
sha256: string;
|
|
sizeBytes: number;
|
|
}
|
|
|
|
export interface StoreArtifactRequest {
|
|
bytes: Uint8Array;
|
|
expectedSha256?: string;
|
|
maximumBytes: number;
|
|
}
|
|
|
|
export interface ReadArtifactRequest {
|
|
storageKey: string;
|
|
expectedSha256: string;
|
|
maximumBytes: number;
|
|
}
|
|
|
|
export interface ArtifactStoreReconciliationRequest {
|
|
referencedStorageKeys: readonly string[];
|
|
allowUnreferencedObjectRemoval: boolean;
|
|
removeUnreferencedBefore: string;
|
|
removeStagingBefore: string;
|
|
maximumRemovals: number;
|
|
}
|
|
|
|
export interface ArtifactStoreReconciliationResult {
|
|
version: 1;
|
|
referencedObjects: number;
|
|
retainedUnreferencedObjects: number;
|
|
removedUnreferencedObjects: number;
|
|
removedStagingFiles: number;
|
|
}
|
|
|
|
export interface BackendArtifactStore {
|
|
/** Cheap live accessibility check; integrity is still verified when reading. */
|
|
availability(storageKey: string): Promise<'available' | 'missing' | 'unavailable'>;
|
|
/** Restore only an absent object, with the original identity and checksum. */
|
|
restore(request: ReadArtifactRequest & { bytes: Uint8Array }): Promise<void>;
|
|
put(request: StoreArtifactRequest): Promise<StoredArtifactReceipt>;
|
|
read(request: ReadArtifactRequest): Promise<Uint8Array>;
|
|
remove(storageKey: string): Promise<boolean>;
|
|
reconcile(request: ArtifactStoreReconciliationRequest): Promise<ArtifactStoreReconciliationResult>;
|
|
}
|
|
|
|
const storageKeyPattern = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/;
|
|
|
|
export function isBackendArtifactStorageKey(value: string): boolean {
|
|
return storageKeyPattern.test(value);
|
|
}
|
|
const sha256Pattern = /^[a-f0-9]{64}$/;
|
|
|
|
function storageError(code: string, message: string, details: Record<string, string | number> = {}): ImportError {
|
|
return new ImportError(message, [{ severity: 'error', code, message, details }]);
|
|
}
|
|
|
|
function validateMaximumBytes(value: number): void {
|
|
if (!Number.isSafeInteger(value) || value < 1) throw storageError('artifact-store-limit-invalid', 'Das Artefaktspeicherlimit ist ungültig.');
|
|
}
|
|
|
|
export function assertArtifactStoreOutsidePublicRoot(artifactRoot: string, publicRoot: string): void {
|
|
if (!isAbsolute(artifactRoot) || !isAbsolute(publicRoot)) {
|
|
throw storageError('artifact-store-root-invalid', 'Artefakt- und Webroot müssen absolute Pfade sein.');
|
|
}
|
|
const artifact = resolve(artifactRoot);
|
|
const published = resolve(publicRoot);
|
|
const fromPublic = relative(published, artifact);
|
|
if (!fromPublic || fromPublic === '.' || (!fromPublic.startsWith('..') && !isAbsolute(fromPublic))) {
|
|
throw storageError('artifact-store-public-root-overlap', 'Der Artefaktspeicher muss außerhalb des veröffentlichten Webroots liegen.');
|
|
}
|
|
}
|
|
async function ensurePrivateDirectory(path: string): Promise<void> {
|
|
await mkdir(path, { recursive: true, mode: 0o700 });
|
|
const information = await lstat(path);
|
|
if (!information.isDirectory() || information.isSymbolicLink()) {
|
|
throw storageError('artifact-store-directory-rejected', 'Der Artefaktspeicherpfad ist kein reguläres Verzeichnis.');
|
|
}
|
|
}
|
|
|
|
async function hashFile(path: string, maximumBytes: number): Promise<{ sha256: string; sizeBytes: number }> {
|
|
const hash = createHash('sha256');
|
|
let sizeBytes = 0;
|
|
try {
|
|
for await (const chunk of createReadStream(path, { highWaterMark: 64 * 1024 })) {
|
|
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
sizeBytes += bytes.byteLength;
|
|
if (sizeBytes > maximumBytes) throw storageError('artifact-store-size-limit', 'Das gespeicherte Artefakt überschreitet das Leselimit.', { sizeBytes, limit: maximumBytes });
|
|
hash.update(bytes);
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof ImportError) throw error;
|
|
throw storageError('artifact-store-read-failed', 'Das gespeicherte Artefakt konnte nicht sicher gelesen werden.');
|
|
}
|
|
return { sha256: hash.digest('hex'), sizeBytes };
|
|
}
|
|
|
|
export class FileSystemBackendArtifactStore implements BackendArtifactStore {
|
|
private readonly root: string;
|
|
private readonly stagingRoot: string;
|
|
private readonly objectRoot: string;
|
|
private initialization?: Promise<void>;
|
|
|
|
constructor(root: string) {
|
|
if (!root.trim() || !isAbsolute(root)) throw storageError('artifact-store-root-invalid', 'Der Artefaktspeicher benötigt einen absoluten Root-Pfad.');
|
|
this.root = resolve(root);
|
|
this.stagingRoot = join(this.root, 'staging');
|
|
this.objectRoot = join(this.root, 'objects');
|
|
}
|
|
|
|
async availability(storageKey: string): Promise<'available' | 'missing' | 'unavailable'> {
|
|
try {
|
|
const path = this.objectPath(storageKey);
|
|
for (const directory of [this.root, this.objectRoot]) {
|
|
const information = await lstat(directory);
|
|
if (!information.isDirectory() || information.isSymbolicLink()) return 'unavailable';
|
|
}
|
|
const information = await lstat(path);
|
|
if (!information.isFile() || information.isSymbolicLink()) return 'unavailable';
|
|
const handle = await open(path, 'r');
|
|
await handle.close();
|
|
return 'available';
|
|
} catch (error) {
|
|
return (error as NodeJS.ErrnoException)?.code === 'ENOENT' ? 'missing' : 'unavailable';
|
|
}
|
|
}
|
|
|
|
async restore(request: ReadArtifactRequest & { bytes: Uint8Array }): Promise<void> {
|
|
const path = this.objectPath(request.storageKey);
|
|
// The complete store may have disappeared since a previous initialization.
|
|
this.initialization = undefined;
|
|
// put verifies the uploaded bytes before publication. link never overwrites
|
|
// an existing object, including one concurrently restored by another request.
|
|
const receipt = await this.put(request);
|
|
try {
|
|
try { await link(this.objectPath(receipt.storageKey), path); }
|
|
catch (error) {
|
|
if ((error as NodeJS.ErrnoException)?.code !== 'EEXIST') throw error;
|
|
}
|
|
await this.read(request);
|
|
} finally {
|
|
await this.remove(receipt.storageKey);
|
|
}
|
|
}
|
|
|
|
async put(request: StoreArtifactRequest): Promise<StoredArtifactReceipt> {
|
|
validateMaximumBytes(request.maximumBytes);
|
|
if (request.bytes.byteLength > request.maximumBytes) {
|
|
throw storageError('artifact-store-size-limit', 'Das Artefakt überschreitet das Schreiblimit.', { sizeBytes: request.bytes.byteLength, limit: request.maximumBytes });
|
|
}
|
|
if (request.expectedSha256 !== undefined && !sha256Pattern.test(request.expectedSha256)) {
|
|
throw storageError('artifact-store-hash-invalid', 'Der erwartete Artefakthash ist ungültig.');
|
|
}
|
|
await this.initialize();
|
|
const storageKey = randomUUID();
|
|
const stagingPath = join(this.stagingRoot, `${storageKey}.staging`);
|
|
const finalPath = this.objectPath(storageKey);
|
|
const handle = await open(stagingPath, 'wx', 0o600);
|
|
try {
|
|
await handle.writeFile(request.bytes);
|
|
await handle.sync();
|
|
} finally {
|
|
await handle.close();
|
|
}
|
|
try {
|
|
const verified = await hashFile(stagingPath, request.maximumBytes);
|
|
if (verified.sizeBytes !== request.bytes.byteLength) {
|
|
throw storageError('artifact-store-size-mismatch', 'Geschriebene und erwartete Artefaktgröße stimmen nicht überein.');
|
|
}
|
|
if (request.expectedSha256 && request.expectedSha256 !== verified.sha256) {
|
|
throw storageError('artifact-store-hash-mismatch', 'Der Artefakthash stimmt nicht mit der erwarteten Prüfsumme überein.');
|
|
}
|
|
await link(stagingPath, finalPath);
|
|
return { version: 1, storageKey, sha256: verified.sha256, sizeBytes: verified.sizeBytes };
|
|
} catch (error) {
|
|
if (error instanceof ImportError) throw error;
|
|
throw storageError('artifact-store-publish-failed', 'Das Artefakt konnte nicht atomar veröffentlicht werden.');
|
|
} finally {
|
|
await unlink(stagingPath).catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
async read(request: ReadArtifactRequest): Promise<Uint8Array> {
|
|
validateMaximumBytes(request.maximumBytes);
|
|
if (!storageKeyPattern.test(request.storageKey)) throw storageError('artifact-store-key-invalid', 'Der Artefaktspeicherschlüssel ist ungültig.');
|
|
if (!sha256Pattern.test(request.expectedSha256)) throw storageError('artifact-store-hash-invalid', 'Der erwartete Artefakthash ist ungültig.');
|
|
await this.initialize();
|
|
const path = this.objectPath(request.storageKey);
|
|
let information;
|
|
try {
|
|
information = await lstat(path);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') {
|
|
throw storageError('artifact-store-object-missing', 'Stored source is missing. Upload the unchanged original file again.');
|
|
}
|
|
throw storageError('artifact-store-read-failed', 'The artifact store cannot be accessed. Check the backend storage connection and permissions.');
|
|
}
|
|
if (!information.isFile() || information.isSymbolicLink()) throw storageError('artifact-store-object-rejected', 'Das gespeicherte Objekt ist keine reguläre Datei.');
|
|
if (information.size > request.maximumBytes) {
|
|
throw storageError('artifact-store-size-limit', 'Das gespeicherte Artefakt überschreitet das Leselimit.', { sizeBytes: information.size, limit: request.maximumBytes });
|
|
}
|
|
const verified = await hashFile(path, request.maximumBytes);
|
|
if (verified.sha256 !== request.expectedSha256 || verified.sizeBytes !== information.size) {
|
|
throw storageError('artifact-store-integrity-mismatch', 'Die Integritätsprüfung des gespeicherten Artefakts ist fehlgeschlagen.');
|
|
}
|
|
const bytes = await readFile(path);
|
|
if (bytes.byteLength !== verified.sizeBytes) throw storageError('artifact-store-size-mismatch', 'Das Artefakt wurde während des Lesens verändert.');
|
|
if (createHash('sha256').update(bytes).digest('hex') !== request.expectedSha256) {
|
|
throw storageError('artifact-store-integrity-mismatch', 'Die Integritätsprüfung der gelesenen Artefaktdaten ist fehlgeschlagen.');
|
|
}
|
|
const finalInformation = await stat(path);
|
|
if (finalInformation.size !== verified.sizeBytes) throw storageError('artifact-store-size-mismatch', 'Das Artefakt wurde während des Lesens verändert.');
|
|
return new Uint8Array(bytes);
|
|
}
|
|
|
|
async remove(storageKey: string): Promise<boolean> {
|
|
await this.initialize();
|
|
const path = this.objectPath(storageKey);
|
|
try {
|
|
const information = await lstat(path);
|
|
if (!information.isFile() || information.isSymbolicLink()) throw storageError('artifact-store-object-rejected', 'Das gespeicherte Objekt ist keine reguläre Datei.');
|
|
await unlink(path);
|
|
return true;
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return false;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async reconcile(request: ArtifactStoreReconciliationRequest): Promise<ArtifactStoreReconciliationResult> {
|
|
const objectCutoff = Date.parse(request.removeUnreferencedBefore);
|
|
const stagingCutoff = Date.parse(request.removeStagingBefore);
|
|
if (!Number.isFinite(objectCutoff) || !Number.isFinite(stagingCutoff)
|
|
|| !Number.isInteger(request.maximumRemovals) || request.maximumRemovals < 1 || request.maximumRemovals > 10_000) {
|
|
throw storageError('artifact-store-reconciliation-invalid', 'Die Artefakt-Reconciliation-Konfiguration ist ungültig.');
|
|
}
|
|
const referenced = new Set(request.referencedStorageKeys);
|
|
if ([...referenced].some(key => !storageKeyPattern.test(key))) throw storageError('artifact-store-key-invalid', 'Eine referenzierte Speicher-ID ist ungültig.');
|
|
await this.initialize();
|
|
let removedObjects = 0;
|
|
let retainedUnreferencedObjects = 0;
|
|
let removedStagingFiles = 0;
|
|
const objectEntries = await readdir(this.objectRoot, { withFileTypes: true });
|
|
for (const entry of objectEntries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
const match = /^([a-f0-9-]{36})\.bin$/.exec(entry.name);
|
|
if (!match || !storageKeyPattern.test(match[1]) || !entry.isFile() || entry.isSymbolicLink()) continue;
|
|
if (referenced.has(match[1])) continue;
|
|
const path = join(this.objectRoot, entry.name);
|
|
const information = await lstat(path);
|
|
if (!request.allowUnreferencedObjectRemoval || !information.isFile() || information.isSymbolicLink() || information.mtimeMs > objectCutoff
|
|
|| removedObjects + removedStagingFiles >= request.maximumRemovals) {
|
|
retainedUnreferencedObjects += 1;
|
|
continue;
|
|
}
|
|
await unlink(path);
|
|
removedObjects += 1;
|
|
}
|
|
const stagingEntries = await readdir(this.stagingRoot, { withFileTypes: true });
|
|
for (const entry of stagingEntries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
if (!/^[a-f0-9-]{36}\.staging$/.test(entry.name) || !entry.isFile() || entry.isSymbolicLink()
|
|
|| removedObjects + removedStagingFiles >= request.maximumRemovals) continue;
|
|
const path = join(this.stagingRoot, entry.name);
|
|
const information = await lstat(path);
|
|
if (!information.isFile() || information.isSymbolicLink() || information.mtimeMs > stagingCutoff) continue;
|
|
await unlink(path);
|
|
removedStagingFiles += 1;
|
|
}
|
|
return {
|
|
version: 1,
|
|
referencedObjects: referenced.size,
|
|
retainedUnreferencedObjects,
|
|
removedUnreferencedObjects: removedObjects,
|
|
removedStagingFiles
|
|
};
|
|
}
|
|
|
|
private initialize(): Promise<void> {
|
|
this.initialization ??= (async () => {
|
|
await ensurePrivateDirectory(this.root);
|
|
await ensurePrivateDirectory(this.stagingRoot);
|
|
await ensurePrivateDirectory(this.objectRoot);
|
|
})();
|
|
return this.initialization;
|
|
}
|
|
|
|
private objectPath(storageKey: string): string {
|
|
if (!storageKeyPattern.test(storageKey)) throw storageError('artifact-store-key-invalid', 'Der Artefaktspeicherschlüssel ist ungültig.');
|
|
return join(this.objectRoot, `${storageKey}.bin`);
|
|
}
|
|
}
|