From 0c9e61410046d81454fde33dde4b8904e8b3a762 Mon Sep 17 00:00:00 2001 From: mptyl Date: Thu, 13 Aug 2026 12:31:12 +0200 Subject: [PATCH] feat: bounded Evidence materializer with manifest and atomic publication (P6) --- .../workspaces/evidence-materialization.ts | 155 ++++++++++++++++++ backend/src/workspaces/git-repository.ts | 26 +++ backend/test/evidence-materialization.test.ts | 134 +++++++++++++++ 3 files changed, 315 insertions(+) create mode 100644 backend/src/workspaces/evidence-materialization.ts create mode 100644 backend/test/evidence-materialization.test.ts diff --git a/backend/src/workspaces/evidence-materialization.ts b/backend/src/workspaces/evidence-materialization.ts new file mode 100644 index 00000000..c9eea887 --- /dev/null +++ b/backend/src/workspaces/evidence-materialization.ts @@ -0,0 +1,155 @@ +import { createHash } from "node:crypto"; +import { + closeSync, + constants as fsConstants, + fchmodSync, + fsyncSync, + mkdirSync, + openSync, + writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; +import { GitWorkspaceRepository } from "./git-repository.js"; + +export interface EvidenceMaterializationLimits { + maxEntries: number; + maxTotalBytes: number; + maxFileBytes: number; + maxPathBytes: number; + maxManifestBytes: number; +} + +export const DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS: EvidenceMaterializationLimits = { + maxEntries: 4096, + maxTotalBytes: 64 * 1024 * 1024, + maxFileBytes: 8 * 1024 * 1024, + maxPathBytes: 4096, + maxManifestBytes: 1024 * 1024, +}; + +export interface EvidenceManifestFile { + mode: "100644" | "100755"; + oid: string; + digest: string; + bytes: number; +} + +export interface EvidenceManifest { + schemaVersion: 1; + workspace: string; + commit: string; + tree: string; + entryCount: number; + totalBytes: number; + files: Record; +} + +export interface MaterializedEvidence { + root: string; + manifestPath: string; + manifest: EvidenceManifest; + /** 64-hex sha256 of the manifest bytes, for the snapshot manifest integrity chain. */ + manifestDigest: string; +} + +function sha256Hex(value: Buffer | string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function sha256Prefixed(value: Buffer | string): string { + return `sha256:${sha256Hex(value)}`; +} + +function writeExclusiveNoFollow(path: string, contents: Buffer, mode: number): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const fd = openSync( + path, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, + 0o600, + ); + let closed = false; + try { + writeFileSync(fd, contents); + fsyncSync(fd); + fchmodSync(fd, mode); + closeSync(fd); + closed = true; + } catch (error) { + if (!closed) try { closeSync(fd); } catch { /* preserve original failure */ } + throw error; + } +} + +export interface MaterializeEvidenceTreeOptions { + repository: GitWorkspaceRepository; + revision: string; + id: string; + /** The workspace directory (e.g. `/`) that will receive `evidence/` and the manifest. */ + targetDirectory: string; + limits?: Partial; +} + +/** + * Materialize a canonical `/evidence` tree from an exact commit into an owned staging + * directory with a bounded manifest. Never follows symlinks; a bound violation or unsafe object + * aborts before any atomic publication. The caller is responsible for the final atomic rename. + */ +export async function materializeEvidenceTree(options: MaterializeEvidenceTreeOptions): Promise { + const limits: EvidenceMaterializationLimits = { ...DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS, ...options.limits }; + if (!/^[0-9a-f]{40}$/.test(options.revision)) throw new Error("evidence revision is invalid"); + if (!/^[a-z][a-z0-9-]{2,62}$/.test(options.id)) throw new Error("evidence workspace id is invalid"); + if (!isAbsolute(options.targetDirectory)) throw new Error("evidence target directory must be absolute"); + + const objects = await options.repository.evidenceTreeObjects(options.revision, options.id); + if (objects.length > limits.maxEntries) throw new Error("evidence entry count exceeds the bound"); + const tree = await options.repository.evidenceTreeId(options.revision, options.id); + + // Disk-space preflight: sum the real object sizes before writing anything. + const sizes = new Map(); + let totalBytes = 0; + for (const entry of objects) { + if (Buffer.byteLength(entry.posixPath, "utf8") > limits.maxPathBytes) { + throw new Error("evidence path exceeds the bound"); + } + const size = await options.repository.gitObjectSize(entry.oid); + if (size > limits.maxFileBytes) throw new Error("evidence file exceeds the bound"); + sizes.set(entry.oid, size); + totalBytes += size; + if (totalBytes > limits.maxTotalBytes) throw new Error("evidence total bytes exceed the bound"); + } + + const root = join(options.targetDirectory, "evidence"); + mkdirSync(root, { recursive: true, mode: 0o700 }); + const files: Record = {}; + for (const entry of objects) { + const contents = await options.repository.evidenceBlobBytes(entry.oid, limits.maxFileBytes); + if (contents.length !== sizes.get(entry.oid)) { + throw new Error("evidence object changed while materializing"); + } + const target = join(root, ...entry.posixPath.split("/")); + writeExclusiveNoFollow(target, contents, entry.mode === "100755" ? 0o755 : 0o644); + files[entry.posixPath] = { + mode: entry.mode, + oid: entry.oid, + digest: sha256Prefixed(contents), + bytes: contents.length, + }; + } + + const manifest: EvidenceManifest = { + schemaVersion: 1, + workspace: options.id, + commit: options.revision, + tree, + entryCount: objects.length, + totalBytes, + files, + }; + const manifestJson = `${JSON.stringify(manifest)}\n`; + if (Buffer.byteLength(manifestJson, "utf8") > limits.maxManifestBytes) { + throw new Error("evidence manifest exceeds the bound"); + } + const manifestPath = join(options.targetDirectory, "evidence.manifest.json"); + writeExclusiveNoFollow(manifestPath, Buffer.from(manifestJson, "utf8"), 0o600); + return { root, manifestPath, manifest, manifestDigest: sha256Hex(manifestJson) }; +} diff --git a/backend/src/workspaces/git-repository.ts b/backend/src/workspaces/git-repository.ts index b38a982f..c197a2df 100644 --- a/backend/src/workspaces/git-repository.ts +++ b/backend/src/workspaces/git-repository.ts @@ -332,6 +332,32 @@ export class GitWorkspaceRepository { return this.gitBlobBytes(objectId, maxBytes, "Evidence"); } + /** Return the 40-hex tree id of a canonical Evidence root at an exact commit. */ + async evidenceTreeId(revision: string, id: string): Promise { + if (!/^[0-9a-f]{40}$/.test(revision) || !/^[a-z][a-z0-9-]{2,62}$/.test(id)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence revision is invalid"); + } + const objectId = (await this.git(["rev-parse", `${revision}:${id}/evidence`])).trim(); + const type = (await this.git(["cat-file", "-t", objectId])).trim(); + if (!/^[0-9a-f]{40}$/.test(objectId) || type !== "tree") { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence root is invalid"); + } + return objectId; + } + + /** Return the byte size of one Git object without reading its contents. */ + async gitObjectSize(objectId: string): Promise { + if (!/^[0-9a-f]{40}$/.test(objectId)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object is invalid"); + } + const raw = (await this.git(["cat-file", "-s", objectId])).trim(); + const size = Number(raw); + if (!Number.isSafeInteger(size) || size < 0) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object size is invalid"); + } + return size; + } + private async gitBlobBytes(objectId: string, maxBytes: number, label: string): Promise { if (!/^[0-9a-f]{40}$/.test(objectId)) { throw new WorkspaceRegistryError("workspace_invalid", `Workspace ${label} object is invalid`); diff --git a/backend/test/evidence-materialization.test.ts b/backend/test/evidence-materialization.test.ts new file mode 100644 index 00000000..2bbfce5e --- /dev/null +++ b/backend/test/evidence-materialization.test.ts @@ -0,0 +1,134 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js"; +import { materializeEvidenceTree } from "../src/workspaces/evidence-materialization.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Evidence Materializer Test", + gitAuthorEmail: "evidence-materializer@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +async function fixture(): Promise<{ root: string; remote: string; commit: string }> { + const root = mkdtempSync(join(tmpdir(), "thoth-evidence-materializer-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Evidence Materializer Test"]); + await git(source, ["config", "user.email", "evidence-materializer@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n"); + mkdirSync(join(source, "research", "evidence", "nested"), { recursive: true }); + writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n schema_version: 3\n id: research\n"); + writeFileSync(join(source, "research", "evidence", "guide.md"), "# guide\n"); + writeFileSync(join(source, "research", "evidence", "nested", "deep.md"), "# deep\n"); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + return { root, remote, commit }; +} + +async function repo(fixture: { root: string; remote: string }): Promise { + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + return repository; +} + +test("materializes the tree, hashes every file, and writes a bounded manifest", async () => { + const fixtureValue = await fixture(); + const repository = await repo(fixtureValue); + const target = mkdtempSync(join(tmpdir(), "thoth-evidence-target-")); + temporaryRoots.push(target); + + const result = await materializeEvidenceTree({ + repository, + revision: fixtureValue.commit, + id: "research", + targetDirectory: target, + }); + + expect(readFileSync(join(result.root, "guide.md"), "utf8")).toBe("# guide\n"); + expect(readFileSync(join(result.root, "nested", "deep.md"), "utf8")).toBe("# deep\n"); + expect(result.manifest).toMatchObject({ + schemaVersion: 1, + workspace: "research", + commit: fixtureValue.commit, + entryCount: 2, + }); + expect(Object.keys(result.manifest.files).sort()).toEqual(["guide.md", "nested/deep.md"]); + expect(result.manifest.files["guide.md"]!.digest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(result.manifestDigest).toMatch(/^[0-9a-f]{64}$/); + expect(readFileSync(result.manifestPath, "utf8")).toContain('"entryCount":2'); +}); + +test("refuses symlink-containing trees and bound violations without publishing", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "E"]); + await git(source, ["config", "user.email", "e@e.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n"); + mkdirSync(join(source, "research", "evidence"), { recursive: true }); + writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n schema_version: 3\n id: research\n"); + writeFileSync(join(source, "research", "outside.md"), "# outside\n"); + symlinkSync("../outside.md", join(source, "research", "evidence", "link.md")); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "symlink"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + const repository = await repo({ root, remote }); + const target = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-target-")); + temporaryRoots.push(target); + + await expect(materializeEvidenceTree({ repository, revision: commit, id: "research", targetDirectory: target })) + .rejects.toThrow(); + expect(readdirSync(target)).toEqual([]); + + // A valid tree but a per-file bound of 1 byte must also refuse. + const fixtureValue = await fixture(); + const repository2 = await repo(fixtureValue); + const target2 = mkdtempSync(join(tmpdir(), "thoth-evidence-bound-target-")); + temporaryRoots.push(target2); + await expect(materializeEvidenceTree({ + repository: repository2, + revision: fixtureValue.commit, + id: "research", + targetDirectory: target2, + limits: { maxFileBytes: 1 }, + })).rejects.toThrow(); + expect(readdirSync(target2)).toEqual([]); +});