feat: bounded Evidence materializer with manifest and atomic publication (P6)

backend/src/workspaces/evidence-materialization.ts

@@ -0,0 +1,155 @@
+import { createHash } from "node:crypto";
+import {
+  closeSync,
+  constants as fsConstants,
+  fchmodSync,
+  fsyncSync,
+  mkdirSync,
+  openSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, isAbsolute, join } from "node:path";
+import { GitWorkspaceRepository } from "./git-repository.js";
+
+export interface EvidenceMaterializationLimits {
+  maxEntries: number;
+  maxTotalBytes: number;
+  maxFileBytes: number;
+  maxPathBytes: number;
+  maxManifestBytes: number;
+}
+
+export const DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS: EvidenceMaterializationLimits = {
+  maxEntries: 4096,
+  maxTotalBytes: 64 * 1024 * 1024,
+  maxFileBytes: 8 * 1024 * 1024,
+  maxPathBytes: 4096,
+  maxManifestBytes: 1024 * 1024,
+};
+
+export interface EvidenceManifestFile {
+  mode: "100644" | "100755";
+  oid: string;
+  digest: string;
+  bytes: number;
+}
+
+export interface EvidenceManifest {
+  schemaVersion: 1;
+  workspace: string;
+  commit: string;
+  tree: string;
+  entryCount: number;
+  totalBytes: number;
+  files: Record<string, EvidenceManifestFile>;
+}
+
+export interface MaterializedEvidence {
+  root: string;
+  manifestPath: string;
+  manifest: EvidenceManifest;
+  /** 64-hex sha256 of the manifest bytes, for the snapshot manifest integrity chain. */
+  manifestDigest: string;
+}
+
+function sha256Hex(value: Buffer | string): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function sha256Prefixed(value: Buffer | string): string {
+  return `sha256:${sha256Hex(value)}`;
+}
+
+function writeExclusiveNoFollow(path: string, contents: Buffer, mode: number): void {
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  const fd = openSync(
+    path,
+    fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW,
+    0o600,
+  );
+  let closed = false;
+  try {
+    writeFileSync(fd, contents);
+    fsyncSync(fd);
+    fchmodSync(fd, mode);
+    closeSync(fd);
+    closed = true;
+  } catch (error) {
+    if (!closed) try { closeSync(fd); } catch { /* preserve original failure */ }
+    throw error;
+  }
+}
+
+export interface MaterializeEvidenceTreeOptions {
+  repository: GitWorkspaceRepository;
+  revision: string;
+  id: string;
+  /** The workspace directory (e.g. `<staging>/<id>`) that will receive `evidence/` and the manifest. */
+  targetDirectory: string;
+  limits?: Partial<EvidenceMaterializationLimits>;
+}
+
+/**
+ * Materialize a canonical `<id>/evidence` tree from an exact commit into an owned staging
+ * directory with a bounded manifest. Never follows symlinks; a bound violation or unsafe object
+ * aborts before any atomic publication. The caller is responsible for the final atomic rename.
+ */
+export async function materializeEvidenceTree(options: MaterializeEvidenceTreeOptions): Promise<MaterializedEvidence> {
+  const limits: EvidenceMaterializationLimits = { ...DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS, ...options.limits };
+  if (!/^[0-9a-f]{40}$/.test(options.revision)) throw new Error("evidence revision is invalid");
+  if (!/^[a-z][a-z0-9-]{2,62}$/.test(options.id)) throw new Error("evidence workspace id is invalid");
+  if (!isAbsolute(options.targetDirectory)) throw new Error("evidence target directory must be absolute");
+
+  const objects = await options.repository.evidenceTreeObjects(options.revision, options.id);
+  if (objects.length > limits.maxEntries) throw new Error("evidence entry count exceeds the bound");
+  const tree = await options.repository.evidenceTreeId(options.revision, options.id);
+
+  // Disk-space preflight: sum the real object sizes before writing anything.
+  const sizes = new Map<string, number>();
+  let totalBytes = 0;
+  for (const entry of objects) {
+    if (Buffer.byteLength(entry.posixPath, "utf8") > limits.maxPathBytes) {
+      throw new Error("evidence path exceeds the bound");
+    }
+    const size = await options.repository.gitObjectSize(entry.oid);
+    if (size > limits.maxFileBytes) throw new Error("evidence file exceeds the bound");
+    sizes.set(entry.oid, size);
+    totalBytes += size;
+    if (totalBytes > limits.maxTotalBytes) throw new Error("evidence total bytes exceed the bound");
+  }
+
+  const root = join(options.targetDirectory, "evidence");
+  mkdirSync(root, { recursive: true, mode: 0o700 });
+  const files: Record<string, EvidenceManifestFile> = {};
+  for (const entry of objects) {
+    const contents = await options.repository.evidenceBlobBytes(entry.oid, limits.maxFileBytes);
+    if (contents.length !== sizes.get(entry.oid)) {
+      throw new Error("evidence object changed while materializing");
+    }
+    const target = join(root, ...entry.posixPath.split("/"));
+    writeExclusiveNoFollow(target, contents, entry.mode === "100755" ? 0o755 : 0o644);
+    files[entry.posixPath] = {
+      mode: entry.mode,
+      oid: entry.oid,
+      digest: sha256Prefixed(contents),
+      bytes: contents.length,
+    };
+  }
+
+  const manifest: EvidenceManifest = {
+    schemaVersion: 1,
+    workspace: options.id,
+    commit: options.revision,
+    tree,
+    entryCount: objects.length,
+    totalBytes,
+    files,
+  };
+  const manifestJson = `${JSON.stringify(manifest)}\n`;
+  if (Buffer.byteLength(manifestJson, "utf8") > limits.maxManifestBytes) {
+    throw new Error("evidence manifest exceeds the bound");
+  }
+  const manifestPath = join(options.targetDirectory, "evidence.manifest.json");
+  writeExclusiveNoFollow(manifestPath, Buffer.from(manifestJson, "utf8"), 0o600);
+  return { root, manifestPath, manifest, manifestDigest: sha256Hex(manifestJson) };
+}

backend/src/workspaces/git-repository.ts

@@ -332,6 +332,32 @@ export class GitWorkspaceRepository {
     return this.gitBlobBytes(objectId, maxBytes, "Evidence");
   }
 
+  /** Return the 40-hex tree id of a canonical Evidence root at an exact commit. */
+  async evidenceTreeId(revision: string, id: string): Promise<string> {
+    if (!/^[0-9a-f]{40}$/.test(revision) || !/^[a-z][a-z0-9-]{2,62}$/.test(id)) {
+      throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence revision is invalid");
+    }
+    const objectId = (await this.git(["rev-parse", `${revision}:${id}/evidence`])).trim();
+    const type = (await this.git(["cat-file", "-t", objectId])).trim();
+    if (!/^[0-9a-f]{40}$/.test(objectId) || type !== "tree") {
+      throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence root is invalid");
+    }
+    return objectId;
+  }
+
+  /** Return the byte size of one Git object without reading its contents. */
+  async gitObjectSize(objectId: string): Promise<number> {
+    if (!/^[0-9a-f]{40}$/.test(objectId)) {
+      throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object is invalid");
+    }
+    const raw = (await this.git(["cat-file", "-s", objectId])).trim();
+    const size = Number(raw);
+    if (!Number.isSafeInteger(size) || size < 0) {
+      throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object size is invalid");
+    }
+    return size;
+  }
+
   private async gitBlobBytes(objectId: string, maxBytes: number, label: string): Promise<Buffer> {
     if (!/^[0-9a-f]{40}$/.test(objectId)) {
       throw new WorkspaceRegistryError("workspace_invalid", `Workspace ${label} object is invalid`);

backend/test/evidence-materialization.test.ts

@@ -0,0 +1,134 @@
+import { execFile } from "node:child_process";
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync, readdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { afterEach, expect, test } from "vitest";
+import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js";
+import { materializeEvidenceTree } from "../src/workspaces/evidence-materialization.js";
+import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
+
+const runFile = promisify(execFile);
+const temporaryRoots: string[] = [];
+
+afterEach(() => {
+  temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
+});
+
+async function git(cwd: string, args: string[]): Promise<string> {
+  const { stdout } = await runFile("git", args, { cwd });
+  return stdout.trim();
+}
+
+function config(root: string, remoteUrl: string): WorkspaceRegistryConfig {
+  return {
+    root,
+    remoteUrl,
+    branch: "main",
+    gitAuthorName: "Evidence Materializer Test",
+    gitAuthorEmail: "evidence-materializer@example.invalid",
+    installationId: "test",
+    secretRoots: [],
+    maxImportBytes: 1024,
+    maxImportEntries: 1,
+  };
+}
+
+async function fixture(): Promise<{ root: string; remote: string; commit: string }> {
+  const root = mkdtempSync(join(tmpdir(), "thoth-evidence-materializer-"));
+  temporaryRoots.push(root);
+  const remote = join(root, "remote.git");
+  const source = join(root, "source");
+  await git(root, ["init", "--bare", "--initial-branch=main", remote]);
+  mkdirSync(source);
+  await git(source, ["init", "--initial-branch=main"]);
+  await git(source, ["config", "user.name", "Evidence Materializer Test"]);
+  await git(source, ["config", "user.email", "evidence-materializer@example.invalid"]);
+  writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n");
+  mkdirSync(join(source, "research", "evidence", "nested"), { recursive: true });
+  writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n  schema_version: 3\n  id: research\n");
+  writeFileSync(join(source, "research", "evidence", "guide.md"), "# guide\n");
+  writeFileSync(join(source, "research", "evidence", "nested", "deep.md"), "# deep\n");
+  await git(source, ["add", "-A"]);
+  await git(source, ["commit", "-m", "initial"]);
+  await git(source, ["remote", "add", "origin", remote]);
+  await git(source, ["push", "origin", "main"]);
+  const commit = await git(source, ["rev-parse", "HEAD"]);
+  return { root, remote, commit };
+}
+
+async function repo(fixture: { root: string; remote: string }): Promise<GitWorkspaceRepository> {
+  const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote));
+  await repository.bootstrap();
+  return repository;
+}
+
+test("materializes the tree, hashes every file, and writes a bounded manifest", async () => {
+  const fixtureValue = await fixture();
+  const repository = await repo(fixtureValue);
+  const target = mkdtempSync(join(tmpdir(), "thoth-evidence-target-"));
+  temporaryRoots.push(target);
+
+  const result = await materializeEvidenceTree({
+    repository,
+    revision: fixtureValue.commit,
+    id: "research",
+    targetDirectory: target,
+  });
+
+  expect(readFileSync(join(result.root, "guide.md"), "utf8")).toBe("# guide\n");
+  expect(readFileSync(join(result.root, "nested", "deep.md"), "utf8")).toBe("# deep\n");
+  expect(result.manifest).toMatchObject({
+    schemaVersion: 1,
+    workspace: "research",
+    commit: fixtureValue.commit,
+    entryCount: 2,
+  });
+  expect(Object.keys(result.manifest.files).sort()).toEqual(["guide.md", "nested/deep.md"]);
+  expect(result.manifest.files["guide.md"]!.digest).toMatch(/^sha256:[0-9a-f]{64}$/);
+  expect(result.manifestDigest).toMatch(/^[0-9a-f]{64}$/);
+  expect(readFileSync(result.manifestPath, "utf8")).toContain('"entryCount":2');
+});
+
+test("refuses symlink-containing trees and bound violations without publishing", async () => {
+  const root = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-"));
+  temporaryRoots.push(root);
+  const remote = join(root, "remote.git");
+  const source = join(root, "source");
+  await git(root, ["init", "--bare", "--initial-branch=main", remote]);
+  mkdirSync(source);
+  await git(source, ["init", "--initial-branch=main"]);
+  await git(source, ["config", "user.name", "E"]);
+  await git(source, ["config", "user.email", "e@e.invalid"]);
+  writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n");
+  mkdirSync(join(source, "research", "evidence"), { recursive: true });
+  writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n  schema_version: 3\n  id: research\n");
+  writeFileSync(join(source, "research", "outside.md"), "# outside\n");
+  symlinkSync("../outside.md", join(source, "research", "evidence", "link.md"));
+  await git(source, ["add", "-A"]);
+  await git(source, ["commit", "-m", "symlink"]);
+  await git(source, ["remote", "add", "origin", remote]);
+  await git(source, ["push", "origin", "main"]);
+  const commit = await git(source, ["rev-parse", "HEAD"]);
+  const repository = await repo({ root, remote });
+  const target = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-target-"));
+  temporaryRoots.push(target);
+
+  await expect(materializeEvidenceTree({ repository, revision: commit, id: "research", targetDirectory: target }))
+    .rejects.toThrow();
+  expect(readdirSync(target)).toEqual([]);
+
+  // A valid tree but a per-file bound of 1 byte must also refuse.
+  const fixtureValue = await fixture();
+  const repository2 = await repo(fixtureValue);
+  const target2 = mkdtempSync(join(tmpdir(), "thoth-evidence-bound-target-"));
+  temporaryRoots.push(target2);
+  await expect(materializeEvidenceTree({
+    repository: repository2,
+    revision: fixtureValue.commit,
+    id: "research",
+    targetDirectory: target2,
+    limits: { maxFileBytes: 1 },
+  })).rejects.toThrow();
+  expect(readdirSync(target2)).toEqual([]);
+});