mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: bounded Evidence materializer with manifest and atomic publication (P6)

Browse Source
This commit is contained in:
marcopan
2026-08-13 12:31:12 +02:00
parent 0c1033889a
commit 0c9e614100
3 changed files with 315 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/workspaces/evidence-materialization.ts
+155
View File
@@ -0,0 +1,155 @@
import { createHash } from "node:crypto";
import {
closeSync,
constants as fsConstants,
fchmodSync,
fsyncSync,
mkdirSync,
openSync,
writeFileSync,
} from "node:fs";
import { dirname, isAbsolute, join } from "node:path";
import { GitWorkspaceRepository } from "./git-repository.js";
export interface EvidenceMaterializationLimits {
maxEntries: number;
maxTotalBytes: number;
maxFileBytes: number;
maxPathBytes: number;
maxManifestBytes: number;
}
export const DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS: EvidenceMaterializationLimits = {
maxEntries: 4096,
maxTotalBytes: 64 * 1024 * 1024,
maxFileBytes: 8 * 1024 * 1024,
maxPathBytes: 4096,
maxManifestBytes: 1024 * 1024,
};
export interface EvidenceManifestFile {
mode: "100644" | "100755";
oid: string;
digest: string;
bytes: number;
}
export interface EvidenceManifest {
schemaVersion: 1;
workspace: string;
commit: string;
tree: string;
entryCount: number;
totalBytes: number;
files: Record<string, EvidenceManifestFile>;
}
export interface MaterializedEvidence {
root: string;
manifestPath: string;
manifest: EvidenceManifest;
/** 64-hex sha256 of the manifest bytes, for the snapshot manifest integrity chain. */
manifestDigest: string;
}
function sha256Hex(value: Buffer | string): string {
return createHash("sha256").update(value).digest("hex");
}
function sha256Prefixed(value: Buffer | string): string {
return `sha256:${sha256Hex(value)}`;
}
function writeExclusiveNoFollow(path: string, contents: Buffer, mode: number): void {
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
const fd = openSync(
path,
fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW,
0o600,
);
let closed = false;
try {
writeFileSync(fd, contents);
fsyncSync(fd);
fchmodSync(fd, mode);
closeSync(fd);
closed = true;
} catch (error) {
if (!closed) try { closeSync(fd); } catch { /* preserve original failure */ }
throw error;
}
}
export interface MaterializeEvidenceTreeOptions {
repository: GitWorkspaceRepository;
revision: string;
id: string;
/** The workspace directory (e.g. `<staging>/<id>`) that will receive `evidence/` and the manifest. */
targetDirectory: string;
limits?: Partial<EvidenceMaterializationLimits>;
}
/**
* Materialize a canonical `<id>/evidence` tree from an exact commit into an owned staging
* directory with a bounded manifest. Never follows symlinks; a bound violation or unsafe object
* aborts before any atomic publication. The caller is responsible for the final atomic rename.
*/
export async function materializeEvidenceTree(options: MaterializeEvidenceTreeOptions): Promise<MaterializedEvidence> {
const limits: EvidenceMaterializationLimits = { ...DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS, ...options.limits };
if (!/^[0-9a-f]{40}$/.test(options.revision)) throw new Error("evidence revision is invalid");
if (!/^[a-z][a-z0-9-]{2,62}$/.test(options.id)) throw new Error("evidence workspace id is invalid");
if (!isAbsolute(options.targetDirectory)) throw new Error("evidence target directory must be absolute");
const objects = await options.repository.evidenceTreeObjects(options.revision, options.id);
if (objects.length > limits.maxEntries) throw new Error("evidence entry count exceeds the bound");
const tree = await options.repository.evidenceTreeId(options.revision, options.id);
// Disk-space preflight: sum the real object sizes before writing anything.
const sizes = new Map<string, number>();
let totalBytes = 0;
for (const entry of objects) {
if (Buffer.byteLength(entry.posixPath, "utf8") > limits.maxPathBytes) {
throw new Error("evidence path exceeds the bound");
}
const size = await options.repository.gitObjectSize(entry.oid);
if (size > limits.maxFileBytes) throw new Error("evidence file exceeds the bound");
sizes.set(entry.oid, size);
totalBytes += size;
if (totalBytes > limits.maxTotalBytes) throw new Error("evidence total bytes exceed the bound");
}
const root = join(options.targetDirectory, "evidence");
mkdirSync(root, { recursive: true, mode: 0o700 });
const files: Record<string, EvidenceManifestFile> = {};
for (const entry of objects) {
const contents = await options.repository.evidenceBlobBytes(entry.oid, limits.maxFileBytes);
if (contents.length !== sizes.get(entry.oid)) {
throw new Error("evidence object changed while materializing");
}
const target = join(root, ...entry.posixPath.split("/"));
writeExclusiveNoFollow(target, contents, entry.mode === "100755" ? 0o755 : 0o644);
files[entry.posixPath] = {
mode: entry.mode,
oid: entry.oid,
digest: sha256Prefixed(contents),
bytes: contents.length,
};
}
const manifest: EvidenceManifest = {
schemaVersion: 1,
workspace: options.id,
commit: options.revision,
tree,
entryCount: objects.length,
totalBytes,
files,
};
const manifestJson = `${JSON.stringify(manifest)}\n`;
if (Buffer.byteLength(manifestJson, "utf8") > limits.maxManifestBytes) {
throw new Error("evidence manifest exceeds the bound");
}
const manifestPath = join(options.targetDirectory, "evidence.manifest.json");
writeExclusiveNoFollow(manifestPath, Buffer.from(manifestJson, "utf8"), 0o600);
return { root, manifestPath, manifest, manifestDigest: sha256Hex(manifestJson) };
}
backend/src/workspaces/git-repository.ts
+26
View File
@@ -332,6 +332,32 @@ export class GitWorkspaceRepository {
return this.gitBlobBytes(objectId, maxBytes, "Evidence");
}
/** Return the 40-hex tree id of a canonical Evidence root at an exact commit. */
async evidenceTreeId(revision: string, id: string): Promise<string> {
if (!/^[0-9a-f]{40}$/.test(revision) || !/^[a-z][a-z0-9-]{2,62}$/.test(id)) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence revision is invalid");
}
const objectId = (await this.git(["rev-parse", `${revision}:${id}/evidence`])).trim();
const type = (await this.git(["cat-file", "-t", objectId])).trim();
if (!/^[0-9a-f]{40}$/.test(objectId) || type !== "tree") {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence root is invalid");
}
return objectId;
}
/** Return the byte size of one Git object without reading its contents. */
async gitObjectSize(objectId: string): Promise<number> {
if (!/^[0-9a-f]{40}$/.test(objectId)) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object is invalid");
}
const raw = (await this.git(["cat-file", "-s", objectId])).trim();
const size = Number(raw);
if (!Number.isSafeInteger(size) || size < 0) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object size is invalid");
}
return size;
}
private async gitBlobBytes(objectId: string, maxBytes: number, label: string): Promise<Buffer> {
if (!/^[0-9a-f]{40}$/.test(objectId)) {
throw new WorkspaceRegistryError("workspace_invalid", `Workspace ${label} object is invalid`);
backend/test/evidence-materialization.test.ts
+134
View File
@@ -0,0 +1,134 @@
import { execFile } from "node:child_process";
import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync, readdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { afterEach, expect, test } from "vitest";
import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js";
import { materializeEvidenceTree } from "../src/workspaces/evidence-materialization.js";
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
const runFile = promisify(execFile);
const temporaryRoots: string[] = [];
afterEach(() => {
temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true }));
});
async function git(cwd: string, args: string[]): Promise<string> {
const { stdout } = await runFile("git", args, { cwd });
return stdout.trim();
}
function config(root: string, remoteUrl: string): WorkspaceRegistryConfig {
return {
root,
remoteUrl,
branch: "main",
gitAuthorName: "Evidence Materializer Test",
gitAuthorEmail: "evidence-materializer@example.invalid",
installationId: "test",
secretRoots: [],
maxImportBytes: 1024,
maxImportEntries: 1,
};
}
async function fixture(): Promise<{ root: string; remote: string; commit: string }> {
const root = mkdtempSync(join(tmpdir(), "thoth-evidence-materializer-"));
temporaryRoots.push(root);
const remote = join(root, "remote.git");
const source = join(root, "source");
await git(root, ["init", "--bare", "--initial-branch=main", remote]);
mkdirSync(source);
await git(source, ["init", "--initial-branch=main"]);
await git(source, ["config", "user.name", "Evidence Materializer Test"]);
await git(source, ["config", "user.email", "evidence-materializer@example.invalid"]);
writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n");
mkdirSync(join(source, "research", "evidence", "nested"), { recursive: true });
writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n schema_version: 3\n id: research\n");
writeFileSync(join(source, "research", "evidence", "guide.md"), "# guide\n");
writeFileSync(join(source, "research", "evidence", "nested", "deep.md"), "# deep\n");
await git(source, ["add", "-A"]);
await git(source, ["commit", "-m", "initial"]);
await git(source, ["remote", "add", "origin", remote]);
await git(source, ["push", "origin", "main"]);
const commit = await git(source, ["rev-parse", "HEAD"]);
return { root, remote, commit };
}
async function repo(fixture: { root: string; remote: string }): Promise<GitWorkspaceRepository> {
const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote));
await repository.bootstrap();
return repository;
}
test("materializes the tree, hashes every file, and writes a bounded manifest", async () => {
const fixtureValue = await fixture();
const repository = await repo(fixtureValue);
const target = mkdtempSync(join(tmpdir(), "thoth-evidence-target-"));
temporaryRoots.push(target);
const result = await materializeEvidenceTree({
repository,
revision: fixtureValue.commit,
id: "research",
targetDirectory: target,
});
expect(readFileSync(join(result.root, "guide.md"), "utf8")).toBe("# guide\n");
expect(readFileSync(join(result.root, "nested", "deep.md"), "utf8")).toBe("# deep\n");
expect(result.manifest).toMatchObject({
schemaVersion: 1,
workspace: "research",
commit: fixtureValue.commit,
entryCount: 2,
});
expect(Object.keys(result.manifest.files).sort()).toEqual(["guide.md", "nested/deep.md"]);
expect(result.manifest.files["guide.md"]!.digest).toMatch(/^sha256:[0-9a-f]{64}$/);
expect(result.manifestDigest).toMatch(/^[0-9a-f]{64}$/);
expect(readFileSync(result.manifestPath, "utf8")).toContain('"entryCount":2');
});
test("refuses symlink-containing trees and bound violations without publishing", async () => {
const root = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-"));
temporaryRoots.push(root);
const remote = join(root, "remote.git");
const source = join(root, "source");
await git(root, ["init", "--bare", "--initial-branch=main", remote]);
mkdirSync(source);
await git(source, ["init", "--initial-branch=main"]);
await git(source, ["config", "user.name", "E"]);
await git(source, ["config", "user.email", "e@e.invalid"]);
writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n");
mkdirSync(join(source, "research", "evidence"), { recursive: true });
writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n schema_version: 3\n id: research\n");
writeFileSync(join(source, "research", "outside.md"), "# outside\n");
symlinkSync("../outside.md", join(source, "research", "evidence", "link.md"));
await git(source, ["add", "-A"]);
await git(source, ["commit", "-m", "symlink"]);
await git(source, ["remote", "add", "origin", remote]);
await git(source, ["push", "origin", "main"]);
const commit = await git(source, ["rev-parse", "HEAD"]);
const repository = await repo({ root, remote });
const target = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-target-"));
temporaryRoots.push(target);
await expect(materializeEvidenceTree({ repository, revision: commit, id: "research", targetDirectory: target }))
.rejects.toThrow();
expect(readdirSync(target)).toEqual([]);
// A valid tree but a per-file bound of 1 byte must also refuse.
const fixtureValue = await fixture();
const repository2 = await repo(fixtureValue);
const target2 = mkdtempSync(join(tmpdir(), "thoth-evidence-bound-target-"));
temporaryRoots.push(target2);
await expect(materializeEvidenceTree({
repository: repository2,
revision: fixtureValue.commit,
id: "research",
targetDirectory: target2,
limits: { maxFileBytes: 1 },
})).rejects.toThrow();
expect(readdirSync(target2)).toEqual([]);
});