refactor(evidence): extract TypeScript lifecycle (#32)
This commit is contained in:
@@ -0,0 +1,158 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
closeSync,
|
||||
constants as fsConstants,
|
||||
fchmodSync,
|
||||
fsyncSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { dirname, isAbsolute, join } from "node:path";
|
||||
import type { GitWorkspaceRepository } from "../git-repository.js";
|
||||
|
||||
export interface EvidenceMaterializationLimits {
|
||||
maxEntries: number;
|
||||
maxTotalBytes: number;
|
||||
maxFileBytes: number;
|
||||
maxPathBytes: number;
|
||||
maxManifestBytes: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS: EvidenceMaterializationLimits = {
|
||||
maxEntries: 4096,
|
||||
maxTotalBytes: 64 * 1024 * 1024,
|
||||
maxFileBytes: 8 * 1024 * 1024,
|
||||
maxPathBytes: 4096,
|
||||
maxManifestBytes: 1024 * 1024,
|
||||
};
|
||||
|
||||
export interface EvidenceManifestFile {
|
||||
mode: "100644" | "100755";
|
||||
oid: string;
|
||||
digest: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface EvidenceManifest {
|
||||
schemaVersion: 1;
|
||||
workspace: string;
|
||||
commit: string;
|
||||
tree: string;
|
||||
entryCount: number;
|
||||
totalBytes: number;
|
||||
files: Record<string, EvidenceManifestFile>;
|
||||
}
|
||||
|
||||
export interface MaterializedEvidence {
|
||||
root: string;
|
||||
manifestPath: string;
|
||||
manifest: EvidenceManifest;
|
||||
/** 64-hex sha256 of the manifest bytes, for the snapshot manifest integrity chain. */
|
||||
manifestDigest: string;
|
||||
}
|
||||
|
||||
function sha256Hex(value: Buffer | string): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function sha256Prefixed(value: Buffer | string): string {
|
||||
return `sha256:${sha256Hex(value)}`;
|
||||
}
|
||||
|
||||
function writeExclusiveNoFollow(path: string, contents: Buffer, mode: number): void {
|
||||
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
||||
const fd = openSync(
|
||||
path,
|
||||
fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW,
|
||||
0o600,
|
||||
);
|
||||
let closed = false;
|
||||
try {
|
||||
writeFileSync(fd, contents);
|
||||
fsyncSync(fd);
|
||||
fchmodSync(fd, mode);
|
||||
closeSync(fd);
|
||||
closed = true;
|
||||
} catch (error) {
|
||||
if (!closed) try { closeSync(fd); } catch { /* preserve original failure */ }
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export interface MaterializeEvidenceTreeOptions {
|
||||
repository: Pick<
|
||||
GitWorkspaceRepository,
|
||||
"evidenceTreeObjects" | "evidenceTreeId" | "gitObjectSize" | "evidenceBlobBytes"
|
||||
>;
|
||||
revision: string;
|
||||
id: string;
|
||||
/** The workspace directory (e.g. `<staging>/<id>`) that will receive `evidence/` and the manifest. */
|
||||
targetDirectory: string;
|
||||
limits?: Partial<EvidenceMaterializationLimits>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Materialize a canonical `<id>/evidence` tree from an exact commit into an owned staging
|
||||
* directory with a bounded manifest. Never follows symlinks; a bound violation or unsafe object
|
||||
* aborts before any atomic publication. The caller is responsible for the final atomic rename.
|
||||
*/
|
||||
export async function materializeEvidenceTree(options: MaterializeEvidenceTreeOptions): Promise<MaterializedEvidence> {
|
||||
const limits: EvidenceMaterializationLimits = { ...DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS, ...options.limits };
|
||||
if (!/^[0-9a-f]{40}$/.test(options.revision)) throw new Error("evidence revision is invalid");
|
||||
if (!/^[a-z][a-z0-9-]{2,62}$/.test(options.id)) throw new Error("evidence workspace id is invalid");
|
||||
if (!isAbsolute(options.targetDirectory)) throw new Error("evidence target directory must be absolute");
|
||||
|
||||
const objects = await options.repository.evidenceTreeObjects(options.revision, options.id);
|
||||
if (objects.length > limits.maxEntries) throw new Error("evidence entry count exceeds the bound");
|
||||
const tree = await options.repository.evidenceTreeId(options.revision, options.id);
|
||||
|
||||
// Disk-space preflight: sum the real object sizes before writing anything.
|
||||
const sizes = new Map<string, number>();
|
||||
let totalBytes = 0;
|
||||
for (const entry of objects) {
|
||||
if (Buffer.byteLength(entry.posixPath, "utf8") > limits.maxPathBytes) {
|
||||
throw new Error("evidence path exceeds the bound");
|
||||
}
|
||||
const size = await options.repository.gitObjectSize(entry.oid);
|
||||
if (size > limits.maxFileBytes) throw new Error("evidence file exceeds the bound");
|
||||
sizes.set(entry.oid, size);
|
||||
totalBytes += size;
|
||||
if (totalBytes > limits.maxTotalBytes) throw new Error("evidence total bytes exceed the bound");
|
||||
}
|
||||
|
||||
const root = join(options.targetDirectory, "evidence");
|
||||
mkdirSync(root, { recursive: true, mode: 0o700 });
|
||||
const files: Record<string, EvidenceManifestFile> = {};
|
||||
for (const entry of objects) {
|
||||
const contents = await options.repository.evidenceBlobBytes(entry.oid, limits.maxFileBytes);
|
||||
if (contents.length !== sizes.get(entry.oid)) {
|
||||
throw new Error("evidence object changed while materializing");
|
||||
}
|
||||
const target = join(root, ...entry.posixPath.split("/"));
|
||||
writeExclusiveNoFollow(target, contents, entry.mode === "100755" ? 0o755 : 0o644);
|
||||
files[entry.posixPath] = {
|
||||
mode: entry.mode,
|
||||
oid: entry.oid,
|
||||
digest: sha256Prefixed(contents),
|
||||
bytes: contents.length,
|
||||
};
|
||||
}
|
||||
|
||||
const manifest: EvidenceManifest = {
|
||||
schemaVersion: 1,
|
||||
workspace: options.id,
|
||||
commit: options.revision,
|
||||
tree,
|
||||
entryCount: objects.length,
|
||||
totalBytes,
|
||||
files,
|
||||
};
|
||||
const manifestJson = `${JSON.stringify(manifest)}\n`;
|
||||
if (Buffer.byteLength(manifestJson, "utf8") > limits.maxManifestBytes) {
|
||||
throw new Error("evidence manifest exceeds the bound");
|
||||
}
|
||||
const manifestPath = join(options.targetDirectory, "evidence.manifest.json");
|
||||
writeExclusiveNoFollow(manifestPath, Buffer.from(manifestJson, "utf8"), 0o600);
|
||||
return { root, manifestPath, manifest, manifestDigest: sha256Hex(manifestJson) };
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import { isIP } from "node:net";
|
||||
import type { WorkspaceDescriptor } from "../schema.js";
|
||||
|
||||
type EvidenceConfig = WorkspaceDescriptor["evidence"];
|
||||
type SemanticFailureCode = "workspace_not_activatable" | "semantic_index_incompatible";
|
||||
|
||||
export interface EvidenceJobState {
|
||||
runId: string;
|
||||
completedStages: string[];
|
||||
childRuns: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface EvidencePreprocessingDependencies {
|
||||
runStage(argv: string[]): Promise<Record<string, unknown>>;
|
||||
persistJob(): void;
|
||||
semanticPreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
|
||||
requireRunId(value: unknown): string;
|
||||
numberRecord(value: unknown): Record<string, number> | undefined;
|
||||
}
|
||||
|
||||
export interface EvidencePreprocessingRequest {
|
||||
evidence: EvidenceConfig;
|
||||
job: EvidenceJobState;
|
||||
dryRun?: boolean;
|
||||
httpPrivateHostAllowlist?: readonly string[];
|
||||
}
|
||||
|
||||
export interface EvidencePreprocessingOutcome {
|
||||
status: "succeeded" | "unchanged" | "dry_run" | "failed";
|
||||
code: "ok" | "egress_policy_refused" | SemanticFailureCode;
|
||||
runId?: string;
|
||||
childRuns?: Record<string, string>;
|
||||
completedStages?: string[];
|
||||
counts?: Record<string, number>;
|
||||
warnings?: string[];
|
||||
}
|
||||
|
||||
function isPrivateHost(hostname: string): boolean {
|
||||
if (hostname === "localhost" || hostname === "metadata.google.internal") return true;
|
||||
const address = isIP(hostname);
|
||||
if (address === 4) {
|
||||
if (/^127\./.test(hostname) || /^10\./.test(hostname) || /^192\.168\./.test(hostname)) {
|
||||
return true;
|
||||
}
|
||||
if (/^169\.254\./.test(hostname) || /^0\./.test(hostname)) return true;
|
||||
const match = /^172\.(\d+)\./.exec(hostname);
|
||||
return Boolean(match && Number(match[1]) >= 16 && Number(match[1]) <= 31);
|
||||
}
|
||||
if (address === 6) {
|
||||
const normalized = hostname.toLowerCase();
|
||||
return normalized === "::1"
|
||||
|| normalized.startsWith("fe80:")
|
||||
|| normalized.startsWith("fd")
|
||||
|| normalized.startsWith("fc");
|
||||
}
|
||||
return hostname.endsWith(".internal");
|
||||
}
|
||||
|
||||
function evidencePolicy(
|
||||
evidence: EvidenceConfig,
|
||||
httpPrivateHostAllowlist?: readonly string[],
|
||||
): EvidencePreprocessingOutcome | undefined {
|
||||
if (!evidence || evidence.source.type === "filesystem") return undefined;
|
||||
if (evidence.source.type === "http") {
|
||||
for (const value of evidence.source.uris) {
|
||||
const host = new URL(value).hostname;
|
||||
if (
|
||||
isPrivateHost(host)
|
||||
&& !(evidence.source.allow_private_hosts && httpPrivateHostAllowlist?.includes(host))
|
||||
) {
|
||||
return { status: "failed", code: "egress_policy_refused" };
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
evidence.source.endpoint_url !== undefined
|
||||
|| evidence.source.credentials === "ambient"
|
||||
|| evidence.source.allow_private_endpoint
|
||||
|| evidence.source.allow_insecure_endpoint
|
||||
) {
|
||||
return { status: "failed", code: "egress_policy_refused" };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function jobResult(job: EvidenceJobState): Pick<
|
||||
EvidencePreprocessingOutcome,
|
||||
"runId" | "childRuns" | "completedStages"
|
||||
> {
|
||||
return {
|
||||
runId: job.runId,
|
||||
childRuns: { ...job.childRuns },
|
||||
completedStages: [...job.completedStages],
|
||||
};
|
||||
}
|
||||
|
||||
async function runEvidenceStage(
|
||||
request: EvidencePreprocessingRequest,
|
||||
deps: EvidencePreprocessingDependencies,
|
||||
): Promise<EvidencePreprocessingOutcome> {
|
||||
const payload = await deps.runStage([
|
||||
"preprocess",
|
||||
"evidence",
|
||||
...(request.dryRun ? ["--dry-run"] : []),
|
||||
...(request.job.childRuns.evidence
|
||||
? ["--resume", request.job.childRuns.evidence]
|
||||
: []),
|
||||
"--json",
|
||||
"-c",
|
||||
"/dev/fd/3",
|
||||
]);
|
||||
if (typeof payload.run_id === "string") {
|
||||
request.job.childRuns.evidence = deps.requireRunId(payload.run_id);
|
||||
}
|
||||
if (!request.dryRun && !request.job.completedStages.includes("evidence")) {
|
||||
request.job.completedStages.push("evidence");
|
||||
}
|
||||
deps.persistJob();
|
||||
return {
|
||||
status: request.dryRun ? "dry_run" : "succeeded",
|
||||
code: "ok",
|
||||
...jobResult(request.job),
|
||||
counts: deps.numberRecord(payload.counts),
|
||||
};
|
||||
}
|
||||
|
||||
export async function preprocessEvidence(
|
||||
request: EvidencePreprocessingRequest,
|
||||
deps: EvidencePreprocessingDependencies,
|
||||
): Promise<EvidencePreprocessingOutcome> {
|
||||
if (!request.evidence) {
|
||||
return {
|
||||
status: "unchanged",
|
||||
code: "ok",
|
||||
warnings: ["workspace has no Evidence source"],
|
||||
};
|
||||
}
|
||||
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
|
||||
if (policy) return policy;
|
||||
const semantic = await deps.semanticPreflight();
|
||||
if (!semantic.ok) {
|
||||
return { status: "failed", code: semantic.code, runId: request.job.runId };
|
||||
}
|
||||
if (request.job.completedStages.includes("evidence") && !request.dryRun) {
|
||||
return {
|
||||
status: "unchanged",
|
||||
code: "ok",
|
||||
runId: request.job.runId,
|
||||
completedStages: [...request.job.completedStages],
|
||||
};
|
||||
}
|
||||
return await runEvidenceStage(request, deps);
|
||||
}
|
||||
|
||||
export async function continueEvidencePreprocessing(
|
||||
request: Omit<EvidencePreprocessingRequest, "dryRun"> & {
|
||||
priorCounts?: Record<string, number>;
|
||||
},
|
||||
deps: EvidencePreprocessingDependencies,
|
||||
): Promise<EvidencePreprocessingOutcome> {
|
||||
if (!request.evidence) {
|
||||
return {
|
||||
status: "succeeded",
|
||||
code: "ok",
|
||||
...jobResult(request.job),
|
||||
warnings: ["workspace has no Evidence source"],
|
||||
...(request.priorCounts ? { counts: request.priorCounts } : {}),
|
||||
};
|
||||
}
|
||||
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
|
||||
if (policy) return { ...policy, ...jobResult(request.job) };
|
||||
if (!request.job.completedStages.includes("evidence")) {
|
||||
return await runEvidenceStage(request, deps);
|
||||
}
|
||||
return { status: "unchanged", code: "ok", ...jobResult(request.job) };
|
||||
}
|
||||
Reference in New Issue
Block a user