fix: harden P1 integration evidence

2026-08-09 21:14:45 +02:00
parent 74ddb86d13
commit 3583659600
5 changed files with 460 additions and 119 deletions
@@ -6,14 +6,20 @@ import {
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { promisify } from "node:util";
+import { fileURLToPath } from "node:url";
 import test from "node:test";

 import {
  canonicalIntegrationBase,
+  CHECK_IDS,
+  buildSafeEnvironment,
+  installExternalFetchGuard,
+  negativeRequestEvidence,
  cleanupOwnedRun,
  createOwnedRun,
  deriveOverall,
  executeChecks,
+  exportArchiveEvidencePath,
  readAndValidateOwnership,
  runCommand,
  runIntegration,
@@ -104,11 +110,14 @@ test("cleanup atomically removes one owned root and preserves siblings", async (
  assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign");
 });

-function validReport(checks = [{
-  id: "preflight", status: "PASS", startedAt: "2026-08-09T00:00:00.000Z",
-  finishedAt: "2026-08-09T00:00:01.000Z", commands: ["git"],
-  artifacts: [{ path: "logs/preflight.json", sha256: "a".repeat(64) }],
-}]) {
+function resultFor(id) {
+  return {
+    id, status: "PASS", startedAt: "2026-08-09T00:00:00.000Z",
+    finishedAt: "2026-08-09T00:00:01.000Z", commands: ["git"],
+    artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }],
+  };
+}
+function validReport(checks = CHECK_IDS.map(resultFor)) {
  return {
    schemaVersion: 1, runId: `p1-${"d".repeat(32)}`, startedAt: "2026-08-09T00:00:00.000Z",
    finishedAt: "2026-08-09T00:00:02.000Z", command: "p1-acceptance integration --keep",
@@ -133,30 +142,53 @@ test("report validation enforces uniqueness, derivation, safe evidence, hashes,
  }
 });

-test("injected failure executes once, retains diagnostics, and returns nonzero", async () => {
+function exactScenarios(run = async () => ({ commands: [], artifacts: [] })) {
+  return CHECK_IDS.map((id) => ({ id, run: () => run(id) }));
+}
+
+test("injected failure executes once, retains a complete ordered diagnostic report, and returns nonzero", async () => {
  const repositoryRoot = await fakeRepository();
-  let calls = 0;
+  const calls = [];
+  const failAt = CHECK_IDS[3];
  const result = await runIntegration({
-    repositoryRoot, keep: false, failAt: "sample",
-    checks: [{ id: "sample", run: async () => { calls += 1; return { commands: [], artifacts: [] }; } }],
+    repositoryRoot, keep: false, failAt,
+    checks: exactScenarios(async (id) => { calls.push(id); return { commands: [], artifacts: [] }; }),
  });
  assert.equal(result.exitCode, 1);
-  assert.equal(calls, 1);
+  assert.deepEqual(calls, CHECK_IDS.slice(0, 4));
  assert.equal((await lstat(result.runRoot)).isDirectory(), true);
  const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
-  assert.equal(report.checks.filter((check) => check.status === "FAIL").length, 1);
-  assert.equal(report.checks[0].id, "sample");
+  assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS);
+  assert.equal(report.checks.filter((check) => check.status === "FAIL").length, CHECK_IDS.length - 3);
+  assert.equal(report.checks[3].error, "Acceptance scenario failed safely.");
+  assert.equal(report.checks[4].error, "Not executed after earlier failure.");
 });

-test("executeChecks never repeats a scenario", async () => {
+test("executeChecks never repeats or executes after first failure but emits the exact check set", async () => {
  const calls = new Map();
  const result = await executeChecks({
-    checks: ["one", "two"].map((id) => ({ id, run: async () => { calls.set(id, (calls.get(id) ?? 0) + 1); return {}; } })),
-    failAt: "two",
+    checks: exactScenarios(async (id) => { calls.set(id, (calls.get(id) ?? 0) + 1); return {}; }),
+    failAt: CHECK_IDS[1],
  });
-  assert.equal(result.length, 2);
-  assert.deepEqual(Object.fromEntries(calls), { one: 1, two: 1 });
+  assert.deepEqual(result.map(({ id }) => id), CHECK_IDS);
+  assert.deepEqual(Object.fromEntries(calls), Object.fromEntries(CHECK_IDS.slice(0, 2).map((id) => [id, 1])));
  assert.equal(result[1].status, "FAIL");
+  assert(result.slice(2).every(({ status, error }) => status === "FAIL" && error === "Not executed after earlier failure."));
+  assert.throws(() => validateReport(validReport(CHECK_IDS.slice(0, -1).map(resultFor))));
+  await assert.rejects(executeChecks({ checks: exactScenarios().reverse() }));
+});
+
+test("owned setup failure still writes one safe result for every exact check", async () => {
+  const repositoryRoot = await fakeRepository();
+  const result = await runIntegration({
+    repositoryRoot, keep: false,
+    setup: async () => { throw new Error("fixture setup raw failure"); },
+  });
+  assert.equal(result.exitCode, 1);
+  const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8"));
+  assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS);
+  assert.equal(report.checks[0].error, "Acceptance setup failed safely.");
+  assert(report.checks.slice(1).every(({ error }) => error === "Not executed after earlier failure."));
 });

 test("scalar fixture secret files contain no harness-invalid whitespace", () => {
@@ -204,11 +236,11 @@ test("secret scanner examines reachable Git blobs, not just loose file bytes", a

 test("successful lifecycle honors keep and cleanup", async () => {
  const repositoryRoot = await fakeRepository();
-  const check = [{ id: "sample", run: async () => ({ commands: [], artifacts: [] }) }];
-  const kept = await runIntegration({ repositoryRoot, keep: true, checks: check });
+  const checks = exactScenarios();
+  const kept = await runIntegration({ repositoryRoot, keep: true, checks });
  assert.equal(kept.exitCode, 0);
  assert.equal((await lstat(kept.runRoot)).isDirectory(), true);
-  const cleaned = await runIntegration({ repositoryRoot, keep: false, checks: check });
+  const cleaned = await runIntegration({ repositoryRoot, keep: false, checks });
  assert.equal(cleaned.exitCode, 0);
  await assert.rejects(lstat(cleaned.runRoot));
 });
@@ -226,3 +258,65 @@ test("command helper accepts only executable plus separate argv", async () => {
  assert.equal(result.stdout, "literal;not-a-shell");
  assert.equal(result.code, 0);
 });
+
+
+test("safe environment rejects ambient THT and keeps only strict process allowlist plus fixture values", () => {
+  const safe = buildSafeEnvironment({
+    ambient: { PATH: "/safe/bin", HOME: "/home/test", LANG: "C", THT_SECRETS_FILE: "/real/secrets", AWS_SECRET_ACCESS_KEY: "real" },
+    fixture: { THT_BIN: "/fixture/tht", THT_WORKSPACE_SECRET_ROOTS: "/fixture/secrets" },
+  });
+  assert.deepEqual(safe, {
+    PATH: "/safe/bin", HOME: "/home/test", LANG: "C",
+    THT_BIN: "/fixture/tht", THT_WORKSPACE_SECRET_ROOTS: "/fixture/secrets",
+  });
+});
+
+test("secret scan fails closed when Git enumeration fails", async () => {
+  const repositoryRoot = await fakeRepository();
+  const run = await createOwnedRun({ repositoryRoot });
+  await mkdir(join(run.root, "remote.git"));
+  await assert.rejects(scanSecrets({ runRoot: run.root, forbiddenValues: ["CANARY-value-123"] }), /Git secret scan failed closed/);
+});
+
+
+test("negative request evidence persists only case label and expected input field", () => {
+  const value = negativeRequestEvidence("credential-field", "evidence.source.password");
+  assert.deepEqual(value, { case: "credential-field", expectedInputField: "evidence.source.password" });
+  assert.equal(JSON.stringify(value).includes("body"), false);
+});
+
+test("external fetch guard permits only the owned loopback API and records external attempts", async () => {
+  const called = [];
+  const guard = installExternalFetchGuard("http://127.0.0.1:12345", async (url) => { called.push(String(url)); return { ok: true }; });
+  await guard.fetch("http://127.0.0.1:12345/workspaces");
+  await assert.rejects(guard.fetch("https://evidence.example.test/guide.md"), /external fetch prohibited/);
+  await assert.rejects(guard.fetch("http://127.0.0.1:9999/health"), /external fetch prohibited/);
+  assert.deepEqual(called, ["http://127.0.0.1:12345/workspaces"]);
+  assert.equal(guard.externalAttempts.length, 2);
+});
+
+
+test("export archive evidence path matches the persisted binary request id", () => {
+  assert.equal(exportArchiveEvidencePath("export-p1-filesystem"), "exports/raw/export-p1-filesystem.zip");
+});
+
+
+test("announce callback observes PASS and manual pending before non-keep cleanup", async () => {
+  const repositoryRoot = await fakeRepository();
+  let observed;
+  const result = await runIntegration({
+    repositoryRoot, keep: false, checks: exactScenarios(),
+    announce: async ({ report, runRoot }) => {
+      observed = { overall: report.overall, manual: "PENDING", rootExists: (await lstat(runRoot)).isDirectory() };
+    },
+  });
+  assert.deepEqual(observed, { overall: "PASS", manual: "PENDING", rootExists: true });
+  assert.equal(result.retained, false);
+});
+
+test("public wrapper replaces ambient environment before invoking the runner", async () => {
+  const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh"), "utf8");
+  assert.match(wrapper, /safe_env=\(env -i/);
+  assert.match(wrapper, /P1_ACCEPTANCE_FAIL_AT/);
+  assert.doesNotMatch(wrapper, /export THT_BIN/);
+});