From a3129b8b81450d982b2dc5183cb720cc1ac248f1 Mon Sep 17 00:00:00 2001 From: mptyl Date: Sun, 9 Aug 2026 22:43:44 +0200 Subject: [PATCH] fix: anchor P1 manual extraction and scans --- backend/scripts/p1-manual-acceptance.mjs | 280 ++++++++++++++++-- backend/scripts/p1-manual-acceptance.test.mjs | 50 +++- docs/testing/p1-manual-acceptance.md | 19 +- 3 files changed, 310 insertions(+), 39 deletions(-) diff --git a/backend/scripts/p1-manual-acceptance.mjs b/backend/scripts/p1-manual-acceptance.mjs index 4e1c5078..a33cf2d0 100755 --- a/backend/scripts/p1-manual-acceptance.mjs +++ b/backend/scripts/p1-manual-acceptance.mjs @@ -44,7 +44,7 @@ async function run(executable,argv,options={}){return await exec(executable,argv function descriptor(id,source){return{workspace:{schema_version:3,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},semantic_index:{vector_store:{engine:"qdrant",collection:id,dimensions:1024,distance:"cosine"},embedding:{provider:"ollama_internal",model:"qwen3-embedding:0.6b",dimensions:1024}},llm_policy:{allowed:["zai/glm-5.2"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};} function descriptors(){return[descriptor("p1-filesystem",{type:"filesystem",uri:"workspace-content/p1-filesystem/evidence",patterns:["**/*.md"],max_bytes:10485760}),descriptor("p1-http",{type:"http",uris:["https://evidence.example.test/guide.md"],authentication:"signed_urls_file",connect_timeout_ms:1250,read_timeout_ms:30001,max_bytes:12345,max_redirects:2,allow_private_hosts:false,max_cache_bytes:67890}),descriptor("p1-s3",{type:"s3",uri:"s3://p1-evidence/published/",endpoint_url:"https://s3.example.test/",region:"eu-west-1",credentials:"static_files",trusted_endpoint:true,allow_private_endpoint:false,allow_insecure_endpoint:false,max_bytes:12345,max_objects:33,max_pages:4,page_size:5})];} function quote(value){return `'${String(value).replaceAll("'",`'"'"'`)}'`;} -async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}} +async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof","python3"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}} async function initializeGit(root){await run("git",["init","--bare","--initial-branch=main",join(root,"remote.git")],{cwd:root});await run("git",["clone",join(root,"remote.git"),join(root,"author")],{cwd:root});for(const [key,value]of [["user.name","P1 Manual Curator"],["user.email","p1-manual@example.invalid"]])await run("git",["config",key,value],{cwd:join(root,"author")});const evidence=join(root,"author","workspace-content","p1-filesystem","evidence");await mkdir(join(evidence,"domain"),{recursive:true});await writeFile(join(evidence,"guide.md"),"# P1 manually curated Evidence\n");await writeFile(join(evidence,"domain","table.md"),"# P1 curated table\n");await run("git",["add","workspace-content"],{cwd:join(root,"author")});await run("git",["commit","-m","Bootstrap P1 manual Evidence"],{cwd:join(root,"author")});await run("git",["push","origin","main"],{cwd:join(root,"author")});} function requestFixtures(items){const result={"status.json":{method:"GET",path:"/workspace-registry/status"},"pull.json":{method:"POST",path:"/workspace-registry/pull"}};for(const workspace of items){const id=workspace.workspace.id;result[`validate-${id}.json`]={workspace};result[`publish-${id}.json`]={action:"create",workspace};result[`read-${id}.json`]={method:"GET",path:`/workspaces/${id}`};result[`export-${id}.json`]={method:"GET",path:`/workspaces/${id}/export`};}Object.assign(result,{"invalid-absolute.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"/etc"}}}},"invalid-traversal.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-filesystem/evidence/../../p1-s3/evidence"}}}},"invalid-cross-workspace.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-s3/evidence"}}}},"invalid-protocol.json":{workspace:{...items[1],evidence:{...items[1].evidence,source:{...items[1].evidence.source,uris:["file:///etc/passwd"]}}}},"invalid-credential.json":{workspace:{...items[2],evidence:{...items[2].evidence,source:{...items[2].evidence.source,access_key:"CANARY-MUST-BE-REJECTED"}}}}});return result;} function curlGet(url,output){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;} @@ -94,7 +94,7 @@ Status: **PENDING**. The reviewer, not this helper, performs and judges every st 8. Inspect runtime identity, absolute reserved filesystem root, Evidence limits, and policy in the rendered YAML; do not inspect secret contents. 9. Personally execute \`${repo}/harness/.venv/bin/tht config check -c ${root}/rendered/runtime-1.yaml\` and the same command for \`runtime-2.yaml\` (or run \`commands/config-check.sh\`). 10. Personally run \`commands/http-15-*.sh\` through \`commands/http-19-*.sh\` to submit the invalid absolute, Evidence-URI traversal, cross-workspace, protocol, and credential validation requests; verify safe rejection, no Git/snapshot mutation, and no rejected canary outside the request fixture. -11. Run \`commands/secret-scan.sh\`; it excludes \`fixture-secrets\` and checks bounded bytes from every Git object, including unreachable blobs and dangling commits, for canary patterns without displaying secret contents. +11. Run \`commands/secret-scan.sh\`; it excludes only the direct \`fixture-secrets\` payload directory, scans bounded filesystem bytes including Git metadata and arbitrary \`.git\` directories, and checks raw bounded bytes from every blob, commit, tree, and tag object, including unreachable objects, without displaying secret contents. 12. Run \`commands/absence-check.sh\`; confirm no preprocessing, Evidence materialization, embedding, Qdrant, ACTIVE, or retention artifact exists. 13. Run \`${repo}/scripts/p1-manual-acceptance.sh stop\`; confirm \`backend.pid\` and the listener on port ${PORT} are gone. 14. Create \`${root}/VERDICT.md\` yourself with reviewer, UTC time, every checklist result, observations, and exactly either \`manual acceptance: PASS\` or \`manual acceptance: FAIL\`. @@ -104,40 +104,258 @@ Preserve a failed lab by stopping it and leaving the owned root in place. Only \ function extractCommand(repo,root){return `#!/usr/bin/env bash set -euo pipefail zip=\${1:?zip required}; out=\${2:?new output required}; expected=\${3:?expected workspace id required} -node --input-type=module - "$zip" "$out" "$expected" ${quote(root)} ${quote(join(repo,"backend/package.json"))} <<'NODE' -import { execFileSync } from "node:child_process"; -import { createHash, randomBytes } from "node:crypto"; -import { createRequire } from "node:module"; -import { lstat, mkdir, open, readFile, realpath, rename, rm } from "node:fs/promises"; -import { basename, dirname, isAbsolute, join, relative } from "node:path"; -const [zip,out,expected,root,packageJson]=process.argv.slice(2),allowed=new Set(["p1-filesystem","p1-http","p1-s3"]),base=join(root,"exports/extracted");let stagedZip,stagedDirectory; -const fail=message=>{throw new Error(message);},exact=(value,keys)=>value&&typeof value==="object"&&!Array.isArray(value)&&JSON.stringify(Object.keys(value).sort())===JSON.stringify([...keys].sort()); -try{ - if(!allowed.has(expected))fail("expected workspace identity is invalid"); - const canonicalRoot=await realpath(root),baseEntry=await lstat(base);if(baseEntry.isSymbolicLink()||!baseEntry.isDirectory()||await realpath(base)!==base||!relative(canonicalRoot,base)||relative(canonicalRoot,base).startsWith("..")||isAbsolute(relative(canonicalRoot,base)))fail("unsafe owned extraction root"); - if(dirname(out)!==base||basename(out).startsWith(".")||basename(out).length===0)fail("unsafe output path");try{await lstat(out);fail("unsafe output path");}catch(error){if(error.code!=="ENOENT")throw error;} - const source=await open(zip,"r");let sourceBytes;try{const stat=await source.stat();if(!stat.isFile()||stat.size<1||stat.size>33554432)fail("source ZIP is unbounded");sourceBytes=await source.readFile();if(sourceBytes.length!==stat.size)fail("source ZIP changed during staging");}finally{await source.close();} - stagedZip=join(root,"exports",".zip-stage-"+randomBytes(16).toString("hex")+".zip");const staged=await open(stagedZip,"wx",0o600);try{await staged.chmod(0o600);await staged.writeFile(sourceBytes);await staged.sync();}finally{await staged.close();} - const names=execFileSync("unzip",["-Z1",stagedZip],{encoding:"utf8",maxBuffer:1048576}).trim().split("\\n"),required=["manifest.json","workspace.yaml","contract.env.example","README.md"]; - if(names.length!==4||new Set(names).size!==4||required.some(name=>!names.includes(name)))fail("unsafe-zip entries"); - const listing=execFileSync("zipinfo",["-l",stagedZip],{encoding:"utf8",maxBuffer:1048576}),regular=listing.split("\\n").filter(line=>line.startsWith("-")).length;if(regular!==4)fail("ZIP contains a symlink or nonregular entry"); - stagedDirectory=join(base,".extract-stage-"+randomBytes(16).toString("hex"));await mkdir(stagedDirectory,{mode:0o700});execFileSync("unzip",["-q",stagedZip,"-d",stagedDirectory],{stdio:"pipe",maxBuffer:1048576}); - const bytes={};for(const name of required){const path=join(stagedDirectory,name),entry=await lstat(path);if(!entry.isFile()||entry.isSymbolicLink()||entry.size<1||entry.size>10485760)fail("extracted file is unsafe");bytes[name]=await readFile(path);} - const randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");for(const name of required){const value=bytes[name].toString("latin1");if(value.includes("P1 manually curated Evidence")||value.includes("P1 curated table")||randomized.test(value)||value.includes(fixed))fail("export contains Evidence or secret canary bytes");} - let manifest;try{manifest=JSON.parse(bytes["manifest.json"].toString("utf8"));}catch{fail("export manifest schema mismatch");}const hashed=required.slice(1);if(!exact(manifest,["schema_version","workspace_id","files"])||manifest.schema_version!==1||manifest.workspace_id!==expected||!exact(manifest.files,hashed)||hashed.some(name=>!/^[0-9a-f]{64}$/.test(manifest.files[name])))fail("export manifest workspace identity or schema mismatch");for(const name of hashed)if(createHash("sha256").update(bytes[name]).digest("hex")!==manifest.files[name])fail("manifest hash mismatch"); - const require=createRequire(packageJson),YAML=require("yaml");let descriptor;try{descriptor=YAML.parse(bytes["workspace.yaml"].toString("utf8"));}catch{fail("export descriptor is malformed");}if(!descriptor||descriptor.workspace?.id!==expected)fail("export descriptor workspace identity mismatch"); - await rename(stagedDirectory,out);stagedDirectory=undefined; -}catch(error){console.error(error.message);process.exitCode=1;}finally{if(stagedDirectory)await rm(stagedDirectory,{recursive:true,force:true}).catch(()=>{});if(stagedZip)await rm(stagedZip,{force:true}).catch(()=>{});} -NODE +python3 - "$zip" "$out" "$expected" ${quote(root)} ${quote(join(repo,"backend/package.json"))} <<'PY' +import hashlib +import io +import json +import os +import re +import secrets +import stat +import subprocess +import sys +import zipfile + +zip_path, output_path, expected, root, package_json = sys.argv[1:] +allowed = {"p1-filesystem", "p1-http", "p1-s3"} +required = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"] +base = os.path.join(root, "exports", "extracted") +base_fd = None +archive_fd = None +stage_fd = None +archive_stage = None +extract_stage = None +published = False + + +def fail(message): + raise RuntimeError(message) + + +def exact(value, keys): + return isinstance(value, dict) and set(value) == set(keys) + + +def write_all(fd, data): + view = memoryview(data) + while view: + written = os.write(fd, view) + if written <= 0: + fail("anchored extraction write failed") + view = view[written:] + + +def read_exact_fd(fd, expected_size, limit, label): + if expected_size < 1 or expected_size > limit: + fail(label + " is unbounded") + chunks = [] + remaining = expected_size + while remaining: + chunk = os.read(fd, min(1024 * 1024, remaining)) + if not chunk: + fail(label + " changed while staging") + chunks.append(chunk) + remaining -= len(chunk) + if os.read(fd, 1): + fail(label + " changed while staging") + return b"".join(chunks) + + +def require_base_identity(): + try: + current = os.stat(base, follow_symlinks=False) + except OSError: + fail("owned extraction root identity changed") + if (not stat.S_ISDIR(current.st_mode) or current.st_dev != base_identity.st_dev + or current.st_ino != base_identity.st_ino or os.path.realpath(base) != base): + fail("owned extraction root identity changed") + + +def remove_anchored_directory(name): + child_fd = None + try: + child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd) + for entry in os.listdir(child_fd): + if entry not in required: + fail("anchored extraction cleanup found an unexpected entry") + os.unlink(entry, dir_fd=child_fd) + os.fsync(child_fd) + except FileNotFoundError: + return + finally: + if child_fd is not None: + os.close(child_fd) + os.rmdir(name, dir_fd=base_fd) + os.fsync(base_fd) + + +try: + for flag in ("O_DIRECTORY", "O_NOFOLLOW"): + if not hasattr(os, flag): + fail("anchored extraction is unavailable on this platform") + if expected not in allowed: + fail("expected workspace identity is invalid") + if os.path.realpath(root) != root or os.path.dirname(output_path) != base: + fail("unsafe owned extraction root or output path") + output_name = os.path.basename(output_path) + if not output_name or output_name.startswith(".") or os.sep in output_name: + fail("unsafe output path") + + base_fd = os.open(base, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + base_identity = os.fstat(base_fd) + if not stat.S_ISDIR(base_identity.st_mode): + fail("unsafe owned extraction root") + require_base_identity() + try: + os.stat(output_name, dir_fd=base_fd, follow_symlinks=False) + fail("unsafe output path") + except FileNotFoundError: + pass + + # Open the caller's source exactly once, then consume only an owned staged copy. + source_fd = os.open(zip_path, os.O_RDONLY | os.O_NOFOLLOW) + try: + source_identity = os.fstat(source_fd) + if not stat.S_ISREG(source_identity.st_mode): + fail("source ZIP is unsafe") + source_bytes = read_exact_fd(source_fd, source_identity.st_size, 33554432, "source ZIP") + source_after = os.fstat(source_fd) + if (source_after.st_dev, source_after.st_ino, source_after.st_size) != (source_identity.st_dev, source_identity.st_ino, source_identity.st_size): + fail("source ZIP changed during staging") + finally: + os.close(source_fd) + archive_sha = hashlib.sha256(source_bytes).digest() + + archive_stage = ".zip-stage-" + secrets.token_hex(16) + ".zip" + archive_fd = os.open(archive_stage, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=base_fd) + write_all(archive_fd, source_bytes) + os.fsync(archive_fd) + del source_bytes + os.lseek(archive_fd, 0, os.SEEK_SET) + staged_bytes = read_exact_fd(archive_fd, os.fstat(archive_fd).st_size, 33554432, "staged archive") + if hashlib.sha256(staged_bytes).digest() != archive_sha: + fail("staged archive SHA mismatch") + + with zipfile.ZipFile(io.BytesIO(staged_bytes), "r") as archive: + infos = archive.infolist() + names = [entry.filename for entry in infos] + if len(names) != 4 or len(set(names)) != 4 or set(names) != set(required): + fail("unsafe-zip entries") + for entry in infos: + mode = (entry.external_attr >> 16) & 0xFFFF + if not stat.S_ISREG(mode) or entry.flag_bits & 1: + fail("ZIP contains a symlink or nonregular entry") + if entry.file_size < 1 or entry.file_size > 10485760: + fail("extracted file is unsafe") + payloads = {name: archive.read(name) for name in required} + if any(len(payloads[entry.filename]) != entry.file_size for entry in infos): + fail("extracted file size mismatch") + + # Exercise the documented unzip prerequisite against the exact staged descriptor, not a path. + listing = subprocess.run( + ["unzip", "-Z1", "/dev/fd/" + str(archive_fd)], pass_fds=(archive_fd,), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20, check=False, + ) + if listing.returncode != 0 or listing.stdout.decode("utf8", "strict").splitlines() != names: + fail("unsafe-zip entries") + os.lseek(archive_fd, 0, os.SEEK_SET) + revalidated = read_exact_fd(archive_fd, len(staged_bytes), 33554432, "staged archive") + if hashlib.sha256(revalidated).digest() != archive_sha or revalidated != staged_bytes: + fail("staged archive SHA mismatch") + require_base_identity() + + randomized = re.compile(br"(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}") + fixed = b"-".join([b"CANARY", b"MUST", b"BE", b"REJECTED"]) + for data in payloads.values(): + if b"P1 manually curated Evidence" in data or b"P1 curated table" in data or randomized.search(data) or fixed in data: + fail("export contains Evidence or secret canary bytes") + + try: + manifest = json.loads(payloads["manifest.json"].decode("utf8")) + except Exception: + fail("export manifest schema mismatch") + hashed = required[1:] + files = manifest.get("files") if isinstance(manifest, dict) else None + if (not exact(manifest, ["schema_version", "workspace_id", "files"]) + or manifest.get("schema_version") != 1 or manifest.get("workspace_id") != expected + or not exact(files, hashed) + or any(not isinstance(files[name], str) or not re.fullmatch(r"[0-9a-f]{64}", files[name]) for name in hashed)): + fail("export manifest workspace identity or schema mismatch") + for name in hashed: + if hashlib.sha256(payloads[name]).hexdigest() != files[name]: + fail("manifest hash mismatch") + + yaml_helper = 'const fs=require("node:fs"),{createRequire}=require("node:module");try{const YAML=createRequire(process.argv[1])("yaml"),v=YAML.parse(fs.readFileSync(0,"utf8"));process.stdout.write(JSON.stringify(v?.workspace?.id??null));}catch{process.exit(2)}' + parsed = subprocess.run(["node", "-e", yaml_helper, package_json], input=payloads["workspace.yaml"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10, check=False) + try: + descriptor_id = json.loads(parsed.stdout.decode("utf8")) if parsed.returncode == 0 else None + except Exception: + descriptor_id = None + if descriptor_id != expected: + fail("export descriptor workspace identity mismatch") + + extract_stage = ".extract-stage-" + secrets.token_hex(16) + os.mkdir(extract_stage, 0o700, dir_fd=base_fd) + stage_fd = os.open(extract_stage, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd) + for name in required: + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=stage_fd) + try: + write_all(fd, payloads[name]) + os.fsync(fd) + finally: + os.close(fd) + os.fsync(stage_fd) + os.close(stage_fd) + stage_fd = None + require_base_identity() + try: + os.stat(output_name, dir_fd=base_fd, follow_symlinks=False) + fail("unsafe output path") + except FileNotFoundError: + pass + os.rename(extract_stage, output_name, src_dir_fd=base_fd, dst_dir_fd=base_fd) + extract_stage = None + published = True + os.fsync(base_fd) + require_base_identity() +except Exception as error: + print(str(error), file=sys.stderr) + sys.exit_code = 1 +finally: + if stage_fd is not None: + os.close(stage_fd) + if extract_stage is not None and base_fd is not None: + try: + remove_anchored_directory(extract_stage) + except Exception: + sys.exit_code = 1 + if published and getattr(sys, "exit_code", 0) and base_fd is not None: + try: + remove_anchored_directory(output_name) + except Exception: + pass + if archive_fd is not None: + os.close(archive_fd) + if archive_stage is not None and base_fd is not None: + try: + os.unlink(archive_stage, dir_fd=base_fd) + os.fsync(base_fd) + except FileNotFoundError: + pass + if base_fd is not None: + os.close(base_fd) +if getattr(sys, "exit_code", 0): + raise SystemExit(sys.exit_code) +PY `;} async function writeCommands(repo,root){const commands=join(root,"commands");for(const [name,body]of [...httpCommands(root),["render-1.sh",renderCommand(repo,root,1)],["render-2.sh",renderCommand(repo,root,2)],["diff-rendered.sh",`#!/bin/sh\nset -eu\ndiff -u ${quote(join(root,"rendered/runtime-1.yaml"))} ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["config-check.sh",`#!/bin/sh\nset -eu\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-1.yaml"))}\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["git-inspect.sh",`#!/bin/sh\nset -eu\ncommit=\${1:?published commit required}\ncase "$commit" in *[!0-9a-f]*|'') exit 2;; esac\n[ \${#commit} -eq 40 ] || exit 2\ngit -C ${quote(join(root,"installation/registry/repo"))} log --oneline --decorate -10 "$commit"\ngit -C ${quote(join(root,"installation/registry/repo"))} ls-tree -r "$commit" -- workspaces workspace-content\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspaces/p1-filesystem.yaml"\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspace-content/p1-filesystem/evidence/guide.md"\n`],["extract-export.sh",extractCommand(repo,root)],["secret-scan.sh",`#!/usr/bin/env bash set -euo pipefail root=${quote(root)} node --input-type=module - "$root" <<'NODE' -import { execFileSync } from "node:child_process";import { lstat, readFile, readdir } from "node:fs/promises";import { basename, join, relative } from "node:path"; -const root=process.argv[2],randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");let found=false;const containsCanary=text=>randomized.test(text)||text.includes(fixed); -async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){const child=join(path,entry.name),rel=relative(root,child);if(entry.isSymbolicLink()){console.error("unsafe symlink during secret scan: "+rel);found=true;continue;}if(entry.isDirectory()){if(rel==="fixture-secrets"||entry.name===".git")continue;await walk(child);}else if(entry.isFile()){const stat=await lstat(child);if(stat.size>33554432)throw Error("secret scan file too large: "+rel);if(containsCanary((await readFile(child)).toString("latin1"))&&rel!=="requests/invalid-credential.json"){console.error("secret canary found: "+rel);found=true;}}}} -function git(args,label){const listing=execFileSync("git",[...args,"cat-file","--batch-all-objects","--unordered","--batch-check=%(objectname) %(objecttype) %(objectsize)"],{encoding:"utf8",maxBuffer:16*1024*1024}).trim(),objects=listing?listing.split("\\n"):[];if(objects.length>100000)throw Error("too many Git objects to scan in "+label);let total=0;for(const line of objects){const [oid,type,sizeText]=line.split(" ");if(!/^[0-9a-f]{40,64}$/.test(oid??"")||!type||!/^\\d+$/.test(sizeText??""))throw Error("malformed Git object listing in "+label);if(type!=="blob")continue;const size=Number(sizeText);total+=size;if(!Number.isSafeInteger(size)||size>33554432||total>536870912)throw Error("Git blob scan bound exceeded in "+label);const blob=execFileSync("git",[...args,"cat-file","blob",oid],{maxBuffer:size+1});if(blob.length!==size)throw Error("Git blob size changed in "+label);if(containsCanary(blob.toString("latin1"))){console.error("secret canary found in Git blob: "+label+":"+oid);found=true;}}} +import { execFileSync } from "node:child_process";import { constants } from "node:fs";import { lstat, open, readdir } from "node:fs/promises";import { join, relative } from "node:path"; +const root=process.argv[2],randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");let found=false,filesystemCount=0,filesystemTotal=0;const containsCanary=text=>randomized.test(text)||text.includes(fixed); +async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){filesystemCount++;if(filesystemCount>200000)throw Error("filesystem secret scan entry bound exceeded");const child=join(path,entry.name),rel=relative(root,child);if(entry.isSymbolicLink()){console.error("unsafe symlink during secret scan (path redacted)");found=true;continue;}if(entry.isDirectory()){if(rel==="fixture-secrets")continue;await walk(child);continue;}if(!entry.isFile())throw Error("unsupported filesystem entry during secret scan");let handle;try{handle=await open(child,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat();if(!before.isFile()||before.size>33554432)throw Error("filesystem secret scan file bound exceeded");filesystemTotal+=before.size;if(filesystemTotal>1073741824)throw Error("filesystem secret scan total bound exceeded");const bytes=await handle.readFile(),after=await handle.stat();if(bytes.length!==before.size||after.dev!==before.dev||after.ino!==before.ino||after.size!==before.size)throw Error("filesystem changed during secret scan");const text=bytes.toString("latin1"),allowedRequest=rel==="requests/invalid-credential.json"&&text.includes(fixed)&&!randomized.test(text);if(containsCanary(text)&&!allowedRequest){console.error("secret canary found in filesystem bytes (path redacted)");found=true;}}finally{if(handle)await handle.close();}}} +function git(args,label){const listing=execFileSync("git",[...args,"cat-file","--batch-all-objects","--unordered","--batch-check=%(objectname) %(objecttype) %(objectsize)"],{encoding:"utf8",maxBuffer:16*1024*1024}).trim(),objects=listing?listing.split("\\n"):[];if(objects.length>100000)throw Error("Git object count bound exceeded in "+label);let total=0;for(const line of objects){const match=line.match(/^([0-9a-f]{40,64}) (blob|commit|tree|tag) (\\d+)$/);if(!match)throw Error("malformed Git object listing in "+label);const[,oid,type,sizeText]=match,size=Number(sizeText);total+=size;if(!Number.isSafeInteger(size)||size>33554432||total>536870912)throw Error("Git object byte bound exceeded in "+label);const raw=execFileSync("git",[...args,"cat-file",type,oid],{maxBuffer:Math.max(1024,size+1)});if(raw.length!==size)throw Error("Git object size changed in "+label);if(containsCanary(raw.toString("latin1"))){console.error(type==="blob"?"secret canary found in Git blob: "+label:"secret canary found in Git object ("+type+"): "+label);found=true;}}} await walk(root);git(["--git-dir",join(root,"remote.git")],"remote.git");git(["-C",join(root,"author")],"author");git(["-C",join(root,"installation/registry/repo")],"installed-registry");if(found)process.exit(1);console.log("no fixture secret canary outside fixture-secrets or in any bounded Git object"); NODE `],["absence-check.sh",`#!/usr/bin/env bash\nset -euo pipefail\nroot=${quote(root)}\nif find "$root" -path '*/.git' -prune -o -type f \\( -iname '*preprocess*' -o -iname '*embedding*' -o -iname '*qdrant*' -o -iname '*retention*' -o -iname '*active*' \\) -print | grep .; then echo 'unexpected P1-scope artifact' >&2; exit 1; fi\necho 'no out-of-scope runtime artifact found'\n`]]){await atomicWrite(join(commands,name),body,0o700);await chmod(join(commands,name),0o700);}} diff --git a/backend/scripts/p1-manual-acceptance.test.mjs b/backend/scripts/p1-manual-acceptance.test.mjs index 8bfda3d2..4c078ed0 100644 --- a/backend/scripts/p1-manual-acceptance.test.mjs +++ b/backend/scripts/p1-manual-acceptance.test.mjs @@ -67,6 +67,11 @@ test("prepare requires the non-Task-8 tht prerequisite before creating state", a await assert.rejects(lstat(fixedManualRoot(repo))); }); +test("prepare and permanent docs declare the python3 extractor prerequisite", async () => { + const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8"),docs=await readFile(new URL("../../docs/testing/p1-manual-acceptance.md",import.meta.url),"utf8"); + assert.match(source,/for\(const command of \[.*["']python3["']/s); assert.match(docs,/python3/); +}); + test("prepare creates independent pending topology, fixtures, commands and guide without verdict", async () => { const repo = await fakeRepo(); const run = await prepareManual({ repositoryRoot: repo, skipBuild: true }); assert.equal(run.root, fixedManualRoot(repo)); @@ -447,7 +452,24 @@ exec ${realUnzip} "$@" `,{mode:0o700}); await execFileAsync("bash",[extract,original,join(run.root,"exports/extracted/staged-source"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:markerPath,P1_SWAP_REPLACEMENT:replacement,P1_SWAP_ORIGINAL:original}}); await lstat(markerPath); await lstat(join(run.root,"exports/extracted/staged-source/manifest.json")); - const generated=await readFile(extract,"utf8"); assert.match(generated,/open\(zip,["']r["']\)/); assert.match(generated,/stage/i); + const generated=await readFile(extract,"utf8"); assert.match(generated,/source_fd = os\.open\(zip_path/); assert.match(generated,/dir_fd=base_fd/); assert.match(generated,/O_NOFOLLOW/); assert.match(generated,/staged archive SHA mismatch/); +}); + +test("generated ZIP verifier anchors output when the extraction base is swapped on first unzip", async () => { + const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh"); + const zip=await makeExportZip(run.root,"ancestor-swap"),base=join(run.root,"exports/extracted"),moved=join(run.root,"exports/extracted-original"),outside=join(repo,"outside-extraction-race"); + const bin=join(repo,"unzip-swap-bin"),marker=join(repo,"unzip-swapped"),realUnzip=(await execFileAsync("which",["unzip"])).stdout.trim(); + await mkdir(bin); await mkdir(outside); + await writeFile(join(bin,"unzip"),`#!/bin/sh +if [ ! -e "$P1_SWAP_MARKER" ]; then + mv "$P1_SWAP_BASE" "$P1_SWAP_MOVED" + ln -s "$P1_SWAP_OUTSIDE" "$P1_SWAP_BASE" + : > "$P1_SWAP_MARKER" +fi +exec ${realUnzip} "$@" +`,{mode:0o700}); + await assert.rejects(execFileAsync("bash",[extract,zip,join(base,"escaped"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:marker,P1_SWAP_BASE:base,P1_SWAP_MOVED:moved,P1_SWAP_OUTSIDE:outside}}),/owned extraction root|identity|changed|unsafe/i); + await lstat(marker); assert.deepEqual(await readdir(outside),[]); }); test("generated ZIP verifier scans all four extracted byte streams for Evidence and canaries", async () => { @@ -469,6 +491,32 @@ test("generated secret scan excludes only the exact request fixture and hides fi await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary)); }); +test("generated secret scan reads Git metadata and arbitrary dot-git directories without printing values", async () => { + for(const rel of ["author/.git/manual-leak","responses/.git/leak"]){ + const canary="SECRET-"+"a".repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),scan=join(run.root,"commands/secret-scan.sh"); + await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]); + await mkdir(dirname(join(run.root,rel)),{recursive:true}); await writeFile(join(run.root,rel),canary); + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary),`${rel} must be scanned with a redacted finding`); + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("generated secret scan reads raw dangling commit, tag, and tree objects without printing values", async () => { + for(const kind of ["commit","tag","tree"]){ + const canary="SESSION-"+({commit:"b",tag:"c",tree:"d"}[kind]).repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),author=join(run.root,"author"),scan=join(run.root,"commands/secret-scan.sh"); + await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]); + if(kind==="commit"){ + await execFileAsync("git",["commit","--allow-empty","-m",canary],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD^"],{cwd:author}); + }else if(kind==="tag"){ + await execFileAsync("git",["tag","-a","temporary-canary-tag","-m",canary],{cwd:author}); await execFileAsync("git",["tag","-d","temporary-canary-tag"],{cwd:author}); + }else{ + await writeFile(join(author,canary),"safe tree payload\n"); await execFileAsync("git",["add",canary],{cwd:author}); await execFileAsync("git",["write-tree"],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD"],{cwd:author}); + } + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git object/.test(error.stderr)&&!error.stderr.includes(canary),`${kind} raw bytes must be scanned with a redacted finding`); + await rm(run.root,{recursive:true,force:true}); + } +}); + test("generated secret scan checks unreachable blobs and dangling commits without printing values", async () => { for(const kind of ["unreachable-blob","dangling-commit"]){ const value=kind==="unreachable-blob"?"SECRET-"+"e".repeat(32):"SECRET-"+"f".repeat(32); diff --git a/docs/testing/p1-manual-acceptance.md b/docs/testing/p1-manual-acceptance.md index e28ae9a6..92cb2244 100644 --- a/docs/testing/p1-manual-acceptance.md +++ b/docs/testing/p1-manual-acceptance.md @@ -7,8 +7,9 @@ result. Automation never creates `VERDICT.md`, never records PASS, and never con ## Prerequisites -From a clean repository checkout, Task 8 must already be implemented. Install Node/npm and Git, -`curl`, `unzip`/`zipinfo`, `lsof`, and the harness development environment so `harness/.venv/bin/tht` is executable. +From a clean repository checkout, Task 8 must already be implemented. Install Node/npm, `python3`, +and Git, `curl`, `unzip`/`zipinfo`, `lsof`, and the harness development environment so +`harness/.venv/bin/tht` is executable. Ports `127.0.0.1:8791` and `127.0.0.1:8792` must be free. The helper builds and serves only the production backend; it does not start Docker or the frontend. @@ -55,11 +56,15 @@ its commit-addressed owned snapshot path, the saved publish commit, and the inst calling the acceptance-only production renderer. The renderer imports the built `ThtRunner`, resolves bindings from environment paths, copies one lease atomically with mode `0600`, and releases it in `finally`. For each exported ZIP, invoke the generated extractor with the exact expected workspace ID -(`p1-filesystem`, `p1-http`, or `p1-s3`); it stages one immutable owned copy, confines extraction, -and binds both the manifest and parsed descriptor identity to that expected ID. The generated secret -scan checks bounded bytes from every Git object, including unreachable blobs and dangling commits. -Do not inspect or print raw secret-file contents; only inspect ownership/mode/path metadata and canary -absence outside `fixture-secrets`. +(`p1-filesystem`, `p1-http`, or `p1-s3`); its `python3` helper opens the source once, stages and +revalidates its SHA-256, anchors every extraction and cleanup operation to an opened no-follow +`exports/extracted` directory descriptor, and binds both the manifest and parsed descriptor identity +to that expected ID. It verifies exactly four regular entries and publishes only their exact checked +bytes. The generated secret scan reads every bounded filesystem file outside the direct +`fixture-secrets` directory, including Git metadata and arbitrary `.git`-named directories, then +enumerates every reachable or unreachable Git object and scans the bounded raw blob, commit, tree, +and tag bytes. Findings redact canary values. Do not inspect or print raw secret-file contents; only +inspect ownership/mode/path metadata and canary absence outside `fixture-secrets`. ## Failures and verdict