fix: anchor P1 manual extraction and scans
This commit is contained in:
@@ -44,7 +44,7 @@ async function run(executable,argv,options={}){return await exec(executable,argv
|
||||
function descriptor(id,source){return{workspace:{schema_version:3,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},semantic_index:{vector_store:{engine:"qdrant",collection:id,dimensions:1024,distance:"cosine"},embedding:{provider:"ollama_internal",model:"qwen3-embedding:0.6b",dimensions:1024}},llm_policy:{allowed:["zai/glm-5.2"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};}
|
||||
function descriptors(){return[descriptor("p1-filesystem",{type:"filesystem",uri:"workspace-content/p1-filesystem/evidence",patterns:["**/*.md"],max_bytes:10485760}),descriptor("p1-http",{type:"http",uris:["https://evidence.example.test/guide.md"],authentication:"signed_urls_file",connect_timeout_ms:1250,read_timeout_ms:30001,max_bytes:12345,max_redirects:2,allow_private_hosts:false,max_cache_bytes:67890}),descriptor("p1-s3",{type:"s3",uri:"s3://p1-evidence/published/",endpoint_url:"https://s3.example.test/",region:"eu-west-1",credentials:"static_files",trusted_endpoint:true,allow_private_endpoint:false,allow_insecure_endpoint:false,max_bytes:12345,max_objects:33,max_pages:4,page_size:5})];}
|
||||
function quote(value){return `'${String(value).replaceAll("'",`'"'"'`)}'`;}
|
||||
async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}}
|
||||
async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof","python3"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}}
|
||||
async function initializeGit(root){await run("git",["init","--bare","--initial-branch=main",join(root,"remote.git")],{cwd:root});await run("git",["clone",join(root,"remote.git"),join(root,"author")],{cwd:root});for(const [key,value]of [["user.name","P1 Manual Curator"],["user.email","p1-manual@example.invalid"]])await run("git",["config",key,value],{cwd:join(root,"author")});const evidence=join(root,"author","workspace-content","p1-filesystem","evidence");await mkdir(join(evidence,"domain"),{recursive:true});await writeFile(join(evidence,"guide.md"),"# P1 manually curated Evidence\n");await writeFile(join(evidence,"domain","table.md"),"# P1 curated table\n");await run("git",["add","workspace-content"],{cwd:join(root,"author")});await run("git",["commit","-m","Bootstrap P1 manual Evidence"],{cwd:join(root,"author")});await run("git",["push","origin","main"],{cwd:join(root,"author")});}
|
||||
function requestFixtures(items){const result={"status.json":{method:"GET",path:"/workspace-registry/status"},"pull.json":{method:"POST",path:"/workspace-registry/pull"}};for(const workspace of items){const id=workspace.workspace.id;result[`validate-${id}.json`]={workspace};result[`publish-${id}.json`]={action:"create",workspace};result[`read-${id}.json`]={method:"GET",path:`/workspaces/${id}`};result[`export-${id}.json`]={method:"GET",path:`/workspaces/${id}/export`};}Object.assign(result,{"invalid-absolute.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"/etc"}}}},"invalid-traversal.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-filesystem/evidence/../../p1-s3/evidence"}}}},"invalid-cross-workspace.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-s3/evidence"}}}},"invalid-protocol.json":{workspace:{...items[1],evidence:{...items[1].evidence,source:{...items[1].evidence.source,uris:["file:///etc/passwd"]}}}},"invalid-credential.json":{workspace:{...items[2],evidence:{...items[2].evidence,source:{...items[2].evidence.source,access_key:"CANARY-MUST-BE-REJECTED"}}}}});return result;}
|
||||
function curlGet(url,output){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;}
|
||||
@@ -94,7 +94,7 @@ Status: **PENDING**. The reviewer, not this helper, performs and judges every st
|
||||
8. Inspect runtime identity, absolute reserved filesystem root, Evidence limits, and policy in the rendered YAML; do not inspect secret contents.
|
||||
9. Personally execute \`${repo}/harness/.venv/bin/tht config check -c ${root}/rendered/runtime-1.yaml\` and the same command for \`runtime-2.yaml\` (or run \`commands/config-check.sh\`).
|
||||
10. Personally run \`commands/http-15-*.sh\` through \`commands/http-19-*.sh\` to submit the invalid absolute, Evidence-URI traversal, cross-workspace, protocol, and credential validation requests; verify safe rejection, no Git/snapshot mutation, and no rejected canary outside the request fixture.
|
||||
11. Run \`commands/secret-scan.sh\`; it excludes \`fixture-secrets\` and checks bounded bytes from every Git object, including unreachable blobs and dangling commits, for canary patterns without displaying secret contents.
|
||||
11. Run \`commands/secret-scan.sh\`; it excludes only the direct \`fixture-secrets\` payload directory, scans bounded filesystem bytes including Git metadata and arbitrary \`.git\` directories, and checks raw bounded bytes from every blob, commit, tree, and tag object, including unreachable objects, without displaying secret contents.
|
||||
12. Run \`commands/absence-check.sh\`; confirm no preprocessing, Evidence materialization, embedding, Qdrant, ACTIVE, or retention artifact exists.
|
||||
13. Run \`${repo}/scripts/p1-manual-acceptance.sh stop\`; confirm \`backend.pid\` and the listener on port ${PORT} are gone.
|
||||
14. Create \`${root}/VERDICT.md\` yourself with reviewer, UTC time, every checklist result, observations, and exactly either \`manual acceptance: PASS\` or \`manual acceptance: FAIL\`.
|
||||
@@ -104,40 +104,258 @@ Preserve a failed lab by stopping it and leaving the owned root in place. Only \
|
||||
function extractCommand(repo,root){return `#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
zip=\${1:?zip required}; out=\${2:?new output required}; expected=\${3:?expected workspace id required}
|
||||
node --input-type=module - "$zip" "$out" "$expected" ${quote(root)} ${quote(join(repo,"backend/package.json"))} <<'NODE'
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { createRequire } from "node:module";
|
||||
import { lstat, mkdir, open, readFile, realpath, rename, rm } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, relative } from "node:path";
|
||||
const [zip,out,expected,root,packageJson]=process.argv.slice(2),allowed=new Set(["p1-filesystem","p1-http","p1-s3"]),base=join(root,"exports/extracted");let stagedZip,stagedDirectory;
|
||||
const fail=message=>{throw new Error(message);},exact=(value,keys)=>value&&typeof value==="object"&&!Array.isArray(value)&&JSON.stringify(Object.keys(value).sort())===JSON.stringify([...keys].sort());
|
||||
try{
|
||||
if(!allowed.has(expected))fail("expected workspace identity is invalid");
|
||||
const canonicalRoot=await realpath(root),baseEntry=await lstat(base);if(baseEntry.isSymbolicLink()||!baseEntry.isDirectory()||await realpath(base)!==base||!relative(canonicalRoot,base)||relative(canonicalRoot,base).startsWith("..")||isAbsolute(relative(canonicalRoot,base)))fail("unsafe owned extraction root");
|
||||
if(dirname(out)!==base||basename(out).startsWith(".")||basename(out).length===0)fail("unsafe output path");try{await lstat(out);fail("unsafe output path");}catch(error){if(error.code!=="ENOENT")throw error;}
|
||||
const source=await open(zip,"r");let sourceBytes;try{const stat=await source.stat();if(!stat.isFile()||stat.size<1||stat.size>33554432)fail("source ZIP is unbounded");sourceBytes=await source.readFile();if(sourceBytes.length!==stat.size)fail("source ZIP changed during staging");}finally{await source.close();}
|
||||
stagedZip=join(root,"exports",".zip-stage-"+randomBytes(16).toString("hex")+".zip");const staged=await open(stagedZip,"wx",0o600);try{await staged.chmod(0o600);await staged.writeFile(sourceBytes);await staged.sync();}finally{await staged.close();}
|
||||
const names=execFileSync("unzip",["-Z1",stagedZip],{encoding:"utf8",maxBuffer:1048576}).trim().split("\\n"),required=["manifest.json","workspace.yaml","contract.env.example","README.md"];
|
||||
if(names.length!==4||new Set(names).size!==4||required.some(name=>!names.includes(name)))fail("unsafe-zip entries");
|
||||
const listing=execFileSync("zipinfo",["-l",stagedZip],{encoding:"utf8",maxBuffer:1048576}),regular=listing.split("\\n").filter(line=>line.startsWith("-")).length;if(regular!==4)fail("ZIP contains a symlink or nonregular entry");
|
||||
stagedDirectory=join(base,".extract-stage-"+randomBytes(16).toString("hex"));await mkdir(stagedDirectory,{mode:0o700});execFileSync("unzip",["-q",stagedZip,"-d",stagedDirectory],{stdio:"pipe",maxBuffer:1048576});
|
||||
const bytes={};for(const name of required){const path=join(stagedDirectory,name),entry=await lstat(path);if(!entry.isFile()||entry.isSymbolicLink()||entry.size<1||entry.size>10485760)fail("extracted file is unsafe");bytes[name]=await readFile(path);}
|
||||
const randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");for(const name of required){const value=bytes[name].toString("latin1");if(value.includes("P1 manually curated Evidence")||value.includes("P1 curated table")||randomized.test(value)||value.includes(fixed))fail("export contains Evidence or secret canary bytes");}
|
||||
let manifest;try{manifest=JSON.parse(bytes["manifest.json"].toString("utf8"));}catch{fail("export manifest schema mismatch");}const hashed=required.slice(1);if(!exact(manifest,["schema_version","workspace_id","files"])||manifest.schema_version!==1||manifest.workspace_id!==expected||!exact(manifest.files,hashed)||hashed.some(name=>!/^[0-9a-f]{64}$/.test(manifest.files[name])))fail("export manifest workspace identity or schema mismatch");for(const name of hashed)if(createHash("sha256").update(bytes[name]).digest("hex")!==manifest.files[name])fail("manifest hash mismatch");
|
||||
const require=createRequire(packageJson),YAML=require("yaml");let descriptor;try{descriptor=YAML.parse(bytes["workspace.yaml"].toString("utf8"));}catch{fail("export descriptor is malformed");}if(!descriptor||descriptor.workspace?.id!==expected)fail("export descriptor workspace identity mismatch");
|
||||
await rename(stagedDirectory,out);stagedDirectory=undefined;
|
||||
}catch(error){console.error(error.message);process.exitCode=1;}finally{if(stagedDirectory)await rm(stagedDirectory,{recursive:true,force:true}).catch(()=>{});if(stagedZip)await rm(stagedZip,{force:true}).catch(()=>{});}
|
||||
NODE
|
||||
python3 - "$zip" "$out" "$expected" ${quote(root)} ${quote(join(repo,"backend/package.json"))} <<'PY'
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
zip_path, output_path, expected, root, package_json = sys.argv[1:]
|
||||
allowed = {"p1-filesystem", "p1-http", "p1-s3"}
|
||||
required = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"]
|
||||
base = os.path.join(root, "exports", "extracted")
|
||||
base_fd = None
|
||||
archive_fd = None
|
||||
stage_fd = None
|
||||
archive_stage = None
|
||||
extract_stage = None
|
||||
published = False
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def exact(value, keys):
|
||||
return isinstance(value, dict) and set(value) == set(keys)
|
||||
|
||||
|
||||
def write_all(fd, data):
|
||||
view = memoryview(data)
|
||||
while view:
|
||||
written = os.write(fd, view)
|
||||
if written <= 0:
|
||||
fail("anchored extraction write failed")
|
||||
view = view[written:]
|
||||
|
||||
|
||||
def read_exact_fd(fd, expected_size, limit, label):
|
||||
if expected_size < 1 or expected_size > limit:
|
||||
fail(label + " is unbounded")
|
||||
chunks = []
|
||||
remaining = expected_size
|
||||
while remaining:
|
||||
chunk = os.read(fd, min(1024 * 1024, remaining))
|
||||
if not chunk:
|
||||
fail(label + " changed while staging")
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
if os.read(fd, 1):
|
||||
fail(label + " changed while staging")
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def require_base_identity():
|
||||
try:
|
||||
current = os.stat(base, follow_symlinks=False)
|
||||
except OSError:
|
||||
fail("owned extraction root identity changed")
|
||||
if (not stat.S_ISDIR(current.st_mode) or current.st_dev != base_identity.st_dev
|
||||
or current.st_ino != base_identity.st_ino or os.path.realpath(base) != base):
|
||||
fail("owned extraction root identity changed")
|
||||
|
||||
|
||||
def remove_anchored_directory(name):
|
||||
child_fd = None
|
||||
try:
|
||||
child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd)
|
||||
for entry in os.listdir(child_fd):
|
||||
if entry not in required:
|
||||
fail("anchored extraction cleanup found an unexpected entry")
|
||||
os.unlink(entry, dir_fd=child_fd)
|
||||
os.fsync(child_fd)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
finally:
|
||||
if child_fd is not None:
|
||||
os.close(child_fd)
|
||||
os.rmdir(name, dir_fd=base_fd)
|
||||
os.fsync(base_fd)
|
||||
|
||||
|
||||
try:
|
||||
for flag in ("O_DIRECTORY", "O_NOFOLLOW"):
|
||||
if not hasattr(os, flag):
|
||||
fail("anchored extraction is unavailable on this platform")
|
||||
if expected not in allowed:
|
||||
fail("expected workspace identity is invalid")
|
||||
if os.path.realpath(root) != root or os.path.dirname(output_path) != base:
|
||||
fail("unsafe owned extraction root or output path")
|
||||
output_name = os.path.basename(output_path)
|
||||
if not output_name or output_name.startswith(".") or os.sep in output_name:
|
||||
fail("unsafe output path")
|
||||
|
||||
base_fd = os.open(base, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
base_identity = os.fstat(base_fd)
|
||||
if not stat.S_ISDIR(base_identity.st_mode):
|
||||
fail("unsafe owned extraction root")
|
||||
require_base_identity()
|
||||
try:
|
||||
os.stat(output_name, dir_fd=base_fd, follow_symlinks=False)
|
||||
fail("unsafe output path")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
# Open the caller's source exactly once, then consume only an owned staged copy.
|
||||
source_fd = os.open(zip_path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
try:
|
||||
source_identity = os.fstat(source_fd)
|
||||
if not stat.S_ISREG(source_identity.st_mode):
|
||||
fail("source ZIP is unsafe")
|
||||
source_bytes = read_exact_fd(source_fd, source_identity.st_size, 33554432, "source ZIP")
|
||||
source_after = os.fstat(source_fd)
|
||||
if (source_after.st_dev, source_after.st_ino, source_after.st_size) != (source_identity.st_dev, source_identity.st_ino, source_identity.st_size):
|
||||
fail("source ZIP changed during staging")
|
||||
finally:
|
||||
os.close(source_fd)
|
||||
archive_sha = hashlib.sha256(source_bytes).digest()
|
||||
|
||||
archive_stage = ".zip-stage-" + secrets.token_hex(16) + ".zip"
|
||||
archive_fd = os.open(archive_stage, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=base_fd)
|
||||
write_all(archive_fd, source_bytes)
|
||||
os.fsync(archive_fd)
|
||||
del source_bytes
|
||||
os.lseek(archive_fd, 0, os.SEEK_SET)
|
||||
staged_bytes = read_exact_fd(archive_fd, os.fstat(archive_fd).st_size, 33554432, "staged archive")
|
||||
if hashlib.sha256(staged_bytes).digest() != archive_sha:
|
||||
fail("staged archive SHA mismatch")
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(staged_bytes), "r") as archive:
|
||||
infos = archive.infolist()
|
||||
names = [entry.filename for entry in infos]
|
||||
if len(names) != 4 or len(set(names)) != 4 or set(names) != set(required):
|
||||
fail("unsafe-zip entries")
|
||||
for entry in infos:
|
||||
mode = (entry.external_attr >> 16) & 0xFFFF
|
||||
if not stat.S_ISREG(mode) or entry.flag_bits & 1:
|
||||
fail("ZIP contains a symlink or nonregular entry")
|
||||
if entry.file_size < 1 or entry.file_size > 10485760:
|
||||
fail("extracted file is unsafe")
|
||||
payloads = {name: archive.read(name) for name in required}
|
||||
if any(len(payloads[entry.filename]) != entry.file_size for entry in infos):
|
||||
fail("extracted file size mismatch")
|
||||
|
||||
# Exercise the documented unzip prerequisite against the exact staged descriptor, not a path.
|
||||
listing = subprocess.run(
|
||||
["unzip", "-Z1", "/dev/fd/" + str(archive_fd)], pass_fds=(archive_fd,),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20, check=False,
|
||||
)
|
||||
if listing.returncode != 0 or listing.stdout.decode("utf8", "strict").splitlines() != names:
|
||||
fail("unsafe-zip entries")
|
||||
os.lseek(archive_fd, 0, os.SEEK_SET)
|
||||
revalidated = read_exact_fd(archive_fd, len(staged_bytes), 33554432, "staged archive")
|
||||
if hashlib.sha256(revalidated).digest() != archive_sha or revalidated != staged_bytes:
|
||||
fail("staged archive SHA mismatch")
|
||||
require_base_identity()
|
||||
|
||||
randomized = re.compile(br"(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}")
|
||||
fixed = b"-".join([b"CANARY", b"MUST", b"BE", b"REJECTED"])
|
||||
for data in payloads.values():
|
||||
if b"P1 manually curated Evidence" in data or b"P1 curated table" in data or randomized.search(data) or fixed in data:
|
||||
fail("export contains Evidence or secret canary bytes")
|
||||
|
||||
try:
|
||||
manifest = json.loads(payloads["manifest.json"].decode("utf8"))
|
||||
except Exception:
|
||||
fail("export manifest schema mismatch")
|
||||
hashed = required[1:]
|
||||
files = manifest.get("files") if isinstance(manifest, dict) else None
|
||||
if (not exact(manifest, ["schema_version", "workspace_id", "files"])
|
||||
or manifest.get("schema_version") != 1 or manifest.get("workspace_id") != expected
|
||||
or not exact(files, hashed)
|
||||
or any(not isinstance(files[name], str) or not re.fullmatch(r"[0-9a-f]{64}", files[name]) for name in hashed)):
|
||||
fail("export manifest workspace identity or schema mismatch")
|
||||
for name in hashed:
|
||||
if hashlib.sha256(payloads[name]).hexdigest() != files[name]:
|
||||
fail("manifest hash mismatch")
|
||||
|
||||
yaml_helper = 'const fs=require("node:fs"),{createRequire}=require("node:module");try{const YAML=createRequire(process.argv[1])("yaml"),v=YAML.parse(fs.readFileSync(0,"utf8"));process.stdout.write(JSON.stringify(v?.workspace?.id??null));}catch{process.exit(2)}'
|
||||
parsed = subprocess.run(["node", "-e", yaml_helper, package_json], input=payloads["workspace.yaml"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10, check=False)
|
||||
try:
|
||||
descriptor_id = json.loads(parsed.stdout.decode("utf8")) if parsed.returncode == 0 else None
|
||||
except Exception:
|
||||
descriptor_id = None
|
||||
if descriptor_id != expected:
|
||||
fail("export descriptor workspace identity mismatch")
|
||||
|
||||
extract_stage = ".extract-stage-" + secrets.token_hex(16)
|
||||
os.mkdir(extract_stage, 0o700, dir_fd=base_fd)
|
||||
stage_fd = os.open(extract_stage, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd)
|
||||
for name in required:
|
||||
fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=stage_fd)
|
||||
try:
|
||||
write_all(fd, payloads[name])
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.fsync(stage_fd)
|
||||
os.close(stage_fd)
|
||||
stage_fd = None
|
||||
require_base_identity()
|
||||
try:
|
||||
os.stat(output_name, dir_fd=base_fd, follow_symlinks=False)
|
||||
fail("unsafe output path")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.rename(extract_stage, output_name, src_dir_fd=base_fd, dst_dir_fd=base_fd)
|
||||
extract_stage = None
|
||||
published = True
|
||||
os.fsync(base_fd)
|
||||
require_base_identity()
|
||||
except Exception as error:
|
||||
print(str(error), file=sys.stderr)
|
||||
sys.exit_code = 1
|
||||
finally:
|
||||
if stage_fd is not None:
|
||||
os.close(stage_fd)
|
||||
if extract_stage is not None and base_fd is not None:
|
||||
try:
|
||||
remove_anchored_directory(extract_stage)
|
||||
except Exception:
|
||||
sys.exit_code = 1
|
||||
if published and getattr(sys, "exit_code", 0) and base_fd is not None:
|
||||
try:
|
||||
remove_anchored_directory(output_name)
|
||||
except Exception:
|
||||
pass
|
||||
if archive_fd is not None:
|
||||
os.close(archive_fd)
|
||||
if archive_stage is not None and base_fd is not None:
|
||||
try:
|
||||
os.unlink(archive_stage, dir_fd=base_fd)
|
||||
os.fsync(base_fd)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if base_fd is not None:
|
||||
os.close(base_fd)
|
||||
if getattr(sys, "exit_code", 0):
|
||||
raise SystemExit(sys.exit_code)
|
||||
PY
|
||||
`;}
|
||||
async function writeCommands(repo,root){const commands=join(root,"commands");for(const [name,body]of [...httpCommands(root),["render-1.sh",renderCommand(repo,root,1)],["render-2.sh",renderCommand(repo,root,2)],["diff-rendered.sh",`#!/bin/sh\nset -eu\ndiff -u ${quote(join(root,"rendered/runtime-1.yaml"))} ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["config-check.sh",`#!/bin/sh\nset -eu\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-1.yaml"))}\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["git-inspect.sh",`#!/bin/sh\nset -eu\ncommit=\${1:?published commit required}\ncase "$commit" in *[!0-9a-f]*|'') exit 2;; esac\n[ \${#commit} -eq 40 ] || exit 2\ngit -C ${quote(join(root,"installation/registry/repo"))} log --oneline --decorate -10 "$commit"\ngit -C ${quote(join(root,"installation/registry/repo"))} ls-tree -r "$commit" -- workspaces workspace-content\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspaces/p1-filesystem.yaml"\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspace-content/p1-filesystem/evidence/guide.md"\n`],["extract-export.sh",extractCommand(repo,root)],["secret-scan.sh",`#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root=${quote(root)}
|
||||
node --input-type=module - "$root" <<'NODE'
|
||||
import { execFileSync } from "node:child_process";import { lstat, readFile, readdir } from "node:fs/promises";import { basename, join, relative } from "node:path";
|
||||
const root=process.argv[2],randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");let found=false;const containsCanary=text=>randomized.test(text)||text.includes(fixed);
|
||||
async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){const child=join(path,entry.name),rel=relative(root,child);if(entry.isSymbolicLink()){console.error("unsafe symlink during secret scan: "+rel);found=true;continue;}if(entry.isDirectory()){if(rel==="fixture-secrets"||entry.name===".git")continue;await walk(child);}else if(entry.isFile()){const stat=await lstat(child);if(stat.size>33554432)throw Error("secret scan file too large: "+rel);if(containsCanary((await readFile(child)).toString("latin1"))&&rel!=="requests/invalid-credential.json"){console.error("secret canary found: "+rel);found=true;}}}}
|
||||
function git(args,label){const listing=execFileSync("git",[...args,"cat-file","--batch-all-objects","--unordered","--batch-check=%(objectname) %(objecttype) %(objectsize)"],{encoding:"utf8",maxBuffer:16*1024*1024}).trim(),objects=listing?listing.split("\\n"):[];if(objects.length>100000)throw Error("too many Git objects to scan in "+label);let total=0;for(const line of objects){const [oid,type,sizeText]=line.split(" ");if(!/^[0-9a-f]{40,64}$/.test(oid??"")||!type||!/^\\d+$/.test(sizeText??""))throw Error("malformed Git object listing in "+label);if(type!=="blob")continue;const size=Number(sizeText);total+=size;if(!Number.isSafeInteger(size)||size>33554432||total>536870912)throw Error("Git blob scan bound exceeded in "+label);const blob=execFileSync("git",[...args,"cat-file","blob",oid],{maxBuffer:size+1});if(blob.length!==size)throw Error("Git blob size changed in "+label);if(containsCanary(blob.toString("latin1"))){console.error("secret canary found in Git blob: "+label+":"+oid);found=true;}}}
|
||||
import { execFileSync } from "node:child_process";import { constants } from "node:fs";import { lstat, open, readdir } from "node:fs/promises";import { join, relative } from "node:path";
|
||||
const root=process.argv[2],randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");let found=false,filesystemCount=0,filesystemTotal=0;const containsCanary=text=>randomized.test(text)||text.includes(fixed);
|
||||
async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){filesystemCount++;if(filesystemCount>200000)throw Error("filesystem secret scan entry bound exceeded");const child=join(path,entry.name),rel=relative(root,child);if(entry.isSymbolicLink()){console.error("unsafe symlink during secret scan (path redacted)");found=true;continue;}if(entry.isDirectory()){if(rel==="fixture-secrets")continue;await walk(child);continue;}if(!entry.isFile())throw Error("unsupported filesystem entry during secret scan");let handle;try{handle=await open(child,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat();if(!before.isFile()||before.size>33554432)throw Error("filesystem secret scan file bound exceeded");filesystemTotal+=before.size;if(filesystemTotal>1073741824)throw Error("filesystem secret scan total bound exceeded");const bytes=await handle.readFile(),after=await handle.stat();if(bytes.length!==before.size||after.dev!==before.dev||after.ino!==before.ino||after.size!==before.size)throw Error("filesystem changed during secret scan");const text=bytes.toString("latin1"),allowedRequest=rel==="requests/invalid-credential.json"&&text.includes(fixed)&&!randomized.test(text);if(containsCanary(text)&&!allowedRequest){console.error("secret canary found in filesystem bytes (path redacted)");found=true;}}finally{if(handle)await handle.close();}}}
|
||||
function git(args,label){const listing=execFileSync("git",[...args,"cat-file","--batch-all-objects","--unordered","--batch-check=%(objectname) %(objecttype) %(objectsize)"],{encoding:"utf8",maxBuffer:16*1024*1024}).trim(),objects=listing?listing.split("\\n"):[];if(objects.length>100000)throw Error("Git object count bound exceeded in "+label);let total=0;for(const line of objects){const match=line.match(/^([0-9a-f]{40,64}) (blob|commit|tree|tag) (\\d+)$/);if(!match)throw Error("malformed Git object listing in "+label);const[,oid,type,sizeText]=match,size=Number(sizeText);total+=size;if(!Number.isSafeInteger(size)||size>33554432||total>536870912)throw Error("Git object byte bound exceeded in "+label);const raw=execFileSync("git",[...args,"cat-file",type,oid],{maxBuffer:Math.max(1024,size+1)});if(raw.length!==size)throw Error("Git object size changed in "+label);if(containsCanary(raw.toString("latin1"))){console.error(type==="blob"?"secret canary found in Git blob: "+label:"secret canary found in Git object ("+type+"): "+label);found=true;}}}
|
||||
await walk(root);git(["--git-dir",join(root,"remote.git")],"remote.git");git(["-C",join(root,"author")],"author");git(["-C",join(root,"installation/registry/repo")],"installed-registry");if(found)process.exit(1);console.log("no fixture secret canary outside fixture-secrets or in any bounded Git object");
|
||||
NODE
|
||||
`],["absence-check.sh",`#!/usr/bin/env bash\nset -euo pipefail\nroot=${quote(root)}\nif find "$root" -path '*/.git' -prune -o -type f \\( -iname '*preprocess*' -o -iname '*embedding*' -o -iname '*qdrant*' -o -iname '*retention*' -o -iname '*active*' \\) -print | grep .; then echo 'unexpected P1-scope artifact' >&2; exit 1; fi\necho 'no out-of-scope runtime artifact found'\n`]]){await atomicWrite(join(commands,name),body,0o700);await chmod(join(commands,name),0o700);}}
|
||||
|
||||
@@ -67,6 +67,11 @@ test("prepare requires the non-Task-8 tht prerequisite before creating state", a
|
||||
await assert.rejects(lstat(fixedManualRoot(repo)));
|
||||
});
|
||||
|
||||
test("prepare and permanent docs declare the python3 extractor prerequisite", async () => {
|
||||
const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8"),docs=await readFile(new URL("../../docs/testing/p1-manual-acceptance.md",import.meta.url),"utf8");
|
||||
assert.match(source,/for\(const command of \[.*["']python3["']/s); assert.match(docs,/python3/);
|
||||
});
|
||||
|
||||
test("prepare creates independent pending topology, fixtures, commands and guide without verdict", async () => {
|
||||
const repo = await fakeRepo(); const run = await prepareManual({ repositoryRoot: repo, skipBuild: true });
|
||||
assert.equal(run.root, fixedManualRoot(repo));
|
||||
@@ -447,7 +452,24 @@ exec ${realUnzip} "$@"
|
||||
`,{mode:0o700});
|
||||
await execFileAsync("bash",[extract,original,join(run.root,"exports/extracted/staged-source"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:markerPath,P1_SWAP_REPLACEMENT:replacement,P1_SWAP_ORIGINAL:original}});
|
||||
await lstat(markerPath); await lstat(join(run.root,"exports/extracted/staged-source/manifest.json"));
|
||||
const generated=await readFile(extract,"utf8"); assert.match(generated,/open\(zip,["']r["']\)/); assert.match(generated,/stage/i);
|
||||
const generated=await readFile(extract,"utf8"); assert.match(generated,/source_fd = os\.open\(zip_path/); assert.match(generated,/dir_fd=base_fd/); assert.match(generated,/O_NOFOLLOW/); assert.match(generated,/staged archive SHA mismatch/);
|
||||
});
|
||||
|
||||
test("generated ZIP verifier anchors output when the extraction base is swapped on first unzip", async () => {
|
||||
const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh");
|
||||
const zip=await makeExportZip(run.root,"ancestor-swap"),base=join(run.root,"exports/extracted"),moved=join(run.root,"exports/extracted-original"),outside=join(repo,"outside-extraction-race");
|
||||
const bin=join(repo,"unzip-swap-bin"),marker=join(repo,"unzip-swapped"),realUnzip=(await execFileAsync("which",["unzip"])).stdout.trim();
|
||||
await mkdir(bin); await mkdir(outside);
|
||||
await writeFile(join(bin,"unzip"),`#!/bin/sh
|
||||
if [ ! -e "$P1_SWAP_MARKER" ]; then
|
||||
mv "$P1_SWAP_BASE" "$P1_SWAP_MOVED"
|
||||
ln -s "$P1_SWAP_OUTSIDE" "$P1_SWAP_BASE"
|
||||
: > "$P1_SWAP_MARKER"
|
||||
fi
|
||||
exec ${realUnzip} "$@"
|
||||
`,{mode:0o700});
|
||||
await assert.rejects(execFileAsync("bash",[extract,zip,join(base,"escaped"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:marker,P1_SWAP_BASE:base,P1_SWAP_MOVED:moved,P1_SWAP_OUTSIDE:outside}}),/owned extraction root|identity|changed|unsafe/i);
|
||||
await lstat(marker); assert.deepEqual(await readdir(outside),[]);
|
||||
});
|
||||
|
||||
test("generated ZIP verifier scans all four extracted byte streams for Evidence and canaries", async () => {
|
||||
@@ -469,6 +491,32 @@ test("generated secret scan excludes only the exact request fixture and hides fi
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary));
|
||||
});
|
||||
|
||||
test("generated secret scan reads Git metadata and arbitrary dot-git directories without printing values", async () => {
|
||||
for(const rel of ["author/.git/manual-leak","responses/.git/leak"]){
|
||||
const canary="SECRET-"+"a".repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),scan=join(run.root,"commands/secret-scan.sh");
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]);
|
||||
await mkdir(dirname(join(run.root,rel)),{recursive:true}); await writeFile(join(run.root,rel),canary);
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary),`${rel} must be scanned with a redacted finding`);
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("generated secret scan reads raw dangling commit, tag, and tree objects without printing values", async () => {
|
||||
for(const kind of ["commit","tag","tree"]){
|
||||
const canary="SESSION-"+({commit:"b",tag:"c",tree:"d"}[kind]).repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),author=join(run.root,"author"),scan=join(run.root,"commands/secret-scan.sh");
|
||||
await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]);
|
||||
if(kind==="commit"){
|
||||
await execFileAsync("git",["commit","--allow-empty","-m",canary],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD^"],{cwd:author});
|
||||
}else if(kind==="tag"){
|
||||
await execFileAsync("git",["tag","-a","temporary-canary-tag","-m",canary],{cwd:author}); await execFileAsync("git",["tag","-d","temporary-canary-tag"],{cwd:author});
|
||||
}else{
|
||||
await writeFile(join(author,canary),"safe tree payload\n"); await execFileAsync("git",["add",canary],{cwd:author}); await execFileAsync("git",["write-tree"],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD"],{cwd:author});
|
||||
}
|
||||
await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git object/.test(error.stderr)&&!error.stderr.includes(canary),`${kind} raw bytes must be scanned with a redacted finding`);
|
||||
await rm(run.root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
|
||||
test("generated secret scan checks unreachable blobs and dangling commits without printing values", async () => {
|
||||
for(const kind of ["unreachable-blob","dangling-commit"]){
|
||||
const value=kind==="unreachable-blob"?"SECRET-"+"e".repeat(32):"SECRET-"+"f".repeat(32);
|
||||
|
||||
@@ -7,8 +7,9 @@ result. Automation never creates `VERDICT.md`, never records PASS, and never con
|
||||
|
||||
## Prerequisites
|
||||
|
||||
From a clean repository checkout, Task 8 must already be implemented. Install Node/npm and Git,
|
||||
`curl`, `unzip`/`zipinfo`, `lsof`, and the harness development environment so `harness/.venv/bin/tht` is executable.
|
||||
From a clean repository checkout, Task 8 must already be implemented. Install Node/npm, `python3`,
|
||||
and Git, `curl`, `unzip`/`zipinfo`, `lsof`, and the harness development environment so
|
||||
`harness/.venv/bin/tht` is executable.
|
||||
Ports `127.0.0.1:8791` and `127.0.0.1:8792` must be free. The helper builds and serves only the
|
||||
production backend; it does not start Docker or the frontend.
|
||||
|
||||
@@ -55,11 +56,15 @@ its commit-addressed owned snapshot path, the saved publish commit, and the inst
|
||||
calling the acceptance-only production renderer. The renderer imports the built `ThtRunner`, resolves
|
||||
bindings from environment paths, copies one lease atomically with mode `0600`, and releases it in
|
||||
`finally`. For each exported ZIP, invoke the generated extractor with the exact expected workspace ID
|
||||
(`p1-filesystem`, `p1-http`, or `p1-s3`); it stages one immutable owned copy, confines extraction,
|
||||
and binds both the manifest and parsed descriptor identity to that expected ID. The generated secret
|
||||
scan checks bounded bytes from every Git object, including unreachable blobs and dangling commits.
|
||||
Do not inspect or print raw secret-file contents; only inspect ownership/mode/path metadata and canary
|
||||
absence outside `fixture-secrets`.
|
||||
(`p1-filesystem`, `p1-http`, or `p1-s3`); its `python3` helper opens the source once, stages and
|
||||
revalidates its SHA-256, anchors every extraction and cleanup operation to an opened no-follow
|
||||
`exports/extracted` directory descriptor, and binds both the manifest and parsed descriptor identity
|
||||
to that expected ID. It verifies exactly four regular entries and publishes only their exact checked
|
||||
bytes. The generated secret scan reads every bounded filesystem file outside the direct
|
||||
`fixture-secrets` directory, including Git metadata and arbitrary `.git`-named directories, then
|
||||
enumerates every reachable or unreachable Git object and scans the bounded raw blob, commit, tree,
|
||||
and tag bytes. Findings redact canary values. Do not inspect or print raw secret-file contents; only
|
||||
inspect ownership/mode/path metadata and canary absence outside `fixture-secrets`.
|
||||
|
||||
## Failures and verdict
|
||||
|
||||
|
||||
Reference in New Issue
Block a user