diff --git a/.artifacts/reviews/task9-quality-audit-final5.md b/.artifacts/reviews/task9-quality-audit-final5.md new file mode 100644 index 00000000..665653b3 --- /dev/null +++ b/.artifacts/reviews/task9-quality-audit-final5.md @@ -0,0 +1,66 @@ +# Task 9 quality audit — final 5 + +**Scope:** the two blocking findings from `task9-quality-audit-final4.md` — unbound production +module graph at manual serve, and commit-addressed snapshots accepted without content identity at +render. Manual acceptance remains **PENDING**; no `VERDICT.md` was created. + +## Verdict: APPROVED for the two final integrity blockers + +### 1. Manual serve binds the complete `backend/dist` module graph, not only `server.js` + +`prepare` now builds a post-build manifest of every regular `backend/dist` file +(relative path, size, SHA-256, device, inode) and writes it as an exclusive `0600` record +(`installation/runtime/backend-dist.manifest.json`) inside the owned root; `ownership.json` +records that record's path/device/inode/size/SHA-256. `serve` revalidates the manifest record +identity and bytes, revalidates every distribution file against it (no-follow, single inode, +size and digest), and refuses before spawning. The manifest descriptor is passed to the child on +fd 4 together with the entrypoint on fd 3. The immutable preload parses the manifest, verifies +the entrypoint cross-digest, reads and hash-verifies **every** file at startup, caches the +verified bytes, and its load hook serves **only** those cached bytes for any import below +`backend/dist` (entry URL still served from the bound fd-3 bytes). A same-path regular +replacement of any imported dependency is therefore refused before `RUNNING` (serve-time +validation), refused at child startup (startup verification), or rendered harmless (cached +bytes), and the parent revalidates the full manifest at `RUNNING` publication and at `stop`. + +### 2. Renderer binds snapshot content to its commit identity + +The generated render command validates the bounded saved read/publish revisions, the +commit-addressed owned snapshot path, the installed Git HEAD, and the bounded +`snapshot.json` manifest of that commit: `head` equals the commit, `files[.yaml]` is the +SHA-256 of the snapshot bytes, the manifest revision binds commit/blob/snapshot path, the saved +revision blob equals the manifest blob, and `git rev-parse :workspaces/.yaml` plus +`git hash-object` of the snapshot bytes both equal that blob. It passes the expected digest as +`--snapshot-sha256`. The renderer re-reads the bounded `snapshot.json` (`head`, +`files[.yaml]` must equal the carried digest), opens the snapshot once with no-follow +semantics and bounded reads, renders only the digest-verified bytes, re-verifies around lease +publication, releases the lease in `finally`, and publishes no output on any refusal. + +## Deterministic regressions added + +- static regular replacement of an imported production dependency after `prepare` is refused, + no marker, no accepted PID record, no orphan; +- deterministic dependency check/load swap (`beforeSpawn` rename) is refused by the child's + startup verification, no marker, no PID record, no orphan; +- after `RUNNING`, a same-path regular dependency replacement is never executed: the loader + serves the verified cached bytes (health-visible source stays the original) and the marker is + absent; +- renderer refuses a same-path regular snapshot byte replacement against the carried digest and + manifest, with lease release and no output; +- renderer refuses manifest `head`, `files` digest, expected-digest, missing, and malformed + cases, with lease release and no output; +- wrapper refuses missing manifest, manifest head/digest/revision tampering, saved-revision blob + mismatch, Git blob mismatch, and snapshot-vs-Git-bytes mismatch, and passes the exact + `--snapshot-sha256` on the valid path (stub renderer records arguments). + +## Verification + +- `bash scripts/test-p1-manual-acceptance.sh` (backend build + both suites): **59 tests, 59 + pass, 0 fail**; no `8791/8792` listener and no `--p1-manual-nonce` process remain. +- `npx tsc --noEmit -p .` (backend): PASS. +- Real-repository `prepare` + `cleanup` cycle: 39 distribution files bound, entrypoint + cross-digest verified, owned root fully removed afterwards. +- Diff check: only the seven Task 9 paths are touched; no Task 8 file was modified. +- This report and the implementation contain no fixture secret or canary values. + +Manual acceptance remains **PENDING** by design; the walkthrough and human verdict are +unchanged. diff --git a/.artifacts/task-15/automated-gates.json b/.artifacts/task-15/automated-gates.json new file mode 100644 index 00000000..80664d7a --- /dev/null +++ b/.artifacts/task-15/automated-gates.json @@ -0,0 +1,282 @@ +{ + "schema": "thothii-task4-certification-v1", + "generated_on": "2026-08-18", + "started_at_utc": "2026-08-18T14:16:40Z", + "ended_at_utc": "2026-08-18T14:20:10Z", + "source_commit": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "source_immutability": { + "status": "PASS", + "tracked_changes_after_freeze": false, + "allowed_untracked": [".playwright-cli/", ".thothctl/"] + }, + "source_commits": { + "task4_candidate": "b31b27e5845ffd3adf311429367319beaba263c7", + "task1": "d43738eeae6d14bb5e470093058b069a983f5372", + "task2": "5f9a3ae066a060b43a11a959b60a1efadd1c2425", + "task3": "0d8e707533fada938c99eb06f8457150e7ef2b40", + "task3_follow_up": "b31b27e5845ffd3adf311429367319beaba263c7", + "fix_round_1_source": "10cd66fe6a5b484a4dc569326a228c1c5484a5d4", + "fix_round_2_source": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "historical_task15_final": "74b062f1a737103524cbe706346cfd65f87cdfd1" + }, + "versions": { + "node_contract": "v24.16.0", + "node_host_default": "v25.6.1", + "go": "go1.26.5", + "pi": "0.80.3" + }, + "retained_report": ".superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md", + "task4_report": ".superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md", + "fix_round_2_report": ".superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-2-report.md", + "workflow": { + "run_id": "32147345625", + "url": "https://github.com/mptyl/ThothII/actions/runs/32147345625", + "event": "workflow_dispatch", + "head_sha": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "status": "completed", + "conclusion": "failure", + "windows_job": { + "name": "Windows clone and Compose contract", + "job_id": "95744249248", + "url": "https://github.com/mptyl/ThothII/actions/runs/32147345625/job/95744249248", + "conclusion": "failure", + "native_step": "Run native Windows retained-capability tests", + "native_step_conclusion": "success", + "command": "go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1", + "requested_packages": ["internal/safeio", "internal/backup", "internal/authstorage"], + "executed_packages": ["internal/safeio", "internal/backup", "internal/authstorage"], + "not_executed_packages": [], + "package_results": { + "internal/safeio": "PASS (22.058s)", + "internal/backup": "PASS (7.161s)", + "internal/authstorage": "PASS (16.088s)" + }, + "failed_step": "Verify Windows clone contract", + "failure_category": "baseline_powershell_parser", + "failure_detail": "scripts/test-windows-clone-contract.ps1:208 parses $remoteYaml: as an invalid variable reference" + }, + "lf_compose_docs_typescript_job": { + "name": "LF, Compose, docs, and TypeScript", + "job_id": "95744249458", + "url": "https://github.com/mptyl/ThothII/actions/runs/32147345625/job/95744249458", + "conclusion": "failure", + "failed_step": "Verify Compose and installation contracts", + "category": "baseline_ci_contract", + "detail": "unified Compose contract passed; test-no-deployment-coupling-scope.sh stopped on TMPDIR: unbound variable", + "downstream_steps": "skipped" + }, + "linux_docker_job": { + "name": "Linux Docker deployment and rollback", + "job_id": "95744249354", + "url": "https://github.com/mptyl/ThothII/actions/runs/32147345625/job/95744249354", + "conclusion": "failure", + "failed_step": "Run unified deployment smoke", + "category": "infrastructure_prerequisite", + "detail": "Task 13 smoke failed before deployment because rg is required", + "cleanup": "PASS", + "image_manifest": "not_generated" + }, + "windows_docker_startup_job": { + "name": "Native Windows Docker Desktop/WSL2 startup", + "job_id": "95744250450", + "url": "https://github.com/mptyl/ThothII/actions/runs/32147345625/job/95744250450", + "status": "NOT_RUN", + "classification": "BLOCKED", + "workflow_conclusion": "skipped", + "reason": "workflow conditions skipped the job; no Windows Docker Desktop/WSL2 command executed" + } + }, + "docker_image_evidence": { + "authentication_smoke": { + "status": "PASS", + "docker_images": [], + "reason": "no_docker_images_exercised" + }, + "unified_docker_smoke": { + "status": "FAIL", + "source_commit": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "run_id": "32147345625", + "workflow_job_id": "95744249354", + "manifest": ".artifacts/task-15/unified-docker-images.json", + "reason": "workflow attempt stopped before deployment because rg is required", + "cleanup": "PASS", + "images": 0, + "historical": { + "status": "PASS", + "source_commit": "74b062f1a737103524cbe706346cfd65f87cdfd1", + "run_id": "20260818070637-66409-30058", + "manifest_sha256": "9c8dec4546909fd93799dbcf374bcb3a89bc46cfe0fd482472c0cbe757ddf5b6", + "images": 5, + "cleanup": "PASS" + } + } + }, + "gates": { + "posix_registry_ownership": { + "status": "PASS", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7", + "evidence": "backend Node 24 full suite including local-registry ownership coverage" + }, + "stagearchive_unix_retained_capability": { + "status": "PASS", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7", + "evidence": "focused safeio/backup tests, Go race suite, and Unix ancestor-swap coverage" + }, + "windows_stagearchive_retained_capability": { + "status": "PASS", + "source_commit": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "evidence": "native Windows backup package passed, including the two-file shared retained-root staging test" + }, + "windows_claim_retained_capability": { + "status": "PASS", + "source_commit": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "evidence": "native Windows safeio and authstorage packages passed concurrent claim/consume coverage" + }, + "workflow_lf_compose_docs_typescript": { + "status": "FAIL", + "classification": "baseline_ci_contract", + "reason": "TMPDIR was unset after the unified Compose contract passed" + }, + "workflow_linux_docker": { + "status": "FAIL", + "classification": "infrastructure_prerequisite", + "reason": "runner did not provide rg; cleanup proof passed and no image manifest was generated" + }, + "go_security_build": { + "status": "PASS", + "source_commit": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "focused_packages": 3, + "race_packages": 18, + "focused_test": "PASS", + "race": "PASS", + "vet": "PASS", + "host_build": "PASS" + }, + "windows_cross_compile": { + "status": "PASS", + "source_commit": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "focused_test_packages": 3, + "cli_build": "PASS", + "execution": "cross_compile_only_not_native_execution" + }, + "backend_node24": { + "status": "PASS", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7", + "node": "v24.16.0", + "files": 76, + "tests": 1092, + "typecheck": "PASS", + "build": "PASS", + "note": "an initial full run had one workspace-registry timeout; focused rerun and complete rerun passed" + }, + "frontend_node24": { + "status": "PASS", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7", + "node": "v24.16.0", + "files": 61, + "tests": 444, + "typecheck": "PASS", + "build": "PASS" + }, + "authentication_and_f1_smoke": { + "status": "PASS", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7", + "node": "v24.16.0", + "filtered_e2e": "1 passed", + "sentinel_leak_scan": "PASS" + }, + "harness_pytest": { + "status": "FAIL", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7", + "passed": 951, + "failed": 1, + "skipped": 4, + "subtests": 232, + "failure": "test_column_decisions::test_f4_emits_column_types: workflow.yaml not found from harness test cwd" + }, + "authentication_docs": { + "status": "PASS", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7" + }, + "shell_syntax": { + "status": "PASS", + "source_commit": "b31b27e5845ffd3adf311429367319beaba263c7" + }, + "authentication_smoke_runtime": { + "status": "PASS", + "node": "v24.16.0", + "sentinel_leak_scan": "PASS" + }, + "compose_default": { + "status": "FAIL", + "reason": "required THT_WORKSPACE_GIT_REMOTE was not available" + }, + "compose_unified": { + "status": "FAIL", + "reason": "compose.unified.yaml is absent from the frozen source" + }, + "unified_docker_smoke": { + "status": "FAIL", + "source_commit": "2a9359071257f9b8a71d36ec2bbb25b161003f81", + "workflow_run_id": "32147345625", + "reason": "remote workflow attempted the smoke but stopped before deployment because rg is required", + "cleanup": "PASS", + "image_manifest": "not_generated" + }, + "ruff": { + "status": "FAIL", + "errors": 192, + "classification": "known_baseline" + }, + "mkdocs_strict": { + "status": "NOT_RUN", + "classification": "BLOCKED", + "historical_status": "FAIL", + "historical_warnings": 69 + }, + "canonical_install_docs": { + "status": "NOT_RUN", + "classification": "BLOCKED", + "historical_status": "FAIL" + }, + "workspace_install_docs": { + "status": "NOT_RUN", + "classification": "BLOCKED", + "historical_status": "FAIL" + }, + "pi_user_auth_compose": { + "status": "NOT_RUN", + "classification": "BLOCKED", + "historical_status": "FAIL" + }, + "deployment_coupling": { + "status": "NOT_RUN", + "classification": "BLOCKED", + "historical_status": "FAIL" + }, + "l2": { + "status": "PENDING", + "reason": "configured secret layout unavailable; gate not run after stop" + }, + "manual_psd": { + "status": "PENDING", + "reason": "approved real identity/access unavailable; gate not run after stop" + }, + "provider_readiness": { + "status": "PENDING", + "reason": "provider prerequisite unavailable; gate not run after stop" + } + }, + "review": { + "original_important_findings_resolved": 3, + "fix_round_2_important_lifecycle": "ADDRESSED", + "fix_round_2_minor_windows_diagnostics": "ADDRESSED", + "verdict": "PASS", + "reason": "the lifecycle controller is bounded and cancellation-aware with cancel, bounded join, and lock-release proof; the temporary Windows diagnostic matrix is removed; exact-source native safeio, backup, and authstorage all pass" + }, + "remediation_status": "PASS", + "release_complete": false, + "authentication_implementation_complete": true, + "release_readiness": "FAIL", + "release_readiness_pending_external_gates": true +} diff --git a/.artifacts/task-15/unified-docker-images.json b/.artifacts/task-15/unified-docker-images.json new file mode 100644 index 00000000..4cbfd1fa --- /dev/null +++ b/.artifacts/task-15/unified-docker-images.json @@ -0,0 +1,54 @@ +{ + "gate": "unified-deployment-smoke", + "status": "pass", + "source_commit": "74b062f1a737103524cbe706346cfd65f87cdfd1", + "run_id": "20260818070637-66409-30058", + "images": [ + { + "id": "sha256:2d7b19491c7eb8c119c3cedb390aaeb2ff5593f6fc43ab66c317565560da6d7d", + "roles": [ + "compose-runtime", + "fixture-runtime" + ], + "repo_digests": [ + "sha256:2d7b19491c7eb8c119c3cedb390aaeb2ff5593f6fc43ab66c317565560da6d7d" + ] + }, + { + "id": "sha256:3b6c31a5d8f8fc58fa3233391b6175bd2fbc793eebb44d5e285ecc6e02e9e687", + "roles": [ + "compose-runtime" + ], + "repo_digests": [ + "sha256:3b6c31a5d8f8fc58fa3233391b6175bd2fbc793eebb44d5e285ecc6e02e9e687" + ] + }, + { + "id": "sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a", + "roles": [ + "compose-runtime" + ], + "repo_digests": [ + "sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a" + ] + }, + { + "id": "sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c", + "roles": [ + "compose-runtime" + ], + "repo_digests": [ + "sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" + ] + }, + { + "id": "sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178", + "roles": [ + "rollback-candidate" + ], + "repo_digests": [ + "sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178" + ] + } + ] +} diff --git a/.dockerignore b/.dockerignore index 22d546d3..492b9eef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,17 +1,31 @@ -# build artefacts & deps +# Build artefacts, local configuration, and runtime data never enter an image context. **/node_modules **/.venv **/__pycache__ **/.pytest_cache **/dist **/*.pyc -harness/.env -harness/workspaces/psd.yaml -deploy/thothii.env .git +.worktrees +.thothctl .gitignore +**/.env +**/.env.* +!.env.example +!deploy/env/*.env.example +deploy/thothii.env +deploy/secrets/ +harness/workspaces/*.yaml +!harness/workspaces/local.yaml +!harness/workspaces/tht.example.yaml +!harness/workspaces/tht-test.yaml **/*.log **/.DS_Store -tht-workspace-psd +coverage/ +.coverage +.artifacts/ +data/ +sessions/ +workspace-registry/ # docs/site (mkdocs build) — non necessari nelle immagini docs/superpowers/plans diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..d4bf3f22 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true + +[*.ps1] +end_of_line = crlf diff --git a/.env.example b/.env.example index 3dbeb9b9..84a06a00 100644 --- a/.env.example +++ b/.env.example @@ -1,31 +1,13 @@ -# ThothII Compose defaults. Copy this file to .env in the repository root. -# The root .env is loaded automatically by Docker Compose; do not put secrets here. +# Common non-secret Compose values. Select local.env or server.env with --env-file. +# Run Compose with both files explicitly, for example: +# docker compose --env-file deploy/env/local.env -f compose.yaml -f deploy/compose.local.yaml up -d --build -COMPOSE_FILE=compose.yaml -COMPOSE_PROFILES= -THT_SECRETS_FILE=deploy/secrets/thothii.secrets - -THOTH_HTTP_PORT=8080 -AUTH_MODE=none -THOTH_PUBLIC_EXPOSURE=false MAX_PI_PROCESSES=4 -PI_PROVIDER= -PI_MODEL= -PI_THINKING= -PI_AUTH_FILE=${HOME}/.pi/agent/auth.json +THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth Workspace Registry" +THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid -# Set these for the selected DWH/vector/embedding adapters. -THT_DB_NAME= -THT_DWH_REST_URL= -THT_VEC_REST_URL= -THT_VEC_WRITE_REST_URL= -THT_OLLAMA_URL= -THT_DOCS_ROOT=/data/workspaces/example/evidence-source -THT_PROFILE=server - -# Local-vector defaults (used by the optional local-vector overlay). -THT_VECTOR_DATABASE=thoth -THT_VECTOR_BOOTSTRAP_USER=postgres -THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator -THT_VECTOR_READER_USER=thoth_vector_reader -THT_VECTOR_WRITER_USER=thoth_vector_writer +THT_DB_NAME=warehouse +THT_DWH_REST_URL=https://dwh.example.invalid +THT_LLM_URL=https://llm.example.invalid diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..9cbe4cac --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +* text=auto +*.sh text eol=lf +Dockerfile* text eol=lf +*.Dockerfile text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.py text eol=lf +*.md text eol=lf +*.ps1 text eol=crlf diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml new file mode 100644 index 00000000..200fa58b --- /dev/null +++ b/.github/workflows/deployment.yml @@ -0,0 +1,174 @@ +name: Deployment release gate + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + windows_docker_startup: + description: Run the native self-hosted Windows Docker Desktop/WSL2 release gate + required: false + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: deployment-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + deterministic: + name: LF, Compose, docs, and TypeScript + runs-on: ubuntu-24.04 + timeout-minutes: 25 + env: + PYTHONDONTWRITEBYTECODE: "1" + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24.16.0" + package-manager-cache: false + - name: Verify shell syntax and LF policy + run: | + git ls-files -z '*.sh' | xargs -0 -n1 bash -n + bash scripts/verify-line-endings.sh + - name: Verify Compose and installation contracts + run: | + bash scripts/test-unified-compose.sh + bash scripts/test-no-deployment-coupling-scope.sh + bash scripts/test-compose-secret-policy.sh + bash scripts/test-no-deployment-coupling.sh + bash scripts/test-preprocess-compose-config.sh + bash scripts/test-verify-workspace-install-docs.sh + git diff --check + - name: Assert clean checkout before release trust bootstrap + run: | + git diff --exit-code + git diff --cached --exit-code + test -z "$(git ls-files --others --exclude-standard)" + - name: Verify schema-v3-only release gate + run: bash scripts/verify-schema-v3-only-release.sh + - name: Verify Task 13 clean-install and runtime fixtures + run: | + bash scripts/test-server-pi-state-topology.sh + bash scripts/unified-deployment-smoke.sh --self-test + - name: Test and type-check backend + working-directory: backend + run: | + npx vitest run + npx tsc --noEmit -p . + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + - name: Test and type-check frontend + working-directory: frontend + run: | + npx vitest run + npx tsc -b + + authentication-browser: + name: Hermetic authentication browser gate + needs: deterministic + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24.16.0" + package-manager-cache: false + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: "1.26.5" + cache-dependency-path: tools/tht/go.sum + - name: Install backend dependencies + working-directory: backend + run: npm ci + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + - name: Install Chromium for Playwright + working-directory: frontend + run: npx playwright install --with-deps chromium + - name: Run authentication and authenticated F1 browser smoke + run: bash scripts/authentication-smoke.sh + + linux-docker: + name: Linux Docker deployment and rollback + runs-on: ubuntu-24.04 + timeout-minutes: 100 + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Run unified deployment smoke + run: timeout --signal=TERM --kill-after=45s 32m bash scripts/unified-deployment-smoke.sh + - name: Run tht update smoke + run: timeout --signal=TERM --kill-after=45s 32m bash scripts/tht-update-smoke.sh + - name: Run Linux server deployment smoke + run: timeout --signal=TERM --kill-after=45s 32m bash scripts/server-deployment-smoke.sh + + windows-clone: + name: Windows clone and Compose contract + runs-on: windows-2025 + timeout-minutes: 20 + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24.16.0" + package-manager-cache: false + - name: Install backend dependencies + working-directory: backend + run: npm ci + - name: Verify clean backend distribution + working-directory: backend + run: node --test --test-concurrency=1 scripts/clean-dist.test.mjs + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: "1.26.5" + cache-dependency-path: tools/tht/go.sum + - name: Run native Windows retained-capability tests + working-directory: tools/tht + run: go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1 + - name: Verify Windows clone contract + shell: pwsh + run: ./scripts/test-windows-clone-contract.ps1 + + windows-docker-release: + name: Native Windows Docker Desktop/WSL2 startup + if: github.event_name == 'workflow_dispatch' && inputs.windows_docker_startup + runs-on: [self-hosted, Windows, X64, docker-desktop] + timeout-minutes: 45 + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: "1.26.5" + cache-dependency-path: tools/tht/go.sum + - name: Run spaced-path Windows Docker release gate + shell: pwsh + run: ./scripts/test-windows-clone-contract.ps1 -DockerStartup diff --git a/.gitignore b/.gitignore index 611f876b..bec6ca03 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ Thoth/ # === Visual companion brainstorming artifacts (local-only) === .superpowers/ .worktrees/ +.tht/ # === Python === __pycache__/ @@ -28,6 +29,7 @@ tools/replay/web/ # === Secrets — NEVER commit === .env harness/.env +harness/workspaces/psd.yaml deploy/thothii.env *.pem ca-chain.pem @@ -35,12 +37,20 @@ config/ca-chain.pem # ThothII deployment configuration and secret values (keep only the README tracked) deploy/.env +deploy/compose.connector-secrets.local.yaml deploy/compose.psd-local.yaml deploy/workspaces/psd.yaml deploy/secrets/* !deploy/secrets/README.md !deploy/secrets/*.example +# Per-installation configuration generated by `tht setup` (examples stay tracked). +deploy/*/thothii-installation.yaml +deploy/*/operator.env +deploy/*/secrets/* +!deploy/*/secrets/.gitkeep +!deploy/*/secrets/*.example + # === Runtime data (sessions contain PII; indexes are derived) === harness/sessions/ harness/indexes/ @@ -67,3 +77,6 @@ site/ # Generated container inventory / SBOM-equivalent verification artifacts .artifacts/ + +# === PrimeAgent local project settings (per-user, not shared) === +.prime/ diff --git a/.superpowers/sdd/2026-08-03-diagnostic-contract-extension/task-3-report.md b/.superpowers/sdd/2026-08-03-diagnostic-contract-extension/task-3-report.md new file mode 100644 index 00000000..246794e4 --- /dev/null +++ b/.superpowers/sdd/2026-08-03-diagnostic-contract-extension/task-3-report.md @@ -0,0 +1,105 @@ +# Task 3 — Diagnostic contract remediation report + +Date: 2026-08-04 + +## Scope + +This remediation is limited to the four approved review findings for the workspace diagnostic +extension. It does not add registry routes, change workspace publication, alter session startup, +or expand transport support. + +## Changes + +1. `RuntimeBindings` now has an explicit `vectorWriter` binding. The new + `resolveRuntimeBindings()` resolves DWH, vector reader, vector writer, and embedding bindings + together. The diagnoser takes the writer credential only from `bindings.vectorWriter`, never + from vector-reader values. +2. Direct PostgreSQL and SSH-tunnelled direct probes accept an absent CA binding while retaining + certificate verification through the runtime system trust store. A supplied CA still uses + verified private-CA trust. REST private-CA refusal is unchanged. +3. A reversible vector probe now requires an authenticated POST declaration with a response map + containing `operation`. The adapter requires the successful JSON response to echo `create` or + `remove` respectively, so an arbitrary 2xx or an upsert-only response cannot activate the + write probe. +4. For DWH and vector REST diagnostics declared with `auth: none`, the resolver no longer + requires an API-key file and the adapter sends no credential. Credential-backed diagnostics + continue to require their local secret file. + +## TDD evidence + +The first focused RED run failed for the intended missing behavior: + +- `resolveRuntimeBindings is not a function` for unauthenticated resolver bindings; +- schema accepted a reversible probe without a response contract; and +- existing diagnostic fixtures rejected the new `response` declaration until schema support was + implemented. + +The focused GREEN run passed `43/43` tests across: + +- `test/workspaces-bindings.test.ts` +- `test/workspaces-schema.test.ts` +- `test/workspaces-diagnostics.test.ts` + +The regression coverage includes resolver-to-diagnoser writer propagation without manually +inserting the writer key into vector-reader bindings, no-CA direct/SSH system-trust requests, +operation-echo validation for create/remove, and `auth: none` bindings without secret files. + +## Documentation and design + +- `docs/workspace-diagnostic-protocol.md` now documents the verified system-trust fallback, + no-secret `auth: none` behavior, and required reversible response contract. +- `docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md` now records the same + response, CA, SSH, and authentication rules. + +## Final verification + +The initial sandboxed full suite could not bind its local SSE listener (`listen EPERM: +operation not permitted 127.0.0.1`). It was rerun unchanged with local-listener permission. + +```text +backend: npx vitest run +31 test files passed; 329 tests passed + +backend: npx tsc --noEmit -p . +exit 0 + +repository: git diff --check +exit 0 +``` + +Expected test harness stderr from existing Pi/process failure-path tests remained present; no test +failed and no diagnostic secret was emitted. + +## Blockers + +None. + +## Round 2 remediation + +The final review found two remaining contract gaps. The binding resolver already treated +`auth: none` as credential-free, but the runtime renderer and diagnostic connector still required +the API-key file. Rendering and connector construction now make that requirement conditional on +the declared REST authentication mode, so a DWH/vector `auth: none` workspace passes resolver, +runtime rendering, and diagnostics with no API-key file. + +SSH forwarding previously changed the PostgreSQL connection host to `127.0.0.1` without retaining +the original target for TLS hostname validation. Forwarded probes now carry `SSH_TARGET_HOST` as +`tlsServername` into the PostgreSQL TLS options; private CA and verified system trust behavior are +unchanged. + +TDD RED: the new end-to-end no-key test failed at the unconditional runtime +`API_KEY_FILE` requirement, while the SSH test showed no `tlsServername` on the loopback probe or +database-client request. TDD GREEN: the focused backend workspace tests passed `40/40`. + +Round 2 final verification: + +```text +backend: npx vitest run +31 test files passed; 332 tests passed + +backend: npx tsc --noEmit -p . +exit 0 + +repository: git diff --check +exit 0 +``` diff --git a/.superpowers/sdd/2026-08-03-git-workspace-registry/task-7-report.md b/.superpowers/sdd/2026-08-03-git-workspace-registry/task-7-report.md new file mode 100644 index 00000000..53111d0d --- /dev/null +++ b/.superpowers/sdd/2026-08-03-git-workspace-registry/task-7-report.md @@ -0,0 +1,101 @@ +# Task 7 report — revision-pinned sessions + +## Delivered + +- New-session requests may carry `workspaceId`, provider, model, and thinking. The backend + resolves the active operational registry revision, enforces its LLM policy, and persists the + workspace ID/revision with the selected LLM settings. +- The harness manifest and `tht session new` support the optional, backward-compatible + `workspace_id` and `workspace_revision` fields. +- Resume resolves the manifest's retained snapshot, including after later registry publication. + A missing retained revision returns a sanitized `workspace_revision_unavailable` response. + Legacy manifests retain the prior workspace behavior and are marked with a visible warning on + `GET /sessions/:id`. +- `/settings` is now a non-mutating compatibility endpoint: installation defaults remain + readable, while anonymous workspace/provider/model/thinking selections are no longer written + to backend settings or principal preferences. + +## TDD evidence + +- RED: `npx vitest run test/routes-sessions.test.ts test/routes-settings.test.ts` failed for the + new immutable-snapshot and no-settings-mutation assertions; the manifest test failed because + `new_session_manifest` did not accept workspace revision fields. +- GREEN: `npx vitest run test/tht-runner.test.ts test/routes-sessions.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .` + completed with 97 passing tests and a clean type check. +- GREEN: `THT_HOME=/private/tmp/thothii-task7-home .venv/bin/pytest tests/test_session_documents.py tests/test_session_mutations.py -q` + completed with 22 passing tests. +- `git diff --check` completed cleanly. + +## Review fixes — round 3 + +- The active registry snapshot that located a session now remains the authorization and mutation + config for response, steer, events, close/delete, archive/group/rename, documents, and detail. + A pruned historical revision cannot block an already-located session's active lifecycle. +- Only Resume resolves the retained pinned descriptor because Pi needs that immutable config to + restart safely. A pruned pin therefore returns the existing sanitized + `workspace_revision_unavailable` 409 solely for Resume. + +### Round 3 verification + +- RED: with a manifest found through an active registry snapshot and `readPinned` forced to fail, + `POST /sessions/:id/response` returned 409 instead of forwarding the active gate response. +- GREEN: `npx vitest run test/routes-sessions.test.ts test/tht-runner.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .` + — 102 tests passed with a clean type check. The regression confirms response, close, and delete + use the locating snapshot without calling `readPinned`, while Resume returns a sanitized 409. +- `git diff --check` completed cleanly. + +## Review fixes — round 2 + +- Lifecycle authorization no longer selects the installation-default workspace. The backend now + finds each session by querying every operational registry snapshot with the authenticated + principal, preserving RLS ownership concealment. +- After locating the manifest, durable pinned sessions resolve their retained descriptor before + any lifecycle mutation/reopen. Legacy sessions continue using the locating registry snapshot. +- Session listing aggregates the owner-visible rows from all operational registry snapshots; + detail, response, steer, resume, events, documents, and lifecycle mutations use the same + server-side locator. No route depends on browser-local workspace state. + +### Round 2 verification + +- RED: the new cross-workspace route integration test created a B session while installation + default A was selected, then demonstrated that `GET /sessions` returned an empty list. +- GREEN: `npx vitest run test/routes-sessions.test.ts test/tht-runner.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .` + — 101 tests passed with a clean type check. The integration test covers create B, list, detail, + response, and resume through B's pinned descriptor while default A remains configured. +- Full backend suite: 342 tests passed. The remaining 7 tests require binding `127.0.0.1` and + fail in this sandbox with `listen EPERM: operation not permitted`; no application assertion + failed. The focused typecheck above passed. +- `git diff --check` completed cleanly. + +## Verification note + +The unscoped backend suite was also run. The Task 7 code regressions in `test/tht-runner.test.ts` +were fixed; the remaining failures were existing sandbox restrictions on tests that listen on +`127.0.0.1` (`listen EPERM: operation not permitted` in SSE/e2e health tests), not application +assertions. + +## Review fixes — round 1 + +- Every new session now resolves `workspaceId` through the registry; an omitted value uses the + configured installation default and persists both the resolved ID and revision. Callers cannot + bypass revision pinning by supplying a workspace ID. +- Browser-local preferences now migrate once from the read-only legacy settings response and hold + workspace, provider, model, and thinking. Session creation includes those selections, including + direct entry points that run before the composer mounts. The frontend no longer `PUT`s shared + settings. +- The settings compatibility endpoint honors a stored installation workspace before falling back + to the first workspace configuration. +- Resume rejects finalized and archived sessions before looking up any pinned snapshot, preserving + the read-only response even when a historical snapshot is unavailable. + +### Review verification + +- RED: the added backend tests failed for omitted-default pinning, read-only resume ordering, and + stored-default precedence; the added frontend preference tests failed because preferences were + neither stored nor included in session requests. +- GREEN: `npx vitest run test/tht-runner.test.ts test/routes-sessions.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .` + — 100 tests passed with a clean type check. +- GREEN: `npx vitest run && npx tsc -b` — 332 frontend tests passed with a clean type check. +- GREEN: `THT_HOME=/private/tmp/thothii-task7-home .venv/bin/pytest tests/test_session_documents.py tests/test_session_mutations.py -q` + — 22 tests passed (one existing testcontainers deprecation warning). +- `git diff --check` completed cleanly. diff --git a/.superpowers/sdd/2026-08-03-git-workspace-registry/task-9-report.md b/.superpowers/sdd/2026-08-03-git-workspace-registry/task-9-report.md new file mode 100644 index 00000000..03b48d80 --- /dev/null +++ b/.superpowers/sdd/2026-08-03-git-workspace-registry/task-9-report.md @@ -0,0 +1,73 @@ +# Task 9 report — Workspace Management CRUD page + +## Delivered + +- Added the Workspace management dialog, launched from the persistent right sidebar and the + Model activity header without touching live-session/SSE state. +- Added a workspace list/detail editor for General, DWH, Semantic index, LLM policy, + Installation requirements, and Git status/history. +- Added browser-only New, Edit, Duplicate, Save draft, and Delete-draft workflows. A deletion + draft stores only ID and immutable revision references; publication remains a Task 10 action. +- Used closed native controls for languages, engines, transports, distance metrics, embedding + providers, and selectable default models. Free values have client-side, accessible errors. +- Made semantic-index dimensions atomic: one editor field always writes the same value to the + vector-store and embedding contracts. +- Added Validate and Test-on-this-installation actions. They display sanitized code/message + diagnostics only; neither action exposes or stores credentials, secrets, or raw response bodies. +- Explicitly excluded publish, pull, import, and export user flows from this task. + +## TDD evidence + +- RED: `npx vitest run src/shell/WorkspaceManager.test.tsx src/shell/WorkspaceEditor.test.tsx` + failed because the manager and editor modules did not exist. +- GREEN: focused manager/editor/AppShell coverage passed after the implementation. +- RED: a deletion-draft persistence regression failed with + `Cannot read properties of undefined (reading 'save')` before the sanitized draft store was added. +- GREEN: the draft-store and manager tests passed once deletion intent persisted locally. + +## Verification + +Executed from `frontend/`: + +```text +npx vitest run +50 test files passed, 358 tests passed +npx tsc -b +exit 0 +``` + +`git diff --check` passed before commit. No workspace secret value, secret-file path, raw +diagnostic body, publish call, import flow, or export flow was introduced. + +## Fix round 1 + +### Root causes and fixes + +- The original duplicate proposal appended `-copy` and then truncated at 63 characters. For an + already-maximal ID, truncation could remove the suffix and reproduce the immutable source ID. + The proposal now reserves suffix space and falls back to a distinct `-2` suffix when a maximal + source already ends in `-copy`. +- `dwh.timeout_ms` was rendered as a positive numeric field but was absent from the client + validation map. It now has the same immediate accessible error treatment as other numeric + fields, so a rejected save never reaches the manager’s saved-draft toast. +- Registry status, workspace list, and selected-detail React Query failures were rendered as + loading, empty, or unselected states. Each now has a named alert and a retry control, distinct + from its corresponding loading and empty state. + +### TDD evidence + +- RED: max-length duplication retained the original 63-character ID; the timeout field produced + no alert; and each of the three failed queries had no accessible retry control. +- GREEN: the focused manager/editor tests passed **12/12**, covering a valid changed duplicate + proposal, rejected zero timeout with no save toast, and status/list/detail retry recovery. + +### Verification + +Executed from `frontend/`: + +```text +npx vitest run +50 test files passed, 364 tests passed +npx tsc -b +exit 0 +``` diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-11-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-11-report.md new file mode 100644 index 00000000..b9c381de --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-11-report.md @@ -0,0 +1,180 @@ +# Task 11 report + +Status: completed on 2026-08-08. + +## Scope delivered + +- Updated operator-facing documentation for the internal Qdrant + Ollama architecture. +- Tightened documentation contract tests to require the current four-service-plus-init topology, + CPU-first/GPU-override guidance, fixed internal model/dimensions, schema-v3 migration wording, + one-collection-per-workspace ownership, and Qdrant backup/restore safety. +- Updated stable repo guidance in `AGENTS.md` and the current snapshot in `PROJECT_STATE.md`. +- Rewrote the workspace diagnostic protocol to the schema-v3/internal-semantic-service contract. +- Updated the memory guide to describe Qdrant as the derived persistent index. +- Updated the runtime secret-bundle guide to remove active vector/embedding secret guidance. + +## Files changed + +- `README.md` +- `AGENTS.md` +- `PROJECT_STATE.md` +- `docs/install/local-workspace-registry.md` +- `docs/install/server-workspace-registry.md` +- `docs/installazione-docker-4-contesti.md` +- `docs/workspace-diagnostic-protocol.md` +- `docs/gestione-memory.md` +- `deploy/secrets/README.md` +- `scripts/verify-workspace-install-docs.sh` +- `scripts/test-verify-workspace-install-docs.sh` + +## Verification + +Fresh successful runs: + +```sh +./scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +git diff --check +``` + +Key outcomes: + +- internal semantic infrastructure documentation contract passed +- all existing install/manual fixture contracts still passed +- diff hygiene passed with no whitespace/errors + +## Self-review notes + +- The updated docs now match the code-backed Compose topology: `frontend`, `core`, `qdrant`, + `embedding`, and `embedding-model-init`. +- Active manuals no longer instruct operators to configure external vector or embedding runtime + endpoints/secrets. +- Qdrant backup/restore wording now matches the helper scripts' exact confirmation and rollback + behavior. +- Legacy descriptor handling is documented as explicit schema-v3 migration only; no silent + semantic-data migration is claimed. + +## Residual concerns + +- The broader repository still contains historical design/spec material that references older + pgvector/external-embedding architecture; this task intentionally updated operator/current-state + documentation and the corresponding contract tests, not historical planning documents. + +## Fix round 1/5 — 2026-08-08 + +Addressed reviewer findings: + +- Moved superseded rollout/state blocks in `PROJECT_STATE.md` behind an explicit + `## Historical snapshots and archived reference notes` boundary. +- Renamed superseded snapshot headings so historical notes no longer present as active `LIVE` + state. +- Added a current-state regression that rejects contradictory active blocks (for example: + schema-v2 operational, two-service active stack, or external vector/embedding runtime claims + before the historical boundary). +- Refactored new internal-semantic doc checks away from exact-sentence coupling: + - parse `compose.yaml` structurally with YAML; + - parse workspace examples structurally with YAML; + - inspect backup/restore stable usage interface; + - keep targeted forbidden-term checks for active docs while allowing historical sections; + - use regex/concept checks for prose. + +Evidence: + +```sh +./scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +git diff --check +``` + +Observed RED before the fix: + +```text +PROJECT_STATE.md: missing Historical snapshots boundary +``` + +## Fix round 2/5 — 2026-08-08 + +Addressed reviewer findings: + +- Renamed every historical `PROJECT_STATE.md` heading after the historical boundary so no heading + level uses `LIVE` or current-state semantics there. +- Strengthened the historical-boundary regression to reject any Markdown heading level + (`#` through `######`) containing `LIVE` or current-state wording after the boundary. +- Added a fixture with a `### ... — LIVE ...` historical heading to prove RED then GREEN. +- Replaced remaining exact phrase checks with concept/semantic validation for: + - one-workspace/one-collection ownership; + - external boundary (DWH/LLM external; vector/embedding internal); + - the Italian compact install note. +- Added paraphrase fixtures that pass and omission/inversion fixtures that fail. + +Evidence: + +```sh +./scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +git diff --check +``` + +## Fix round 4/5 — 2026-08-08 + +Addressed reviewer finding: + +- Eliminated semantic-index verifier/test contract drift by extracting the production + semantic-index ownership row matcher into `semantic_index_relationship_spec` and reusing it in + the fixture-level paraphrase, omission, and scattered-token checks. +- Kept the relationship constrained to one structured Markdown table row via + `verify_markdown_table_relationships`; the scattered-token fixture still removes the row and + appends the same words outside the table, where it must be rejected. +- Added a direct regression that copies the repository docs into an isolated root, applies the + accepted paraphrase “A workspace keeps exactly one Qdrant collection reserved for itself”, and + runs that root's actual `scripts/verify-workspace-install-docs.sh --fixtures-only` instead of a + separate temporary spec. + +Observed RED before the fix: + +```text +production verifier rejected the accepted semantic-index paraphrase +local workspace manual: missing relationship in 'Semantic index ownership contract': {'scope': 'workspace semantic index', 'ownership rule': '(each|one|single).*(workspace).*(single|one).*(Qdrant).*(collection)|(each workspace reserves a single qdrant collection)', 'isolation rule': 'schema.*evidence.*memory.*(one|that).*(collection).*(kind|payload)'} +``` + +Evidence: + +```sh +./scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +git diff --check +``` + +Observed RED during this round: + +```text +PROJECT_STATE.md: historical section still contains active/live heading markers +compact manual paraphrase lacks required pattern: (esterni solo|solo esterni|restano esterni) +``` + +## Fix round 3/5 — 2026-08-08 + +Addressed reviewer findings: + +- Added table-driven historical-heading fixtures for every Markdown heading level `#` through + `######`; all are rejected after the historical boundary when they contain `LIVE`/current-state + semantics. +- Added small structured ownership tables to the active local/server manuals and to the compact + Italian operator note. +- Added small structured semantic-index ownership tables to the active local/server manuals. +- Replaced the remaining scattered-token relationship checks with explicit structured-section + parsing: + - architecture ownership rows map DWH → external, LLM → external, Qdrant → internal, + Ollama embedding → internal; + - semantic-index ownership rows localize the one-workspace/one-collection contract and the + schema/Evidence/Memory isolation rule. +- Added adversarial fixtures that fail when the same tokens are merely scattered in free text. +- Added structured paraphrase fixtures that pass and omission/inversion fixtures that fail. + +Evidence: + +```sh +./scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +git diff --check +``` diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-12-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-12-report.md new file mode 100644 index 00000000..1ea763f5 --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-12-report.md @@ -0,0 +1,43 @@ +# Task 12 Report — Remove unreachable pgvector runtime code + +Status: completed + +Summary: +- Proved the retired pgvector runtime had no remaining operational adapter call sites after migration by re-running the required grep; only the packaging assertion still mentions `migrations/vector`. +- Removed the obsolete pgvector/HTTP/direct vector runtime modules, vector SQL migrations, and their affected runtime tests. +- Kept the operational semantic path on Qdrant and migrated the remaining runtime callers to that path. +- Kept `psycopg2-binary` because DWH direct PostgreSQL and session PostgreSQL code still depend on it. + +Implementation notes: +- Extracted shared collection/kind validation into `harness/tht/adapters/vector/_shared.py` so `QdrantVectorStore` no longer depends on the deleted pgvector module. +- Simplified `build_vector_store()` to return only `QdrantVectorStore`. +- Migrated vector/evidence/memory CLI paths away from legacy pgvector loaders and REST vector clients. +- Updated packaging coverage so the built wheel asserts session SQL migrations are present and vector SQL migrations are absent. + +Verification: +- `cd harness && .venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py tests/test_semantic_kind_isolation.py tests/test_vector_migration_packaging.py -q` +- `cd harness && .venv/bin/pytest tests/test_adapter_factory.py tests/test_solved_search_cli.py -q` +- `cd harness && .venv/bin/python -c "import tht.cli, tht.adapters.factory, tht.adapters.vector, tht.vectorstore.reader"` +- `cd harness && uv build` +- `harness/.venv/bin/ruff check harness/tests/test_adapter_factory.py harness/tests/test_solved_search_cli.py harness/tests/test_vector_migration_packaging.py harness/tests/test_vector_port_contract.py harness/tht/adapters/factory.py harness/tht/adapters/vector/__init__.py harness/tht/adapters/vector/_shared.py harness/tht/adapters/vector/qdrant.py harness/tht/cli/evidence_cmd.py harness/tht/cli/memory_cmd.py harness/tht/cli/search_cmd.py harness/tht/cli/vector_cmd.py harness/tht/solved.py harness/tht/vectorstore/reader.py` +- `git diff --check` + +Notes / concerns: +- Repository-wide `harness/.venv/bin/ruff check .` still reports many pre-existing findings outside this task’s touched files; it is not clean on this branch baseline. +- Some legacy config compatibility parsing still exists outside the deleted runtime path. This task removed the unreachable runtime/migration code without broad config-schema refactoring. + +## Fix round 1 evidence + +Changes: +- Removed dead `vector migrate` registration from `harness/tht/cli/__init__.py` and deleted `harness/tht/cli/vector_migrate_cmd.py`. +- Added CLI regressions proving `vector migrate` is absent while `vector init` and `vector index-schema` remain available. +- Restored the accidentally removed non-vector regressions by moving report coverage into `harness/tests/test_report.py` and restoring the taskdoc promoted-table slicing check in `harness/tests/test_taskdoc.py`. +- Reworded surviving active help/docstrings away from pgvector-specific wording in the touched Qdrant-backed command surface. + +Verification: +- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_report.py tests/test_taskdoc.py tests/test_vector_migration_packaging.py -q` +- `cd harness && .venv/bin/python -c "from typer.testing import CliRunner; from tht.cli import app; r=CliRunner().invoke(app, ['vector','--help']); assert r.exit_code == 0, r.output; assert 'migrate' not in r.output; r=CliRunner().invoke(app, ['vector','migrate','--help']); assert r.exit_code != 0, r.output; print('cli-help-ok')"` +- `cd harness && .venv/bin/python -c "import tht.cli, tht.cli.vector_cmd, tht.report, tht.taskdoc; print('imports-ok')"` +- `cd harness && uv build` +- `harness/.venv/bin/ruff check harness/tests/test_qdrant_cli_commands.py harness/tests/test_report.py harness/tests/test_taskdoc.py harness/tests/test_vector_migration_packaging.py harness/tht/cli/__init__.py harness/tht/cli/search_cmd.py harness/tht/cli/vector_cmd.py harness/tht/cli/memory_cmd.py harness/tht/solved.py` +- `git diff --check` diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-13-implementation.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-13-implementation.md new file mode 100644 index 00000000..9f798a3a --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-13-implementation.md @@ -0,0 +1,175 @@ +# Task 13 Implementation Report + +## Status + +DONE_WITH_CONCERNS + +## Changes + +- Updated stale harness/backend/frontend tests and fixtures to the Task 13 internal Qdrant/Ollama contract. +- Made `deploy/workspaces/psd.yaml.example` generic while preserving schema-v3 Qdrant/Ollama shape. +- Fixed `scripts/workspace-registry-smoke.sh` to pass the required legacy migration `--collection` and prove exact Docker cleanup, including its smoke image. +- Updated `PROJECT_STATE.md` with only evidence observed in this run. + +Changed files: + +- `PROJECT_STATE.md` +- `backend/test/routes-workspaces.test.ts` +- `backend/test/workspace-runtime-handoff.test.ts` +- `backend/test/workspaces-contracts.test.ts` +- `backend/test/workspaces-git-repository.test.ts` +- `deploy/workspaces/psd.yaml.example` +- `frontend/src/shell/NewSessionDialog.test.tsx` +- `harness/tests/test_adapter_command_regressions.py` +- `harness/tests/test_workspace.py` +- `scripts/task13-runtime-fixture-check.ts` +- `scripts/test-verify-workspace-install-docs.sh` +- `scripts/workspace-registry-smoke.sh` + +## Verification + +Deterministic gates: + +- `cd harness && .venv/bin/pytest -q && .venv/bin/ruff check .` + - Initial red: 2 harness pytest failures. + - After fixture fixes: harness pytest passed `819 passed, 4 deselected, 74 warnings in 27.73s`. + - Ruff still failed with `Found 220 errors`; treated as existing unrelated debt. + - Touched harness files verified clean with `cd harness && .venv/bin/ruff check tests/test_adapter_command_regressions.py tests/test_workspace.py && .venv/bin/pytest -q tests/test_adapter_command_regressions.py::test_solved_index_writes_through_writer_only_factory_store tests/test_workspace.py::test_load_workspace_expands_env_vars`: `All checks passed!` and `2 passed, 2 warnings in 0.14s`. +- `cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build` + - Initial red: 4 backend Vitest failures. + - After fixes: `Test Files 39 passed (39)`, `Tests 464 passed (464)`, TypeScript passed, build passed. +- `cd frontend && npx vitest run && npx tsc -b && npm run build` + - Initial red: 1 frontend Vitest failure. + - After fix: frontend Vitest passed `374/374`, TypeScript passed, build passed with Vite `built in 6.55s`. +- `git diff --check` + - Passed with no output. + +Focused reruns: + +- `cd backend && npx vitest run test/workspaces-migrate-legacy.test.ts test/workspaces-contracts.test.ts test/routes-workspaces.test.ts test/workspace-runtime-handoff.test.ts test/workspaces-git-repository.test.ts && cd .. && ./scripts/test-no-deployment-coupling.sh && ./scripts/verify-workspace-install-docs.sh --fixtures-only && git diff --check` + - `Test Files 5 passed (5)`, `Tests 35 passed (35)`. + - Coupling guard passed: `no active retired deployment or external semantic coupling found.` + - Install docs fixtures passed through `relative secret-source fixture rejected passed`. + +Deployment contracts: + +- `./scripts/test-default-compose.sh && ./scripts/test-unified-compose.sh && ./scripts/test-internal-semantic-compose.sh && ./scripts/test-no-deployment-coupling.sh && ./scripts/test-compose-secret-policy.sh && ./scripts/verify-workspace-install-docs.sh --fixtures-only` + - Passed. Output included: + - `default Compose contract passed.` + - `unified Compose contract passed.` + - `internal semantic Compose/script contracts passed.` + - `no active retired deployment or external semantic coupling found.` + - `Compose secret policy passed.` + - install-doc fixture checks through `relative secret-source fixture rejected passed`. + +Docker smokes: + +- `/usr/bin/time -p ./scripts/internal-semantic-smoke.sh` + - Passed: `Task 13 internal semantic smoke passed.` + - Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808200245-83368-17823.` + - Duration: `real 217.34`. +- `/usr/bin/time -p ./scripts/workspace-registry-smoke.sh` + - Initial red: `usage: migrate-legacy --input --output --collection [--id ]`. + - After fix: `workspace registry smoke passed`. + - Cleanup proof: `no compose containers, volumes, networks, or image remain for thoth-workspace-registry-smoke-89671.` + - Duration: `real 9.93`. +- `/usr/bin/time -p ./scripts/unified-deployment-smoke.sh` + - Passed: `Task 13 full deployment smoke passed.` + - Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808200706-85638-13391.` + - Duration: `real 125.57`. +- `/usr/bin/time -p ./scripts/thothctl-update-smoke.sh` + - Passed: `Task 13 update deployment smoke passed.` + - Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808200918-87340-10404.` + - Duration: `real 85.40`. +- `/usr/bin/time -p ./scripts/server-deployment-smoke.sh` + - Passed: `Task 13 Linux server deployment smoke passed.` + - Cleanup proof: `no labeled containers, volumes, networks, or images remain for 20260808201047-88645-20675.` + - Duration: `real 55.99`. + +Final audit: + +- `rg -n "pgvector|local-vector|THT_VECTOR_|EMBEDDING_BASE_URL|openai_compatible|ollama_compatible" . --glob '!docs/plans/**' --glob '!docs/superpowers/**' --glob '!**/node_modules/**' --glob '!**/.venv/**' --glob '!**/.git/**'` + - Returned matches in legacy schema-v1/v2 support, migration tests, negative guards, historical notes, and older harness docs/code. + - This remains a concern: the audit is not clean under the brief's strict expected outcome. +- `git status --short` + - Before report/commit, contained only intentional Task 13 changes. + +## Image and Host Evidence + +- Host CPU: `Apple M4 Pro`. +- Host OS: `Darwin MacProM4-di-Marco.local 25.5.0 Darwin Kernel Version 25.5.0: Tue Jun 9 22:28:34 PDT 2026; root:xnu-12377.121.10~1/RELEASE_ARM64_T6041 arm64`. +- Docker server: `29.6.2 linux/arm64`. +- Verified pinned images: + - `qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c`. + - `ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a`. +- Workspace registry smoke ephemeral image: + - Manifest list: `sha256:4d056bf2cb38d0e8ede91fbf121df1f9f18caee0d401581618ccef9ed8a55e73`. + - Config: `sha256:613f8fb28c0517adee4085f41bc447f2c3813b0fdbb7b26624bfb4cb192b6fd8`. + - Removed during cleanup. + +## Manual Gates + +- GPU exposure gate (`THOTH_ENABLE_EMBEDDING_GPU=1` on Linux): not executed in this run. +- Windows Docker Desktop startup/manual job: not executed in this run. + +## Commits + +- `4e810af` (`test: align qdrant ollama verification fixtures`) +- `7c09b98` (`docs: record qdrant ollama verification`) + +## Known Limitations + +- Broad harness Ruff remains existing unrelated debt: `Found 220 errors`. +- Final active-reference audit is not clean; it still finds legacy/negative-guard references outside explicit migration fixture files. +- Ephemeral Task 13 core/frontend image IDs from `internal-semantic-smoke.sh`, `unified-deployment-smoke.sh`, `thothctl-update-smoke.sh`, and `server-deployment-smoke.sh` were removed by exact cleanup and were not emitted in stdout; pinned Qdrant/Ollama digests and the workspace-registry smoke image digest were captured. + +## Fix Round 1 — reviewer findings + +Status: DONE + +Changes: + +- `scripts/workspace-registry-smoke.sh` now derives the smoke image reference from the already unique Compose project instead of using the global tag `thothii-workspace-registry-smoke:local`. +- The workspace-registry cleanup helpers remove and verify only the exact per-run image reference, plus Compose resources labeled with the exact project. +- Added deterministic self-test coverage in `backend/test/workspaces-migrate-legacy.test.ts` via `WORKSPACE_REGISTRY_SMOKE_SELF_TEST=image-cleanup-identity`; it stubs Docker and fails if cleanup touches same-repository foreign tags such as `:local` or another project tag. +- Updated active harness/testing/PRD docs and Python comments that still described the current semantic store as pgvector/vectordb. Preserved schema-v1/v2 and harness legacy compatibility fixtures. +- Updated `PROJECT_STATE.md` with fix-round smoke evidence and a precise, non-overclaiming audit limitation. + +Focused verification: + +- `cd backend && npx vitest run test/workspaces-migrate-legacy.test.ts` + - Passed: `7 passed`. +- `cd harness && .venv/bin/pytest -q tests/test_memory_save_one.py tests/test_adapter_command_regressions.py tests/test_solved_search_cli.py tests/test_search_pack.py` + - Passed: `22 passed, 14 warnings`. +- `cd harness && .venv/bin/ruff check tht/memory.py tht/search/__init__.py tht/workspace.py tht/vectorstore/store.py tests/test_memory_save_one.py tests/test_adapter_command_regressions.py tests/test_solved_search_cli.py` + - Passed: `All checks passed!` +- `bash -n scripts/workspace-registry-smoke.sh && WORKSPACE_REGISTRY_SMOKE_SELF_TEST=image-cleanup-identity bash scripts/workspace-registry-smoke.sh` + - Passed: `workspace registry smoke image cleanup identity self-test passed`. +- `./scripts/test-no-deployment-coupling.sh` + - Passed: `no active retired deployment or external semantic coupling found.` +- `./scripts/verify-workspace-install-docs.sh --fixtures-only` + - Passed through `relative secret-source fixture rejected passed`. +- `cd backend && npx tsc --noEmit -p .` + - Passed with no output. +- `/usr/bin/time -p ./scripts/workspace-registry-smoke.sh` + - Passed: `workspace registry smoke passed`. + - Built exact per-run tag: `thothii-workspace-registry-smoke:thoth-workspace-registry-smoke-thoth-workspace-registry-smoke-10vi3a-19157`. + - Manifest list: `sha256:715b943057929418cad4aa71806d9edbaf823555d19bda6b875297617463fd4a`. + - Config: `sha256:a566521981e08958aae9a12bfc7803bb5f3f835536b4bb8c39df8fcf26063161`. + - Cleanup proof: `no compose containers, volumes, networks, or image remain for thoth-workspace-registry-smoke-thoth-workspace-registry-smoke-10vi3a-19157.` + - Duration: `real 42.06`. + +Fix-round audit command: + +- `rg -n "pgvector|local-vector|THT_VECTOR_|EMBEDDING_BASE_URL|openai_compatible|ollama_compatible" . --glob '!docs/plans/**' --glob '!docs/superpowers/**' --glob '!**/node_modules/**' --glob '!**/.venv/**' --glob '!**/.git/**'` + +Categorized remaining hits: + +- Backend legacy parser/migration compatibility, kept deliberately non-operational for schema-v1/v2 descriptors: `backend/src/workspaces/schema.ts`, `types.ts`, `migrate-legacy.ts`, `runtime-renderer.ts`, `bindings.ts`, `contracts.ts`, `diagnostics.ts`. +- Backend negative guards and legacy fixture tests: `backend/test/workspaces-schema.test.ts`, `workspaces-migrate-v2-qdrant.test.ts`, `workspace-registry.test.ts`, `workspace-runtime-renderer.test.ts`, `workspaces-bindings.test.ts`, `workspaces-contracts.test.ts`, `workspaces-diagnostics.test.ts`, `workspaces-git-repository.test.ts`, `routes-workspaces.test.ts`, `routes-sessions.test.ts`, `provider-credentials.test.ts`. +- Secret/env scrub guards for retired variables: `backend/src/config.ts`, `backend/src/config/secret-bundle.ts`, `backend/src/pi/provider-credentials.ts`, `scripts/compose-with-preflight.sh`, `scripts/test-external-compose-lifecycle.sh`. +- Deployment negative guards and fixture-scope tests: `scripts/test-no-deployment-coupling.sh`, `scripts/test-no-deployment-coupling-scope.sh`, `scripts/test-preprocess-compose-config.sh`, `scripts/test-verify-workspace-install-docs.sh`, `scripts/verify-workspace-install-docs.sh`, `scripts/vector-rotate-bootstrap-password.sh`. +- Harness legacy config compatibility and fixtures: `harness/tht/config.py`, `harness/tht/config_compat.py`, `harness/tests/test_config_resources.py`, `harness/tests/l2/test_session_ablazione.py`, `harness/workspaces/tht.example.yaml`, `harness/workspaces/tht-test.yaml`. +- Retained off-repository migration SQL fixtures: `harness/scripts/create_vector_reader_rpc.sql`, `harness/scripts/create_vector_writer_rpc.sql`. +- Historical/reference notes, not active operator contracts: `brain/codebase/datamart-builder-deployment-gotchas.md`, `PROJECT_STATE.md`. +- Gitignored task report self-reference: `.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-13-implementation.md`. diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-2-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-2-report.md new file mode 100644 index 00000000..9f88efee --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-2-report.md @@ -0,0 +1,165 @@ +Task 2 report — Make collection ownership unique in the Git registry + +Summary + +- Implemented unique Qdrant collection ownership enforcement during registry snapshot activation. +- Registry session revision leases now reject `migration_required` descriptors. +- Legacy migration now requires an explicit target collection and emits schema v3 descriptors. +- Preserved active snapshot rollback behavior on invalid pulled snapshots. + +RED evidence + +Focused RED command from the brief: + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \ + -t "collection|migration_required" +``` + +Observed failures before implementation: + +- `rejects duplicate schema v3 collection ownership and keeps the previous active snapshot` + - `registry.pull()` resolved instead of rejecting. +- `does not acquire a session revision lease for a migration_required workspace` + - `acquireSessionRevision()` resolved instead of rejecting. +- `migrates a legacy descriptor only with an explicit target collection into schema v3` + - received schema version `1` instead of `3`. +- `requires an explicit target collection for legacy migration` + - migration did not throw without a collection. + +GREEN evidence + +Focused GREEN command from the brief: + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \ + -t "collection|migration_required" +``` + +Fresh result after implementation: + +- 2 files passed +- 4 tests passed +- 0 failures + +Additional verification run after final cleanup: + +```bash +cd backend +npx vitest run test/routes-workspaces.test.ts +npx vitest run +npx tsc --noEmit -p . +git diff --check +``` + +Fresh results: + +- `test/routes-workspaces.test.ts`: 7 passed +- full backend Vitest: 39 files passed, 454 tests passed +- backend typecheck: passed +- `git diff --check`: passed + +Changed files + +- `backend/src/workspaces/registry.ts` +- `backend/src/workspaces/migrate-legacy.ts` +- `backend/test/workspace-registry.test.ts` +- `backend/test/workspaces-migrate-legacy.test.ts` +- `backend/test/routes-workspaces.test.ts` + +Why one extra file changed + +- `backend/test/routes-workspaces.test.ts` needed updating because Task 1 made schema v3 the only operational descriptor shape, and the route test still assumed the old pre-Task-3 runtime behavior. Updating that expectation was necessary to keep the required backend suite verification meaningful. + +Implementation notes + +- Duplicate collection detection is enforced only for operational schema v3 descriptors by tracking `collection -> workspaceId` during activation. +- Duplicate failures are sanitized back to `workspace_invalid` / `Workspace repository content is invalid`. +- `acquireSessionRevision()` now fails closed for `migration_required` revisions. +- Legacy migration CLI now requires `--collection `. +- Legacy migration output is schema v3 with the fixed internal semantic contract: + - `vector_store.engine = qdrant` + - explicit `collection` + - embedding provider `ollama_internal` + - embedding model `qwen3-embedding:0.6b` + +self-review + +- Confirmed invalid pulled snapshots do not replace the previous active snapshot. +- Confirmed duplicate collection enforcement does not affect legacy migration-required descriptors. +- Confirmed create/update publication tests still pass with unique per-workspace collections. +- Confirmed no JSON stdout contract regressions in the migration CLI. +- Kept runtime/data mutation scope descriptor-only; no user workspace repo or Qdrant data changes. + +Concerns + +- No code concerns remaining for Task 2. +- One deliberate scope exception: a route test was updated to align with the already-established Task 1 / Task 3 fail-closed contract. + +Fix round 1 + +Scope + +- Restored meaningful route-level diagnoser coverage without reopening schema-v3 semantic runtime paths. +- Added direct schema-v2 registry coverage for `migration_required` listing and lease rejection. + +Covering test files + +- `backend/test/routes-workspaces.test.ts` +- `backend/test/workspace-registry.test.ts` + +RED command and output + +Command: + +```bash +cd backend +npx vitest run test/routes-workspaces.test.ts test/workspace-registry.test.ts +``` + +Observed result on top of `76bc94d` after adding the restored/new assertions: + +- 2 files passed +- 37 tests passed +- 0 failures + +Why no RED appeared: + +- The review items exposed missing/weakened coverage, not a production behavior bug. +- `/workspaces/:id/test` already reaches the diagnoser for resolvable legacy v2 descriptors. +- Schema-v3 `/workspaces/:id/test` already fails closed before diagnoser entry. +- Schema-v2 descriptors were already listed as `migration_required` and already rejected by `acquireSessionRevision()`. + +GREEN command and output + +Command: + +```bash +cd backend +npx vitest run test/routes-workspaces.test.ts test/workspace-registry.test.ts +npx tsc --noEmit -p . +``` + +Fresh results: + +- covering tests: 2 files passed, 37 tests passed +- backend typecheck: passed + +Changed files + +- `backend/test/routes-workspaces.test.ts` +- `backend/test/workspace-registry.test.ts` +- `.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-2-report.md` + +What changed + +- Split route coverage so `POST /workspaces/validate` still checks canonical validation independently. +- Restored route-level diagnoser coverage through a migration-required schema-v2 descriptor with resolvable legacy bindings. +- Added an explicit schema-v3 fail-closed regression for `POST /workspaces/:id/test`. +- Added a direct schema-v2 registry regression proving `list()` returns `migration_required` and `acquireSessionRevision()` rejects it. + +Concerns + +- No production concerns. This round only tightened coverage and corrected the weakened test expectation. diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-3-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-3-report.md new file mode 100644 index 00000000..82bb1f00 --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-3-report.md @@ -0,0 +1,132 @@ +# Task 3 report — Remove external semantic bindings and render internal endpoints + +Date: 2026-08-08 + +## Scope + +Implemented backend-owned schema-v3 semantic runtime rendering so workspace descriptors and installation contracts remain free of external Qdrant/Ollama endpoints and credentials, while DWH bindings stay unchanged. + +## RED evidence + +Focused RED command: + +`cd backend && npx vitest run test/workspaces-contracts.test.ts test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts test/config.test.ts` + +Observed failures before implementation: + +- `config.test.ts` + - missing `internalQdrantUrl` + - missing `internalEmbeddingUrl` +- `workspaces-bindings.test.ts` + - schema v3 semantic binding resolution threw unsupported errors +- `workspace-runtime-renderer.test.ts` + - schema v3 runtime rendering threw `Schema version 3 runtime rendering is unsupported until the internal semantic runtime is implemented` + +## GREEN evidence + +Focused GREEN command: + +`cd backend && npx vitest run test/workspaces-contracts.test.ts test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts test/config.test.ts` + +Result: + +- 4 test files passed +- 36 tests passed + +Typecheck: + +`cd backend && npx tsc --noEmit -p .` + +Result: + +- passed + +Hygiene: + +- `git diff --check` passed + +## Files changed + +Listed-task files changed: + +- `backend/src/config.ts` +- `backend/src/workspaces/bindings.ts` +- `backend/src/workspaces/runtime-renderer.ts` +- `backend/test/config.test.ts` +- `backend/test/workspace-runtime-renderer.test.ts` +- `backend/test/workspaces-bindings.test.ts` +- `backend/test/workspaces-contracts.test.ts` + +Listed-task files inspected but not changed: + +- `backend/src/workspaces/contracts.ts` + +Unavoidable additional wiring changes: + +- `backend/src/app.ts` +- `backend/src/tht/tht-runner.ts` + +Reason: the new typed internal semantic runtime config had to flow from backend config into ephemeral harness config rendering at runtime. + +## Behavior delivered + +- schema-v3 installation contract exposes DWH bindings only +- schema-v3 binding resolution ignores external semantic env vars instead of sourcing runtime semantics from them +- runtime rendering for schema v3 emits backend-owned internal semantic endpoints: + - Qdrant: `http://qdrant:6333` + - Embedding: `http://embedding:11434` + - Model: `qwen3-embedding:0.6b` + - Dimensions: `1024` +- internal semantic URLs are validated to allow only `qdrant` / `embedding` / `localhost` / loopback hosts +- DWH transport/runtime behavior remains unchanged + +## Self-review + +- Confirmed schema-v3 contracts/docs no longer advertise VECTOR or EMBEDDING installation variables. +- Confirmed schema-v3 runtime output ignores injected external semantic endpoints from env bindings. +- Confirmed semantic endpoints are rendered only in the ephemeral backend-owned harness config path. +- Confirmed type wiring is explicit from `AppConfig` → `ThtRunner` → runtime renderer. + +## Concerns + +- Host validation currently permits both `http` and `https` on the allowed internal hosts. That keeps the configuration flexible, but if the installation contract intended `http` only, that restriction is not enforced here. + +## Fix round 1/5 + +Scope: + +- moved schema-v3 internal embeddings under `resources.embeddings` +- enforced `http`-only internal semantic URLs + +RED evidence: + +`cd backend && npx vitest run test/workspace-runtime-renderer.test.ts test/config.test.ts` + +Observed failures on `bc8afe0`: + +- `workspace-runtime-renderer.test.ts` + - schema-v3 output omitted `resources.embeddings` + - schema-v3 still exposed top-level `embeddings` +- `config.test.ts` + - `https://qdrant:6333` was accepted + +GREEN evidence: + +`cd backend && npx vitest run test/workspace-runtime-renderer.test.ts test/config.test.ts` + +Result: + +- 2 test files passed +- 16 tests passed + +Typecheck: + +`cd backend && npx tsc --noEmit -p .` + +Result: + +- passed + +Updated concerns: + +- none for this round beyond future tightening if exact-port rejection is later requested explicitly. diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-4-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-4-report.md new file mode 100644 index 00000000..d3248faf --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-4-report.md @@ -0,0 +1,149 @@ +# Task 4 Report — Narrow harness embedding configuration to internal Ollama + +## Status + +Implemented on 2026-08-08 in `/Users/mp/projects/ThothII/.worktrees/git-workspace-registry`. + +## RED evidence + +Command: + +```bash +cd harness +./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py -q +``` + +Observed before implementation: + +- exit code `1` +- `10 failed, 10 passed` +- failures proved the missing `OllamaInternalEmbeddings` client and missing internal-only config validation + +Representative failures: + +- `ImportError: cannot import name 'OllamaInternalEmbeddings'` +- `AttributeError: 'EmbeddingsConfig' object has no attribute 'provider'` +- config tests `DID NOT RAISE ConfigError` for external provider, API key, and non-private base URL + +## GREEN evidence + +Focused behavior suite: + +```bash +cd harness +./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py -q +``` + +- exit code `0` +- `20 passed` + +Relevant harness verification: + +```bash +cd harness +./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py tests/test_ollama_ensure.py -q +``` + +- exit code `0` +- `36 passed, 2 warnings` + +Changed-file lint: + +```bash +cd harness +./.venv/bin/ruff check tht/config.py tht/config_compat.py tht/vectorstore/embeddings.py tht/cli/ollama_cmd.py tests/test_config_resources.py tests/test_internal_embeddings.py +``` + +- exit code `0` +- `All checks passed!` + +Patch hygiene: + +```bash +git diff --check +``` + +- exit code `0` + +## What changed + +- translated schema-v3 `resources.embeddings` into the harness-compatible embedding config view +- validated the internal embedding contract only for that runtime-owned `resources.embeddings` path: + - provider must be `ollama_internal` + - model must be `qwen3-embedding:0.6b` + - dimensions must be `1024` + - base URL must be `http://embedding:11434` or loopback HTTP on port `11434` + - extra fields like `api_key` are rejected +- replaced the active embed client with `OllamaInternalEmbeddings`, using one bounded `/api/embed` request per batch +- removed task/query prefix rewriting from the active embedding path +- validated response count, vector dimension, and finite numeric values before returning embeddings +- kept `tht ollama ensure --json` stdout pristine while warming through the internal client + +## Self-review + +- kept changes inside the brief-listed files +- preserved DWH and session-persistence behavior +- preserved the legacy `OllamaEmbeddings` import path as an alias to avoid unrelated call-site churn + +## Concerns + +- the focused harness verification still emits two pre-existing warnings: + - `DeprecationWarning` from `testcontainers.postgres` + - `FutureWarning` because `resources` currently flows through the legacy config translation path + +## Fix round 1 — 2026-08-08 + +### Findings addressed + +- HIGH: external top-level `embeddings` remained an operational fallback and could still load +- MEDIUM: non-object embed JSON payloads escaped as raw `AttributeError` + +### RED evidence + +Command: + +```bash +cd harness +./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py tests/test_ollama_ensure.py -q +``` + +Observed before the fix: + +- exit code `1` +- `2 failed, 36 passed, 2 warnings` + +Representative failures: + +- `AttributeError: 'list' object has no attribute 'get'` from `response.json()` returning a JSON array +- `Failed: DID NOT RAISE ConfigError` for top-level external `embeddings.provider=openai_compatible` + +### GREEN evidence + +Command: + +```bash +cd harness +./.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py tests/test_ollama_ensure.py -q +``` + +Observed after the fix: + +- exit code `0` +- `38 passed, 2 warnings` + +Touched-file lint: + +```bash +cd harness +./.venv/bin/ruff check tht/config.py tht/vectorstore/embeddings.py tests/test_internal_embeddings.py tests/test_config_resources.py +``` + +- exit code `0` +- `All checks passed!` + +### Minimal fix + +- validated the final active `cfg.embeddings` contract after config loading, so legacy top-level + embedding inputs now fail explicitly unless they exactly match the internal Ollama contract +- converted non-mapping embed JSON payloads into controlled `EmbeddingsError` failures with + sanitized diagnostics instead of raw attribute errors diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-5-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-5-report.md new file mode 100644 index 00000000..d4243cc5 --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-5-report.md @@ -0,0 +1,170 @@ +# Task 5 Report — Implement the Qdrant VectorStore adapter + +## Status + +Implemented on 2026-08-08 in `/Users/mp/projects/ThothII/.worktrees/git-workspace-registry`. + +## RED evidence + +Command: + +```bash +cd harness +./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q +``` + +Observed before implementation: + +- exit code `2` +- collection failed during import because the adapter did not exist yet + +Representative failures: + +- `ModuleNotFoundError: No module named 'tht.adapters.vector.qdrant'` + +## GREEN evidence + +Focused behavior suite: + +```bash +cd harness +./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q +``` + +- exit code `0` +- `31 passed, 1 warning` + +Touched-file lint: + +```bash +cd harness +./.venv/bin/ruff check tht/adapters/vector/qdrant.py tht/adapters/vector/__init__.py \ + tht/ports/vector.py tht/vectorstore/records.py tht/vectorstore/store.py \ + tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py +``` + +- exit code `0` +- `All checks passed!` + +Patch hygiene: + +```bash +git diff --check +``` + +- exit code `0` + +## What changed + +- added `QdrantVectorStore` with direct `requests`-based REST calls for: + - `GET /collections/{collection}` + - `PUT /collections/{collection}` + - `PUT /collections/{collection}/index` + - `PUT /collections/{collection}/points?wait=true` + - `POST /collections/{collection}/points/query` + - `POST /collections/{collection}/points/scroll` + - `POST /collections/{collection}/points/delete?wait=true` +- implemented idempotent collection provisioning for `1024` dimensions and `Cosine` distance +- created deterministic UUIDv5 point IDs from workspace, semantic kind, and canonical record key +- preserved canonical record identity and only upserted/deleted points matching the exact workspace + and generation filters +- added Qdrant payload helpers so stored payloads carry: + - `workspace_id` + - grouped semantic `kind` (`schema`, `evidence`, `memory`) + - original `record_kind` + - canonical `record_key` + - `content_hash` + - existing Thoth metadata fields +- mapped Qdrant payloads back into existing `VectorHit` objects without losing the original + Thoth kind +- exported the new adapter from the public vector adapter package and added focused contract tests +- sanitized timeout and malformed-response failures so CLI-facing callers do not leak raw endpoint + details + +## Self-review + +- confirmed collection mismatch fails without any delete/recreate path +- confirmed every query/scroll/delete operation includes a workspace filter +- confirmed the adapter never deletes or rewrites unrelated Qdrant points +- added keyword payload indexes for all filter-critical fields used here, including `document_id` + for exact Evidence filtering + +## Concerns + +- the requested `adversarial-review` skill could not run its full external reviewer flow in this + environment because the skill’s referenced `brain/` files are missing at + `/Users/mp/.agents/skills/adversarial-review`; I performed a manual adversarial self-review + instead +- the focused suite still emits one pre-existing warning from `testcontainers.postgres` + +## Fix round 1 — 2026-08-08 + +### Findings addressed + +- IMPORTANT: metadata collisions could override canonical Qdrant payload identity fields and break + workspace isolation +- IMPORTANT: scroll-based operations only read the first page and did not follow + `next_page_offset`, making `existing_hashes`, `list_evidence_generations`, and delete counts + inexact beyond one page + +### RED evidence + +Command: + +```bash +cd harness +./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q +``` + +Observed before the fix: + +- exit code `1` +- `2 failed, 31 passed, 1 warning` + +Representative failures: + +- `assert payload["workspace_id"] == "demo"` failed because colliding `record.metadata` + overwrote canonical payload fields +- paginated scroll test missed later pages, so `existing_hashes` and generation cleanup counts + were incomplete + +### GREEN evidence + +Command: + +```bash +cd harness +./.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q +``` + +Observed after the fix: + +- exit code `0` +- `33 passed, 1 warning` + +Touched-file lint: + +```bash +cd harness +./.venv/bin/ruff check tht/adapters/vector/qdrant.py tht/vectorstore/records.py \ + tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py +``` + +- exit code `0` +- `All checks passed!` + +Patch hygiene: + +```bash +git diff --check +``` + +- exit code `0` + +### Minimal fix + +- made `qdrant_payload` apply canonical fields after `record.metadata` so workspace ID, semantic + kind, original record kind, canonical record key, and content hash cannot be overridden by + metadata collisions +- paginated `_scroll` until `next_page_offset` is absent, sent the returned `offset` back on the + next request, and reject repeated offsets as malformed to avoid infinite loops diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-6-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-6-report.md new file mode 100644 index 00000000..8b70773e --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-6-report.md @@ -0,0 +1,144 @@ +# Task 6 Report + +Date: 2026-08-08 + +Status: implemented and verified + +Summary: + +- Added schema-v3 Qdrant runtime support to the harness config/resource layer and vector factory. +- Made Qdrant payloads carry `workspace_id` and `workspace_revision` on every point. +- Routed schema and memory bulk indexing through the transport-neutral vector port with canonical hash-based dedup. +- Kept Evidence canonical on filesystem and Memory canonical in JSONL; Qdrant remains derived/rebuildable. +- Added focused tests for semantic-kind isolation, shared identity fields, search-pack kind boundaries, and the schema-v3 factory/config path. + +Files changed: + +- `harness/tht/config.py` +- `harness/tht/config_compat.py` +- `harness/tht/adapters/factory.py` +- `harness/tht/adapters/vector/qdrant.py` +- `harness/tht/vectorstore/records.py` +- `harness/tht/cli/vector_cmd.py` +- `harness/tht/cli/memory_cmd.py` +- `harness/tests/test_semantic_kind_isolation.py` +- `harness/tests/test_memory_save_one.py` +- `harness/tests/test_search_pack.py` +- `harness/tests/test_qdrant_vector_store.py` +- `harness/tests/test_adapter_factory.py` +- `harness/tests/test_config_resources.py` + +Verification: + +- Focused RED/GREEN task suite: + - `cd harness && .venv/bin/pytest tests/test_semantic_kind_isolation.py tests/test_memory_save_one.py tests/test_search_pack.py -q` +- Relevant harness suite: + - `cd harness && .venv/bin/pytest tests/test_semantic_kind_isolation.py tests/test_memory_save_one.py tests/test_search_pack.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py tests/test_vector_port_contract.py tests/test_corpus_pipeline.py -q` + - Result: `131 passed` +- Changed-file Ruff: + - `cd harness && .venv/bin/ruff check tht/vectorstore/records.py tht/adapters/vector/qdrant.py tht/config_compat.py tht/config.py tht/adapters/factory.py tht/cli/vector_cmd.py tht/cli/memory_cmd.py tests/test_memory_save_one.py tests/test_search_pack.py tests/test_semantic_kind_isolation.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py` + - Result: clean + +Concerns / follow-up: + +- `memory clear` still retains its older direct-vector assumptions and was not expanded in this task because the brief focused on canonical builders and schema/evidence/memory routing through the active Qdrant path. +- The relevant suite still emits pre-existing warnings (legacy config deprecation in older fixtures, plus existing Pydantic serializer warnings in corpus tests), but they are not introduced by this task. + +## Fix round 1 (2026-08-08) + +Scope: + +- Fixed qdrant-only schema-v3 command gating for `vector index-schema`, `memory promote`, and `memory index`. +- Replaced `memory clear`'s direct-pgvector-only path with vector-port deletion by kind. +- Added focused qdrant-only CLI regression tests and refreshed older CLI fixtures to the enforced internal embedding contract. + +RED evidence: + +- `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py -q` +- Initial result against commit `5e39cfa`: `4 failed` +- Failure signatures: + - `ERRORE: sezioni mancanti nel workspace yaml: vector_db o vector_write_rest.` + - `ERRORE: sezioni mancanti nel workspace yaml: vector_db.` + +GREEN evidence: + +- Focused fix suite: + - `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py tests/test_memory_save_one.py tests/test_search_pack.py -q` + - Result: `51 passed` +- Relevant broader vector/memory/schema/search suite: + - `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_qdrant_vector_store.py tests/test_adapter_factory.py tests/test_config_resources.py tests/test_memory_save_one.py tests/test_search_pack.py tests/test_vector_port_contract.py tests/test_adapter_command_regressions.py tests/test_solved_search_cli.py tests/test_schema_introspect_guard.py tests/test_semantic_kind_isolation.py tests/test_corpus_pipeline.py -q` + - Result: `154 passed` +- Ruff on the fix surface: + - `cd harness && .venv/bin/ruff check tht/ports/vector.py tht/adapters/vector/qdrant.py tht/adapters/vector/pgvector.py tht/adapters/vector/thoth_http.py tht/vectorstore/rest_client.py tht/cli/vector_cmd.py tht/cli/memory_cmd.py tests/test_qdrant_cli_commands.py tests/test_solved_search_cli.py` + - Result: clean + +Notes: + +- `memory clear` now deletes derived `kind=memory` points through the configured writable vector store, while leaving the JSONL registry as the source of truth until the registry file is removed by the command. +- The broader suite still carries the same pre-existing warnings noted above; this fix round did not add new warnings or failures. + +## Fix round 2 (2026-08-08) + +Scope: + +- Removed the accidental HTTP writer `delete_kinds` capability expansion from `ThothHttpVectorStore` and `VectorRestClient`. +- Reworked `memory clear` so schema-v3 Qdrant uses scoped `kind=memory` deletion, while legacy transports keep the pre-task direct-sync path instead of advertising a nonexistent RPC. +- Tightened the qdrant-only memory-clear regression to assert the exact `("memory", ["memory"])` delete scope. + +RED evidence: + +- Re-review found a transport contract mismatch in fix round 1: + - `ThothHttpVectorStore` exposed `delete_kinds(...)` + - `VectorRestClient` exposed `delete_kinds(...)` + - but the legacy HTTP writer migration only allowlists `delete_vector_generation`, not `delete_vector_kinds` +- The new regressions added in this round capture that mismatch and the missing qdrant delete-scope assertion: + - `tests/test_vector_port_contract.py::test_http_store_supports_writer_without_reader` + - `tests/l0/test_vector_adapter_parity.py::test_http_rest_client_does_not_advertise_nonexistent_delete_kinds_rpc` + - `tests/test_qdrant_cli_commands.py::test_memory_clear_accepts_qdrant_only_runtime_config` + +GREEN evidence: + +- Focused regression suite: + - `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_vector_port_contract.py tests/l0/test_vector_adapter_parity.py tests/test_adapter_command_regressions.py -q` + - Result: `53 passed` +- Broader relevant vector/memory/search suite: + - `cd harness && .venv/bin/pytest tests/test_qdrant_cli_commands.py tests/test_adapter_command_regressions.py tests/test_vector_port_contract.py tests/l0/test_vector_adapter_parity.py tests/test_solved_search_cli.py tests/test_qdrant_vector_store.py tests/test_search_similar_kinds.py tests/test_corpus_pipeline.py -q` + - Result: `135 passed` +- Ruff on the changed fix surface: + - `cd harness && .venv/bin/ruff check tht/cli/memory_cmd.py tht/ports/vector.py tht/adapters/vector/thoth_http.py tht/vectorstore/rest_client.py tests/test_qdrant_cli_commands.py tests/test_vector_port_contract.py tests/l0/test_vector_adapter_parity.py` + - Result: clean + +Notes: + +- Legacy HTTP/vector-rest deployments do not gain a new destructive RPC surface from this fix; they keep their previous behavior and continue to fail closed for unsupported cleanup. +- The broader suite still emits the same pre-existing deprecation and serializer warnings already noted above; this round did not introduce new warnings. + +## Fix round 3 (2026-08-08) + +Scope: + +- Added an adapter-level Qdrant regression for mixed semantic kinds within one workspace plus a second workspace memory point. +- Verified that `delete_kinds("memory", ["memory"])` emits the real adapter filter with both `workspace_id=demo` and `record_kind=memory`. +- Verified that non-memory semantic kinds in the same workspace and memory from another workspace survive the delete. + +RED evidence: + +- Re-review identified a test gap rather than a confirmed runtime bug: + - existing coverage asserted only the CLI mock call shape for qdrant memory clear + - there was no adapter-level regression proving the real Qdrant delete filter and resulting fake-Qdrant state across mixed semantic kinds/workspaces +- Added regression: + - `tests/test_qdrant_vector_store.py::test_delete_kinds_is_workspace_scoped_and_preserves_other_semantic_kinds` + +GREEN evidence: + +- Requested focused suite: + - `cd harness && .venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_qdrant_cli_commands.py tests/test_semantic_kind_isolation.py -q` + - Result: `18 passed` +- Ruff on changed files: + - `cd harness && .venv/bin/ruff check tests/test_qdrant_vector_store.py` + - Result: clean + +Notes: + +- This round required no production change; the new adapter regression passed against the existing Qdrant implementation. +- The focused suite still emits the same pre-existing `testcontainers.postgres` deprecation warning from `tests/conftest.py`; no new warnings were introduced. diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-7-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-7-report.md new file mode 100644 index 00000000..a0d72e66 --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-7-report.md @@ -0,0 +1,98 @@ +# Task 7 report — mandatory Qdrant and Ollama Compose services + +Date: 2026-08-08 + +Status: completed + +Summary: + +- Added mandatory private `qdrant`, `embedding`, and `embedding-model-init` services to the base Compose stack. +- Pinned Qdrant `v1.18.2` and Ollama `0.32.0` by immutable multi-arch digest. +- Persisted Qdrant storage in `qdrant-data` and Ollama model cache in `embedding-models`. +- Wired `core` to fixed internal semantic endpoints: + - `THT_INTERNAL_QDRANT_URL=http://qdrant:6333` + - `THT_INTERNAL_EMBEDDING_URL=http://embedding:11434` + - `THT_INTERNAL_EMBEDDING_MODEL=qwen3-embedding:0.6b` + - `THT_INTERNAL_EMBEDDING_DIMENSIONS=1024` +- Removed external vector / embedding endpoint requirements from the local and server env examples. +- Added an idempotent Ollama model bootstrap script that: + - waits up to a bounded deadline for `/api/tags` + - skips `ollama pull` when the model is already cached + - pulls `qwen3-embedding:0.6b` only when needed + - verifies the model appears in `/api/tags` after pull +- Added optional GPU override file `deploy/compose.embedding-gpu.yaml`; base Compose remains CPU-only. +- Updated `scripts/run-stack.sh` so the GPU override is included only when `THOTH_ENABLE_EMBEDDING_GPU=1`. + +Verification: + +- RED confirmed before implementation: + - `./scripts/test-default-compose.sh` failed on missing required services. + - `./scripts/test-unified-compose.sh` failed on missing required services. + - `./scripts/test-internal-semantic-compose.sh` failed because the GPU override file did not exist. +- GREEN after implementation: + - `./scripts/test-default-compose.sh` + - `./scripts/test-unified-compose.sh` + - `./scripts/test-internal-semantic-compose.sh` + - `git diff --check` +- Additional shell verification: + - `scripts/run-stack.sh --wait` includes only base + local Compose files by default. + - `THOTH_ENABLE_EMBEDDING_GPU=1 scripts/run-stack.sh --wait` adds `deploy/compose.embedding-gpu.yaml`. + +Resolved image digests: + +- `qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c` +- `ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a` + +Self-review: + +- The first bootstrap-script draft depended on tools not guaranteed inside the Ollama image. This was corrected after image inspection; the final script uses only confirmed image tools (`bash`, `ollama`, `grep`) plus raw HTTP over `/dev/tcp`. +- The server overlay intentionally replaces most named core volumes with bind mounts, so the unified contract was tightened to require named semantic-cache volumes there while preserving the local/base named-volume checks. + +Concerns: + +- The model bootstrap waits for Ollama readiness and verifies cache state, but the first real cold-start will still take time to download `qwen3-embedding:0.6b`. +- The GPU override requests generic Docker GPU capability only; actual GPU availability remains host/runtime dependent and intentionally stays opt-in. + +## Fix round 1 / 5 — 2026-08-08 + +Rulings applied: + +- Kept the Task 1 boundary intact: schema-v3 remains the only operational workspace descriptor shape. +- Did not restore any external semantic fallback for schema-v2 live sessions. +- Treated `PROJECT_STATE.md` as stale documentation for this point, not runtime truth. + +Focused schema-v2 evidence: + +- Re-ran the existing targeted registry test: + - `cd backend && npx vitest run test/workspace-registry.test.ts -t "lists a schema v2 descriptor as migration_required and refuses to acquire it"` +- Result: pass. +- Evidence from that test: + - schema-v2 descriptors list as `migration_required` + - `acquireSessionRevision("psd-clinical")` rejects with `code: "workspace_invalid"` +- Conclusion: schema-v2 acquisition remains blocked; no external semantic fallback was reintroduced. + +Contract consistency fixes: + +- Updated `harness/tests/test_local_compose_contract.py` to assert the mandatory internal semantic stack, fixed internal core semantic env, private-service topology, persistent volumes, and Ollama health/dependency contract. +- Updated shell Compose contracts to require: + - Ollama healthcheck on `embedding` + - `embedding-model-init` dependency on `embedding: service_healthy` +- Updated `scripts/unified-deployment-smoke.sh` rendered-contract helper to expect the mandatory internal semantic topology and internal semantic env names, and to reject retired external semantic bindings. +- Updated `scripts/test-task13-runtime-fixtures.sh` to exercise `task13_assert_rendered_contract` for both local and server fixture renders. + +Fix round 1 verification: + +- RED before implementation: + - `cd harness && .venv/bin/pytest tests/test_local_compose_contract.py -q` failed because `embedding` had no healthcheck. + - `./scripts/test-default-compose.sh` failed because `embedding` had no healthcheck. + - `./scripts/test-unified-compose.sh` failed because `embedding` had no healthcheck. + - `./scripts/test-task13-runtime-fixtures.sh local` failed because `unified-deployment-smoke.sh` still expected `core,frontend`. +- GREEN after implementation: + - `./scripts/test-default-compose.sh` + - `./scripts/test-unified-compose.sh` + - `./scripts/test-internal-semantic-compose.sh` + - `cd harness && .venv/bin/pytest tests/test_local_compose_contract.py -q` + - `./scripts/test-task13-runtime-fixtures.sh local` + - `./scripts/test-task13-runtime-fixtures.sh server` + - `cd backend && npx vitest run test/workspace-registry.test.ts -t "lists a schema v2 descriptor as migration_required and refuses to acquire it"` + - `docker compose --env-file deploy/env/local.env.example -f compose.yaml -f deploy/compose.local.yaml config --format json` diff --git a/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-9-report.md b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-9-report.md new file mode 100644 index 00000000..da2f169d --- /dev/null +++ b/.superpowers/sdd/2026-08-08-internal-qdrant-ollama/task-9-report.md @@ -0,0 +1,65 @@ +Status: completed on August 8, 2026. + +Summary: +- Updated the frontend workspace contract from schema v2 editing to schema v3 publishing. +- Kept only `semantic_index.vector_store.collection` editable; rendered qdrant / internal Ollama semantic values as fixed read-only architecture values. +- Removed external vector transport / endpoint / credential / embedding diagnostics branches from frontend draft sanitization, conflict parsing, and editor UI. +- Added a migration-required banner in workspace management and blocked `migration_required` workspaces from new-session selection. +- Aligned the example workspace YAML comments with the fixed internal qdrant/Ollama architecture. + +Files changed: +- `frontend/src/api/workspaces.ts` +- `frontend/src/api/workspaces.test.ts` +- `frontend/src/workspaces/drafts.ts` +- `frontend/src/workspaces/drafts.test.ts` +- `frontend/src/shell/WorkspaceEditor.tsx` +- `frontend/src/shell/WorkspaceEditor.test.tsx` +- `frontend/src/shell/WorkspaceManager.tsx` +- `frontend/src/shell/WorkspaceManager.test.tsx` +- `frontend/src/shell/WorkspacePublishDialog.test.tsx` +- `frontend/src/api/sessions.ts` +- `frontend/src/shell/SteerInput.tsx` +- `frontend/src/shell/SteerInput.test.tsx` +- `deploy/workspaces/example.yaml` +- `deploy/workspaces/psd.yaml.example` + +Verification: +- `cd frontend && npx vitest run src/shell/SteerInput.test.tsx src/shell/WorkspaceEditor.test.tsx src/shell/WorkspaceManager.test.tsx src/shell/WorkspacePublishDialog.test.tsx src/workspaces/drafts.test.ts src/api/workspaces.test.ts` + - Result: 6 files passed, 59 tests passed. +- `cd frontend && npx tsc -b` + - Result: passed. +- `git diff --check` + - Result: passed. + +Self-review: +- The frontend now publishes the exact schema v3 semantic shape and no longer persists legacy semantic transport/credential branches. +- Migration-required workspaces are visible in management with an explicit banner and are excluded from the composer workspace selector. +- One dependent test file outside the original brief list (`WorkspacePublishDialog.test.tsx`) and the composer/session-selection path (`api/sessions.ts`, `SteerInput.tsx`, related test) were updated because they were directly coupled to the old v2 semantic/edit-selection behavior. + +Concerns: +- The composer still retains backward-compatible behavior for summaries that omit `revision` entirely; only explicit `revision.state === "migration_required"` is blocked. That matches the current mixed-test environment, but once summary responses are guaranteed to include `revision`, that fallback may be removable. + +Fix round 1/5 — August 8, 2026 + +Summary: +- Made missing or invalid workspace summaries fail safe in frontend session creation and composer selection instead of falling open as legacy. +- Added an actionable unavailable message in workspace management for incomplete summaries with no canonical revision. +- Replaced the old runtime-oriented example descriptor files with exact backend WorkspaceV3 descriptor YAML. + +Additional files changed: +- `frontend/src/api/sessions.test.ts` +- `backend/test/workspaces-schema.test.ts` + +Fix-round verification: +- `cd frontend && npx vitest run src/api/sessions.test.ts src/shell/SteerInput.test.tsx src/shell/WorkspaceManager.test.tsx src/shell/WorkspaceEditor.test.tsx src/shell/WorkspacePublishDialog.test.tsx src/workspaces/drafts.test.ts src/api/workspaces.test.ts` + - Result: 7 files passed, 73 tests passed. +- `cd frontend && npx tsc -b` + - Result: passed. +- `cd backend && npx vitest run test/workspaces-schema.test.ts` + - Result: 1 file passed, 17 tests passed. +- `git diff --check` + - Result: passed. + +Notes: +- Missing `revision` in a workspace summary now fails with the same session/composer safety posture as `migration_required`, using the existing safe workspace-policy error for session creation and an explicit unavailable message in workspace management. +- The committed example files now validate as actual schema-v3 descriptors instead of deployment/runtime templates with forbidden semantic endpoint fields. diff --git a/.superpowers/sdd/2026-08-15-unified-tht-cli-product-step/task-5-report.md b/.superpowers/sdd/2026-08-15-unified-tht-cli-product-step/task-5-report.md new file mode 100644 index 00000000..ef8da22c --- /dev/null +++ b/.superpowers/sdd/2026-08-15-unified-tht-cli-product-step/task-5-report.md @@ -0,0 +1,83 @@ +# Task 5 Report — `tht setup` lifecycle orchestration + +## Status + +Completed. `tht setup` now validates the checkout and host prerequisites, creates or validates +the non-secret installation files, validates Compose, and by default builds, starts, health-checks, +and verifies the installation. `tht setup --configure-only` stops immediately after successful +Compose rendering. + +## Implementation + +- Added `setup.Run`, with an ordered host preflight: project/worktree discovery, Docker Engine, + Docker Compose, supported architecture, and LF line-ending checks. +- Reused `config.Installation.ComposeArgs` for all Compose calls and added a narrow + `compose.InstallationRunner` adapter for Pi diagnostics; no shell command construction was added + to the top-level CLI parser. +- Default setup performs `compose build`, `compose up --detach --remove-orphans`, bounded polling + for `core`, `frontend`, `qdrant`, `embedding`, and `embedding-model-init`, then aggregate volume + diagnostics and `pi.Doctor`. +- Health timeout errors identify the last failing service and preserve containers for diagnosis, + with `tht logs ` and `tht status` guidance. +- Completion output includes the frontend URL, selected descriptor, and next action. + +## TDD evidence + +The initial focused test run failed because `setup.Run` did not exist. Tests were then written +against a fake Compose runner before the orchestration was implemented. They cover the complete +ordered flow, configure-only stop, preflight failure before writing configuration, health retry, +timeout guidance, and CLI default versus `--configure-only` dispatch. + +## Verification + +Executed from `tools/tht`: + +```bash +go test ./internal/setup ./internal/compose ./cmd/tht -run 'TestRun|TestSetupCommand|TestInstallationRunner' -count=1 +go test ./internal/setup ./internal/compose ./cmd/tht -count=1 +go test ./... +git diff --check +``` + +All commands passed. No actual Docker build, container start, live-stack restart, system +installation, Pi configuration edit, or documentation rewrite was performed. + +## Commit + +`feat(setup): build start and verify ThothII` (this report is included in that commit). + +## Concerns + +- The bounded health wait is verified with fakes only, as required for this task. Real Docker + lifecycle verification belongs to the later live acceptance task. +- The existing aggregate `tht doctor` command remains a separate implementation; Task 5 performs + its equivalent setup-time prerequisite checks plus `pi.Doctor` without invoking a nested CLI + process. + +## Fix round 1 + +The independent review identified three gaps. All were reproduced with RED tests before the +production change: + +- A rendered Compose document containing any one volume was accepted. `requireVolumes` now + requires `settings`, `pi-state`, `workspace-registry`, `workspace-secrets`, `sessions`, + `qdrant-data`, and `embedding-models`; tests reject each individual omission and an + unrelated-only volume set. +- Failures after `compose up` could return without recovery instructions. A single recovery + wrapper now preserves the underlying error while adding the retained-container, `tht logs + `, and `tht status` guidance for failed `up`, health, aggregate doctor, and Pi doctor + phases. Focused tests also prove build failure stops before attempting startup. +- LF inspection previously walked the full checkout. It now inspects only `compose.yaml`, + `deploy/`, and `docker/`; a test proves CRLF content under `node_modules/` is ignored. + +Verification added for this round: + +```bash +go test ./internal/setup -run 'TestRequireVolumes|TestRun(BuildFailure|UpFailure|AggregateDoctorFailure|PiDoctorFailure|IgnoresIrrelevant|TimesOut)' -count=1 +go test ./internal/setup -count=1 +``` + +Both passed before the final full-suite verification. No Docker or live operation was run. + +Implementation commit evidence: `ea70cc95b04532043744a9de6c5912e30a214595` — +`fix(setup): harden verification and recovery`. diff --git a/.superpowers/sdd/2026-08-15-unified-tht-cli-product-step/task-6-report.md b/.superpowers/sdd/2026-08-15-unified-tht-cli-product-step/task-6-report.md new file mode 100644 index 00000000..90a7d55b --- /dev/null +++ b/.superpowers/sdd/2026-08-15-unified-tht-cli-product-step/task-6-report.md @@ -0,0 +1,55 @@ +# Task 6 — Version, aggregate doctor, and build-aware start + +Status: complete. + +Implemented the host-side `tht version`, aggregate `tht doctor [--json]`, and `tht start [--build]` contracts. + +- `version` is descriptor-free and reports semantic version, commit, build time, OS, and architecture. +- `doctor` emits typed, redacted checks for descriptor state, Docker/Compose, rendered volumes, file permissions, service health, workspace registry, the container-local workflow doctor, and Pi doctor. Its JSON mode writes exactly one JSON document to stdout. +- The Python workflow doctor is invoked only as `docker compose exec -T core tht doctor --json` after core is running. +- `start` uses the shared lifecycle service: default `up → health`; `--build` is `build → up → health`. +- `setup` now reuses the shared lifecycle and aggregate diagnostics rather than keeping parallel health/volume implementations. + +Verification performed without live Docker/container commands: + +```bash +cd tools/tht +go test ./internal/version ./internal/doctor ./internal/service ./cmd/tht \ + -run 'TestVersion|TestDoctor|TestStart|TestCurrent|TestRun' -count=1 +go test ./internal/setup -count=1 -run 'TestRun' -v +go test ./... -count=1 +git diff --check +``` + +All completed successfully. The intentionally fake runner coverage includes unavailable Docker, +stopped/running core, workflow failure redaction, pristine JSON output, and start ordering. + +Concerns: no live Docker validation or host installation was run, by explicit task constraint. + +## Fix round 1 + +Completed the independent-review follow-up without live Docker operations. + +- `workspace-registry` now executes a container-local, read-only Node validation of + `/data/workspace-registry/state/active.json` and every declared snapshot descriptor. It no + longer passes merely because Compose declares a volume. +- Host file permissions are checked before Docker/Compose availability and therefore remain + visible as failures when Docker is unavailable. +- Separate typed, bounded HTTP probes verify core (`curl --max-time 5`) and frontend + (`wget -T 5`) reachability, independently of Compose health. The probe is injectable in tests. +- The successful report tests assert the stable full checklist: + `descriptor`, `files`, `docker`, `compose`, `configuration`, `services`, `core-http`, + `frontend-http`, `workspace-registry`, `workflow`, `pi`. + +Additional verification: + +```bash +cd tools/tht +go test ./internal/doctor -run 'TestRun(ChecksUnsafeFilesEvenWhenDockerIsUnavailable|FailsAnInvalidContainerLocalRegistryState|ReportsEachHTTPReachabilityProbeFailure|UsesOnlyContainerLocalWorkflowAndPiDiagnosticsWhenCoreRuns)' -count=1 -v +go test ./internal/doctor ./internal/setup ./internal/service ./cmd/tht -count=1 +go test ./... -count=1 +git diff --check +``` + +All passed with fake runners/probes only. No live container, HTTP endpoint, or host installation +was touched. diff --git a/.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md b/.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md new file mode 100644 index 00000000..6b173538 --- /dev/null +++ b/.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md @@ -0,0 +1,163 @@ +# Task 15 retained release-gate report — fix round 5 (sanitized) + +## Final-review fix-round-2 addendum — frozen source `2a9359071257f9b8a71d36ec2bbb25b161003f81` + +This addendum supersedes the fix-round-1 addendum for current authentication remediation status +while preserving the fix-round-5 material below as historical provenance. + +- Authentication remediation status: `PASS`. The three original remediation Important findings + remain `RESOLVED`; the fix-round-2 fully bounded lifecycle Important is `ADDRESSED`; and the + temporary Windows diagnostic-matrix Minor is `ADDRESSED`. +- Overall branch/release readiness is separately `FAIL`, with unavailable external/manual gates + `PENDING`. +- Completed exact-source workflow run `32147345625` concluded `failure` on baseline release jobs. + Its `Windows clone and Compose contract` job (`95744249248`) executed the unfiltered command + `go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1`; the native step + passed all three packages: safeio `22.058s`, backup `7.161s`, authstorage `16.088s`. +- The Windows job failed only afterward in the baseline clone-contract script at + `scripts/test-windows-clone-contract.ps1:208`, where PowerShell rejects the undelimited + `$remoteYaml:` variable reference. +- `LF, Compose, docs, and TypeScript` job `95744249458` reproduced the baseline unset-`TMPDIR` + failure after unified Compose passed. Linux Docker job `95744249354` reproduced the missing-`rg` + prerequisite failure; cleanup passed and no image manifest was generated. +- The skipped Windows Docker Desktop/WSL2 job is recorded as `NOT_RUN` / `BLOCKED`, not FAIL. + Downstream commands skipped after executed baseline failures use the same classification. The + matrix contains an explicit native `windows_stagearchive_retained_capability` PASS row. +- Historical Node/auth/browser/docs PASS and harness/Ruff/Compose FAIL evidence remains bound to + its recorded source where not rerun. L2, real PSD/manual acceptance, and provider readiness + remain `PENDING`. +- Current machine-readable evidence and the requested Task 4 report are recorded in + `.artifacts/task-15/automated-gates.json` and + `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md`. +- The full fix-round-2 RED/GREEN and finding disposition is recorded in + `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-2-report.md`. +- Current automated-gates SHA-256: + `6c516db5c2064c4a4a2e5f25961b993cd4a8fe020bbbb822fbac7faa0c119599`. +- Historical unified Docker manifest SHA-256: `9c8dec4546909fd93799dbcf374bcb3a89bc46cfe0fd482472c0cbe757ddf5b6`. + +The complete sanitized Task 4 matrix and the separate remediation/release verdicts are in the +requested Task 4 report. + +- Final tested source commit: `74b062f1a737103524cbe706346cfd65f87cdfd1`. +- Historical retained source commits: fix-round-2 `fe190e7046acc173f510dddcb32f46ed142858c1`, + maintenance follow-up `4d230b87afdcd24f02264f8f937c8628b92db05a`, prior final Docker + source `e20bf33e2a00102192e5be66b178037aeca3a7b1`, and fix-round-4 streamed + archive privacy `54698e73400a54ce7c3e6c10099e14eb471ce8b9`. +- Versions: Node contract `v24.16.0`; host default Node `v25.6.1`; Go `go1.26.5`; + Pi `0.80.3`. +- Historical automated gate artifact: `.artifacts/task-15/automated-gates.json`; + SHA-256 `7d9ec93af15510605f1aa7179b26a7ee46d78122f647854300f7a9922057a63f`. +- Docker image manifest: `.artifacts/task-15/unified-docker-images.json`; + SHA-256 `9c8dec4546909fd93799dbcf374bcb3a89bc46cfe0fd482472c0cbe757ddf5b6`. + +## Fix-round-5 evidence + +- PASS, RED then GREEN: `TestCreateCanonicalNewPrivateFileUsesPinnedParentAfterAncestorSwap` + first failed because the creator had not retained its parent before creation. It now opens every + Unix ancestor once, creates the leaf with `openat(O_NOFOLLOW|O_CREAT|O_EXCL)`, applies and checks + `0600` by descriptor (`fchmod`/`fstat`), and uses `unlinkat` for creator failure cleanup. The + deterministic test moves the opened parent, replaces its lexical name with an outside symlink, + validates the archive under the moved original parent, and proves no outside archive was written. +- PASS: the Windows implementation uses NT `RootDirectory`-relative traversal for every component + after the volume root and for final file creation. The retained final parent receives only the + required child-create right (`FILE_WRITE_DATA` for a file, `FILE_APPEND_DATA` for a directory), + reparse points are rejected, and the owner-only protected DACL is installed in the same + `NtCreateFile` operation. The native-Windows test attempts the pre-create parent swap and calls + `safeio.ValidatePrivateRegular`; it is compiled but not executed on this host. +- PASS: `go test ./internal/safeio ./internal/backup -count=1`, `go test -race ./...` across + `18` packages, `go vet ./...`, and a native host `tht` CLI build. Existing StageArchive + capacity, lifecycle, rollback, streaming, and cleanup tests remain passing. +- PASS, compile-only: Windows amd64 static test/build compilation across `18` packages, including + the retained-handle Windows tests. No Windows executable was run; native execution remains + PENDING and is not inferred from compilation. +- PASS on Node `v24.16.0`: the hermetic OIDC/F1 authentication browser smoke passed all current + `8` checks in `frontend/e2e/auth.spec.ts` and `frontend/e2e/f1.spec.ts`; the runtime sentinel + leak scan passed. +- PASS: shell syntax, unified-smoke safety self-test, default Compose contract, unified Compose + contract, and Compose secret-policy contract. +- PASS: final unified Docker deployment smoke run `20260818070637-66409-30058`, bound exactly to + source `74b062f1a737103524cbe706346cfd65f87cdfd1`. It exercised maintenance-auth isolation, + restore, registry lifecycle, bad-candidate rollback, image revalidation, and task-scoped cleanup. + +## Sanitized final unified Docker output + +```text +== Build and start isolated local Compose distribution == +== Recreate offline and retain the validated registry snapshot == +== Pull a valid catalog+descriptor metadata update == +== Pull a content-only Git Evidence update == +== Reject catalog/descriptor metadata mismatch and retain the valid snapshot == +== Reject orphan descriptor directories not listed in the catalog == +== Reject the retired flat workspace layout and retain the valid snapshot == +== Inject a bad pinned Pi candidate and prove automatic rollback == +Task 13 full deployment smoke passed. +Task 13 cleanup proof: no labeled containers, volumes, networks, or images remain for 20260818070637-66409-30058. +``` + +## Sanitized Docker image identities + +- `sha256:2d7b19491c7eb8c119c3cedb390aaeb2ff5593f6fc43ab66c317565560da6d7d`; + roles `compose-runtime`, `fixture-runtime`. +- `sha256:3b6c31a5d8f8fc58fa3233391b6175bd2fbc793eebb44d5e285ecc6e02e9e687`; + role `compose-runtime`. +- `sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a`; + role `compose-runtime`. +- `sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c`; + role `compose-runtime`. +- `sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178`; + role `rollback-candidate`. + +For each image, the retained repository-digest component equals the listed image digest. Registry +names and credentials are deliberately omitted. + +## Complete observed matrix + +- PASS: Task 13 lifecycle carry-ins; retained-handle owner-private restore staging; provider fixture + round-one `6/6`; backend Node 24 round-one suite `75 files / 1081 tests`; frontend Node 24 + round-one suite `61 files / 444 tests`; current Node 24 authentication/F1 browser smoke `8/8`; + final-source Go race/build `18 packages`; Windows static cross-compile `18 packages`; harness + round-one suite `921 passed / 4 L2 deselected`; authentication docs round-one gate; shell/Compose + contracts; final unified Docker smoke; five-image traceability; and Docker cleanup. +- FAIL: Ruff `192` known-baseline errors; MkDocs strict `69` known-baseline warnings; existing + canonical/workspace install wording checks; existing Pi model-policy check; deployment-coupling + scan against preserved ignored private material. +- PENDING: native Windows execution because required host prerequisites are unavailable; L2 because + the configured secret layout is unavailable; real PSD/manual acceptance because no real + identity/access is available; isolated provider readiness because an unrelated host port is + occupied. + +## Final Task 15 review after fix round 5 + +The fresh Terra review verdict is **CHANGES REQUIRED**. The five-round breaker is exhausted; no +sixth implementation round was started. Two Important findings remain: + +- `StageArchive` does not retain the opaque parent/directory capability through the complete + stream and `Close` lifecycle. Staging-directory creation and final cleanup still use pathname + operations, so an ancestor swap after creation can strand the secret-bearing archive or redirect + cleanup. Deterministic StageArchive swap-and-cleanup coverage is still required on Unix and + native Windows. +- Windows claim removal closes its validated retained parent handles before calling pathname-based + `DeleteFile`. Removal must instead remain handle-relative (or delete through the opened handle), + with a native-Windows ancestor-swap test. + +The focused/full Go, cross-compile, Node 24, browser, Compose, Docker lifecycle, image-traceability, +and cleanup results above remain valid evidence for source `74b062f1a737103524cbe706346cfd65f87cdfd1`. +They do not override the final code-review verdict. Native Windows execution remains PENDING. + +The authentication feature is **not implementation-complete or release-complete** while these code +findings and the required FAIL/PENDING gates remain. No secret values, real identities, internal +endpoints, or registry names are retained. + +## Final whole-branch review + +The final read-only Terra review of `351361f..39b5453` also returned **CHANGES REQUIRED** and found +one additional Important issue: the POSIX local-user registry validates file type, link count, and +mode for `users.yaml` and its parent directory, but does not require ownership by the effective UID. +A foreign-owned `0600` registry inside a runtime-owned `0700` directory can remain writable by the +foreign owner and be used to alter credentials or grant the administrator role. The registry must +enforce effective-UID ownership on every POSIX `lstat`/`fstat` path and add foreign-owner rejection +coverage. + +No new Critical issue or load-bearing Minor issue was found. The branch is **not ready to merge**: +this ownership defect and the two retained-capability cleanup defects above require fixes and renewed +review, independently of the remaining FAIL/PENDING release gates. diff --git a/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-1-report.md b/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-1-report.md new file mode 100644 index 00000000..92d2a4cd --- /dev/null +++ b/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-1-report.md @@ -0,0 +1,162 @@ +# Final-review fix round 1 report (sanitized) + +## Verdict + +- Base: `fa499a9bdd37011833691b0f447470d8b7e8a3a6`. +- Final frozen source: `10cd66fe6a5b484a4dc569326a228c1c5484a5d4` on + `feat/thoth-auth`. +- Authentication remediation: **PASS / ADDRESSED**. All four final-review Important findings are + resolved relative to the remediation brief. +- Terra Minor evidence corrections: **ADDRESSED**. +- Branch/release readiness: **FAIL**. The completed exact-source workflow still contains executed + baseline clone-contract, LF/Compose, and Linux Docker failures. Unavailable external/manual + gates remain **PENDING**. +- Source and evidence remain separate commits. No workflow was dispatched from the evidence-only + phase. + +## Finding disposition + +| Finding | Disposition | Evidence | +|---|---|---| +| Important 1 — exhaustive Windows cleanup | RESOLVED | Cleanup now attempts close/delete/validation operations in deterministic order and returns sanitized `ErrUnsafeFile` after aggregating failures. `TestWindowsPrivateRegularCleanupClosesAfterDeleteDispositionFailure` and `TestWindowsClaimCleanupAttemptsLaterOperationsAfterEarlierFailure` cover the non-short-circuit contract. Global no-delete sharing remains unchanged. | +| Important 2 — usable native Windows authority | RESOLVED | Owner-only descriptors use the current user SID, protected/non-defaulted DACL semantics, valid NT attributes/access masks, self-relative creation descriptors, and semantic full-control validation. Equal-or-stronger Windows fixture adaptations retain no-delete handles instead of weakening ACL/identity checks. The final native three-package gate passes. | +| Important 3 — restore-test deadlock | RESOLVED | Lifecycle-stage release observes the buffered worker outcome, uses a bounded/cancellable release, reports premature completion directly, and never waits indefinitely on `done`. `TestReleaseLifecycleStageReturnsPrematureWorkerOutcome` and the lifecycle-lock terminal-cleanup test are green. | +| Important 4 — complete native package gate | RESOLVED | Workflow and remediation plan both use the exact unfiltered command `go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1`. Final logs prove all three packages executed natively. | +| Minor — non-executed gate classification | RESOLVED | Non-executed/skipped commands are `NOT_RUN` / `BLOCKED`; `FAIL` is reserved for commands that ran and failed. Historical results remain separately labelled. | +| Minor — explicit Windows StageArchive row | RESOLVED | `.artifacts/task-15/automated-gates.json` contains `windows_stagearchive_retained_capability` = PASS, bound to the final source and native backup result. | + +Additional failures exposed by the required unfiltered gate were fixed without narrowing the +workflow: Windows secret-bearing archive reservation is protected before use; StageArchive shares +one retained root capability across both staged files; claim/consume transitions serialize the +complete public validation and retained-handle operation while preserving ACL, hard-link identity, +reparse rejection, and no-delete invariants. + +## RED → GREEN record + +### Initial RED + +- Run `32122302381`: + https://github.com/mptyl/ThothII/actions/runs/32122302381 +- Source: `b31b27e5845ffd3adf311429367319beaba263c7`. +- Windows job: `95665197885`. +- Result: native `safeio`/`backup` failure, including the 10-minute restore lifecycle timeout; + `authstorage` was absent from the command. This established the RED for Important 2–4 and the + required native authority. +- Cleanup failure-injection tests added for Important 1 first exposed the short-circuit behavior + before the implementation was changed. + +### Final concurrency RED + +- Run `32140481263`: + https://github.com/mptyl/ThothII/actions/runs/32140481263 +- Source: `b48e9e9189dd0e8083db9bd0378704524e670edb`. +- Windows job: `95721724645`. +- Native results: backup PASS (`20.757s`), authstorage PASS (`104.180s`), safeio FAIL + (`63.502s`). The only failures were: + - `TestCanonicalPrivateClaimWaitsForRetainedRemoveOperation`: the concurrent claim returned + `false, unsafe file` before retained removal completed; + - `TestCanonicalPrivateClaimConsumeHasOneConcurrentWinner`: iteration 8 returned `unsafe file`. +- Diagnosis: the process mutex started below `validateClaimPaths`; a concurrent caller could fail + while reopening the retained no-delete directory before reaching the lock. + +### GREEN implementation and local gates + +The lock boundary was moved to the three public claim/read/remove APIs, covering validation, +relative operation, and handle close. The Unix implementation uses a no-op boundary and retains its +existing descriptor-relative semantics. + +Final-source local commands passed: + +```text +go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1 +go test -race ./... +go vet ./... +go build -o /tmp/thothii-tht-host ./cmd/tht +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o /tmp/thothii-tht-windows.exe ./cmd/tht +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c ... ./internal/{safeio,backup,authstorage} +``` + +- Focused host package times: safeio `8.750s`, backup `8.378s`, authstorage `8.854s`. +- Race suite and vet: PASS. +- Host CLI: Mach-O arm64; Windows CLI and all three Windows test binaries: PE32+ x86-64. +- Cross-compilation remains compile-only and is not used as native proof. + +## Exact-source native certification + +- Run: `32141428407` +- URL: https://github.com/mptyl/ThothII/actions/runs/32141428407 +- Event/status/conclusion: `workflow_dispatch` / `completed` / `failure`. +- Head SHA: `10cd66fe6a5b484a4dc569326a228c1c5484a5d4` — exact final source match. +- Windows job: `Windows clone and Compose contract`, job `95724751282`: + https://github.com/mptyl/ThothII/actions/runs/32141428407/job/95724751282 +- Native step: `Run native Windows retained-capability tests` — **PASS**. +- Exact command: `go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1`. +- Native package results: + - safeio PASS (`8.230s`); + - backup PASS (`5.195s`); + - authstorage PASS (`8.383s`). +- Job conclusion: `failure` only because the following `Verify Windows clone contract` baseline + step failed with a PowerShell `ParserError` at + `scripts/test-windows-clone-contract.ps1:208`; `$remoteYaml:` is not delimited before `:`. + +## Remaining branch/release blockers + +| Gate | Classification | Exact outcome | +|---|---|---| +| Windows native authentication packages | PASS | All three required packages executed on final source. | +| Windows clone contract | FAIL / baseline | Executed after native PASS; PowerShell parser error at line 208. | +| LF, Compose, docs, and TypeScript | FAIL / baseline CI contract | Job `95724751205`; unified Compose passed, then `test-no-deployment-coupling-scope.sh` failed because `TMPDIR` was unset. Downstream skipped commands are `NOT_RUN` / `BLOCKED`. | +| Linux Docker deployment and rollback | FAIL / infrastructure prerequisite | Job `95724751356`; executed smoke stopped because `rg` was unavailable. Cleanup proof passed; no new image manifest was generated. | +| Native Windows Docker Desktop/WSL2 startup | NOT_RUN / BLOCKED | Job `95724752028` was skipped by workflow conditions; no Docker/WSL2 command executed. | +| Harness/Ruff/other historical baseline gates | FAIL | Retained with their recorded source and results; not rewritten as final-source proof. | +| L2, real PSD/manual acceptance, provider readiness | PENDING | Required secrets, identity/access, or provider prerequisites remain unavailable. | + +The historical Docker image manifest remains bound to source +`74b062f1a737103524cbe706346cfd65f87cdfd1`; it was not reused as proof for the final source. + +## Principal source commits + +- `cd5f505` — exhaustive cleanup, Windows authority foundation, restore deadlock tests/fix, and + complete workflow/plan package command. +- `a0e05ad` through `b6396e6` — effective full-control DACL semantics, valid NT attributes/access, + self-relative descriptors, retained no-delete fixture ordering, and Windows installation fixture + protection. +- `824245d` — preserve existing lifecycle ACL trees instead of mutating inherited authority. +- `455fffb`, `2d1670e`, `c01482c`, `9fc1a15` — concurrent claim/consume and settled-loss handling. +- `6474118` — one retained StageArchive root capability shared across staged files. +- `feee4ee` — unified Windows path wrappers on the retained primitive. +- `b261dd4` — bounded private-root sharing contention handling. +- `b48e9e9` — deterministic retained-remove concurrency regression and claim-operation lock. +- `10cd66f` — final lock boundary includes public path validation; frozen source. + +## Files changed + +Source changes relative to the fix-round base: + +- `.github/workflows/deployment.yml`; +- `docs/superpowers/plans/2026-08-18-thothii-authentication-remediation.md`; +- `tools/tht/internal/authstorage/storage_test.go`; +- `tools/tht/internal/backup/{create.go,create_test.go,fixture_security_unix_test.go,fixture_security_windows_test.go,preflight.go,preflight_test.go,preflight_windows_test.go,restore.go,restore_test.go}`; +- `tools/tht/internal/safeio/{claim_unix.go,claim_windows.go,claim_windows_test.go,files.go,files_test.go,private_root_windows.go,private_windows.go,private_windows_test.go}`. + +Evidence/status changes are restricted to: + +- `.artifacts/task-15/automated-gates.json`; +- `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md`; +- `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-1-report.md`; +- `.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md`; +- `PROJECT_STATE.md`. + +Machine-readable evidence SHA-256: +`5c110b7b2607693de078def441b10290c5a29024c83b7e5a0ced894b72b7507f`. + +## Git and protection status + +- The evidence commit contains only the five evidence/status files listed above; no source is + changed after frozen source `10cd66fe6a5b484a4dc569326a228c1c5484a5d4`. +- After the evidence commit and push, the intended status is synchronized + `feat/thoth-auth...origin/feat/thoth-auth` with only protected untracked `.playwright-cli/` and + `.thothctl/`. +- `AGENTS.md`, `CLAUDE.md`, and `docs/agents/` are untouched. No generated `tools/tht/tht` exists. +- Evidence commit SHA is reported externally after commit creation because a commit cannot contain + its own final hash. diff --git a/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-2-report.md b/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-2-report.md new file mode 100644 index 00000000..77f2fbbd --- /dev/null +++ b/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-2-report.md @@ -0,0 +1,133 @@ +# Final-review fix round 2 report (sanitized) + +## Verdict + +- Base evidence head: `0f762ad6b67675356389cc546421a1c46ad5a736`. +- Frozen source: `2a9359071257f9b8a71d36ec2bbb25b161003f81` on `feat/thoth-auth`. +- Authentication remediation: **PASS**. +- Three original remediation Important findings: **RESOLVED**. +- Fix-round-2 bounded lifecycle Important: **ADDRESSED**. +- Fix-round-2 temporary Windows diagnostics Minor: **ADDRESSED**. +- Release readiness: **FAIL** for executed unrelated baseline gates, with unavailable + external/manual gates separately **PENDING**. +- Source and evidence are separate commits. The evidence-only phase changed no source or tests and + dispatched no workflow. + +## Finding disposition + +| Finding | Disposition | Evidence | +|---|---|---| +| Original Important — POSIX local-registry ownership | RESOLVED | Effective-UID ownership enforcement and its Node 24 coverage remain green at their recorded source. Fix round 2 did not alter this boundary. | +| Original Important — retained-capability StageArchive lifecycle | RESOLVED | Native Windows `internal/backup` passed on the exact source, preserving the retained-root staging and cleanup coverage. | +| Original Important — handle-relative Windows claim removal | RESOLVED | Native Windows `internal/safeio` and `internal/authstorage` passed on the exact source, including retained claim/consume coverage. | +| Fix-round-2 Important — fully bounded restore lifecycle test | ADDRESSED | Gate publication and release are context-aware; stage, outcome, admission, checkpoint, and verification waits are bounded; aborts cancel, safely release, bounded-join, then assert lock-free. The deterministic withheld-gate test proves prompt timeout/cancellation, worker join, and eventual lock release. | +| Fix-round-2 Minor — temporary Windows diagnostic matrix | ADDRESSED | `windowsRelativeOpenMatrix` and its diagnostic-only call/import were removed. Owner-only DACL shape, NT access normalization, full-control, cleanup, and retained no-delete tests remain. | + +The round-1 restore lifecycle finding was broadened by the scoped round-2 review: bounded release +alone was insufficient while stage publication, gate waits, and nearby outcome/admission waits +could still outlive a controller abort. The round-2 implementation closes that broader test +orchestration gap without changing production authentication semantics. + +## RED → GREEN record + +### RED + +The deterministic withheld-gate regression was introduced first and run without relying on a +global ten-minute package timeout: + +```text +go test ./internal/backup -run '^TestRestoreLifecycleCancellationJoinsWithWithheldGate$' -count=1 +``` + +It failed in approximately `0.64s` with: + +```text +cancelled restore worker did not join within the bounded deadline +``` + +This proved that cancellation did not yet unblock and join a worker retained at the lifecycle +gate. + +### GREEN and refactor + +- The gate uses a cancellation source shared by controller and worker. Both publication and + release are `select`-based and cancellation-aware. +- Shared bounded helpers cover stage, outcome, error, signal, release, and admission waits. +- Abort cleanup is ordered: cancel, cancel the controller gate when distinct, safely release a + pending gate, bounded-join the worker, then prove the lifecycle lock is free. +- Premature worker outcomes retain and surface their original error. +- The existing success, recovery, maintenance-barrier, stale-checkpoint, and verification + assertions remain active. + +Final local gates on the frozen source: + +```text +go test ./internal/backup -run '^(TestRestoreLifecycleCancellationJoinsWithWithheldGate|TestReleaseLifecycleStage|TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup|TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore|TestRestoreKeepsAdmissionBarrierActiveUntilVerificationCommits)$' -count=1 +go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1 +go test ./... -count=1 +go test -race ./... +go vet ./... +go build -o /tmp/thothii-tht-host-fix-round-2 ./cmd/tht +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c ./internal/safeio -o /tmp/tht-safeio-fix-round-2-windows.test.exe +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c ./internal/backup -o /tmp/tht-backup-fix-round-2-windows.test.exe +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c ./internal/authstorage -o /tmp/tht-authstorage-fix-round-2-windows.test.exe +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o /tmp/thothii-tht-fix-round-2-windows.exe ./cmd/tht +``` + +All commands passed. The final focused lifecycle run completed in `0.672s`; the full security +package run passed safeio, backup, and authstorage; race, vet, host build, Windows test-package +cross-compiles, and Windows CLI cross-compile also passed. Cross-compilation is recorded only as +compile evidence and is not used as native authority. + +## Exact-source native certification + +- Controller-authorized run: `32147345625` — + https://github.com/mptyl/ThothII/actions/runs/32147345625. +- Event/status/conclusion: `workflow_dispatch` / `completed` / `failure`. +- Head SHA: `2a9359071257f9b8a71d36ec2bbb25b161003f81`, exactly matching the frozen source. +- Windows job: `Windows clone and Compose contract`, job `95744249248` — + https://github.com/mptyl/ThothII/actions/runs/32147345625/job/95744249248. +- Native step: `Run native Windows retained-capability tests` — **PASS**. +- Exact unfiltered command: + `go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1`. +- Native package results: + - `internal/safeio` PASS (`22.058s`); + - `internal/backup` PASS (`7.161s`); + - `internal/authstorage` PASS (`16.088s`). + +The Windows job failed only in the following baseline clone-contract step. PowerShell reported a +parser error at `scripts/test-windows-clone-contract.ps1:208` because `$remoteYaml:` is not a +delimited variable reference. This later failure does not alter the successful native Go step. + +## Separate release-readiness verdict + +| Gate | Classification | Exact outcome | +|---|---|---| +| Authentication remediation | PASS | Source and exact-source native three-package authority are green. | +| Windows clone contract | FAIL / baseline | Job `95744249248`; parser error at `scripts/test-windows-clone-contract.ps1:208`, after native PASS. | +| LF, Compose, docs, and TypeScript | FAIL / baseline CI contract | Job `95744249458`; unified Compose passed, then the existing unset-`TMPDIR` failure stopped the contract step. Downstream commands were skipped. | +| Linux Docker deployment and rollback | FAIL / infrastructure prerequisite | Job `95744249354`; the existing missing-`rg` prerequisite stopped the smoke before deployment. Cleanup passed and no new image manifest was generated. | +| Native Windows Docker Desktop/WSL2 startup | NOT_RUN / BLOCKED | Job `95744250450` was skipped by workflow conditions; no native Docker/WSL2 command ran. | +| L2, real PSD/manual acceptance, provider readiness | PENDING | Required secrets, identity/access, or provider prerequisites remain unavailable. | + +Executed failures remain `FAIL`; skipped commands are `NOT_RUN` / `BLOCKED`; unavailable external +gates remain `PENDING`. Therefore remediation PASS does not imply release readiness PASS. + +## Evidence and protection status + +- Machine-readable evidence: `.artifacts/task-15/automated-gates.json`; SHA-256 + `6c516db5c2064c4a4a2e5f25961b993cd4a8fe020bbbb822fbac7faa0c119599`. +- Current Task 4 report: + `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md`. +- Retained Task 15 report: + `.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md`. +- Project snapshot: `PROJECT_STATE.md`. +- Historical Docker evidence remains bound to its recorded older source and is not reused as proof + for `2a9359071257f9b8a71d36ec2bbb25b161003f81`. +- `.playwright-cli/` and `.thothctl/` remain protected and untracked. No source/test file, + instruction file, workflow, or `docs/agents/` content changed in this evidence phase. +- The separate evidence commit SHA is reported after commit creation because a commit cannot + contain its own final hash. + +No credentials, tokens, internal endpoints, identities, registry names, raw environments, or +browser traces are retained in this report. diff --git a/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md b/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md new file mode 100644 index 00000000..6a6d3b8c --- /dev/null +++ b/.superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md @@ -0,0 +1,131 @@ +# Task 4 authentication remediation recertification (sanitized) + +## Fix-round-2 recertification — remediation PASS + +- Exact source: `2a9359071257f9b8a71d36ec2bbb25b161003f81` on `feat/thoth-auth`. +- Authorized workflow: completed run `32147345625`, + https://github.com/mptyl/ThothII/actions/runs/32147345625, exact matching head SHA. +- Native job: `Windows clone and Compose contract`, job `95744249248`. +- Required native step: `Run native Windows retained-capability tests` — **PASS**. +- Exact unfiltered command: + `go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1`. +- Package evidence: `internal/safeio` PASS (`22.058s`), `internal/backup` PASS (`7.161s`), + `internal/authstorage` PASS (`16.088s`). This includes explicit native Windows + StageArchive retained-capability and concurrent claim-consume coverage. +- The later `Verify Windows clone contract` step failed independently at + `scripts/test-windows-clone-contract.ps1:208`: PowerShell parsed `$remoteYaml:` as an invalid + variable reference. This baseline deployment-contract failure does not change the native Go + package result. +- The optional `Native Windows Docker Desktop/WSL2 startup` job was skipped by workflow + conditions. It is `NOT_RUN` / `BLOCKED`, because no Docker Desktop/WSL2 command executed. +- The workflow reached `completed` with conclusion `failure`: the native authentication step is + PASS, while the later clone-contract, LF/Compose, and Linux Docker baseline steps are FAIL. +- Existing LF/Compose job `95744249458` and Linux Docker job `95744249354` failures repeated + before downstream work. Skipped commands are `NOT_RUN` / `BLOCKED`, not executed failures. + External L2/PSD/provider gates remain `PENDING`. + +Finding disposition is explicit: the three original remediation Important findings remain +**RESOLVED**; the fix-round-2 lifecycle Important is **ADDRESSED**; and the temporary Windows +diagnostic-matrix Minor is **ADDRESSED**. Authentication remediation is **PASS**. This does not +change overall release readiness: executed baseline gates remain **FAIL**, while unavailable +external/manual gates remain **PENDING**. + +The section below is retained as historical evidence for the pre-fix frozen source. + +## Historical pre-fix result + +- Frozen source under test: `b31b27e5845ffd3adf311429367319beaba263c7` on `feat/thoth-auth`. +- Freeze check: PASS. No tracked source changed during certification. The only untracked paths + retained are `.playwright-cli/` and `.thothctl/`. +- Certification window: `2026-08-18T09:26Z` to `2026-08-18T09:48:36Z` (UTC; the start marker is + minute-precision because no earlier second-level operator timestamp was captured). +- Overall result: `FAIL` / `CHANGES_REQUIRED`. The three Important findings are not closed and + authentication is not implementation-complete or release-complete. + +## Local gate matrix + +| Gate | Result | Sanitized evidence | +|---|---|---| +| Go focused security tests | PASS | `safeio`, `backup`, and `authstorage`; 3 packages | +| Go race/vet/host build | PASS | 18 race-tested packages; vet and host CLI build exit 0 | +| Windows amd64 cross-compile | PASS | focused safeio/backup test binaries and CLI build; compile-only | +| POSIX registry ownership | PASS | Node 24 backend suite includes local-registry ownership coverage | +| Unix StageArchive retained capability | PASS | focused safeio/backup and race coverage passed on host | +| Backend Node 24 | PASS | 76 files / 1092 tests; typecheck and build passed | +| Frontend Node 24 | PASS | 61 files / 444 tests; typecheck and build passed | +| Authentication/F1 browser smoke | PASS | Node `v24.16.0`; filtered E2E 1 passed; sentinel scan passed | +| Harness pytest | FAIL | 951 passed, 1 failed, 4 skipped, 232 subtests; `test_f4_emits_column_types` could not find `workflow.yaml` from its test cwd | +| Ruff | FAIL | 192 errors; known baseline | +| Authentication docs smoke | PASS | required-term and forbidden-word checks passed | +| Shell syntax | PASS | `bash -n scripts/*.sh` | +| Default Compose contract | FAIL | required `THT_WORKSPACE_GIT_REMOTE` was unavailable | +| Unified Compose contract | FAIL | `compose.unified.yaml` is absent from the frozen source | +| Unified Docker smoke | FAIL | workflow attempted it on the frozen SHA but stopped before deployment because `rg` was unavailable; cleanup proof passed and no new image manifest was generated | +| L2 / PSD manual / provider readiness | PENDING | required external secrets, identities/access, or provider prerequisites unavailable/not reached | + +The first full backend Vitest attempt had one workspace-registry timeout. The focused test and a +fresh complete rerun passed, so the current backend result above is the fresh complete rerun. + +## Native Windows authority + +The authorized dispatch was bound to the frozen SHA: + +- Run: `32122302381` +- URL: https://github.com/mptyl/ThothII/actions/runs/32122302381 +- Head SHA: `b31b27e5845ffd3adf311429367319beaba263c7` +- Workflow conclusion: `failure` +- Job: `Windows clone and Compose contract`, job `95665197885` +- Job URL: https://github.com/mptyl/ThothII/actions/runs/32122302381/job/95665197885 +- Native step: `Run native Windows retained-capability tests` — `failure` +- Executed command: `go test ./internal/safeio ./internal/backup -count=1` +- Observed focused failures include `TestRemoveCanonicalPrivateClaimRetainsParentDuringDeletion` + and `TestRemoveCanonicalPrivateClaimPreservesOrphan`. +- The backup package timed out in + `TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup` after `10m0s`. +- Additional backup failures included retained-staging `unsafe file` results, Windows temporary-file + cleanup reporting that a file was still in use, and fixture cases that could not read external + secret declarations. The first two categories are remediation/security-boundary failures; the + fixture declaration failures are recorded as an accompanying CI-fixture issue. +- `internal/authstorage` was not requested by the frozen workflow step and therefore has no native + Windows execution evidence. Cross-compilation does not substitute for this gate. + +This native failure is the blocking gate. No source fix was attempted, and no later Docker smoke +was run locally after the failure. + +## Other workflow failures + +- `LF, Compose, docs, and TypeScript` (job `95665197839`) failed in + `Verify Compose and installation contracts` after the unified Compose contract itself passed. + `test-no-deployment-coupling-scope.sh` aborted on `TMPDIR: unbound variable`; this is classified + as a baseline/CI contract prerequisite, and later docs/TypeScript steps were skipped. +- `Linux Docker deployment and rollback` (job `95665197846`) failed before deployment because the + runner did not provide `rg` (`Task 13 smoke failed: rg is required`). The sanitized cleanup proof + passed and no Docker image manifest was generated. This is classified as an infrastructure + prerequisite failure, not as evidence of a remediation regression. + +## Evidence and provenance + +- Current machine-readable matrix: `.artifacts/task-15/automated-gates.json`; SHA-256 + `6c516db5c2064c4a4a2e5f25961b993cd4a8fe020bbbb822fbac7faa0c119599`. +- Current requested report: this file (SHA-256 recorded after the evidence commit if needed for + external indexing). +- Current fix-round report: + `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-2-report.md`. +- Historical Docker image manifest: `.artifacts/task-15/unified-docker-images.json`, unchanged + because no new immutable-source Docker smoke ran. Its retained historical SHA-256 is + `9c8dec4546909fd93799dbcf374bcb3a89bc46cfe0fd482472c0cbe757ddf5b6`, bound to historical source + `74b062f1a737103524cbe706346cfd65f87cdfd1`, not to this Task 4 candidate. +- The historical Task 15 report remains provenance for earlier source SHAs; its current addendum + records this recertification separately. + +No credentials, tokens, internal endpoints, provider identities, registry names, raw environments, +or browser traces are retained here. + +## Separate verdicts + +- Three Important findings: `CHANGES_REQUIRED`. Native Windows retained-capability authority + failed, and the frozen workflow omits the required `authstorage` package from its native command. +- Overall release readiness: `FAIL` with additional `PENDING` gates. The native Windows remediation + gate failed; the remote Docker attempt failed on a missing runner prerequisite; existing + Ruff/harness/Compose failures and external/manual prerequisites remain unresolved; and no + successful new unified Docker image evidence exists. diff --git a/AGENTS.md b/AGENTS.md index 34fc1175..46840fc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,10 +11,14 @@ detail. Design history lives in `docs/superpowers/specs/` and `docs/superpowers/ ## Commands -The repo has three independently-built layers. Run the **full stack** (real Pi + DWH, needs -VPN + `harness/.env` + `pi` on PATH) with `./scripts/run-stack.sh` (frontend :5173 → backend :8787). +The repo has three independently-built layers. Run the local Docker stack with `./scripts/run-stack.sh` after creating `deploy/env/local.env`; it starts the base+local Compose profile with `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. The core image contains Pi. Qdrant and Ollama are internal Compose services; DWH and LLM remain external configuration endpoints. -**harness/** (Python `tht` CLI + Pi gate extension) +**Native host CLI `tht`** (`tools/tht/`) +- Operator surface: `setup`, `start`, `stop`, `status`, `doctor`, `auth`, `workspace`, and `pi`. +- Use `tht --installation /thothii-installation.yaml ` for installation, + authentication, diagnostics, lifecycle, and workspace operations. + +**harness/** (Python workflow `tht` CLI + Pi gate extension) - Install: `cd harness && python -m venv .venv && pip install -e ".[dev]"` (puts `tht` on PATH) - Test: `.venv/bin/pytest -q` — `l2` (real GLM + remote DB) is opt-in via `addopts = -m 'not l2'`; `l0` (testcontainers) needs Docker - Single test: `.venv/bin/pytest tests/test_session_mutations.py::test_set_name -v` (or `-k `); include e2e with `-m l2` @@ -38,8 +42,8 @@ No ESLint on the TS layers — `tsc` is the gate. Tests use vitest + MSW (no net frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → DWH (read-only) ``` -- **The harness owns the workflow and all persistence.** `tht` (Python) is a deterministic - CLI; `harness/.pi/extensions/tht-gate.js` is a Pi extension that drives an **8-phase +- **The harness owns the workflow and all persistence.** The Python workflow CLI `tht` inside + `core` is deterministic; `harness/.pi/extensions/tht-gate.js` is a Pi extension that drives an **8-phase NL→SQL workflow**. The single source of workflow truth is `harness/workflow.yaml`; the orchestration rules the model must follow are `harness/.pi/skills/tht-sessione/SKILL.md`. "Current phase" is computed by folding the decision ledger (`harness/tht/phase.py`), not @@ -52,7 +56,8 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → There is no verbatim transcript store. A resumed Pi process rebuilds context from `tht session show ` + the on-disk artifacts. -- **The backend is a thin bridge with no database.** `ThtRunner` shells `tht` subcommands; +- **The backend is a thin bridge with no database.** `ThtRunner` shells the Python workflow `tht` + subcommands inside `core`; `PiProcessManager` runs one Pi child per session and bridges its RPC stream; `SessionBridge` maps Pi RPC events → client events (`ui_request`/`text_delta`/`info`); `SseHub` fans them out over SSE to the browser. App settings live in a JSON file diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 6218edd0..32225972 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,9 +1,524 @@ # ThothII — Project State -> Starting-point snapshot for new sessions. Last updated: 2026-07-23 (session summary redesign live). +> Starting-point snapshot for new sessions. +> **Requisito finale del progetto (owner, 2026-08-11):** al termine dell'ultima fase tecnica deve +> essere prodotto un documento unico che guidi l'utente passo-passo su (1) come preparare il +> repository dei workspace su Git secondo le regole del progetto, (2) come usare gli strumenti di +> ThothII per il repository (app + CLI `tht`), (3) come usare l'applicazione ThothII di base +> (sessioni, domande, gate). Il documento userà parole semplici ed esempi; i dettagli tecnici +> resteranno nei contratti esistenti. Esempio pratico completo: Policlinico San Donato. +> Last updated: 2026-08-18 (final-review fix round 2 recorded; native Windows authentication gate +> passed, remediation is complete, and unrelated release gates remain open). > Point a fresh session here ("read PROJECT_STATE.md") before substantial work. -## Session summary redesign — LIVE 2026-07-23 +### Authentication final-review fix round 2 — remediation PASS, release gates remain (2026-08-18) + +- Frozen source is `2a9359071257f9b8a71d36ec2bbb25b161003f81` on `feat/thoth-auth`. + Source and evidence are separate commits; generated local runtime-state directories remain + untracked and must not be staged. +- Local PASS on the frozen source: exact `safeio`/`backup`/`authstorage` tests, full Go race suite, + `go vet`, macOS host build, Windows amd64 package cross-compiles, and Windows CLI build. +- The lifecycle tests now use context-aware gate publication/release, bounded waits for stages, + outcomes and admission, and cancel plus bounded worker join before lock-release assertions. A + deterministic withheld-gate case proves timeout, cancellation, join, and eventual lock release. + The temporary Windows relative-open diagnostic matrix was removed without reducing DACL, NT + normalization, or retained no-delete assertions. +- Authorized exact-source workflow run `32147345625` completed on the exact frozen SHA and + executed the unfiltered native command + `go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1`. The required step + passed: safeio `22.058s`, backup `7.161s`, authstorage `16.088s`. Native Windows StageArchive and + concurrent claim-consume evidence are therefore PASS, not inferred from cross-compilation. +- The same Windows job later failed the unrelated clone-contract script at + `scripts/test-windows-clone-contract.ps1:208` because `$remoteYaml:` is not a valid PowerShell + variable reference. LF/Compose and Linux Docker baseline failures also repeated. The optional + Windows Docker startup job was skipped without executing and is `NOT_RUN` / `BLOCKED`; the + overall completed run conclusion is `failure` because the baseline jobs remain red. +- Historical Node/auth/browser/docs PASS and harness/Ruff/Compose FAIL evidence remains bound to + its recorded source where not rerun. L2, PSD/manual, and provider prerequisites remain + `PENDING`; no new Docker image manifest was generated. +- Durable evidence: `.artifacts/task-15/automated-gates.json`, + `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/task-4-report.md`, and + `.superpowers/sdd/2026-08-18-thothii-authentication-remediation/fix-round-2-report.md`. +- Current automated-gates SHA-256 is + `6c516db5c2064c4a4a2e5f25961b993cd4a8fe020bbbb822fbac7faa0c119599`; the historical Docker + manifest remains bound to its recorded older source and was not reused for this candidate. +- **State:** the three original remediation Important findings remain `RESOLVED`; the fix-round-2 + lifecycle Important is `ADDRESSED`; the Windows diagnostics Minor is `ADDRESSED`; authentication + remediation is `PASS`. Separately, release readiness remains `FAIL`, with L2, PSD/manual, and + provider gates `PENDING`, until unrelated deployment, runner, baseline, and external gates close. + +### P3 effective configuration and `.tht-dwh` — implementation complete, automated PASS, manual PASS (2026-08-13) + +- **Scope:** P3 (PRD D3): a versioned shared canonicalizer produces the non-secret effective + DWH/preprocessing configuration and a stable logical identity + (`workspace://@v1:`), used identically by the application sessions and the operator + CLI. `OWNER.json` writes are versioned; legacy roots remain readable; content-only/Evidence-only + changes keep the identity (no forced reconfiguration), while DWH-affecting changes fail closed + (never silently reusing the old generation). +- **Memory:** explicit workspace-global `paths.memory` root with a guarded migration command + (`tht memory migrate`) that copies and verifies exactly one legacy JSONL under the workspace + lock and fails closed on conflicts. +- **Revision-scoped records:** schema and Evidence Qdrant point IDs, payloads and queries include + `workspace_revision`; memory/solved stay workspace-wide. +- **Operator contract:** `tht` now carries `effectiveConfigIdentity`/`configFingerprint`/ + `inputFingerprint` in results; the operator config lease path is deterministic for the same + revision+identity. +- **Retained evidence:** `.artifacts/p3-integration/p3-da9428d84f152fe059d41a89436496b7/` + (15/15 checks PASS), bound to clean source commit + `3b0726472e15c157…`. +- **Manual gate:** P3 walkthrough in `docs/testing/p2-p6-manual-verification.md`; decision + **PASS** (owner approval 2026-08-13). +### P4 Qdrant collection lifecycle — implementation complete, automated PASS, manual PASS (2026-08-13) + +- **Scope:** P4 (PRD D4): one shared TypeScript collection manager owns the Qdrant collection + and payload-index contract; session admission self-heals a missing collection (1024/cosine + + the 8 required keyword payload indexes) and adds missing indexes, but never mutates an + incompatible collection (`semantic_index_incompatible`); the operator path keeps + `require_existing` semantics. +- **Host CLI:** `tht workspace vector inspect` (read-only contract report) and + `tht workspace vector rebuild --workspace --collection --confirm --destroy` + (guarded delete/recreate of only the descriptor-owned collection, with durable state before + deletion and verification after recreation; mismatched confirmation or missing `--destroy` + → exit 2). +- **Key files:** `backend/src/workspaces/qdrant-collection.ts` (+test), `backend/src/tht/tht-runner.ts` + (`qdrantEnsure` self-heal for admission; default `require_existing` elsewhere), + `backend/src/workspaces/runtime-config-lease.ts` (lease exposes `semanticQdrantUrl`), + `backend/src/workspace-maintenance.ts` + `preprocessing-service.ts` (`vector-inspect`/`vector-rebuild` + operator commands), `tools/tht/internal/workspaceops/operations.go` (+tests). +- **Automated acceptance:** PASS 11/11 (run `p4-466bbfdea9ef3111f36baa99fc2d64aa`, + report `.artifacts/p4-integration/p4-466bbfdea9ef3111f36baa99fc2d64aa/` retained via `--keep`, + bound to clean source commit `e056c19e6214254a9e3b2390e24c389920b84e95`): preflight, clean_state, + ownership, qdrant_up, self_heal_create_missing, self_heal_repairs_missing_index, + incompatible_refused, require_existing_refused, rebuild_recreates_contract, secret_scan, + cleanup_confinement. +- **Gates:** backend 666/666 + tsc clean; Go build+test 9/9; p4 runner unit tests 3/3; harness + 841 passed (only the two pre-existing debt failures unchanged). +- **Manual acceptance:** PASS (owner approval 2026-08-13) — walkthrough section P4 in + `docs/testing/p2-p6-manual-verification.md`. + + +### P5 curated FK annotations in Git — implementation complete, automated PASS, manual PASS (2026-08-13) + +- **Scope:** P5 (PRD D5): the canonical curated FK file is `/schema/annotations.yaml`, + a regular Git blob at the same commit as the descriptor. Absence is compatible (empty canonical set + + warning); symlinks, trees/gitlinks, cross-namespace paths, oversized (>16 MiB), non-UTF-8, and + malformed objects are refused at activation. Activation synchronizes the blob to the immutable + revision root `/data/sessions//revisions//artifacts/mschema/annotations.yaml` with a + restrictive mode and an adjacent ownership manifest (`workspace`, `commit`, `blobId`, + `contentDigest`, `destination`); re-sync is idempotent and re-verifies, and tampered destinations + fail closed. +- **Runtime root:** the backend renders `paths.annotations_root` for the pinned revision while + `paths.artifacts`/`indexes`/`memory`/`sessions` stay workspace-global (the binding-keyed DWH cache + at `artifacts.parent` is untouched); the harness resolves annotations from `annotations_root` with a + legacy fallback. +- **Review primitive:** `tht ... workspace schema accept --run --yes` is the only human FK + review path. It validates the current synced Git blob with the harness parser and records + `{ reviewedCandidatesDigest, annotationsDigest, workspaceRevision, blobId }`. Missing `--yes`, an + unknown run, an empty/malformed blob, or a non-matching candidate fails closed (`annotation_invalid`) + without recording a review. The P2 host-file `schema check --annotations --reviewed-candidates` + review write is superseded (read-only validation only). +- **Continuation gate:** `preprocess run` continues only when the accepted review's blob digest equals + the current revision's synced annotations digest and the DWH binding is compatible; otherwise it + records a new `manual_review_required` checkpoint. +- **Key files:** `backend/src/workspaces/annotations-sync.ts` (+test), `backend/src/workspaces/ + annotations.ts`, `backend/src/workspaces/git-repository.ts` (`annotationsObject`), + `backend/src/workspaces/registry.ts` (activation validation + sync), `backend/src/workspaces/ + preprocessing-service.ts` (`acceptSchema` + continuation gate), `backend/src/workspace-maintenance.ts` + (`schema-accept`), `tools/tht/internal/workspaceops/operations.go` (+tests), `harness/tht/ + config.py` + `cli/schema_cmd.py` (`paths.annotations_root`), `docs/contracts/ + workspace-preprocessing-cli.md`. +- **Gates:** backend **689/689** + tsc clean; Go build+test 9/9; harness focused schema/annotations + 52 passed. Full-suite re-run and the clean-state process goal are recorded at the acceptance gate. +- **Automated acceptance:** PASS 10/10 (run `p5-66b1f1e74f147a23c0a4bff04e6d2a4c`, report + `.artifacts/p5-integration/p5-66b1f1e74f147a23c0a4bff04e6d2a4c/` retained via `--keep`, bound to + clean source commit `9db0299063d5068198c05dc467d7f86dc34de85b`): preflight, clean_state, ownership, + activation_sync, accept_happy_path, revision_isolation, accept_negatives, continuation_gate, + secret_scan, cleanup_confinement. Runner: `scripts/p5-acceptance.sh` / + `backend/scripts/p5-acceptance.mjs` (+unit test `scripts/test-p5-acceptance.sh`). +- **Manual acceptance:** PASS (owner approval 2026-08-13) — walkthrough section P5 in + `docs/testing/p2-p6-manual-verification.md`. + +### P6 commit-addressed Evidence materialization — implementation complete, automated PASS, manual PASS (2026-08-13) + +- **Scope:** P6 (PRD D6): filesystem Evidence `/evidence` is materialized from the exact pinned + Git commit into the immutable revision content root `/snapshots///evidence` + at activation, with a sibling bounded manifest `/evidence.manifest.json` whose digest is chained + into `snapshot.json`. +- **Safety:** fixed Git plumbing (`ls-tree -r -z` + `cat-file blob`), no shell, no mobile checkout; + symlinks/gitlinks at any depth, traversal/absolute/duplicate/cross-namespace paths, and non-regular + modes are refused. Installation-local bounds (defaults): 4096 entries, 64 MiB total, 8 MiB per + file, 4096 path bytes, 1 MiB manifest; a size-sum preflight runs before writing and no partial root + is published. Re-activation reuses a valid root and fails closed on a tampered manifest. +- **Engine:** `evidencePolicy` no longer stops filesystem sources (`evidence_materialization_required` + retired); `preprocess evidence`/`preprocess run` operate on the materialized root. Evidence Qdrant + records remain revision-scoped; corpus ACTIVE is revision-qualified. HTTP/S3 Evidence is unchanged. +- **Retention:** materialized roots live inside the commit-addressed snapshot directory, so they are + retained while pinned and removed by the existing snapshot retention scan when unreferenced. +- **Key files:** `backend/src/workspaces/evidence-materialization.ts` (+test), + `backend/src/workspaces/git-repository.ts` (`evidenceTreeObjects`/`evidenceTreeId`/ + `evidenceBlobBytes`/`gitObjectSize`), `backend/src/workspaces/registry.ts` (activation staging + + integrity chain), `backend/src/workspaces/preprocessing-service.ts` (stop removal), + `backend/src/workspaces/types.ts` + `config.ts` (limits), `docs/contracts/ + workspace-preprocessing-cli.md`. +- **Gates:** backend **698/698** + tsc clean; Go build+test 9/9 (unchanged); harness focused suites + pass. Full-suite re-run and the clean-state process goal recorded at the acceptance gate. +- **Automated acceptance:** PASS 10/10 (run `p6-7a4c4d0ebb63399cfa9f674738b9e8fc`, report + `.artifacts/p6-integration/p6-7a4c4d0ebb63399cfa9f674738b9e8fc/` retained via `--keep`, bound to + clean source commit `124891bbfe8dc8270e8b58c4206150eb8bebeaa7`): preflight, clean_state, ownership, + activation_materialization, evidence_preprocess, revision_isolation, unsafe_tree_refused, + bound_refused, secret_scan, cleanup_confinement. Runner: `scripts/p6-acceptance.sh` / + `backend/scripts/p6-acceptance.mjs` (+unit test `scripts/test-p6-acceptance.sh`). +- **Manual acceptance:** PASS (owner approval 2026-08-13) — walkthrough section P6 in + `docs/testing/p2-p6-manual-verification.md`. + +### P7 PSD migration — plan + repository restructured + local validation PASS; owner-gated (2026-08-13) + +- **Plan:** `docs/superpowers/plans/2026-08-13-p7-psd-migration.md`. +- **Done (autonomous):** `/Users/mp/projects/tht-workspace-psd` restructured to the P1.1 layout and + committed (`thoth-workspaces.yaml` + `psd-clinical/workspace.yaml` schema v3 + `psd-clinical/ + evidence/` 36 `.md` + `psd-clinical/schema/annotations.yaml` 42 KB); legacy runtime dirs gitignored + and the old flat `psd.yaml` retired. A local `WorkspaceRegistry.bootstrap()` against a scratch bare + clone **activated `psd-clinical`** (descriptor valid, 36 Evidence materialized + manifest, 42 KB + annotations synced, `workspace-docs` generated) with no DWH/secret access. +- **Templates:** `deploy/psd/{workspace-bindings,operator,thothii-installation}.env.example` + + gitignored `secrets/`; operator checklist in `docs/install/psd-workspace-setup.md` (registered in + MkDocs nav). +- **Published (2026-08-13):** private repo `https://github.com/mptyl/tht-workspace-psd` (main = + `d4f9185`), consumed via SSH deploy key `thothii-psd` (read-write, passphrase-less, generated in + `deploy/psd/secrets/`). Real operator config is wired (gitignored): `deploy/psd/operator.env`, + `workspace-bindings.env`, `thothii-installation.yaml`, `connector-secrets.yaml` + `secrets/` + (DWH X-API-Key reused from the legacy `.env`; no CA — the DWH REST is public HTTPS). +- **Stack live:** started via `tht start` (project `thothii-70417a3e30ea`), all services + healthy, `qwen3-embedding:0.6b` present; the registry cloned + activated `psd-clinical` + (`ready`); `tht workspace inspect` returns `ok` with descriptor/catalog/runtime identities. + Gotcha recorded: `tht` uses a per-descriptor Compose project name, so the stack must be + started with `tht start` (not a raw `compose-with-preflight.sh up`). +- **Preprocessing live (2026-08-13):** with VPN active, `tht workspace preprocess run + --workspace psd-clinical` **succeeded** against the real PSD DWH — DWH introspection + LSH + (163 tables / 2275 columns), FK review (no new candidates: the 42 KB curated annotations are + authoritative), schema index (2438 records) and filesystem Evidence index (36 docs / 43 chunks). + Qdrant `psd-clinical` now holds **2482 revision-scoped points** (`schema_table` 164, + `schema_column` 2275, `evidence` 43; all carry `workspace_revision`). Rerun is idempotent + (Evidence `unchanged: 36`). +- **Fixes shipped during the live run** (real-DWH scale revealed them): (1) pruned ~95 GB of orphaned + acceptance-run Docker volumes; (2) raised `workspace-maintenance` tmpfs `/tmp` 64 MiB → 1 GiB + (PSD LSH snapshot is ~105 MB); (3) `vector rebuild` now recreates the 8 keyword payload indexes + (it only created dimensions/distance); (4) Qdrant upserts are chunked (256 points/batch) — a 2438- + record schema batch exceeded Qdrant's 32 MiB JSON limit; (5) frozen Evidence metadata lists now + stay lists (`FrozenList`) instead of tuples, preserving JSON shape; (6) embedding timeout 30 s → + 300 s and batch 32 → 16 for large CPU corpora. +- **Remaining:** live session smoke on `psd-clinical` (P8 L2) — create a session with a real + natural-language question and reach the first reviewer gate. + +### Final aggregate P2–P6 verification — automated PASS, manual PENDING (2026-08-13) + +- **Aggregate process goal:** one clean-state run exercises the complete DWH → FK → schema → + filesystem Evidence chain through `tht`/the operator surface, proves idempotency and + revision isolation, proves a second installation consumes the same Git workspace with its own + state, exercises unsafe-tree and bound negatives, and cleans only owned resources. +- **Automated acceptance:** PASS 12/12 (run `p2p6-ee542112c526ef0d4c25ddf6c8bc164b`, report + `.artifacts/p2p6-integration/p2p6-ee542112c526ef0d4c25ddf6c8bc164b/` retained via `--keep`, bound + to clean source commit `1dcf4051b0d9db8ae163e4d7c53871564ca3c564`): preflight, clean_state, + ownership, activation_materialization, dwh_chain, fk_schema_evidence_chain, revision_isolation, + second_installation, unsafe_tree_refused, bound_refused, secret_scan, cleanup_confinement. Runner: + `scripts/p2p6-acceptance.sh` / `backend/scripts/p2p6-acceptance.mjs` (+unit test). +- **Full suites + builds (design §10):** harness **873 passed / 4 deselected** (with color disabled; + the forced-color environment splits `--help` flags and trips the gate-CLI consistency test only); + backend **698/698** + tsc + build; frontend **364/364** + `tsc -b` + build; `tht` Go + build+test **9/9**; `git diff --check` clean. +- **Manual acceptance:** PENDING — "Final aggregate P2–P6 verification" in + `docs/testing/p2-p6-manual-verification.md`. + +### User-guide deliverable (owner requirement) — written, review PENDING (2026-08-13) + +- **`docs/guida-utente.md`** (Italian, simple words + examples) covers: (1) preparing the workspace + Git repository (catalog + schema-v3 descriptor + Evidence + curated annotations), (2) using the + ThothII tools for the repository (`tht` commands + read-only workspace management), and (3) + using the base ThothII application (sessions, questions, gates). It ends with a complete + Policlinico San Donato walkthrough and links to the technical contracts. +- Registered in the MkDocs nav (`mkdocs.yml`). Owner review PENDING. + +### P2 host preprocessing CLI — implementation complete, automated PASS, manual PENDING (2026-08-11) + +- **Scope:** P2 (PRD D2, based on the P1.1 registry contract): the installed native `tht` + binary is the only host interface for workspace preprocessing. Commands: `workspace inspect`, + `preprocess dwh`, `schema suggest-fks`, `schema check`, `index-schema`, `preprocess evidence`, + `preprocess run`, with the exact grammar, file-ingress bounds, result contract and exit codes in + `docs/contracts/workspace-preprocessing-cli.md`. +- **Operator:** `workspace-maintenance` is a profile-gated Compose service sharing the core image, + with no Pi auth/state, no backend/Pi/frontend listener, no Git credentials, and a compiled Node + entrypoint (`backend/src/workspace-maintenance.ts`) driving the existing harness engine through + pristine JSON machine interfaces (`schema_cmd.py`, `vector_cmd.py`, `preprocess_cmd.py`). +- **Boundaries honored:** FK review is digest-bound (candidate digest == persisted artifact; a + review accepted for the same candidate content counts); Qdrant collections are never created by + the product path (`require_existing` + pre-provisioned fixture, P4 owns lifecycle); filesystem + Evidence stops with `evidence_materialization_required` (P6); HTTP Evidence enforces an + installation private-host allowlist; `ssh_tunnel` stays fail-closed (P10); cross-revision DWH + reuse is explicitly P3. +- **Retained evidence:** `.artifacts/p2-integration/p2-b109757b26388a5ed6b1d173dee86584/` + (11/11 checks PASS), bound to clean source commit + `de5de36f9a4edfd4fbebf277822090871ccdd61f`. +- **Manual gate:** P2 walkthrough in `docs/testing/p2-p6-manual-verification.md`; the owner + approved P2 on 2026-08-11 (manual acceptance PASS). P3 and later start only after an explicit + new authorization. + +### P1.1 workspace-directory registry — automated integration PASS, manual PENDING (2026-08-11) + +- **Scope:** P1 correction (not preprocessing). Root curator-owned catalog `thoth-workspaces.yaml`; + one self-contained directory per workspace (`/workspace.yaml`, optional `/evidence/**`); + generated docs stay API-owned under `workspace-docs/`; internal immutable snapshots remain + flat (`//.yaml`) to preserve session pins and runtime trust. +- **Ownership:** the API may create a descriptor once when its catalog slot exists and the + descriptor Git object is absent at the exact base commit. Existing descriptors and curated + content are curator-owned and change only through Git commit/push then installation pull. + Update/delete publish payloads are refused as HTTP 409 `workspace_curator_owned`. Catalog and + Evidence are never written/staged/cleaned by the API. Explicit pull may produce one deterministic + docs-only follow-up commit that never touches curator bytes. +- **Schema/UI:** schema v3 remains the only descriptor schema; filesystem Evidence URI is exactly + `/evidence`. Browser workspace management is read-only for ready workspaces (Pull/Sync, + Validate, installation Test, Export, Evidence summary, curator Git guidance) and offers an + editable bootstrap form only for `configuration_required` catalog slots. +- **Retained evidence:** `.artifacts/p11-integration/p11-ac0b047024fb09eeca218512526a6b23/` + (`report.json` sha256 `44250145fede36de5de941262beb833c920e8c73366d987cdd738856aac6f6d6`), + 19/19 checks PASS, bound to clean source commit + `eac472011e465c24572d9a6bae14de0fb3e246c0` / tree `4fd15ec28d3b7967b7b8757158013307fb20f3b9`. +- **Verification:** backend Vitest **634 passed / 41 files** + tsc + build; frontend Vitest + **364 passed / 54 files** + tsc + build; harness focused Evidence/config pytest **39 passed**; + install-docs and schema-v3-only gates PASS; `workspace-registry-smoke.sh` and + `unified-deployment-smoke.sh` full Docker runs PASS with exact cleanup. +- **Known limitations:** P2–P6 plans/designs are unchanged and their old source paths are + inventoried for a later owner-approved adaptation plan. Windows Docker startup and native + PowerShell contract were not executed on a Windows host. P1's accepted historical evidence and + process artifacts remain untouched; the old P1 process commands are not rerunnable against the + superseding P1.1 repository contract. +- **Manual gate:** `.artifacts/manual-acceptance/p11/` prepared for the reviewer; + follow `docs/testing/p11-manual-acceptance.md`. The owner reviewed the walkthrough and + approved the implementation on 2026-08-11. + +```text +P1.1 automated integration: PASS +P1.1 manual acceptance: PASS (owner approval 2026-08-11) +``` + +# P1 configuration process — ACCEPTED 2026-08-10 + +- Retained evidence: `.artifacts/p1-integration/p1-038bf31360180dc831220b33fbadcfe6/report.md` +- Final report hashes: `report.json` `f07d49097966de6f0307490089fdb2ae61379c04b7fc7177d3c44cf001e1b46a`; `report.md` `09b6a9e9ad9eed2b049e286af452e12fa1f3174ea8d633253542604470890c6c`. +- automated integration: PASS +- manual acceptance: PASS — explicitly approved by the project reviewer on 2026-08-10. +- The retained run is bound to clean source commit + `c7338969d7c7c1c396d9099b7ab2d309b70ab6cf` and tree + `5f7013904806054b9f587230be89f34cbc80f5fc`. Its hash-bound provenance contains exact + 43-file backend source and 39-file compiled `dist` manifests (manifest SHA-256 + `eb6d6c77c78d14c798b50d0be430ad124b8fd8965afbc4bb07d358089d23f49` and `9f9e8899f8aca882ff08d49ec6cd00caeea75691c8280095a9b88bc74939a30a`). +- The retained audit has exactly 15 PASS checks and 134 unique declared artifacts whose final + bytes match every SHA-256 declaration. It records 749 PASS command events and 1,664 production + child/network events, with listener shutdown and refusal checks recorded in the final ownership + artifact. Raw Git rejects configured executable diff drivers and other helper-bearing state. +- Manual production acceptance binds every regular compiled distribution file through an immutable + manifest and cached verified module bytes; imported dependency replacement is refused before + RUNNING. Snapshot rendering validates the bounded `snapshot.json`, expected digest, and Git blob + identity, refusing regular source replacement without publishing output. +- Final Task 8/9 focused suites pass (48/48 Task 8; 59/59 manual acceptance and renderer checks), + backend TypeScript/build pass, frontend tests/build pass. Historical harness pytest/Ruff debt + remains unrelated to this P1 work. + +## Internal Qdrant + Ollama semantic infrastructure — LIVE 2026-08-08 + +- **Compose topology.** The mandatory application stack is `frontend`, `core`, `qdrant`, + `embedding`, and the one-shot `embedding-model-init`. Startup is CPU-first by default; Linux + hosts may opt into GPU exposure with `THOTH_ENABLE_EMBEDDING_GPU=1`. Qdrant is private on the + Compose network and persists `/qdrant/storage` in `qdrant-data`. Ollama persists its local model + cache in `embedding-models`, and `embedding-model-init` blocks `core` until + `qwen3-embedding:0.6b` is present. + +- **Semantic contract.** Internal semantic indexing is fixed to `qwen3-embedding:0.6b`, + `1024` dimensions, and cosine distance. Schema v3 is the only accepted workspace descriptor. + Schema v1 and v2 workspace descriptors are rejected before activation. Candidate snapshot + validation makes activation or a pull fail atomically and leaves the prior valid snapshot active; + there is no in-product migrator or automatic conversion. One workspace owns one Qdrant + collection, and + schema, Evidence, and Memory records coexist inside that collection with payload `kind` + separation. + +- **Final review runtime barriers.** Operational routes, retained session pins, and runtime + rendering now require schema version 3 before resolving bindings, readiness, diagnostics, or + Pi. Session admission verifies the exact internal Qdrant collection (dimensions, cosine + distance, and required keyword payload indexes) before Ollama and before manifest persistence. + The Qdrant adapter binds every search/list/delete filter to its constructed workspace identity + and rejects conflicting caller namespaces. +- **Boundary and persistence.** Only DWH and LLM remain external runtime application endpoints. + There are no active external vector or embedding endpoint instructions, bindings, or secrets in + the supported operator manuals. Qdrant remains a derived but persistent semantic index: the + canonical sources of truth stay the workspace Git descriptors, phase artifacts, and memory + registry/ledger. The Ollama model cache is recoverable for offline startup but is not the + canonical source of semantic content. +- **Backup and recovery.** `./scripts/vector-backup.sh --project-name --output ` + archives exactly one labeled `_qdrant-data` volume and preserves the prior `qdrant` + running state. `./scripts/vector-restore.sh --project-name --input + --confirm-project ` requires the exact repeated project confirmation, validates manifest + and archive safety before stopping `qdrant`, stages rollback content, restores semantic storage + in place, and restarts `qdrant` only if it was previously running. Recovery requires the registry + to already hold a reviewed v3 descriptor revision compatible with the restored collection; the + helper does not restore descriptors, rename collections, or repair a semantic-index + incompatibility. Backup and restore share one atomic Docker-daemon lock per Compose + project/Qdrant volume; contenders fail before volume resolution, and cleanup removes the lock + only when its ownership labels still match. +- **Verification recorded for Task 13 final audit.** On Apple M4 Pro + (`Darwin 25.5.0`, Docker Server `29.6.2 linux/arm64`), harness pytest passed + **827 passed / 4 deselected**; backend Vitest passed **477/477** plus TypeScript and build; + frontend Vitest passed **374/374** plus TypeScript and build; `git diff --check` passed. + Deployment contracts passed: + `test-default-compose.sh`, `test-unified-compose.sh`, `test-internal-semantic-compose.sh`, + `test-no-deployment-coupling.sh`, `test-compose-secret-policy.sh`, and + `verify-workspace-install-docs.sh --fixtures-only`. +- **Task 13 Docker smoke evidence.** CPU semantic smoke passed in **217.34s** and proved + offline Qdrant/Ollama persistence plus exact cleanup. Workspace registry smoke passed in + **42.06s** in fix round 1 with a per-run image tag derived from the unique Compose project, + and proves exact cleanup of compose containers, volumes, networks, and only that smoke image. + Unified deployment smoke passed in **125.57s**; update-only rollback smoke + passed in **85.40s**; Linux server deployment smoke passed in **55.99s**. The previously + observed `tht` rollback failure did not recur. +- **Task 13 image and manual-gate notes.** Verified pinned runtime images: + `qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c` + and + `ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a`. + The workspace-registry smoke fix-round image used tag + `thothii-workspace-registry-smoke:thoth-workspace-registry-smoke-thoth-workspace-registry-smoke-10vi3a-19157`, + built manifest list `sha256:715b943057929418cad4aa71806d9edbaf823555d19bda6b875297617463fd4a` + with config `sha256:a566521981e08958aae9a12bfc7803bb5f3f835536b4bb8c39df8fcf26063161`, + and removed that exact reference during cleanup. Local GPU exposure (`THOTH_ENABLE_EMBEDDING_GPU=1`) and + Windows Docker Desktop startup were not manually executed in this run. +- **Task 13 known limitations.** Broad harness Ruff remains existing unrelated debt + (**220 errors**); touched harness files were verified Ruff-clean. The final active-reference + audit remains non-empty only in deterministic negative guards, retained off-repository migration + SQL, L2 compatibility fixtures, gitignored task notes, and historical reference notes. No active + schema-v3 operator manual or supported runtime deployment path retains external vector or + embedding endpoint coupling. +- **Final review fix verification.** Backend Vitest passed **477/477** plus TypeScript and build; + harness pytest passed **827 passed / 4 deselected** with the existing 74 warnings; touched Python + files are Ruff-clean. The complete `tht` Go suite, deterministic backup/restore safety test, + internal semantic Compose contract, no-deployment-coupling gate, CPU/offline semantic smoke, and + unified deployment smoke all pass after the final fix. The intermittent `tht` rollback failure was + traced to Docker Desktop alternating equivalent bind sources between `/private/...` and + `/host_mnt/private/...`. Exact state-v4 source hashes remain unchanged; only fresh bind + observations made by a Darwin `tht` carry non-serialized aliases for the rollback + comparison, so pre-fix recovery state remains readable and Linux `/host_mnt` paths remain + distinct. The rollback-only smoke passed twice consecutively after each fix revision, and the + subsequent full unified smoke passed with exact cleanup. + +# Historical archive +## Historical snapshots and archived reference notes + +### Historical snapshot — Unified deployment release gate, Task 13 (2026-08-05) + +- **Release coverage.** `scripts/unified-deployment-smoke.sh` gates the two-service render/build, + frontend-to-core routing, embedded pinned Pi, Git registry bootstrap, offline recreation, valid + update, invalid-update retention, and the four persistent stores. `scripts/tht-update-smoke.sh` + independently exercises the bad-Pi update and automatic rollback path. + `scripts/server-deployment-smoke.sh` starts the server plus required session overlays with the + same smoke-built core/frontend images, disposable bind roots/secrets/session configuration, + upstream-auth checks, and fail-closed unavailable-session behavior. +- **Isolation and disclosure boundary.** Every run generates a unique temporary root, Compose + project, container/image names, transaction image tags, and run label. The rollback fixture uses + an immutable `hello-world` digest whose preflight exits successfully, guaranteeing the stopped + core state required by `tht` compensation. Cleanup includes stopped project containers in + its final ownership check immediately before teardown and removes only exact containers, + Compose resources, image references, control state, and temporary files. There is no global + prune. Failure diagnostics are bounded and sanitized, and all credentials/endpoints used by the + smokes are disposable fixtures rather than operator or repository secrets. Every public smoke + also has an internal 30-minute process-group supervisor with TERM/KILL of the complete group. +- **Cross-platform CI contract.** `.github/workflows/deployment.yml` uses immutable action commits, + pinned supported Node and Go versions, runs LF/Compose/secret/coupling/docs/TypeScript gates on + Linux, runs each Linux Docker smoke once under its own outer timeout, and copies the Windows + source into a path containing spaces before building/invoking native `tht` and rendering + Compose. The optional `windows_docker_startup` dispatch targets a labelled self-hosted Windows + Docker Desktop/WSL2 runner and performs bounded two-service startup and exact cleanup. No local + Windows or Windows Docker execution is claimed until that manual job is recorded. +- **Validation status.** Deterministic Phase A gates, backend **434/434** plus TypeScript, + frontend **386/386** plus TypeScript, and harness **862 passed / 5 L2 deselected** are green. + Review round 1 ran each Docker smoke exactly once without retry. Unified (`103.86s`) and + update-only (`46.45s`) passed build/start, core/Pi/registry/persistence setup and the stopped + candidate preflight, but `tht` stopped before mutation at its active-session inventory gate. + Round 2 replaces presence-only fixture checks with generated Compose renders plus the production + workspace resolver; this found and fixed missing explicit direct transport selections. The + clean-server preflight now atomically initializes the three hidden Pi-agent targets under the + writable parent bind while protected/tracked sources remain separate read-only mounts. Clean + empty-root render/setup and wrong-service/value/mount mutations are green. The corrected server + one-shot built and started both healthy services from an empty Pi-state root, then stopped at an + incorrectly addressed authenticated frontend hop. Fix round 3 adds the exact fourth private + non-admin claim and proves its nginx/backend transformation in a focused auth test. It also + centralizes schema-v2 registry descriptor resolution and secret-safe runtime rendering in + `ThtRunner`, preserving canonical revision identity and durable session roots for inventory, + create/resume/show, SQL, and Pi calls. The fresh update-only one-shot now passes mutation, + automatic `rolled_back` compensation, exact prior-image restoration, unchanged registry head + and mount identities, all four persistence sentinels, post-rollback doctor/workspace checks, + and exact labeled-resource cleanup. The one authorized server invocation was blocked at its + first Docker readiness call by the execution sandbox's socket permission before any Compose + resource could be created, so authenticated workspace/fail-closed session behavior remains an + explicit release gate. Native Windows PowerShell/Docker execution also remains pending. + +### Historical snapshot — Portable deployment decoupling (superseded 2026-08-08) + +- **Mandatory stack.** The supported Compose stack is exactly `frontend` plus `core`; use the + base file with `deploy/compose.local.yaml`, or with `deploy/compose.server.yaml` plus the + required public-server session overlay. `run-stack.sh` + invokes the base+local Compose command and the core image provides Pi, so no host Pi binary is + part of the launch contract. +- **External boundaries.** DWH, vector DB, embedding, LLM, and reverse-proxy services are + external configurable endpoints even when deployed on the same infrastructure. The two + superseded PSD/portal deployment overlays were removed. Workspace descriptors and migration + utilities remain separate from deployment runtime configuration. +- **Legacy PSD deployment ruling.** The PSD bootstrap was deleted because it generated the + retired overlay and was therefore deployment machinery, not a data migration utility. Its + remaining live contract checks were renamed for the generic local Compose profile. The coupling + gate rejects stale active deployment filenames and content while deliberately excluding + historical plans/specs, canonical workspace descriptors, and non-runtime migration helpers. +- **Fresh provider and secret contract.** Local, server, and standalone development mount the + protected Pi auth JSON plus tracked declarative model/settings files read-only under + `/home/thoth/.pi/agent`. The existing strict application bundle is a core-only Docker secret at + `/run/secrets/thothii.secrets`; operator env files contain only its absolute source path. + Provider readiness is exercised from a fresh Compose volume through model listing, configuration, + and sanitized credential status. +- **Install and scan closure.** Superseded copied one-service installation examples and the + provider-owned-network test are retired. Active manuals use the canonical base plus local/server + and optional overrides, while the category-based coupling scan covers runtime, Docker smoke, + install, operator, and positive deployment-test contracts and propagates scanner errors. + +### Historical snapshot — Portable Git workspace registry, pre-schema-v3 (superseded 2026-08-08) + +- **Source of truth and scope.** The canonical workspace repository is a generic Git remote, + configured only by `THT_WORKSPACE_GIT_REMOTE` and `THT_WORKSPACE_GIT_BRANCH` (there is no + committed PSD/Chirone remote or branch default). Both a local Docker installation and a server + persist its checkout, validated snapshots, state, and locks at `/data/workspace-registry`. + Connector endpoints, transport choices, and secret-file paths remain local bindings; secret + contents are never stored in Git, API responses, browser storage, diagnostics, or bundles. +- **Migration and session safety.** Schema-v2 descriptors are operational; legacy descriptors are + visible as `migration_required` until migrated by the documented operator workflow. New sessions + acquire a persistent revision lease before readiness and persist workspace ID plus immutable Git + revision. Retention hands that lease off only after an authoritative scan observes the manifest, + so a stale concurrent scan cannot prune the pinned snapshot. Resume resolves that historical + snapshot, while retention preserves every revision referenced by an open, closed, or failed + unarchived manifest. + Reconciliation runs only with a complete local installation list or an administrator's complete + server list, never from a remote user's partial view. +- **SSH connector boundary.** The current OpenSSH forward is owned by one bounded diagnostic and is + always cleaned up afterward. DWH/vector `ssh_tunnel` bindings therefore return + `workspace_not_activatable`, and new-session creation rejects them before persistence. Direct and + REST runtime connectors remain supported; Git remote access over SSH is unaffected. +- **Operator manuals.** Follow [the local manual](docs/install/local-workspace-registry.md) for + macOS/Windows/Linux Docker Desktop deployment and [the server manual](docs/install/server-workspace-registry.md) + for Gitea-compatible remotes, reverse proxy, migration, backup, and recovery. The release + workflow is Git review/push → installation pull → validate → local diagnostic test → browser-local + workspace/model/reasoning selection → revision-pinned session. +- **Verification recorded for this source branch.** `git diff --check` passed; backend Vitest + **371/371** and TypeScript passed; frontend Vitest **398/398** and TypeScript passed; the + harness document regression passed **10/10**. `./scripts/workspace-registry-smoke.sh` and the + executable installation-manual fixture verifier passed with Docker. A final unrestricted full + harness run remains a release command for the deployment environment; the earlier local + long-running harness run was intentionally cancelled before it produced a final result. + +### Historical snapshot — Session summary redesign (2026-07-23) - Session documents are projected at read time in outcome-first order: original question, final SQL, persisted data preview, revised question, assumptions, one memory list, then @@ -26,7 +541,7 @@ `sha256:8311ca1308b459ece7236bf143da7b1a226ff4082fed924e1b5a207c24b6ca29` is running. Frontend and `/api/health` both returned HTTP 200. -## Local Pi user auth + startup failure handling — LIVE 2026-07-21 +### Historical snapshot — Local Pi user auth + startup failure handling (2026-07-21) - The PSD Docker profile now bind-mounts the configurable host `PI_AUTH_FILE` read-only at `/home/thoth/.pi/agent/auth.json`; on this Mac it resolves to the real user profile @@ -48,7 +563,7 @@ `a390c8b8-0a91-4a37-967b-ce7ff9be9797`, `a2f974b2-4c48-4967-b4b6-afdbc2b2d541`, and `f66e1959-3c71-4b10-8aa1-606992046b7e` (API delete 204, subsequent lookup 404 for each). -## User-owned sessions cutover — prepared, manual gate pending (2026-07-16) +### Historical snapshot — User-owned sessions cutover (2026-07-16) - **Target contract:** the public server runs `AUTH_MODE=upstream` with Task 4 portal identity forwarding and Task 5 principal enforcement deployed together. Its session source of truth is @@ -70,7 +585,7 @@ release; never re-enable filesystem persistence, restore the archive into production, or dual-write during rollback. -## Deployment — Docker locale (Profile A, co-located) — LIVE 2026-07-12 +### Historical snapshot — Docker locale deployment, Profile A (superseded 2026-08-05) ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal** a `https://aritmolab.policlinicosandonato.it/datamart-builder` (backend invisibile, tutto same-origin via nginx del portale). @@ -83,7 +598,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal - **Standalone/dev**: `docker-compose.dev.yml` (rete propria, porte host 8787/8090) + `scripts/docker-smoke.sh`. - Piano dettagliato: `docs/superpowers/plans/2026-07-12-local-docker-deploy-implementation.md`. -### Runtime incident fixes — LIVE 2026-07-13 +### Archived snapshot — Runtime incident fixes (2026-07-13) - The bind-mounted Pi profile came from host paths and did not trust `/app/harness`. Pi 0.80 consequently loaded **zero** project extensions, prompts and skills, silently @@ -102,7 +617,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal resumed directly at F1, ran `tht session show`, and completed `tht search pack` (12 tables, 0 evidence, 2 solved) without repository exploration or adapter errors. -### Workflow/UI regression fixes — LIVE 2026-07-14 +### Archived snapshot — Workflow/UI regression fixes (2026-07-14) - **F1 Model Activity restored.** Session create/resume now preserves configured/persisted thinking instead of forcing `off`. Pi's nested `thinking_delta` is bridged to a dedicated @@ -127,7 +642,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal Running image ids: core `sha256:55acef2f12151ea97144c2f5e9164d63f2ca734bc2746fef553df94849e3fb3f`; frontend `sha256:1043f79392420149655cc63d70461e2ca2005b2290a1e3e21dcf845ec3bd1c81`. -### Pi-enabled model selector — LIVE 2026-07-14 +### Archived snapshot — Pi-enabled model selector (2026-07-14) - **Pi is the allowlist authority.** `/models` reads the mounted Pi `enabledModels`, intersects it with models currently available from Pi, and preserves the configured order. Enumeration @@ -148,7 +663,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal ID and running container image ID both equal `sha256:577f99754fd0731251c8ddd8608b1b8baee09d02fad66c759b23f8221083e676`. -### Qwen connectivity + state-aware Resume recovery — LIVE 2026-07-14 +### Archived snapshot — Qwen connectivity + state-aware Resume recovery (2026-07-14) - **Pi turns have an explicit lifecycle.** The bridge tracks `idle`, `running`, `waiting`, and `failed`; a reviewer gate is `waiting`, responses/steering return to `running`, and an @@ -183,7 +698,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal exited 0 with `controller.abort()` in cleanup, preserving the gate, session cleanup, and exact settings-restoration evidence. -### Complete activity timeline + CTE spacing — LIVE 2026-07-15 +### Archived snapshot — Complete activity timeline + CTE spacing (2026-07-15) - **Model activity is complete from F1.** The left panel now records the submitted prompt before session creation completes, then projects thinking, assistant output, sanitized tool lifecycle, @@ -216,7 +731,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal field crossed SSE. Cleanup closed with 200, deleted only that session with 204, restored the exact settings object, and left no Pi runtime or smoke session. -### Filtered Model activity projection — LIVE 2026-07-15 +### Archived snapshot — Filtered Model activity projection (2026-07-15) - **Resolved contract.** `activityLog` still folds the complete in-memory prompt, thinking, assistant, sanitized tool, reviewer-gate, status, and turn-lifecycle history. The left panel now @@ -239,7 +754,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal four public fields. The probe accepted the close response, deleted only that session with 204, restored the exact saved settings object, confirmed the session absent, and left no Pi runtime. -### Central live log + compact CTE density — LIVE 2026-07-15 +### Archived snapshot — Central activity log + compact CTE density (2026-07-15) - **Resolved UI contract.** The central working body now renders every chronological non-blank assistant transcript line in one bounded accessible log, without user-entry echoes, @@ -269,7 +784,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal unrelated Pi runtime existed to disturb. The count-only frontend sensitive/error pattern scan was **0**. No live model smoke was run, and settings and sessions were intentionally untouched. -### Resizable activity split + compact CTE rows — LIVE 2026-07-15 +### Archived snapshot — Resizable activity split + compact CTE rows (2026-07-15) - **Resolved UI contract.** `activityLog` remains the complete in-memory chronological fold. The left Model activity panel default-denies every kind except prompt, thinking, and assistant, @@ -303,7 +818,7 @@ ThothII gira in Docker sul server co-locato, **embedded nel portale omics_portal sensitive/error pattern scan was **0**. No live model smoke was run; settings and sessions were intentionally untouched. -### Final activity-split fix — LIVE 2026-07-15 +### Archived snapshot — Final activity-split fix (2026-07-15) - **Source and verification (`2026-07-15T15:56:57+02:00`).** Deployed source commit `1f540fcb78ac9e552e56a21e47edf66e9872b323` (`1f540fc`). Frontend Vitest passed **298/298** diff --git a/README.md b/README.md index 7d23e7fa..7a912f46 100644 --- a/README.md +++ b/README.md @@ -2,64 +2,110 @@ ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fastify/Pi/`tht` core. The portable deployment runs exactly two application services; data services remain -external in this profile. +external in this profile, except for the mandatory internal semantic services bundled in Compose. -## Docker Compose: one-command startup +Authentication is configured through the single host CLI tht: see the [local authentication guide](docs/install/authentication-local.md), +[generic OIDC guide](docs/install/authentication-oidc.md), and [manual acceptance matrix](docs/testing/authentication-manual-acceptance.md). -Requirements: Docker Engine with Compose v2. The default project starts only the two -application images; DWH, vector and embedding services can be remote or supplied by an -optional overlay. +## Docker Compose: local startup + +Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external, +configurable endpoints—even when they are co-located with ThothII. From a fresh clone, run these commands from the repository root: ```sh -cp .env.example .env -cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets -chmod 600 deploy/secrets/thothii.secrets -# Edit .env (non-secret endpoints) and deploy/secrets/thothii.secrets (KEY=VALUE lines). -docker compose up --build -d +cp deploy/env/local.env.example deploy/env/local.env +# Edit deploy/env/local.env, including PI_AUTH_FILE, THT_SECRETS_FILE, and external endpoints. +docker compose --env-file deploy/env/local.env \ + -f compose.yaml -f deploy/compose.local.yaml up --build -d ``` -The root `.env` is loaded automatically by Compose. It defaults to `compose.yaml`, an empty -profile, and `THT_SECRETS_FILE=deploy/secrets/thothii.secrets`; no `--env-file`, `-f`, or -`--profile` flag is required for the normal installation. Add or edit YAML workspace descriptors -under `deploy/workspaces/`; they are mounted read-only and relative `roots` resolve beneath -`/data/workspaces/`. Open (set `THOTH_HTTP_PORT` in -`.env` to choose another loopback port). +`./scripts/run-stack.sh` runs this same base+local command in the foreground. The core image +contains its Pi runtime; no host `pi` executable is used. For a server installation: -The bundle contains only values, one per line (`THT_MODEL_API_KEY=...`, DWH/vector keys, and -the optional local-vector passwords). It is ignored by Git and never copied into either image. -Do not put credentials in `.env`, workspace YAML, URLs, or Compose interpolation values. - -### Optional overlays - -Overlays are selected in `.env`, so the operational command remains the same. On Unix-like -systems use `:` between files; on Windows use `;`: - -```dotenv -# Remote DWH/vector/embedding services with authenticated reverse proxy: -COMPOSE_FILE=compose.yaml:deploy/compose.production.yaml -COMPOSE_PROFILES= - -# Local pgvector (Mac/Windows or a standalone application server): -COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml -COMPOSE_PROFILES=local-vector +```sh +cp deploy/env/server.env.example deploy/env/server.env +# Edit all absolute storage, Pi/secret/session files, and endpoint paths. +sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001 +docker compose --env-file deploy/env/server.env \ + -f compose.yaml -f deploy/compose.server.yaml \ + -f deploy/compose.session-server.yaml.example up --build -d ``` -After changing `.env`, apply the selected configuration with `docker compose up --build -d`. -Preprocessing is an explicit opt-in preset: append -`deploy/compose.preprocess.yaml:deploy/compose.preprocess-local-vector.yaml` and set -`COMPOSE_PROFILES=local-vector,preprocess`; then run the job with -`docker compose run --rm preprocess-evidence` or `preprocess-dwh`. +The initializer is required for an empty or restored server Pi-state bind. It atomically creates +the three regular targets hidden below the writable parent bind; protected Pi auth and tracked +model/settings sources remain separate read-only mounts. See the server manual before substituting +a root other than `/srv/thothii/pi-state`. -Application state, including settings, sessions, artifacts, and indexes, lives in the named -`thoth_data` volume mounted at `/data`. `docker compose down` keeps that volume. Only an -explicit destructive command such as `docker compose down --volumes` removes it. +Workspace descriptors come from the Git remote configured by `THT_WORKSPACE_GIT_REMOTE`; their +runtime endpoint and secret bindings remain installation-local. Open + (set `THOTH_HTTP_PORT` in `deploy/env/local.env` to choose another +loopback port). + +Credentials and certificates are local protected files. Do not put them in environment examples, +workspace YAML, URLs, or Compose interpolation values. + +Application state is split across the named `settings`, `pi-state`, `workspace-registry`, +`sessions`, `qdrant-data`, and `embedding-models` volumes. `docker compose down` keeps them. +`qdrant-data` is a derived but persistent index store; `embedding-models` is an Ollama model +cache for `qwen3-embedding:0.6b` with fixed `1024`-dimension embeddings. Only an explicit destructive command such as `docker compose +down --volumes` removes them. The frontend depends on the core health check and proxies `/health` and `/api/*` to it. The application health endpoint intentionally checks process readiness only; external dependency diagnostics are exposed by `tht doctor` and do not prevent the UI from starting. +## Git-backed workspace repository + +Workspace descriptors are shared through a validated Git repository while endpoint bindings and +secret files remain installation-local. Use the [local Mac/PC installation manual](docs/install/local-workspace-registry.md) +for Docker Desktop or a local engine, the [server installation manual](docs/install/server-workspace-registry.md) +for the Gitea, reverse-proxy, backup, upgrade, and recovery workflow, and the +[P1→P1.1 migration guide](docs/migrations/p1-to-p1-1-registry-layout.md) before upgrading an +older flat-layout registry. + +The curator-owned repository layout is: + +```text +thoth-workspaces.yaml +/workspace.yaml +/evidence/** +``` + +`thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered `workspaces` list of +`{id, name, description?}` entries. It is authoritative for workspace ID, name, description, and +display order. Every catalog entry must have a matching descriptor in the same commit; otherwise +the complete candidate is rejected. Descriptors remain curator-owned and change only through a +Git commit and push from a separate authoring clone, followed by an installation pull. ThothII +never writes any workspace repository content. + +The operator workflow is: curate catalog/descriptor/Evidence changes in Git, commit and push, +**Update workspace repository** from each ThothII installation, select the workspace, complete its +write-only runtime-secret fields, run **Validate workspace source** and **Test workspace +connections**, then select the workspace locally before creating sessions. Each new session +pins the Git revision it used; a later pull cannot change a Resume. Snapshot cleanup retains every +revision referenced by an open, closed, or failed unarchived session. It reconciles from the +single local installation list or from a server administrator's complete session list, never from +a remote user's partial list. The isolated deployment exercise is +`./scripts/workspace-registry-smoke.sh`; both manuals are checked with +`./scripts/verify-workspace-install-docs.sh --profile local` or `--profile server`. + + +Schema v3 is the only accepted workspace descriptor. Schema v1 and v2 workspace descriptors are +rejected before activation. Candidate snapshot validation therefore makes activation or a pull fail +atomically while the prior valid snapshot remains active. There is no in-product migrator or +automatic conversion. A repository must already contain reviewed v3 descriptors. One workspace +owns one Qdrant collection; +schema, Evidence, and Memory records share that collection and stay separated by indexed payload +`kind`. + + +Connector `ssh_tunnel` bindings are diagnostic-only in this release: their bounded probe always +cleans up the loopback forward and returns `workspace_not_activatable`; session creation is rejected +before persistence. Git registry access over SSH is unaffected. Use direct or REST connector +transport for runtime sessions. + `docker-compose.dev.yml` is deliberately local: both published ports bind to `127.0.0.1`, `THT_SESSION_STORAGE=local`, and `THT_HOME=/data/local-home`. Do not set `THOTH_PUBLIC_EXPOSURE=true` for that profile; the backend rejects that public/local combination @@ -79,28 +125,97 @@ volume afterward. It never targets the fixed `thothii` operator project or its v `KEEP_SMOKE_RESOURCES=1` to retain that smoke project's resources for inspection; remove them later with `docker compose --project-name "$SMOKE_PROJECT" down --volumes`. -## Optional local pgvector and recovery +## Unified deployment release gates -The local-vector overlay reads `THT_VECTOR_BOOTSTRAP_PASSWORD`, -`THT_VECTOR_MIGRATOR_PASSWORD`, `THT_VECTOR_READER_PASSWORD`, and -`THT_VECTOR_WRITER_PASSWORD` from the same bundle. Its `vector_data` volume is independent of -application state; passwords are selected at runtime and are never passed as URL arguments. +Task 13 adds no-secret release gates around the canonical local and server Compose profiles. Its +deterministic safety check does not contact the Docker daemon: + +```sh +bash scripts/unified-deployment-smoke.sh --self-test +``` + +The three Linux Docker smokes are separate release commands. Each creates a unique Compose project, temporary +Git workspace remote, fixture provider, image names, and run label. Its exit trap removes only +resources carrying that exact run identity and never performs a global Docker prune. Cleanup +enumerates running and stopped project containers immediately before `compose down` and refuses +the teardown if any container, volume, or network has a foreign run label. + +```sh +bash scripts/unified-deployment-smoke.sh +bash scripts/tht-update-smoke.sh +bash scripts/server-deployment-smoke.sh +``` + +The unified smoke builds and starts `frontend` and `core`, verifies the embedded Pi and internal +registry, recreates with the Git remote offline, activates a valid Git update, rejects invalid Git +content while retaining the valid snapshot, and checks the four persistence volumes. The unified +and update-only smokes +inject a digest-pinned non-core candidate under a deliberately mismatched Pi version and require +`tht pi update` to roll back while preserving settings, sessions, Pi state, registry revision, +and mount identity. The rollback candidate is the digest-pinned `hello-world` executable: a +preflight proves that it exits successfully, so the failed replacement core satisfies +`tht`'s stopped-core compensation precondition. The server smoke uses the same smoke-built +core/frontend images with the server and required session overlays, disposable bind roots and +secret files, upstream-auth checks, and a fail-closed `503` assertion for its deliberately +unavailable disposable session endpoint. No real provider, database credential, or repository +secret is required. + +For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular +`agent/auth.json`, `agent/models.json`, and `agent/settings.json` mount targets atomically before +Compose. The server smoke starts from an empty Pi-state root and applies this same preflight; the +real protected/tracked sources remain separate read-only mounts. Deterministic fixture tests render +both profiles, verify that bindings stay on `core`, check mount readability, and run the production +workspace resolver. Wrong-service, wrong-value, and broken-secret-mount mutations must fail. + +Each public smoke has its own 30-minute process-group supervisor with TERM/KILL cleanup; CI retains +an independent 32-minute outer timeout and does not retry a failed command. + +Current release status (2026-08-05): clean-root render/setup and the production runtime-binding +resolver contracts are green. The server fixture supplies all four private trusted claims, +including exact non-admin value `0`, and a focused test proves nginx normalization produces the +accepted non-admin backend principal. Canonical schema-v3 registry descriptors now pass through +one backend-owned, secret-safe runtime handoff for inventory and session execution; canonical +identity and durable session/artifact/index roots are retained. The fresh update-only smoke passed +bad-candidate mutation, automatic `rolled_back` compensation, exact prior-image restoration, +unchanged registry/mount identity, all four sentinels, post-rollback doctor/workspace checks, and +exact cleanup. The one authorized server-smoke invocation was denied access to the Docker socket +by its execution sandbox before startup, so the complete authenticated workspace and fail-closed +session assertions still require a fresh authorized release run. Native Windows Docker +Desktop/WSL2 remains a separate manual/self-hosted gate. + +The deterministic native Windows contract is: + +```powershell +.\scripts\test-windows-clone-contract.ps1 +``` + +It checks Git's CRLF/LF attributes and bytes, copies tracked source into a temporary path containing +spaces, builds and invokes native Windows `tht` there, and renders exactly `core` plus +`frontend` without starting containers. On a supported self-hosted Windows Docker Desktop/WSL2 +runner, dispatch the deployment workflow with `windows_docker_startup=true`; that job executes: + +```powershell +.\scripts\test-windows-clone-contract.ps1 -DockerStartup +``` + +Startup mode adds bounded image build/two-service health startup, installation-aware `tht` +status, stopped-container-aware ownership checks, and exact cleanup. The ordinary hosted Windows +job remains deterministic and does not claim Docker startup. ## Preprocessing jobs and S3 Evidence -The included job workspaces target the local-vector profile. Put the four local-vector password -keys in the bundle, set `THT_OLLAMA_URL`, mount Evidence at `/data/source/evidence`, then select -the preprocessing preset in `.env`: +The included preprocessing services reuse the internal Qdrant/Ollama stack. Mount Evidence at +`/data/source/evidence`, then run the explicit preprocessing preset: -```dotenv -COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml:deploy/compose.preprocess.yaml:deploy/compose.preprocess-local-vector.yaml -COMPOSE_PROFILES=local-vector,preprocess +```sh +docker compose --env-file deploy/env/local.env \ + -f compose.yaml -f deploy/compose.local.yaml \ + -f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-evidence ``` -Run `docker compose run --rm preprocess-evidence` or -`docker compose run --rm preprocess-dwh`. The overlay makes each job wait for the vector -database health check, role reconciliation, and a successful migration; no separate database -startup or migration command is required. +Replace the final service with `preprocess-dwh` when required. The overlay makes each job wait for the internal Qdrant +service health checks and embedding model initialization; no separate semantic-service startup is +required. S3 Evidence uses the optional `tht[s3]` dependency and canonical `s3://bucket/key` provenance. AWS endpoints are used when no custom URL is supplied. Every custom endpoint is an explicit egress @@ -111,39 +226,39 @@ egress policy. Store access key, secret key, and session token as secret referen deployment configuration—never in Compose environment values or source URIs. Discovery and reads are bounded by configured page, object, and byte limits. -Create a versioned PostgreSQL custom-format backup (the filename is operator-controlled, so use -an immutable timestamp or release identifier): +Create a versioned Qdrant volume backup for one exact Compose project (the filename is +operator-controlled, so use an immutable timestamp or release identifier): ```sh ./scripts/vector-backup.sh \ - --host 127.0.0.1 --port 5432 --database thoth --user thoth_backup \ - --password-file /secure/thoth/vector-backup-password \ - --output /secure/backups/thoth-vectors-2026-07-12.dump + --project-name thothii \ + --output /secure/backups/thoth-qdrant-2026-08-08.tar ``` -The dump contains the three allowlisted `vectors` tables, their data and ACLs, plus the -`public.tht_vector_migrations` ledger. Login roles and passwords are deliberately not copied: -provision/reconcile the approved role names on the target first, and install the `vector` -extension in its `vectors` schema. The target must otherwise contain no vector tables or ledger. +The script resolves exactly one Docker volume with the labels +`com.docker.compose.project=` and `com.docker.compose.volume=qdrant-data`, stops the +`qdrant` service if it is running, archives that volume's persistent contents, then restores the +prior service state. It never performs global Docker cleanup and refuses to overwrite an existing +archive path. -Restore always names both the currently active source and a target on a physically distinct -PostgreSQL cluster. The script compares PostgreSQL system identity, so host aliases or a different -database in the active cluster cannot bypass the guard. It refuses a non-empty target unless -`--force-nonempty` is explicit, and the clean restore is one transaction: +Restore targets that same exact project-scoped `qdrant-data` volume. Because restore replaces the +persistent Qdrant data in place, it requires an explicit confirmation that exactly repeats the +Compose project name by passing `--confirm-project`: ```sh ./scripts/vector-restore.sh \ - --active-host vector-db --active-database thoth --active-user thoth_backup \ - --active-password-file /secure/thoth/vector-active-password \ - --target-host vector-db-restore --target-database thoth --target-user thoth_restore \ - --target-password-file /secure/thoth/vector-restore-password \ - --input /secure/backups/thoth-vectors-2026-07-12.dump + --project-name thothii \ + --input /secure/backups/thoth-qdrant-2026-08-08.tar \ + --confirm-project thothii ``` -After restore, run `tht vector migrate --status --json`, adapter health, and a known retrieval -query against the target before changing any deployment endpoint. Never test recovery against the -active `vector_data` volume. `./scripts/local-vector-smoke.sh --backup-restore` performs this drill -with disposable source and target volumes. +The restore script stops `qdrant`, validates the exact labeled target, stages the current volume +contents for rollback, extracts the requested archive into the volume, and then returns the +service to its prior running state. It restores semantic storage only. Before reopening write +traffic, the workspace registry must already be at a reviewed v3 descriptor revision compatible +with the restored collection; then run backend health checks and a known retrieval query. The +helper does not restore descriptors, rename collections, or reconcile an incompatible collection +contract. ## Production trust boundary and secrets @@ -161,15 +276,10 @@ with the organization's reviewed identity proxy. `AUTH_MODE=upstream` trusts thi rejects requests without the identity header. Setting `THOTH_PUBLIC_EXPOSURE=true` with any other auth mode fails during core startup. -Production credentials use the one Compose secret bundle, not `.env`. Put the required keys in -`deploy/secrets/thothii.secrets` and select the production overlay in `.env`: - -```dotenv -THT_MODEL_API_KEY=replace-me -THT_DWH_API_KEY=replace-me -THT_VEC_API_KEY=replace-me -THT_VEC_WRITE_API_KEY=replace-me -``` +Production credentials use the existing Compose secret-bundle contract, never environment values. +Copy `deploy/secrets/thothii.secrets.example` to a protected host file, include only the required +keys, and set its absolute path as `THT_SECRETS_FILE` in the operator env. Keep Pi's native +provider auth in the separate protected file named by `PI_AUTH_FILE`. The bundle is mounted read-only as `/run/secrets/thothii.secrets` and must be mode `0600` or `0400` on the host. Docker's runtime `0444` mode is accepted only beneath `/run/secrets`; see @@ -201,9 +311,9 @@ still scrubbed. Supporting them requires a future dedicated provider-specific co The server profile stores sessions and per-user preferences directly in PostgreSQL schema `thoth_sessions`; it does not use PostgREST, browser storage, a shared session directory, or a -dual write. Start from [`deploy/compose.session-server.yaml.example`](deploy/compose.session-server.yaml.example) -and copy [`deploy/workspaces/server-sessions.yaml.example`](deploy/workspaces/server-sessions.yaml.example) -to the untracked `deploy/workspaces/server-sessions.yaml` mounted into the core container. +dual write. Use [`deploy/compose.session-server.yaml.example`](deploy/compose.session-server.yaml.example) +with the canonical base+server files and set `THT_SERVER_WORKSPACE_CONFIG` to an absolute, +protected copy of [`deploy/workspaces/server-sessions.yaml.example`](deploy/workspaces/server-sessions.yaml.example). The runtime login needs membership in the no-login database role `thoth_sessions_runtime` only. The distinct, one-shot migrator login needs migration authority and uses @@ -233,13 +343,14 @@ session store without upstream authentication, direct DB host/name/runtime user/ The migrator independently rejects every other TLS mode before reading its password secret or constructing a database URL. -Perform the cutover in one maintenance window, with the Task 4 portal proxy headers and Task 5 -backend principal parser deployed together. Neither change is safe to deploy independently: Task -4 clears the legacy identity header and Task 5 rejects it. Drain/stop active Pi work, enable a -maintenance response at the portal, then run the migrator once and inspect its pristine JSON: +Perform the cutover in one maintenance window, with the upstream identity-proxy headers and +backend principal parser deployed together. Neither change is safe to deploy independently: the +proxy clears the legacy identity header and the backend rejects it. Drain/stop active Pi work, +enable a maintenance response at the proxy, then run the migrator once and inspect its pristine JSON: ```sh -docker compose -f compose.yaml -f deploy/compose.session-server.yaml \ +docker compose --env-file deploy/env/server.env \ + -f compose.yaml -f deploy/compose.server.yaml -f deploy/compose.session-server.yaml.example \ --profile session-migrate run --rm session-migrate ``` diff --git a/backend/package-lock.json b/backend/package-lock.json index 37d3296a..9478eec4 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -6,11 +6,18 @@ "": { "name": "thothii-backend", "dependencies": { + "@fastify/cookie": "11.1.2", "@fastify/cors": "^11.2.0", - "fastify": "^5.0.0" + "@fastify/rate-limit": "11.2.0", + "@types/pg": "^8.20.3", + "fastify": "^5.0.0", + "openid-client": "6.8.5", + "pg": "^8.22.0", + "yaml": "^2.9.0", + "zod": "^4.4.3" }, "devDependencies": { - "@types/node": "^22.0.0", + "@types/node": "24.13.3", "tsx": "^4.19.0", "typescript": "^5.6.0", "vitest": "^2.1.0" @@ -479,6 +486,55 @@ "fast-uri": "^3.0.0" } }, + "node_modules/@fastify/cookie": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-11.1.2.tgz", + "integrity": "sha512-Dtrpk/YOGUsbRMvP/8ZqPpwnMRv0qSqodFdoQ2B589Obc7jw4s4Qla+cV72Bsm7WsZJnqlYFX/i7uSBq0xzg6g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "cookie": "^2.0.0", + "fastify-plugin": "^6.0.0" + } + }, + "node_modules/@fastify/cookie/node_modules/cookie": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-2.0.1.tgz", + "integrity": "sha512-yuToqVvRrj6pfDXREyQAAv8SkAEk/8GS3jQRTiUMm66TVtBYmqQeoEjL2Lmq8Rpo6271vH76InTChTitEAm65w==", + "license": "MIT", + "engines": { + "node": ">=22" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@fastify/cookie/node_modules/fastify-plugin": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-6.0.0.tgz", + "integrity": "sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/@fastify/cors": { "version": "11.2.0", "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-11.2.0.tgz", @@ -589,6 +645,44 @@ "ipaddr.js": "^2.1.0" } }, + "node_modules/@fastify/rate-limit": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-11.2.0.tgz", + "integrity": "sha512-X7osJd4XSvMoejYrnJkSZYYjY1eNYoBqhjlzf1RakC2204qExFqZFTKj5+T7VuzA/iUI9Z3UoSqQRkB2HpG0oQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@lukeed/ms": "^2.0.2", + "fastify-plugin": "^6.0.0", + "ip-address": "^10.2.0", + "toad-cache": "^3.7.0" + } + }, + "node_modules/@fastify/rate-limit/node_modules/fastify-plugin": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-6.0.0.tgz", + "integrity": "sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -596,6 +690,15 @@ "dev": true, "license": "MIT" }, + "node_modules/@lukeed/ms": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", + "integrity": "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", @@ -960,13 +1063,23 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.20.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.0.tgz", - "integrity": "sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==", - "dev": true, + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", "license": "MIT", "dependencies": { - "undici-types": "~6.21.0" + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/pg": { + "version": "8.20.3", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.3.tgz", + "integrity": "sha512-4Tvg+HO6+oQaAkpT8GTYoSExzpGGZz532GXgbbCElWJQeQdMozBWxEKNBhJJpHFjWXsMxqPbyypvj/89FWNoSQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" } }, "node_modules/@vitest/expect": { @@ -1362,9 +1475,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", - "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "funding": [ { "type": "github", @@ -1436,9 +1549,9 @@ } }, "node_modules/find-my-way": { - "version": "9.6.0", - "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.6.0.tgz", - "integrity": "sha512-Zf4Xve4RymLl7NgaavNebZ01joJ8MfVerOG43wy7SHLO+r+K0C6d/SE0BiR7AV5V1VOCFlOP7ecdo+I4qmiHrQ==", + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.7.0.tgz", + "integrity": "sha512-f2JHn75x2JlwUwLenZypgczR7YWMb/uO9BvUXtus+JMgkbIkLADd38cI4EiV+OQqrGo1Zlq6V8wnqMJ8e62wUQ==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -1464,6 +1577,15 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/ip-address": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", + "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", @@ -1473,6 +1595,15 @@ "node": ">= 10" } }, + "node_modules/jose": { + "version": "6.2.9", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.9.tgz", + "integrity": "sha512-XrchZOFZUl/T3vTwRe8XK+cJrGtMF4th1ARnDfwbBXFKThGhlsxEE4Zu03AD/bjJSt/9jT/mxrOCkJWOg77aPA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/json-schema-ref-resolver": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz", @@ -1578,6 +1709,15 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/oauth4webapi": { + "version": "3.8.7", + "resolved": "https://registry.npmjs.org/oauth4webapi/-/oauth4webapi-3.8.7.tgz", + "integrity": "sha512-4RxcKxXjuItDFZ20RRPf4YTw3kpeXJyCgJFxVzJ068A7PNJ18st2Dg90tlC1LkSDS0GecroagCLHYEIVUhCAkw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -1587,6 +1727,19 @@ "node": ">=14.0.0" } }, + "node_modules/openid-client": { + "version": "6.8.5", + "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-6.8.5.tgz", + "integrity": "sha512-jNGC/5wnTYwCcEUe2ss0IRUmVRQcgxM0A1nLb3eX/9llqNbMWOQd2xd+qDAgfVCpA5Qh96Y1cdnkfbva6+bSdA==", + "license": "MIT", + "dependencies": { + "jose": "^6.2.8", + "oauth4webapi": "^3.8.7" + }, + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/pathe": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", @@ -1604,6 +1757,95 @@ "node": ">= 14.16" } }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -1677,6 +1919,45 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/process-warning": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", @@ -2006,10 +2287,9 @@ } }, "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "license": "MIT" }, "node_modules/vite": { @@ -2607,6 +2887,39 @@ "engines": { "node": ">=8" } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/backend/package.json b/backend/package.json index 0ebaeabc..359c7605 100644 --- a/backend/package.json +++ b/backend/package.json @@ -4,16 +4,25 @@ "type": "module", "scripts": { "dev": "tsx watch src/server.ts", + "prebuild": "node scripts/clean-dist.mjs", "build": "tsc -p tsconfig.json", "test": "vitest run", - "start": "node dist/server.js" + "start": "node dist/server.js", + "test:schema-v3-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs" }, "dependencies": { + "@fastify/cookie": "11.1.2", "@fastify/cors": "^11.2.0", - "fastify": "^5.0.0" + "@fastify/rate-limit": "11.2.0", + "@types/pg": "^8.20.3", + "fastify": "^5.0.0", + "openid-client": "6.8.5", + "pg": "^8.22.0", + "yaml": "^2.9.0", + "zod": "^4.4.3" }, "devDependencies": { - "@types/node": "^22.0.0", + "@types/node": "24.13.3", "tsx": "^4.19.0", "typescript": "^5.6.0", "vitest": "^2.1.0" diff --git a/backend/scripts/bash-heredoc.mjs b/backend/scripts/bash-heredoc.mjs new file mode 100644 index 00000000..891d649b --- /dev/null +++ b/backend/scripts/bash-heredoc.mjs @@ -0,0 +1,157 @@ +/** Shared Bash heredoc word parser for descriptor extraction and policy masking. */ + +function physicalLines(source) { + const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? []; + if (rawLines.length === 0) rawLines.push(""); + let offset = 0; + return rawLines.map((raw) => { + const record = { raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, ""), start: offset }; + offset += raw.length; + return record; + }); +} + +function heredocOperator(line) { + let quote = null; + let arithmeticDepth = 0; + for (let index = 0; index < line.length - 1; index += 1) { + const character = line[index]; + if (quote !== null) { + if (character === quote) quote = null; + else if (quote === '"' && character === "\\") index += 1; + continue; + } + if (character === "'" || character === '"') { quote = character; continue; } + if (character === "\\") { index += 1; continue; } + if (character === "#" && (index === 0 || /[ \t;|&()]/u.test(line[index - 1]))) break; + if (character === "(" && line[index + 1] === "(") { arithmeticDepth += 1; index += 1; continue; } + if (character === ")" && line[index + 1] === ")" && arithmeticDepth > 0) { arithmeticDepth -= 1; index += 1; continue; } + if (arithmeticDepth > 0 || character !== "<" || line[index + 1] !== "<") continue; + if (line[index - 1] === "<" || line[index + 2] === "<") { index += 1; continue; } + return index; + } + return -1; +} + +function endsWithBashContinuation(line) { + let quote = null; + for (let index = 0; index < line.length; index += 1) { + const character = line[index]; + if (quote === null && character === "`") { index += 1; continue; } + if (quote === "'") { if (character === "'") quote = null; continue; } + if (character === '"') { if (quote === '"') quote = null; else if (quote === null) quote = '"'; continue; } + if (character !== "\\") continue; + if (index === line.length - 1) return true; + if (quote === null || (quote === '"' && '$`"\\'.includes(line[index + 1]))) index += 1; + } + return false; +} + +function bashLogicalLine(lines, start) { + let line = lines[start]; + let end = start; + while (endsWithBashContinuation(line)) { + if (end + 1 >= lines.length) break; + line = `${line.slice(0, -1)}${lines[end + 1]}`; + end += 1; + } + return { line, end }; +} + +function bashHeredocOpener(line, operator, label, lineNumber) { + let cursor = operator + 2; + let stripTabs = false; + if (line[cursor] === "-") { stripTabs = true; cursor += 1; } + while (line[cursor] === " " || line[cursor] === "\t") cursor += 1; + const unsupported = () => { throw new Error(`${label}:${lineNumber}: unsupported Bash heredoc opener`); }; + if (cursor >= line.length || line[cursor] === "#") unsupported(); + let delimiter = ""; + let quotedDelimiter = false; + while (cursor < line.length) { + const character = line[cursor]; + if (character === " " || character === "\t" || ";|&<>".includes(character)) break; + if (character === "'" || character === '"') { + quotedDelimiter = true; + const quote = character; + cursor += 1; + let closed = false; + while (cursor < line.length) { + const quoted = line[cursor]; + if (quoted === quote) { closed = true; cursor += 1; break; } + if (quote === '"' && quoted === "\\") { + cursor += 1; + if (cursor >= line.length) unsupported(); + const escaped = line[cursor]; + delimiter += '$`"\\'.includes(escaped) ? escaped : `\\${escaped}`; + cursor += 1; + continue; + } + delimiter += quoted; + cursor += 1; + } + if (!closed) unsupported(); + continue; + } + if (character === "\\") { + quotedDelimiter = true; + cursor += 1; + if (cursor >= line.length) unsupported(); + delimiter += line[cursor]; + cursor += 1; + continue; + } + if (character === "$" || character === "`" || "(){}[]*?".includes(character)) unsupported(); + delimiter += character; + cursor += 1; + } + if (delimiter.length === 0) unsupported(); + if (heredocOperator(line.slice(cursor)) >= 0) unsupported(); + return { delimiter, stripTabs, expandable: !quotedDelimiter }; +} + +function parsedBashHeredocs(source, label) { + const records = physicalLines(source); + const lines = records.map((record) => record.text); + const extracted = []; + for (let index = 0; index < lines.length; index += 1) { + const logical = bashLogicalLine(lines, index); + const operator = heredocOperator(logical.line); + if (operator < 0) { index = logical.end; continue; } + const opener = index; + const { delimiter, stripTabs, expandable } = bashHeredocOpener(logical.line, operator, label, index + 1); + index = logical.end; + const body = []; + const startLine = index + 2; + const bodyStart = records[index + 1]?.start ?? source.length; + let closed = false; + for (index += 1; index < lines.length; index += 1) { + const candidate = stripTabs ? lines[index].replace(/^\t+/u, "") : lines[index]; + if (candidate === delimiter) { closed = true; break; } + body.push(candidate); + } + const bodyEnd = closed ? records[index].start : source.length; + extracted.push({ + source: `${body.join("\n")}\n`, label: `${label}:${startLine} Bash heredoc${closed ? "" : " (unclosed)"}`, + expandable, closed, bodyStart, bodyEnd, path: label, + rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""), + }); + } + return extracted; +} + +function extractBashDocuments(source, label) { + return parsedBashHeredocs(source, label).map(({ bodyStart: _start, bodyEnd: _end, closed: _closed, ...document }) => document); +} + +function literalBashHeredocBodyRanges(source, label) { + const ranges = []; + for (const heredoc of parsedBashHeredocs(source, label)) { + if (!heredoc.expandable) { + if (!heredoc.closed) throw new Error(`${label}: revision-state policy found an unclosed literal Bash heredoc`); + ranges.push({ start: heredoc.bodyStart, end: heredoc.bodyEnd }); + } + } + return ranges; +} + +export { extractBashDocuments, literalBashHeredocBodyRanges }; diff --git a/backend/scripts/clean-dist.mjs b/backend/scripts/clean-dist.mjs new file mode 100644 index 00000000..8abea9f0 --- /dev/null +++ b/backend/scripts/clean-dist.mjs @@ -0,0 +1,13 @@ +import { rm } from "node:fs/promises"; +import { basename, dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDirectory = dirname(fileURLToPath(import.meta.url)); +const backendRoot = resolve(scriptDirectory, ".."); +const target = resolve(backendRoot, "dist"); + +if (dirname(target) !== backendRoot || basename(target) !== "dist") { + throw new Error(`Refusing to clean non-dist target: ${target}`); +} + +await rm(target, { recursive: true, force: true }); diff --git a/backend/scripts/clean-dist.test.mjs b/backend/scripts/clean-dist.test.mjs new file mode 100644 index 00000000..d5425ab5 --- /dev/null +++ b/backend/scripts/clean-dist.test.mjs @@ -0,0 +1,147 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { + access, cp, lstat, mkdir, mkdtemp, readFile, rm, symlink, writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const execFileAsync = promisify(execFile); +const backendRoot = fileURLToPath(new URL("../", import.meta.url)); +const ownedRoots = []; + +function packageBuildInvocation(platform = process.platform, environment = process.env) { + if (platform === "win32") { + const comspec = environment.ComSpec ?? environment.COMSPEC; + if (!comspec) throw new Error("ComSpec is required to run npm on Windows."); + return { executable: comspec, args: ["/d", "/s", "/c", "npm.cmd run build"] }; + } + return { executable: "npm", args: ["run", "build"] }; +} + +async function isMissing(path) { + try { + await access(path); + return false; + } catch (error) { + if (error?.code === "ENOENT") return true; + throw error; + } +} + +async function createOwnedRoot(prefix) { + const root = await mkdtemp(join(tmpdir(), prefix)); + ownedRoots.push(root); + return root; +} + +async function copyCleaner(fixtureRoot) { + await mkdir(join(fixtureRoot, "scripts"), { recursive: true }); + const cleaner = join(fixtureRoot, "scripts", "clean-dist.mjs"); + await cp(join(backendRoot, "scripts", "clean-dist.mjs"), cleaner); + return cleaner; +} + +async function createBackendFixture() { + const fixtureRoot = await createOwnedRoot("thoth-backend-clean-dist-"); + await Promise.all([ + cp(join(backendRoot, "package.json"), join(fixtureRoot, "package.json")), + cp(join(backendRoot, "tsconfig.json"), join(fixtureRoot, "tsconfig.json")), + cp(join(backendRoot, "src"), join(fixtureRoot, "src"), { recursive: true }), + copyCleaner(fixtureRoot), + ]); + const dependencyRoot = join(backendRoot, "node_modules"); + const dependencyEntry = await lstat(dependencyRoot); + if (!dependencyEntry.isDirectory() || dependencyEntry.isSymbolicLink()) { + throw new Error("Backend node_modules must be a real directory."); + } + await symlink( + dependencyRoot, + join(fixtureRoot, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + return fixtureRoot; +} + +async function removeOwnedRoot(root) { + for (const childName of ["node_modules", "dist"]) { + const child = join(root, childName); + try { + const entry = await lstat(child); + if (entry.isSymbolicLink()) { + await rm(child, { recursive: true, force: true }); + } else if (childName === "node_modules") { + throw new Error(`Refusing to clean fixture with a non-link node_modules: ${root}`); + } + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + } + await rm(root, { recursive: true, force: true }); +} + +test.afterEach(async () => { + for (const root of ownedRoots.splice(0)) await removeOwnedRoot(root); +}); + +test("Windows package builds use ComSpec instead of executing npm.cmd directly", () => { + assert.deepEqual( + packageBuildInvocation("win32", { ComSpec: "C:\\Windows\\System32\\cmd.exe" }), + { + executable: "C:\\Windows\\System32\\cmd.exe", + args: ["/d", "/s", "/c", "npm.cmd run build"], + }, + ); + assert.throws(() => packageBuildInvocation("win32", {}), /ComSpec is required/); +}); + +test("cleaner is idempotent and removes a dist link without following it", async () => { + const fixtureRoot = await createOwnedRoot("thoth-backend-cleaner-"); + const cleaner = await copyCleaner(fixtureRoot); + const fixtureDist = join(fixtureRoot, "dist"); + + await execFileAsync(process.execPath, [cleaner], { cwd: fixtureRoot }); + assert.equal(await isMissing(fixtureDist), true); + await execFileAsync(process.execPath, [cleaner], { cwd: fixtureRoot }); + assert.equal(await isMissing(fixtureDist), true); + + const outsideRoot = await createOwnedRoot("thoth-backend-cleaner-outside-"); + const outsideSentinel = join(outsideRoot, "sentinel.txt"); + await writeFile(outsideSentinel, "outside-owned-data\n", "utf8"); + await symlink(outsideRoot, fixtureDist, process.platform === "win32" ? "junction" : "dir"); + + await execFileAsync(process.execPath, [cleaner], { cwd: fixtureRoot }); + + assert.equal(await isMissing(fixtureDist), true, "dist link survived cleaner"); + assert.equal(await readFile(outsideSentinel, "utf8"), "outside-owned-data\n"); +}); + +test("package build replaces the complete backend distribution in an owned fixture", async () => { + const fixtureRoot = await createBackendFixture(); + const copiedPackage = JSON.parse(await readFile(join(fixtureRoot, "package.json"), "utf8")); + assert.equal(copiedPackage.scripts.prebuild, "node scripts/clean-dist.mjs"); + + const workspacesDist = join(fixtureRoot, "dist", "workspaces"); + const staleModules = [ + "stale-build-sentinel.js", + "migrate-legacy.js", + "migrate-v2-qdrant.js", + ].map((name) => join(workspacesDist, name)); + await mkdir(workspacesDist, { recursive: true }); + await Promise.all(staleModules.map((path) => writeFile(path, "export const stale = true;\n", "utf8"))); + + const { executable, args } = packageBuildInvocation(); + await execFileAsync(executable, args, { cwd: fixtureRoot }); + + for (const path of staleModules) { + assert.equal(await isMissing(path), true, `stale module survived the package build: ${path}`); + } + assert.equal( + await isMissing(join(fixtureRoot, "dist", "server.js")), + false, + "server output was not compiled", + ); +}); diff --git a/backend/scripts/p1-acceptance.mjs b/backend/scripts/p1-acceptance.mjs new file mode 100755 index 00000000..1752504a --- /dev/null +++ b/backend/scripts/p1-acceptance.mjs @@ -0,0 +1,2243 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from "node:crypto"; +import { + accessSync, closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, + openSync, readFileSync, readdirSync, realpathSync, statSync, +} from "node:fs"; +import { + access, chmod, lstat, mkdir, open, readFile, readdir, realpath, rename, rm, stat, symlink, writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { createRequire, syncBuiltinESMExports } from "node:module"; +import { Socket, isIP } from "node:net"; +import { + basename, dirname, isAbsolute, join, relative, resolve, sep, +} from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const require = createRequire(import.meta.url); +const mutableChildProcess = require("node:child_process"); +const mutableDgram = require("node:dgram"); +const mutableDns = require("node:dns"); +const mutableWorkerThreads = require("node:worker_threads"); +let commandEventSink; +let activeCommandCheckId; +let activeExecutablePolicy; +let activePolicyRejectionSink; +let integrationOwner; +let productionSurfaceOwner; +const RUN_ID = /^p1-[0-9a-f]{32}$/; +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const SAFE_RELATIVE = /^(?!\/)(?!.*(?:^|\/)\.\.(?:\/|$))(?!.*\\)[A-Za-z0-9._/-]+$/; +const COMMAND = /^[A-Za-z0-9._+-]+$/; +export const CHECK_IDS = Object.freeze([ + "preflight", "clean_state", "ownership", "local_git_bootstrap", + "http_validate_publish_pull_read_export", "same_revision_git_objects", + "content_only_revision", "snapshot_and_docs", "runtime_render_determinism", + "tht_config_check", "negative_schema_cases", "negative_context_case", + "no_p1_scope_artifacts", "secret_scan", "cleanup_confinement", +]); +const TOPOLOGY = [ + "remote.git", "author", "installation/registry", "installation/data", "installation/runtime", + "fixture-secrets", "fixtures/descriptors", "fixtures/requests", "requests", "responses", + "exports/raw", "exports/extracted", "rendered", "logs", +]; +const ZIP_FILES = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"]; +const MAX_OUTPUT = 16 * 1024 * 1024; +const PYTHON_LOCK_HOLDER_PROGRAM = [ + "import fcntl, os, sys", + "fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT | getattr(os, 'O_NOFOLLOW', 0), 0o600)", + "try:", + " fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)", + "except BlockingIOError:", + " sys.exit(73)", + "sys.stdout.write('locked\\n')", + "sys.stdout.flush()", + "sys.stdin.buffer.read()", +].join("\n"); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +export function scalarSecretBytes(value) { + if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid"); + return Buffer.from(value); +} +function canonicalRoot(repositoryRoot) { return realpathSync(repositoryRoot); } +export function canonicalIntegrationBase(repositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p1-integration"); +} +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!existsSync(cursor)) break; + const entry = lstatSync(cursor); + if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink"); + } +} +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(16).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + let failure = error; + if (handle) { + try { await handle.close(); } catch (closeError) { failure = closeError; } + } + try { await rm(staging, { force: true }); } catch (cleanupError) { failure = cleanupError; } + throw failure; + } +} +function exactOwnedResources(run) { + const contextual = join(run.root, "installation", "runtime", "contextual"); + return [ + run.root, + join(run.root, "remote.git"), + join(run.root, "author"), + join(run.root, "installation", "registry"), + join(run.root, "installation", "data"), + join(run.root, "installation", "runtime"), + contextual, + join(contextual, "remote.git"), + join(contextual, "author"), + join(contextual, "registry"), + join(contextual, "data"), + join(contextual, "runtime"), + ]; +} +function initialListeners(pid) { + return ["primary", "contextual"].map((name) => ({ + name, kind: "fastify", host: "127.0.0.1", requestedPort: 0, pid, state: "not_started", + })); +} +function ownership(run, listeners = run.listeners) { + return { + schemaVersion: 1, runId: run.runId, runNonce: run.nonce, root: run.root, + repositoryRoot: run.repositoryRoot, startedAt: run.startedAt, pid: run.pid, + listeners, + resources: exactOwnedResources(run), + }; +} +async function writeOwnership(run, listenerUpdate) { + const listeners = listenerUpdate + ? run.listeners.map((listener) => listener.name === listenerUpdate.name ? listenerUpdate : listener) + : run.listeners; + await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownership(run, listeners), null, 2)}\n`); + run.listeners = listeners; +} +export async function createOwnedRun({ repositoryRoot, runId, nonce, now, pid } = {}) { + const repo = canonicalRoot(repositoryRoot); + const base = canonicalIntegrationBase(repo); + validateNoSymlinkAncestors(repo, base); + await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + validateNoSymlinkAncestors(repo, base); + const id = runId ?? `p1-${randomBytes(16).toString("hex")}`; + const root = validateRunRoot(repo, join(base, id), id); + const run = { + repositoryRoot: repo, root, runId: id, nonce: nonce ?? randomBytes(32).toString("hex"), + startedAt: now ?? nowIso(), pid: pid ?? process.pid, + listeners: initialListeners(pid ?? process.pid), + }; + if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity"); + await mkdir(root, { mode: 0o700 }); + await writeOwnership(run); + return run; +} +function strictOwnership(value, run, expectedNonce) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed"); + const validListeners = Array.isArray(value.listeners) && value.listeners.length === 2 + && value.listeners.every((listener, index) => { + const expectedName = ["primary", "contextual"][index]; + const common = listener?.name === expectedName && listener.kind === "fastify" && listener.host === "127.0.0.1" + && listener.requestedPort === 0 && listener.pid === process.pid && ["not_started", "listening", "closed", "close_failed"].includes(listener.state); + return common && (listener.state === "not_started" + ? !("actualPort" in listener) + : Number.isInteger(listener.actualPort) && listener.actualPort >= 1 && listener.actualPort <= 65535); + }); + if (value.schemaVersion !== 1 || value.runId !== run.runId || value.runNonce !== expectedNonce + || value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid + || !ISO_UTC.test(value.startedAt ?? "") || !validListeners + || JSON.stringify(value.resources) !== JSON.stringify(exactOwnedResources(run))) throw new Error("ownership identity mismatch"); + return value; +} +export async function readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }) { + const repo = canonicalRoot(repositoryRoot); + const id = basename(resolve(runRoot)); + const lexical = validateRunRoot(repo, runRoot, id); + const rootEntry = await lstat(lexical); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory"); + if (await realpath(lexical) !== lexical) throw new Error("owned run root is not canonical"); + const ownershipPath = join(lexical, "ownership.json"); + const ownershipEntry = await lstat(ownershipPath); + if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe"); + let value; + try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); } + return strictOwnership(value, { + repositoryRoot: repo, root: lexical, runId: id, nonce: expectedNonce, + startedAt: value.startedAt, pid: process.pid, + }, expectedNonce); +} + +export async function cleanupOwnedRun({ repositoryRoot, runRoot, expectedNonce }) { + const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }); + const base = canonicalIntegrationBase(repositoryRoot); + const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`); + await rename(runRoot, tombstone); + await rm(tombstone, { recursive: true }); +} +export async function finalizeOwnedRun({ run, success, keep }) { + if (!success || keep) return false; + await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + return true; +} + +function resolveTrustedSystemExecutableSync(name) { + const candidates = process.platform === "win32" + ? [] + : [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]; + for (const candidate of candidates) { + try { + accessSync(candidate, fsConstants.X_OK); + const canonical = realpathSync(candidate); + if (statSync(canonical).isFile()) return canonical; + } catch { /* try the next fixed trusted executable location */ } + } + throw new Error(`cannot resolve trusted system executable: ${name}`); +} + +const THT_EDITABLE_FINDER_NORMALIZED_SHA256 = "3489b09b63511e27e1ae4d3f858d2bc576fe77beb5ea97607673781a32d2723e"; + +function assertRegularNonSymlink(path, label) { + let entry; + try { entry = lstatSync(path); } catch { throw new Error(`${label} is unavailable`); } + if (!entry.isFile() || entry.isSymbolicLink() || realpathSync(path) !== path) throw new Error(`${label} identity is invalid`); + return entry; +} + +function assertSafeSitePackages(identity) { + const entries = readdirSync(identity.sitePackages, { withFileTypes: true }); + const pthNames = entries.filter(({ name }) => name.endsWith(".pth")).map(({ name }) => name); + const finderNames = entries.filter(({ name }) => /^__editable___tht_.*_finder\.py$/.test(name)).map(({ name }) => name); + if (pthNames.length !== 1 || pthNames[0] !== identity.pthName || finderNames.length !== 1 + || finderNames[0] !== identity.finderName || entries.some((entry) => entry.isSymbolicLink())) { + throw new Error("trusted THT editable binding is ambiguous"); + } + const startup = /^(?:sitecustomize|usercustomize|tht)(?:\..*)?$/; + if (entries.some(({ name }) => startup.test(name))) throw new Error("trusted THT editable binding has an import startup override"); + const cache = join(identity.sitePackages, "__pycache__"); + if (existsSync(cache) && readdirSync(cache).some((name) => startup.test(name) + || /^__editable___tht_.*_finder\..*\.pyc$/.test(name))) { + throw new Error("trusted THT editable binding has an import startup override"); + } + assertRegularNonSymlink(identity.pthPath, "trusted THT editable pth"); + assertRegularNonSymlink(identity.finderPath, "trusted THT editable finder"); + const pth = readFileSync(identity.pthPath, "utf8"); + const rawFinder = readFileSync(identity.finderPath, "utf8"); + const finder = rawFinder.replaceAll("\r\n", "\n"); + const occurrences = finder.split(identity.sourceRoot).length - 1; + const normalized = finder.replaceAll(identity.sourceRoot, ""); + if (pth !== identity.expectedPth || occurrences !== 5 || sha256(normalized) !== THT_EDITABLE_FINDER_NORMALIZED_SHA256) { + throw new Error("trusted THT editable binding is invalid"); + } + return { pth, finder: rawFinder }; +} + +function sourceTreeFilesSync(root, current = root, files = []) { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) throw new Error("trusted THT source contains a symlink"); + if (entry.isDirectory()) sourceTreeFilesSync(root, path, files); + else if (entry.isFile()) files.push(relative(root, path).split(sep).join("/")); + else throw new Error("trusted THT source contains a special file"); + } + return files; +} + +function assertBoundThtFiles(identity) { + assertRegularNonSymlink(identity.entrypointPath, "trusted THT entrypoint"); + assertRegularNonSymlink(identity.pythonCanonicalPath, "trusted THT interpreter"); + if (realpathSync(identity.pythonPath) !== identity.pythonCanonicalPath) throw new Error("trusted THT identity changed: interpreter"); + if (sha256(readFileSync(identity.entrypointPath)) !== identity.entrypointSha256 + || sha256(readFileSync(identity.pythonCanonicalPath)) !== identity.pythonSha256) { + throw new Error("trusted THT identity changed: entrypoint or interpreter"); + } + for (const root of [dirname(identity.sourceRoot), dirname(identity.entrypointPath)]) { + for (const name of ["sitecustomize.py", "sitecustomize.pyc", "usercustomize.py", "usercustomize.pyc", "tht.py", "tht.pyc"]) { + if (existsSync(join(root, name))) throw new Error("trusted THT identity changed: import startup override"); + } + } + const { pth, finder } = assertSafeSitePackages(identity); + if (sha256(pth) !== identity.pthSha256 || sha256(finder) !== identity.editableFinderSha256) { + throw new Error("trusted THT identity changed: editable binding"); + } + const actualPaths = sourceTreeFilesSync(identity.sourceRoot).map((path) => `harness/tht/${path}`); + actualPaths.push("harness/pyproject.toml"); + actualPaths.sort(); + const expectedPaths = identity.sourceManifest.map(({ path }) => path).sort(); + if (JSON.stringify(actualPaths) !== JSON.stringify(expectedPaths)) throw new Error("trusted THT identity changed: source manifest"); + for (const file of identity.sourceManifest) { + const path = join(identity.repositoryRoot, ...file.path.split("/")); + assertRegularNonSymlink(path, "trusted THT source file"); + if (sha256(readFileSync(path)) !== file.sha256) throw new Error("trusted THT identity changed: source bytes"); + } +} + +export function revalidateThtIdentity(identity) { + try { assertBoundThtFiles(identity); } catch (error) { + if (/^trusted THT identity changed/.test(error.message)) throw error; + throw new Error(`trusted THT identity changed: ${error.message}`); + } + return true; +} + +async function gitTrackedSourceManifest(repo, gitPath) { + const listing = await runCommand({ + executable: gitPath, + argv: ["-C", repo, "ls-files", "-s", "-z", "--", "harness/tht", "harness/pyproject.toml"], + env: baseSafeGitEnvironment(gitPath), + }); + const treeListing = await runCommand({ + executable: gitPath, + argv: ["-C", repo, "ls-tree", "-r", "-z", "HEAD", "--", "harness/tht", "harness/pyproject.toml"], + env: baseSafeGitEnvironment(gitPath), + }); + const treeEntries = new Map(treeListing.stdout.split("\0").filter(Boolean).map((record) => { + const match = /^(100644|100755) blob ([0-9a-f]{40,64})\t(harness\/(?:tht\/[^\0]+|pyproject\.toml))$/.exec(record); + if (!match) throw new Error("trusted THT Git tree identity is invalid"); + return [match[3], { mode: match[1], oid: match[2] }]; + })); + const manifest = []; + for (const record of listing.stdout.split("\0").filter(Boolean)) { + const match = /^(100644|100755) ([0-9a-f]{40,64}) 0\t(harness\/(?:tht\/[^\0]+|pyproject\.toml))$/.exec(record); + if (!match) throw new Error("trusted THT Git index identity is invalid"); + const [, mode, oid, path] = match; + const tree = treeEntries.get(path); + if (!tree || tree.mode !== mode || tree.oid !== oid) throw new Error(`trusted THT Git index differs from HEAD tree: ${path}`); + treeEntries.delete(path); + const blob = await runCommand({ executable: gitPath, argv: ["-C", repo, "cat-file", "blob", oid], env: baseSafeGitEnvironment(gitPath) }); + const bytes = Buffer.from(blob.stdout); + const worktreePath = join(repo, ...path.split("/")); + assertRegularNonSymlink(worktreePath, "trusted THT source file"); + if (!bytes.equals(readFileSync(worktreePath))) throw new Error(`trusted THT source bytes differ from Git: ${path}`); + manifest.push({ path, mode, gitBlob: oid, sha256: sha256(bytes) }); + } + if (treeEntries.size !== 0) throw new Error("trusted THT Git index differs from HEAD tree"); + manifest.sort((left, right) => left.path.localeCompare(right.path)); + if (!manifest.some(({ path }) => path === "harness/pyproject.toml") + || !manifest.some(({ path }) => path === "harness/tht/cli/__init__.py")) throw new Error("trusted THT tracked source manifest is incomplete"); + return manifest; +} + +export async function resolveProductionExecutables({ repositoryRoot } = {}) { + const repo = canonicalRoot(repositoryRoot); + const gitPath = resolveTrustedSystemExecutableSync("git"); + const pythonPath = resolveTrustedSystemExecutableSync("python3"); + const thtPath = join(repo, "harness", ".venv", "bin", "tht"); + assertRegularNonSymlink(thtPath, "trusted THT entrypoint"); + accessSync(thtPath, fsConstants.X_OK); + const entrypointBytes = await readFile(thtPath, "utf8"); + const lines = entrypointBytes.replaceAll("\r\n", "\n").split("\n"); + const shebang = lines.shift() ?? ""; + const pythonLexical = shebang.startsWith("#!") ? shebang.slice(2) : ""; + const expectedBody = [ + "import sys", "from tht.cli import app", "if __name__ == '__main__':", + " if sys.argv[0].endswith('.exe'):", " sys.argv[0] = sys.argv[0][:-4]", + " sys.exit(app())", "", + ].join("\n"); + const venvBin = join(repo, "harness", ".venv", "bin"); + if (dirname(pythonLexical) !== venvBin || !/^python3(?:\.\d+)?$/.test(basename(pythonLexical)) + || realpathSync(pythonLexical) !== realpathSync(join(venvBin, "python")) + || lines.join("\n") !== expectedBody) throw new Error("trusted THT generated entrypoint is invalid"); + const sourceRoot = join(repo, "harness", "tht"); + const pyproject = await readFile(join(repo, "harness", "pyproject.toml"), "utf8"); + if (!/^tht\s*=\s*["']tht\.cli:app["']$/m.test(pyproject)) throw new Error("trusted THT console-script declaration is invalid"); + const pythonVersion = basename(pythonLexical); + const sitePackages = join(repo, "harness", ".venv", "lib", pythonVersion, "site-packages"); + const siteEntries = await readdir(sitePackages); + const pthNames = siteEntries.filter((name) => name.endsWith(".pth")); + const finderNames = siteEntries.filter((name) => /^__editable___tht_.*_finder\.py$/.test(name)); + if (pthNames.length !== 1 || finderNames.length !== 1) throw new Error("trusted THT editable binding is ambiguous"); + const finderModule = finderNames[0].slice(0, -3); + const identity = { + repositoryRoot: repo, entrypoint: "generated-console-script", entrypointPath: thtPath, + entrypointSha256: sha256(entrypointBytes), pythonPath: pythonLexical, + pythonCanonicalPath: realpathSync(pythonLexical), pythonSha256: sha256(await readFile(realpathSync(pythonLexical))), + sourceRoot, sourceStatus: "git-index-byte-identical", sitePackages, + pthName: pthNames[0], pthPath: join(sitePackages, pthNames[0]), expectedPth: `import ${finderModule}; ${finderModule}.install()`, + finderName: finderNames[0], finderPath: join(sitePackages, finderNames[0]), + }; + const binding = assertSafeSitePackages(identity); + identity.pthSha256 = sha256(binding.pth); + identity.editableFinderSha256 = sha256(binding.finder); + identity.sourceManifest = await gitTrackedSourceManifest(repo, gitPath); + assertBoundThtFiles(identity); + return { gitPath, pythonPath, thtPath, thtIdentity: identity }; +} + +let fallbackGitHooksPath; +function ownedFallbackGitHooksPath() { + if (!fallbackGitHooksPath) fallbackGitHooksPath = realpathSync(mkdtempSync(join(tmpdir(), `p1-git-hooks-${process.pid}-`))); + return fallbackGitHooksPath; +} +function gitSafeConfig(hooksPath) { + return [ + ["core.hooksPath", hooksPath], ["core.attributesFile", "/dev/null"], ["core.fsmonitor", "false"], + ["core.pager", "/bin/cat"], ["pager.status", "false"], ["diff.external", ""], + ["interactive.diffFilter", ""], ["commit.gpgSign", "false"], ["tag.gpgSign", "false"], + ["user.signingKey", ""], ["gpg.program", "/bin/false"], ["credential.helper", ""], + ["core.askPass", "/bin/false"], ["sequence.editor", "/bin/false"], ["core.editor", "/bin/false"], + ["protocol.allow", "never"], ["protocol.file.allow", "always"], ["protocol.ext.allow", "never"], + ]; +} +function hardenedGitArgv(argv, hooksPath) { + return [...gitSafeConfig(hooksPath).flatMap(([key, value]) => ["-c", `${key}=${value}`]), ...argv]; +} +function baseSafeGitEnvironment(gitPath) { + return { + PATH: dirname(gitPath), HOME: "/nonexistent", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_SYSTEM: "/dev/null", + GIT_CONFIG_GLOBAL: "/dev/null", GIT_ATTR_NOSYSTEM: "1", GIT_TERMINAL_PROMPT: "0", + GIT_ASKPASS: "/bin/false", SSH_ASKPASS: "/bin/false", GIT_ALLOW_PROTOCOL: "file", + GIT_PROTOCOL_FROM_USER: "0", GIT_PAGER: "/bin/cat", + }; +} +function assertEmptyHooksDirectory(path) { + let entry; + try { entry = lstatSync(path); } catch { throw new Error("unsafe Git repository state: hooks directory is unavailable"); } + if (!entry.isDirectory() || entry.isSymbolicLink() || realpathSync(path) !== path || readdirSync(path).length !== 0) { + throw new Error("unsafe Git repository state: hooks directory is not owned and empty"); + } +} +function parseLocalGitConfig(bytes) { + let section; + const entries = []; + for (const raw of bytes.replaceAll("\r\n", "\n").split("\n")) { + const line = raw.trim(); + if (!line || line.startsWith("#") || line.startsWith(";")) continue; + const sectionMatch = /^\[([A-Za-z0-9.-]+)(?:\s+"([^"\\]*)")?\]$/.exec(line); + if (sectionMatch) { section = sectionMatch[2] ? `${sectionMatch[1].toLowerCase()}.${sectionMatch[2]}` : sectionMatch[1].toLowerCase(); continue; } + const valueMatch = /^([A-Za-z0-9.-]+)\s*=\s*(.*)$/.exec(line); + if (!section || !valueMatch || /[\\\0]/.test(valueMatch[2])) throw new Error("unsafe Git repository state: local config is malformed"); + entries.push([`${section}.${valueMatch[1].toLowerCase()}`, valueMatch[2]]); + } + return entries; +} +function safeLocalGitConfigEntry(key, value, runRoot) { + if (key === "core.repositoryformatversion") return value === "0"; + if (["core.filemode", "core.bare", "core.logallrefupdates", "core.ignorecase", "core.precomposeunicode"].includes(key)) return /^(?:true|false)$/.test(value); + if (key === "remote.origin.url") return ownedGitPath(value, runRoot); + if (key === "remote.origin.fetch") return /^\+refs\/heads\/(?:\*|main|invalid-context):refs\/remotes\/origin\/(?:\*|main|invalid-context)$/.test(value); + if (/^branch\.(?:main|invalid-context)\.remote$/.test(key)) return value === "origin"; + if (/^branch\.(?:main|invalid-context)\.merge$/.test(key)) return /^refs\/heads\/(?:main|invalid-context)$/.test(value); + if (key === "user.name") return FIXTURE_GIT_CONFIG.get("user.name")?.has(value) === true; + if (key === "user.email") return FIXTURE_GIT_CONFIG.get("user.email")?.has(value) === true; + return false; +} +function repositoryGitDirectory(argv, cwd, runRoot) { + let candidate; + if (argv[0] === "--git-dir") candidate = argv[1]; + else if (argv[0] === "-C") candidate = join(argv[1], ".git"); + else if (cwd) candidate = join(cwd, ".git"); + if (!candidate || !ownedGitPath(resolve(candidate), runRoot) || !existsSync(candidate)) return undefined; + const entry = lstatSync(candidate); + if (entry.isFile() && !entry.isSymbolicLink()) { + const match = /^gitdir: (.+)\n?$/.exec(readFileSync(candidate, "utf8")); + if (!match) throw new Error("unsafe Git repository state: gitdir file is malformed"); + candidate = resolve(dirname(candidate), match[1]); + } else if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("unsafe Git repository state: git directory is unsafe"); + return realpathSync(candidate); +} +function findAttributes(current, gitDirectory, findings = []) { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (path === gitDirectory || (entry.name === ".git" && (entry.isDirectory() || entry.isFile()))) continue; + if (entry.isSymbolicLink()) throw new Error("unsafe Git repository state: worktree contains a symlink"); + if (entry.isDirectory()) findAttributes(path, gitDirectory, findings); + else if (entry.name === ".gitattributes") findings.push(path); + } + return findings; +} +function validateGitDirectoryState(gitDirectory, runRoot) { + const configPath = join(gitDirectory, "config"); + const configEntry = lstatSync(configPath); + if (!configEntry.isFile() || configEntry.isSymbolicLink()) throw new Error("unsafe Git repository state: local config is unsafe"); + const entries = parseLocalGitConfig(readFileSync(configPath, "utf8")); + if (entries.some(([key, value]) => !safeLocalGitConfigEntry(key, value, runRoot))) { + throw new Error("unsafe Git repository state: local config is not exact"); + } + const infoAttributes = join(gitDirectory, "info", "attributes"); + if (existsSync(infoAttributes)) { + const entry = lstatSync(infoAttributes); + if (!entry.isFile() || entry.isSymbolicLink() || readFileSync(infoAttributes).length !== 0) { + throw new Error("unsafe Git repository state: info attributes are not empty"); + } + } + const hooks = join(gitDirectory, "hooks"); + if (existsSync(hooks)) for (const entry of readdirSync(hooks, { withFileTypes: true })) { + if (entry.isSymbolicLink() || !entry.isFile() || !entry.name.endsWith(".sample")) { + throw new Error("unsafe Git repository state: repository hook is present"); + } + } + const worktree = dirname(gitDirectory); + if (basename(gitDirectory) === ".git" && findAttributes(worktree, gitDirectory).length > 0) { + throw new Error("unsafe Git repository state: worktree attributes are present"); + } + return entries; +} +function exactRemoteTarget(argv, entries) { + const offset = argv[0] === "-c" || argv[0] === "-C" || argv[0] === "--git-dir" ? 2 : 0; + const verb = argv[offset]; const args = argv.slice(offset + 1); + if (verb === "clone") { + const operands = args.filter((value) => value !== "--bare" && value !== "--single-branch" && value !== "--" + && value !== "--branch" && value !== "main" && value !== "invalid-context"); + return operands.at(-2); + } + if (["fetch", "push"].includes(verb) && args.includes("origin")) { + return entries.find(([key]) => key === "remote.origin.url")?.[1]; + } + return undefined; +} +function assertSafeGitRepositoryState(argv, cwd, runRoot, fixedHooksPath) { + assertEmptyHooksDirectory(fixedHooksPath); + if (argv[0] === "-c") assertEmptyHooksDirectory(argv[1].slice("core.hooksPath=".length)); + const gitDirectory = repositoryGitDirectory(argv, cwd, runRoot); + const entries = gitDirectory ? validateGitDirectoryState(gitDirectory, runRoot) : []; + const target = exactRemoteTarget(argv, entries); + if (target !== undefined) { + if (!ownedGitPath(target, runRoot) || !existsSync(join(target, "config"))) { + throw new Error("unsafe Git repository state: remote target is not exact and owned"); + } + validateGitDirectoryState(realpathSync(target), runRoot); + } +} +function rawGitCommonDirectory(gitDirectory) { + const path = join(gitDirectory, "commondir"); + if (!existsSync(path)) return gitDirectory; + const entry = lstatSync(path); + const value = entry.isFile() && !entry.isSymbolicLink() ? readFileSync(path, "utf8") : ""; + if (!/^[^\0\n\r]+\n?$/.test(value)) throw new Error("unsafe Git repository state: common directory is unsafe"); + const common = resolve(gitDirectory, value.trimEnd()); + const commonEntry = lstatSync(common); + if (!commonEntry.isDirectory() || commonEntry.isSymbolicLink() || realpathSync(common) !== common) { + throw new Error("unsafe Git repository state: common directory is unsafe"); + } + return common; +} +function rawGitConfigEntries(path, required = false) { + if (!existsSync(path)) { + if (required) throw new Error("unsafe Git repository state: local config is unavailable"); + return []; + } + const entry = lstatSync(path); + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("unsafe Git repository state: local config is unsafe"); + return parseLocalGitConfig(readFileSync(path, "utf8")); +} +function unsafeRawGitConfigKey(key, value = "") { + const normalized = key.toLowerCase(); + if (normalized === "core.fsmonitor" && /^(?:true|false|1|0)$/i.test(String(value).trim())) return false; + return /^(?:filter|diff)\..+\.(?:clean|smudge|process|required|textconv|external|command)$/.test(normalized) + || /^(?:remote\..+\.(?:uploadpack|receivepack)|uploadpack\..+|receivepack\..+)$/.test(normalized) + || /^credential(?:\..+)?\.helper$/.test(normalized) + || /^(?:core\.(?:hookspath|attributesfile|fsmonitor|sshcommand|askpass|pager|editor|sequence\.editor)|sequence\.editor|interactive\.difffilter|diff\.external|gpg\.program|gpg\.ssh\.program)$/.test(normalized) + || /^merge\..+\.driver$/.test(normalized) + || /^(?:pager\..+|alias\..+|difftool\..+\.(?:cmd|path)|mergetool\..+\.(?:cmd|path))$/.test(normalized) + || /^include(?:if\..+)?\.path$/.test(normalized); +} +function validateRawGitDirectoryState(gitDirectory) { + const common = rawGitCommonDirectory(gitDirectory); + const entries = [ + ...rawGitConfigEntries(join(common, "config"), true), + ...rawGitConfigEntries(join(gitDirectory, "config.worktree")), + ]; + if (entries.some(([key, value]) => unsafeRawGitConfigKey(key, value))) { + throw new Error("unsafe Git repository state: executable local config is present"); + } + for (const hooks of new Set([join(common, "hooks"), join(gitDirectory, "hooks")])) { + if (!existsSync(hooks)) continue; + const hooksEntry = lstatSync(hooks); + if (!hooksEntry.isDirectory() || hooksEntry.isSymbolicLink()) throw new Error("unsafe Git repository state: repository hooks are unsafe"); + for (const entry of readdirSync(hooks, { withFileTypes: true })) { + if (entry.isSymbolicLink() || !entry.isFile() || !entry.name.endsWith(".sample")) { + throw new Error("unsafe Git repository state: repository hook is present"); + } + } + } + return entries; +} +function rawRepositoryGitDirectory(argv, cwd) { + if (argv[0] === "--git-dir") return repositoryGitDirectory(argv, cwd); + let current = argv[0] === "-C" ? argv[1] : cwd; + if (!current) return undefined; + current = realpathSync(current); + while (true) { + if (existsSync(join(current, ".git"))) return repositoryGitDirectory(["-C", current]); + const parent = dirname(current); + if (parent === current) return undefined; + current = parent; + } +} +function assertSafeRawGitRepositoryState(argv, cwd, fixedHooksPath) { + assertEmptyHooksDirectory(fixedHooksPath); + const gitDirectory = rawRepositoryGitDirectory(argv, cwd); + const entries = gitDirectory ? validateRawGitDirectoryState(gitDirectory) : []; + const target = exactRemoteTarget(argv, entries); + if (target !== undefined) { + if (!ownedGitPath(target) || !existsSync(join(target, "config"))) { + throw new Error("unsafe Git repository state: remote target is not exact and owned"); + } + validateRawGitDirectoryState(realpathSync(target)); + } +} + +const FIXTURE_GIT_CONFIG = new Map([ + ["user.name", new Set(["P1 Fixture Curator", "P1 Context Curator"])], + ["user.email", new Set(["p1-curator@example.invalid", "p1-context@example.invalid"])], +]); +function ownedGitPath(value, runRoot) { + if (typeof value !== "string" || !isAbsolute(value) || value.includes("\0")) return false; + if (!runRoot) return true; + const lexical = resolve(value); const rel = relative(runRoot, lexical); + if (rel.startsWith("..") || isAbsolute(rel)) return false; + try { validateNoSymlinkAncestors(runRoot, lexical); } catch { return false; } + return true; +} +function ownedEmptyHooksPath(value, runRoot) { + if (!ownedGitPath(value, runRoot) || !/(?:^|\/)registry\/locks\/empty-hooks$/.test(value)) return false; + try { + const entry = lstatSync(value); + return entry.isDirectory() && !entry.isSymbolicLink() && realpathSync(value) === value; + } catch { return false; } +} +function safeGitOperand(value) { return typeof value === "string" && value.length > 0 && !value.startsWith("-") && !/[\0\n\r]/.test(value); } +function exactArray(value, expected) { return value.length === expected.length && value.every((item, index) => item === expected[index]); } +export function validateGitInvocation(argv, { runRoot } = {}) { + if (!Array.isArray(argv) || argv.some((value) => typeof value !== "string")) throw new Error("Git command is prohibited"); + if (exactArray(argv, ["--version"])) return "--version"; + let index = 0; let prefix; + if (["-C", "--git-dir"].includes(argv[0])) { + if (!ownedGitPath(argv[1], runRoot)) throw new Error("Git command is prohibited"); + prefix = argv[0]; index = 2; + } else if (argv[0] === "-c") { + if (typeof argv[1] !== "string" || !argv[1].startsWith("core.hooksPath=")) throw new Error("Git command is prohibited"); + const hooksPath = argv[1].slice("core.hooksPath=".length); + if (!ownedEmptyHooksPath(hooksPath, runRoot)) throw new Error("Git command is prohibited"); + prefix = "hooks"; index = 2; + } else if (argv[0]?.startsWith("-")) throw new Error("Git command is prohibited"); + const verb = argv[index]; const args = argv.slice(index + 1); + const branch = (value) => /^(?:main|invalid-context)$/.test(value ?? ""); + const object = (value) => safeGitOperand(value) && !/^[a-z][a-z0-9+.-]*:\/\//i.test(value) && !/^[^/\s]+@[^:\s]+:/.test(value); + const ownedPair = (values) => values.length === 2 && values.every((value) => ownedGitPath(value, runRoot)); + let valid = false; + switch (verb) { + case "init": valid = !prefix && args.length === 3 && args[0] === "--bare" && args[1] === "--initial-branch=main" && ownedGitPath(args[2], runRoot); break; + case "clone": valid = (!prefix && ownedPair(args)) + || (!prefix && args[0] === "--bare" && ownedPair(args.slice(1))) + || (prefix === "hooks" && args.length === 6 && args[0] === "--branch" && branch(args[1]) && args[2] === "--single-branch" && args[3] === "--" && ownedPair(args.slice(4))); break; + case "config": valid = !prefix && args.length === 2 && FIXTURE_GIT_CONFIG.get(args[0])?.has(args[1]) === true; break; + case "add": valid = (!prefix || prefix === "hooks") && ((args.length === 1 && /^(?:workspace-content|workspace-content\/p1-filesystem\/evidence\/guide\.md)$/.test(args[0])) + || (args.length === 2 && args[0] === "-A" && args[1] === "workspace-content/p1-filesystem/evidence") + || (args[0] === "--" && args.length >= 2 && args.slice(1).every((value) => /^(?:workspaces|workspace-docs)\/[A-Za-z0-9./-]+$/.test(value)))); break; + case "commit": valid = (!prefix || prefix === "hooks") && args.length === 2 && args[0] === "-m" && /^(?:Bootstrap curated P1 content|Update curated Evidence only|Invalid contextual Evidence state|Publish workspace p1-(?:filesystem|http|s3))$/.test(args[1]); break; + case "push": valid = (!prefix || prefix === "hooks") && (exactArray(args, ["origin", "main"]) || exactArray(args, ["origin", "invalid-context"]) + || exactArray(args, ["-u", "origin", "invalid-context"]) || exactArray(args, ["origin", "HEAD:main"])); break; + case "checkout": valid = !prefix && exactArray(args, ["-b", "invalid-context"]); break; + case "fetch": valid = (!prefix || prefix === "hooks") && (exactArray(args, ["origin", "main"]) || (args.length === 3 && args[0] === "--no-tags" && args[1] === "origin" && branch(args[2]))); break; + case "remote": valid = prefix === "hooks" && args.length === 4 && exactArray(args.slice(0, 3), ["set-url", "origin", "--"]) && ownedGitPath(args[3], runRoot); break; + case "merge": valid = prefix === "hooks" && exactArray(args, ["--ff-only", "FETCH_HEAD"]); break; + case "merge-base": valid = prefix === "hooks" && exactArray(args, ["HEAD", "FETCH_HEAD"]); break; + case "reset": valid = (!prefix || prefix === "hooks") && args.length === 2 && args[0] === "--hard" && /^(?:origin\/main|refs\/remotes\/origin\/(?:main|invalid-context))$/.test(args[1]); break; + case "clean": valid = prefix === "hooks" && exactArray(args, ["-fd", "--", "workspaces", "workspace-docs"]); break; + case "status": valid = (!prefix && (exactArray(args, ["--porcelain=v1"]) || exactArray(args, ["--porcelain"]))) + || (prefix === "hooks" && exactArray(args, ["--porcelain"])) + || (prefix === "-C" && (exactArray(args, ["--porcelain=v1", "--untracked-files=all"]) + || exactArray(args, ["--porcelain=v1", "--untracked-files=all", "--", "harness/tht", "harness/pyproject.toml"]))); break; + case "ls-files": valid = prefix === "-C" && exactArray(args, ["-s", "-z", "--", "harness/tht", "harness/pyproject.toml"]); break; + case "write-tree": valid = !prefix && args.length === 0; break; + case "show-ref": valid = (!prefix || prefix === "--git-dir" || prefix === "hooks") && args.length === 0; break; + case "symbolic-ref": valid = (!prefix || prefix === "hooks") && exactArray(args, ["--short", "HEAD"]); break; + case "rev-list": valid = ((prefix === "--git-dir" || prefix === "-C") && exactArray(args, ["--objects", "--all"])) + || (prefix === "hooks" && exactArray(args, ["--left-right", "--count", "HEAD...@{upstream}"])); break; + case "ls-tree": valid = (prefix === "hooks" && exactArray(args, ["-r", "--name-only", "HEAD", "--", "workspaces"])) + || (prefix === "-C" && exactArray(args, ["-r", "-z", "HEAD", "--", "harness/tht", "harness/pyproject.toml"])); break; + case "cat-file": valid = (!prefix || prefix === "--git-dir" || prefix === "-C" || prefix === "hooks") && args.length === 2 + && ((args[0] === "-e" || args[0] === "-t" || args[0] === "blob") && object(args[1])); break; + case "show": valid = prefix === "hooks" && args.length === 1 && object(args[0]); break; + case "rev-parse": valid = (!prefix || prefix === "--git-dir" || prefix === "hooks" || prefix === "-C") && args.length === 1 && object(args[0]); break; + default: valid = false; + } + if (!valid) throw new Error("Git command is prohibited"); + return verb; +} +function boundedChildEnvironment(value, expected) { + const environment = value ?? process.env; + if (!environment || typeof environment !== "object" || Array.isArray(environment)) throw new Error("child environment is invalid"); + const allowedExtra = new Set([ + "GIT_AUTHOR_NAME", "GIT_AUTHOR_EMAIL", "GIT_COMMITTER_NAME", "GIT_COMMITTER_EMAIL", + "THT_AUTH_USER_ID", "THT_AUTH_USERNAME", "THT_AUTH_IS_ADMIN", "THT_DWH_API_KEY", "THT_VEC_API_KEY", + "THT_VEC_WRITE_API_KEY", "THT_CA", "THT_SSL_CA", + ]); + for (const [key, value] of Object.entries(environment)) { + if (typeof value !== "string" || (!(key in expected) && !allowedExtra.has(key))) throw new Error("child environment exceeds acceptance bounds"); + } + for (const [key, value] of Object.entries(expected)) if (environment[key] !== value) throw new Error("child environment changed acceptance bounds"); + return environment; +} +function sanitizedArgvLabels(argv = []) { + return Array.isArray(argv) ? argv.filter((value) => typeof value === "string").map((value) => + isAbsolute(value) || value.includes(sep) ? "[path]" : /^[a-z]+:\/\//i.test(value) ? "[url]" : value.length > 80 ? "[value]" : value) : []; +} +function safeChildEvent(events, { surface = "child_process", api, executable, argv = [], outcome, detail, bounds }) { + events.push({ + surface, api, executable: executable ? basename(executable) : undefined, + argvLabels: sanitizedArgvLabels(argv), + outcome, ...(activeCommandCheckId ? { checkId: activeCommandCheckId } : {}), ...(detail ? { detail } : {}), ...(bounds ? { bounds } : {}), + }); +} +function recordPolicyRejection({ executable, argv, api = "validation", detail = "policy" } = {}) { + const event = { + surface: "child_process", api, executable: typeof executable === "string" ? basename(executable) : undefined, + argvLabels: sanitizedArgvLabels(argv), outcome: "REJECTED", detail, + ...(activeCommandCheckId ? { checkId: activeCommandCheckId } : {}), + }; + if (activePolicyRejectionSink) activePolicyRejectionSink.push(event); + else if (commandEventSink) commandEventSink.push(event); + return event; +} +function policyError(message, details) { recordPolicyRejection(details); throw new Error(message); } +function validateThtInvocation(argv, { thtPath, runRoot, cwd } = {}) { + const configPath = Array.isArray(argv) ? argv[3] : undefined; + let configEntry; + try { configEntry = typeof configPath === "string" ? lstatSync(configPath) : undefined; } catch { configEntry = undefined; } + if (!configEntry || !exactArray(argv, ["config", "check", "-c", configPath]) || !ownedGitPath(configPath, runRoot) + || !/\.ya?ml$/.test(configPath) || !configEntry.isFile() || configEntry.isSymbolicLink() + || realpathSync(configPath) !== configPath || cwd !== dirname(dirname(dirname(thtPath)))) { + throw new Error("THT command is prohibited"); + } + return "config-check"; +} + +export function installProductionSurfaceGuard({ + gitPath, pythonPath, thtPath, thtIdentity, runRoot, environment, + originalFetch = globalThis.fetch, failPatchAt, +}) { + if (productionSurfaceOwner) throw new Error("production surface guard is already active"); + for (const value of [gitPath, pythonPath, thtPath, runRoot]) if (!isAbsolute(value)) throw new Error("production guard paths must be absolute"); + if (typeof originalFetch !== "function") throw new Error("global fetch is unavailable"); + if (!thtIdentity || thtIdentity.sourceStatus !== "git-index-byte-identical") throw new Error("trusted THT identity is absent"); + const gitHooksPath = join(runRoot, "installation", "runtime", "acceptance-git-hooks"); + mkdirSync(gitHooksPath, { recursive: true, mode: 0o700 }); + assertEmptyHooksDirectory(gitHooksPath); + if (failPatchAt !== undefined && (!Number.isInteger(failPatchAt) || failPatchAt < 1 || failPatchAt > 12)) throw new Error("invalid production patch failure probe"); + const token = Symbol("p1-production-surface"); + const events = []; + const originals = { + execFile: mutableChildProcess.execFile, spawn: mutableChildProcess.spawn, + exec: mutableChildProcess.exec, execSync: mutableChildProcess.execSync, execFileSync: mutableChildProcess.execFileSync, + spawnSync: mutableChildProcess.spawnSync, fork: mutableChildProcess.fork, + createSocket: mutableDgram.createSocket, Worker: mutableWorkerThreads.Worker, + dns: new Map(), dnsPromises: new Map(), dlopen: process.dlopen, + }; + for (const [name, value] of Object.entries(originals)) { + if (!["dns", "dnsPromises"].includes(name) && typeof value !== "function") throw new Error("production patch prerequisite is unavailable"); + } + const validateOptions = (options, allowed, api) => { + if (!options || typeof options !== "object" || Array.isArray(options)) throw new Error(`${api} options are invalid`); + for (const key of Object.keys(options)) if (!allowed.has(key)) throw new Error(`${api} option is prohibited`); + if ("timeout" in options && (!Number.isSafeInteger(options.timeout) || options.timeout < 1 || options.timeout > 300_000)) throw new Error("command bounds are invalid"); + if ("maxBuffer" in options && (!Number.isSafeInteger(options.maxBuffer) || options.maxBuffer < 1 || options.maxBuffer > MAX_OUTPUT)) throw new Error("command bounds are invalid"); + if ("shell" in options && options.shell !== false) throw new Error(`${api} shell is prohibited`); + }; + const resolveChild = (executable, argv, options, api) => { + const canonical = executable === "git" ? gitPath : executable === "python3" ? pythonPath : executable; + if (canonical === gitPath) { + validateGitInvocation(argv, { runRoot }); + if (options.cwd !== undefined && !ownedGitPath(options.cwd, runRoot)) throw new Error("Git working directory is prohibited"); + assertSafeGitRepositoryState(argv, options.cwd, runRoot, gitHooksPath); + const logical = argv[0] === "-c" ? argv.slice(2) : argv; + return { executable: gitPath, kind: "git", argv: hardenedGitArgv(logical, gitHooksPath) }; + } + if (canonical === thtPath) { + validateThtInvocation(argv, { thtPath, runRoot, cwd: options.cwd }); + revalidateThtIdentity(thtIdentity); + return { executable: thtPath, kind: "tht", argv }; + } + if (canonical === pythonPath) { + if (api !== "spawn" || argv.length !== 3 || argv[0] !== "-c" || argv[1] !== PYTHON_LOCK_HOLDER_PROGRAM + || !isAbsolute(argv[2]) || relative(runRoot, argv[2]).startsWith("..") || basename(argv[2]) !== "repository.lock") { + throw new Error("child command is prohibited"); + } + return { executable: pythonPath, kind: "python-lock-holder", argv }; + } + throw new Error("child command is prohibited"); + }; + const rejectChild = (api, args, message = "child command is prohibited") => { + safeChildEvent(events, { api, executable: typeof args[0] === "string" ? args[0] : undefined, argv: Array.isArray(args[1]) ? args[1] : [], outcome: "REJECTED" }); + throw new Error(message); + }; + const guardedExecFile = function guardedExecFile(executable, argv, options, callback) { + if (!Array.isArray(argv)) return rejectChild("execFile", [executable]); + if (typeof options === "function") { callback = options; options = {}; } + options ??= {}; + let resolved; + try { + validateOptions(options, new Set(["cwd", "env", "timeout", "maxBuffer", "encoding", "shell"]), "execFile"); + resolved = resolveChild(executable, argv, options, "execFile"); + boundedChildEnvironment(options.env, environment); + } catch (error) { safeChildEvent(events, { api: "execFile", executable, argv, outcome: "REJECTED" }); throw error; } + const bounded = { ...options, env: options.env ?? environment, timeout: options.timeout ?? 30_000, maxBuffer: options.maxBuffer ?? MAX_OUTPUT, shell: false }; + safeChildEvent(events, { api: "execFile", executable: resolved.executable, argv, outcome: "STARTED", detail: resolved.kind, + bounds: { timeoutMs: bounded.timeout, maxOutputBytes: bounded.maxBuffer, environment: "owned" } }); + return originals.execFile(resolved.executable, resolved.argv, bounded, (error, stdout, stderr) => { + safeChildEvent(events, { api: "execFile", executable: resolved.executable, argv, outcome: error ? "FAIL" : "PASS", detail: resolved.kind }); + callback?.(error, stdout, stderr); + }); + }; + Object.defineProperty(guardedExecFile, promisify.custom, { value: (executable, argv, options) => new Promise((resolvePromise, reject) => { + guardedExecFile(executable, argv, options, (error, stdout, stderr) => error ? reject(Object.assign(error, { stdout, stderr })) : resolvePromise({ stdout, stderr })); + }) }); + const guardedSpawn = function guardedSpawn(executable, argv, options = {}) { + if (!Array.isArray(argv)) return rejectChild("spawn", [executable]); + let resolved; + try { + validateOptions(options, new Set(["cwd", "env", "stdio", "shell"]), "spawn"); + if ("stdio" in options && JSON.stringify(options.stdio) !== JSON.stringify(["pipe", "pipe", "pipe"])) throw new Error("spawn stdio is prohibited"); + resolved = resolveChild(executable, argv, options, "spawn"); + boundedChildEnvironment(options.env, environment); + } catch (error) { safeChildEvent(events, { api: "spawn", executable, argv, outcome: "REJECTED" }); throw error; } + const bounded = { ...options, env: options.env ?? environment, shell: false }; + safeChildEvent(events, { api: "spawn", executable: resolved.executable, argv, outcome: "STARTED", detail: resolved.kind, + bounds: { timeoutMs: 300_000, maxOutputBytes: MAX_OUTPUT, environment: "owned" } }); + const child = originals.spawn(resolved.executable, resolved.argv, bounded); + let bytes = 0; + const count = (chunk) => { bytes += chunk.length; if (bytes > MAX_OUTPUT) child.kill("SIGKILL"); }; + child.stdout?.on("data", count); child.stderr?.on("data", count); + const timer = setTimeout(() => child.kill("SIGKILL"), 300_000); timer.unref(); + child.once("exit", (code) => { clearTimeout(timer); safeChildEvent(events, { api: "spawn", executable: resolved.executable, argv, outcome: code === 0 ? "PASS" : "FAIL", detail: resolved.kind }); }); + child.once("error", () => { clearTimeout(timer); }); + return child; + }; + const childWrappers = new Map(); + for (const api of ["exec", "execSync", "execFileSync", "spawnSync", "fork"]) childWrappers.set(api, (...args) => rejectChild(api, args)); + const guardedCreateSocket = (..._args) => { safeChildEvent(events, { surface: "dgram", api: "createSocket", outcome: "REJECTED" }); throw new Error("prohibited production surface: dgram"); }; + class ProhibitedWorker { constructor() { safeChildEvent(events, { surface: "worker_threads", api: "Worker", outcome: "REJECTED" }); throw new Error("prohibited production surface: worker_threads"); } } + const dnsWrappers = new Map(); const dnsPromiseWrappers = new Map(); + for (const name of ["lookup", "resolve", "resolve4", "resolve6", "resolveAny", "resolveCaa", "resolveCname", "resolveMx", "resolveNaptr", "resolveNs", "resolvePtr", "resolveSoa", "resolveSrv", "resolveTxt", "reverse", "Resolver"]) { + if (typeof mutableDns[name] !== "function") continue; + const original = mutableDns[name]; originals.dns.set(name, original); + dnsWrappers.set(name, (...args) => { + if (name === "lookup" && ["127.0.0.1", "::1"].includes(args[0])) { + safeChildEvent(events, { surface: "dns", api: name, outcome: "PASS", detail: "owned-loopback-literal" }); return original(...args); + } + safeChildEvent(events, { surface: "dns", api: name, outcome: "REJECTED" }); throw new Error("prohibited production surface: dns"); + }); + } + for (const [name, value] of Object.entries(mutableDns.promises ?? {})) if (typeof value === "function") { + originals.dnsPromises.set(name, value); + dnsPromiseWrappers.set(name, async (...args) => { + if (name === "lookup" && ["127.0.0.1", "::1"].includes(args[0])) { + safeChildEvent(events, { surface: "dns", api: `promises.${name}`, outcome: "PASS", detail: "owned-loopback-literal" }); return await value(...args); + } + safeChildEvent(events, { surface: "dns", api: `promises.${name}`, outcome: "REJECTED" }); throw new Error("prohibited production surface: dns"); + }); + } + const guardedDlopen = (..._args) => { safeChildEvent(events, { surface: "native_addon", api: "dlopen", outcome: "REJECTED" }); throw new Error("prohibited production surface: native addon"); }; + const changes = []; let network; let restored = false; let patchStep = 0; + const assign = (target, key, value) => { const original = target[key]; target[key] = value; changes.push({ target, key, value, original }); }; + const checkpoint = () => { patchStep += 1; if (failPatchAt === patchStep) throw new Error("injected production patch failure"); }; + const rollback = () => { + const errors = []; + if (network) { try { network.restore(); } catch (error) { errors.push(error); } network = undefined; } + for (const { target, key, original } of [...changes].reverse()) { try { target[key] = original; } catch (error) { errors.push(error); } } + try { syncBuiltinESMExports(); } catch (error) { errors.push(error); } + if (productionSurfaceOwner === token) productionSurfaceOwner = undefined; + if (activePolicyRejectionSink === events) activePolicyRejectionSink = undefined; + if (activeExecutablePolicy?.owner === token) activeExecutablePolicy = undefined; + return errors; + }; + try { + productionSurfaceOwner = token; checkpoint(); + assign(mutableChildProcess, "execFile", guardedExecFile); checkpoint(); + assign(mutableChildProcess, "spawn", guardedSpawn); checkpoint(); + for (const [api, wrapper] of childWrappers) assign(mutableChildProcess, api, wrapper); checkpoint(); + assign(mutableDgram, "createSocket", guardedCreateSocket); checkpoint(); + assign(mutableWorkerThreads, "Worker", ProhibitedWorker); checkpoint(); + for (const [name, wrapper] of dnsWrappers) assign(mutableDns, name, wrapper); checkpoint(); + for (const [name, wrapper] of dnsPromiseWrappers) assign(mutableDns.promises, name, wrapper); checkpoint(); + assign(process, "dlopen", guardedDlopen); checkpoint(); + syncBuiltinESMExports(); checkpoint(); + network = installNetworkGuard(originalFetch); checkpoint(); + activePolicyRejectionSink = events; + activeExecutablePolicy = { gitPath, pythonPath, thtPath, thtIdentity, runRoot, owner: token }; checkpoint(); + } catch (error) { + const rollbackErrors = rollback(); + if (rollbackErrors.length) throw new Error("production surface guard installation rollback failed", { cause: error }); + throw error; + } + return { + events, externalAttempts: network.externalAttempts, + gitPolicy: { hooksPath: gitHooksPath, fixedConfig: gitSafeConfig(gitHooksPath) }, + addOwnedOrigin: network.addOwnedOrigin, hasOwnedOrigin: network.hasOwnedOrigin, + restore() { + if (restored) throw new Error("production surface guard restored twice"); + restored = true; + let tampered = productionSurfaceOwner !== token || activePolicyRejectionSink !== events || activeExecutablePolicy?.owner !== token; + for (const { target, key, value } of changes) if (target[key] !== value) tampered = true; + const errors = rollback(); + if (tampered || errors.length) throw new Error("production surface guard ownership restoration failed"); + }, + }; +} + +export async function runCommand(options) { + const details = () => ({ executable: options && typeof options === "object" ? options.executable : undefined, + argv: options && typeof options === "object" ? options.argv : undefined, api: "runCommand", detail: "policy" }); + if (!options || typeof options !== "object" || Array.isArray(options)) policyError("command requires an options object", details()); + const allowed = new Set(["executable", "argv", "cwd", "env", "timeoutMs", "stdin", "maxOutputBytes"]); + for (const key of Object.keys(options)) if (!allowed.has(key)) policyError(`unsupported command option ${key}`, details()); + const { executable, argv, cwd, env, timeoutMs = 30_000, stdin, maxOutputBytes = MAX_OUTPUT } = options; + if (typeof executable !== "string" || !isAbsolute(executable) || /[;&|`$><\n\r]/.test(executable)) policyError("command executable is invalid", details()); + let canonical; + try { canonical = realpathSync(executable); } catch { policyError("command executable is not allowlisted", details()); } + const allowedGit = activeExecutablePolicy?.gitPath ?? resolveTrustedSystemExecutableSync("git"); + const allowedTht = activeExecutablePolicy?.thtPath; + if (canonical !== allowedGit && canonical !== allowedTht) policyError("command executable is not allowlisted", details()); + if (!Array.isArray(argv) || argv.some((value) => typeof value !== "string")) policyError("command argv must be a string array", details()); + const guardedByProduction = productionSurfaceOwner !== undefined; + let rawGitHooksPath; let rawGitArgv; + try { + if (canonical === allowedGit) { + validateGitInvocation(argv, { runRoot: activeExecutablePolicy?.runRoot }); + if (!guardedByProduction) { + rawGitHooksPath = ownedFallbackGitHooksPath(); + rawGitArgv = argv[0] === "-c" ? argv.slice(2) : argv; + assertSafeRawGitRepositoryState(rawGitArgv, cwd, rawGitHooksPath); + } + } + if (canonical === allowedTht) validateThtInvocation(argv, { thtPath: allowedTht, runRoot: activeExecutablePolicy.runRoot, cwd }); + } catch (error) { policyError(error.message, details()); } + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 300_000 || !Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1 || maxOutputBytes > MAX_OUTPUT) { + policyError("command bounds are invalid", details()); + } + return await new Promise((resolvePromise, reject) => { + const childArgv = canonical === allowedGit && !guardedByProduction ? hardenedGitArgv(rawGitArgv, rawGitHooksPath) : argv; + const childEnv = canonical === allowedGit && !guardedByProduction ? { ...(env ?? {}), ...baseSafeGitEnvironment(canonical) } : env; + const child = mutableChildProcess.execFile(canonical, childArgv, { cwd, env: childEnv, timeout: timeoutMs, maxBuffer: maxOutputBytes, encoding: "utf8", shell: false }, (error, stdout, stderr) => { + const code = error && typeof error.code === "number" ? error.code : error ? 1 : 0; + const result = { code, stdout: stdout ?? "", stderr: stderr ?? "" }; + if (commandEventSink) commandEventSink.push({ + executable: basename(canonical), argvLabels: sanitizedArgvLabels(argv), outcome: error ? "FAIL" : "PASS", + ...(activeCommandCheckId ? { checkId: activeCommandCheckId } : {}), + }); + if (error) Object.assign(error, { result }); + error ? reject(error) : resolvePromise(result); + }); + if (stdin !== undefined) { child.stdin.end(stdin); } + }); +} +async function git(argv, options = {}) { return await runCommand({ executable: activeExecutablePolicy?.gitPath ?? resolveTrustedSystemExecutableSync("git"), argv, ...options }); } +async function tht(executable, argv, options = {}) { return await runCommand({ executable, argv, ...options }); } +function safeArtifactPath(path) { + if (typeof path !== "string" || !SAFE_RELATIVE.test(path) || path.startsWith(".") || path.includes("//")) throw new Error("unsafe artifact path"); + return path; +} +async function fileArtifact(runRoot, path) { + safeArtifactPath(path); + return { path, sha256: sha256(await readFile(join(runRoot, path))) }; +} +function forbiddenKey(value) { + if (!value || typeof value !== "object") return false; + if (Array.isArray(value)) return value.some(forbiddenKey); + for (const [key, nested] of Object.entries(value)) { + if (/^(attempt|attempts|retry|retries)$/i.test(key) || forbiddenKey(nested)) return true; + } + return false; +} +export function deriveOverall(checks) { return checks.length > 0 && checks.every(({ status }) => status === "PASS") ? "PASS" : "FAIL"; } +function hasExactCheckIds(checks) { + return checks.length === CHECK_IDS.length && checks.every(({ id }, index) => id === CHECK_IDS[index]); +} +export function validateReport(report) { + if (!report || report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "") + || !ISO_UTC.test(report.finishedAt ?? "") || typeof report.command !== "string" || forbiddenKey(report) + || !Array.isArray(report.checks) || !hasExactCheckIds(report.checks)) throw new Error("report is invalid"); + const ids = new Set(); + const artifactPaths = new Set(); + for (const check of report.checks) { + if (!check || !/^[a-z0-9_]+$/.test(check.id ?? "") || ids.has(check.id) || !["PASS", "FAIL"].includes(check.status) + || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "") + || !Array.isArray(check.commands) || check.commands.some((name) => !COMMAND.test(name)) + || !Array.isArray(check.artifacts) || check.artifacts.some(({ path, sha256 }) => { + try { safeArtifactPath(path); } catch { return true; } + return !HEX64.test(sha256 ?? ""); + })) throw new Error("report check is invalid"); + ids.add(check.id); + for (const artifact of check.artifacts) { + if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated"); + artifactPaths.add(artifact.path); + } + } + if (report.overall !== deriveOverall(report.checks)) throw new Error("report overall is not derived"); + return report; +} +function renderReportMarkdown(report) { + validateReport(report); + const rows = report.checks.map((check) => `| ${check.id} | ${check.status} |`).join("\n"); + return `# P1 automated integration\n\nRun: \`${report.runId}\`\n\n| Check | Status |\n|---|---|\n${rows}\n\nautomated integration: ${report.overall}\nmanual acceptance: PENDING\n`; +} +function containsAny(bytes, forbiddenValues) { + return forbiddenValues.some((value) => value && bytes.includes(Buffer.from(value))); +} +async function walkFiles(root, current = root, out = []) { + for (const entry of await readdir(current, { withFileTypes: true })) { + const path = join(current, entry.name); + const rel = relative(root, path).split(sep).join("/"); + if (entry.isSymbolicLink()) throw new Error(`secret scan failed closed: symlink outside fixture-secrets: ${rel}`); + if (entry.isDirectory()) { + if (rel === "fixture-secrets") continue; + await walkFiles(root, path, out); + } else if (entry.isFile()) out.push({ path, rel }); + } + return out; +} +async function gitObjectScan(runRoot, forbiddenValues, expectedGitRepositories) { + const findings = []; const repositories = []; + for (const rel of expectedGitRepositories) { + const directory = join(runRoot, rel); + if (!existsSync(directory)) throw new Error(`Git secret scan failed closed: missing expected Git repository: ${rel}`); + const args = basename(directory) === "remote.git" ? ["--git-dir", directory] : ["-C", directory]; + let objects; + try { + objects = (await git([...args, "rev-list", "--objects", "--all"])).stdout.trim().split("\n").filter(Boolean); + } catch { throw new Error(`Git secret scan failed closed during enumeration: ${basename(directory)}`); } + let blobCount = 0; + for (const line of objects) { + const oid = line.split(" ", 1)[0]; + let type; let bytes; + try { + type = (await git([...args, "cat-file", "-t", oid])).stdout.trim(); + if (!/^(blob|tree|commit|tag)$/.test(type)) throw new Error("invalid object type"); + if (type !== "blob") continue; + blobCount += 1; + bytes = Buffer.from((await git([...args, "cat-file", "blob", oid], { maxOutputBytes: 16 * 1024 * 1024 })).stdout); + } catch { throw new Error(`Git secret scan failed closed during object inspection: ${basename(directory)}:${oid}`); } + if (containsAny(bytes, forbiddenValues)) findings.push({ path: `git-object:${rel}:${oid}` }); + } + repositories.push({ path: rel, objectCount: objects.length, blobCount }); + } + return { findings, repositories }; +} +export async function scanSecretsDetailed({ runRoot, forbiddenValues, virtualFiles = [], expectedGitRepositories = ["remote.git", "author"] }) { + const values = forbiddenValues.filter((value) => typeof value === "string" && value.length >= 8); + const findings = []; + for (const file of await walkFiles(runRoot)) if (containsAny(await readFile(file.path), values)) findings.push({ path: file.rel }); + for (const file of virtualFiles) if (containsAny(Buffer.from(file.bytes), values)) findings.push({ path: file.path }); + const gitScan = await gitObjectScan(runRoot, values, expectedGitRepositories); + findings.push(...gitScan.findings); + return { findings, repositories: gitScan.repositories }; +} +export async function scanSecrets(options) { return (await scanSecretsDetailed(options)).findings; } + +export function negativeRequestEvidence(caseLabel, expectedInputField) { + if (!/^[a-z0-9-]+$/.test(caseLabel) || !/^[a-z_]+(?:\.[a-z_]+)*$/.test(expectedInputField)) throw new Error("unsafe negative-case evidence"); + return { case: caseLabel, expectedInputField }; +} + +function loopbackOrigin(value) { + const url = new URL(value); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1" || !url.port) throw new Error("owned API must be loopback HTTP"); + return url.origin; +} + +export function installExternalFetchGuard(ownedBaseUrl, fetchImplementation = globalThis.fetch) { + const ownedOrigin = loopbackOrigin(ownedBaseUrl); + const externalAttempts = []; + const guardedFetch = async (input, init) => { + const candidate = new URL(typeof input === "string" || input instanceof URL ? input : input.url); + if (candidate.origin !== ownedOrigin) { + externalAttempts.push({ transport: "fetch", protocol: candidate.protocol, loopback: candidate.hostname === "127.0.0.1" }); + throw new Error("external fetch prohibited"); + } + return await fetchImplementation(input, init); + }; + return { fetch: guardedFetch, externalAttempts }; +} + +function socketDestination(args) { + const first = Array.isArray(args[0]) ? args[0][0] : args[0]; + if (typeof first === "object" && first !== null) { + if (first.path !== undefined) return { path: String(first.path) }; + return { host: String(first.host ?? first.hostname ?? "localhost"), port: Number(first.port) }; + } + if (typeof first === "number") return { host: typeof args[1] === "string" ? args[1] : "localhost", port: first }; + return { path: String(first) }; +} + +export function installNetworkGuard(fetchImplementation = globalThis.fetch) { + if (typeof fetchImplementation !== "function") throw new Error("global fetch is unavailable"); + const ownedOrigins = new Set(); + const externalAttempts = []; + const originalFetch = globalThis.fetch; + const originalConnect = Socket.prototype.connect; + const isOwned = (host, port) => { + if (!Number.isInteger(port) || port < 1 || port > 65535) return false; + const normalized = host === "localhost" || host === "::1" ? "127.0.0.1" : host; + return isIP(normalized) !== 0 && normalized === "127.0.0.1" && ownedOrigins.has(`http://127.0.0.1:${port}`); + }; + const guardedFetch = async (input, init) => { + const candidate = new URL(typeof input === "string" || input instanceof URL ? input : input.url); + if (!ownedOrigins.has(candidate.origin)) { + externalAttempts.push({ transport: "fetch", protocol: candidate.protocol, loopback: candidate.hostname === "127.0.0.1" }); + throw new Error("external network connection prohibited"); + } + return await fetchImplementation(input, init); + }; + const guardedSocketConnect = function guardedSocketConnect(...args) { + const destination = socketDestination(args); + if (!("host" in destination) || !isOwned(destination.host, destination.port)) { + externalAttempts.push({ transport: "socket", loopback: destination.host === "127.0.0.1" }); + throw new Error("external network connection prohibited"); + } + return originalConnect.apply(this, args); + }; + let fetchChanged = false; let socketChanged = false; + try { + globalThis.fetch = guardedFetch; fetchChanged = true; + Socket.prototype.connect = guardedSocketConnect; socketChanged = true; + } catch (error) { + if (socketChanged) Socket.prototype.connect = originalConnect; + if (fetchChanged) globalThis.fetch = originalFetch; + throw error; + } + let restored = false; + return { + externalAttempts, + addOwnedOrigin(value) { ownedOrigins.add(loopbackOrigin(value)); }, + hasOwnedOrigin(value) { return ownedOrigins.has(loopbackOrigin(value)); }, + restore() { + if (restored) throw new Error("network guard restored twice"); + restored = true; + const tampered = globalThis.fetch !== guardedFetch || Socket.prototype.connect !== guardedSocketConnect; + globalThis.fetch = originalFetch; + Socket.prototype.connect = originalConnect; + if (tampered) throw new Error("network guard ownership changed"); + }, + }; +} + +function sanitizeForEvidence(value, forbiddenValues = []) { + if (typeof value === "string") { + let safe = value; + for (const forbidden of forbiddenValues) if (forbidden) safe = safe.split(forbidden).join("[REDACTED]"); + return safe.length > 16_384 ? `${safe.slice(0, 16_384)}[TRUNCATED]` : safe; + } + if (Array.isArray(value)) return value.map((item) => sanitizeForEvidence(item, forbiddenValues)); + if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, nested]) => [key, sanitizeForEvidence(nested, forbiddenValues)])); + return value; +} +async function evidence(run, path, value, forbiddenValues = []) { + const safe = sanitizeForEvidence(value, forbiddenValues); + await atomicWrite(join(run.root, path), `${JSON.stringify(safe, null, 2)}\n`); + return await fileArtifact(run.root, path); +} +export async function executeChecks({ checks, failAt, recorder } = {}) { + if (!Array.isArray(checks) || !hasExactCheckIds(checks)) throw new Error("scenarios must match the exact ordered check set"); + if (failAt !== undefined && !CHECK_IDS.includes(failAt)) throw new Error("failure hook must name an exact check"); + const results = []; + let stopped = false; + for (const scenario of checks) { + const startedAt = nowIso(); + let result; + if (stopped) { + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: "Not executed after earlier failure." }; + } else { + try { + const output = await scenario.run(); + if (scenario.id === failAt) { + const injected = new Error("injected acceptance failure"); + injected.acceptancePartial = { commands: output.commands ?? [], artifacts: output.artifacts ?? [] }; + throw injected; + } + result = { id: scenario.id, status: "PASS", startedAt, finishedAt: nowIso(), commands: output.commands ?? [], artifacts: output.artifacts ?? [] }; + } catch (error) { + const partial = error?.acceptancePartial ?? {}; + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: partial.commands ?? [], artifacts: partial.artifacts ?? [], error: "Acceptance scenario failed safely." }; + stopped = true; + } + } + results.push(result); + if (recorder) await recorder(result); + } + return results; +} + +function baseWorkspace(id, evidenceSource) { + return { + workspace: { schema_version: 3, id, name: `P1 ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } }, + }; +} +function descriptors() { + return [ + baseWorkspace("p1-filesystem", { type: "filesystem", uri: "workspace-content/p1-filesystem/evidence", patterns: ["**/*.md"], max_bytes: 10485760 }), + baseWorkspace("p1-http", { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", connect_timeout_ms: 1250, read_timeout_ms: 30001, max_bytes: 12345, max_redirects: 2, allow_private_hosts: false, max_cache_bytes: 67890 }), + baseWorkspace("p1-s3", { type: "s3", uri: "s3://p1-evidence/published/", endpoint_url: "https://s3.example.test/", region: "eu-west-1", credentials: "static_files", trusted_endpoint: true, allow_private_endpoint: false, allow_insecure_endpoint: false, max_bytes: 12345, max_objects: 33, max_pages: 4, page_size: 5 }), + ]; +} +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } +async function createTopology(run) { + for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); +} +async function setupSecrets(ctx) { + const secretDir = join(ctx.run.root, "fixture-secrets"); + const values = { + dwh: `DWH-${randomBytes(16).toString("hex")}`, + signed: `SIGNED-${randomBytes(16).toString("hex")}`, + access: `ACCESS-${randomBytes(16).toString("hex")}`, + secret: `SECRET-${randomBytes(16).toString("hex")}`, + session: `SESSION-${randomBytes(16).toString("hex")}`, + rejected: `REJECTED-${randomBytes(16).toString("hex")}`, + }; + ctx.forbiddenValues = Object.values(values); + ctx.secretValues = values; + const paths = { + dwh: join(secretDir, "dwh-password"), signed: join(secretDir, "evidence-signed-urls.json"), + access: join(secretDir, "evidence-access"), secret: join(secretDir, "evidence-secret"), session: join(secretDir, "evidence-session"), + }; + await atomicWrite(paths.dwh, scalarSecretBytes(values.dwh)); + await atomicWrite(paths.signed, JSON.stringify([`https://evidence.example.test/guide.md?token=${values.signed}`])); + await atomicWrite(paths.access, scalarSecretBytes(values.access)); + await atomicWrite(paths.secret, scalarSecretBytes(values.secret)); + await atomicWrite(paths.session, scalarSecretBytes(values.session)); + const env = {}; + for (const workspace of ctx.descriptors) { + const ns = namespace(workspace.workspace.id); const prefix = `THT_WS_${ns}`; + Object.assign(env, { + [`${prefix}_DWH_TRANSPORT`]: "postgres_direct", [`${prefix}_DWH_HOST`]: "dwh.invalid", + [`${prefix}_DWH_PORT`]: "5432", [`${prefix}_DWH_USER`]: "reader", [`${prefix}_DWH_PASSWORD_FILE`]: paths.dwh, + }); + } + Object.assign(env, { + THT_WS_P1_HTTP_EVIDENCE_SIGNED_URLS_FILE: paths.signed, + THT_WS_P1_S3_EVIDENCE_ACCESS_KEY_FILE: paths.access, + THT_WS_P1_S3_EVIDENCE_SECRET_KEY_FILE: paths.secret, + THT_WS_P1_S3_EVIDENCE_SESSION_TOKEN_FILE: paths.session, + }); + Object.assign(ctx.env, env); + env.THT_WORKSPACE_SECRET_ROOTS = secretDir; + await atomicWrite(join(ctx.run.root, "installation", "bindings.env"), `${Object.entries(env).map(([key, value]) => `${key}=${value}`).join("\n")}\n`); + await atomicWrite(join(ctx.run.root, "installation", "runtime", "base.yaml"), "{}\n"); +} +async function initializeGit(ctx) { + await git(["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], { cwd: ctx.run.root }); + await git(["clone", join(ctx.run.root, "remote.git"), join(ctx.run.root, "author")], { cwd: ctx.run.root }); + await git(["config", "user.name", "P1 Fixture Curator"], { cwd: join(ctx.run.root, "author") }); + await git(["config", "user.email", "p1-curator@example.invalid"], { cwd: join(ctx.run.root, "author") }); + const evidenceRoot = join(ctx.run.root, "author", "workspace-content", "p1-filesystem", "evidence"); + await mkdir(join(evidenceRoot, "domain"), { recursive: true }); + await writeFile(join(evidenceRoot, "guide.md"), "# P1 curated Evidence\n"); + await writeFile(join(evidenceRoot, "domain", "table.md"), "# Curated table\n"); + await git(["add", "workspace-content"], { cwd: join(ctx.run.root, "author") }); + await git(["commit", "-m", "Bootstrap curated P1 content"], { cwd: join(ctx.run.root, "author") }); + await git(["push", "origin", "main"], { cwd: join(ctx.run.root, "author") }); + ctx.bootstrapCommit = (await git(["rev-parse", "HEAD"], { cwd: join(ctx.run.root, "author") })).stdout.trim(); +} +async function loadProductionBackend() { + const [{ loadConfig }, { buildApp }, { WorkspaceRegistry }, { ThtRunner }] = await Promise.all([ + import("../dist/config.js"), import("../dist/app.js"), import("../dist/workspaces/registry.js"), import("../dist/tht/tht-runner.js"), + ]); + return { loadConfig, buildApp, WorkspaceRegistry, ThtRunner }; +} +async function startProductionBackend(ctx, { name, env, runtimeConfigPath }) { + const { loadConfig, buildApp, WorkspaceRegistry, ThtRunner } = await loadProductionBackend(); + const config = loadConfig(env); + const registry = new WorkspaceRegistry(config.workspaceRegistry); + const thtRunner = new ThtRunner({ + thtBin: config.thtBin, harnessDir: config.harnessDir, configPath: runtimeConfigPath, + dataRoot: config.dataRoot, runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"), + secretRoots: config.workspaceRegistry.secretRoots, secretsFile: config.secretsFile, secretFiles: config.secretFiles, + semanticRuntime: { internalQdrantUrl: config.internalQdrantUrl, internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, internalEmbeddingDimensions: config.internalEmbeddingDimensions }, + }); + const app = buildApp(config, { thtRunner, workspaceRegistry: registry }); + let address; + try { + address = await app.listen({ host: "127.0.0.1", port: 0 }); + } catch (error) { + try { await app.close(); } catch { throw new Error("production listener start and close both failed"); } + throw error; + } + const url = new URL(address); + const baseUrl = `http://127.0.0.1:${url.port}`; + ctx.networkGuard.addOwnedOrigin(baseUrl); + const service = { name, app, baseUrl, registry, thtRunner, config }; + ctx.services.push(service); + await writeOwnership(ctx.run, { + name, kind: "fastify", host: "127.0.0.1", requestedPort: 0, + actualPort: Number(url.port), pid: process.pid, state: "listening", + }); + return service; +} + +async function startBackend(ctx) { + const service = await startProductionBackend(ctx, { + name: "primary", env: ctx.env, + runtimeConfigPath: join(ctx.run.root, "installation", "runtime", "base.yaml"), + }); + ctx.registryConfig = service.config.workspaceRegistry; + ctx.registry = service.registry; + ctx.thtRunner = service.thtRunner; + ctx.app = service.app; + ctx.baseUrl = service.baseUrl; +} + +export function exportArchiveEvidencePath(requestId) { + if (!/^export-[a-z0-9-]+$/.test(requestId)) throw new Error("invalid export request id"); + return `exports/raw/${requestId}.zip`; +} +function trackArtifact(ctx, artifact) { + if (ctx.activeArtifacts) { + if (ctx.activeArtifacts.some(({ path }) => path === artifact.path)) throw new Error("artifact path is duplicated within check"); + ctx.activeArtifacts.push(artifact); + } + return artifact; +} + +async function request(ctx, id, method, path, body, binary = false, requestEvidence, baseUrl = ctx.baseUrl) { + const requestSummary = requestEvidence === undefined + ? { method, path, ...(body === undefined ? {} : { body: sanitizeForEvidence(body, ctx.forbiddenValues) }) } + : { method, path, input: requestEvidence }; + trackArtifact(ctx, await evidence(ctx.run, `requests/${id}.json`, requestSummary, ctx.forbiddenValues)); + ctx.httpRequests.push({ method, path }); + const response = await globalThis.fetch(`${baseUrl}${path}`, { + method, headers: body === undefined ? {} : { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), signal: AbortSignal.timeout(15_000), + }); + if (binary) { + const bytes = Buffer.from(await response.arrayBuffer()); + await atomicWrite(join(ctx.run.root, exportArchiveEvidencePath(id)), bytes); + trackArtifact(ctx, await evidence(ctx.run, `responses/${id}.json`, { status: response.status, contentType: response.headers.get("content-type"), bytes: bytes.length })); + return { status: response.status, bytes }; + } + const text = await response.text(); let parsed; + try { parsed = text ? JSON.parse(text) : null; } catch { parsed = { invalidJson: true }; } + const safe = sanitizeForEvidence(parsed, ctx.forbiddenValues); + trackArtifact(ctx, await evidence(ctx.run, `responses/${id}.json`, { status: response.status, body: safe }, ctx.forbiddenValues)); + return { status: response.status, body: parsed }; +} +async function extractZip(ctx, id, bytes) { + const yauzl = (await import("yauzl")).default; + const output = join(ctx.run.root, "exports", "extracted", id); await mkdir(output, { recursive: true }); + const files = await new Promise((resolvePromise, reject) => { + yauzl.fromBuffer(bytes, { lazyEntries: true, strictFileNames: true, validateEntrySizes: true }, (error, zip) => { + if (error || !zip) return reject(error ?? new Error("zip open failed")); + const collected = new Map(); let total = 0; + zip.on("error", reject); zip.on("end", () => resolvePromise(collected)); + zip.on("entry", (entry) => { + const type = (entry.externalFileAttributes >>> 16) & 0o170000; + if (!ZIP_FILES.includes(entry.fileName) || entry.fileName.includes("\\") || entry.fileName.includes("..") || entry.fileName.startsWith("/") || entry.fileName.endsWith("/") || type === 0o120000 || collected.has(entry.fileName) || entry.uncompressedSize > 2_000_000) return reject(new Error("unsafe export entry")); + zip.openReadStream(entry, (streamError, stream) => { + if (streamError || !stream) return reject(streamError ?? new Error("zip stream failed")); + const chunks = []; stream.on("data", (chunk) => { total += chunk.length; if (total > 8_000_000) reject(new Error("export too large")); else chunks.push(chunk); }); + stream.on("end", () => { collected.set(entry.fileName, Buffer.concat(chunks)); zip.readEntry(); }); stream.on("error", reject); + }); + }); + zip.readEntry(); + }); + }); + assert(files.size === ZIP_FILES.length, "export file allowlist mismatch"); + const manifest = JSON.parse(files.get("manifest.json").toString("utf8")); + assert(manifest.schema_version === 1 && manifest.workspace_id === id, "export manifest identity mismatch"); + for (const name of ZIP_FILES.slice(1)) assert(sha256(files.get(name)) === manifest.files[name], `export hash mismatch ${name}`); + for (const [name, contents] of files) await atomicWrite(join(output, name), contents, 0o600); + return manifest; +} +function assert(condition, message) { if (!condition) throw new Error(message); } +function assertGenericWorkspaceInvalid(response, label) { + assert(response.status === 400 && response.body?.code === "workspace_invalid", `${label} was not rejected through HTTP`); + assert(Object.keys(response.body).sort().join(",") === "code,message", `${label} response envelope was not exact`); + assert(response.body.message === "Workspace request or bundle is invalid.", `${label} response message was not generic`); + assert(!/fatal:|stderr|git command|rev-parse|ls-tree/i.test(JSON.stringify(response.body)), `${label} exposed Git stderr`); +} +async function snapshotDigest(path) { + const files = await walkFiles(path); const result = {}; + for (const file of files) result[file.rel] = sha256(await readFile(file.path)); + return result; +} +const SAFE_AMBIENT_ENV = Object.freeze(["LANG", "LC_ALL", "TZ"]); +export function buildSafeEnvironment({ ambient = process.env, fixture = {} } = {}) { + const safe = {}; + for (const key of SAFE_AMBIENT_ENV) if (typeof ambient[key] === "string") safe[key] = ambient[key]; + for (const [key, value] of Object.entries(fixture)) { + if (typeof value !== "string") throw new Error(`fixture environment value must be a string: ${key}`); + safe[key] = value; + } + return safe; +} + +async function manifestFiles(root, paths) { + const files = []; + const visit = async (absolute, rel) => { + const entry = await lstat(absolute); + if (entry.isSymbolicLink()) throw new Error(`provenance path is a symlink: ${rel}`); + if (entry.isDirectory()) { + for (const child of (await readdir(absolute, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + await visit(join(absolute, child.name), rel ? `${rel}/${child.name}` : child.name); + } + } else if (entry.isFile()) { + const bytes = await readFile(absolute); + files.push({ path: rel, bytes: bytes.length, sha256: sha256(bytes) }); + } else throw new Error(`provenance path is not a regular file: ${rel}`); + }; + for (const path of paths) await visit(join(root, path), path); + files.sort((a, b) => a.path.localeCompare(b.path)); + return { files, manifestSha256: sha256(JSON.stringify(files)) }; +} + +export async function collectRepositoryProvenance({ repositoryRoot, gitPath = resolveTrustedSystemExecutableSync("git") }) { + const repo = canonicalRoot(repositoryRoot); + const gitEnv = baseSafeGitEnvironment(gitPath); + const readIdentity = async () => { + const [head, tree, status] = await Promise.all([ + runCommand({ executable: gitPath, argv: ["-C", repo, "rev-parse", "HEAD"], env: gitEnv }), + runCommand({ executable: gitPath, argv: ["-C", repo, "rev-parse", "HEAD^{tree}"], env: gitEnv }), + runCommand({ executable: gitPath, argv: ["-C", repo, "status", "--porcelain=v1", "--untracked-files=all"], env: gitEnv }), + ]); + return { head: head.stdout.trim(), tree: tree.stdout.trim(), status: status.stdout }; + }; + const before = await readIdentity(); + if (!HEX40.test(before.head) || !HEX40.test(before.tree) || before.status !== "") throw new Error("repository is not clean at exact HEAD"); + const backendRoot = join(repo, "backend"); + const backendSource = await manifestFiles(backendRoot, [ + "src", "scripts/p1-acceptance.mjs", "package.json", "package-lock.json", "tsconfig.json", + ]); + const backendDist = await manifestFiles(backendRoot, ["dist"]); + const after = await readIdentity(); + if (JSON.stringify(after) !== JSON.stringify(before)) throw new Error("repository provenance changed during binding"); + return { schemaVersion: 1, head: before.head, tree: before.tree, clean: true, backendSource, backendDist }; +} + +async function setupContext(run, repositoryRoot, env, ctx = {}) { + const provenance = await collectRepositoryProvenance({ repositoryRoot, gitPath: resolveTrustedSystemExecutableSync("git") }); + const executables = await resolveProductionExecutables({ repositoryRoot }); + const harnessDir = realpathSync(join(repositoryRoot, "harness")); + const gitTracePath = join(run.root, "logs", "production-git-trace.jsonl"); + const ownedHome = join(run.root, "installation", "runtime", "acceptance-home"); + const ownedTmp = join(run.root, "installation", "runtime", "tmp"); + await mkdir(ownedHome, { recursive: true, mode: 0o700 }); + await mkdir(ownedTmp, { recursive: true, mode: 0o700 }); + const executablePath = [...new Set([dirname(executables.gitPath), dirname(executables.pythonPath), dirname(executables.thtPath)])].join(":"); + const fixtureEnv = { + PATH: executablePath, HOME: ownedHome, TMPDIR: ownedTmp, + GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_SYSTEM: "/dev/null", GIT_CONFIG_GLOBAL: "/dev/null", GIT_ATTR_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "/bin/false", SSH_ASKPASS: "/bin/false", GIT_ALLOW_PROTOCOL: "file", GIT_PROTOCOL_FROM_USER: "0", + GIT_CONFIG_COUNT: "4", GIT_CONFIG_KEY_0: "commit.gpgSign", GIT_CONFIG_VALUE_0: "false", + GIT_CONFIG_KEY_1: "tag.gpgSign", GIT_CONFIG_VALUE_1: "false", + GIT_CONFIG_KEY_2: "credential.helper", GIT_CONFIG_VALUE_2: "", + GIT_CONFIG_KEY_3: "core.fsmonitor", GIT_CONFIG_VALUE_3: "false", GIT_PAGER: "/bin/cat", + HOST: "127.0.0.1", PORT: "0", AUTH_MODE: "none", THT_BIN: executables.thtPath, + THT_HARNESS_DIR: harnessDir, THT_DATA_ROOT: join(run.root, "installation", "data"), + SETTINGS_FILE: join(run.root, "installation", "data", "settings.json"), + MAINTENANCE_STATE_FILE: join(run.root, "installation", "data", "maintenance.json"), + THT_WORKSPACE_REGISTRY_ROOT: join(run.root, "installation", "registry"), + THT_WORKSPACE_GIT_REMOTE: join(run.root, "remote.git"), THT_WORKSPACE_GIT_BRANCH: "main", + THT_WORKSPACE_GIT_AUTHOR_NAME: "P1 API Publisher", THT_WORKSPACE_GIT_AUTHOR_EMAIL: "p1-api@example.invalid", + THT_WORKSPACE_INSTALLATION_ID: "p1-acceptance", THT_WORKSPACE_SECRET_ROOTS: join(run.root, "fixture-secrets"), + THT_HOME: join(run.root, "installation", "runtime", "tht-home"), PYTHONDONTWRITEBYTECODE: "1", PYTHONNOUSERSITE: "1", + GIT_TRACE2_EVENT: gitTracePath, + }; + Object.assign(ctx, { + run, repositoryRoot, descriptors: descriptors(), forbiddenValues: ctx.forbiddenValues ?? [], + env: buildSafeEnvironment({ ambient: env, fixture: fixtureEnv }), + httpRequests: [], services: [], gitTracePath, executables, provenance, + expectedGitRepositories: ["remote.git", "author"], + }); + await createTopology(run); + await setupSecrets(ctx); + return ctx; +} + +async function treeHash(path, excludedPrefixes = []) { + const digest = await snapshotDigest(path); + for (const key of Object.keys(digest)) if (excludedPrefixes.some((prefix) => key === prefix || key.startsWith(`${prefix}/`))) delete digest[key]; + return sha256(JSON.stringify(digest)); +} +async function checkoutSemanticState(path) { + const head = await git(["rev-parse", "HEAD"], { cwd: path }); + const branch = await git(["symbolic-ref", "--short", "HEAD"], { cwd: path }); + const statusResult = await git(["status", "--porcelain=v1"], { cwd: path }); + const indexTree = await git(["write-tree"], { cwd: path }); + const refs = await git(["show-ref"], { cwd: path }); + return { + head: head.stdout.trim(), branch: branch.stdout.trim(), status: statusResult.stdout, + indexTree: indexTree.stdout.trim(), refs: sha256(refs.stdout), + worktree: await treeHash(path, [".git"]), + }; +} +async function bareSemanticState(path, branch) { + const [head, tree, refs] = await Promise.all([ + git(["--git-dir", path, "rev-parse", `refs/heads/${branch}`]), + git(["--git-dir", path, "rev-parse", `refs/heads/${branch}^{tree}`]), + git(["--git-dir", path, "show-ref"]), + ]); + return { head: head.stdout.trim(), tree: tree.stdout.trim(), refs: sha256(refs.stdout) }; +} +async function primarySemanticState(ctx) { + return { + remote: await bareSemanticState(join(ctx.run.root, "remote.git"), "main"), + author: await checkoutSemanticState(join(ctx.run.root, "author")), + checkout: await checkoutSemanticState(join(ctx.run.root, "installation", "registry", "repo")), + active: await treeHash(join(ctx.run.root, "installation", "registry", "state")), + snapshots: await treeHash(join(ctx.run.root, "installation", "registry", "snapshots")), + data: await treeHash(join(ctx.run.root, "installation", "data")), + runtime: await treeHash(join(ctx.run.root, "installation", "runtime"), ["contextual"]), + }; +} +async function registryState(ctx) { + return await primarySemanticState(ctx); +} +async function contextualSemanticState(root) { + return { + remote: await bareSemanticState(join(root, "remote.git"), "invalid-context"), + author: await checkoutSemanticState(join(root, "author")), + checkout: await checkoutSemanticState(join(root, "registry", "repo")), + active: await treeHash(join(root, "registry", "state")), + snapshots: await treeHash(join(root, "registry", "snapshots")), + data: await treeHash(join(root, "data")), + runtime: await treeHash(join(root, "runtime")), + }; +} +async function invariantArtifact(ctx, path, value) { + return trackArtifact(ctx, await evidence(ctx.run, path, value, ctx.forbiddenValues)); +} + +function assertByteIdentical(left, right, label) { + assert(JSON.stringify(left) === JSON.stringify(right), `${label} state changed`); +} +async function currentSnapshotManifest(ctx, commit) { + const path = join(ctx.run.root, "installation", "registry", "snapshots", commit, "snapshot.json"); + const manifest = JSON.parse(await readFile(path, "utf8")); + assert(manifest.head === commit, "snapshot manifest head mismatch"); + return { path, manifest }; +} +function revisionFromManifest(manifest, id) { + const revision = manifest.revisions.find((candidate) => candidate.id === id); + assert(revision, `manifest revision absent ${id}`); + return revision; +} +function renderedRoot(parsed) { return parsed.evidence.sources[0].root; } +function assertRuntimeContract(ctx, id, parsed, revision) { + assert(parsed.runtime_identity.workspace_id === id, "runtime workspace identity mismatch"); + assert(parsed.runtime_identity.workspace_revision === revision.commit, "runtime revision mismatch"); + assert(parsed.runtime_identity.source_identity === `workspace://${id}`, "runtime source identity mismatch"); + assert(parsed.vector.max_chunk_chars === 4000 && parsed.vector.retain_published_generations === 3, "runtime policy mismatch"); + const source = parsed.evidence.sources[0]; + if (id === "p1-filesystem") { + const exactRoot = join(dirname(revision.snapshotPath), "workspace-content", id, "evidence"); + assert(source.type === "filesystem" && source.root === exactRoot, "filesystem root mismatch"); + assert(JSON.stringify(source.patterns) === JSON.stringify(["**/*.md"]) && source.max_bytes === 10485760, "filesystem source contract mismatch"); + assert(!existsSync(source.root), "filesystem Evidence root was materialized"); + } else if (id === "p1-http") { + assert(source.type === "http", "HTTP source type mismatch"); + assert(JSON.stringify(source.provenance_urls) === JSON.stringify(["https://evidence.example.test/guide.md"]), "HTTP provenance mismatch"); + assert(source.signed_urls_file === join(ctx.run.root, "fixture-secrets", "evidence-signed-urls.json"), "HTTP binding mismatch"); + assert(source.connect_timeout === 1.25 && source.read_timeout === 30.001 && source.max_bytes === 12345 + && source.max_redirects === 2 && source.allow_private_hosts === false && source.max_cache_bytes === 67890, "HTTP limits mismatch"); + } else if (id === "p1-s3") { + assert(source.type === "s3" && source.bucket === "p1-evidence" && source.prefix === "published/", "S3 identity mismatch"); + assert(source.endpoint_url === "https://s3.example.test/" && source.region === "eu-west-1", "S3 endpoint mismatch"); + assert(source.access_key_file === join(ctx.run.root, "fixture-secrets", "evidence-access") + && source.secret_key_file === join(ctx.run.root, "fixture-secrets", "evidence-secret") + && source.session_token_file === join(ctx.run.root, "fixture-secrets", "evidence-session"), "S3 bindings mismatch"); + assert(source.trusted_endpoint === true && source.allow_private_endpoint === false && source.allow_insecure_endpoint === false + && source.max_bytes === 12345 && source.max_objects === 33 && source.max_pages === 4 && source.page_size === 5, "S3 limits mismatch"); + } +} + + +async function traceSize(path) { + try { return (await stat(path)).size; } catch (error) { if (error.code === "ENOENT") return 0; throw error; } +} +function uniqueArtifacts(artifacts) { + const seen = new Set(); + for (const artifact of artifacts) { + if (seen.has(artifact.path)) throw new Error("artifact path is duplicated within check"); + seen.add(artifact.path); + } + return artifacts; +} +async function commandsObservedForCheck(ctx, checkId, traceBefore) { + const commands = new Set((commandEventSink ?? []).filter((event) => event.checkId === checkId).map((event) => event.executable)); + if (await traceSize(ctx.gitTracePath) > traceBefore) commands.add("git"); + return [...commands].sort(); +} +function wrapProductionCheck(ctx, scenario) { + return { + id: scenario.id, + run: async () => { + ctx.activeArtifacts = []; + activeCommandCheckId = scenario.id; + const traceBefore = await traceSize(ctx.gitTracePath); + try { + const output = await scenario.run(); + return { + commands: await commandsObservedForCheck(ctx, scenario.id, traceBefore), + artifacts: uniqueArtifacts([...(output.artifacts ?? []), ...ctx.activeArtifacts]), + }; + } catch (error) { + error.acceptancePartial = { + commands: await commandsObservedForCheck(ctx, scenario.id, traceBefore), + artifacts: uniqueArtifacts(ctx.activeArtifacts), + }; + throw error; + } finally { + activeCommandCheckId = undefined; + ctx.activeArtifacts = undefined; + } + }, + }; +} + +async function assertProductionGitTrace(ctx) { + const text = await readFile(ctx.gitTracePath, "utf8"); + const events = text.split("\n").filter(Boolean).map((line) => JSON.parse(line)); + const productionStarts = events.filter((event) => event.event === "start" && Array.isArray(event.argv) + && event.argv.some((arg) => typeof arg === "string" && arg.startsWith("core.hooksPath="))); + const publication = productionStarts.some(({ argv }) => argv.includes("commit") && argv.some((arg) => /^Publish workspace /.test(arg))); + const pull = productionStarts.some(({ argv }) => argv.includes("fetch")); + assert(publication && pull, "production Git publication/pull operations absent from Trace2 evidence"); + return { eventCount: events.length, productionStartCount: productionStarts.length, publication, pull }; +} + +async function assertNoP1ScopeEntrypoints(ctx) { + const workspacesRoot = join(ctx.repositoryRoot, "backend", "dist", "workspaces"); + const modules = (await readdir(workspacesRoot)).filter((name) => name.endsWith(".js")); + const forbiddenModuleNames = modules.filter((name) => /evidence[-_.]?(?:adapter|acquisition|preprocess)|(?:acquisition|preprocess)[-_.]?evidence/i.test(name)); + const forbiddenExports = []; + for (const name of modules) { + const source = await readFile(join(workspacesRoot, name), "utf8"); + if (/export\s+(?:class|function|const)\s+(?:acquire|preprocess)Evidence|export\s+(?:class|function|const)\s+Evidence(?:Adapter|Acquisition|Preprocessor)/.test(source)) forbiddenExports.push(name); + } + const routeSurfaces = ctx.services.map(({ name, app }) => ({ name, routes: app.printRoutes({ commonPrefix: false }) })); + const forbiddenRoutes = routeSurfaces.filter(({ routes }) => /\/(?:evidence|acquisition|preprocess)(?:\W|$)/i.test(routes)); + const packageJson = JSON.parse(await readFile(join(ctx.repositoryRoot, "backend", "package.json"), "utf8")); + const entrypointBytes = JSON.stringify({ main: packageJson.main, bin: packageJson.bin, exports: packageJson.exports, scripts: packageJson.scripts }); + const forbiddenPackageEntrypoints = /(?:acquire|preprocess)Evidence|Evidence(?:Adapter|Acquisition|Preprocessor)/i.test(entrypointBytes); + const auditedSurfaceFiles = []; + for (const group of ["workspaces", "routes"]) { + const root = join(ctx.repositoryRoot, "backend", "dist", group); + for (const name of (await readdir(root)).filter((value) => value.endsWith(".js")).sort()) { + auditedSurfaceFiles.push({ path: `${group}/${name}`, source: await readFile(join(root, name), "utf8") }); + } + } + const prohibitedProcessSurfaces = auditedSurfaceFiles.filter(({ source }) => /node:(?:dgram|worker_threads|dns)|\.node(?:["']|$)|process\.dlopen|node-gyp|bindings\s*\(/.test(source)); + const networkAdapterModules = auditedSurfaceFiles.filter(({ source }) => /node:(?:net|http|https)|globalThis\.fetch|\bfetch\s*\(/.test(source)).map(({ path }) => path); + const childProcessModules = auditedSurfaceFiles.filter(({ source }) => /node:child_process/.test(source)).map(({ path }) => path); + assert(JSON.stringify(networkAdapterModules) === JSON.stringify(["workspaces/diagnostics.js"]) + && JSON.stringify(childProcessModules) === JSON.stringify(["workspaces/diagnostics.js", "workspaces/git-repository.js"]) + && prohibitedProcessSurfaces.length === 0, + "unexpected production process/network adapter entrypoint surface present"); + assert(forbiddenModuleNames.length === 0 && forbiddenExports.length === 0 && forbiddenRoutes.length === 0 && !forbiddenPackageEntrypoints, + "prohibited P1 adapter/acquisition/preprocessing entrypoint surface present"); + return { + moduleFilesAudited: modules.sort(), routeAppsAudited: routeSurfaces.map(({ name }) => name), packageEntrypointsAudited: true, + productionSurfaceFilesAudited: auditedSurfaceFiles.map(({ path }) => path), networkAdapterModules, + childProcessModules, workerDgramDnsNativeEntrypoints: [], + }; +} + +function productionChecks(ctx) { + const log = async (id, value) => ({ commands: [], artifacts: [await evidence(ctx.run, `logs/${id}.json`, value, ctx.forbiddenValues)] }); + const scenarios = [ + { id: "preflight", run: async () => { + const gitVersion = await git(["--version"]); await access(ctx.env.THT_BIN, fsConstants.X_OK); + const preflight = await log("preflight", { git: gitVersion.stdout.trim(), node: process.version, thtExecutable: true, + repositoryHead: ctx.provenance.head, repositoryTree: ctx.provenance.tree, repositoryClean: ctx.provenance.clean }); + preflight.artifacts.push(await evidence(ctx.run, "logs/provenance.json", ctx.provenance)); + return preflight; + } }, + { id: "clean_state", run: async () => { + assert(RUN_ID.test(ctx.run.runId), "run identity invalid"); + return await log("clean_state", { exclusiveRoot: true, reused: false }); + } }, + { id: "ownership", run: async () => { + await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); + return await log("ownership", { valid: true, listener: "not_started" }); + } }, + { id: "local_git_bootstrap", run: async () => { + await initializeGit(ctx); + const descriptorArtifacts = []; + for (const workspace of ctx.descriptors) { + const path = `fixtures/descriptors/${workspace.workspace.id}.json`; + await atomicWrite(join(ctx.run.root, path), `${JSON.stringify(workspace, null, 2)}\n`); + descriptorArtifacts.push(await fileArtifact(ctx.run.root, path)); + } + assert(!existsSync(join(ctx.run.root, "author", "workspaces")), "fixture authored a descriptor"); + return { artifacts: [await evidence(ctx.run, "logs/local_git_bootstrap.json", { bootstrapCommit: ctx.bootstrapCommit, descriptorEmpty: true }), ...descriptorArtifacts] }; + } }, + { id: "http_validate_publish_pull_read_export", run: async () => { + await startBackend(ctx); + const status = await request(ctx, "registry-status", "GET", "/workspace-registry/status"); + assert(status.status === 200 && status.body.head === ctx.bootstrapCommit, "empty registry status failed"); + let base = status.body.head; + for (const workspace of ctx.descriptors) { + const id = workspace.workspace.id; + const validated = await request(ctx, `validate-${id}`, "POST", "/workspaces/validate", { workspace }); + assert(validated.status === 200 && validated.body.workspace.workspace.id === id, `validation failed ${id}`); + const published = await request(ctx, `publish-${id}`, "POST", "/workspaces/publish", { action: "create", workspace, baseCommit: base }); + assert(published.status === 200 && HEX40.test(published.body.revision.commit), `publication failed ${id}`); + base = published.body.revision.commit; + } + ctx.publicationHead = base; + const pulled = await request(ctx, "registry-pull", "POST", "/workspace-registry/pull"); + assert(pulled.status === 200 && pulled.body.head === base, "pull failed"); + const listed = await request(ctx, "workspace-list", "GET", "/workspaces"); + assert(listed.status === 200 && listed.body.length === 3, "list failed"); + ctx.reads = {}; ctx.exportManifests = {}; + const artifacts = []; + for (const workspace of ctx.descriptors) { + const id = workspace.workspace.id; const read = await request(ctx, `read-${id}`, "GET", `/workspaces/${id}`); + assert(read.status === 200, `read failed ${id}`); ctx.reads[id] = read.body; + const exported = await request(ctx, `export-${id}`, "GET", `/workspaces/${id}/export`, undefined, true); + assert(exported.status === 200, `export failed ${id}`); ctx.exportManifests[id] = await extractZip(ctx, id, exported.bytes); + artifacts.push(await fileArtifact(ctx.run.root, exportArchiveEvidencePath(`export-${id}`))); + for (const name of ZIP_FILES) artifacts.push(await fileArtifact(ctx.run.root, `exports/extracted/${id}/${name}`)); + } + artifacts.unshift(await evidence(ctx.run, "logs/http-flow.json", { workspaceIds: Object.keys(ctx.reads), head: base, realListener: true, fetch: true })); + return { commands: [], artifacts }; + } }, + { id: "same_revision_git_objects", run: async () => { + const revision = ctx.reads["p1-filesystem"].revision; + ctx.oldRevision = revision; ctx.oldSnapshotDigest = await snapshotDigest(dirname(revision.snapshotPath)); + const checkoutHead = (await git(["rev-parse", "HEAD"], { cwd: join(ctx.run.root, "installation", "registry", "repo") })).stdout.trim(); + const manifestPath = join(dirname(revision.snapshotPath), "snapshot.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + const lease = ctx.thtRunner.acquireWorkspaceRuntime(revision.snapshotPath); let rendered; + try { rendered = (await import("yaml")).parse(await readFile(lease.path, "utf8")); } finally { lease.release(); } + ctx.oldFilesystemRoot = renderedRoot(rendered); + const identities = [revision.commit, checkoutHead, manifest.head, rendered.runtime_identity.workspace_revision]; + assert(new Set(identities).size === 1, "revision identities diverged"); + const repo = join(ctx.run.root, "installation", "registry", "repo"); + await git(["cat-file", "-e", `${revision.commit}:workspaces/p1-filesystem.yaml`], { cwd: repo }); + await git(["cat-file", "-e", `${revision.commit}:workspace-content/p1-filesystem/evidence/guide.md`], { cwd: repo }); + const type = (await git(["cat-file", "-t", `${revision.commit}:workspace-content/p1-filesystem/evidence`], { cwd: repo })).stdout.trim(); + assert(type === "tree", "Evidence object is not a tree"); + assert(!existsSync(join(dirname(revision.snapshotPath), "workspace-content")), "snapshot materialized workspace-content"); + return { commands: ["git"], artifacts: [ + await evidence(ctx.run, "logs/git-object-proof.json", { commit: revision.commit, checkoutHead, manifestHead: manifest.head, runtimeRevision: rendered.runtime_identity.workspace_revision, filesystemRoot: ctx.oldFilesystemRoot, evidenceType: type }), + await fileArtifact(ctx.run.root, relative(ctx.run.root, manifestPath)), await fileArtifact(ctx.run.root, relative(ctx.run.root, revision.snapshotPath)), + ] }; + } }, + { id: "content_only_revision", run: async () => { + const author = join(ctx.run.root, "author"); + await git(["fetch", "origin", "main"], { cwd: author }); await git(["reset", "--hard", "origin/main"], { cwd: author }); + const descriptorBefore = (await git(["rev-parse", "HEAD:workspaces/p1-filesystem.yaml"], { cwd: author })).stdout.trim(); + await writeFile(join(author, "workspace-content", "p1-filesystem", "evidence", "guide.md"), "# P1 curated Evidence v2\n"); + await git(["add", "workspace-content/p1-filesystem/evidence/guide.md"], { cwd: author }); await git(["commit", "-m", "Update curated Evidence only"], { cwd: author }); await git(["push", "origin", "main"], { cwd: author }); + ctx.contentCommit = (await git(["rev-parse", "HEAD"], { cwd: author })).stdout.trim(); + const pulled = await request(ctx, "content-only-pull", "POST", "/workspace-registry/pull"); + assert(pulled.status === 200 && pulled.body.head === ctx.contentCommit, "content pull failed"); + const currentRead = (await request(ctx, "read-filesystem-content", "GET", "/workspaces/p1-filesystem")).body; + const current = currentRead.revision; + const descriptorAfter = (await git(["rev-parse", "HEAD:workspaces/p1-filesystem.yaml"], { cwd: author })).stdout.trim(); + assert(current.commit === ctx.contentCommit && current.blob === ctx.oldRevision.blob && descriptorAfter === descriptorBefore, "content revision identity failed"); + assertByteIdentical(await snapshotDigest(dirname(ctx.oldRevision.snapshotPath)), ctx.oldSnapshotDigest, "old snapshot"); + const lease = ctx.thtRunner.acquireWorkspaceRuntime(current.snapshotPath); let rendered; + try { rendered = (await import("yaml")).parse(await readFile(lease.path, "utf8")); } finally { lease.release(); } + const newRoot = renderedRoot(rendered); + const expectedOldRoot = join(dirname(ctx.oldRevision.snapshotPath), "workspace-content", "p1-filesystem", "evidence"); + const expectedNewRoot = join(dirname(current.snapshotPath), "workspace-content", "p1-filesystem", "evidence"); + assert(ctx.oldFilesystemRoot === expectedOldRoot, "old filesystem root was not old commit-addressed root"); + assert(newRoot === expectedNewRoot && newRoot !== ctx.oldFilesystemRoot, "new filesystem root did not change exactly with commit"); + ctx.currentRevision = current; + const currentSnapshot = await currentSnapshotManifest(ctx, current.commit); ctx.currentManifest = currentSnapshot.manifest; + return { commands: ["git"], artifacts: [ + await evidence(ctx.run, "logs/content-only-revision.json", { oldCommit: ctx.oldRevision.commit, newCommit: current.commit, descriptorBlob: current.blob, oldFilesystemRoot: ctx.oldFilesystemRoot, newFilesystemRoot: newRoot, oldSnapshotImmutable: true }), + ] }; + } }, + { id: "snapshot_and_docs", run: async () => { + const artifacts = []; + for (const id of ctx.descriptors.map((item) => item.workspace.id)) { + const extracted = join(ctx.run.root, "exports", "extracted", id); + for (const name of ZIP_FILES) { + assert((await lstat(join(extracted, name))).isFile(), `missing extracted ${name}`); + await access(join(extracted, name), fsConstants.R_OK); + } + const revision = revisionFromManifest(ctx.currentManifest, id); + for (const suffix of [".yaml", ".env.example", ".md"]) { + const file = join(dirname(revision.snapshotPath), `${id}${suffix}`); + assert(existsSync(file), `snapshot artifact absent ${file}`); + artifacts.push(await fileArtifact(ctx.run.root, relative(ctx.run.root, file))); + } + assert(!existsSync(join(dirname(revision.snapshotPath), "workspace-content")), "snapshot materialized source tree"); + } + artifacts.push(await fileArtifact(ctx.run.root, relative(ctx.run.root, join(dirname(ctx.currentRevision.snapshotPath), "snapshot.json")))); + artifacts.unshift(await evidence(ctx.run, "logs/snapshot-and-docs.json", { exactBundleFiles: ZIP_FILES, generatedDocs: true, immutableSnapshots: true, derivedFromManifest: true })); + return { commands: [], artifacts }; + } }, + { id: "runtime_render_determinism", run: async () => { + ctx.configChecks = []; const artifacts = []; const YAML = await import("yaml"); + for (const id of ctx.descriptors.map((item) => item.workspace.id)) { + const revision = revisionFromManifest(ctx.currentManifest, id); const bytes = []; + for (let n = 1; n <= 2; n += 1) { + const lease = ctx.thtRunner.acquireWorkspaceRuntime(revision.snapshotPath); + try { + const contents = await readFile(lease.path); bytes.push(contents); + await atomicWrite(join(ctx.run.root, "rendered", `${id}-${n}.yaml`), contents); + const checked = await tht(ctx.env.THT_BIN, ["config", "check", "-c", lease.path], { cwd: ctx.env.THT_HARNESS_DIR, env: ctx.env, timeoutMs: 30_000 }); + ctx.configChecks.push({ id, observation: n, code: checked.code }); + } finally { lease.release(); } + const runtimeDir = join(ctx.run.root, "installation", "registry", "snapshots", "runtime"); + if (existsSync(runtimeDir)) assert((await readdir(runtimeDir)).length === 0, "runtime lease leaked"); + artifacts.push(await fileArtifact(ctx.run.root, `rendered/${id}-${n}.yaml`)); + } + assert(bytes[0].equals(bytes[1]), `render nondeterministic ${id}`); + assertRuntimeContract(ctx, id, YAML.parse(bytes[0].toString("utf8")), revision); + } + artifacts.unshift(await evidence(ctx.run, "logs/runtime-render.json", { deterministic: true, released: true, fullSourcePolicyAssertions: true, rootsUnmaterialized: true, workspaces: ctx.descriptors.map((item) => item.workspace.id) })); + return { commands: ["tht"], artifacts }; + } }, + { id: "tht_config_check", run: async () => { + assert(ctx.configChecks.length === 6 && ctx.configChecks.every(({ code }) => code === 0), "tht config checks incomplete"); + return { commands: ["tht"], artifacts: [await evidence(ctx.run, "logs/tht-config-check.json", ctx.configChecks)] }; + } }, + { id: "negative_schema_cases", run: async () => { + const base = structuredClone(ctx.descriptors[0]); + const cases = [ + ["absolute", (w) => { w.evidence.source.uri = "/tmp/evidence"; }, "evidence.source.uri"], + ["traversal", (w) => { w.evidence.source.uri = "workspace-content/p1-filesystem/../evidence"; }, "evidence.source.uri"], + ["backslash", (w) => { w.evidence.source.uri = "workspace-content\\p1-filesystem\\evidence"; }, "evidence.source.uri"], + ["cross-workspace", (w) => { w.evidence.source.uri = "workspace-content/other/evidence"; }, "evidence.source.uri"], + ["unsupported-source", (w) => { w.evidence.source.type = "ftp"; w.evidence.source.uri = "ftp://example.test/file"; }, "evidence.source.type"], + ["unsupported-protocol", (w) => { w.evidence.source = { type: "http", uris: ["ftp://evidence.example.test/file"], authentication: "none" }; }, "evidence.source.uris"], + ["credential-field", (w) => { w.evidence.source.password = ctx.secretValues.rejected; }, "evidence.source.password"], + ["http-userinfo-query", (w) => { w.evidence.source = { type: "http", uris: [`https://user:${ctx.secretValues.rejected}@evidence.example.test/guide?x=${ctx.secretValues.rejected}`], authentication: "none" }; }, "evidence.source.uris"], + ["malformed-policy", (w) => { w.evidence.policy.max_chunk_chars = 0; }, "evidence.policy.max_chunk_chars"], + ["malformed-limit", (w) => { w.evidence.source.max_bytes = 0; }, "evidence.source.max_bytes"], + ]; + const outcomes = []; + for (const [id, mutate, field] of cases) { + const before = await registryState(ctx); const workspace = structuredClone(base); mutate(workspace); + const safeInput = negativeRequestEvidence(id, field); + trackArtifact(ctx, await evidence(ctx.run, `fixtures/requests/negative-${id}.json`, safeInput)); + const response = await request(ctx, `negative-${id}`, "POST", "/workspaces/validate", { workspace }, false, safeInput); + assertGenericWorkspaceInvalid(response, `negative ${id}`); + assert(JSON.stringify(response.body).includes(ctx.secretValues.rejected) === false, `negative leaked ${id}`); + assertByteIdentical(await registryState(ctx), before, `negative ${id}`); + outcomes.push({ case: id, status: response.status, code: response.body.code, expectedInputField: field, genericSafeEnvelope: true, stateByteIdentical: true }); + } + return { commands: ["git"], artifacts: [await evidence(ctx.run, "logs/negative-schema.json", outcomes)] }; + } }, + { id: "negative_context_case", run: async () => { + const contextual = join(ctx.run.root, "installation", "runtime", "contextual"); + const contextualRemote = join(contextual, "remote.git"); + const contextualAuthor = join(contextual, "author"); + const primaryBefore = await primarySemanticState(ctx); + assert(primaryBefore.remote.head === ctx.contentCommit, "primary main was not last-valid before contextual scenario"); + + await mkdir(contextual, { recursive: true }); + await git(["clone", "--bare", join(ctx.run.root, "remote.git"), contextualRemote], { cwd: contextual }); + await git(["clone", contextualRemote, contextualAuthor], { cwd: contextual }); + await git(["config", "user.name", "P1 Context Curator"], { cwd: contextualAuthor }); + await git(["config", "user.email", "p1-context@example.invalid"], { cwd: contextualAuthor }); + await git(["checkout", "-b", "invalid-context"], { cwd: contextualAuthor }); + await git(["push", "-u", "origin", "invalid-context"], { cwd: contextualAuthor }); + await mkdir(join(contextual, "runtime"), { recursive: true }); + await mkdir(join(contextual, "data"), { recursive: true }); + await atomicWrite(join(contextual, "runtime", "base.yaml"), "{}\n"); + const contextualEnv = buildSafeEnvironment({ ambient: ctx.env, fixture: { + ...ctx.env, + THT_DATA_ROOT: join(contextual, "data"), + SETTINGS_FILE: join(contextual, "data", "settings.json"), + MAINTENANCE_STATE_FILE: join(contextual, "data", "maintenance.json"), + THT_WORKSPACE_REGISTRY_ROOT: join(contextual, "registry"), + THT_WORKSPACE_GIT_REMOTE: contextualRemote, + THT_WORKSPACE_GIT_BRANCH: "invalid-context", + THT_WORKSPACE_INSTALLATION_ID: "p1-contextual-acceptance", + THT_HOME: join(contextual, "runtime", "tht-home"), + } }); + const contextualService = await startProductionBackend(ctx, { + name: "contextual", env: contextualEnv, runtimeConfigPath: join(contextual, "runtime", "base.yaml"), + }); + ctx.expectedGitRepositories.push( + "installation/runtime/contextual/remote.git", + "installation/runtime/contextual/author", + ); + const contextualStatus = await request(ctx, "context-registry-status", "GET", "/workspace-registry/status", undefined, false, undefined, contextualService.baseUrl); + assert(contextualStatus.status === 200 && contextualStatus.body.head === ctx.contentCommit, "contextual registry bootstrap failed"); + const contextualBaselinePull = await request(ctx, "context-registry-baseline-pull", "POST", "/workspace-registry/pull", undefined, false, undefined, contextualService.baseUrl); + assert(contextualBaselinePull.status === 200 && contextualBaselinePull.body.head === ctx.contentCommit, "contextual baseline pull failed"); + const primaryAfterSetup = await primarySemanticState(ctx); + await invariantArtifact(ctx, "logs/negative-context-setup-state.json", { before: primaryBefore, after: primaryAfterSetup }); + assertByteIdentical(primaryAfterSetup, primaryBefore, "primary state during contextual setup"); + + const missing = baseWorkspace("missing-context", { type: "filesystem", uri: "workspace-content/missing-context/evidence", patterns: ["**/*.md"], max_bytes: 100 }); + const missingFixture = "fixtures/descriptors/missing-context.json"; + await atomicWrite(join(ctx.run.root, missingFixture), `${JSON.stringify(missing, null, 2)}\n`); + trackArtifact(ctx, await fileArtifact(ctx.run.root, missingFixture)); + const missingBefore = { primary: await primarySemanticState(ctx), secondary: await contextualSemanticState(contextual) }; + await invariantArtifact(ctx, "logs/negative-context-missing-before.json", missingBefore); + const rejectedPublish = await request(ctx, "context-missing-publish", "POST", "/workspaces/publish", { action: "create", workspace: missing, baseCommit: ctx.contentCommit }, false, undefined, contextualService.baseUrl); + const missingAfter = { primary: await primarySemanticState(ctx), secondary: await contextualSemanticState(contextual) }; + await invariantArtifact(ctx, "logs/negative-context-missing-after.json", missingAfter); + assertGenericWorkspaceInvalid(rejectedPublish, "context publish"); + assertByteIdentical(missingAfter.secondary, missingBefore.secondary, "failed contextual publish full semantic state"); + assertByteIdentical(missingAfter.primary, primaryBefore, "primary state after contextual publish"); + + await rm(join(contextualAuthor, "workspace-content", "p1-filesystem", "evidence"), { recursive: true }); + await git(["add", "-A", "workspace-content/p1-filesystem/evidence"], { cwd: contextualAuthor }); + await git(["commit", "-m", "Invalid contextual Evidence state"], { cwd: contextualAuthor }); + await git(["push", "origin", "invalid-context"], { cwd: contextualAuthor }); + const invalidRemoteCommit = (await git(["rev-parse", "HEAD"], { cwd: contextualAuthor })).stdout.trim(); + const invalidBefore = { primary: await primarySemanticState(ctx), secondary: await contextualSemanticState(contextual) }; + await invariantArtifact(ctx, "logs/negative-context-invalid-before.json", invalidBefore); + const rejectedPull = await request(ctx, "context-invalid-pull", "POST", "/workspace-registry/pull", undefined, false, undefined, contextualService.baseUrl); + const invalidAfter = { primary: await primarySemanticState(ctx), secondary: await contextualSemanticState(contextual) }; + await invariantArtifact(ctx, "logs/negative-context-invalid-after.json", invalidAfter); + assertGenericWorkspaceInvalid(rejectedPull, "context pull"); + assertByteIdentical(invalidAfter.primary, primaryBefore, "primary state after contextual pull"); + assertByteIdentical(invalidAfter.secondary.remote, invalidBefore.secondary.remote, "pull mutated contextual fixture remote"); + assertByteIdentical(invalidAfter.secondary.author, invalidBefore.secondary.author, "pull mutated contextual fixture author"); + for (const key of ["active", "snapshots", "data", "runtime"]) { + assert(invalidAfter.secondary[key] === invalidBefore.secondary[key], `invalid pull changed last-valid ${key}`); + } + assert(invalidBefore.secondary.checkout.head === ctx.contentCommit, "contextual checkout was not last-valid before invalid pull"); + assert(invalidAfter.secondary.checkout.head === invalidRemoteCommit, "invalid checkout did not advance as explicitly allowed"); + assert(invalidAfter.secondary.checkout.refs !== invalidBefore.secondary.checkout.refs, "invalid checkout refs did not advance as explicitly allowed"); + assert(invalidAfter.secondary.checkout.branch === "invalid-context" && invalidAfter.secondary.checkout.status === "", "invalid checkout branch/status mismatch"); + assert(invalidAfter.secondary.checkout.indexTree === invalidAfter.secondary.remote.tree, "invalid checkout index did not match invalid remote tree"); + assert(invalidAfter.primary.remote.head === ctx.contentCommit, "contextual scenario mutated primary main"); + return { artifacts: [await evidence(ctx.run, "logs/negative-context.json", { + realSecondaryHttp: true, secondaryBranch: "invalid-context", primaryFullSemanticStateByteIdentical: true, + primaryMainUnchanged: true, missingPublishSecondaryFullSemanticStateByteIdentical: true, + invalidRemoteCommit, checkoutHeadIndexRefsAdvancedExplicitlyAllowed: true, remoteUnchangedByRequest: true, + lastValidActiveSnapshotsDataRuntimeByteIdentical: true, exactGenericEnvelopesNoStderr: true, + gitTransportTelemetryExcluded: [".git/logs", ".git/FETCH_HEAD", ".git/ORIG_HEAD", ".git/objects"], + }, ctx.forbiddenValues)] }; + } }, + { id: "no_p1_scope_artifacts", run: async () => { + const forbidden = ["artifacts/evidence", "corpus/ACTIVE", "embedding-output", "qdrant-records", "preprocessing-invocation"]; + const files = (await walkFiles(ctx.run.root)).map(({ rel }) => rel); + const present = files.filter((path) => forbidden.some((part) => path.includes(part))); + const prohibitedRoutesCalled = ctx.httpRequests.filter(({ path }) => /\/evidence|\/acquisition|\/preprocess/i.test(path)); + const prohibitedCommands = (commandEventSink ?? []).filter(({ argvLabels }) => argvLabels.some((label) => /preprocess|acquire.*evidence|embedding|qdrant/i.test(label))); + const surfaceAudit = await assertNoP1ScopeEntrypoints(ctx); + const gitTraceProof = await assertProductionGitTrace(ctx); + assert(present.length === 0 && prohibitedRoutesCalled.length === 0 && prohibitedCommands.length === 0, "prohibited P1 scope operation observed"); + assert(ctx.networkGuard.externalAttempts.length === 0, "external network connection attempted"); + const rejectedSurfaces = [ + ...ctx.networkGuard.events, + ...(commandEventSink ?? []).filter((event) => event.outcome === "REJECTED" && !ctx.networkGuard.events.includes(event)), + ].filter(({ outcome }) => outcome === "REJECTED"); + const rejectedChildren = rejectedSurfaces.filter(({ surface }) => surface === "child_process"); + const childKinds = new Set(ctx.networkGuard.events.filter(({ surface }) => surface === "child_process").map(({ detail }) => detail).filter(Boolean)); + assert(rejectedSurfaces.length === 0 && rejectedChildren.length === 0 && ["git", "python-lock-holder", "tht"].every((kind) => childKinds.has(kind)), + "production child allowlist evidence is incomplete"); + assert(ctx.services.length === 2 && ctx.services.every(({ baseUrl }) => ctx.networkGuard.hasOwnedOrigin(baseUrl)), "listener was not an owned loopback origin"); + return await log("no-p1-scope-artifacts", { + absentArtifacts: forbidden, moduleEntrypointSurfaceAbsent: true, routeEntrypointSurfaceAbsent: true, + ...surfaceAudit, productionGitTrace: gitTraceProof, networkGuardInstalledBeforeProduction: true, externalNetworkAttempts: [], + exactProductionChildApi: true, productionChildKinds: [...childKinds].sort(), rejectedProductionChildren: [], + ownedLoopbackOrigins: ctx.services.map(({ name }) => name), + }); + } }, + { id: "secret_scan", run: async () => { + const scan = await scanSecretsDetailed({ + runRoot: ctx.run.root, forbiddenValues: ctx.forbiddenValues, + expectedGitRepositories: ctx.expectedGitRepositories, + }); + assert(scan.findings.length === 0, "secret canary found outside exclusion"); + ctx.gitSecretScanFrozen = true; + return await log("secret-scan", { + scanned: true, gitEnumerationFailClosed: true, excluded: "fixture-secrets", + expectedGitRepositories: [...ctx.expectedGitRepositories], repositories: scan.repositories, findings: [], + }); + } }, + { id: "cleanup_confinement", run: async () => { + const fakeRepo = join(ctx.run.root, "installation", "runtime", "cleanup-test"); + await mkdir(join(fakeRepo, ".artifacts", "p1-integration"), { recursive: true }); + const synthetic = await createOwnedRun({ repositoryRoot: fakeRepo }); + const sibling = join(fakeRepo, ".artifacts", "p1-integration", `p1-${"e".repeat(32)}`); + await mkdir(sibling); await writeFile(join(sibling, "sentinel"), "foreign"); + await cleanupOwnedRun({ repositoryRoot: fakeRepo, runRoot: synthetic.root, expectedNonce: synthetic.nonce }); + assert(await readFile(join(sibling, "sentinel"), "utf8") === "foreign", "cleanup removed sibling"); + await rm(fakeRepo, { recursive: true }); + assert(!existsSync(fakeRepo), "cleanup test resource remained"); + return await log("cleanup-confinement", { ownedRemoved: true, siblingPreservedDuringAssertion: true, testResourceRemoved: true }); + } }, + ]; + return scenarios.map((scenario) => wrapProductionCheck(ctx, scenario)); +} + +async function assertRejectsCode(fn, code) { + try { await fn(); } catch (error) { if (error?.code === code) return; throw error; } + throw new Error(`expected ${code}`); +} + +function replaceProcessEnvironment(values) { + for (const key of Object.keys(process.env)) delete process.env[key]; + Object.assign(process.env, values); +} +function failedCheck(id, startedAt, error) { + return { id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error }; +} +function completeFailedResults(results, firstError = "Acceptance setup failed safely.") { + const completed = [...results]; + for (let index = completed.length; index < CHECK_IDS.length; index += 1) { + completed.push(failedCheck(CHECK_IDS[index], nowIso(), index === 0 ? firstError : "Not executed after earlier failure.")); + } + return completed; +} + +function assertUniqueResultArtifacts(results) { + const seen = new Set(); + for (const result of results) for (const artifact of result.artifacts) { + if (seen.has(artifact.path)) throw new Error("artifact path is duplicated across checks"); + seen.add(artifact.path); + } + return results; +} +async function verifyDeclaredArtifacts(runRoot, results) { + assertUniqueResultArtifacts(results); + for (const result of results) for (const artifact of result.artifacts) { + safeArtifactPath(artifact.path); + const current = sha256(await readFile(join(runRoot, artifact.path))); + if (current !== artifact.sha256) throw new Error("declared artifact hash mismatch"); + } +} +function minimalFailClosedReport(run, keep) { + const checks = CHECK_IDS.map((id, index) => failedCheck( + id, nowIso(), index === 0 ? "Acceptance audit failed closed." : "Not executed after fail-closed audit.", + )); + return { + schemaVersion: 1, runId: run.runId, startedAt: run.startedAt, finishedAt: nowIso(), + command: `p1-acceptance integration${keep ? " --keep" : ""}`, overall: "FAIL", checks, + }; +} +function reportBytes(report) { + validateReport(report); + return { + json: Buffer.from(`${JSON.stringify(report, null, 2)} +`), + markdown: Buffer.from(renderReportMarkdown(report)), + }; +} +function attachResultArtifact(results, checkId, artifact) { + const result = results.find(({ id }) => id === checkId); + if (!result) throw new Error("trace artifact owner is absent"); + result.artifacts.push(artifact); +} + +async function listenerRefuses(baseUrl, timeoutMs = 1_000) { + const url = new URL(baseUrl); + return await new Promise((resolvePromise) => { + const socket = new Socket(); let settled = false; + const finish = (refuses) => { if (settled) return; settled = true; socket.destroy(); resolvePromise(refuses); }; + const timer = setTimeout(() => finish(false), timeoutMs); timer.unref(); + socket.once("connect", () => { clearTimeout(timer); finish(false); }); + socket.once("error", () => { clearTimeout(timer); finish(true); }); + try { socket.connect({ host: "127.0.0.1", port: Number(url.port) }); } catch { clearTimeout(timer); finish(true); } + }); +} +function environmentMatches(expected) { + const currentKeys = Object.keys(process.env).sort(); const expectedKeys = Object.keys(expected).sort(); + return JSON.stringify(currentKeys) === JSON.stringify(expectedKeys) + && expectedKeys.every((key) => process.env[key] === expected[key]); +} + +export async function runIntegration({ + repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, + failAt = env.P1_ACCEPTANCE_FAIL_AT, checks, setup, announce, ownershipWriter = writeOwnership, +} = {}) { + if (integrationOwner) throw new Error("P1 acceptance integration is already active"); + const acquisitionToken = Symbol("p1-integration-owner"); + integrationOwner = acquisitionToken; + const savedEnv = { ...process.env }; + let installedEnv; let run; let ctx; let results = []; let fatal; let auditFailed = false; + try { + try { + run = await createOwnedRun({ repositoryRoot }); + ctx = { + run, repositoryRoot, forbiddenValues: [], expectedGitRepositories: [], services: [], listenerClosureEvidence: [], + originalFetch: globalThis.fetch, + }; + commandEventSink = []; + const selectedSetup = setup ?? (checks === undefined ? setupContext : undefined); + if (selectedSetup) { + const configured = await selectedSetup(run, repositoryRoot, env, ctx); + if (configured && configured !== ctx) Object.assign(ctx, configured); + } + if (checks === undefined) { + if (!ctx.env || !ctx.executables) throw new Error("acceptance setup returned no bounded environment"); + installedEnv = { ...ctx.env }; + replaceProcessEnvironment(installedEnv); + ctx.networkGuard = installProductionSurfaceGuard({ + ...ctx.executables, runRoot: run.root, environment: installedEnv, originalFetch: ctx.originalFetch, + }); + checks = productionChecks(ctx); + } else if (ctx.env) { + installedEnv = { ...ctx.env }; + replaceProcessEnvironment(installedEnv); + } + results = await executeChecks({ checks, failAt }); + } catch (error) { + fatal = error; + if (run) results = completeFailedResults(results); + } finally { + if (ctx?.services) { + for (const service of [...ctx.services].reverse()) { + const actualPort = Number(new URL(service.baseUrl).port); + let closed = false; let closeError; + try { + await service.app.close(); + closed = await listenerRefuses(service.baseUrl); + if (!closed) closeError = new Error("listener still accepts connections after close"); + } catch (error) { closeError = error; } + const state = closed ? "closed" : "close_failed"; + ctx.listenerClosureEvidence.push({ name: service.name, host: "127.0.0.1", actualPort, state, listenerRefusedConnection: closed }); + try { + await ownershipWriter(run, { + name: service.name, kind: "fastify", host: "127.0.0.1", requestedPort: 0, + actualPort, pid: process.pid, state, + }); + } catch (error) { closeError ??= error; } + if (closeError) { fatal ??= closeError; auditFailed = true; } + } + } + try { ctx?.networkGuard?.restore(); } catch (error) { fatal ??= error; auditFailed = true; } + if (installedEnv && !environmentMatches(installedEnv)) { fatal ??= new Error("acceptance environment ownership changed"); auditFailed = true; } + try { replaceProcessEnvironment(savedEnv); } catch (error) { fatal ??= error; auditFailed = true; } + if (!environmentMatches(savedEnv)) { fatal ??= new Error("acceptance environment restoration failed"); auditFailed = true; } + } + if (!run) throw fatal; + results = completeFailedResults(results); + + const commandEvents = commandEventSink ?? []; + commandEventSink = undefined; + activeCommandCheckId = undefined; + try { + assertUniqueResultArtifacts(results); + const finalOwnershipBytes = await readFile(join(run.root, "ownership.json")); + const finalOwnership = await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + if (ctx.services.length > 0) { + assert(ctx.listenerClosureEvidence.length === ctx.services.length + && ctx.listenerClosureEvidence.every(({ state, listenerRefusedConnection }) => state === "closed" && listenerRefusedConnection), + "final listener closure evidence is incomplete"); + } + const finalOwnershipArtifact = await evidence(run, "logs/final-ownership.json", { + ownershipSha256: sha256(finalOwnershipBytes), listeners: finalOwnership.listeners, + listenerRefusalChecks: ctx.listenerClosureEvidence, + }, ctx.forbiddenValues); + attachResultArtifact(results, "ownership", finalOwnershipArtifact); + const commandArtifact = await evidence(run, "logs/command-events.json", { eventCount: commandEvents.length, events: commandEvents }, ctx.forbiddenValues); + attachResultArtifact(results, "preflight", commandArtifact); + if (ctx.networkGuard) { + const executablePolicy = {}; + for (const name of ["gitPath", "pythonPath", "thtPath"]) { + const executablePath = ctx.executables[name]; + executablePolicy[name] = { path: executablePath, sha256: sha256(await readFile(executablePath)) }; + } + executablePolicy.thtIdentity = ctx.executables.thtIdentity; + const childArtifact = await evidence(run, "logs/production-child-events.json", { + executablePolicy, environmentPolicy: { + PATH: ctx.env.PATH, HOME: ctx.env.HOME, TMPDIR: ctx.env.TMPDIR, + gitGlobalConfigDisabled: true, gitSystemConfigDisabled: true, gitSystemAttributesDisabled: true, + gitPromptsHelpersSigningDisabled: true, gitLocalConfigAttributesHooksValidatedBeforeInvocation: true, + gitFixedConfigPrefix: ctx.networkGuard.gitPolicy.fixedConfig, gitOwnedEmptyHooksPath: ctx.networkGuard.gitPolicy.hooksPath, + gitAllowedProtocol: "file", maxOutputBytes: MAX_OUTPUT, maxTimeoutMs: 300_000, + }, + eventCount: ctx.networkGuard.events.length, events: ctx.networkGuard.events, + }, ctx.forbiddenValues); + attachResultArtifact(results, "no_p1_scope_artifacts", childArtifact); + } + if (ctx.gitTracePath) { + const gitTraceBytes = await readFile(ctx.gitTracePath); + for (const line of gitTraceBytes.toString("utf8").split("\n").filter(Boolean)) JSON.parse(line); + attachResultArtifact(results, "no_p1_scope_artifacts", await fileArtifact(run.root, relative(run.root, ctx.gitTracePath))); + } + assertUniqueResultArtifacts(results); + if (ctx.executables && !ctx.gitSecretScanFrozen) throw new Error("expected Git secret scan was not frozen inside secret_scan"); + } catch { auditFailed = true; } + + let report = { + schemaVersion: 1, runId: run.runId, startedAt: run.startedAt, finishedAt: nowIso(), + command: `p1-acceptance integration${keep ? " --keep" : ""}`, + overall: !fatal && !auditFailed && deriveOverall(results) === "PASS" ? "PASS" : "FAIL", checks: results, + }; + let bytes; + try { + bytes = reportBytes(report); + const findings = await scanSecrets({ + runRoot: run.root, forbiddenValues: ctx.forbiddenValues, expectedGitRepositories: [], + virtualFiles: [{ path: "report.json", bytes: bytes.json }, { path: "report.md", bytes: bytes.markdown }], + }); + if (findings.length > 0) throw new Error("final filesystem or virtual secret scan failed"); + await verifyDeclaredArtifacts(run.root, results); + } catch { auditFailed = true; } + if (auditFailed) { + report = minimalFailClosedReport(run, keep); + bytes = reportBytes(report); + if (containsAny(bytes.json, ctx.forbiddenValues) || containsAny(bytes.markdown, ctx.forbiddenValues)) { + throw new Error("sanitized fail-closed report unexpectedly contains a forbidden value"); + } + } + await atomicWrite(join(run.root, "report.json"), bytes.json); + await atomicWrite(join(run.root, "report.md"), bytes.markdown); + const finalSuccess = report.overall === "PASS"; + if (announce) await announce({ report, runRoot: run.root, keep }); + const removed = await finalizeOwnedRun({ run, success: finalSuccess, keep }); + return { exitCode: finalSuccess ? 0 : 1, runRoot: run.root, retained: !removed, report }; + } finally { + commandEventSink = undefined; + activeCommandCheckId = undefined; + if (integrationOwner === acquisitionToken) integrationOwner = undefined; + else if (integrationOwner !== undefined) throw new Error("P1 acceptance integration ownership changed"); + } +} +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv.length === 2 && argv[1] !== "--keep")) { + console.error("usage: p1-acceptance integration [--keep]"); return 2; + } + try { + const result = await runIntegration({ + repositoryRoot: defaultRepositoryRoot, keep: argv.includes("--keep"), env, + announce: async ({ report, runRoot, keep }) => { + console.log(`automated integration: ${report.overall}`); + console.log("manual acceptance: PENDING"); + if (keep || report.overall !== "PASS") console.log(runRoot); + }, + }); + return result.exitCode; + } catch (error) { + console.error("P1 acceptance failed before owning a reportable run."); return 1; + } +} +if (resolve(process.argv[1] ?? "") === modulePath) process.exitCode = await main(); diff --git a/backend/scripts/p1-acceptance.test.mjs b/backend/scripts/p1-acceptance.test.mjs new file mode 100644 index 00000000..95200605 --- /dev/null +++ b/backend/scripts/p1-acceptance.test.mjs @@ -0,0 +1,958 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { + chmod, cp, lstat, mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { createServer, connect } from "node:net"; +import dgram from "node:dgram"; +import { Worker } from "node:worker_threads"; +import test from "node:test"; + +import { + canonicalIntegrationBase, + CHECK_IDS, + buildSafeEnvironment, + collectRepositoryProvenance, + installExternalFetchGuard, + installNetworkGuard, + installProductionSurfaceGuard, + resolveProductionExecutables, + negativeRequestEvidence, + cleanupOwnedRun, + createOwnedRun, + deriveOverall, + executeChecks, + exportArchiveEvidencePath, + readAndValidateOwnership, + runCommand, + runIntegration, + scalarSecretBytes, + scanSecrets, + validateReport, + validateRunRoot, +} from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p1 acceptance repository with spaces-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true }); + return await realpath(root); +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("run roots are only canonical direct integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p1-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [ + base, + join(repositoryRoot, ".artifacts", "manual-acceptance", id), + join(base, id, "nested"), + join(base, "foreign"), + join(dirname(base), id), + ]) assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p1-${"A".repeat(32)}`), `p1-${"A".repeat(32)}`)); +}); + +test("cleanup refuses every unowned or ambiguous root", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const cases = [ + ["missing ownership", async (run) => rm(join(run.root, "ownership.json"))], + ["malformed ownership", async (run) => writeFile(join(run.root, "ownership.json"), "{")], + ["mismatched root", async (run) => { + const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8")); + value.root = join(base, `p1-${"b".repeat(32)}`); + await writeFile(join(run.root, "ownership.json"), JSON.stringify(value)); + }], + ["mismatched pid", async (run) => { + const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8")); + value.pid += 1; + await writeFile(join(run.root, "ownership.json"), JSON.stringify(value)); + }], + ["wrong resource list", async (run) => { + const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8")); + value.resources.push(join(repositoryRoot, "foreign")); + await writeFile(join(run.root, "ownership.json"), JSON.stringify(value)); + }], + ]; + for (const [, mutate] of cases) { + const run = await createOwnedRun({ repositoryRoot }); + await mutate(run); + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce })); + assert.equal((await lstat(run.root)).isDirectory(), true); + } + const wrongNonce = await createOwnedRun({ repositoryRoot }); + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: wrongNonce.root, expectedNonce: "0".repeat(64) })); + const symlinkRun = await createOwnedRun({ repositoryRoot }); + const target = `${symlinkRun.root}-target`; + await rm(symlinkRun.root, { recursive: true }); + await mkdir(target); + await symlink(target, symlinkRun.root); + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: symlinkRun.root, expectedNonce: symlinkRun.nonce })); + for (const bad of [base, join(repositoryRoot, ".artifacts", "manual-acceptance"), join(base, "foreign")]) { + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: "0".repeat(64) })); + } +}); + +test("cleanup atomically removes one owned root and preserves siblings", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const sibling = join(canonicalIntegrationBase(repositoryRoot), `p1-${"c".repeat(32)}`); + await mkdir(sibling); + await writeFile(join(sibling, "sentinel"), "foreign"); + await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await assert.rejects(lstat(run.root)); + assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign"); +}); + +function resultFor(id) { + return { + id, status: "PASS", startedAt: "2026-08-09T00:00:00.000Z", + finishedAt: "2026-08-09T00:00:01.000Z", commands: ["git"], + artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }], + }; +} +function validReport(checks = CHECK_IDS.map(resultFor)) { + return { + schemaVersion: 1, runId: `p1-${"d".repeat(32)}`, startedAt: "2026-08-09T00:00:00.000Z", + finishedAt: "2026-08-09T00:00:02.000Z", command: "p1-acceptance integration --keep", + overall: deriveOverall(checks), checks, + }; +} + +test("report validation enforces uniqueness, derivation, safe evidence, hashes, times, and commands", () => { + assert.doesNotThrow(() => validateReport(validReport())); + const mutations = [ + (r) => r.checks.push(structuredClone(r.checks[0])), + (r) => { r.checks[0].attempt = 1; }, + (r) => { r.checks[0].artifacts[0].path = "../secret"; }, + (r) => { r.checks[0].artifacts[0].sha256 = "bad"; }, + (r) => { r.checks[0].startedAt = "today"; }, + (r) => { r.checks[0].commands = ["git status"]; }, + (r) => { r.overall = "PASS"; r.checks[0].status = "FAIL"; }, + (r) => { r.nested = { retries: 2 }; }, + ]; + for (const mutate of mutations) { + const report = validReport(); mutate(report); assert.throws(() => validateReport(report)); + } +}); + +function exactScenarios(run = async () => ({ commands: [], artifacts: [] })) { + return CHECK_IDS.map((id) => ({ id, run: () => run(id) })); +} + +test("injected failure executes once, retains a complete ordered diagnostic report, and returns nonzero", async () => { + const repositoryRoot = await fakeRepository(); + const calls = []; + const failAt = CHECK_IDS[3]; + const result = await runIntegration({ + repositoryRoot, keep: false, failAt, + checks: exactScenarios(async (id) => { calls.push(id); return { commands: [], artifacts: [] }; }), + }); + assert.equal(result.exitCode, 1); + assert.deepEqual(calls, CHECK_IDS.slice(0, 4)); + assert.equal((await lstat(result.runRoot)).isDirectory(), true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS); + assert.equal(report.checks.filter((check) => check.status === "FAIL").length, CHECK_IDS.length - 3); + assert.equal(report.checks[3].error, "Acceptance scenario failed safely."); + assert.equal(report.checks[4].error, "Not executed after earlier failure."); +}); + +test("failed scenario retains partial request and response evidence with observed commands", async () => { + const partial = { + commands: ["git"], + artifacts: [ + { path: "requests/partial.json", sha256: "a".repeat(64) }, + { path: "responses/partial.json", sha256: "b".repeat(64) }, + ], + }; + const checks = exactScenarios(async (id) => { + if (id === CHECK_IDS[4]) { + const error = new Error("HTTP scenario failed after response persistence"); + error.acceptancePartial = partial; + throw error; + } + return {}; + }); + const results = await executeChecks({ checks }); + assert.deepEqual(results[4].commands, partial.commands); + assert.deepEqual(results[4].artifacts, partial.artifacts); + assert.equal(results[4].error, "Acceptance scenario failed safely."); +}); + +test("executeChecks never repeats or executes after first failure but emits the exact check set", async () => { + const calls = new Map(); + const result = await executeChecks({ + checks: exactScenarios(async (id) => { calls.set(id, (calls.get(id) ?? 0) + 1); return {}; }), + failAt: CHECK_IDS[1], + }); + assert.deepEqual(result.map(({ id }) => id), CHECK_IDS); + assert.deepEqual(Object.fromEntries(calls), Object.fromEntries(CHECK_IDS.slice(0, 2).map((id) => [id, 1]))); + assert.equal(result[1].status, "FAIL"); + assert(result.slice(2).every(({ status, error }) => status === "FAIL" && error === "Not executed after earlier failure.")); + assert.throws(() => validateReport(validReport(CHECK_IDS.slice(0, -1).map(resultFor)))); + await assert.rejects(executeChecks({ checks: exactScenarios().reverse() })); +}); + +test("owned setup failure still writes one safe result for every exact check", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ + repositoryRoot, keep: false, + setup: async () => { throw new Error("fixture setup raw failure"); }, + }); + assert.equal(result.exitCode, 1); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS); + assert.equal(report.checks[0].error, "Acceptance setup failed safely."); + assert(report.checks.slice(1).every(({ error }) => error === "Not executed after earlier failure.")); +}); + +test("scalar fixture secret files contain no harness-invalid whitespace", () => { + const bytes = scalarSecretBytes("CANARY-secret-value-123456"); + assert.equal(bytes.toString("utf8"), "CANARY-secret-value-123456"); + assert.equal([...bytes].some((byte) => /\s/.test(String.fromCharCode(byte))), false); + assert.throws(() => scalarSecretBytes("bad secret")); +}); + +test("secret scanner excludes only the direct fixture-secrets subtree", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const canary = "CANARY-secret-value-123456"; + await mkdir(join(run.root, "fixture-secrets")); + await writeFile(join(run.root, "fixture-secrets", "allowed"), canary); + const paths = [ + "logs/a.log", "responses/a.json", "rendered/a.yaml", "exports/raw/a.zip", + "exports/extracted/a.md", "requests/a.json", "report-preview.md", "nested/fixture-secrets/not-excluded", + ]; + for (const path of paths) { + await mkdir(dirname(join(run.root, path)), { recursive: true }); + await writeFile(join(run.root, path), `prefix ${canary} suffix`); + } + const findings = await scanSecrets({ runRoot: run.root, forbiddenValues: [canary], expectedGitRepositories: [] }); + assert.deepEqual(new Set(findings.map((finding) => finding.path)), new Set(paths)); +}); + +test("secret scanner examines reachable Git blobs, not just loose file bytes", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const canary = "GIT-CANARY-secret-value-987654"; + const gitRoot = join(run.root, "author"); + await mkdir(gitRoot); + await execFileAsync("git", ["init", "--initial-branch=main"], { cwd: gitRoot }); + await execFileAsync("git", ["config", "user.name", "Scanner Test"], { cwd: gitRoot }); + await execFileAsync("git", ["config", "user.email", "scanner@example.invalid"], { cwd: gitRoot }); + await writeFile(join(gitRoot, "secret.txt"), canary); + await execFileAsync("git", ["add", "secret.txt"], { cwd: gitRoot }); + await execFileAsync("git", ["commit", "-m", "secret blob"], { cwd: gitRoot }); + await execFileAsync("git", ["rm", "secret.txt"], { cwd: gitRoot }); + await execFileAsync("git", ["commit", "-m", "remove worktree copy"], { cwd: gitRoot }); + const findings = await scanSecrets({ runRoot: run.root, forbiddenValues: [canary], expectedGitRepositories: ["author"] }); + assert.equal(findings.some((finding) => finding.path.startsWith("git-object:")), true); +}); + +test("successful lifecycle honors keep and cleanup", async () => { + const repositoryRoot = await fakeRepository(); + const checks = exactScenarios(); + const kept = await runIntegration({ repositoryRoot, keep: true, checks }); + assert.equal(kept.exitCode, 0); + assert.equal((await lstat(kept.runRoot)).isDirectory(), true); + const cleaned = await runIntegration({ repositoryRoot, keep: false, checks }); + assert.equal(cleaned.exitCode, 0); + await assert.rejects(lstat(cleaned.runRoot)); +}); + +test("command helper accepts only executable plus separate argv", async () => { + await assert.rejects(runCommand("git status")); + await assert.rejects(runCommand({ executable: "/bin/echo", argv: "hello" })); + await assert.rejects(runCommand({ executable: "/bin/echo", argv: [], shell: true })); + await assert.rejects(runCommand({ executable: "git status; rm -rf /", argv: [] })); + await assert.rejects(runCommand({ executable: "/tmp/git", argv: ["--version"] }), /command executable is not allowlisted/); + await assert.rejects(runCommand({ executable: "tht", argv: ["config", "check"] }), /command executable is invalid/); + const scratchRoot = await fakeRepository(); + const executable = join(scratchRoot, "executable with spaces"); + await writeFile(executable, "#!/bin/sh\nprintf '%s' \"$1\"\n", { mode: 0o700 }); + await chmod(executable, 0o700); + await assert.rejects(runCommand({ executable, argv: ["literal;not-a-shell"] }), /command executable is not allowlisted/); + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const { gitPath } = await resolveProductionExecutables({ repositoryRoot }); + const result = await runCommand({ executable: gitPath, argv: ["--version"] }); + assert.match(result.stdout, /^git version /); + assert.equal(result.code, 0); +}); + +test("raw runCommand rejects a configured clean filter before exact Git add", async () => { + const repositoryRoot = await fakeRepository(); + const content = join(repositoryRoot, "workspace-content"); + const helper = join(repositoryRoot, "clean-helper"); + const marker = join(repositoryRoot, "clean-helper-ran"); + await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Fixture Curator"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-curator@example.invalid"], { cwd: repositoryRoot }); + await mkdir(content); + await writeFile(join(content, "guide.md"), "content\n"); + await writeFile(join(repositoryRoot, ".gitattributes"), "workspace-content/** filter=bad\n"); + await writeFile(helper, `#!/bin/sh\nprintf ran > '${marker}'\ncat\n`, { mode: 0o700 }); + await execFileAsync("/usr/bin/git", ["config", "filter.bad.clean", `'${helper}'`], { cwd: repositoryRoot }); + + const { gitPath } = await resolveProductionExecutables({ + repositoryRoot: await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")), + }); + await assert.rejects( + runCommand({ executable: gitPath, argv: ["add", "workspace-content"], cwd: repositoryRoot, env: process.env }), + /unsafe Git repository state/, + ); + await assert.rejects(lstat(marker)); +}); + +test("raw runCommand rejects a diff driver textconv before exact Git show", async () => { + const repositoryRoot = await fakeRepository(); + const marker = join(repositoryRoot, "textconv-helper-ran"); + const helper = join(repositoryRoot, "textconv-helper"); + await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Fixture Curator"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-curator@example.invalid"], { cwd: repositoryRoot }); + await writeFile(join(repositoryRoot, ".gitattributes"), "file diff=evil\n"); + await execFileAsync("/usr/bin/git", ["add", ".gitattributes"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["commit", "-m", "attributes"], { cwd: repositoryRoot }); + await writeFile(join(repositoryRoot, "file"), "v1\n"); + await execFileAsync("/usr/bin/git", ["add", "file"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["commit", "-m", "v1"], { cwd: repositoryRoot }); + await writeFile(join(repositoryRoot, "file"), "v2\n"); + await execFileAsync("/usr/bin/git", ["add", "file"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["commit", "-m", "v2"], { cwd: repositoryRoot }); + await writeFile(helper, `#!/bin/sh\nprintf ran > '${marker}'\nexit 0\n`, { mode: 0o700 }); + await execFileAsync("/usr/bin/git", ["config", "diff.evil.textconv", `'${helper}'`], { cwd: repositoryRoot }); + const emptyHooks = join(repositoryRoot, "registry", "locks", "empty-hooks"); + await mkdir(emptyHooks, { recursive: true }); + + const { gitPath } = await resolveProductionExecutables({ + repositoryRoot: await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")), + }); + await assert.rejects( + runCommand({ executable: gitPath, argv: ["-c", `core.hooksPath=${emptyHooks}`, "show", "HEAD"], cwd: repositoryRoot, env: process.env }), + /unsafe Git repository state/, + ); + await assert.rejects(lstat(marker)); +}); + + +test("safe environment rejects ambient THT and keeps only strict process allowlist plus fixture values", () => { + const safe = buildSafeEnvironment({ + ambient: { PATH: "/safe/bin", HOME: "/home/test", LANG: "C", THT_SECRETS_FILE: "/real/secrets", AWS_SECRET_ACCESS_KEY: "real" }, + fixture: { THT_BIN: "/fixture/tht", THT_WORKSPACE_SECRET_ROOTS: "/fixture/secrets" }, + }); + assert.deepEqual(safe, { + LANG: "C", THT_BIN: "/fixture/tht", THT_WORKSPACE_SECRET_ROOTS: "/fixture/secrets", + }); +}); + +test("secret scan fails closed when Git enumeration fails", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + await mkdir(join(run.root, "remote.git")); + await assert.rejects(scanSecrets({ runRoot: run.root, forbiddenValues: ["CANARY-value-123"] }), /Git secret scan failed closed/); +}); + + +test("negative request evidence persists only case label and expected input field", () => { + const value = negativeRequestEvidence("credential-field", "evidence.source.password"); + assert.deepEqual(value, { case: "credential-field", expectedInputField: "evidence.source.password" }); + assert.equal(JSON.stringify(value).includes("body"), false); +}); + +test("external fetch guard permits only the owned loopback API and records external attempts", async () => { + const called = []; + const guard = installExternalFetchGuard("http://127.0.0.1:12345", async (url) => { called.push(String(url)); return { ok: true }; }); + await guard.fetch("http://127.0.0.1:12345/workspaces"); + await assert.rejects(guard.fetch("https://evidence.example.test/guide.md"), /external fetch prohibited/); + await assert.rejects(guard.fetch("http://127.0.0.1:9999/health"), /external fetch prohibited/); + assert.deepEqual(called, ["http://127.0.0.1:12345/workspaces"]); + assert.equal(guard.externalAttempts.length, 2); +}); + + +test("export archive evidence path matches the persisted binary request id", () => { + assert.equal(exportArchiveEvidencePath("export-p1-filesystem"), "exports/raw/export-p1-filesystem.zip"); +}); + + +test("announce callback observes PASS and manual pending before non-keep cleanup", async () => { + const repositoryRoot = await fakeRepository(); + let observed; + const result = await runIntegration({ + repositoryRoot, keep: false, checks: exactScenarios(), + announce: async ({ report, runRoot }) => { + observed = { overall: report.overall, manual: "PENDING", rootExists: (await lstat(runRoot)).isDirectory() }; + }, + }); + assert.deepEqual(observed, { overall: "PASS", manual: "PENDING", rootExists: true }); + assert.equal(result.retained, false); +}); + +test("public wrapper replaces ambient environment before invoking the runner", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh"), "utf8"); + assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/); + assert.doesNotMatch(wrapper, /P1_ACCEPTANCE_FAIL_AT|LANG|LC_ALL|TZ/); + assert.doesNotMatch(wrapper, /export THT_BIN/); +}); + + +test("network guard is installed globally, rejects non-loopback sockets, and permits one owned listener", async () => { + const server = createServer((socket) => socket.end("ok")); + await new Promise((resolvePromise, reject) => server.listen(0, "127.0.0.1", (error) => error ? reject(error) : resolvePromise())); + const address = server.address(); + assert(address && typeof address === "object"); + const guard = installNetworkGuard(); + try { + guard.addOwnedOrigin(`http://127.0.0.1:${address.port}`); + const contents = await new Promise((resolvePromise, reject) => { + const socket = connect({ host: "127.0.0.1", port: address.port }); + let value = ""; + socket.setEncoding("utf8"); + socket.on("data", (chunk) => { value += chunk; }); + socket.on("end", () => resolvePromise(value)); + socket.on("error", reject); + }); + assert.equal(contents, "ok"); + assert.throws(() => connect({ host: "example.com", port: 80 }), /external network connection prohibited/); + await assert.rejects(globalThis.fetch("https://example.com/"), /external network connection prohibited/); + assert.equal(guard.externalAttempts.length, 2); + } finally { + guard.restore(); + await new Promise((resolvePromise) => server.close(resolvePromise)); + } +}); + +test("report validation rejects duplicate artifact paths across checks", () => { + const report = validReport(); + report.checks[1].artifacts[0].path = report.checks[0].artifacts[0].path; + assert.throws(() => validateReport(report), /report artifact path is duplicated/); +}); + +test("virtual report leakage yields a minimal sanitized exact-15 FAIL report", async () => { + const repositoryRoot = await fakeRepository(); + const canary = "VIRTUAL-CANARY-12345678"; + const checks = exactScenarios(async (id) => ({ + commands: [], + artifacts: id === CHECK_IDS[0] ? [{ path: `logs/${canary}.json`, sha256: "a".repeat(64) }] : [], + })); + const result = await runIntegration({ + repositoryRoot, + checks, + setup: async (_run, _repositoryRoot, _env, ctx) => { + ctx.forbiddenValues = [canary]; + return ctx; + }, + }); + assert.equal(result.exitCode, 1); + const bytes = await readFile(join(result.runRoot, "report.json")); + assert.equal(bytes.includes(Buffer.from(canary)), false); + const report = JSON.parse(bytes); + assert.deepEqual(report.checks.map(({ id }) => id), CHECK_IDS); + assert(report.checks.every(({ status, commands, artifacts }) => status === "FAIL" && commands.length === 0 && artifacts.length === 0)); +}); + +test("partial setup preserves forbidden values and never writes secret-bearing report bytes", async () => { + const repositoryRoot = await fakeRepository(); + const canary = "PARTIAL-SETUP-CANARY-12345678"; + const result = await runIntegration({ + repositoryRoot, + setup: async (run, _repositoryRoot, _env, ctx) => { + ctx.forbiddenValues = [canary]; + await mkdir(join(run.root, "logs"), { recursive: true }); + await writeFile(join(run.root, "logs", "partial-setup.log"), canary); + throw new Error(`unsafe ${canary}`); + }, + }); + assert.equal(result.exitCode, 1); + const bytes = await readFile(join(result.runRoot, "report.json")); + assert.equal(bytes.includes(Buffer.from(canary)), false); + const report = JSON.parse(bytes); + assert.equal(report.checks.length, 15); + assert(report.checks.every(({ status }) => status === "FAIL")); +}); + +test("secret scan fails closed when either expected Git repository is missing", async () => { + for (const missing of ["remote.git", "author"]) { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const present = missing === "remote.git" ? "author" : "remote.git"; + await mkdir(join(run.root, present)); + await execFileAsync("git", present === "remote.git" ? ["init", "--bare", join(run.root, present)] : ["init", join(run.root, present)]); + await assert.rejects(scanSecrets({ runRoot: run.root, forbiddenValues: ["CANARY-value-123"] }), new RegExp(`missing expected Git repository: ${missing.replace(".", "\\.")}`)); + } +}); + + +test("runIntegration fails closed when a later duplicate overwrites stale artifact evidence", async () => { + const repositoryRoot = await fakeRepository(); + const checks = exactScenarios(async (id) => { + if (id === CHECK_IDS[0]) { + await mkdir(join(repositoryRoot, ".artifacts", "p1-integration", "scratch"), { recursive: true }); + } + return { commands: [], artifacts: [] }; + }); + const result = await runIntegration({ + repositoryRoot, keep: true, + setup: async (run, _repositoryRoot, _env, ctx) => { + const path = join(run.root, "logs", "overwritten.json"); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, "first"); + const stale = { path: "logs/overwritten.json", sha256: "a7937b64b8caa58f03721bb6bacf9e92a2c78987f5d1692a065a4698e006c4ca" }; + checks[0].run = async () => ({ commands: [], artifacts: [stale] }); + checks[1].run = async () => { + await writeFile(path, "second"); + return { commands: [], artifacts: [{ path: stale.path, sha256: "16367aacb67a4a017c8da8ab95682ccb389c61bb315f3425e2f2666f2476d1ce" }] }; + }; + return ctx; + }, + checks, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.report.checks.length, 15); + assert(result.report.checks.every(({ status, artifacts }) => status === "FAIL" && artifacts.length === 0)); +}); + +test("production surface guard rejects and records UDP, Worker, git ls-remote, and unexpected python", async () => { + const runRoot = await fakeRepository(); + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const executables = await resolveProductionExecutables({ repositoryRoot }); + const guard = installProductionSurfaceGuard({ + ...executables, runRoot, environment: { ...process.env }, originalFetch: globalThis.fetch, + }); + try { + assert.throws(() => dgram.createSocket("udp4"), /prohibited production surface/); + assert.throws(() => new Worker("", { eval: true }), /prohibited production surface/); + await assert.rejects(runCommand({ executable: executables.gitPath, argv: ["ls-remote", "https://example.com/repo.git"] }), /Git command is prohibited/); + const childProcess = await import("node:child_process"); + assert.throws(() => childProcess.spawn(executables.pythonPath, ["-c", "print('unexpected')"]), /child command is prohibited/); + assert.deepEqual(new Set(guard.events.filter(({ outcome }) => outcome === "REJECTED").map(({ surface }) => surface)), + new Set(["dgram", "worker_threads", "child_process"])); + } finally { + guard.restore(); + } +}); + +test("listener close rejection retains listening truth and forces exact-15 FAIL", async () => { + const repositoryRoot = await fakeRepository(); + const server = createServer(); + await new Promise((resolvePromise, reject) => server.listen(0, "127.0.0.1", (error) => error ? reject(error) : resolvePromise())); + const address = server.address(); + assert(address && typeof address === "object"); + const result = await runIntegration({ + repositoryRoot, keep: false, checks: exactScenarios(), + setup: async (run, _repositoryRoot, _env, ctx) => { + ctx.services = [{ name: "primary", baseUrl: `http://127.0.0.1:${address.port}`, app: { close: async () => { throw new Error("close rejected"); } } }]; + const value = JSON.parse(await readFile(join(run.root, "ownership.json"), "utf8")); + value.listeners[0] = { name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0, actualPort: address.port, pid: process.pid, state: "listening" }; + await writeFile(join(run.root, "ownership.json"), `${JSON.stringify(value, null, 2)}\n`); + return ctx; + }, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.retained, true); + const owner = JSON.parse(await readFile(join(result.runRoot, "ownership.json"), "utf8")); + assert.notEqual(owner.listeners[0].state, "closed"); + assert(result.report.checks.every(({ status }) => status === "FAIL")); + await new Promise((resolvePromise) => server.close(resolvePromise)); +}); + +test("ownership close write failure forces retained exact-15 FAIL", async () => { + const repositoryRoot = await fakeRepository(); + const server = createServer(); + await new Promise((resolvePromise, reject) => server.listen(0, "127.0.0.1", (error) => error ? reject(error) : resolvePromise())); + const address = server.address(); + assert(address && typeof address === "object"); + const result = await runIntegration({ + repositoryRoot, keep: false, checks: exactScenarios(), + ownershipWriter: async (_run, update) => { if (update?.state === "closed") throw new Error("owned write rejected"); }, + setup: async (_run, _repositoryRoot, _env, ctx) => { + ctx.services = [{ name: "primary", baseUrl: `http://127.0.0.1:${address.port}`, app: { close: async () => await new Promise((resolvePromise) => server.close(resolvePromise)) } }]; + return ctx; + }, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.retained, true); + assert(result.report.checks.every(({ status }) => status === "FAIL")); +}); + +test("nested runIntegration is rejected before process-global mutation and outer restoration remains owned", async () => { + const repositoryRoot = await fakeRepository(); + const originalFetch = globalThis.fetch; + const originalPath = process.env.PATH; + let nestedError; + const result = await runIntegration({ + repositoryRoot, keep: true, checks: exactScenarios(), + setup: async (_run, _repositoryRoot, _env, ctx) => { + try { await runIntegration({ repositoryRoot, keep: true, checks: exactScenarios() }); } catch (error) { nestedError = error; } + assert.equal(globalThis.fetch, originalFetch); + assert.equal(process.env.PATH, originalPath); + return ctx; + }, + }); + assert.match(nestedError?.message ?? "", /already active/); + assert.equal(result.exitCode, 0); + assert.equal(globalThis.fetch, originalFetch); + assert.equal(process.env.PATH, originalPath); +}); + + +test("environment tampering fails the audit and restores the caller environment", async () => { + const repositoryRoot = await fakeRepository(); + const before = { ...process.env }; + const checks = exactScenarios(async (id) => { + if (id === CHECK_IDS[0]) process.env.P1_ACCEPTANCE_UNOWNED = "tampered"; + return { commands: [], artifacts: [] }; + }); + const result = await runIntegration({ + repositoryRoot, keep: true, checks, + setup: async (_run, _repositoryRoot, _env, ctx) => { ctx.env = { P1_ACCEPTANCE_OWNED: "yes" }; return ctx; }, + }); + assert.equal(result.exitCode, 1); + assert(result.report.checks.every(({ status }) => status === "FAIL")); + assert.deepEqual({ ...process.env }, before); +}); + +test("production guard detects global tampering and restores without stranding patches", async () => { + const runRoot = await fakeRepository(); + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const executables = await resolveProductionExecutables({ repositoryRoot }); + const originalFetch = globalThis.fetch; + const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env }, originalFetch }); + globalThis.fetch = originalFetch; + assert.throws(() => guard.restore(), /ownership restoration failed/); + assert.equal(globalThis.fetch, originalFetch); + const childProcess = await import("node:child_process"); + assert.doesNotThrow(() => childProcess.spawn); +}); + + +test("Git grammar rejects helper, config, alias, and network-capable spellings with one event each", async () => { + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const runRoot = await fakeRepository(); + const executables = await resolveProductionExecutables({ repositoryRoot, thtBin: "/tmp/hostile-tht" }); + const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } }); + const source = join(runRoot, "source.git"); + const destination = join(runRoot, "destination"); + const marker = join(runRoot, "helper-ran"); + await execFileAsync(executables.gitPath, ["init", "--bare", source]); + const helper = join(runRoot, "upload-helper"); + await writeFile(helper, `#!/bin/sh\nprintf ran > "${marker}"\nexit 99\n`, { mode: 0o700 }); + const prohibited = [ + ["clone", `--upload-pack=${helper}`, source, destination], + ["clone", "--receive-pack=/tmp/helper", source, destination], + ["--exec-path=/tmp", "status"], + ["-c", "alias.status=!touch /tmp/pwn", "status"], + ["-c", "core.hooksPath=/tmp/hooks", "status"], + ["-c", "diff.external=/tmp/helper", "status"], + ["config", "filter.bad.clean", "/tmp/helper"], + ["ls-remote", "https://example.com/repo.git"], + ]; + try { + for (const argv of prohibited) { + const before = guard.events.length; + await assert.rejects(runCommand({ executable: executables.gitPath, argv }), /Git command is prohibited/); + assert.equal(guard.events.length - before, 1); + assert.equal(guard.events.at(-1).outcome, "REJECTED"); + } + await assert.rejects(lstat(marker)); + } finally { guard.restore(); } +}); + +test("production executables ignore ambient THT and bind the generated tht entrypoint to reviewed source", async () => { + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const hostile = join(await fakeRepository(), "tht"); + await writeFile(hostile, "#!/bin/sh\nexit 0\n", { mode: 0o700 }); + const executables = await resolveProductionExecutables({ repositoryRoot, thtBin: hostile }); + assert.equal(executables.thtPath, join(repositoryRoot, "harness", ".venv", "bin", "tht")); + assert.equal(executables.thtIdentity.sourceRoot, join(repositoryRoot, "harness", "tht")); + assert.equal(executables.thtIdentity.sourceStatus, "git-index-byte-identical"); + assert.equal(executables.thtIdentity.entrypoint, "generated-console-script"); + assert.match(executables.thtIdentity.pythonPath, /python3(?:\.\d+)?$/); +}); + +test("tht accepts only config check for one owned rendered yaml", async () => { + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const runRoot = await fakeRepository(); + const rendered = join(runRoot, "rendered", "workspace.yaml"); + await mkdir(dirname(rendered), { recursive: true }); + await writeFile(rendered, "profile: acceptance\n"); + const executables = await resolveProductionExecutables({ repositoryRoot }); + const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } }); + const childProcess = await import("node:child_process"); + try { + for (const argv of [ + ["config", "check"], ["config", "check", "-c", "/tmp/unowned.yaml"], + ["doctor"], ["config", "check", "-c", rendered, "--extra"], + ]) { + const before = guard.events.length; + assert.throws(() => childProcess.execFile(executables.thtPath, argv), /THT command is prohibited/); + assert.equal(guard.events.length - before, 1); + } + } finally { guard.restore(); } +}); + +test("production guard installation rolls back every patch and owner on every injected patch failure", async () => { + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const runRoot = await fakeRepository(); + const executables = await resolveProductionExecutables({ repositoryRoot }); + const childProcess = await import("node:child_process"); + const originalSpawn = childProcess.spawn; + const originalDgram = dgram.createSocket; + const originalFetch = globalThis.fetch; + for (let failPatchAt = 1; failPatchAt <= 12; failPatchAt += 1) { + assert.throws(() => installProductionSurfaceGuard({ + ...executables, runRoot, environment: { ...process.env }, failPatchAt, + }), /injected production patch failure/); + assert.equal(childProcess.spawn, originalSpawn); + assert.equal(dgram.createSocket, originalDgram); + assert.equal(globalThis.fetch, originalFetch); + const reacquired = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } }); + reacquired.restore(); + } +}); + +test("command bounds reject zero, negative, fractional, and nonnumeric timeouts with one sanitized event", async () => { + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const runRoot = await fakeRepository(); + const executables = await resolveProductionExecutables({ repositoryRoot }); + const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } }); + try { + for (const timeoutMs of [0, -1, 1.5, NaN]) { + const before = guard.events.length; + await assert.rejects(runCommand({ executable: executables.gitPath, argv: ["--version"], timeoutMs }), /command bounds are invalid/); + assert.equal(guard.events.length - before, 1); + } + } finally { guard.restore(); } +}); + +test("public wrapper has no ambient command resolution and isolates the build and runner", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh"), "utf8"); + assert.doesNotMatch(wrapper, /command\s+-v/); + assert.doesNotMatch(wrapper, /\b(?:node|npm)\s+--prefix/); + assert.match(wrapper, /env -i/); + assert.match(wrapper, /npm-cli\.js/); + assert.match(wrapper, /"\$node_path" "\$npm_path"/); + assert.match(wrapper, /\/bin\/rm -rf -- "\$repo_root\/backend\/dist"/); +}); + + +test("hostile PATH Node npm and THT substitutes never execute at the public wrapper boundary", async () => { + const hostileRoot = await fakeRepository(); + const marker = join(hostileRoot, "ambient-tool-ran"); + for (const name of ["node", "npm", "tht"]) { + const path = join(hostileRoot, name); + await writeFile(path, `#!/bin/sh\nprintf '%s' '${name}' >> '${marker}'\nexit 97\n`, { mode: 0o700 }); + await chmod(path, 0o700); + } + const wrapper = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh"); + await assert.rejects(execFileAsync(wrapper, ["invalid"], { + env: { ...process.env, PATH: hostileRoot, THT_BIN: join(hostileRoot, "tht") }, timeout: 30_000, + })); + await assert.rejects(lstat(marker)); +}); + + +async function fakeTrustedThtRepository() { + const repositoryRoot = await fakeRepository(); + const realRepository = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const harness = join(repositoryRoot, "harness"); + const sourceRoot = join(harness, "tht"); + await mkdir(harness, { recursive: true }); + await cp(join(realRepository, "harness", "tht"), sourceRoot, { + recursive: true, filter: (path) => !path.split("/").includes("__pycache__") && !path.endsWith(".pyc"), + }); + await cp(join(realRepository, "harness", "pyproject.toml"), join(harness, "pyproject.toml")); + const realExecutables = await resolveProductionExecutables({ repositoryRoot: realRepository }); + const pythonName = realExecutables.thtIdentity.pythonPath.split("/").at(-1); + const venvBin = join(harness, ".venv", "bin"); + const sitePackages = join(harness, ".venv", "lib", pythonName, "site-packages"); + await mkdir(venvBin, { recursive: true }); + await mkdir(sitePackages, { recursive: true }); + await symlink(realExecutables.thtIdentity.pythonCanonicalPath, join(venvBin, "python")); + await symlink(realExecutables.thtIdentity.pythonCanonicalPath, join(venvBin, pythonName)); + const entrypoint = `#!${join(venvBin, pythonName)}\nimport sys\nfrom tht.cli import app\nif __name__ == '__main__':\n if sys.argv[0].endswith('.exe'):\n sys.argv[0] = sys.argv[0][:-4]\n sys.exit(app())\n`; + await writeFile(join(venvBin, "tht"), entrypoint, { mode: 0o700 }); + const realSite = join(realRepository, "harness", ".venv", "lib", pythonName, "site-packages"); + const realFinderName = (await import("node:fs/promises")).readdir(realSite).then((entries) => entries.find((name) => /^__editable___tht_.*_finder\.py$/.test(name))); + const finderName = await realFinderName; + const realFinder = await readFile(join(realSite, finderName), "utf8"); + const finder = realFinder.replaceAll(join(realRepository, "harness", "tht"), sourceRoot); + await writeFile(join(sitePackages, finderName), finder); + const moduleName = finderName.slice(0, -3); + await writeFile(join(sitePackages, "__editable__.tht-0.1.0.pth"), `import ${moduleName}; ${moduleName}.install()`); + await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Test"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-test@example.invalid"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["add", "harness/tht", "harness/pyproject.toml"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["commit", "-m", "trusted source"], { cwd: repositoryRoot }); + return { repositoryRoot, sourceRoot, sitePackages, finderName }; +} + +test("Git rejects configured upload-pack, clean filter, and hook state before exact allowed operations", async () => { + const repositoryRoot = await realpath(join(dirname(fileURLToPath(import.meta.url)), "..", "..")); + const runRoot = await fakeRepository(); + const remote = join(runRoot, "remote.git"); + const author = join(runRoot, "author"); + await execFileAsync("/usr/bin/git", ["init", "--bare", "--initial-branch=main", remote]); + await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main", author]); + await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Fixture Curator"], { cwd: author }); + await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-curator@example.invalid"], { cwd: author }); + await writeFile(join(author, "seed"), "seed\n"); + await execFileAsync("/usr/bin/git", ["add", "seed"], { cwd: author }); + await execFileAsync("/usr/bin/git", ["commit", "-m", "seed"], { cwd: author }); + await execFileAsync("/usr/bin/git", ["remote", "add", "origin", remote], { cwd: author }); + await execFileAsync("/usr/bin/git", ["push", "origin", "main"], { cwd: author }); + const executables = await resolveProductionExecutables({ repositoryRoot }); + const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } }); + try { + for (const [kind, configure, argv] of [ + ["upload", async (helper) => execFileAsync("/usr/bin/git", ["config", "remote.origin.uploadpack", helper], { cwd: author }), ["fetch", "origin", "main"]], + ["filter", async (helper) => { + await mkdir(join(author, "workspace-content"), { recursive: true }); + await writeFile(join(author, ".gitattributes"), "workspace-content/** filter=bad\n"); + await execFileAsync("/usr/bin/git", ["config", "filter.bad.clean", helper], { cwd: author }); + }, ["add", "workspace-content"]], + ["hook", async (helper) => { await cp(helper, join(author, ".git", "hooks", "pre-commit")); }, ["commit", "-m", "Bootstrap curated P1 content"]], + ]) { + await execFileAsync("/usr/bin/git", ["config", "--unset-all", "remote.origin.uploadpack"], { cwd: author }).catch(() => {}); + await execFileAsync("/usr/bin/git", ["config", "--remove-section", "filter.bad"], { cwd: author }).catch(() => {}); + await rm(join(author, ".gitattributes"), { force: true }); + await rm(join(author, ".git", "hooks", "pre-commit"), { force: true }); + const marker = join(runRoot, `${kind}-marker`); + const helper = join(runRoot, `${kind}-helper`); + await writeFile(helper, `#!/bin/sh\nprintf ran > '${marker}'\nexec /usr/bin/git-upload-pack \"$@\"\n`, { mode: 0o700 }); + await configure(helper); + const before = guard.events.length; + await assert.rejects(runCommand({ executable: executables.gitPath, argv, cwd: author, env: { ...process.env } }), /unsafe Git repository state/); + assert.equal(guard.events.length - before, 1); + assert.equal(guard.events.at(-1).outcome, "REJECTED"); + await assert.rejects(lstat(marker)); + } + } finally { guard.restore(); } +}); + +test("trusted tht rejects executable finder code and Git-hidden source changes", async () => { + const maliciousFinder = await fakeTrustedThtRepository(); + const finderPath = join(maliciousFinder.sitePackages, maliciousFinder.finderName); + await writeFile(finderPath, `open('${join(maliciousFinder.repositoryRoot, "finder-marker")}', 'w').write('ran')\n${await readFile(finderPath, "utf8")}`); + await assert.rejects(resolveProductionExecutables({ repositoryRoot: maliciousFinder.repositoryRoot }), /editable binding is invalid/); + + const ignoredPyc = await fakeTrustedThtRepository(); + await mkdir(join(ignoredPyc.sitePackages, "__pycache__")); + await writeFile(join(ignoredPyc.sitePackages, "__pycache__", `${ignoredPyc.finderName.slice(0, -3)}.cpython-313.pyc`), "malicious bytecode"); + await assert.rejects(resolveProductionExecutables({ repositoryRoot: ignoredPyc.repositoryRoot }), /import startup override/); + + const hiddenSource = await fakeTrustedThtRepository(); + const sourcePath = join(hiddenSource.sourceRoot, "cli", "__init__.py"); + await execFileAsync("/usr/bin/git", ["update-index", "--assume-unchanged", "harness/tht/cli/__init__.py"], { cwd: hiddenSource.repositoryRoot }); + await writeFile(sourcePath, `${await readFile(sourcePath, "utf8")}\n# malicious hidden swap\n`); + await assert.rejects(resolveProductionExecutables({ repositoryRoot: hiddenSource.repositoryRoot }), /source bytes differ from Git/); +}); + +test("trusted tht guard rejects and records post-resolution entrypoint finder and source swaps at spawn", async () => { + for (const target of ["entrypoint", "finder", "source"]) { + const fixture = await fakeTrustedThtRepository(); + const executables = await resolveProductionExecutables({ repositoryRoot: fixture.repositoryRoot }); + const runRoot = await fakeRepository(); + const configPath = join(runRoot, "rendered", "workspace.yaml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile(configPath, "profile: acceptance\n"); + const guard = installProductionSurfaceGuard({ ...executables, runRoot, environment: { ...process.env } }); + try { + const path = target === "entrypoint" ? executables.thtPath + : target === "finder" ? join(fixture.sitePackages, fixture.finderName) + : join(fixture.sourceRoot, "cli", "__init__.py"); + await writeFile(path, `${await readFile(path, "utf8")}\n# post-resolution swap\n`, target === "entrypoint" ? { mode: 0o700 } : undefined); + const childProcess = await import("node:child_process"); + assert.throws(() => childProcess.execFile(executables.thtPath, ["config", "check", "-c", configPath], { + cwd: join(fixture.repositoryRoot, "harness"), env: { ...process.env }, + }), /trusted THT identity changed/); + assert.equal(guard.events.at(-1).outcome, "REJECTED"); + } finally { guard.restore(); } + } +}); + +test("secret scan fails closed on a recoverable symlink outside fixture-secrets", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const canary = "CANARY-symlink-secret-123456"; + await mkdir(join(run.root, "fixture-secrets")); + await writeFile(join(run.root, "fixture-secrets", "token"), canary); + await mkdir(join(run.root, "responses")); + await symlink(join(run.root, "fixture-secrets", "token"), join(run.root, "responses", "leak")); + await assert.rejects(scanSecrets({ runRoot: run.root, forbiddenValues: [canary], expectedGitRepositories: [] }), /symlink outside fixture-secrets/); +}); + +test("direct public wrapper clears startup files and exported functions before Bash starts", async () => { + const root = await fakeRepository(); + const bashStartup = join(root, "bash-startup"); + const envStartup = join(root, "env-startup"); + const bashMarker = join(root, "bash-env-ran"); + const envMarker = join(root, "env-ran"); + const functionMarker = join(root, "exported-function-ran"); + await writeFile(bashStartup, `printf sourced > '${bashMarker}'\n`); + await writeFile(envStartup, `printf sourced > '${envMarker}'\n`); + const wrapper = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p1-acceptance.sh"); + await assert.rejects(execFileAsync(wrapper, ["invalid"], { + env: { + ...process.env, + BASH_ENV: bashStartup, + ENV: envStartup, + "BASH_FUNC_cd%%": `() { printf function > '${functionMarker}'; builtin cd "$@"; }`, + }, + })); + for (const marker of [bashMarker, envMarker, functionMarker]) await assert.rejects(lstat(marker)); + assert.match(await readFile(wrapper, "utf8"), /^#!\/usr\/bin\/env -S -i PATH=\/usr\/bin:\/bin \/bin\/bash\n/); +}); + +test("final listener ownership state is a declared hash-bound report artifact", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: true, checks: exactScenarios() }); + const artifact = result.report.checks.flatMap(({ artifacts }) => artifacts).find(({ path }) => path === "logs/final-ownership.json"); + assert(artifact); + const bytes = await readFile(join(result.runRoot, artifact.path)); + const { createHash } = await import("node:crypto"); + assert.equal(createHash("sha256").update(bytes).digest("hex"), artifact.sha256); + const value = JSON.parse(bytes); + assert.deepEqual(value.listeners.map(({ state }) => state), ["not_started", "not_started"]); +}); + +test("repository provenance binds clean HEAD tree and backend source/dist manifests and rejects dirty state", async () => { + const repositoryRoot = await fakeRepository(); + await mkdir(join(repositoryRoot, "backend", "src"), { recursive: true }); + await mkdir(join(repositoryRoot, "backend", "scripts"), { recursive: true }); + await mkdir(join(repositoryRoot, "backend", "dist"), { recursive: true }); + await writeFile(join(repositoryRoot, "backend", "src", "app.ts"), "export const value = 1;\n"); + await writeFile(join(repositoryRoot, "backend", "scripts", "p1-acceptance.mjs"), "export {};\n"); + await writeFile(join(repositoryRoot, "backend", "dist", "app.js"), "export const value = 1;\n"); + await writeFile(join(repositoryRoot, "backend", "package.json"), "{}\n"); + await writeFile(join(repositoryRoot, "backend", "package-lock.json"), "{}\n"); + await writeFile(join(repositoryRoot, "backend", "tsconfig.json"), "{}\n"); + await execFileAsync("/usr/bin/git", ["init", "--initial-branch=main"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.name", "P1 Test"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["config", "user.email", "p1-test@example.invalid"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["add", "backend"], { cwd: repositoryRoot }); + await execFileAsync("/usr/bin/git", ["commit", "-m", "clean tree"], { cwd: repositoryRoot }); + const provenance = await collectRepositoryProvenance({ repositoryRoot, gitPath: "/usr/bin/git" }); + assert.match(provenance.head, /^[0-9a-f]{40}$/); + assert.match(provenance.tree, /^[0-9a-f]{40}$/); + assert.equal(provenance.clean, true); + assert.equal(provenance.backendSource.files.some(({ path }) => path === "src/app.ts"), true); + assert.equal(provenance.backendDist.files.some(({ path }) => path === "dist/app.js"), true); + await writeFile(join(repositoryRoot, "backend", "src", "app.ts"), "dirty\n"); + await assert.rejects(collectRepositoryProvenance({ repositoryRoot, gitPath: "/usr/bin/git" }), /repository is not clean/); +}); diff --git a/backend/scripts/p1-manual-acceptance.mjs b/backend/scripts/p1-manual-acceptance.mjs new file mode 100755 index 00000000..89407388 --- /dev/null +++ b/backend/scripts/p1-manual-acceptance.mjs @@ -0,0 +1,609 @@ +#!/usr/bin/env node +import { execFile, spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { closeSync, constants, fstatSync, fsyncSync, lstatSync, openSync, realpathSync } from "node:fs"; +import { access, chmod, lstat, mkdir, open, readFile, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; +import http from "node:http"; +import net from "node:net"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); const modulePath=fileURLToPath(import.meta.url); const defaultRepositoryRoot=realpathSync(resolve(dirname(modulePath),"../..")); +const HEX64=/^[0-9a-f]{64}$/; const PORT=8791; const HOST="127.0.0.1"; +export function fixedManualRoot(repositoryRoot=defaultRepositoryRoot){return join(realpathSync(repositoryRoot),".artifacts","manual-acceptance","p1");} +function below(parent,child){const rel=relative(parent,child);return rel!==""&&!rel.startsWith(`..${sep}`)&&rel!==".."&&!isAbsolute(rel);} +function noSymlinkExisting(repo,target){const rel=relative(repo,target);if(rel.startsWith("..")||isAbsolute(rel))throw new Error("root leaves repository");let cursor=repo;for(const part of rel.split(sep).filter(Boolean)){cursor=join(cursor,part);try{if(lstatSync(cursor).isSymbolicLink())throw new Error("owned root ancestor is a symlink");}catch(error){if(error.code==="ENOENT")break;throw error;}}} +async function atomicWrite(path,bytes,mode=0o600){await mkdir(dirname(path),{recursive:true});const staging=join(dirname(path),`.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`);let h;try{h=await open(staging,"wx",mode);await h.writeFile(bytes);await h.sync();await h.close();h=undefined;await rename(staging,path);const fd=openSync(dirname(path),constants.O_RDONLY);try{fsyncSync(fd);}finally{closeSync(fd);}}finally{if(h)await h.close().catch(()=>{});await rm(staging,{force:true}).catch(()=>{});}} +function directorySync(path){const fd=openSync(path,constants.O_RDONLY);try{fsyncSync(fd);}finally{closeSync(fd);}} +async function exclusiveRecord(path,value,label){const bytes=`${JSON.stringify(value,null,2)}\n`;let handle,createdEntry;try{handle=await open(path,"wx",0o600);createdEntry=await handle.stat();await handle.chmod(0o600);await handle.writeFile(bytes);await handle.sync();await handle.close();handle=undefined;directorySync(dirname(path));return{path,bytes,dev:createdEntry.dev,ino:createdEntry.ino};}catch(error){if(handle)await handle.close().catch(()=>{});if(createdEntry)try{const current=await lstat(path);if(current.dev===createdEntry.dev&¤t.ino===createdEntry.ino)await rm(path);}catch{}if(error.code==="EEXIST")throw new Error(`${label} already exists; operator inspection required`);throw error;}} +async function requireExactRecord(record){const entry=await lstat(record.path);if(!entry.isFile()||entry.isSymbolicLink()||(entry.mode&0o777)!==0o600||(record.dev!==undefined&&(entry.dev!==record.dev||entry.ino!==record.ino)))throw new Error("owned lifecycle record is unsafe");if(await readFile(record.path,"utf8")!==record.bytes)throw new Error("owned lifecycle record changed; operator inspection required");const after=await lstat(record.path);if(after.dev!==entry.dev||after.ino!==entry.ino)throw new Error("owned lifecycle record changed; operator inspection required");return after;} +async function removeExactRecord(record){await requireExactRecord(record);await requireExactRecord(record);await rm(record.path);directorySync(dirname(record.path));} +async function replaceExactRecord(record,value){await requireExactRecord(record);const bytes=`${JSON.stringify(value,null,2)}\n`,staging=join(dirname(record.path),`.${basename(record.path)}.${randomBytes(12).toString("hex")}.tmp`);let handle;try{handle=await open(staging,"wx",0o600);await handle.chmod(0o600);await handle.writeFile(bytes);await handle.sync();await handle.close();handle=undefined;await requireExactRecord(record);await rename(staging,record.path);const entry=await lstat(record.path);directorySync(dirname(record.path));return{path:record.path,bytes,dev:entry.dev,ino:entry.ino};}finally{if(handle)await handle.close().catch(()=>{});await rm(staging,{force:true}).catch(()=>{});}} +function sameEntry(actual,expected){return actual.dev===expected.dev&&actual.ino===expected.ino;} +async function requirePathIdentity(path,expected,label){let entry;try{entry=await lstat(path);}catch{throw new Error(`${label} identity changed`);}if(entry.isSymbolicLink()||!sameEntry(entry,expected))throw new Error(`${label} identity changed`);return entry;} +async function acquireLifecycle(repo,operation){ + const lockPath=join(repo,".p1-manual-acceptance.lifecycle.lock"),artifacts=join(repo,".artifacts"),manualParent=join(artifacts,"manual-acceptance"),root=fixedManualRoot(repo); + noSymlinkExisting(repo,manualParent);await mkdir(manualParent,{recursive:true,mode:0o700});noSymlinkExisting(repo,manualParent); + const repoEntry=await lstat(repo),artifactsEntry=await lstat(artifacts),parentEntry=await lstat(manualParent); + if(!repoEntry.isDirectory()||!artifactsEntry.isDirectory()||!parentEntry.isDirectory())throw new Error("lifecycle namespace identity is unsafe"); + const lifecycleNonce=randomBytes(32).toString("hex"),record=await exclusiveRecord(lockPath,{schemaVersion:1,kind:"p1-manual-lifecycle",operation,lifecycleNonce,root,repositoryRoot:repo},"external lifecycle lock"),entry=await requireExactRecord(record); + return{...record,dev:entry.dev,ino:entry.ino,repoPath:repo,repoEntry,artifactsPath:artifacts,artifactsEntry,parentPath:manualParent,parentEntry,rootEntry:undefined}; +} +async function requireLifecycleContext(lifecycle,{root=false}={}){ + await requireExactRecord(lifecycle);await requirePathIdentity(lifecycle.repoPath,lifecycle.repoEntry,"repository root");await requirePathIdentity(lifecycle.artifactsPath,lifecycle.artifactsEntry,"artifact root");await requirePathIdentity(lifecycle.parentPath,lifecycle.parentEntry,"manual acceptance parent"); + if(root&&lifecycle.rootEntry)await requirePathIdentity(join(lifecycle.parentPath,"p1"),lifecycle.rootEntry,"manual acceptance root"); +} +async function bindLifecycleRoot(lifecycle,root){const entry=await lstat(root);if(!entry.isDirectory()||entry.isSymbolicLink())throw new Error("manual acceptance root identity is unsafe");lifecycle.rootEntry=entry;await requireLifecycleContext(lifecycle,{root:true});return entry;} +async function findRootByIdentity(repo,identity){ + const artifacts=join(repo,".artifacts");let count=0; + for(const parent of await readdir(artifacts,{withFileTypes:true})){if(++count>1024)throw new Error("manual cleanup search bound exceeded");if(!parent.isDirectory()||parent.isSymbolicLink())continue;const candidate=join(artifacts,parent.name,"p1");try{const entry=await lstat(candidate);if(entry.isDirectory()&&!entry.isSymbolicLink()&&sameEntry(entry,identity))return candidate;}catch{} + }return undefined; +} +async function cleanupFailedPrepare(repo,lifecycle){if(!lifecycle.rootEntry)return;const candidate=await findRootByIdentity(repo,lifecycle.rootEntry);if(!candidate)return;const entry=await lstat(candidate);if(!sameEntry(entry,lifecycle.rootEntry)||entry.isSymbolicLink())throw new Error("failed prepare root identity changed");await rm(candidate,{recursive:true});} +function legacySupervisorPath(root){return join(root,"installation/runtime/p1-backend-supervisor.mjs");} +const CONTROL_PORT=8792; +const PRELOAD_SOURCE=`import net from "node:net"; +import { createHash } from "node:crypto"; +import { closeSync, constants, fstatSync, openSync, readFileSync, readSync, realpathSync } from "node:fs"; +import { registerHooks } from "node:module"; +import { dirname, join, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +const HOST="127.0.0.1",PORT=8792,HTTP_PORT=8791,HEX=/^[0-9a-f]{64}$/; +const argv=process.argv.slice(2),noncePrefix="--p1-manual-nonce=",rootPrefix="--p1-root=",controlPrefix="--p1-control-nonce=",shaPrefix="--p1-entry-sha256=",devPrefix="--p1-entry-dev=",inoPrefix="--p1-entry-ino="; +const prefixes=[noncePrefix,rootPrefix,controlPrefix,shaPrefix,devPrefix,inoPrefix]; +if(argv.length!==6||argv.some((value,index)=>!value.startsWith(prefixes[index])))throw new Error("manual control identity arguments refused"); +const nonce=argv[0].slice(noncePrefix.length),root=argv[1].slice(rootPrefix.length),controlNonce=argv[2].slice(controlPrefix.length),entrySha=argv[3].slice(shaPrefix.length),entryDev=argv[4].slice(devPrefix.length),entryIno=argv[5].slice(inoPrefix.length); +if(!HEX.test(nonce)||!root.startsWith("/")||!HEX.test(controlNonce)||!HEX.test(entrySha)||!/^[0-9]+$/.test(entryDev)||!/^[0-9]+$/.test(entryIno))throw new Error("manual control identity refused"); +if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("manual distribution no-follow protection is unavailable"); +const entryStat=fstatSync(3),entrySource=readFileSync(3);if(!entryStat.isFile()||String(entryStat.dev)!==entryDev||String(entryStat.ino)!==entryIno||createHash("sha256").update(entrySource).digest("hex")!==entrySha)throw new Error("manual entrypoint FD identity refused"); +const manifestStat=fstatSync(4);if(!manifestStat.isFile()||manifestStat.size<1||manifestStat.size>8388608)throw new Error("manual distribution manifest FD identity refused"); +let manifest;try{manifest=JSON.parse(readFileSync(4));}catch{throw new Error("manual distribution manifest is malformed");} +const entryPath=realpathSync(process.argv[1]),distRoot=dirname(entryPath); +if(manifest?.schemaVersion!==1||manifest.kind!=="p1-manual-dist-manifest"||manifest.root!==distRoot||!manifest.files||typeof manifest.files!=="object"||Array.isArray(manifest.files))throw new Error("manual distribution manifest identity refused"); +const distEntries=Object.entries(manifest.files);if(distEntries.length<1||distEntries.length>20000)throw new Error("manual distribution manifest identity refused"); +const distBytes=new Map(); +for(const[rel,file]of distEntries){ + if(typeof rel!=="string"||!rel||rel.startsWith("/")||rel.startsWith("..")||rel.includes("\\\\")||rel.includes("/./")||rel.endsWith("/")||!Number.isSafeInteger(file?.size)||file.size<1||file.size>33554432||!HEX.test(file?.sha256??"")||!/^[0-9]+$/.test(String(file?.dev))||!/^[0-9]+$/.test(String(file?.ino)))throw new Error("manual distribution manifest is malformed"); + const path=join(distRoot,rel),fd=openSync(path,constants.O_RDONLY|constants.O_NOFOLLOW); + try{ + const before=fstatSync(fd); + if(!before.isFile()||before.nlink!==1||String(before.dev)!==String(file.dev)||String(before.ino)!==String(file.ino)||before.size!==file.size)throw new Error("manual distribution module identity changed"); + const bytes=Buffer.alloc(before.size);let offset=0; + while(offset{const address=ownedListener?.listening?ownedListener.address():undefined;return{listening:Boolean(ownedListener?.listening&&address&&address.address===HOST&&address.port===HTTP_PORT),host:address?.address,port:address?.port,generation:listenGeneration};}; +const originalListen=net.Server.prototype.listen;net.Server.prototype.listen=function(...args){const candidate=this;candidate.once("listening",()=>{const address=candidate.address();if(address&&address.address===HOST&&address.port===HTTP_PORT){ownedListener=candidate;listenGeneration++;}});candidate.on("close",()=>{if(ownedListener===candidate){ownedListener=undefined;if(state==="READY")state="LISTENER_CLOSED";}});return originalListen.apply(candidate,args);}; +const identity=()=>({status:state,pid:process.pid,nonce,controlNonce,root,control:{host:HOST,port:PORT},listener:listenerIdentity()}); +const control=net.createServer(socket=>{let bytes="";socket.setEncoding("utf8");socket.on("data",chunk=>{bytes+=chunk;if(bytes.length>2048)socket.destroy();});socket.on("end",()=>{let request;try{request=JSON.parse(bytes);}catch{socket.end();return;}if(request?.nonce!==controlNonce){socket.end();return;}if(request.action==="status"){socket.end(JSON.stringify(identity())+"\\n");return;}if(request.action==="ready"&&!stopping&&listenerIdentity().listening){state="READY";clearTimeout(watchdog);socket.end(JSON.stringify(identity())+"\\n");return;}if(request.action==="stop"&&!stopping){stopping=true;state="STOPPING";clearTimeout(watchdog);socket.end(JSON.stringify(identity())+"\\n",()=>{control.close();if(ownedListener?.listening)ownedListener.close(()=>process.exit(0));else setImmediate(()=>process.exit(0));});return;}socket.end(JSON.stringify(identity())+"\\n");});}); +await new Promise((resolve,reject)=>{control.once("error",reject);control.listen({host:HOST,port:PORT,exclusive:true},resolve);}); +const watchdog=setTimeout(()=>{if(state!=="STARTING")return;console.error("manual backend readiness watchdog expired");control.close(()=>process.exit(1));setTimeout(()=>process.exit(1),100).unref();},8000); +`; +const PRELOAD=`data:text/javascript;base64,${Buffer.from(PRELOAD_SOURCE,"utf8").toString("base64")}`; +async function controlRequest(control,payload){if(control?.host!==HOST||!Number.isSafeInteger(control?.port)||control.port<1||control.port>65535)throw new Error("backend control identity mismatch");return await new Promise((resolvePromise,reject)=>{const socket=net.createConnection({host:control.host,port:control.port}),timer=setTimeout(()=>socket.destroy(new Error("backend control timeout")),2000);let bytes="";socket.setEncoding("utf8");socket.on("connect",()=>socket.end(JSON.stringify(payload)));socket.on("data",chunk=>{bytes+=chunk;if(bytes.length>2048)socket.destroy(new Error("backend control response too large"));});socket.on("error",reject);socket.on("close",()=>{clearTimeout(timer);let value;try{value=JSON.parse(bytes);}catch{return reject(new Error("backend control response is malformed"));}resolvePromise(value);});});} + +function ownedValue(repo,root,nonce,{backendLog=null,entrypoint,distManifest=null,stage="PREPARING",createdAt=new Date().toISOString()}={}){return{schemaVersion:1,kind:"p1-manual-acceptance",nonce,repositoryRoot:repo,root,status:"PENDING",stage,createdAt,listener:{host:HOST,port:PORT,state:"stopped"},backendLog,entrypoint,distManifest,resources:[root,{kind:"fastify",host:HOST,port:PORT}]};} +function validEntrypoint(value,repo){return value?.path===join(repo,"backend/dist/server.js")&&Number.isSafeInteger(value.dev)&&Number.isSafeInteger(value.ino)&&Number.isSafeInteger(value.size)&&value.size>0&&HEX64.test(value.sha256??"");} +async function readBoundEntrypoint(repo){ + if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("production entrypoint no-follow protection is unavailable");const path=join(repo,"backend/dist/server.js");let handle; + try{handle=await open(path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat(),pathEntry=await lstat(path);if(!before.isFile()||before.nlink!==1||pathEntry.isSymbolicLink()||!pathEntry.isFile()||!sameEntry(before,pathEntry))throw new Error("production server identity is unsafe");if(before.size<1||before.size>33554432)throw new Error("production entrypoint is unbounded");const bytes=Buffer.alloc(before.size);let offset=0;while(offset{});throw error;} +} +async function requireEntrypointPathIdentity(entrypoint){const entry=await lstat(entrypoint.path);if(!entry.isFile()||entry.isSymbolicLink()||entry.nlink!==1||entry.dev!==entrypoint.dev||entry.ino!==entrypoint.ino||entry.size!==entrypoint.size)throw new Error("production entrypoint identity changed");const bytes=await readFile(entrypoint.path);if(bytes.length!==entrypoint.size||createHash("sha256").update(bytes).digest("hex")!==entrypoint.sha256)throw new Error("production entrypoint bytes changed");return entry;} +function validDistManifest(value,root){return value?.path===join(root,"installation","runtime","backend-dist.manifest.json")&&Number.isSafeInteger(value.dev)&&Number.isSafeInteger(value.ino)&&Number.isSafeInteger(value.size)&&value.size>0&&HEX64.test(value.sha256??"");} +function parseDistManifest(bytes,distRoot){let value;try{value=JSON.parse(bytes.toString("utf8"));}catch{throw new Error("production distribution manifest is malformed");}const files=value?.files;if(value?.schemaVersion!==1||value.kind!=="p1-manual-dist-manifest"||value.root!==distRoot||!files||typeof files!=="object"||Array.isArray(files))throw new Error("production distribution manifest is malformed");const entries=Object.entries(files);if(entries.length<1||entries.length>20000)throw new Error("production distribution manifest is malformed");for(const[rel,file]of entries){if(!/^[^./\\][^/\\]*(?:\/[^./\\][^/\\]*)*$/.test(rel)||!Number.isSafeInteger(file?.size)||file.size<1||file.size>33554432||!HEX64.test(file?.sha256??"")||!Number.isSafeInteger(file?.dev)||!Number.isSafeInteger(file?.ino))throw new Error("production distribution manifest is malformed");}return{value,files};} +async function buildDistManifest(repo){const dist=join(repo,"backend","dist"),files={};let count=0,total=0;async function walk(dir){for(const entry of await readdir(dir,{withFileTypes:true})){const path=join(dir,entry.name);if(entry.isSymbolicLink())throw new Error("production distribution contains a symlink");if(entry.isDirectory()){await walk(path);continue;}if(!entry.isFile())throw new Error("production distribution contains a nonregular entry");if(++count>20000)throw new Error("production distribution is unbounded");const rel=relative(dist,path).split(sep).join("/");let handle;try{handle=await open(path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat();if(!before.isFile()||before.nlink!==1||before.size<1||before.size>33554432)throw new Error("production distribution module is unsafe");total+=before.size;if(total>536870912)throw new Error("production distribution is unbounded");const bytes=Buffer.alloc(before.size);let offset=0;while(offset{});}}}await walk(dist);return{schemaVersion:1,kind:"p1-manual-dist-manifest",root:dist,files};} +async function readBoundDistManifest(repo,owned){ + if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("production distribution manifest no-follow protection is unavailable");const distManifest=owned.distManifest;let handle; + try{handle=await open(distManifest.path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat(),pathEntry=await lstat(distManifest.path);if(!before.isFile()||before.nlink!==1||pathEntry.isSymbolicLink()||!pathEntry.isFile()||!sameEntry(before,pathEntry)||before.dev!==distManifest.dev||before.ino!==distManifest.ino||before.size!==distManifest.size)throw new Error("production distribution manifest identity changed");if(before.size<1||before.size>8388608)throw new Error("production distribution manifest is unbounded");const bytes=Buffer.alloc(before.size);let offset=0;while(offset{});throw error;} +} +async function validateDistFiles(repo,files){const dist=join(repo,"backend","dist");for(const[rel,file]of Object.entries(files)){const path=join(dist,...rel.split("/"));let handle;try{handle=await open(path,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat(),pathEntry=await lstat(path);if(!before.isFile()||before.nlink!==1||pathEntry.isSymbolicLink()||!pathEntry.isFile()||!sameEntry(before,pathEntry)||before.dev!==file.dev||before.ino!==file.ino||before.size!==file.size)throw new Error("production distribution module identity changed");const bytes=Buffer.alloc(before.size);let offset=0;while(offset{});}}} + +export async function readManualOwnership({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),root=fixedManualRoot(repo);noSymlinkExisting(repo,root);let rootEntry,ownershipEntry;try{rootEntry=await lstat(root);ownershipEntry=await lstat(join(root,"ownership.json"));}catch{throw new Error("manual ownership is missing");}if(!rootEntry.isDirectory()||rootEntry.isSymbolicLink()||await realpath(root)!==root||!ownershipEntry.isFile()||ownershipEntry.isSymbolicLink())throw new Error("manual ownership is unsafe");let value;try{value=JSON.parse(await readFile(join(root,"ownership.json"),"utf8"));}catch{throw new Error("manual ownership is malformed");}const baseValid=value.schemaVersion===1&&value.kind==="p1-manual-acceptance"&&HEX64.test(value.nonce??"")&&value.repositoryRoot===repo&&value.root===root&&value.status==="PENDING"&&["PREPARING","READY"].includes(value.stage)&&value.listener?.host===HOST&&value.listener?.port===PORT&&value.listener?.state==="stopped"&&typeof value.createdAt==="string"&&validEntrypoint(value.entrypoint,repo)&&validDistManifest(value.distManifest,root)&&JSON.stringify(value.resources)===JSON.stringify([root,{kind:"fastify",host:HOST,port:PORT}]);const readyLog=value.backendLog?.path===join(root,"logs/backend.log")&&Number.isSafeInteger(value.backendLog?.dev)&&Number.isSafeInteger(value.backendLog?.ino);if(!baseValid||(value.stage==="READY"?!readyLog:value.backendLog!==null))throw new Error("manual ownership identity mismatch");return value;} +async function run(executable,argv,options={}){return await exec(executable,argv,{...options,maxBuffer:2*1024*1024,encoding:"utf8"});} +function descriptor(id,source){return{workspace:{schema_version:3,id,name:`P1 ${id}`,language:"en"},dwh:{engine:"postgres",database:"postgres",schema:"public",supported_transports:["postgres_direct"]},semantic_index:{vector_store:{engine:"qdrant",collection:id,dimensions:1024,distance:"cosine"},embedding:{provider:"ollama_internal",model:"qwen3-embedding:0.6b",dimensions:1024}},llm_policy:{allowed:["zai/glm-5.2"]},evidence:{source,policy:{max_chunk_chars:4000,retain_published_generations:3}}};} +function descriptors(){return[descriptor("p1-filesystem",{type:"filesystem",uri:"workspace-content/p1-filesystem/evidence",patterns:["**/*.md"],max_bytes:10485760}),descriptor("p1-http",{type:"http",uris:["https://evidence.example.test/guide.md"],authentication:"signed_urls_file",connect_timeout_ms:1250,read_timeout_ms:30001,max_bytes:12345,max_redirects:2,allow_private_hosts:false,max_cache_bytes:67890}),descriptor("p1-s3",{type:"s3",uri:"s3://p1-evidence/published/",endpoint_url:"https://s3.example.test/",region:"eu-west-1",credentials:"static_files",trusted_endpoint:true,allow_private_endpoint:false,allow_insecure_endpoint:false,max_bytes:12345,max_objects:33,max_pages:4,page_size:5})];} +function quote(value){return `'${String(value).replaceAll("'",`'"'"'`)}'`;} +async function checkPrerequisites(repo){for(const path of ["scripts/p1-acceptance.sh","scripts/test-p1-acceptance.sh","backend/scripts/p1-acceptance.mjs","backend/dist/server.js"]){try{await access(join(repo,path));}catch{throw new Error(`Task 8 prerequisite is missing: ${path}`);}}for(const command of ["node","npm","git","curl","unzip","zipinfo","lsof","python3"]){try{await run(command,[command==="unzip"||command==="lsof"?"-v":command==="zipinfo"?"-h":"--version"]);}catch{throw new Error(`missing prerequisite: ${command}`);}}const tht=join(repo,"harness",".venv","bin","tht");try{await access(tht,constants.X_OK);}catch{throw new Error("missing prerequisite: harness/.venv/bin/tht");}} +async function initializeGit(root){await run("git",["init","--bare","--initial-branch=main",join(root,"remote.git")],{cwd:root});await run("git",["clone",join(root,"remote.git"),join(root,"author")],{cwd:root});for(const [key,value]of [["user.name","P1 Manual Curator"],["user.email","p1-manual@example.invalid"]])await run("git",["config",key,value],{cwd:join(root,"author")});const evidence=join(root,"author","workspace-content","p1-filesystem","evidence");await mkdir(join(evidence,"domain"),{recursive:true});await writeFile(join(evidence,"guide.md"),"# P1 manually curated Evidence\n");await writeFile(join(evidence,"domain","table.md"),"# P1 curated table\n");await run("git",["add","workspace-content"],{cwd:join(root,"author")});await run("git",["commit","-m","Bootstrap P1 manual Evidence"],{cwd:join(root,"author")});await run("git",["push","origin","main"],{cwd:join(root,"author")});} +function requestFixtures(items){const result={"status.json":{method:"GET",path:"/workspace-registry/status"},"pull.json":{method:"POST",path:"/workspace-registry/pull"}};for(const workspace of items){const id=workspace.workspace.id;result[`validate-${id}.json`]={workspace};result[`publish-${id}.json`]={action:"create",workspace};result[`read-${id}.json`]={method:"GET",path:`/workspaces/${id}`};result[`export-${id}.json`]={method:"GET",path:`/workspaces/${id}/export`};}Object.assign(result,{"invalid-absolute.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"/etc"}}}},"invalid-traversal.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-filesystem/evidence/../../p1-s3/evidence"}}}},"invalid-cross-workspace.json":{workspace:{...items[0],evidence:{...items[0].evidence,source:{...items[0].evidence.source,uri:"workspace-content/p1-s3/evidence"}}}},"invalid-protocol.json":{workspace:{...items[1],evidence:{...items[1].evidence,source:{...items[1].evidence.source,uris:["file:///etc/passwd"]}}}},"invalid-credential.json":{workspace:{...items[2],evidence:{...items[2].evidence,source:{...items[2].evidence.source,access_key:"CANARY-MUST-BE-REJECTED"}}}}});return result;} +function curlGet(url,output){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;} +function curlPost(url,output,body){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(body)} --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;} +function curlPostEmpty(url,output){return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`;} +function publishCurl(root,id,previousResponse){const descriptor=join(root,"fixtures/descriptors",`${id}.json`),body=join(root,"requests",`publish-${id}.concrete.json`),response=join(root,"responses",`publish-${id}.json`);return `#!/usr/bin/env bash +set -euo pipefail +node --input-type=module - ${quote(previousResponse)} ${quote(descriptor)} ${quote(body)} <<'NODE' +import { open, readFile, rename, stat } from "node:fs/promises";import { basename, dirname, join } from "node:path";import { randomBytes } from "node:crypto"; +const [priorPath,descriptorPath,output]=process.argv.slice(2);const bounded=async(path)=>{let s;try{s=await stat(path);}catch{throw Error("required saved response is missing");}if(!s.isFile()||s.size<2||s.size>1048576)throw Error("saved response is unbounded");let value;try{value=JSON.parse(await readFile(path,"utf8"));}catch{throw Error("saved response is malformed JSON");}return value;}; +const prior=await bounded(priorPath),workspace=await bounded(descriptorPath);const base=prior.head??prior.revision?.commit;if(!/^[0-9a-f]{40}$/.test(base??""))throw Error("saved response has no valid current base commit");const bytes=JSON.stringify({action:"create",workspace,baseCommit:base},null,2)+"\\n",tmp=join(dirname(output),"."+basename(output)+"."+randomBytes(8).toString("hex")+".tmp");const h=await open(tmp,"wx",0o600);try{await h.writeFile(bytes);await h.sync();}finally{await h.close();}await rename(tmp,output); +NODE +curl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(body)} --output ${quote(response)} --write-out 'HTTP %{http_code}\n' 'http://127.0.0.1:8791/workspaces/publish' +`;} +function httpCommands(root){const base="http://127.0.0.1:8791",entries=[];entries.push(["http-01-status.sh",curlGet(`${base}/workspace-registry/status`,join(root,"responses/status.json"))]);let n=2;for(const id of ["p1-filesystem","p1-http","p1-s3"])entries.push([`http-0${n++}-validate-${id}.sh`,curlPost(`${base}/workspaces/validate`,join(root,"responses",`validate-${id}.json`),join(root,"requests",`validate-${id}.json`))]);let prior=join(root,"responses/status.json");for(const id of ["p1-filesystem","p1-http","p1-s3"]){entries.push([`http-0${n++}-publish-${id}.sh`,publishCurl(root,id,prior)]);prior=join(root,"responses",`publish-${id}.json`);}entries.push([`http-0${n++}-pull.sh`,curlPostEmpty(`${base}/workspace-registry/pull`,join(root,"responses/pull.json"))]);for(const id of ["p1-filesystem","p1-http","p1-s3"])entries.push([`http-${String(n++).padStart(2,"0")}-read-${id}.sh`,curlGet(`${base}/workspaces/${id}`,join(root,"responses",`read-${id}.json`))]);for(const id of ["p1-filesystem","p1-http","p1-s3"])entries.push([`http-${String(n++).padStart(2,"0")}-export-${id}.sh`,curlGet(`${base}/workspaces/${id}/export`,join(root,"exports/raw",`${id}.zip`))]);for(const kind of ["absolute","traversal","cross-workspace","protocol","credential"])entries.push([`http-${String(n++).padStart(2,"0")}-invalid-${kind}.sh`,curlPost(`${base}/workspaces/validate`,join(root,"responses",`invalid-${kind}.json`),join(root,"requests",`invalid-${kind}.json`))]);return entries;} +function renderCommand(repo,root,n){const output=join(root,"rendered",`runtime-${n}.yaml`),response=join(root,"responses","read-p1-filesystem.json"),published=join(root,"responses","pull.json"),snapshots=join(root,"installation","registry","snapshots"),checkout=join(root,"installation","registry","repo");return `#!/usr/bin/env bash +set -euo pipefail +repo=${quote(repo)} +root=${quote(root)} +set -a +. ${quote(join(root,"installation","bindings.env"))} +set +a +node --input-type=module - "$root" ${quote(response)} ${quote(published)} ${quote(snapshots)} ${quote(checkout)} ${quote(output)} "$repo/backend/scripts/p1-render-snapshot.mjs" <<'NODE' +import { createHash } from "node:crypto"; +import { readFile, realpath, stat } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { spawnSync } from "node:child_process"; +const [root,readPath,publishPath,snapshots,checkout,output,renderer]=process.argv.slice(2); +const HEX40=/^[0-9a-f]{40}$/,HEX64=/^[0-9a-f]{64}$/,REVISION_KEYS=["blob","commit","id","snapshotPath"]; +const bounded=async(path,label="saved response")=>{let s;try{s=await stat(path);}catch{throw new Error(label+" is missing or unbounded");}if(!s.isFile()||s.size<2||s.size>1048576)throw new Error(label+" is missing or unbounded");let v;try{v=JSON.parse(await readFile(path,"utf8"));}catch{throw new Error(label+" is malformed JSON");}return v;}; +const boundedBytes=async(path)=>{let s;try{s=await stat(path);}catch{throw new Error("saved snapshot is missing or unbounded");}if(!s.isFile()||s.size<2||s.size>1048576)throw new Error("saved snapshot is missing or unbounded");return await readFile(path);}; +const read=await bounded(readPath),published=await bounded(publishPath);const revision=read?.revision,commit=revision?.commit,snapshot=revision?.snapshotPath,publishedCommit=published?.head??published?.revision?.commit; +if(!HEX40.test(commit??"")||commit!==publishedCommit)throw new Error("saved read/publish revisions differ"); +if(typeof snapshot!=="string"||!isAbsolute(snapshot))throw new Error("snapshot path is not absolute");const canonical=await realpath(snapshot);const rel=relative(snapshots,canonical);if(rel.startsWith("..")||isAbsolute(rel)||dirname(canonical)!==resolve(snapshots,commit))throw new Error("snapshot escapes owned commit root"); +const id=basename(canonical).slice(0,-".yaml".length);if(!/^[a-z][a-z0-9-]{2,62}$/.test(id))throw new Error("snapshot workspace identity is invalid"); +const git=spawnSync("git",["-C",checkout,"rev-parse","HEAD"],{encoding:"utf8"});if(git.status!==0||git.stdout.trim()!==commit)throw new Error("saved revision differs from installed Git commit"); +const manifest=await bounded(join(snapshots,commit,"snapshot.json"),"snapshot manifest");const files=manifest?.files,revisions=manifest?.revisions; +if(manifest?.head!==commit||!files||typeof files!=="object"||Array.isArray(files))throw new Error("snapshot manifest identity is invalid"); +const expected=files[id+".yaml"];if(!HEX64.test(expected??""))throw new Error("snapshot manifest digest is invalid"); +const snapshotBytes=await boundedBytes(canonical);if(createHash("sha256").update(snapshotBytes).digest("hex")!==expected)throw new Error("snapshot bytes differ from manifest digest"); +const entry=Array.isArray(revisions)?revisions.find(candidate=>candidate?.id===id):undefined; +const exactEntry=entry&&typeof entry==="object"&&!Array.isArray(entry)&&Object.keys(entry).sort().every((key,index)=>key===REVISION_KEYS[index])&&Object.keys(entry).length===REVISION_KEYS.length; +if(!exactEntry||entry.id!==id||entry.commit!==commit||typeof entry.blob!=="string"||!HEX40.test(entry.blob)||entry.snapshotPath!==canonical)throw new Error("snapshot manifest revision is invalid"); +if(!HEX40.test(revision?.blob??"")||revision.blob!==entry.blob)throw new Error("saved revision blob differs from snapshot manifest"); +const blobCheck=spawnSync("git",["-C",checkout,"rev-parse",commit+":workspaces/"+id+".yaml"],{encoding:"utf8"}); +if(blobCheck.status!==0||blobCheck.stdout.trim()!==entry.blob)throw new Error("snapshot blob differs from installed Git commit"); +const hashObject=spawnSync("git",["hash-object","--stdin"],{input:snapshotBytes,encoding:"utf8"}); +if(hashObject.status!==0||hashObject.stdout.trim()!==entry.blob)throw new Error("snapshot bytes differ from Git blob"); +const child=spawnSync(process.execPath,[renderer,"--ownership",resolve(root,"ownership.json"),"--snapshot",canonical,"--output",output,"--snapshot-sha256",expected],{stdio:"inherit",env:process.env});if(child.status!==0)process.exit(child.status??1); +NODE +`;} +function guide(repo,root){const base=`http://${HOST}:${PORT}`;return `# P1 manual configuration walkthrough + +Status: **PENDING**. The reviewer, not this helper, performs and judges every step. Never inspect raw secret-file contents. Every lifecycle action uses the stable repository-root \`.p1-manual-acceptance.lifecycle.lock\`; successful prepare has advanced its ownership-first recovery record from \`PREPARING\` to \`READY\`. + +1. Inspect \`${root}/ownership.json\`, including the bound production entrypoint identity and the complete \`backend/dist\` module manifest identity, the pre-publication Evidence tree under \`author/workspace-content/p1-filesystem/evidence\`, descriptor fixtures, and binding **paths and modes** in \`installation/bindings.env\`. +2. Run \`${repo}/scripts/p1-manual-acceptance.sh serve\`; verify one production Node PID owns both \`${HOST}:${PORT}\` and its authenticated \`${HOST}:${CONTROL_PORT}\` control listener (for example, use \`lsof -nP -iTCP:${PORT} -sTCP:LISTEN\` and repeat for port ${CONTROL_PORT}). Serve executes the ownership-bound production entrypoint and complete verified \`backend/dist\` module graph from opened no-follow descriptors and publishes \`RUNNING\` only after the same authenticated child acknowledges its owned HTTP listener and passes bounded health checks. +3. Personally run each concrete \`commands/http-01-*.sh\` through \`commands/http-14-*.sh\` script, one at a time in numeric order: real curl status → three validates → three sequential publishes → pull → three reads → three exports against \`${base}\`. Each script saves the exact JSON response under \`responses/\` or ZIP bytes under \`exports/raw/\`; each publish derives its current base from the preceding bounded saved response. Do not advance on a non-2xx response. +4. Only after publish, run \`commands/git-inspect.sh \`: inspect \`git log\`, \`git ls-tree\`, \`git show :workspaces/.yaml\`, and \`git show :workspace-content//evidence/...\` at that same commit. +5. Inspect generated \`workspace-docs\`, the immutable commit-addressed descriptor snapshot, and its \`snapshot.json\` manifest. +6. Run \`commands/extract-export.sh exports/raw/p1-filesystem.zip exports/extracted/p1-filesystem p1-filesystem\`, then the equivalent exact commands for \`p1-http\` and \`p1-s3\`; verify each manifest and descriptor identity, hashes, and absence of Evidence bytes and secret/canary material. +7. After saving \`responses/read-p1-filesystem.json\` and the final API/Git head in \`responses/pull.json\`, run \`commands/render-1.sh\`, \`commands/render-2.sh\`, then \`commands/diff-rendered.sh\`. The render commands bind the snapshot bytes to the commit\'s \`snapshot.json\` digest and Git blob identity; the renderer revalidates that digest and renders only the verified bytes through one opened no-follow \`rendered\` directory identity, refusing an ancestor swap. +8. Inspect runtime identity, absolute reserved filesystem root, Evidence limits, and policy in the rendered YAML; do not inspect secret contents. +9. Personally execute \`${repo}/harness/.venv/bin/tht config check -c ${root}/rendered/runtime-1.yaml\` and the same command for \`runtime-2.yaml\` (or run \`commands/config-check.sh\`). +10. Personally run \`commands/http-15-*.sh\` through \`commands/http-19-*.sh\` to submit the invalid absolute, Evidence-URI traversal, cross-workspace, protocol, and credential validation requests; verify safe rejection, no Git/snapshot mutation, and no rejected canary outside the request fixture. +11. Run \`commands/secret-scan.sh\`; it excludes only the direct \`fixture-secrets\` payload directory, scans bounded filesystem content and name/path bytes, discovers every bounded arbitrary \`.git\` repository plus the owned bare remote, and checks loose-ref names plus raw bounded bytes from every blob, commit, tree, and tag object, including unreachable objects. Findings and operational errors redact secret-bearing paths and values. +12. Run \`commands/absence-check.sh\`; confirm no file or directory represents preprocessing, Evidence materialization (including \`artifacts/evidence\`), embedding, Qdrant, ACTIVE, or retention state. +13. Run \`${repo}/scripts/p1-manual-acceptance.sh stop\`; confirm \`backend.pid\` and both listeners on ports ${PORT} and ${CONTROL_PORT} are gone. +14. Create \`${root}/VERDICT.md\` yourself with reviewer, UTC time, every checklist result, observations, and exactly either \`manual acceptance: PASS\` or \`manual acceptance: FAIL\`. + +Preserve a failed lab by stopping it and leaving the owned root in place. Only \`cleanup\` removes this exact stopped lab. +`;} +function extractCommand(repo,root){return `#!/usr/bin/env bash +set -euo pipefail +zip=\${1:?zip required}; out=\${2:?new output required}; expected=\${3:?expected workspace id required} +python3 - "$zip" "$out" "$expected" ${quote(root)} ${quote(join(repo,"backend/package.json"))} <<'PY' +import hashlib +import io +import json +import os +import re +import secrets +import stat +import subprocess +import sys +import zipfile + +zip_path, output_path, expected, root, package_json = sys.argv[1:] +allowed = {"p1-filesystem", "p1-http", "p1-s3"} +required = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"] +base = os.path.join(root, "exports", "extracted") +base_fd = None +archive_fd = None +stage_fd = None +archive_stage = None +extract_stage = None +published = False + + +def fail(message): + raise RuntimeError(message) + + +def exact(value, keys): + return isinstance(value, dict) and set(value) == set(keys) + + +def write_all(fd, data): + view = memoryview(data) + while view: + written = os.write(fd, view) + if written <= 0: + fail("anchored extraction write failed") + view = view[written:] + + +def read_exact_fd(fd, expected_size, limit, label): + if expected_size < 1 or expected_size > limit: + fail(label + " is unbounded") + chunks = [] + remaining = expected_size + while remaining: + chunk = os.read(fd, min(1024 * 1024, remaining)) + if not chunk: + fail(label + " changed while staging") + chunks.append(chunk) + remaining -= len(chunk) + if os.read(fd, 1): + fail(label + " changed while staging") + return b"".join(chunks) + + +def require_base_identity(): + try: + current = os.stat(base, follow_symlinks=False) + except OSError: + fail("owned extraction root identity changed") + if (not stat.S_ISDIR(current.st_mode) or current.st_dev != base_identity.st_dev + or current.st_ino != base_identity.st_ino or os.path.realpath(base) != base): + fail("owned extraction root identity changed") + + +def remove_anchored_directory(name): + child_fd = None + try: + child_fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd) + for entry in os.listdir(child_fd): + if entry not in required: + fail("anchored extraction cleanup found an unexpected entry") + os.unlink(entry, dir_fd=child_fd) + os.fsync(child_fd) + except FileNotFoundError: + return + finally: + if child_fd is not None: + os.close(child_fd) + os.rmdir(name, dir_fd=base_fd) + os.fsync(base_fd) + + +try: + for flag in ("O_DIRECTORY", "O_NOFOLLOW"): + if not hasattr(os, flag): + fail("anchored extraction is unavailable on this platform") + if expected not in allowed: + fail("expected workspace identity is invalid") + if os.path.realpath(root) != root or os.path.dirname(output_path) != base: + fail("unsafe owned extraction root or output path") + output_name = os.path.basename(output_path) + if not output_name or output_name.startswith(".") or os.sep in output_name: + fail("unsafe output path") + + base_fd = os.open(base, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + base_identity = os.fstat(base_fd) + if not stat.S_ISDIR(base_identity.st_mode): + fail("unsafe owned extraction root") + require_base_identity() + try: + os.stat(output_name, dir_fd=base_fd, follow_symlinks=False) + fail("unsafe output path") + except FileNotFoundError: + pass + + # Open the caller's source exactly once, then consume only an owned staged copy. + source_fd = os.open(zip_path, os.O_RDONLY | os.O_NOFOLLOW) + try: + source_identity = os.fstat(source_fd) + if not stat.S_ISREG(source_identity.st_mode): + fail("source ZIP is unsafe") + source_bytes = read_exact_fd(source_fd, source_identity.st_size, 33554432, "source ZIP") + source_after = os.fstat(source_fd) + if (source_after.st_dev, source_after.st_ino, source_after.st_size) != (source_identity.st_dev, source_identity.st_ino, source_identity.st_size): + fail("source ZIP changed during staging") + finally: + os.close(source_fd) + archive_sha = hashlib.sha256(source_bytes).digest() + + archive_stage = ".zip-stage-" + secrets.token_hex(16) + ".zip" + archive_fd = os.open(archive_stage, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=base_fd) + write_all(archive_fd, source_bytes) + os.fsync(archive_fd) + del source_bytes + os.lseek(archive_fd, 0, os.SEEK_SET) + staged_bytes = read_exact_fd(archive_fd, os.fstat(archive_fd).st_size, 33554432, "staged archive") + if hashlib.sha256(staged_bytes).digest() != archive_sha: + fail("staged archive SHA mismatch") + + with zipfile.ZipFile(io.BytesIO(staged_bytes), "r") as archive: + infos = archive.infolist() + names = [entry.filename for entry in infos] + if len(names) != 4 or len(set(names)) != 4 or set(names) != set(required): + fail("unsafe-zip entries") + for entry in infos: + mode = (entry.external_attr >> 16) & 0xFFFF + if not stat.S_ISREG(mode) or entry.flag_bits & 1: + fail("ZIP contains a symlink or nonregular entry") + if entry.file_size < 1 or entry.file_size > 10485760: + fail("extracted file is unsafe") + payloads = {name: archive.read(name) for name in required} + if any(len(payloads[entry.filename]) != entry.file_size for entry in infos): + fail("extracted file size mismatch") + + # Exercise the documented unzip prerequisite against the exact staged descriptor, not a path. + listing = subprocess.run( + ["unzip", "-Z1", "/dev/fd/" + str(archive_fd)], pass_fds=(archive_fd,), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=20, check=False, + ) + if listing.returncode != 0 or listing.stdout.decode("utf8", "strict").splitlines() != names: + fail("unsafe-zip entries") + os.lseek(archive_fd, 0, os.SEEK_SET) + revalidated = read_exact_fd(archive_fd, len(staged_bytes), 33554432, "staged archive") + if hashlib.sha256(revalidated).digest() != archive_sha or revalidated != staged_bytes: + fail("staged archive SHA mismatch") + require_base_identity() + + randomized = re.compile(br"(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}") + fixed = b"-".join([b"CANARY", b"MUST", b"BE", b"REJECTED"]) + for data in payloads.values(): + if b"P1 manually curated Evidence" in data or b"P1 curated table" in data or randomized.search(data) or fixed in data: + fail("export contains Evidence or secret canary bytes") + + try: + manifest = json.loads(payloads["manifest.json"].decode("utf8")) + except Exception: + fail("export manifest schema mismatch") + hashed = required[1:] + files = manifest.get("files") if isinstance(manifest, dict) else None + if (not exact(manifest, ["schema_version", "workspace_id", "files"]) + or manifest.get("schema_version") != 1 or manifest.get("workspace_id") != expected + or not exact(files, hashed) + or any(not isinstance(files[name], str) or not re.fullmatch(r"[0-9a-f]{64}", files[name]) for name in hashed)): + fail("export manifest workspace identity or schema mismatch") + for name in hashed: + if hashlib.sha256(payloads[name]).hexdigest() != files[name]: + fail("manifest hash mismatch") + + yaml_helper = 'const fs=require("node:fs"),{createRequire}=require("node:module");try{const YAML=createRequire(process.argv[1])("yaml"),v=YAML.parse(fs.readFileSync(0,"utf8"));process.stdout.write(JSON.stringify(v?.workspace?.id??null));}catch{process.exit(2)}' + parsed = subprocess.run(["node", "-e", yaml_helper, package_json], input=payloads["workspace.yaml"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10, check=False) + try: + descriptor_id = json.loads(parsed.stdout.decode("utf8")) if parsed.returncode == 0 else None + except Exception: + descriptor_id = None + if descriptor_id != expected: + fail("export descriptor workspace identity mismatch") + + extract_stage = ".extract-stage-" + secrets.token_hex(16) + os.mkdir(extract_stage, 0o700, dir_fd=base_fd) + stage_fd = os.open(extract_stage, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=base_fd) + for name in required: + fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=stage_fd) + try: + write_all(fd, payloads[name]) + os.fsync(fd) + finally: + os.close(fd) + os.fsync(stage_fd) + os.close(stage_fd) + stage_fd = None + require_base_identity() + try: + os.stat(output_name, dir_fd=base_fd, follow_symlinks=False) + fail("unsafe output path") + except FileNotFoundError: + pass + os.rename(extract_stage, output_name, src_dir_fd=base_fd, dst_dir_fd=base_fd) + extract_stage = None + published = True + os.fsync(base_fd) + require_base_identity() +except Exception as error: + if isinstance(error, RuntimeError): + print(str(error), file=sys.stderr) + else: + print("extraction operational failure (details redacted)", file=sys.stderr) + sys.exit_code = 1 +finally: + if stage_fd is not None: + os.close(stage_fd) + if extract_stage is not None and base_fd is not None: + try: + remove_anchored_directory(extract_stage) + except Exception: + sys.exit_code = 1 + if published and getattr(sys, "exit_code", 0) and base_fd is not None: + try: + remove_anchored_directory(output_name) + except Exception: + pass + if archive_fd is not None: + os.close(archive_fd) + if archive_stage is not None and base_fd is not None: + try: + os.unlink(archive_stage, dir_fd=base_fd) + os.fsync(base_fd) + except FileNotFoundError: + pass + if base_fd is not None: + os.close(base_fd) +if getattr(sys, "exit_code", 0): + raise SystemExit(sys.exit_code) +PY +`;} +async function writeCommands(repo,root){const commands=join(root,"commands");for(const [name,body]of [...httpCommands(root),["render-1.sh",renderCommand(repo,root,1)],["render-2.sh",renderCommand(repo,root,2)],["diff-rendered.sh",`#!/bin/sh\nset -eu\ndiff -u ${quote(join(root,"rendered/runtime-1.yaml"))} ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["config-check.sh",`#!/bin/sh\nset -eu\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-1.yaml"))}\n${quote(join(repo,"harness/.venv/bin/tht"))} config check -c ${quote(join(root,"rendered/runtime-2.yaml"))}\n`],["git-inspect.sh",`#!/bin/sh\nset -eu\ncommit=\${1:?published commit required}\ncase "$commit" in *[!0-9a-f]*|'') exit 2;; esac\n[ \${#commit} -eq 40 ] || exit 2\ngit -C ${quote(join(root,"installation/registry/repo"))} log --oneline --decorate -10 "$commit"\ngit -C ${quote(join(root,"installation/registry/repo"))} ls-tree -r "$commit" -- workspaces workspace-content\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspaces/p1-filesystem.yaml"\ngit -C ${quote(join(root,"installation/registry/repo"))} show "$commit:workspace-content/p1-filesystem/evidence/guide.md"\n`],["extract-export.sh",extractCommand(repo,root)],["secret-scan.sh",`#!/usr/bin/env bash +set -euo pipefail +root=${quote(root)} +node --input-type=module - "$root" <<'NODE' +import { spawnSync } from "node:child_process";import { constants } from "node:fs";import { lstat, open, readdir } from "node:fs/promises";import { join, relative } from "node:path"; +const root=process.argv[2],randomized=/(?:DWH|SIGNED|ACCESS|SECRET|SESSION)-[0-9a-f]{32}/,fixed=["CANARY","MUST","BE","REJECTED"].join("-");let found=false,filesystemCount=0,filesystemTotal=0,nameTotal=0;const gitDirs=new Set([join(root,"remote.git")]);const containsCanary=value=>randomized.test(value)||value.includes(fixed);const finding=kind=>{console.error("secret canary found in "+kind+" (path and value redacted)");found=true;}; +async function maybeGitDir(path){try{const head=await lstat(join(path,"HEAD")),objects=await lstat(join(path,"objects"));if(head.isFile()&&objects.isDirectory()&&!head.isSymbolicLink()&&!objects.isSymbolicLink())gitDirs.add(path);}catch{}} +async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){if(++filesystemCount>200000)throw Error("bound");const child=join(path,entry.name),rel=relative(root,child),nameBytes=Buffer.from(entry.name),pathBytes=Buffer.from(rel);if(nameBytes.length>255||pathBytes.length>4096||(nameTotal+=nameBytes.length+pathBytes.length)>67108864)throw Error("bound");if(containsCanary(nameBytes.toString("latin1"))||containsCanary(pathBytes.toString("latin1")))finding("filesystem name bytes");if(entry.isSymbolicLink()){console.error("unsafe symlink during secret scan (path redacted)");found=true;continue;}if(entry.isDirectory()){if(entry.name===".git")await maybeGitDir(child);if(rel==="fixture-secrets")continue;await walk(child);continue;}if(!entry.isFile())throw Error("unsupported");let handle;try{handle=await open(child,constants.O_RDONLY|constants.O_NOFOLLOW);const before=await handle.stat();if(!before.isFile()||before.size>33554432)throw Error("bound");filesystemTotal+=before.size;if(filesystemTotal>1073741824)throw Error("bound");const bytes=await handle.readFile(),after=await handle.stat();if(bytes.length!==before.size||after.dev!==before.dev||after.ino!==before.ino||after.size!==before.size)throw Error("changed");const value=bytes.toString("latin1"),allowedRequest=rel==="requests/invalid-credential.json"&&value.includes(fixed)&&!randomized.test(value);if(containsCanary(value)&&!allowedRequest)finding("filesystem bytes");}finally{if(handle)await handle.close();}}} +function gitRun(args,options={}){const result=spawnSync("git",args,{...options,stdio:[options.input===undefined?"ignore":"pipe","pipe","pipe"]});if(result.error||result.status!==0)throw Error("git");return result.stdout;} +function scanGit(gitDir){const listing=gitRun(["--git-dir",gitDir,"cat-file","--batch-all-objects","--unordered","--batch-check=%(objectname) %(objecttype) %(objectsize)"],{encoding:"utf8",maxBuffer:16*1024*1024}).trim(),objects=listing?listing.split("\\n"):[];if(objects.length>100000)throw Error("bound");let total=0;for(const line of objects){const match=line.match(/^([0-9a-f]{40,64}) (blob|commit|tree|tag) (\\d+)$/);if(!match)throw Error("git");const[,oid,type,sizeText]=match,size=Number(sizeText);total+=size;if(!Number.isSafeInteger(size)||size>33554432||total>536870912)throw Error("bound");const raw=gitRun(["--git-dir",gitDir,"cat-file",type,oid],{maxBuffer:Math.max(1024,size+1)});if(raw.length!==size)throw Error("changed");if(containsCanary(raw.toString("latin1")))finding(type==="blob"?"Git blob":"Git object");}} +try{await walk(root);for(const gitDir of gitDirs)scanGit(gitDir);if(found)process.exitCode=1;else console.log("no fixture secret canary outside fixture-secrets or in any bounded Git object");}catch{console.error("secret scan operational failure (details redacted)");process.exitCode=2;} +NODE +`],["absence-check.sh",`#!/usr/bin/env bash +set -euo pipefail +root=${quote(root)} +node --input-type=module - "$root" <<'NODE' +import { readdir } from "node:fs/promises";import { join,relative } from "node:path"; +const root=process.argv[2];let count=0,rejected=false;const artifactName=name=>name.toUpperCase()==="ACTIVE"||/(?:materiali[sz](?:e|ed|ation)|preprocess|embedding|qdrant|retention)/i.test(name); +async function walk(path){for(const entry of await readdir(path,{withFileTypes:true})){if(++count>200000)throw Error("bound");const child=join(path,entry.name),parts=relative(root,child).split("/");if(parts.some((part,index)=>part==="artifacts"&&parts[index+1]==="evidence")||artifactName(entry.name))rejected=true;if(entry.isSymbolicLink())continue;if(entry.isDirectory()&&entry.name!==".git")await walk(child);}} +try{await walk(root);if(rejected){console.error("unexpected out-of-scope P2+ artifact (path redacted)");process.exitCode=1;}else console.log("no out-of-scope runtime artifact found");}catch{console.error("out-of-scope artifact check failed safely (details redacted)");process.exitCode=2;} +NODE +`]]){await atomicWrite(join(commands,name),body,0o700);await chmod(join(commands,name),0o700);}} +export async function prepareManual(options={}){ + const unknown=Object.keys(options).filter(key=>!["repositoryRoot","skipBuild"].includes(key));if(unknown.length)throw new Error(`unknown or automated-run prepare input: ${unknown.join(", ")}`); + const{repositoryRoot=defaultRepositoryRoot,skipBuild=false}=options,repo=realpathSync(repositoryRoot),root=fixedManualRoot(repo),lifecycle=await acquireLifecycle(repo,"prepare");let entryBinding,ownershipCreated=false; + try{ + await requireLifecycleContext(lifecycle);await checkPrerequisites(repo);if(!skipBuild)await run("npm",["--prefix",join(repo,"backend"),"run","build"]);await requireLifecycleContext(lifecycle); + entryBinding=await readBoundEntrypoint(repo);const entrypoint=entryBinding.identity;await entryBinding.handle.close();entryBinding=undefined; + noSymlinkExisting(repo,root);try{await mkdir(root,{recursive:false,mode:0o700});}catch(error){if(error.code==="EEXIST")throw new Error("manual acceptance root already exists; stop/cleanup it explicitly");throw error;}await bindLifecycleRoot(lifecycle,root); + for(const path of ["installation/registry","installation/data","installation/runtime","fixture-secrets","fixtures/descriptors","requests","responses","exports/raw","exports/extracted","rendered","logs","commands"]){await mkdir(join(root,path),{recursive:true,mode:path==="fixture-secrets"?0o700:0o755});await requireLifecycleContext(lifecycle,{root:true});} + const distManifestValue=await buildDistManifest(repo),distManifestRecord=await exclusiveRecord(join(root,"installation/runtime/backend-dist.manifest.json"),distManifestValue,"production distribution manifest"),distManifest={path:distManifestRecord.path,dev:distManifestRecord.dev,ino:distManifestRecord.ino,size:distManifestRecord.bytes.length,sha256:createHash("sha256").update(distManifestRecord.bytes).digest("hex")};await requireLifecycleContext(lifecycle,{root:true}); + const nonce=randomBytes(32).toString("hex"),createdAt=new Date().toISOString();await exclusiveRecord(join(root,"ownership.json"),ownedValue(repo,root,nonce,{entrypoint,distManifest,createdAt}),"manual ownership");ownershipCreated=true;await requireLifecycleContext(lifecycle,{root:true}); + const backendLogPath=join(root,"logs/backend.log"),backendLogHandle=await open(backendLogPath,"wx",0o600);let backendLogEntry;try{await backendLogHandle.chmod(0o600);await backendLogHandle.sync();backendLogEntry=await backendLogHandle.stat();}finally{await backendLogHandle.close();}directorySync(dirname(backendLogPath));const backendLog={path:backendLogPath,dev:backendLogEntry.dev,ino:backendLogEntry.ino}; + await requireLifecycleContext(lifecycle,{root:true}); + try{await initializeGit(root);}catch(error){try{await requireLifecycleContext(lifecycle,{root:true});}catch{await cleanupFailedPrepare(repo,lifecycle);throw new Error("manual acceptance parent or root identity changed during prepare");}throw error;}await requireLifecycleContext(lifecycle,{root:true}); + const items=descriptors();for(const workspace of items)await atomicWrite(join(root,"fixtures/descriptors",`${workspace.workspace.id}.json`),`${JSON.stringify(workspace,null,2)}\n`); + const secrets={"dwh-password":`DWH-${randomBytes(16).toString("hex")}`,"evidence-signed-urls.json":JSON.stringify([`https://evidence.example.test/guide.md?token=SIGNED-${randomBytes(16).toString("hex")}`]),"evidence-access":`ACCESS-${randomBytes(16).toString("hex")}`,"evidence-secret":`SECRET-${randomBytes(16).toString("hex")}`,"evidence-session":`SESSION-${randomBytes(16).toString("hex")}`};for(const[name,value]of Object.entries(secrets))await atomicWrite(join(root,"fixture-secrets",name),value,0o600); + const env={};for(const workspace of items){const ns=workspace.workspace.id.toUpperCase().replaceAll("-","_"),prefix=`THT_WS_${ns}`;Object.assign(env,{[`${prefix}_DWH_TRANSPORT`]:"postgres_direct",[`${prefix}_DWH_HOST`]:"dwh.invalid",[`${prefix}_DWH_PORT`]:"5432",[`${prefix}_DWH_USER`]:"reader",[`${prefix}_DWH_PASSWORD_FILE`]:join(root,"fixture-secrets/dwh-password")});}Object.assign(env,{THT_WORKSPACE_SECRET_ROOTS:join(root,"fixture-secrets"),THT_WS_P1_HTTP_EVIDENCE_SIGNED_URLS_FILE:join(root,"fixture-secrets/evidence-signed-urls.json"),THT_WS_P1_S3_EVIDENCE_ACCESS_KEY_FILE:join(root,"fixture-secrets/evidence-access"),THT_WS_P1_S3_EVIDENCE_SECRET_KEY_FILE:join(root,"fixture-secrets/evidence-secret"),THT_WS_P1_S3_EVIDENCE_SESSION_TOKEN_FILE:join(root,"fixture-secrets/evidence-session")}); + await atomicWrite(join(root,"installation/bindings.env"),Object.entries(env).map(([k,v])=>`${k}=${quote(v)}`).join("\n")+"\n");await atomicWrite(join(root,"installation/base.yaml"),"{}\n");for(const[name,value]of Object.entries(requestFixtures(items)))await atomicWrite(join(root,"requests",name),`${JSON.stringify(value,null,2)}\n`);await writeCommands(repo,root);await atomicWrite(join(root,"GUIDE.md"),guide(repo,root),0o600);await requireLifecycleContext(lifecycle,{root:true});await atomicWrite(join(root,"ownership.json"),`${JSON.stringify(ownedValue(repo,root,nonce,{backendLog,entrypoint,distManifest,stage:"READY",createdAt}),null,2)}\n`);await requireLifecycleContext(lifecycle,{root:true});return{repositoryRoot:repo,root,nonce}; + }catch(error){ + if(entryBinding)await entryBinding.handle.close().catch(()=>{}); + if(ownershipCreated){try{await requireLifecycleContext(lifecycle,{root:true});}catch{await cleanupFailedPrepare(repo,lifecycle).catch(()=>{});throw new Error("manual acceptance parent or root identity changed during prepare");}} + throw error; + }finally{await removeExactRecord(lifecycle);} +} +function portAvailable(port,label=`${HOST}:${port}`){return new Promise((resolvePromise,reject)=>{const server=net.createServer();server.once("error",error=>error.code==="EADDRINUSE"?reject(new Error(`${label} is occupied`)):reject(error));server.listen({host:HOST,port,exclusive:true},()=>server.close(()=>resolvePromise()));});} +async function requireCanonicalDirectory(path,label){const entry=await lstat(path);if(!entry.isDirectory()||entry.isSymbolicLink()||await realpath(path)!==path)throw new Error(`${label} directory identity is unsafe`);return entry;} +async function requireAbsent(path,label){try{await lstat(path);throw new Error(`${label} is legacy or unsafe`);}catch(error){if(error.code!=="ENOENT")throw error;}} +async function validateServeFilesystem(repo,root,owned){ + if(root!==fixedManualRoot(repo))throw new Error("owned root identity is unsafe"); + for(const [path,label] of [ + [repo,"repository root"],[join(repo,".artifacts"),"artifact root"],[join(repo,".artifacts/manual-acceptance"),"manual root ancestor"],[root,"owned root"], + [join(root,"installation"),"owned installation"],[join(root,"installation/runtime"),"owned runtime"],[join(root,"installation/data"),"owned data"], + [join(root,"installation/registry"),"owned registry"],[join(root,"fixture-secrets"),"owned secrets"],[join(root,"logs"),"owned logs"], + [join(repo,"backend"),"backend root"],[join(repo,"backend/dist"),"backend distribution"], + ])await requireCanonicalDirectory(path,label); + await requireAbsent(legacySupervisorPath(root),"legacy supervisor"); + if(owned.stage!=="READY")throw new Error("manual acceptance preparation is incomplete"); + const script=join(repo,"backend/dist/server.js");await requireEntrypointPathIdentity(owned.entrypoint);const manifestRecord=await readBoundDistManifest(repo,owned);try{await validateDistFiles(repo,manifestRecord.files);}finally{await manifestRecord.handle.close();} + const logPath=join(root,"logs/backend.log"); + if(owned.backendLog?.path!==logPath)throw new Error("backend log ownership identity is unsafe"); + return{script,logPath}; +} +function openOwnedBackendLog(owned,logPath){ + if(!Number.isInteger(constants.O_NOFOLLOW))throw new Error("backend log no-follow protection is unavailable"); + let fd; + try{ + fd=openSync(logPath,constants.O_WRONLY|constants.O_APPEND|constants.O_NOFOLLOW); + const entry=fstatSync(fd),pathEntry=lstatSync(logPath); + if(!entry.isFile()||(entry.mode&0o777)!==0o600||entry.nlink!==1||entry.dev!==owned.backendLog.dev||entry.ino!==owned.backendLog.ino||pathEntry.isSymbolicLink()||!pathEntry.isFile()||pathEntry.dev!==entry.dev||pathEntry.ino!==entry.ino)throw new Error("backend log identity is unsafe"); + return fd; + }catch(error){if(fd!==undefined)closeSync(fd);throw error;} +} +async function ensureRuntimeDirectory(path){try{await mkdir(path,{mode:0o700});}catch(error){if(error.code!=="EEXIST")throw error;}const entry=await requireCanonicalDirectory(path,"owned runtime child");if((entry.mode&0o077)!==0)throw new Error("owned runtime child mode is unsafe");} +async function processStart(pid){return (await run("ps",["-p",String(pid),"-o","lstart="])).stdout.trim();} +async function processArgs(pid){return (await run("ps",["-ww","-p",String(pid),"-o","command="])).stdout.trim();} +async function processCwd(pid){try{return await realpath(`/proc/${pid}/cwd`);}catch{try{const out=(await run("lsof",["-a","-p",String(pid),"-d","cwd","-Fn"])).stdout.split("\n").find(x=>x.startsWith("n"));return out?await realpath(out.slice(1)):"";}catch{return"";}}} +async function processExecutable(pid){try{return await realpath(`/proc/${pid}/exe`);}catch{try{const paths=(await run("lsof",["-a","-p",String(pid),"-d","txt","-Fn"])).stdout.split("\n").filter(x=>x.startsWith("n")).map(x=>x.slice(1));for(const path of paths){try{const canonical=await realpath(path);if(canonical===realpathSync(process.execPath))return canonical;}catch{}}return"";}catch{return"";}}} +function alive(pid){try{process.kill(pid,0);return true;}catch{return false;}} +async function readPid(root){const path=join(root,"backend.pid"),entry=await lstat(path);if(!entry.isFile()||entry.isSymbolicLink()||(entry.mode&0o777)!==0o600)throw new Error("backend PID record is unsafe");const bytes=await readFile(path,"utf8");let value;try{value=JSON.parse(bytes);}catch{throw new Error("backend PID record is malformed");}return{path,bytes,value,dev:entry.dev,ino:entry.ino};} +async function validateProcess(repo,root,owned,pidRecord){ + const script=join(repo,"backend/dist/server.js"),entrypoint=owned.entrypoint; + if(pidRecord.schemaVersion!==1||pidRecord.kind!=="p1-manual-backend"||pidRecord.status!=="RUNNING"||!Number.isSafeInteger(pidRecord.pid)||pidRecord.pid<2||!HEX64.test(pidRecord.reservationNonce??"")||pidRecord.nonce!==owned.nonce||pidRecord.root!==root||pidRecord.repositoryRoot!==repo||pidRecord.executable!==process.execPath||pidRecord.preload!==PRELOAD||pidRecord.script!==script||JSON.stringify(pidRecord.entrypoint)!==JSON.stringify(entrypoint)||!pidRecord.startIdentity||pidRecord.control?.host!==HOST||pidRecord.control?.port!==CONTROL_PORT)throw new Error("backend process identity mismatch; refusing cooperative control"); + await requireEntrypointPathIdentity(entrypoint);const manifestRecord=await readBoundDistManifest(repo,owned);try{await validateDistFiles(repo,manifestRecord.files);}finally{await manifestRecord.handle.close();}if(!alive(pidRecord.pid))throw new Error("backend PID is stale; operator inspection required"); + const[start,args,cwd,executable]=await Promise.all([processStart(pidRecord.pid),processArgs(pidRecord.pid),processCwd(pidRecord.pid),processExecutable(pidRecord.pid)]); + const expectedArgs=[pidRecord.executable,"--import",pidRecord.preload,pidRecord.script,`--p1-manual-nonce=${owned.nonce}`,`--p1-root=${root}`,`--p1-control-nonce=${pidRecord.reservationNonce}`,`--p1-entry-sha256=${entrypoint.sha256}`,`--p1-entry-dev=${entrypoint.dev}`,`--p1-entry-ino=${entrypoint.ino}`].join(" "); + if(start!==pidRecord.startIdentity||cwd!==repo||executable!==realpathSync(pidRecord.executable)||args!==expectedArgs)throw new Error("backend process identity mismatch; refusing cooperative control");return true; +} +async function waitForChildExit(child,milliseconds){if(!child||child.exitCode!==null||child.signalCode!==null)return true;return await Promise.race([new Promise(resolvePromise=>child.once("exit",()=>resolvePromise(true))),new Promise(resolvePromise=>setTimeout(()=>resolvePromise(child.exitCode!==null||child.signalCode!==null),milliseconds))]);} +async function healthStatus(){return await new Promise((resolvePromise,reject)=>{const request=http.get({host:HOST,port:PORT,path:"/health",timeout:500},response=>{const status=response.statusCode;response.resume();response.once("end",()=>resolvePromise(status));});request.once("timeout",()=>request.destroy(new Error("backend health readiness timeout")));request.once("error",reject);});} +function exactControlIdentity(answer,child,owned,root,reservationNonce){return answer?.pid===child.pid&&answer?.nonce===owned.nonce&&answer?.controlNonce===reservationNonce&&answer?.root===root&&answer?.control?.host===HOST&&answer?.control?.port===CONTROL_PORT;} +function exactOwnedListener(answer,generation){return answer?.listener?.listening===true&&answer.listener.host===HOST&&answer.listener.port===PORT&&Number.isSafeInteger(answer.listener.generation)&&answer.listener.generation>0&&(generation===undefined||answer.listener.generation===generation);} +export async function serveManual({repositoryRoot=defaultRepositoryRoot,beforeSpawn}={}){ + const repo=realpathSync(repositoryRoot),lifecycle=await acquireLifecycle(repo,"serve");let pidRecord,child,controlObserved=false,logFd,entryBinding,manifestBinding; + try{ + const owned=await readManualOwnership({repositoryRoot:repo}),root=owned.root;await bindLifecycleRoot(lifecycle,root); + if(owned.listener.host!==HOST||owned.listener.port!==PORT)throw new Error("non-loopback or unexpected bind refused"); + const{script,logPath}=await validateServeFilesystem(repo,root,owned);await requireLifecycleContext(lifecycle,{root:true}); + logFd=openOwnedBackendLog(owned,logPath);entryBinding=await readBoundEntrypoint(repo);if(JSON.stringify(entryBinding.identity)!==JSON.stringify(owned.entrypoint))throw new Error("production entrypoint identity changed");manifestBinding=await readBoundDistManifest(repo,owned); + const reservationNonce=randomBytes(32).toString("hex");pidRecord=await exclusiveRecord(join(root,"backend.pid"),{schemaVersion:1,kind:"p1-manual-backend-reservation",status:"RESERVED",reservationNonce,nonce:owned.nonce,root,repositoryRoot:repo},"backend PID record"); + await Promise.all([portAvailable(PORT),portAvailable(CONTROL_PORT,`${HOST}:${CONTROL_PORT} control port`)]);await requireLifecycleContext(lifecycle,{root:true}); + await ensureRuntimeDirectory(join(root,"installation/runtime/home"));await ensureRuntimeDirectory(join(root,"installation/runtime/tmp"));await ensureRuntimeDirectory(join(root,"installation/runtime/tht-home"));await requireLifecycleContext(lifecycle,{root:true}); + const inherited={};for(const key of ["PATH","LANG","LC_ALL","TZ"])if(process.env[key]!==undefined)inherited[key]=process.env[key]; + const env={...inherited,HOME:join(root,"installation/runtime/home"),TMPDIR:join(root,"installation/runtime/tmp"),HOST,PORT:String(PORT),AUTH_MODE:"none",THT_BIN:join(repo,"harness/.venv/bin/tht"),THT_HARNESS_DIR:join(repo,"harness"),THT_DATA_ROOT:join(root,"installation/data"),SETTINGS_FILE:join(root,"installation/data/settings.json"),MAINTENANCE_STATE_FILE:join(root,"installation/data/maintenance.json"),THT_WORKSPACE_REGISTRY_ROOT:join(root,"installation/registry"),THT_WORKSPACE_GIT_REMOTE:join(root,"remote.git"),THT_WORKSPACE_GIT_BRANCH:"main",THT_WORKSPACE_GIT_AUTHOR_NAME:"P1 Manual API Publisher",THT_WORKSPACE_GIT_AUTHOR_EMAIL:"p1-manual-api@example.invalid",THT_WORKSPACE_INSTALLATION_ID:"p1-manual-acceptance",THT_WORKSPACE_SECRET_ROOTS:join(root,"fixture-secrets"),THT_HOME:join(root,"installation/runtime/tht-home")}; + if(beforeSpawn)await beforeSpawn({script,entrypoint:{...owned.entrypoint}});await requireLifecycleContext(lifecycle,{root:true}); + const entryArgs=[`--p1-entry-sha256=${owned.entrypoint.sha256}`,`--p1-entry-dev=${owned.entrypoint.dev}`,`--p1-entry-ino=${owned.entrypoint.ino}`]; + child=spawn(process.execPath,["--import",PRELOAD,script,`--p1-manual-nonce=${owned.nonce}`,`--p1-root=${root}`,`--p1-control-nonce=${reservationNonce}`,...entryArgs],{cwd:repo,env,detached:true,stdio:["ignore",logFd,logFd,entryBinding.handle.fd,manifestBinding.handle.fd]}); + await entryBinding.handle.close();entryBinding=undefined;await manifestBinding.handle.close();manifestBinding=undefined;closeSync(logFd);logFd=undefined; + let start="";for(let n=0;n<80;n++){if(child.exitCode!==null)break;try{start=await processStart(child.pid);if(start)break;}catch{}await new Promise(r=>setTimeout(r,25));} + if(!start)throw new Error("backend failed before process identity could be recorded");await requireLifecycleContext(lifecycle,{root:true}); + pidRecord=await replaceExactRecord(pidRecord,{schemaVersion:1,kind:"p1-manual-backend-reservation",status:"STARTING",pid:child.pid,reservationNonce,nonce:owned.nonce,root,repositoryRoot:repo,executable:process.execPath,preload:PRELOAD,script,entrypoint:owned.entrypoint,startIdentity:start,control:{host:HOST,port:CONTROL_PORT}}); + const deadline=Date.now()+7000;let readyAnswer,listenerGeneration; + while(Date.now()setTimeout(r,50));continue;} + if(!exactControlIdentity(status,child,owned,root,reservationNonce)||status.status!=="STARTING")throw new Error("backend control status identity mismatch");controlObserved=true; + if(!exactOwnedListener(status)){await new Promise(r=>setTimeout(r,50));continue;}listenerGeneration=status.listener.generation; + await requireLifecycleContext(lifecycle,{root:true});await requireEntrypointPathIdentity(owned.entrypoint); + let httpCode;try{httpCode=await healthStatus();}catch{await new Promise(r=>setTimeout(r,50));continue;}if(!Number.isSafeInteger(httpCode)||httpCode<200||httpCode>=300)throw new Error(`backend health readiness returned HTTP ${httpCode}`); + readyAnswer=await controlRequest({host:HOST,port:CONTROL_PORT},{action:"ready",nonce:reservationNonce});if(!exactControlIdentity(readyAnswer,child,owned,root,reservationNonce)||readyAnswer.status!=="READY"||!exactOwnedListener(readyAnswer,listenerGeneration))throw new Error("backend READY listener acknowledgement identity mismatch"); + const finalHealth=await healthStatus();if(!Number.isSafeInteger(finalHealth)||finalHealth<200||finalHealth>=300)throw new Error("backend final health readiness failed"); + const finalStatus=await controlRequest({host:HOST,port:CONTROL_PORT},{action:"status",nonce:reservationNonce});if(!exactControlIdentity(finalStatus,child,owned,root,reservationNonce)||finalStatus.status!=="READY"||!exactOwnedListener(finalStatus,listenerGeneration))throw new Error("backend final listener identity mismatch");readyAnswer=finalStatus;break; + } + if(!readyAnswer)throw new Error("backend readiness failed; inspect owned backend log and starting PID record"); + const runningValue={schemaVersion:1,kind:"p1-manual-backend",status:"RUNNING",pid:child.pid,reservationNonce,nonce:owned.nonce,root,repositoryRoot:repo,executable:process.execPath,preload:PRELOAD,script,entrypoint:owned.entrypoint,startIdentity:start,control:{host:HOST,port:CONTROL_PORT},listener:{host:HOST,port:PORT,generation:listenerGeneration}}; + await requireLifecycleContext(lifecycle,{root:true});await validateProcess(repo,root,owned,runningValue);if(child.exitCode!==null||!alive(child.pid))throw new Error("backend exited before RUNNING publication");pidRecord=await replaceExactRecord(pidRecord,runningValue);await requireLifecycleContext(lifecycle,{root:true}); + const publishedStatus=await controlRequest(runningValue.control,{action:"status",nonce:reservationNonce});if(!exactControlIdentity(publishedStatus,child,owned,root,reservationNonce)||publishedStatus.status!=="READY"||!exactOwnedListener(publishedStatus,listenerGeneration)){await removeExactRecord(pidRecord);throw new Error("backend listener changed during RUNNING publication");} + child.unref();return child.pid; + }catch(error){ + if(entryBinding)await entryBinding.handle.close().catch(()=>{});if(manifestBinding)await manifestBinding.handle.close().catch(()=>{});if(logFd!==undefined){closeSync(logFd);logFd=undefined;} + if(child&&controlObserved){try{const value=pidRecord?JSON.parse(pidRecord.bytes):undefined;await controlRequest({host:HOST,port:CONTROL_PORT},{action:"stop",nonce:value?.reservationNonce});}catch{}await waitForChildExit(child,3000);}else if(child)await waitForChildExit(child,8500); + if(pidRecord&&(!child||child.exitCode!==null||child.signalCode!==null||!alive(child.pid)))await removeExactRecord(pidRecord).catch(()=>{});throw error; + }finally{if(entryBinding)await entryBinding.handle.close().catch(()=>{});if(manifestBinding)await manifestBinding.handle.close().catch(()=>{});if(logFd!==undefined)closeSync(logFd);await removeExactRecord(lifecycle);} +} +export async function stopManual({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),lifecycle=await acquireLifecycle(repo,"stop");try{const owned=await readManualOwnership({repositoryRoot:repo}),root=owned.root;if(owned.stage!=="READY")throw new Error("owned backend was never prepared");await bindLifecycleRoot(lifecycle,root);let record;try{record=await readPid(root);}catch(error){if(error.code==="ENOENT")throw new Error("owned backend is not served");throw error;}await requireLifecycleContext(lifecycle,{root:true});await validateProcess(repo,root,owned,record.value);const answer=await controlRequest(record.value.control,{action:"stop",nonce:record.value.reservationNonce});if(answer.status!=="STOPPING"||answer.pid!==record.value.pid||answer.nonce!==owned.nonce||answer.controlNonce!==record.value.reservationNonce||answer.root!==root||answer.control?.host!==HOST||answer.control?.port!==CONTROL_PORT)throw new Error("backend cooperative stop acknowledgement mismatch; PID record retained");for(let n=0;n<100;n++){if(!alive(record.value.pid)){await requireLifecycleContext(lifecycle,{root:true});await removeExactRecord(record);await requireLifecycleContext(lifecycle,{root:true});return;}await new Promise(r=>setTimeout(r,100));}throw new Error("owned backend did not stop cooperatively; operator must intervene; PID record retained");}finally{await removeExactRecord(lifecycle);}} +const ANCHORED_REMOVE_SOURCE=String.raw`import os,stat,sys +parent,parent_dev,parent_ino,root_dev,root_ino,tomb=sys.argv[1:] +pfd=rfd=None +def die(): raise RuntimeError("anchored cleanup refused") +def clear(fd): + names=os.listdir(fd) + if len(names)>200000: die() + for name in names: + if name in (".",".."): die() + item=os.stat(name,dir_fd=fd,follow_symlinks=False) + if stat.S_ISDIR(item.st_mode): + child=os.open(name,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd) + try: clear(child) + finally: os.close(child) + os.rmdir(name,dir_fd=fd) + elif stat.S_ISREG(item.st_mode) or stat.S_ISLNK(item.st_mode): os.unlink(name,dir_fd=fd) + else: die() +try: + pfd=os.open(parent,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + ps=os.fstat(pfd) + if (ps.st_dev,ps.st_ino)!=(int(parent_dev),int(parent_ino)): die() + rfd=os.open("p1",os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=pfd) + rs=os.fstat(rfd) + if (rs.st_dev,rs.st_ino)!=(int(root_dev),int(root_ino)): die() + try: os.stat(tomb,dir_fd=pfd,follow_symlinks=False); die() + except FileNotFoundError: pass + os.rename("p1",tomb,src_dir_fd=pfd,dst_dir_fd=pfd);os.fsync(pfd) + clear(rfd);os.close(rfd);rfd=None;os.rmdir(tomb,dir_fd=pfd);os.fsync(pfd) +except Exception: + print("anchored cleanup refused (details redacted)",file=sys.stderr);raise SystemExit(1) +finally: + if rfd is not None: os.close(rfd) + if pfd is not None: os.close(pfd) +`; +async function anchoredRemoveOwnedRoot(lifecycle,owned){const tomb=`.deleting-p1-${owned.nonce.slice(0,16)}`;try{await run("python3",["-c",ANCHORED_REMOVE_SOURCE,lifecycle.parentPath,String(lifecycle.parentEntry.dev),String(lifecycle.parentEntry.ino),String(lifecycle.rootEntry.dev),String(lifecycle.rootEntry.ino),tomb]);}catch{throw new Error("anchored cleanup refused; owned identities changed");}} +export async function cleanupManual({repositoryRoot=defaultRepositoryRoot}={}){const repo=realpathSync(repositoryRoot),lifecycle=await acquireLifecycle(repo,"cleanup");try{const owned=await readManualOwnership({repositoryRoot:repo}),root=owned.root;await bindLifecycleRoot(lifecycle,root);try{const record=await readPid(root);if(record.value.status==="RUNNING"&&alive(record.value.pid)){await validateProcess(repo,root,owned,record.value);throw new Error("owned backend is live; run stop first");}throw new Error("stale or starting backend PID record requires operator inspection and stop validation");}catch(error){if(error.code!=="ENOENT")throw error;}if(root!==fixedManualRoot(repo)||!below(join(repo,".artifacts"),root))throw new Error("cleanup root identity mismatch");await requireLifecycleContext(lifecycle,{root:true});await anchoredRemoveOwnedRoot(lifecycle,owned);await requireLifecycleContext(lifecycle);}finally{await removeExactRecord(lifecycle);}} +async function main(){const[action,...rest]=process.argv.slice(2);if(rest.length||!["prepare","serve","stop","cleanup"].includes(action??""))throw new Error("usage: p1-manual-acceptance.mjs prepare|serve|stop|cleanup");if(action==="prepare")await prepareManual();if(action==="serve")await serveManual();if(action==="stop")await stopManual();if(action==="cleanup")await cleanupManual();console.log(`P1 manual acceptance ${action}: ${action==="prepare"?"PENDING":"complete"}`);} +if(process.argv[1]&&realpathSync(process.argv[1])===modulePath)main().catch(error=>{console.error(`p1 manual acceptance refused: ${error.message}`);process.exitCode=1;}); diff --git a/backend/scripts/p1-manual-acceptance.test.mjs b/backend/scripts/p1-manual-acceptance.test.mjs new file mode 100644 index 00000000..2ae83d01 --- /dev/null +++ b/backend/scripts/p1-manual-acceptance.test.mjs @@ -0,0 +1,787 @@ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, symlink, writeFile } from "node:fs/promises"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +import { + cleanupManual, fixedManualRoot, prepareManual, readManualOwnership, serveManual, stopManual, +} from "./p1-manual-acceptance.mjs"; + +const roots = []; +async function fakeRepo() { + const root = await realpath(await mkdtemp(join(tmpdir(), "p1-manual-repo-"))); + roots.push(root); + for (const path of ["scripts/p1-acceptance.sh", "scripts/test-p1-acceptance.sh", "backend/scripts/p1-acceptance.mjs", "backend/dist/server.js"]) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), path.endsWith(".sh") ? "#!/bin/sh\n" : "export {};\n", { mode: 0o700 }); + } + await symlink(new URL("../node_modules", import.meta.url).pathname, join(root, "backend", "node_modules"), "dir"); + await mkdir(join(root, "harness", ".venv", "bin"), { recursive: true }); + await writeFile(join(root, "harness", ".venv", "bin", "tht"), "#!/bin/sh\n", { mode: 0o700 }); + await chmod(join(root, "harness", ".venv", "bin", "tht"), 0o700); + await mkdir(join(root, "harness", "workspaces"), { recursive: true }); + return root; +} +test.afterEach(async () => Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))); + +test("prepare refuses a pre-existing or symlink fixed root", async () => { + const repo = await fakeRepo(); const root = fixedManualRoot(repo); + await mkdir(root, { recursive: true }); + await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /already exists/); + await rm(root, { recursive: true }); + const target = `${root}-target`; await mkdir(target, { recursive: true }); await symlink(target, root); + await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /already exists|symlink/); +}); + +test("prepare requires Task 8 and prerequisites before creating state", async () => { + const repo = await fakeRepo(); await rm(join(repo, "scripts", "p1-acceptance.sh")); + await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /Task 8/); + await assert.rejects(lstat(fixedManualRoot(repo))); +}); + +test("public wrapper exposes only four actions and rejects automated-run prepare input", async () => { + const wrapper=new URL("../../scripts/p1-manual-acceptance.sh",import.meta.url),source=await readFile(wrapper,"utf8"); + assert.match(source,/prepare\|serve\|stop\|cleanup/); assert.doesNotMatch(source,/integration\|automated|prepare\|serve\|stop\|cleanup\|/); + await assert.rejects(execFileAsync("bash",[wrapper.pathname,"prepare",".artifacts/p1-integration/run"]),error=>error.code===2&&/usage:/.test(error.stderr)); +}); + +test("prepare rejects unknown automated-run input before creating its root", async () => { + const repo = await fakeRepo(); + await assert.rejects( + prepareManual({ repositoryRoot: repo, skipBuild: true, automatedRun: join(repo, ".artifacts", "p1-integration") }), + /unknown|automated/i, + ); + await assert.rejects(lstat(fixedManualRoot(repo))); +}); + +test("prepare requires the non-Task-8 tht prerequisite before creating state", async () => { + const repo = await fakeRepo(); await rm(join(repo, "harness", ".venv", "bin", "tht")); + await assert.rejects(prepareManual({ repositoryRoot: repo, skipBuild: true }), /missing prerequisite.*tht/); + await assert.rejects(lstat(fixedManualRoot(repo))); +}); + +test("prepare and permanent docs declare the python3 extractor prerequisite", async () => { + const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8"),docs=await readFile(new URL("../../docs/testing/p1-manual-acceptance.md",import.meta.url),"utf8"); + assert.match(source,/for\(const command of \[.*["']python3["']/s); assert.match(docs,/python3/); +}); + +test("prepare creates independent pending topology, fixtures, commands and guide without verdict", async () => { + const repo = await fakeRepo(); const run = await prepareManual({ repositoryRoot: repo, skipBuild: true }); + assert.equal(run.root, fixedManualRoot(repo)); + const owned = await readManualOwnership({ repositoryRoot: repo }); + assert.equal(owned.status, "PENDING"); assert.equal(owned.listener.host, "127.0.0.1"); assert.equal(owned.listener.port, 8791); + for (const path of ["remote.git/HEAD", "author/.git", "installation/registry", "fixture-secrets/dwh-password", "fixtures/descriptors/p1-filesystem.json", "requests/status.json", "responses", "exports", "rendered", "logs", "commands/render-1.sh", "commands/render-2.sh", "logs/backend.log", "GUIDE.md"]) await lstat(join(run.root, path)); + await assert.rejects(lstat(join(run.root, "VERDICT.md"))); + const guide = await readFile(join(run.root, "GUIDE.md"), "utf8"); + let previous = -1; for (let n = 1; n <= 14; n++) { const at = guide.indexOf(`${n}. `); assert.ok(at > previous, `step ${n} ordered`); previous = at; } + assert.doesNotMatch(guide, /cat .*fixture-secrets|show.*secret contents/i); for(const id of ["p1-filesystem","p1-http","p1-s3"])assert.match(guide,new RegExp(`extract-export\\.sh[^\\n]+${id}`)); + for(const contract of [/stable repository-root/,/ownership-first recovery/,/one production Node PID owns both/,/opened no-follow descriptor/,/arbitrary `.git` repository/,/name\/path bytes/,/artifacts\/evidence/,/opened no-follow `rendered` directory/])assert.match(guide,contract); + const traversal=JSON.parse(await readFile(join(run.root,"requests","invalid-traversal.json"),"utf8")); assert.match(traversal.workspace.evidence.source.uri,/\.\./); + const bindings=await readFile(join(run.root,"installation","bindings.env"),"utf8"); assert.match(bindings,new RegExp(`^THT_WORKSPACE_SECRET_ROOTS=.*fixture-secrets`,"m")); const scan=await readFile(join(run.root,"commands","secret-scan.sh"),"utf8"),extract=await readFile(join(run.root,"commands","extract-export.sh"),"utf8"); assert.match(scan,/batch-all-objects/); assert.match(scan,/cat-file/); assert.match(scan,/maybeGitDir/); assert.match(extract,/ZIP contains a symlink or nonregular entry/); + const pubFs=await readFile(join(run.root,"commands","http-05-publish-p1-filesystem.sh"),"utf8"),pubHttp=await readFile(join(run.root,"commands","http-06-publish-p1-http.sh"),"utf8"),pubS3=await readFile(join(run.root,"commands","http-07-publish-p1-s3.sh"),"utf8"); assert.match(pubFs,/responses\/status\.json/); assert.match(pubHttp,/responses\/publish-p1-filesystem\.json/); assert.match(pubS3,/responses\/publish-p1-http\.json/); assert.doesNotMatch(pubFs,/REPLACE_WITH/); + const render = await readFile(join(run.root, "commands", "render-1.sh"), "utf8"); + for(const name of await readdir(join(run.root,"commands")))if(name.endsWith(".sh"))await execFileAsync("bash",["-n",join(run.root,"commands",name)]); + assert.match(render, /read-p1-filesystem\.json/); assert.match(render, /responses\/pull\.json/); assert.doesNotMatch(render, /responses\/publish-p1-filesystem\.json/); assert.match(render, /snapshotPath/); assert.match(render, /p1-render-snapshot\.mjs/); +}); + +test("cleanup rejects unowned, live, mismatched and symlink state and preserves siblings", async () => { + const repo = await fakeRepo(); const integration = join(repo, ".artifacts", "p1-integration"); const sibling = join(repo, ".artifacts", "manual-acceptance", "foreign"); + await mkdir(integration, { recursive: true }); await writeFile(join(integration, "sentinel"), "keep"); + await mkdir(sibling, { recursive: true }); await writeFile(join(sibling, "sentinel"), "keep"); + await assert.rejects(cleanupManual({ repositoryRoot: repo }), /ownership|root/); + const run = await prepareManual({ repositoryRoot: repo, skipBuild: true }); + const ownershipPath = join(run.root, "ownership.json"); const owned = JSON.parse(await readFile(ownershipPath)); owned.root += "-wrong"; await writeFile(ownershipPath, JSON.stringify(owned)); + await assert.rejects(cleanupManual({ repositoryRoot: repo }), /identity/); assert.equal((await lstat(run.root)).isDirectory(), true); + assert.equal(await readFile(join(integration, "sentinel"), "utf8"), "keep"); assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "keep"); +}); + +test("cleanup removes only the exact stopped owned root and never creates verdict", async () => { + const repo = await fakeRepo(); const run = await prepareManual({ repositoryRoot: repo, skipBuild: true }); + await cleanupManual({ repositoryRoot: repo }); await assert.rejects(lstat(run.root)); +}); + + +async function installFakeServer(repo, { startupDelay = 0, healthStatus = 200, marker } = {}) { + await writeFile(join(repo, "backend", "dist", "server.js"), `import http from "node:http"; +${marker ? `import { writeFileSync } from "node:fs"; writeFileSync(${JSON.stringify(marker)}, "executed");` : ""} +const server=http.createServer((req,res)=>{res.statusCode=req.url==="/health"?${healthStatus}:200;res.setHeader("content-type","application/json");res.end(JSON.stringify({status:"ok",ambient:process.env.THT_DWH_API_KEY,maintenance:process.env.MAINTENANCE_STATE_FILE,wrongMaintenance:process.env.THT_MAINTENANCE_STATE_FILE}));}); +setTimeout(()=>server.listen(Number(process.env.PORT),process.env.HOST),${startupDelay}); +`); +} + + +async function matchingManualServerPids(root, nonce) { + const { stdout } = await execFileAsync("ps", ["ax", "-o", "pid=,command="]); + const nonceArg = `--p1-manual-nonce=${nonce}`, rootArg = `--p1-root=${root}`; + return stdout.split("\n").filter(line => line.includes(nonceArg) && line.includes(rootArg)) + .map(line => Number(line.trim().match(/^(\d+)/)?.[1])).filter(Number.isSafeInteger); +} +async function listenerPids() { + try { + const { stdout } = await execFileAsync("lsof", ["-nP", "-t", "-iTCP:8791", "-sTCP:LISTEN"]); + return [...new Set(stdout.trim().split("\n").filter(Boolean).map(Number))]; + } catch (error) { + if (error.code === 1) return []; + throw error; + } +} + +test("prepare and cleanup share one external lifecycle lock for the whole transaction", { concurrency: false }, async () => { + const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim(); + const bin=join(repo,"blocking-bin"),entered=join(repo,"prepare-entered"),release=join(repo,"prepare-release"); + await mkdir(bin); await writeFile(join(bin,"git"),`#!/bin/sh +if [ "$1" = init ] && [ ! -e ${JSON.stringify(entered)} ]; then + : > ${JSON.stringify(entered)} + n=0 + while [ ! -e ${JSON.stringify(release)} ] && [ "$n" -lt 250 ]; do sleep 0.02; n=$((n + 1)); done + [ -e ${JSON.stringify(release)} ] || exit 99 +fi +exec ${JSON.stringify(realGit)} "$@" +`,{mode:0o700}); + const priorPath=process.env.PATH; process.env.PATH=`${bin}:${priorPath}`; + try { + const preparing=prepareManual({repositoryRoot:repo,skipBuild:true}); + for(let n=0;n<200;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,10));} + await lstat(entered); + const lock=join(repo,".p1-manual-acceptance.lifecycle.lock"); + const lockEntry=await lstat(lock); assert.equal(lockEntry.isFile(),true); assert.equal(lockEntry.mode&0o777,0o600); + const lockBytes=await readFile(lock,"utf8"),lockValue=JSON.parse(lockBytes); + assert.deepEqual(Object.keys(lockValue).sort(),["kind","lifecycleNonce","operation","repositoryRoot","root","schemaVersion"].sort()); + assert.equal(lockValue.kind,"p1-manual-lifecycle"); assert.equal(lockValue.operation,"prepare"); + assert.match(lockValue.lifecycleNonce,/^[0-9a-f]{64}$/); assert.equal(lockValue.repositoryRoot,repo); assert.equal(lockValue.root,fixedManualRoot(repo)); + assert.equal(lockBytes,`${JSON.stringify(lockValue,null,2)}\n`); + await assert.rejects(cleanupManual({repositoryRoot:repo}),/lifecycle lock.*exists|operator inspection/i); + await writeFile(release,"go"); const run=await preparing; + await readManualOwnership({repositoryRoot:repo}); await assert.rejects(lstat(lock)); + await cleanupManual({repositoryRoot:repo}); await assert.rejects(lstat(run.root)); + } finally { process.env.PATH=priorPath; await writeFile(release,"go").catch(()=>{}); } +}); + +test("the external lifecycle lock prevents old-root/new-root ABA and ownership is read only under lock", async () => { + const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8"); + assert.match(source,/\.p1-manual-acceptance\.lifecycle\.lock/); + const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const lockPath=join(repo,".p1-manual-acceptance.lifecycle.lock"),nonce="f".repeat(64),bytes=`${nonce}\n`; + const handle=await open(lockPath,"wx",0o600); await handle.writeFile(bytes); await handle.sync(); + try { + const old=JSON.parse(await readFile(join(run.root,"ownership.json"),"utf8")); + await rm(run.root,{recursive:true}); await mkdir(run.root,{recursive:true}); + await writeFile(join(run.root,"ownership.json"),JSON.stringify({...old,nonce:"e".repeat(64)}),{mode:0o600}); + for(const operation of [serveManual,stopManual,cleanupManual]){ + await assert.rejects(operation({repositoryRoot:repo}),/lifecycle lock.*exists|operator inspection/i); + assert.equal((await lstat(run.root)).isDirectory(),true); + } + } finally { await handle.close(); await rm(lockPath,{force:true}); } +}); + +test("external lifecycle lock release preserves an exact-byte inode replacement", { concurrency: false }, async () => { + const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim(); + const bin=join(repo,"replacement-bin"),entered=join(repo,"replacement-entered"),release=join(repo,"replacement-release"); + await mkdir(bin); await writeFile(join(bin,"git"),`#!/bin/sh +if [ "$1" = init ] && [ ! -e ${JSON.stringify(entered)} ]; then + : > ${JSON.stringify(entered)} + n=0 + while [ ! -e ${JSON.stringify(release)} ] && [ "$n" -lt 250 ]; do sleep 0.02; n=$((n + 1)); done + [ -e ${JSON.stringify(release)} ] || exit 99 +fi +exec ${JSON.stringify(realGit)} "$@" +`,{mode:0o700}); + const priorPath=process.env.PATH; process.env.PATH=`${bin}:${priorPath}`; let preparing; + try { + preparing=prepareManual({repositoryRoot:repo,skipBuild:true}); + for(let n=0;n<200;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,10));} + await lstat(entered); + const lock=join(repo,".p1-manual-acceptance.lifecycle.lock"),bytes=await readFile(lock); + const original=await lstat(lock),replacement=join(dirname(lock),".replacement-lifecycle-lock"); + await writeFile(replacement,bytes,{mode:0o600}); const replacementEntry=await lstat(replacement); + assert.notEqual(replacementEntry.ino,original.ino); await rename(replacement,lock); await writeFile(release,"go"); + await assert.rejects(preparing,/lifecycle record.*unsafe|lifecycle record.*changed|operator inspection/i); preparing=undefined; + const retained=await lstat(lock); assert.equal(retained.dev,replacementEntry.dev); assert.equal(retained.ino,replacementEntry.ino); + assert.deepEqual(await readFile(lock),bytes); + } finally { + process.env.PATH=priorPath; await writeFile(release,"go").catch(()=>{}); if(preparing)await preparing.catch(()=>{}); + } +}); + +test("prepare records one regular 0600 backend log and no generated supervisor", async () => { + const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const owned=await readManualOwnership({repositoryRoot:repo}),entry=await lstat(join(run.root,"logs/backend.log")); + assert.equal(entry.isFile(),true); assert.equal(entry.isSymbolicLink(),false); assert.equal(entry.mode&0o777,0o600); + assert.deepEqual(owned.backendLog,{path:join(run.root,"logs/backend.log"),dev:entry.dev,ino:entry.ino}); + await assert.rejects(lstat(join(run.root,"installation/runtime/p1-backend-supervisor.mjs"))); +}); + +// A delayed real listener leaves the pre-fix port-check/spawn window open long enough for every +// overlapping call. The backend.pid reservation, rather than scheduler timing, must pick one owner. +test("concurrent serves reserve one exact process and leave no orphan after stop", { concurrency: false }, async () => { + const repo=await fakeRepo(); await installFakeServer(repo,{startupDelay:400}); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const owned=await readManualOwnership({repositoryRoot:repo}); let winner; + try { + const results=await Promise.allSettled(Array.from({length:12},()=>serveManual({repositoryRoot:repo}))); + const fulfilled=results.filter(result=>result.status==="fulfilled"); + assert.equal(fulfilled.length,1,`one serve fulfills: ${results.map(result=>result.status).join(",")}`); + assert.equal(results.filter(result=>result.status==="rejected").length,11); + winner=fulfilled[0].value; + const pidPath=join(run.root,"backend.pid"),record=JSON.parse(await readFile(pidPath,"utf8")),entry=await lstat(pidPath); + assert.equal(entry.mode&0o777,0o600); assert.equal(record.status,"RUNNING"); + assert.match(record.reservationNonce,/^[0-9a-f]{64}$/); assert.equal(record.pid,winner); + assert.equal(record.nonce,owned.nonce); assert.equal(record.root,run.root); assert.equal(record.repositoryRoot,repo); + assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[winner]); + assert.deepEqual(await listenerPids(),[winner]); assert.doesNotThrow(()=>process.kill(winner,0)); + await stopManual({repositoryRoot:repo}); + await assert.rejects(lstat(pidPath)); assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); + assert.deepEqual(await listenerPids(),[]); assert.throws(()=>process.kill(winner,0)); + await cleanupManual({repositoryRoot:repo}); await assert.rejects(lstat(run.root)); + } finally { + for(const pid of await matchingManualServerPids(run.root,owned.nonce))try{process.kill(pid,"SIGTERM");}catch{} + await new Promise(resolvePromise=>setTimeout(resolvePromise,50)); + for(const pid of await matchingManualServerPids(run.root,owned.nonce))try{process.kill(pid,"SIGKILL");}catch{} + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("cooperative stop is serialized and production never sends a numeric terminating signal", { concurrency: false }, async () => { + const source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8"); + assert.doesNotMatch(source,/process\.kill\([^,]+,\s*["']SIG(?:TERM|KILL|INT)/); + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const owned=await readManualOwnership({repositoryRoot:repo}); const pid=await serveManual({repositoryRoot:repo}); + const record=JSON.parse(await readFile(join(run.root,"backend.pid"),"utf8")); assert.equal(record.control.host,"127.0.0.1"); + const unauthorized=await new Promise((resolvePromise,reject)=>{const socket=net.createConnection(record.control),timer=setTimeout(()=>socket.destroy(new Error("control timeout")),1000);let bytes="";socket.setEncoding("utf8");socket.on("connect",()=>socket.end(JSON.stringify({action:"stop",nonce:"0".repeat(64)})));socket.on("data",chunk=>bytes+=chunk);socket.on("error",reject);socket.on("close",()=>{clearTimeout(timer);resolvePromise(bytes);});}); + assert.equal(unauthorized,""); assert.doesNotThrow(()=>process.kill(pid,0)); + const stopped=await Promise.allSettled([stopManual({repositoryRoot:repo}),stopManual({repositoryRoot:repo})]); + assert.equal(stopped.filter(result=>result.status==="fulfilled").length,1); + assert.equal(stopped.filter(result=>result.status==="rejected").length,1); + await assert.rejects(lstat(join(run.root,"backend.pid"))); assert.deepEqual(await listenerPids(),[]); + assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); assert.throws(()=>process.kill(pid,0)); + await cleanupManual({repositoryRoot:repo}); +}); + +test("serve binds the one fixed loopback address, refuses a second PID, and guarded stop removes identity", { concurrency: false }, async () => { + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const priorAmbient=process.env.THT_DWH_API_KEY; process.env.THT_DWH_API_KEY="AMBIENT-MUST-NOT-PASS"; const pid=await serveManual({repositoryRoot:repo}); assert.equal(Number.isSafeInteger(pid),true); + const health=await (await fetch("http://127.0.0.1:8791/health")).json(); assert.equal(health.status,"ok"); assert.equal(health.ambient,undefined); assert.equal(health.wrongMaintenance,undefined); assert.equal(health.maintenance,join(run.root,"installation/data/maintenance.json")); if(priorAmbient===undefined)delete process.env.THT_DWH_API_KEY;else process.env.THT_DWH_API_KEY=priorAmbient; + await assert.rejects(serveManual({repositoryRoot:repo}),/PID record/); + await stopManual({repositoryRoot:repo}); await assert.rejects(lstat(join(run.root,"backend.pid"))); + await assert.rejects(fetch("http://127.0.0.1:8791/health",{signal:AbortSignal.timeout(200)})); + await cleanupManual({repositoryRoot:repo}); +}); + +test("serve requires a 2xx HTTP health check and leaves no orphan on 503", { concurrency: false }, async () => { + const repo=await fakeRepo(); await installFakeServer(repo,{healthStatus:503}); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const owned=await readManualOwnership({repositoryRoot:repo}); + await assert.rejects(serveManual({repositoryRoot:repo}),/health|readiness/i); + await assert.rejects(lstat(join(run.root,"backend.pid"))); + assert.deepEqual(await listenerPids(),[]); assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); +}); + +test("serve launches exact server.js with immutable preload and fixed owned control port", { concurrency: false }, async () => { + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const pid=await serveManual({repositoryRoot:repo}),record=JSON.parse(await readFile(join(run.root,"backend.pid"),"utf8")); + assert.equal(record.pid,pid); assert.equal(record.script,join(repo,"backend/dist/server.js")); + assert.equal(record.control.host,"127.0.0.1"); assert.equal(record.control.port,8792); + assert.match(record.preload,/^data:text\/javascript;base64,/); + const args=(await execFileAsync("ps",["-ww","-p",String(pid),"-o","command="])).stdout.trim(); + assert.equal(args,[process.execPath,"--import",record.preload,record.script,`--p1-manual-nonce=${record.nonce}`,`--p1-root=${run.root}`,`--p1-control-nonce=${record.reservationNonce}`,`--p1-entry-sha256=${record.entrypoint.sha256}`,`--p1-entry-dev=${record.entrypoint.dev}`,`--p1-entry-ino=${record.entrypoint.ino}`].join(" ")); + await stopManual({repositoryRoot:repo}); +}); + +test("serve refuses legacy supervisor, runtime, server and log substitutions before code or outside writes", { concurrency: false }, async () => { + for(const kind of ["legacy-supervisor","runtime-symlink","server-symlink","log-symlink","log-replaced"]){ + const repo=await fakeRepo(),marker=join(repo,`outside-${kind}.marker`); await installFakeServer(repo,{marker}); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),outside=join(repo,`outside-${kind}`); await mkdir(outside); + if(kind==="legacy-supervisor")await symlink(join(outside,"outside.mjs"),join(run.root,"installation/runtime/p1-backend-supervisor.mjs")); + if(kind==="runtime-symlink"){await rm(join(run.root,"installation/runtime"),{recursive:true});await symlink(outside,join(run.root,"installation/runtime"));} + if(kind==="server-symlink"){ + const external=join(outside,"server.js"); await writeFile(external,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(marker)},"bad");`); + await rm(join(repo,"backend/dist/server.js")); await symlink(external,join(repo,"backend/dist/server.js")); + } + if(kind==="log-symlink"){await rm(join(run.root,"logs/backend.log"));await symlink(join(outside,"captured.log"),join(run.root,"logs/backend.log"));} + if(kind==="log-replaced"){await rm(join(run.root,"logs/backend.log"));await writeFile(join(run.root,"logs/backend.log"),"",{mode:0o600});} + await assert.rejects(serveManual({repositoryRoot:repo}),/unsafe|identity|symlink|legacy|realpath|log/i,kind); + await assert.rejects(lstat(marker),undefined,`${kind} must refuse before server execution`); + assert.deepEqual(await readdir(outside),kind==="server-symlink"?["server.js"]:[]); + await assert.rejects(lstat(join(run.root,"backend.pid"))); + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("serve refuses an occupied fixed control port before spawning", { concurrency: false }, async () => { + const repo=await fakeRepo(),marker=join(repo,"server-executed"); await installFakeServer(repo,{marker}); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const blocker=net.createServer(); await new Promise((resolvePromise,reject)=>blocker.once("error",reject).listen(8792,"127.0.0.1",resolvePromise)); + try { await assert.rejects(serveManual({repositoryRoot:repo}),/8792.*occupied|control.*occupied/); } finally { await new Promise(resolvePromise=>blocker.close(resolvePromise)); } + await assert.rejects(lstat(marker)); await assert.rejects(lstat(join(run.root,"backend.pid"))); +}); + +test("serve refuses an occupied fixed port and never creates a PID or verdict", { concurrency: false }, async () => { + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const blocker=net.createServer(); await new Promise((resolvePromise,reject)=>blocker.once("error",reject).listen(8791,"127.0.0.1",resolvePromise)); + try { await assert.rejects(serveManual({repositoryRoot:repo}),/occupied/); } finally { await new Promise(resolvePromise=>blocker.close(resolvePromise)); } + await assert.rejects(lstat(join(run.root,"backend.pid"))); await assert.rejects(lstat(join(run.root,"VERDICT.md"))); +}); + +test("serve and cleanup refuse stale or mismatched PID records without signaling", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + await writeFile(join(run.root,"backend.pid"),JSON.stringify({pid:999999,nonce:"wrong"}),{mode:0o600}); + await assert.rejects(serveManual({repositoryRoot:repo}),/PID record/); + await assert.rejects(stopManual({repositoryRoot:repo}),/identity mismatch/); + await assert.rejects(cleanupManual({repositoryRoot:repo}),/identity|stale/); + assert.equal((await lstat(run.root)).isDirectory(),true); +}); + +test("serve refuses non-loopback ownership without creating process state", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const ownershipPath=join(run.root,"ownership.json"),owned=JSON.parse(await readFile(ownershipPath,"utf8")); + owned.listener.host="0.0.0.0"; await writeFile(ownershipPath,JSON.stringify(owned)); + await assert.rejects(serveManual({repositoryRoot:repo}),/identity|loopback|bind/); + await assert.rejects(lstat(join(run.root,"backend.pid"))); +}); + +test("cleanup refuses a correctly owned live server until guarded stop", { concurrency: false }, async () => { + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const pid=await serveManual({repositoryRoot:repo}); + try { + await assert.rejects(cleanupManual({repositoryRoot:repo}),/owned backend is live|stop first/); + assert.doesNotThrow(()=>process.kill(pid,0)); + } finally { + try { await stopManual({repositoryRoot:repo}); } catch { try { process.kill(pid,"SIGTERM"); } catch {} } + } + await cleanupManual({repositoryRoot:repo}); await assert.rejects(lstat(run.root)); +}); + +async function processStartIdentity(pid) { + return (await execFileAsync("ps",["-p",String(pid),"-o","lstart="])).stdout.trim(); +} +async function stopTestProcess(child) { + if (child.exitCode === null) child.kill("SIGTERM"); + if (child.exitCode === null) await new Promise(resolvePromise=>child.once("exit",resolvePromise)); +} + +test("live foreign executable, cwd, start and args mismatches are never signaled", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const owned=JSON.parse(await readFile(join(run.root,"ownership.json"),"utf8")); + const script=join(run.root,"installation/runtime/p1-backend-supervisor.mjs"),nonceArg=`--p1-manual-nonce=${owned.nonce}`,rootArg=`--p1-root=${run.root}`,reservationNonce="a".repeat(64),controlArg=`--p1-control-nonce=${reservationNonce}`; + await writeFile(script,"setInterval(()=>{},1000);\n",{mode:0o600}); + const cases=[ + ["executable",()=>spawn("bash",["-c","while :; do sleep 1; done",script,nonceArg,rootArg,controlArg],{cwd:repo,stdio:"ignore"}),{}], + ["cwd",()=>spawn(process.execPath,[script,nonceArg,rootArg,controlArg],{cwd:tmpdir(),stdio:"ignore"}),{}], + ["start",()=>spawn(process.execPath,[script,nonceArg,rootArg,controlArg],{cwd:repo,stdio:"ignore"}),{startIdentity:"foreign-start"}], + ["args",()=>spawn(process.execPath,[script],{cwd:repo,stdio:"ignore"}),{}], + ]; + for(const [name,start,override] of cases){ + const child=start(); + try { + let actualStart=""; for(let n=0;n<50&&!actualStart;n++){try{actualStart=await processStartIdentity(child.pid);}catch{} if(!actualStart)await new Promise(r=>setTimeout(r,20));} + assert.ok(actualStart,`live ${name} process started`); + const record={schemaVersion:1,kind:"p1-manual-backend",status:"RUNNING",pid:child.pid,reservationNonce,nonce:owned.nonce,root:run.root,repositoryRoot:repo,executable:process.execPath,script,startIdentity:actualStart,control:{host:"127.0.0.1",port:1},...override}; + await writeFile(join(run.root,"backend.pid"),JSON.stringify(record),{mode:0o600}); + await assert.rejects(stopManual({repositoryRoot:repo}),/process identity mismatch|refusing cooperative control/); + assert.doesNotThrow(()=>process.kill(child.pid,0)); + await rm(join(run.root,"backend.pid")); + } finally { await stopTestProcess(child); await rm(join(run.root,"backend.pid"),{force:true}); } + } +}); + +test("generated render command validates saved responses and owned snapshot before renderer", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const script=join(run.root,"commands/render-1.sh"), output=join(run.root,"rendered/runtime-1.yaml"); + const invoke=()=>execFileAsync("bash",[script],{cwd:repo}); + await assert.rejects(invoke(),/saved response is missing/); + await writeFile(join(run.root,"responses/read-p1-filesystem.json"),"{"); await writeFile(join(run.root,"responses/pull.json"),"{}"); + await assert.rejects(invoke(),/malformed JSON/); + const a="a".repeat(40),b="b".repeat(40),outside=join(repo,"outside.yaml"); await writeFile(outside,"x"); + await writeFile(join(run.root,"responses/read-p1-filesystem.json"),JSON.stringify({revision:{commit:a,snapshotPath:outside}})); await writeFile(join(run.root,"responses/pull.json"),JSON.stringify({head:b})); + await assert.rejects(invoke(),/revisions differ/); + await writeFile(join(run.root,"responses/pull.json"),JSON.stringify({head:a})); await assert.rejects(invoke(),/snapshot escapes/); + await assert.rejects(lstat(output)); +}); + +const renderSnapshotYaml=`workspace: + schema_version: 3 + id: p1-filesystem + name: P1 filesystem + language: en +dwh: + engine: postgres + database: postgres + schema: public + supported_transports: [postgres_direct] +semantic_index: + vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine} + embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024} +llm_policy: + allowed: [zai/glm-5.2] +evidence: + source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760} + policy: {max_chunk_chars: 4000, retain_published_generations: 3} +`; + +test("generated render command binds snapshot bytes to the commit manifest and Git blob end to end", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); + const author=join(run.root,"author"); await mkdir(join(author,"workspaces"),{recursive:true}); + await writeFile(join(author,"workspaces","p1-filesystem.yaml"),renderSnapshotYaml); + await execFileAsync("git",["add","workspaces"],{cwd:author}); await execFileAsync("git",["commit","-m","publish p1"],{cwd:author}); await execFileAsync("git",["push","origin","main"],{cwd:author}); + const commit=(await execFileAsync("git",["rev-parse","HEAD"],{cwd:author})).stdout.trim(); + const blob=(await execFileAsync("git",["rev-parse","HEAD:workspaces/p1-filesystem.yaml"],{cwd:author})).stdout.trim(); + await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]); + const commitDir=join(run.root,"installation/registry/snapshots",commit); await mkdir(commitDir,{recursive:true}); + const snapshot=join(commitDir,"p1-filesystem.yaml"),snapshotPath=snapshot,snapshotSha=sha256(renderSnapshotYaml); + await writeFile(snapshot,renderSnapshotYaml); + const readPath=join(run.root,"responses/read-p1-filesystem.json"),pullPath=join(run.root,"responses/pull.json"),script=join(run.root,"commands/render-1.sh"),script2=join(run.root,"commands/render-2.sh"),output=join(run.root,"rendered/runtime-1.yaml"),output2=join(run.root,"rendered/runtime-2.yaml"); + const rendererStub=join(repo,"backend/scripts/p1-render-snapshot.mjs"),stubArgs=join(run.root,"rendered/stub-args.json"); + await writeFile(rendererStub,`import { writeFileSync } from "node:fs";\nwriteFileSync(${JSON.stringify(stubArgs)}, JSON.stringify(process.argv.slice(2)));\n`); + const revision={id:"p1-filesystem",commit,blob,snapshotPath}; + const manifest=(entry=revision)=>({head:commit,revisions:[entry],files:{"p1-filesystem.yaml":snapshotSha}}); + await writeFile(readPath,JSON.stringify({revision})); await writeFile(pullPath,JSON.stringify({head:commit})); + await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest.*(missing|unbounded)/i); + await assert.rejects(lstat(output)); + const legacyRevision={...revision}; legacyRevision[["st","ate"].join("")]=["oper","ational"].join(""); + await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest(legacyRevision))); + await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest revision is invalid/); + await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest({...revision,unexpected:"field"}))); + await assert.rejects(execFileAsync("bash",[script],{cwd:repo}),/snapshot manifest revision is invalid/); + await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest())); + await execFileAsync("bash",[script],{cwd:repo}); + assert.deepEqual(JSON.parse(await readFile(stubArgs,"utf8")),["--ownership",join(run.root,"ownership.json"),"--snapshot",snapshot,"--output",output,"--snapshot-sha256",snapshotSha]); + await writeFile(snapshot,renderSnapshotYaml.replace("max_chunk_chars: 4000","max_chunk_chars: 3999")); + await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/snapshot bytes differ from manifest digest/); + await assert.rejects(lstat(output2)); + await writeFile(snapshot,renderSnapshotYaml); + await writeFile(join(commitDir,"snapshot.json"),JSON.stringify({...manifest(),head:"c".repeat(40)})); + await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/snapshot manifest identity is invalid/); + await assert.rejects(lstat(output2)); + for(const malformed of [ + {id:"p1-filesystem",commit,blob}, + {...revision,id:"p1-http"}, + {...revision,commit:"c".repeat(40)}, + {...revision,blob:"f".repeat(39)}, + {...revision,blob:[blob]}, + {...revision,snapshotPath:join(commitDir,"wrong.yaml")}, + ]){ + await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest(malformed))); + await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/snapshot manifest revision is invalid/); + } + await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest({...revision,blob:"f".repeat(40)}))); + await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/saved revision blob differs from snapshot manifest/); + await assert.rejects(lstat(output2)); + await writeFile(join(commitDir,"snapshot.json"),JSON.stringify(manifest())); + await writeFile(readPath,JSON.stringify({revision:{...revision,blob:"f".repeat(40)}})); + await assert.rejects(execFileAsync("bash",[script2],{cwd:repo}),/saved revision blob differs/); + await assert.rejects(lstat(output2)); +}); + + +const sha256=bytes=>createHash("sha256").update(bytes).digest("hex"); +async function makeExportZip(directory,name,{payloads={},manifest,workspaceId="p1-filesystem",extra=false,symlinkReadme=false}={}) { + const source=join(directory,`${name}-source`),zip=join(directory,`${name}.zip`); await mkdir(source,{recursive:true}); + const files={"workspace.yaml":`workspace:\n id: ${workspaceId}\n`,"contract.env.example":"SAFE=path\n","README.md":"# Safe\n",...payloads}; + const value=manifest??{schema_version:1,workspace_id:workspaceId,files:Object.fromEntries(Object.entries(files).map(([n,b])=>[n,sha256(b)]))}; + await writeFile(join(source,"manifest.json"),JSON.stringify(value)); + for(const [file,bytes] of Object.entries(files))if(!(symlinkReadme&&file==="README.md"))await writeFile(join(source,file),bytes); + if(symlinkReadme)await symlink("workspace.yaml",join(source,"README.md")); + if(extra)await writeFile(join(source,"extra.txt"),"extra"); + const names=["manifest.json","workspace.yaml","contract.env.example","README.md",...(extra?["extra.txt"]:[])]; + await execFileAsync("zip",["-q",...(symlinkReadme?["-y"]:[]),zip,...names],{cwd:source}); return zip; +} + +test("generated ZIP verifier enforces exact manifest mapping, hashes, entries and regular files", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh"); + const invoke=async(name,options={})=>execFileAsync("bash",[extract,await makeExportZip(run.root,name,options),join(run.root,"exports/extracted",name),"p1-filesystem"],{cwd:repo}); + await invoke("valid"); + const safe={"workspace.yaml":"workspace:\n id: p1-filesystem\n","contract.env.example":"SAFE=path\n","README.md":"# Safe\n"}; + const hashes=Object.fromEntries(Object.entries(safe).map(([n,b])=>[n,sha256(b)])); + await assert.rejects(invoke("missing-map",{manifest:{schema_version:1,workspace_id:"p1-filesystem",files:{"workspace.yaml":hashes["workspace.yaml"],"contract.env.example":hashes["contract.env.example"]}}}),/manifest/i); + await assert.rejects(invoke("short-hash",{manifest:{schema_version:1,workspace_id:"p1-filesystem",files:{...hashes,"README.md":"abc"}}}),/manifest/i); + await assert.rejects(invoke("extra-entry",{extra:true}),/unsafe-zip/); + await assert.rejects(invoke("nonregular",{symlinkReadme:true}),/symlink|nonregular/); +}); + +test("generated ZIP verifier binds identity, stages source once, and rejects symlink output ancestry", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh"); + for(const id of ["p1-filesystem","p1-http","p1-s3"]){ + const zip=await makeExportZip(run.root,`valid-${id}`,{workspaceId:id}); + await execFileAsync("bash",[extract,zip,join(run.root,"exports/extracted",id),id],{cwd:repo}); + } + const wrong=await makeExportZip(run.root,"wrong-valid-id",{workspaceId:"p1-http"}); + await assert.rejects(execFileAsync("bash",[extract,wrong,join(run.root,"exports/extracted/wrong-id"),"p1-s3"],{cwd:repo}),/workspace.*identity|workspace_id/i); + const descriptorMismatch=await makeExportZip(run.root,"descriptor-mismatch",{workspaceId:"p1-http",payloads:{"workspace.yaml":"workspace:\n id: p1-s3\n"}}); + await assert.rejects(execFileAsync("bash",[extract,descriptorMismatch,join(run.root,"exports/extracted/descriptor-mismatch"),"p1-http"],{cwd:repo}),/workspace.*identity|descriptor/i); + + const outside=join(repo,"outside-extract"); await mkdir(outside); await rm(join(run.root,"exports/extracted"),{recursive:true}); await symlink(outside,join(run.root,"exports/extracted")); + const safe=await makeExportZip(run.root,"symlink-parent"); + await assert.rejects(execFileAsync("bash",[extract,safe,join(run.root,"exports/extracted/escape"),"p1-filesystem"],{cwd:repo}),/symlink|owned|unsafe/i); + assert.deepEqual(await readdir(outside),[]); await rm(join(run.root,"exports/extracted")); await mkdir(join(run.root,"exports/extracted")); + + const original=await makeExportZip(run.root,"replace-original"),replacement=await makeExportZip(run.root,"replace-malicious",{extra:true}); + const bin=join(run.root,"swap-bin"),markerPath=join(run.root,"swap-once"); await mkdir(bin); + const realUnzip=(await execFileAsync("which",["unzip"])).stdout.trim(); + await writeFile(join(bin,"unzip"),`#!/bin/sh +if [ ! -e "$P1_SWAP_MARKER" ]; then cp "$P1_SWAP_REPLACEMENT" "$P1_SWAP_ORIGINAL"; : > "$P1_SWAP_MARKER"; fi +exec ${realUnzip} "$@" +`,{mode:0o700}); + await execFileAsync("bash",[extract,original,join(run.root,"exports/extracted/staged-source"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:markerPath,P1_SWAP_REPLACEMENT:replacement,P1_SWAP_ORIGINAL:original}}); + await lstat(markerPath); await lstat(join(run.root,"exports/extracted/staged-source/manifest.json")); + const generated=await readFile(extract,"utf8"); assert.match(generated,/source_fd = os\.open\(zip_path/); assert.match(generated,/dir_fd=base_fd/); assert.match(generated,/O_NOFOLLOW/); assert.match(generated,/staged archive SHA mismatch/); +}); + +test("generated ZIP verifier anchors output when the extraction base is swapped on first unzip", async () => { + const repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh"); + const zip=await makeExportZip(run.root,"ancestor-swap"),base=join(run.root,"exports/extracted"),moved=join(run.root,"exports/extracted-original"),outside=join(repo,"outside-extraction-race"); + const bin=join(repo,"unzip-swap-bin"),marker=join(repo,"unzip-swapped"),realUnzip=(await execFileAsync("which",["unzip"])).stdout.trim(); + await mkdir(bin); await mkdir(outside); + await writeFile(join(bin,"unzip"),`#!/bin/sh +if [ ! -e "$P1_SWAP_MARKER" ]; then + mv "$P1_SWAP_BASE" "$P1_SWAP_MOVED" + ln -s "$P1_SWAP_OUTSIDE" "$P1_SWAP_BASE" + : > "$P1_SWAP_MARKER" +fi +exec ${realUnzip} "$@" +`,{mode:0o700}); + await assert.rejects(execFileAsync("bash",[extract,zip,join(base,"escaped"),"p1-filesystem"],{cwd:repo,env:{...process.env,PATH:`${bin}:${process.env.PATH}`,P1_SWAP_MARKER:marker,P1_SWAP_BASE:base,P1_SWAP_MOVED:moved,P1_SWAP_OUTSIDE:outside}}),/owned extraction root|identity|changed|unsafe/i); + await lstat(marker); assert.deepEqual(await readdir(outside),[]); +}); + +test("generated ZIP verifier scans all four extracted byte streams for Evidence and canaries", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),extract=join(run.root,"commands/extract-export.sh"); + const markers=["P1 manually curated Evidence","DWH-"+"d".repeat(32),"CANARY-MUST-BE-REJECTED"]; + for(const marker of markers)for(const target of ["manifest.json","workspace.yaml","contract.env.example","README.md"]){ + const name=`scan-${markers.indexOf(marker)}-${target.replaceAll(".","-")}`,payloads=target==="manifest.json"?{}:{[target]:marker}; + const files={"workspace.yaml":"workspace:\n id: p1-filesystem\n","contract.env.example":"SAFE=path\n","README.md":"# Safe\n",...payloads}; + const manifest={schema_version:1,workspace_id:target==="manifest.json"?marker:"p1-filesystem",files:Object.fromEntries(Object.entries(files).map(([n,b])=>[n,sha256(b)]))}; + const zip=await makeExportZip(run.root,name,{payloads,manifest}); + await assert.rejects(execFileAsync("bash",[extract,zip,join(run.root,"exports/extracted",name),"p1-filesystem"],{cwd:repo}),error=>/Evidence|canary/.test(error.stderr)&&!error.stderr.includes(marker),`${target} must reject ${marker.slice(0,8)}`); + } +}); + +test("generated secret scan excludes only the exact request fixture and hides fixed canary", async () => { + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const installed=join(run.root,"installation/registry/repo"),scan=join(run.root,"commands/secret-scan.sh"),canary="CANARY-MUST-BE-REJECTED"; + await execFileAsync("git",["clone",join(run.root,"remote.git"),installed]); await execFileAsync("bash",[scan],{cwd:repo}); + const leak=join(run.root,"responses/requests/invalid-credential.json"); await mkdir(dirname(leak),{recursive:true}); await writeFile(leak,canary); + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary)); +}); + +test("generated secret scan reads Git metadata and arbitrary dot-git directories without printing values", async () => { + for(const rel of ["author/.git/manual-leak","responses/.git/leak"]){ + const canary="SECRET-"+"a".repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),scan=join(run.root,"commands/secret-scan.sh"); + await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]); + await mkdir(dirname(join(run.root,rel)),{recursive:true}); await writeFile(join(run.root,rel),canary); + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary found/.test(error.stderr)&&!error.stderr.includes(canary),`${rel} must be scanned with a redacted finding`); + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("generated secret scan reads raw dangling commit, tag, and tree objects without printing values", async () => { + for(const kind of ["commit","tag","tree"]){ + const canary="SESSION-"+({commit:"b",tag:"c",tree:"d"}[kind]).repeat(32),repo=await fakeRepo(),run=await prepareManual({repositoryRoot:repo,skipBuild:true}),author=join(run.root,"author"),scan=join(run.root,"commands/secret-scan.sh"); + await execFileAsync("git",["clone",join(run.root,"remote.git"),join(run.root,"installation/registry/repo")]); + if(kind==="commit"){ + await execFileAsync("git",["commit","--allow-empty","-m",canary],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD^"],{cwd:author}); + }else if(kind==="tag"){ + await execFileAsync("git",["tag","-a","temporary-canary-tag","-m",canary],{cwd:author}); await execFileAsync("git",["tag","-d","temporary-canary-tag"],{cwd:author}); + }else{ + await writeFile(join(author,canary),"safe tree payload\n"); await execFileAsync("git",["add",canary],{cwd:author}); await execFileAsync("git",["write-tree"],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD"],{cwd:author}); + } + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git object/.test(error.stderr)&&!error.stderr.includes(canary),`${kind} raw bytes must be scanned with a redacted finding`); + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("generated secret scan checks unreachable blobs and dangling commits without printing values", async () => { + for(const kind of ["unreachable-blob","dangling-commit"]){ + const value=kind==="unreachable-blob"?"SECRET-"+"e".repeat(32):"SECRET-"+"f".repeat(32); + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const author=join(run.root,"author"),installed=join(run.root,"installation/registry/repo"),scan=join(run.root,"commands/secret-scan.sh"); + await execFileAsync("git",["clone",join(run.root,"remote.git"),installed]); const file=join(author,"dangling-secret"); await writeFile(file,value); + if(kind==="unreachable-blob"){await execFileAsync("git",["hash-object","-w",file],{cwd:author}); await rm(file);} + else {await execFileAsync("git",["add","dangling-secret"],{cwd:author}); await execFileAsync("git",["commit","-m","dangling secret"],{cwd:author}); await execFileAsync("git",["reset","--hard","HEAD^"],{cwd:author});} + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git blob/.test(error.stderr)&&!error.stderr.includes(value),`${kind} must be scanned`); + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("generated secret scan checks randomized and fixed canaries in reachable Git without printing values", async () => { + for(const canary of ["DWH-"+"c".repeat(32),"CANARY-MUST-BE-REJECTED"]){ + const repo=await fakeRepo(); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const installed=join(run.root,"installation/registry/repo"),author=join(run.root,"author"),scan=join(run.root,"commands/secret-scan.sh"); + await execFileAsync("git",["clone",join(run.root,"remote.git"),installed]); + await writeFile(join(author,"temporary-secret"),canary); await execFileAsync("git",["add","temporary-secret"],{cwd:author}); await execFileAsync("git",["commit","-m","temporary canary"],{cwd:author}); await execFileAsync("git",["rm","temporary-secret"],{cwd:author}); await execFileAsync("git",["commit","-m","remove canary"],{cwd:author}); + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git blob/.test(error.stderr)&&!error.stderr.includes(canary)); + await rm(run.root,{recursive:true,force:true}); + } +}); + + +test("prepare publishes cleanable ownership before lab population", { concurrency: false }, async () => { + const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim(),bin=join(repo,"failing-bin"); + await installFakeServer(repo); await mkdir(bin); + await writeFile(join(bin,"git"),`#!/bin/sh +if [ "$1" = init ]; then exit 71; fi +exec ${JSON.stringify(realGit)} "$@" +`,{mode:0o700}); + const prior=process.env.PATH; process.env.PATH=`${bin}:${prior}`; + try { await assert.rejects(prepareManual({repositoryRoot:repo,skipBuild:true})); } + finally { process.env.PATH=prior; } + const owned=await readManualOwnership({repositoryRoot:repo}); + assert.equal(owned.stage,"PREPARING"); + await cleanupManual({repositoryRoot:repo}); + await assert.rejects(lstat(fixedManualRoot(repo))); +}); + +test("stable repo-root lifecycle namespace survives manual-parent rename and cleans partial prepare", { concurrency: false }, async () => { + const repo=await fakeRepo(),realGit=(await execFileAsync("which",["git"])).stdout.trim(); + const bin=join(repo,"rename-lock-bin"),entered=join(repo,"rename-entered"),release=join(repo,"rename-release"); + await installFakeServer(repo); await mkdir(bin); + await writeFile(join(bin,"git"),`#!/bin/sh +if [ "$1" = init ] && [ ! -e ${JSON.stringify(entered)} ]; then + : > ${JSON.stringify(entered)} + while [ ! -e ${JSON.stringify(release)} ]; do sleep 0.01; done +fi +exec ${JSON.stringify(realGit)} "$@" +`,{mode:0o700}); + const prior=process.env.PATH; process.env.PATH=`${bin}:${prior}`; let preparing; + const parent=join(repo,".artifacts/manual-acceptance"),moved=join(repo,".artifacts/manual-acceptance-moved"); + try { + preparing=prepareManual({repositoryRoot:repo,skipBuild:true}); + for(let n=0;n<300;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,10));} + await lstat(entered); await rename(parent,moved); await mkdir(parent,{recursive:true}); await writeFile(join(parent,"public-sibling"),"keep"); await writeFile(join(moved,"moved-sibling"),"keep"); + await assert.rejects(cleanupManual({repositoryRoot:repo}),/lifecycle lock|operator inspection/i); + await writeFile(release,"go"); await assert.rejects(preparing,/identity|changed|unsafe|manual/i); preparing=undefined; + assert.equal(await readFile(join(parent,"public-sibling"),"utf8"),"keep"); + assert.equal(await readFile(join(moved,"moved-sibling"),"utf8"),"keep"); + await assert.rejects(lstat(join(moved,"p1"))); + await assert.rejects(lstat(join(repo,".p1-manual-acceptance.lifecycle.lock"))); + } finally { process.env.PATH=prior; await writeFile(release,"go").catch(()=>{}); if(preparing)await preparing.catch(()=>{}); } +}); + +test("all four lifecycle operations serialize on the stable repo-root lock", async () => { + const repo=await fakeRepo(); await installFakeServer(repo); await prepareManual({repositoryRoot:repo,skipBuild:true}); + const lock=join(repo,".p1-manual-acceptance.lifecycle.lock"); await writeFile(lock,"foreign",{mode:0o600}); + try { + for(const operation of [prepareManual,serveManual,stopManual,cleanupManual]) + await assert.rejects(operation({repositoryRoot:repo,skipBuild:true}),/lifecycle lock|operator inspection/i); + } finally { await rm(lock,{force:true}); } +}); + +test("prepare binds production entry bytes and serve rejects a regular replacement", { concurrency: false }, async () => { + const repo=await fakeRepo(),malicious=join(repo,"malicious-executed"); await installFakeServer(repo); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo}),script=join(repo,"backend/dist/server.js"); + const prepared=await readFile(script); assert.equal(owned.entrypoint.sha256,sha256(prepared)); + await rm(script); await writeFile(script,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(malicious)},"bad");setInterval(()=>{},1000);`); + await assert.rejects(serveManual({repositoryRoot:repo}),/entrypoint|production server.*identity/i); + await assert.rejects(lstat(malicious)); await assert.rejects(lstat(join(run.root,"backend.pid"))); +}); + +test("serve rejects replacement of an imported production dependency", { concurrency: false }, async () => { + const repo=await fakeRepo(),malicious=join(repo,"dependency-executed"); await installFakeServer(repo); + const server=join(repo,"backend/dist/server.js"), original=await readFile(server); await writeFile(join(repo,"backend/dist/dep.js"),"export const dependency = true;\n"); await writeFile(server,`import \"./dep.js\";\n${original}`); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}); const before=await readFile(server); await writeFile(join(repo,"backend/dist/dep.js"),`import {writeFileSync} from \"node:fs\"; writeFileSync(${JSON.stringify(malicious)},\"bad\");\n`); + assert.deepEqual(await readFile(server),before); await assert.rejects(serveManual({repositoryRoot:repo}),/distribution|identity|manifest/i); await assert.rejects(lstat(malicious)); await assert.rejects(lstat(join(run.root,"backend.pid"))); +}); + +test("serve refuses a replaced backend dist dependency after prepare", { concurrency: false }, async () => { + const repo=await fakeRepo(); await installFakeServer(repo); + const dependency=join(repo,"backend/dist/dependency.js"); await writeFile(dependency,"export const value = 1;\n"); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),marker=join(repo,"dependency-replaced-executed"); + await writeFile(dependency,`import { writeFileSync } from "node:fs";writeFileSync(${JSON.stringify(marker)},"bad");export const value = 2;\n`); + await assert.rejects(serveManual({repositoryRoot:repo}),/distribution|manifest|identity/i); + await assert.rejects(lstat(marker)); await assert.rejects(lstat(join(run.root,"backend.pid"))); +}); + +test("serve refuses a deterministic dependency check/load swap before execution", { concurrency: false }, async () => { + const repo=await fakeRepo(),marker=join(repo,"dep-swap-executed"); + await writeFile(join(repo,"backend/dist/dep.js"),`export function start(){}\n`); + await writeFile(join(repo,"backend/dist/server.js"),`import http from "node:http";\nimport { start } from "./dep.js";\nstart();\nconst server=http.createServer((req,res)=>{res.statusCode=req.url==="/health"?200:200;res.setHeader("content-type","application/json");res.end(JSON.stringify({status:"ok"}));});\nserver.listen(Number(process.env.PORT),process.env.HOST);\n`); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo}),replacement=join(repo,"dep-replacement.js"); + await writeFile(replacement,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(marker)},"executed");export function start(){}\n`); + await assert.rejects(serveManual({repositoryRoot:repo,beforeSpawn:async()=>rename(replacement,join(repo,"backend/dist/dep.js"))}),/identity|changed|refused|distribution|module|readiness|failed/i); + await assert.rejects(lstat(marker)); await assert.rejects(lstat(join(run.root,"backend.pid"))); + assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); assert.deepEqual(await listenerPids(),[]); +}); + +test("immutable loader serves verified cached dependency bytes after a same-path regular replacement", { concurrency: false }, async () => { + const repo=await fakeRepo(),marker=join(repo,"dep-replacement-executed"); + await writeFile(join(repo,"backend/dist/dep.js"),`export function mark(){ globalThis.__depSource = "original"; }\n`); + await writeFile(join(repo,"backend/dist/server.js"),`import http from "node:http";\nlet n = 0;\nconst server=http.createServer(async (req,res)=>{ if(req.url==="/load"){ await import(\`./dep.js?v=\${++n}\`).then(m=>m.mark()); } res.setHeader("content-type","application/json"); res.end(JSON.stringify({status:"ok",dep:globalThis.__depSource??"unset"})); });\nserver.listen(Number(process.env.PORT),process.env.HOST);\n`); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo}); + const pid=await serveManual({repositoryRoot:repo}); + try { + const health=async()=>(await (await fetch("http://127.0.0.1:8791/health")).json()); + const load=async()=>{ await fetch("http://127.0.0.1:8791/load"); return (await health()).dep; }; + for(let n=0;n<60;n++){try{if((await health()).status==="ok")break;}catch{}await new Promise(r=>setTimeout(r,50));} + assert.equal(await load(),"original"); + await writeFile(join(repo,"backend/dist/dep.js"),`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(marker)},"executed");export function mark(){ globalThis.__depSource = "replaced"; }\n`); + assert.equal(await load(),"original"); await assert.rejects(lstat(marker)); + await writeFile(join(repo,"backend/dist/dep.js"),`export function mark(){ globalThis.__depSource = "original"; }\n`); + } finally { + try { await stopManual({repositoryRoot:repo}); } catch { try { process.kill(pid,"SIGTERM"); } catch {} } + } + await cleanupManual({repositoryRoot:repo}); +}); + +test("opened production FD prevents deterministic check-spawn replacement execution", { concurrency: false }, async () => { + const repo=await fakeRepo(),safe=join(repo,"safe-executed"),malicious=join(repo,"malicious-executed"); await installFakeServer(repo,{marker:safe}); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),script=join(repo,"backend/dist/server.js"),replacement=join(repo,"replacement-server.js"),owned=await readManualOwnership({repositoryRoot:repo}); + await writeFile(replacement,`import {writeFileSync} from "node:fs";writeFileSync(${JSON.stringify(malicious)},"bad");setInterval(()=>{},1000);`); + await assert.rejects(serveManual({repositoryRoot:repo,beforeSpawn:async()=>rename(replacement,script)}),/entrypoint|identity|changed|readiness|failed|distribution|module/i); + await assert.rejects(lstat(malicious)); await assert.rejects(lstat(join(run.root,"backend.pid"))); + assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); +}); + +test("foreign 8791 health can never make a delayed authenticated child RUNNING", { concurrency: false }, async () => { + const repo=await fakeRepo(),entered=join(repo,"entry-loaded"); await installFakeServer(repo,{startupDelay:700,marker:entered}); + const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),owned=await readManualOwnership({repositoryRoot:repo}); + const serving=serveManual({repositoryRoot:repo}); + for(let n=0;n<300;n++){try{await lstat(entered);break;}catch{} await new Promise(r=>setTimeout(r,5));} + await lstat(entered); const foreign=net.createServer((socket)=>socket.end("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nforeign")); + await new Promise((resolvePromise,reject)=>foreign.once("error",reject).listen(8791,"127.0.0.1",resolvePromise)); + try { + await assert.rejects(serving,/readiness|listener|entrypoint|backend failed/i); + await assert.rejects(lstat(join(run.root,"backend.pid"))); + assert.equal((await listenerPids()).includes(process.pid),true); + assert.deepEqual(await matchingManualServerPids(run.root,owned.nonce),[]); + } finally { await new Promise(resolvePromise=>foreign.close(resolvePromise)); } +}); + +test("absence gate rejects evidence and every P2 materialization artifact name", async () => { + for(const rel of ["artifacts/evidence/generation/chunk.md","responses/materialization","responses/preprocess-state","responses/embedding-cache","responses/qdrant-state","responses/ACTIVE","responses/retention-policy"]){ + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),check=join(run.root,"commands/absence-check.sh"),target=join(run.root,rel); + if(rel.endsWith("materialization"))await mkdir(target,{recursive:true}); else {await mkdir(dirname(target),{recursive:true}); await writeFile(target,"safe");} + await assert.rejects(execFileAsync("bash",[check],{cwd:repo}),/out-of-scope/i,rel); + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("secret scan discovers every arbitrary git repository including unreachable objects", async () => { + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),gitdir=join(run.root,"responses/.git"),scan=join(run.root,"commands/secret-scan.sh"),canary="SECRET-"+"9".repeat(32); + await execFileAsync("git",["init","--bare",gitdir]); const blob=join(run.root,"responses/canary-blob"); await writeFile(blob,canary); await execFileAsync("git",["--git-dir",gitdir,"hash-object","-w",blob]); await rm(blob); + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/Git object|secret canary/.test(error.stderr)&&!error.stderr.includes(canary)); +}); + +test("secret scan bounds and scans filesystem names plus loose ref names", async () => { + for(const kind of ["file","directory","loose-ref"]){ + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),scan=join(run.root,"commands/secret-scan.sh"),canary="SESSION-"+"8".repeat(32); + if(kind==="file")await writeFile(join(run.root,"responses",canary),"safe"); + if(kind==="directory")await mkdir(join(run.root,"responses",canary)); + if(kind==="loose-ref"){const ref=join(run.root,"author/.git/refs/heads",canary);await mkdir(dirname(ref),{recursive:true});await writeFile(ref,"0".repeat(40)+"\n");} + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>/secret canary/.test(error.stderr)&&!error.stderr.includes(canary),kind); + await rm(run.root,{recursive:true,force:true}); + } +}); + +test("extractor and scanner operational diagnostics redact canary-bearing paths", async () => { + const repo=await fakeRepo(); await installFakeServer(repo); const run=await prepareManual({repositoryRoot:repo,skipBuild:true}),canary="SECRET-"+"7".repeat(32),extract=join(run.root,"commands/extract-export.sh"),scan=join(run.root,"commands/secret-scan.sh"); + const missing=join(run.root,"exports/raw",`${canary}.zip`),output=join(run.root,"exports/extracted",canary); + await assert.rejects(execFileAsync("bash",[extract,missing,output,"p1-filesystem"],{cwd:repo}),error=>!error.stderr.includes(canary)&&/redacted|refused|unsafe/i.test(error.stderr)); + const oversized=join(run.root,"responses","oversized"); const handle=await open(oversized,"w"); await handle.truncate(33554433); await handle.close(); + await assert.rejects(execFileAsync("bash",[scan],{cwd:repo}),error=>!error.stderr.includes(canary)&&/redacted|failed|bound/i.test(error.stderr)); +}); + + +test("public prepare build is inside the stable lifecycle transaction", async()=>{ + const wrapper=await readFile(new URL("../../scripts/p1-manual-acceptance.sh",import.meta.url),"utf8"),source=await readFile(new URL("./p1-manual-acceptance.mjs",import.meta.url),"utf8"); + assert.doesNotMatch(wrapper,/npm .*run build/); assert.match(source,/action==="prepare"\)await prepareManual\(\)/); +}); diff --git a/backend/scripts/p1-render-snapshot.mjs b/backend/scripts/p1-render-snapshot.mjs new file mode 100755 index 00000000..d423d696 --- /dev/null +++ b/backend/scripts/p1-render-snapshot.mjs @@ -0,0 +1,172 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { constants, lstatSync, realpathSync } from "node:fs"; +import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +// This acceptance-only adapter deliberately imports the built production runner. +import { ThtRunner } from "../dist/tht/tht-runner.js"; + +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; + +function fixedRoot(repositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p1"); } +function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); } +function assertNoSymlinks(root, path, allowMissingLeaf = false) { + const rel = relative(root, path); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path is outside owned root"); + let cursor = root; + for (const [index, part] of rel.split(sep).filter(Boolean).entries()) { + cursor = join(cursor, part); + try { if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink"); } + catch (error) { + if (allowMissingLeaf && error.code === "ENOENT" && index === rel.split(sep).filter(Boolean).length - 1) return; + throw error; + } + } +} +async function ownership(repositoryRoot, ownershipPath) { + const root = fixedRoot(repositoryRoot); + const expected = join(root, "ownership.json"); + if (resolve(ownershipPath) !== expected) throw new Error("ownership path is not owned"); + const rootEntry = await lstat(root); const ownershipEntry = await lstat(expected); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership is unsafe"); + if (await realpath(root) !== root) throw new Error("ownership root is not canonical"); + let value; try { value = JSON.parse(await readFile(expected, "utf8")); } catch { throw new Error("ownership is malformed"); } + if (value?.schemaVersion !== 1 || value.kind !== "p1-manual-acceptance" || !HEX64.test(value.nonce ?? "") + || value.repositoryRoot !== realpathSync(repositoryRoot) || value.root !== root || value.status !== "PENDING" + || value.listener?.host !== "127.0.0.1" || value.listener?.port !== 8791) throw new Error("ownership identity mismatch"); + return { root, value }; +} +const ANCHORED_PUBLISH_SOURCE=String.raw`import os,secrets,stat,sys +parent,name,expected_dev,expected_ino=sys.argv[1:] +pfd=fd=None;stage=".render-stage-"+secrets.token_hex(16);published=False +def fail(): raise RuntimeError("anchored publication refused") +try: + pfd=os.open(parent,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + identity=os.fstat(pfd) + if (identity.st_dev,identity.st_ino)!=(int(expected_dev),int(expected_ino)): fail() + try: os.stat(name,dir_fd=pfd,follow_symlinks=False); fail() + except FileNotFoundError: pass + fd=os.open(stage,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=pfd) + data=sys.stdin.buffer.read(33554433) + if len(data)>33554432: fail() + view=memoryview(data) + while view: + written=os.write(fd,view) + if written<=0: fail() + view=view[written:] + os.fsync(fd);os.close(fd);fd=None;os.rename(stage,name,src_dir_fd=pfd,dst_dir_fd=pfd);published=True;os.fsync(pfd) + current=os.stat(parent,follow_symlinks=False) + if not stat.S_ISDIR(current.st_mode) or (current.st_dev,current.st_ino)!=(identity.st_dev,identity.st_ino): fail() +except Exception: + if published: + try: os.unlink(name,dir_fd=pfd);os.fsync(pfd) + except Exception: pass + print("anchored output publication refused (details redacted)",file=sys.stderr);raise SystemExit(1) +finally: + if fd is not None: os.close(fd) + if pfd is not None: + try: os.unlink(stage,dir_fd=pfd) + except FileNotFoundError: pass + os.close(pfd) +`; +async function atomicCopy(source,output) { + const parent=dirname(output),entry=await lstat(parent);if(!entry.isDirectory()||entry.isSymbolicLink())throw new Error("rendered parent identity is unsafe");const bytes=await readFile(source); + const result=spawnSync("python3",["-c",ANCHORED_PUBLISH_SOURCE,parent,basename(output),String(entry.dev),String(entry.ino)],{input:bytes,encoding:"utf8",maxBuffer:1024*1024}); + if(result.error||result.status!==0)throw new Error("anchored output publication refused; rendered parent identity changed or output is unsafe"); +} + +function sameEntry(actual, expected) { return actual.dev === expected.dev && actual.ino === expected.ino; } +async function readBounded(path, max, label) { + let handle; + try { + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = await handle.stat(), pathEntry = await lstat(path); + if (!before.isFile() || pathEntry.isSymbolicLink() || !pathEntry.isFile() || !sameEntry(before, pathEntry)) throw new Error(`${label} is unsafe`); + if (before.size < 1 || before.size > max) throw new Error(`${label} is unbounded`); + const bytes = Buffer.alloc(before.size); let offset = 0; + while (offset < bytes.length) { + const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset); + if (bytesRead < 1) throw new Error(`${label} changed while reading`); + offset += bytesRead; + } + const after = await handle.stat(); + if (!sameEntry(before, after) || after.size !== before.size) throw new Error(`${label} changed while reading`); + return bytes; + } finally { + if (handle) await handle.close().catch(() => {}); + } +} +async function readSnapshotManifest(root, manifestPath, commit, yamlName, expectedDigest) { + let manifestEntry; + try { assertNoSymlinks(root, manifestPath); manifestEntry = await lstat(manifestPath); } + catch (error) { if (error?.code === "ENOENT") throw new Error("snapshot manifest is missing or unbounded"); throw error; } + if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink() || await realpath(manifestPath) !== manifestPath) throw new Error("snapshot manifest is unsafe"); + const bytes = await readBounded(manifestPath, 1048576, "snapshot manifest"); + let manifest; try { manifest = JSON.parse(bytes.toString("utf8")); } catch { throw new Error("snapshot manifest is malformed"); } + const files = manifest?.files; + if (manifest?.head !== commit || !files || typeof files !== "object" || Array.isArray(files)) throw new Error("snapshot manifest identity is unsafe"); + if (!HEX64.test(files[yamlName] ?? "") || files[yamlName] !== expectedDigest) throw new Error("snapshot manifest digest is unsafe"); + return manifest; +} + +export async function renderOwnedSnapshot({ repositoryRoot = defaultRepositoryRoot, ownershipPath, snapshotPath, outputPath, snapshotSha256, env = process.env, beforePublish }) { + const repo = realpathSync(repositoryRoot); const { root } = await ownership(repo, resolve(repo, ownershipPath)); + const snapshot = resolve(repo, snapshotPath); const output = resolve(repo, outputPath); + const snapshotsRoot = join(root, "installation", "registry", "snapshots"); + const renderedRoot = join(root, "rendered"); + if (!isAbsolute(snapshotPath) || !below(snapshotsRoot, snapshot)) throw new Error("snapshot is not an owned absolute path"); + const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(relative(snapshotsRoot, snapshot).split(sep).join("/")); + if (!match || !HEX40.test(match[1])) throw new Error("snapshot is not commit addressed"); + if (!HEX64.test(snapshotSha256 ?? "")) throw new Error("snapshot digest identity is unsafe"); + assertNoSymlinks(root, snapshot); const snapshotEntry = await lstat(snapshot); + if (!snapshotEntry.isFile() || snapshotEntry.isSymbolicLink() || await realpath(snapshot) !== snapshot) throw new Error("snapshot is unsafe"); + const yamlName = `${match[2]}.yaml`; + const manifestPath = join(snapshotsRoot, match[1], "snapshot.json"); + await readSnapshotManifest(root, manifestPath, match[1], yamlName, snapshotSha256); + const snapshotBytes = await readBounded(snapshot, 1048576, "snapshot"); + if (createHash("sha256").update(snapshotBytes).digest("hex") !== snapshotSha256) throw new Error("snapshot bytes changed"); + if (!below(renderedRoot, output) || dirname(output) !== renderedRoot || !output.endsWith(".yaml")) throw new Error("output is not an owned rendered path"); + assertNoSymlinks(root, dirname(output)); + try { if ((await lstat(output)).isSymbolicLink()) throw new Error("output is unsafe"); } catch (error) { if (error.code !== "ENOENT") throw error; } + await mkdir(join(snapshotsRoot, "runtime"), { recursive: true, mode: 0o700 }); + const prior = {}; + for (const [key, value] of Object.entries(env)) { prior[key] = process.env[key]; if (value === undefined) delete process.env[key]; else process.env[key] = value; } + const runner = new ThtRunner({ + thtBin: join(repo, "harness", ".venv", "bin", "tht"), harnessDir: join(repo, "harness"), + configPath: join(root, "installation", "base.yaml"), dataRoot: join(root, "installation", "data"), + runtimeSnapshotRoot: join(snapshotsRoot, "runtime"), secretRoots: [join(root, "fixture-secrets")], + semanticRuntime: { internalQdrantUrl: "http://qdrant:6333", internalEmbeddingUrl: "http://embedding:11434", internalEmbeddingModel: "qwen3-embedding:0.6b", internalEmbeddingDimensions: 1024 }, + }); + let lease; + try { + lease = runner.acquireWorkspaceRuntime(snapshot); + const verifySnapshot = async () => { + const current = await readBounded(snapshot, 1048576, "snapshot"); + if (createHash("sha256").update(current).digest("hex") !== snapshotSha256) throw new Error("snapshot content changed during rendering"); + }; + await verifySnapshot(); + if(beforePublish)await beforePublish({output,renderedRoot}); + await verifySnapshot(); + await atomicCopy(lease.path, output); + } + finally { + if (lease) lease.release(); + for (const key of Object.keys(env)) { if (prior[key] === undefined) delete process.env[key]; else process.env[key] = prior[key]; } + } + return output; +} +function parseArgs(argv) { + if (argv.length !== 8) throw new Error("usage: p1-render-snapshot.mjs --ownership PATH --snapshot ABSOLUTE_PATH --output PATH --snapshot-sha256 HEX"); + const result = {}; for (let i=0;icreateHash("sha256").update(bytes).digest("hex"); +async function fixture() { + const repo=await realpath(await mkdtemp(join(tmpdir(),"p1-render-repo-"))); roots.push(repo); + const root=join(repo,".artifacts/manual-acceptance/p1"); const commit="a".repeat(40); const snapshot=join(root,"installation/registry/snapshots",commit,"p1-filesystem.yaml"); + for (const p of [dirname(snapshot),join(root,"rendered"),join(root,"installation/registry/snapshots/runtime"),join(root,"installation/data"),join(root,"fixture-secrets"),join(repo,"harness")]) await mkdir(p,{recursive:true,mode:0o700}); + await writeFile(join(root,"ownership.json"),JSON.stringify({schemaVersion:1,kind:"p1-manual-acceptance",nonce:"b".repeat(64),repositoryRoot:repo,root,status:"PENDING",listener:{host:"127.0.0.1",port:8791}})); + await writeFile(join(root,"installation/base.yaml"),"{}\n"); + const secret=join(root,"fixture-secrets/dwh-password"); await writeFile(secret,"not-inspected",{mode:0o600}); + await writeFile(snapshot,`workspace: + schema_version: 3 + id: p1-filesystem + name: P1 filesystem + language: en +dwh: + engine: postgres + database: postgres + schema: public + supported_transports: [postgres_direct] +semantic_index: + vector_store: {engine: qdrant, collection: p1-filesystem, dimensions: 1024, distance: cosine} + embedding: {provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024} +llm_policy: + allowed: [zai/glm-5.2] +evidence: + source: {type: filesystem, uri: workspace-content/p1-filesystem/evidence, patterns: ["**/*.md"], max_bytes: 10485760} + policy: {max_chunk_chars: 4000, retain_published_generations: 3} +`); + const snapshotBytes=await readFile(snapshot); const snapshotSha256=sha256(snapshotBytes); + const manifestPath=join(dirname(snapshot),"snapshot.json"); + await writeFile(manifestPath,JSON.stringify({head:commit,revisions:[{id:"p1-filesystem",commit,blob:"0".repeat(40),snapshotPath:snapshot}],files:{"p1-filesystem.yaml":snapshotSha256}})); + const env={THT_WS_P1_FILESYSTEM_DWH_TRANSPORT:"postgres_direct",THT_WS_P1_FILESYSTEM_DWH_HOST:"dwh.invalid",THT_WS_P1_FILESYSTEM_DWH_PORT:"5432",THT_WS_P1_FILESYSTEM_DWH_USER:"reader",THT_WS_P1_FILESYSTEM_DWH_PASSWORD_FILE:secret}; + return {repo,root,snapshot,snapshotSha256,manifestPath,env}; +} +test.afterEach(async()=>Promise.all(roots.splice(0).map(r=>rm(r,{recursive:true,force:true})))); + +const call=(f,extra={})=>renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:f.snapshot,snapshotSha256:f.snapshotSha256,env:{...process.env,...f.env},...extra}); +const runtimeLeases=async f=>await (await import("node:fs/promises")).readdir(join(f.root,"installation/registry/snapshots/runtime")); + +test("renderer copies a production lease deterministically with mode 0600 and no leases",async()=>{ const f=await fixture(); const one=join(f.root,"rendered/one.yaml"),two=join(f.root,"rendered/two.yaml"); await call(f,{outputPath:one}); await call(f,{outputPath:two}); assert.deepEqual(await readFile(one),await readFile(two)); assert.equal((await lstat(one)).mode&0o777,0o600); assert.deepEqual(await runtimeLeases(f),[]); }); + +test("renderer rejects unowned, symlink, out-of-root and missing-digest paths",async()=>{ const f=await fixture(); const outside=join(f.repo,"outside.yaml"); await writeFile(outside,"x"); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:outside,outputPath:join(f.root,"rendered/x.yaml"),snapshotSha256:f.snapshotSha256,env:f.env}),/owned|snapshot/); const link=join(dirname(f.snapshot),"linked.yaml"); await symlink(f.snapshot,link); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:link,outputPath:join(f.root,"rendered/x.yaml"),snapshotSha256:f.snapshotSha256,env:f.env}),/snapshot|symlink/); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:f.snapshot,outputPath:outside,snapshotSha256:f.snapshotSha256,env:f.env}),/output/); await assert.rejects(renderOwnedSnapshot({repositoryRoot:f.repo,ownershipPath:join(f.root,"ownership.json"),snapshotPath:f.snapshot,outputPath:join(f.root,"rendered/x.yaml"),env:f.env}),/snapshot digest identity/); }); + +test("renderer releases its acquired lease when atomic output copy fails",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/existing.yaml"); await mkdir(output); await assert.rejects(call(f,{outputPath:output}),/anchored|publication|unsafe/); assert.deepEqual(await runtimeLeases(f),[]); }); + +test("renderer refuses a same-path regular snapshot byte replacement against manifest and expected digest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); const replaced=(await readFile(f.snapshot,"utf8")).replace("max_chunk_chars: 4000","max_chunk_chars: 3999"); await writeFile(f.snapshot,replaced); await assert.rejects(call(f,{outputPath:output}),/snapshot bytes changed/); await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); }); + +test("renderer refuses snapshot manifest head, digest, and expected-digest tampering",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/tampered.yaml"); const manifest=JSON.parse(await readFile(f.manifestPath,"utf8")); + await writeFile(f.manifestPath,JSON.stringify({...manifest,head:"c".repeat(40)})); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest identity/); + await writeFile(f.manifestPath,JSON.stringify({...manifest,files:{"p1-filesystem.yaml":"d".repeat(64)}})); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest digest/); + await writeFile(f.manifestPath,JSON.stringify(manifest)); await assert.rejects(call(f,{outputPath:output,snapshotSha256:"e".repeat(64)}),/snapshot manifest digest/); + await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); }); + +test("renderer refuses a missing or malformed snapshot manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/nomanifest.yaml"); await rm(f.manifestPath); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*(missing|unbounded|unsafe)/); await writeFile(f.manifestPath,"{not json"); await assert.rejects(call(f,{outputPath:output}),/snapshot manifest.*malformed/); await assert.rejects(lstat(output)); assert.deepEqual(await runtimeLeases(f),[]); }); + +test("renderer rejects a regular snapshot replacement against its manifest",async()=>{ const f=await fixture(); const output=join(f.root,"rendered/replaced.yaml"); await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await writeFile(f.snapshot,"workspace:\n schema_version: 3\n id: p1-filesystem\n name: replaced\n")}}),/snapshot content changed/); await assert.rejects(lstat(output)); }); + +test("renderer anchors publication when rendered parent is concurrently swapped", async()=>{ + const f=await fixture(),output=join(f.root,"rendered/raced.yaml"),moved=join(f.root,"rendered-moved"),outside=join(f.repo,"outside-rendered"); await mkdir(outside); + await assert.rejects(call(f,{outputPath:output,beforePublish:async()=>{await rename(join(f.root,"rendered"),moved);await symlink(outside,join(f.root,"rendered"));}}),/identity|changed|unsafe|publication/i); + assert.deepEqual(await (await import("node:fs/promises")).readdir(outside),[]); +}); diff --git a/backend/scripts/p11-acceptance.mjs b/backend/scripts/p11-acceptance.mjs new file mode 100644 index 00000000..0a2c4864 --- /dev/null +++ b/backend/scripts/p11-acceptance.mjs @@ -0,0 +1,905 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from "node:crypto"; +import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync } from "node:fs"; +import { access, lstat, mkdir, open, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; + +import { + buildSafeEnvironment, + collectRepositoryProvenance, + deriveOverall, + scanSecrets, +} from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const RUN_ID = /^p11-[0-9a-f]{32}$/; +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const ZIP_FILES = ["manifest.json", "workspace.yaml", "contract.env.example", "README.md"]; + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (lstatSync(resolved).isFile()) return resolved; + } catch {} + } + throw new Error(`required executable not found: ${name}`); +} +function resolveExecutables(repositoryRoot) { + const repo = canonicalRoot(repositoryRoot); + const thtPath = join(repo, "harness", ".venv", "bin", "tht"); + if (!existsSync(thtPath)) throw new Error("required executable not found: tht"); + return { gitPath: resolveSystemExecutable("git"), pythonPath: resolveSystemExecutable("python3"), thtPath: realpathSync(thtPath) }; +} +const TOPOLOGY = [ + "remote.git", "author", "installation/registry", "installation/data", "installation/runtime", + "fixture-secrets", "fixtures/descriptors", "fixtures/requests", "requests", "responses", + "exports/raw", "exports/extracted", "rendered", "logs", +]; +export const CHECK_IDS = Object.freeze([ + "preflight", + "clean_state", + "ownership", + "catalog_bootstrap", + "catalog_only_listing", + "bootstrap_create_once", + "api_curator_boundary", + "curator_descriptor_update", + "content_only_revision", + "docs_only_reconciliation", + "same_revision_git_objects", + "snapshot_and_export", + "runtime_render_determinism", + "tht_config_check", + "negative_catalog_layout_cases", + "negative_schema_context_cases", + "no_p2_scope_artifacts", + "secret_scan", + "cleanup_confinement", +]); + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function assert(condition, message) { if (!condition) throw new Error(message); } +function scalarSecretBytes(value) { + if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid"); + return Buffer.from(value); +} +function canonicalRoot(repositoryRoot) { return realpathSync(repositoryRoot); } +export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p11-integration"); +} +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!existsSync(cursor)) break; + const entry = lstatSync(cursor); + if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink"); + } +} +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + if (handle) await handle.close().catch(() => {}); + await rm(staging, { force: true }).catch(() => {}); + throw error; + } +} +function exactOwnedResources(run) { + return [ + run.root, + join(run.root, "remote.git"), + join(run.root, "author"), + join(run.root, "installation", "registry"), + join(run.root, "installation", "data"), + join(run.root, "installation", "runtime"), + ]; +} +function initialListeners(pid) { + return [{ name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0, pid, state: "not_started" }]; +} +function ownershipValue(run, listeners = run.listeners) { + return { + schemaVersion: 1, + kind: "p11-acceptance", + runId: run.runId, + runNonce: run.nonce, + root: run.root, + repositoryRoot: run.repositoryRoot, + startedAt: run.startedAt, + pid: run.pid, + listeners, + resources: exactOwnedResources(run), + }; +} +async function writeOwnership(run, listenerUpdate) { + const listeners = listenerUpdate + ? run.listeners.map((listener) => listener.name === listenerUpdate.name ? listenerUpdate : listener) + : run.listeners; + await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run, listeners), null, 2)}\n`); + run.listeners = listeners; +} +export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) { + const repo = canonicalRoot(repositoryRoot); + const base = canonicalIntegrationBase(repo); + validateNoSymlinkAncestors(repo, base); + await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + const id = runId ?? `p11-${randomBytes(16).toString("hex")}`; + const root = validateRunRoot(repo, join(base, id), id); + const run = { + repositoryRoot: repo, + root, + runId: id, + nonce: nonce ?? randomBytes(32).toString("hex"), + startedAt: now ?? nowIso(), + pid: pid ?? process.pid, + listeners: initialListeners(pid ?? process.pid), + }; + if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity"); + await mkdir(root, { mode: 0o700 }); + await writeOwnership(run); + return run; +} +function strictOwnership(value, run, expectedNonce) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed"); + const listener = value.listeners?.[0]; + const validListener = Array.isArray(value.listeners) && value.listeners.length === 1 + && listener?.name === "primary" && listener.kind === "fastify" && listener.host === "127.0.0.1" + && listener.requestedPort === 0 && listener.pid === process.pid + && ["not_started", "listening", "closed", "close_failed"].includes(listener.state) + && (listener.state === "not_started" ? !("actualPort" in listener) + : Number.isInteger(listener.actualPort) && listener.actualPort >= 1 && listener.actualPort <= 65535); + if (value.schemaVersion !== 1 || value.kind !== "p11-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce + || value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid + || !ISO_UTC.test(value.startedAt ?? "") || !validListener + || JSON.stringify(value.resources) !== JSON.stringify(exactOwnedResources(run))) throw new Error("ownership identity mismatch"); + return value; +} +export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const repo = canonicalRoot(repositoryRoot); + const id = basename(resolve(runRoot)); + const lexical = validateRunRoot(repo, runRoot, id); + const rootEntry = await lstat(lexical); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory"); + const ownershipPath = join(lexical, "ownership.json"); + const ownershipEntry = await lstat(ownershipPath); + if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe"); + let value; + try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); } + return strictOwnership(value, { + repositoryRoot: repo, + root: lexical, + runId: id, + nonce: expectedNonce, + startedAt: value.startedAt, + pid: process.pid, + }, expectedNonce); +} +export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }); + const base = canonicalIntegrationBase(repositoryRoot); + const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`); + await rename(runRoot, tombstone); + await rm(tombstone, { recursive: true, force: false }); +} +async function finalizeOwnedRun({ run, success, keep }) { + if (!success || keep) return false; + await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + return true; +} + +function sanitizeForEvidence(value, forbiddenValues = []) { + const forbidden = forbiddenValues.filter((item) => typeof item === "string" && item.length > 0); + const redactString = (input) => forbidden.reduce((text, secret) => text.split(secret).join("[REDACTED]"), input); + if (typeof value === "string") return redactString(value); + if (Array.isArray(value)) return value.map((item) => sanitizeForEvidence(item, forbiddenValues)); + if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, sanitizeForEvidence(item, forbiddenValues)])); + return value; +} +async function fileArtifact(root, relativePath) { + const bytes = await readFile(join(root, relativePath)); + return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) }; +} +async function evidence(run, relativePath, value, forbiddenValues = []) { + await atomicWrite(join(run.root, relativePath), `${JSON.stringify(sanitizeForEvidence(value, forbiddenValues), null, 2)}\n`); + return await fileArtifact(run.root, relativePath); +} +async function writeJson(path, value) { + await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`); +} +async function walkFiles(root) { + const files = []; + async function visit(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) await visit(path); + else if (entry.isFile()) files.push({ path, rel: relative(root, path).split(sep).join("/") }); + } + } + if (existsSync(root)) await visit(root); + return files.sort((a, b) => a.rel.localeCompare(b.rel)); +} +async function snapshotDigest(root) { + const result = {}; + for (const file of await walkFiles(root)) result[file.rel] = sha256(await readFile(file.path)); + return result; +} +function assertByteIdentical(left, right, label) { + if (JSON.stringify(left) !== JSON.stringify(right)) throw new Error(`${label} changed unexpectedly`); +} +async function writeReportFiles({ run, report }) { + validateReport(report); + await writeJson(join(run.root, "report.json"), report); + const lines = [ + `# P1.1 acceptance report`, + "", + `Run ID: ${report.runId}`, + `Overall: ${report.overall}`, + "", + ...report.checks.map((check) => `- ${check.id}: ${check.status}`), + "", + `report.json sha256: ${sha256(await readFile(join(run.root, "report.json")))}`, + `P1.1 automated integration: ${report.overall}`, + "P1.1 manual acceptance: PENDING", + ]; + await atomicWrite(join(run.root, "report.md"), `${lines.join("\n")}\n`); +} +export function validateReport(report) { + if (!report || typeof report !== "object" || Array.isArray(report)) throw new Error("report is malformed"); + if (report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "") + || !ISO_UTC.test(report.finishedAt ?? "") || report.command !== "p11-acceptance integration --keep") throw new Error("report identity is invalid"); + if (report.overall !== deriveOverall(report.checks ?? [])) throw new Error("report overall is not derived"); + if (!Array.isArray(report.checks) || report.checks.length !== CHECK_IDS.length) throw new Error("report checks are incomplete"); + const ids = report.checks.map((check) => check.id); + if (JSON.stringify(ids) !== JSON.stringify(CHECK_IDS)) throw new Error("report checks are not exact"); + const artifactPaths = new Set(); + for (const check of report.checks) { + if (!["PASS", "FAIL"].includes(check.status) || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "")) { + throw new Error("report check metadata is invalid"); + } + if (!Array.isArray(check.commands) || check.commands.some((command) => typeof command !== "string" || !/^[A-Za-z0-9._+-]+$/.test(command))) { + throw new Error("report command is invalid"); + } + if (!Array.isArray(check.artifacts)) throw new Error("report artifacts are invalid"); + for (const artifact of check.artifacts) { + if (typeof artifact.path !== "string" || artifact.path.startsWith("/") || artifact.path.includes("..") || !/^[A-Za-z0-9._/-]+$/.test(artifact.path)) { + throw new Error("report artifact path is invalid"); + } + if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report artifact hash is invalid"); + if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated"); + artifactPaths.add(artifact.path); + } + } +} + +async function execCommand(executable, argv, { cwd, env, timeoutMs = 30_000, stdin } = {}) { + if (!Array.isArray(argv) || argv.some((value) => typeof value !== "string")) throw new Error("command argv must be a string array"); + const result = await execFileAsync(executable, argv, { + cwd, + env, + timeout: timeoutMs, + maxBuffer: 16 * 1024 * 1024, + encoding: "utf8", + ...(stdin === undefined ? {} : { input: stdin }), + }); + return { code: 0, stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; +} +async function git(ctx, argv, options = {}) { + return await execCommand(ctx.executables.gitPath, argv, { ...options, env: ctx.env }); +} +async function tht(ctx, argv, options = {}) { + try { + return await execCommand(ctx.executables.thtPath, argv, { ...options, env: ctx.env }); + } catch (error) { + if (typeof error?.code === "number") return { code: error.code, stdout: error.stdout ?? "", stderr: error.stderr ?? "" }; + throw error; + } +} +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } +function baseWorkspace(id, evidenceSource) { + return { + workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } }, + }; +} +function descriptors() { + return [ + baseWorkspace("p11-filesystem", { type: "filesystem", uri: "p11-filesystem/evidence", patterns: ["**/*.md"], max_bytes: 10485760 }), + baseWorkspace("p11-http", { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", connect_timeout_ms: 1250, read_timeout_ms: 30001, max_bytes: 12345, max_redirects: 2, allow_private_hosts: false, max_cache_bytes: 67890 }), + baseWorkspace("p11-s3", { type: "s3", uri: "s3://p11-evidence/published/", endpoint_url: "https://s3.example.test/", region: "eu-west-1", credentials: "static_files", trusted_endpoint: true, allow_private_endpoint: false, allow_insecure_endpoint: false, max_bytes: 12345, max_objects: 33, max_pages: 4, page_size: 5 }), + ]; +} +async function createTopology(run) { + for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); +} +async function setupSecrets(ctx) { + const secretDir = join(ctx.run.root, "fixture-secrets"); + const values = { + dwh: `DWH-${randomBytes(12).toString("hex")}`, + signed: `SIGNED-${randomBytes(12).toString("hex")}`, + access: `ACCESS-${randomBytes(12).toString("hex")}`, + secret: `SECRET-${randomBytes(12).toString("hex")}`, + session: `SESSION-${randomBytes(12).toString("hex")}`, + rejected: `REJECTED-${randomBytes(12).toString("hex")}`, + }; + ctx.forbiddenValues = Object.values(values); + ctx.secretValues = values; + const paths = { + dwh: join(secretDir, "dwh-password"), + signed: join(secretDir, "evidence-signed-urls.json"), + access: join(secretDir, "evidence-access"), + secret: join(secretDir, "evidence-secret"), + session: join(secretDir, "evidence-session"), + }; + await atomicWrite(paths.dwh, scalarSecretBytes(values.dwh)); + await atomicWrite(paths.signed, JSON.stringify([`https://evidence.example.test/guide.md?token=${values.signed}`])); + await atomicWrite(paths.access, scalarSecretBytes(values.access)); + await atomicWrite(paths.secret, scalarSecretBytes(values.secret)); + await atomicWrite(paths.session, scalarSecretBytes(values.session)); + const env = {}; + for (const workspace of ctx.descriptors) { + const prefix = `THT_WS_${namespace(workspace.workspace.id)}`; + Object.assign(env, { + [`${prefix}_DWH_TRANSPORT`]: "postgres_direct", + [`${prefix}_DWH_HOST`]: "dwh.invalid", + [`${prefix}_DWH_PORT`]: "5432", + [`${prefix}_DWH_USER`]: "reader", + [`${prefix}_DWH_PASSWORD_FILE`]: paths.dwh, + }); + } + Object.assign(env, { + THT_WS_P11_HTTP_EVIDENCE_SIGNED_URLS_FILE: paths.signed, + THT_WS_P11_S3_EVIDENCE_ACCESS_KEY_FILE: paths.access, + THT_WS_P11_S3_EVIDENCE_SECRET_KEY_FILE: paths.secret, + THT_WS_P11_S3_EVIDENCE_SESSION_TOKEN_FILE: paths.session, + }); + Object.assign(ctx.env, env); + await atomicWrite(join(ctx.run.root, "installation", "bindings.env"), `${Object.entries(env).map(([key, value]) => `${key}=${value}`).join("\n")}\n`); + await atomicWrite(join(ctx.run.root, "installation", "runtime", "base.yaml"), "{}\n"); +} +function catalog(entries = ctxDescriptors) { + return { schema_version: 1, workspaces: entries.map(({ workspace }) => ({ id: workspace.id, name: workspace.name, description: workspace.description })) }; +} +const ctxDescriptors = descriptors(); +async function initializeGit(ctx) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], { cwd: ctx.run.root }); + await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], { cwd: ctx.run.root }); + await git(ctx, ["config", "user.name", "P1 Fixture Curator"], { cwd: author }); + await git(ctx, ["config", "user.email", "p1-curator@example.invalid"], { cwd: author }); + const catalogBytes = `${JSON.stringify({ + schema_version: 1, + workspaces: [ + ...catalog(ctx.descriptors).workspaces, + { id: "p11-pending", name: "P1.1 pending", description: "Catalog-only slot awaiting bootstrap" }, + ], + }, null, 2)}\n`; + await atomicWrite(join(author, "thoth-workspaces.yaml"), catalogBytes, 0o644); + const evidenceRoot = join(author, "p11-filesystem", "evidence"); + await mkdir(join(evidenceRoot, "domain"), { recursive: true }); + await atomicWrite(join(evidenceRoot, "guide.md"), "# P1.1 curated Evidence\n", 0o644); + await atomicWrite(join(evidenceRoot, "domain", "table.md"), "# Curated table\n", 0o644); + await git(ctx, ["add", "thoth-workspaces.yaml"], { cwd: author }); + await git(ctx, ["add", "p11-filesystem/evidence/guide.md"], { cwd: author }); + await git(ctx, ["add", "-A", "p11-filesystem/evidence"], { cwd: author }); + await git(ctx, ["commit", "-m", "Bootstrap curated P1 content"], { cwd: author }); + await git(ctx, ["push", "origin", "main"], { cwd: author }); + ctx.bootstrapCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim(); + ctx.catalogBlobBefore = (await git(ctx, ["rev-parse", `HEAD:thoth-workspaces.yaml`], { cwd: author })).stdout.trim(); + ctx.evidenceTreeBefore = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/evidence`], { cwd: author })).stdout.trim(); +} +async function loadProductionBackend() { + const [{ loadConfig }, { buildApp }, { WorkspaceRegistry }, { ThtRunner }] = await Promise.all([ + import("../dist/config.js"), + import("../dist/app.js"), + import("../dist/workspaces/registry.js"), + import("../dist/tht/tht-runner.js"), + ]); + return { loadConfig, buildApp, WorkspaceRegistry, ThtRunner }; +} +async function startBackend(ctx) { + const { loadConfig, buildApp, WorkspaceRegistry, ThtRunner } = await loadProductionBackend(); + const config = loadConfig(ctx.env); + const registry = new WorkspaceRegistry(config.workspaceRegistry); + const thtRunner = new ThtRunner({ + thtBin: config.thtBin, + harnessDir: config.harnessDir, + configPath: join(ctx.run.root, "installation", "runtime", "base.yaml"), + dataRoot: config.dataRoot, + runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"), + secretRoots: config.workspaceRegistry.secretRoots, + secretsFile: config.secretsFile, + secretFiles: config.secretFiles, + semanticRuntime: { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }, + }); + const app = buildApp(config, { thtRunner, workspaceRegistry: registry }); + const address = await app.listen({ host: "127.0.0.1", port: 0 }); + const baseUrl = `http://127.0.0.1:${new URL(address).port}`; + ctx.registry = registry; + ctx.thtRunner = thtRunner; + ctx.app = app; + ctx.baseUrl = baseUrl; + await writeOwnership(ctx.run, { + name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0, + actualPort: Number(new URL(address).port), pid: process.pid, state: "listening", + }); +} +async function stopBackend(ctx) { + if (ctx.app) { + await ctx.app.close().catch(() => {}); + await writeOwnership(ctx.run, { + name: "primary", kind: "fastify", host: "127.0.0.1", requestedPort: 0, + actualPort: Number(new URL(ctx.baseUrl).port), pid: process.pid, state: "closed", + }).catch(() => {}); + } +} +async function request(ctx, id, method, path, body, binary = false, safeInput) { + const requestSummary = safeInput === undefined + ? { method, path, ...(body === undefined ? {} : { body: sanitizeForEvidence(body, ctx.forbiddenValues) }) } + : { method, path, input: safeInput }; + await evidence(ctx.run, `requests/${id}.json`, requestSummary, ctx.forbiddenValues); + const response = await fetch(`${ctx.baseUrl}${path}`, { + method, + headers: body === undefined ? {} : { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(15_000), + }); + if (binary) { + const bytes = Buffer.from(await response.arrayBuffer()); + await atomicWrite(join(ctx.run.root, `exports/raw/${id}.zip`), bytes); + await evidence(ctx.run, `responses/${id}.json`, { status: response.status, bytes: bytes.length, contentType: response.headers.get("content-type") }); + return { status: response.status, bytes }; + } + const text = await response.text(); + let parsed; + try { parsed = text ? JSON.parse(text) : null; } catch { parsed = { invalidJson: true, raw: text }; } + await evidence(ctx.run, `responses/${id}.json`, { status: response.status, body: sanitizeForEvidence(parsed, ctx.forbiddenValues) }, ctx.forbiddenValues); + return { status: response.status, body: parsed }; +} +async function extractZip(ctx, id, bytes) { + const yauzl = (await import("yauzl")).default; + const output = join(ctx.run.root, "exports", "extracted", id); + await mkdir(output, { recursive: true }); + const files = await new Promise((resolvePromise, reject) => { + yauzl.fromBuffer(bytes, { lazyEntries: true, strictFileNames: true, validateEntrySizes: true }, (error, zip) => { + if (error || !zip) return reject(error ?? new Error("zip open failed")); + const collected = new Map(); + zip.on("error", reject); + zip.on("entry", (entry) => { + if (!ZIP_FILES.includes(entry.fileName) || entry.fileName.includes("..") || entry.fileName.startsWith("/") || entry.fileName.endsWith("/")) return reject(new Error("unsafe export entry")); + zip.openReadStream(entry, (streamError, stream) => { + if (streamError || !stream) return reject(streamError ?? new Error("zip stream failed")); + const chunks = []; + stream.on("data", (chunk) => chunks.push(chunk)); + stream.on("error", reject); + stream.on("end", async () => { + const buffer = Buffer.concat(chunks); + collected.set(entry.fileName, buffer); + await atomicWrite(join(output, entry.fileName), buffer); + zip.readEntry(); + }); + }); + }); + zip.on("end", () => resolvePromise(collected)); + zip.readEntry(); + }); + }); + assert(files.size === ZIP_FILES.length, "export bundle entry mismatch"); + return JSON.parse(files.get("manifest.json").toString("utf8")); +} +function checkResult(id, startedAt, status, artifacts = [], commands = [], error) { + return { id, status, startedAt, finishedAt: nowIso(), artifacts, commands, ...(error ? { error } : {}) }; +} +async function executeChecks({ checks }) { + const results = []; + let stopped = false; + for (const scenario of checks) { + const startedAt = nowIso(); + if (stopped) { + results.push(checkResult(scenario.id, startedAt, "FAIL", [], [], "Not executed after earlier failure.")); + continue; + } + try { + const output = await scenario.run(); + results.push(checkResult(scenario.id, startedAt, "PASS", output.artifacts ?? [], output.commands ?? [])); + } catch (error) { + const partial = error?.acceptancePartial ?? {}; + results.push(checkResult(scenario.id, startedAt, "FAIL", partial.artifacts ?? [], partial.commands ?? [], "Acceptance scenario failed safely.")); + stopped = true; + } + } + return results; +} +async function registryState(ctx) { + const statePath = join(ctx.run.root, "installation", "registry", "state", "active.json"); + const active = JSON.parse(await readFile(statePath, "utf8")); + return { + head: active.head, + revisions: active.revisions.map((revision) => ({ id: revision.id, commit: revision.commit, blob: revision.blob })), + catalog: active.catalog ?? null, + }; +} +function safeErrorEnvelope(response, code, status) { + assert(response.status === status, `expected ${status}`); + assert(response.body?.code === code, `expected error code ${code}`); + assert(Object.keys(response.body).sort().join(",") === "code,message", "error envelope is not exact"); +} +async function productionChecks(ctx) { + const check = async (id, value, commands = []) => ({ commands, artifacts: [await evidence(ctx.run, `logs/${id}.json`, value, ctx.forbiddenValues)] }); + return [ + { id: "preflight", run: async () => check("preflight", { node: process.version, repositoryHead: ctx.provenance.head, repositoryTree: ctx.provenance.tree, clean: ctx.provenance.clean, thtExecutable: true }) }, + { id: "clean_state", run: async () => check("clean_state", { runId: ctx.run.runId, reused: false }) }, + { id: "ownership", run: async () => { await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); return await check("ownership", { valid: true }); } }, + { id: "catalog_bootstrap", run: async () => { + await initializeGit(ctx); + for (const workspace of ctx.descriptors) await atomicWrite(join(ctx.run.root, "fixtures", "descriptors", `${workspace.workspace.id}.json`), `${JSON.stringify(workspace, null, 2)}\n`); + return { + commands: ["git"], + artifacts: [ + await evidence(ctx.run, "logs/catalog-bootstrap.json", { bootstrapCommit: ctx.bootstrapCommit, catalogOnly: true }), + await fileArtifact(ctx.run.root, "author/thoth-workspaces.yaml"), + await fileArtifact(ctx.run.root, "author/p11-filesystem/evidence/guide.md"), + ], + }; + } }, + { id: "catalog_only_listing", run: async () => { + await startBackend(ctx); + const status = await request(ctx, "registry-status", "GET", "/workspace-registry/status"); + assert(status.status === 200 && status.body.head === ctx.bootstrapCommit, "status head mismatch"); + const listed = await request(ctx, "workspace-list-initial", "GET", "/workspaces"); + assert(listed.status === 200 && listed.body.length === 4, "catalog listing failed"); + assert(listed.body.every((entry) => entry.configurationState === "configuration_required"), "catalog entries were not configuration_required"); + ctx.baseCommit = status.body.head; + return await check("catalog_only_listing", { head: status.body.head, ids: listed.body.map((entry) => entry.id), allConfigurationRequired: true }); + } }, + { id: "bootstrap_create_once", run: async () => { + let base = ctx.baseCommit; + ctx.bootstrapResponses = {}; + for (const workspace of ctx.descriptors) { + const validated = await request(ctx, `validate-${workspace.workspace.id}`, "POST", "/workspaces/validate", { workspace }); + assert(validated.status === 200, `validate failed ${workspace.workspace.id}`); + const published = await request(ctx, `publish-${workspace.workspace.id}`, "POST", "/workspaces/publish", { action: "create", workspace, baseCommit: base }); + assert(published.status === 200 && HEX40.test(published.body.revision.commit), `publish failed ${workspace.workspace.id}`); + ctx.bootstrapResponses[workspace.workspace.id] = published.body; + base = published.body.revision.commit; + } + ctx.publishHead = base; + const listed = await request(ctx, "workspace-list-ready", "GET", "/workspaces"); + assert(listed.body.filter((entry) => entry.configurationState === "ready").length === 3, "bootstrap did not activate all published entries"); + assert(listed.body.find((entry) => entry.id === "p11-pending")?.configurationState === "configuration_required", "pending slot was not left unconfigured"); + return await check("bootstrap_create_once", { head: base, readyIds: listed.body.filter((entry) => entry.configurationState === "ready").map((entry) => entry.id) }); + } }, + { id: "api_curator_boundary", run: async () => { + const author = join(ctx.run.root, "author"); + const catalogAfter = (await git(ctx, ["rev-parse", `HEAD:thoth-workspaces.yaml`], { cwd: author })).stdout.trim(); + const evidenceAfter = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/evidence`], { cwd: author })).stdout.trim(); + assert(catalogAfter === ctx.catalogBlobBefore, "catalog blob changed during bootstrap"); + assert(evidenceAfter === ctx.evidenceTreeBefore, "evidence tree changed during bootstrap"); + ctx.apiBoundaryState = await registryState(ctx); + return await check("api_curator_boundary", { catalogUnchanged: true, evidenceUnchanged: true, state: ctx.apiBoundaryState }, ["git"]); + } }, + { id: "curator_descriptor_update", run: async () => { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], { cwd: author }); + await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author }); + const workspace = structuredClone(ctx.descriptors[0]); + workspace.workspace.name = "P1.1 Curated Filesystem"; + workspace.workspace.description = "Curator updated descriptor and catalog metadata"; + ctx.curatedWorkspace = workspace; + const updatedCatalog = catalog([workspace, ctx.descriptors[1], ctx.descriptors[2]]); + await atomicWrite(join(author, "thoth-workspaces.yaml"), `${JSON.stringify(updatedCatalog, null, 2)}\n`, 0o644); + await atomicWrite(join(author, "p11-filesystem", "workspace.yaml"), `${(await import("yaml")).stringify(workspace)}`, 0o644); + await git(ctx, ["add", "thoth-workspaces.yaml"], { cwd: author }); + await git(ctx, ["add", "--", "p11-filesystem/workspace.yaml"], { cwd: author }); + await git(ctx, ["commit", "-m", "Publish workspace p1-filesystem"], { cwd: author }); + await git(ctx, ["push", "origin", "main"], { cwd: author }); + ctx.curatorCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim(); + ctx.curatorDescriptorBlob = (await git(ctx, ["rev-parse", `HEAD:p11-filesystem/workspace.yaml`], { cwd: author })).stdout.trim(); + const pulled = await request(ctx, "pull-after-curator-update", "POST", "/workspace-registry/pull"); + assert(pulled.status === 200 && HEX40.test(pulled.body.head), "pull after curator update failed"); + ctx.docsFollowupHead = pulled.body.head; + const read = await request(ctx, "read-after-curator-update", "GET", "/workspaces/p11-filesystem"); + assert(read.status === 200 && read.body.workspace.workspace.name === workspace.workspace.name, "curator update did not activate"); + assert(read.body.revision.blob === ctx.curatorDescriptorBlob, "api rewrote curator descriptor bytes"); + return await check("curator_descriptor_update", { curatorCommit: ctx.curatorCommit, activeHead: ctx.docsFollowupHead, descriptorBlob: ctx.curatorDescriptorBlob }, ["git"]); + } }, + { id: "content_only_revision", run: async () => { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], { cwd: author }); + await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author }); + await atomicWrite(join(author, "p11-filesystem", "evidence", "guide.md"), "# P1.1 curated Evidence v2\n", 0o644); + await git(ctx, ["add", "p11-filesystem/evidence/guide.md"], { cwd: author }); + await git(ctx, ["commit", "-m", "Update curated Evidence only"], { cwd: author }); + await git(ctx, ["push", "origin", "main"], { cwd: author }); + ctx.contentCommit = (await git(ctx, ["rev-parse", "HEAD"], { cwd: author })).stdout.trim(); + const pulled = await request(ctx, "pull-after-content-update", "POST", "/workspace-registry/pull"); + assert(pulled.status === 200 && pulled.body.head === ctx.contentCommit, "content pull head mismatch"); + const read = await request(ctx, "read-after-content-update", "GET", "/workspaces/p11-filesystem"); + assert(read.body.revision.commit === ctx.contentCommit, "content commit did not activate"); + assert(read.body.revision.blob === ctx.curatorDescriptorBlob, "descriptor blob changed on content-only update"); + ctx.currentRead = read.body; + return await check("content_only_revision", { commit: ctx.contentCommit, descriptorBlobUnchanged: true }, ["git"]); + } }, + { id: "docs_only_reconciliation", run: async () => { + const repo = join(ctx.run.root, "installation", "registry", "repo"); + const diff = (await git(ctx, ["show", "--name-only", "--format=", ctx.docsFollowupHead], { cwd: repo })).stdout.trim().split(/\n+/).filter(Boolean); + assert(diff.length > 0 && diff.every((path) => path.startsWith("workspace-docs/")), "docs follow-up touched non-doc paths"); + const finalDescriptor = (await git(ctx, ["rev-parse", `${ctx.docsFollowupHead}:p11-filesystem/workspace.yaml`], { cwd: repo })).stdout.trim(); + assert(finalDescriptor === ctx.curatorDescriptorBlob, "docs follow-up rewrote descriptor"); + return await check("docs_only_reconciliation", { head: ctx.docsFollowupHead, files: diff, descriptorBlobPreserved: true }, ["git"]); + } }, + { id: "same_revision_git_objects", run: async () => { + const repo = join(ctx.run.root, "installation", "registry", "repo"); + const revision = ctx.currentRead.revision; + const manifestPath = join(dirname(revision.snapshotPath), "snapshot.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + const catalogBlob = (await git(ctx, ["rev-parse", `${revision.commit}:thoth-workspaces.yaml`], { cwd: repo })).stdout.trim(); + const descriptorBlob = (await git(ctx, ["rev-parse", `${revision.commit}:p11-filesystem/workspace.yaml`], { cwd: repo })).stdout.trim(); + const evidenceTree = (await git(ctx, ["rev-parse", `${revision.commit}:p11-filesystem/evidence`], { cwd: repo })).stdout.trim(); + assert(manifest.head === revision.commit, "snapshot manifest head mismatch"); + assert(descriptorBlob === revision.blob, "descriptor blob mismatch"); + ctx.snapshotManifest = manifest; + return { + commands: ["git"], + artifacts: [ + await evidence(ctx.run, "logs/same-revision-git-objects.json", { commit: revision.commit, catalogBlob, descriptorBlob, evidenceTree, snapshotHead: manifest.head }), + await fileArtifact(ctx.run.root, relative(ctx.run.root, revision.snapshotPath)), + await fileArtifact(ctx.run.root, relative(ctx.run.root, manifestPath)), + ], + }; + } }, + { id: "snapshot_and_export", run: async () => { + ctx.exportManifests = {}; + const artifacts = []; + for (const workspace of ctx.descriptors) { + const id = workspace.workspace.id; + const exported = await request(ctx, `export-${id}`, "GET", `/workspaces/${id}/export`, undefined, true); + assert(exported.status === 200, `export failed ${id}`); + ctx.exportManifests[id] = await extractZip(ctx, id, exported.bytes); + artifacts.push(await fileArtifact(ctx.run.root, `exports/raw/export-${id}.zip`)); + for (const name of ZIP_FILES) artifacts.push(await fileArtifact(ctx.run.root, `exports/extracted/${id}/${name}`)); + } + return { commands: [], artifacts: [await evidence(ctx.run, "logs/snapshot-and-export.json", { exported: Object.keys(ctx.exportManifests), files: ZIP_FILES }), ...artifacts] }; + } }, + { id: "runtime_render_determinism", run: async () => { + const YAML = await import("yaml"); + ctx.configChecks = []; + const artifacts = []; + for (const workspace of ctx.descriptors) { + const revision = (await request(ctx, `read-render-${workspace.workspace.id}`, "GET", `/workspaces/${workspace.workspace.id}`)).body.revision; + const renders = []; + for (let n = 1; n <= 2; n += 1) { + const lease = ctx.thtRunner.acquireWorkspaceRuntime(revision.snapshotPath); + try { + const bytes = await readFile(lease.path); + renders.push(bytes); + await atomicWrite(join(ctx.run.root, "rendered", `${workspace.workspace.id}-${n}.yaml`), bytes); + const checked = await tht(ctx, ["config", "check", "-c", lease.path], { cwd: ctx.env.THT_HARNESS_DIR, timeoutMs: 30_000 }); + ctx.configChecks.push({ id: workspace.workspace.id, observation: n, code: checked.code }); + } finally { + lease.release(); + } + artifacts.push(await fileArtifact(ctx.run.root, `rendered/${workspace.workspace.id}-${n}.yaml`)); + } + assert(renders[0].equals(renders[1]), `render was nondeterministic ${workspace.workspace.id}`); + const rendered = YAML.parse(renders[0].toString("utf8")); + assert(rendered.runtime_identity.workspace_revision === revision.commit, `runtime identity mismatch ${workspace.workspace.id}`); + } + return { commands: ["tht"], artifacts: [await evidence(ctx.run, "logs/runtime-render-determinism.json", { deterministic: true, checks: ctx.configChecks }), ...artifacts] }; + } }, + { id: "tht_config_check", run: async () => { + assert(ctx.configChecks.length === ctx.descriptors.length * 2 && ctx.configChecks.every((item) => item.code === 0), "tht config checks failed"); + return await check("tht-config-check", ctx.configChecks, ["tht"]); + } }, + { id: "negative_catalog_layout_cases", run: async () => { + const baseline = await registryState(ctx); + const author = join(ctx.run.root, "author"); + const current = (await request(ctx, "current-list-before-negatives", "GET", "/workspaces")).body; + const secondCreate = await request(ctx, "second-create", "POST", "/workspaces/publish", { action: "create", workspace: ctx.descriptors[0], baseCommit: baseline.head }); + safeErrorEnvelope(secondCreate, "workspace_curator_owned", 409); + const update = await request(ctx, "legacy-update", "POST", "/workspaces/publish", { action: "update", workspace: ctx.descriptors[0], baseCommit: baseline.head, baseBlob: ctx.curatorDescriptorBlob }); + safeErrorEnvelope(update, "workspace_curator_owned", 409); + const deletion = await request(ctx, "legacy-delete", "POST", "/workspaces/publish", { action: "delete", id: "p11-filesystem", baseCommit: baseline.head, baseBlob: ctx.curatorDescriptorBlob }); + safeErrorEnvelope(deletion, "workspace_curator_owned", 409); + const unknown = structuredClone(ctx.descriptors[0]); + unknown.workspace.id = "p11-unknown"; + const unknownPublish = await request(ctx, "unknown-catalog-id", "POST", "/workspaces/publish", { action: "create", workspace: unknown, baseCommit: baseline.head }); + safeErrorEnvelope(unknownPublish, "workspace_invalid", 400); + const mismatch = structuredClone(ctx.descriptors[0]); + mismatch.workspace.id = "p11-pending"; + mismatch.workspace.name = "Mismatched pending name"; + mismatch.semantic_index.vector_store.collection = "p11-pending"; + const mismatchPublish = await request(ctx, "catalog-metadata-mismatch", "POST", "/workspaces/publish", { action: "create", workspace: mismatch, baseCommit: baseline.head }); + safeErrorEnvelope(mismatchPublish, "workspace_invalid", 400); + const after = await registryState(ctx); + assertByteIdentical(after, baseline, "registry state after curator-owned refusals"); + assert(JSON.stringify((await request(ctx, "current-list-after-negatives", "GET", "/workspaces")).body) === JSON.stringify(current), "workspace listing mutated after negative cases"); + await git(ctx, ["fetch", "origin", "main"], { cwd: author }); + await git(ctx, ["reset", "--hard", "origin/main"], { cwd: author }); + await mkdir(join(author, "workspaces"), { recursive: true }); + await atomicWrite(join(author, "workspaces", "legacy.yaml"), "workspace: bad\n", 0o644); + await git(ctx, ["add", "--", "workspaces/legacy.yaml"], { cwd: author }); + await git(ctx, ["commit", "-m", "Invalid contextual Evidence state"], { cwd: author }); + await git(ctx, ["push", "origin", "HEAD:main"], { cwd: author }); + const rejectedPull = await request(ctx, "invalid-layout-pull", "POST", "/workspace-registry/pull"); + safeErrorEnvelope(rejectedPull, "workspace_invalid", 400); + const afterInvalidPull = await registryState(ctx); + assertByteIdentical(afterInvalidPull, baseline, "registry state after invalid pull"); + return await check("negative_catalog_layout_cases", { secondCreate: true, update: true, delete: true, unknownCatalogId: true, metadataMismatch: true, oldLayoutRejected: true }, ["git"]); + } }, + { id: "negative_schema_context_cases", run: async () => { + const base = structuredClone(ctx.descriptors[0]); + const cases = [ + ["invalid-uri", (workspace) => { workspace.evidence.source.uri = "/etc/passwd"; }, "evidence.source.uri"], + ["invalid-secret-field", (workspace) => { workspace.evidence.source.password = ctx.secretValues.rejected; }, "evidence.source.password"], + ["missing-evidence-tree", (workspace) => { workspace.workspace.id = "p11-pending"; workspace.workspace.name = "P1.1 pending"; workspace.workspace.description = "Catalog-only slot awaiting bootstrap"; workspace.semantic_index.vector_store.collection = "p11-pending"; workspace.evidence.source.uri = "p11-pending/evidence"; }, "evidence.source.uri"], + ]; + const outcomes = []; + for (const [id, mutate, field] of cases) { + const workspace = structuredClone(base); + mutate(workspace); + const endpoint = id === "missing-evidence-tree" ? "/workspaces/publish" : "/workspaces/validate"; + const payload = id === "missing-evidence-tree" ? { action: "create", workspace, baseCommit: ctx.publishHead } : { workspace }; + const response = await request(ctx, `negative-schema-${id}`, "POST", endpoint, payload, false, { case: id, expectedInputField: field }); + safeErrorEnvelope(response, "workspace_invalid", 400); + outcomes.push({ case: id, status: response.status, field }); + } + return await check("negative_schema_context_cases", outcomes); + } }, + { id: "no_p2_scope_artifacts", run: async () => { + const forbidden = ["artifacts/evidence", "materialized", "qdrant", "embedding", "ACTIVE", "retention"]; + const present = forbidden.filter((path) => existsSync(join(ctx.run.root, path))); + assert(present.length === 0, "p2 scope artifacts present"); + return await check("no_p2_scope_artifacts", { absent: forbidden }); + } }, + { id: "secret_scan", run: async () => { + const findings = await scanSecrets({ runRoot: ctx.run.root, forbiddenValues: ctx.forbiddenValues, expectedGitRepositories: ["remote.git", "author"] }); + assert(findings.length === 0, "secret scan found leaked secret material"); + return await check("secret_scan", { findings: 0 }); + } }, + { id: "cleanup_confinement", run: async () => { + const parent = canonicalIntegrationBase(ctx.repositoryRoot); + const siblings = (await readdir(parent)).filter((name) => name !== ctx.run.runId); + return await check("cleanup_confinement", { listenerState: ctx.run.listeners[0].state, siblingCount: siblings.length }); + } }, + ]; +} + +async function setupContext({ repositoryRoot = defaultRepositoryRoot, env = process.env } = {}) { + const run = await createOwnedRun({ repositoryRoot }); + const provenance = await collectRepositoryProvenance({ repositoryRoot }); + const executables = resolveExecutables(repositoryRoot); + const harnessDir = realpathSync(join(repositoryRoot, "harness")); + const ownedHome = join(run.root, "installation", "runtime", "acceptance-home"); + const ownedTmp = join(run.root, "installation", "runtime", "tmp"); + await mkdir(ownedHome, { recursive: true, mode: 0o700 }); + await mkdir(ownedTmp, { recursive: true, mode: 0o700 }); + const executablePath = [...new Set([dirname(executables.gitPath), dirname(executables.pythonPath), dirname(executables.thtPath)])].join(":"); + const fixtureEnv = { + PATH: executablePath, + HOME: ownedHome, + TMPDIR: ownedTmp, + HOST: "127.0.0.1", + PORT: "0", + AUTH_MODE: "none", + THT_BIN: executables.thtPath, + THT_HARNESS_DIR: harnessDir, + THT_DATA_ROOT: join(run.root, "installation", "data"), + SETTINGS_FILE: join(run.root, "installation", "data", "settings.json"), + MAINTENANCE_STATE_FILE: join(run.root, "installation", "data", "maintenance.json"), + THT_WORKSPACE_REGISTRY_ROOT: join(run.root, "installation", "registry"), + THT_WORKSPACE_GIT_REMOTE: join(run.root, "remote.git"), + THT_WORKSPACE_GIT_BRANCH: "main", + THT_WORKSPACE_GIT_AUTHOR_NAME: "P1 API Publisher", + THT_WORKSPACE_GIT_AUTHOR_EMAIL: "p1-api@example.invalid", + THT_WORKSPACE_INSTALLATION_ID: "p11-acceptance", + THT_WORKSPACE_SECRET_ROOTS: join(run.root, "fixture-secrets"), + THT_HOME: join(run.root, "installation", "runtime", "tht-home"), + PYTHONDONTWRITEBYTECODE: "1", + PYTHONNOUSERSITE: "1", + }; + const ctx = { + run, + repositoryRoot: canonicalRoot(repositoryRoot), + provenance, + executables, + descriptors: descriptors(), + env: buildSafeEnvironment({ ambient: env, fixture: fixtureEnv }), + forbiddenValues: [], + }; + await createTopology(run); + await setupSecrets(ctx); + return ctx; +} + +export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) { + const ctx = await setupContext({ repositoryRoot, env }); + const priorEnv = {}; + for (const [key, value] of Object.entries(ctx.env)) { + priorEnv[key] = process.env[key]; + process.env[key] = value; + } + let success = false; + try { + const checks = await productionChecks(ctx); + const results = await executeChecks({ checks }); + const report = { + schemaVersion: 1, + runId: ctx.run.runId, + startedAt: ctx.run.startedAt, + finishedAt: nowIso(), + command: "p11-acceptance integration --keep", + overall: deriveOverall(results), + checks: results, + }; + await writeReportFiles({ run: ctx.run, report }); + success = report.overall === "PASS"; + if (announce) await announce({ report, runRoot: ctx.run.root }); + return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) }; + } finally { + await stopBackend(ctx).catch(() => {}); + for (const [key, value] of Object.entries(ctx.env)) { + if (priorEnv[key] === undefined) delete process.env[key]; + else process.env[key] = priorEnv[key]; + } + } +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) { + throw new Error("usage: p11-acceptance.mjs integration [--keep]"); + } + const result = await runIntegration({ keep: argv.includes("--keep"), env }); + return result.exitCode; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { + const code = await main(); + process.exitCode = code; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/backend/scripts/p11-acceptance.test.mjs b/backend/scripts/p11-acceptance.test.mjs new file mode 100644 index 00000000..a53fbf68 --- /dev/null +++ b/backend/scripts/p11-acceptance.test.mjs @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_IDS, + canonicalIntegrationBase, + cleanupOwnedRun, + createOwnedRun, + readAndValidateOwnership, + validateReport, + validateRunRoot, +} from "./p11-acceptance.mjs"; + +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p11-acceptance-repo-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p11-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true }); + return root; +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("run roots are only canonical direct p11 integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p11-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [ + base, + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(repositoryRoot, ".artifacts", "p1-integration", id), + join(base, id, "nested"), + join(base, "foreign"), + ]) { + assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + } + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p11-${"A".repeat(32)}`), `p11-${"A".repeat(32)}`)); +}); + +test("cleanup refuses p1, manual, sibling, and wrong-nonce roots", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + for (const bad of [ + join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`), + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(canonicalIntegrationBase(repositoryRoot), `p11-${"c".repeat(32)}`), + ]) { + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce })); + } + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) })); +}); + +test("cleanup removes exactly one owned p11 root", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const sibling = join(canonicalIntegrationBase(repositoryRoot), `p11-${"d".repeat(32)}`); + await mkdir(sibling); + await writeFile(join(sibling, "sentinel"), "foreign"); + await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await assert.rejects(readFile(join(run.root, "ownership.json"))); + assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign"); +}); + +function resultFor(id) { + return { + id, + status: "PASS", + startedAt: "2026-08-11T00:00:00.000Z", + finishedAt: "2026-08-11T00:00:01.000Z", + commands: ["git"], + artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }], + }; +} + +test("report validation requires exact p11 identity, check order, and unique artifacts", () => { + const report = { + schemaVersion: 1, + runId: `p11-${"e".repeat(32)}`, + startedAt: "2026-08-11T00:00:00.000Z", + finishedAt: "2026-08-11T00:00:10.000Z", + command: "p11-acceptance integration --keep", + overall: "PASS", + checks: CHECK_IDS.map(resultFor), + }; + assert.doesNotThrow(() => validateReport(report)); + const invalid = structuredClone(report); + invalid.runId = `p1-${"e".repeat(32)}`; + assert.throws(() => validateReport(invalid)); + const duplicate = structuredClone(report); + duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path; + assert.throws(() => validateReport(duplicate), /duplicated/); + const reordered = structuredClone(report); + reordered.checks.reverse(); + reordered.overall = "FAIL"; + assert.throws(() => validateReport(reordered)); +}); + +test("public wrapper uses a strict empty environment", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p11-acceptance.sh"), "utf8"); + assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/); + assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/); + assert.doesNotMatch(wrapper, /P11_ACCEPTANCE_FAIL_AT/); +}); diff --git a/backend/scripts/p11-manual-acceptance.mjs b/backend/scripts/p11-manual-acceptance.mjs new file mode 100644 index 00000000..18c9cd0d --- /dev/null +++ b/backend/scripts/p11-manual-acceptance.mjs @@ -0,0 +1,404 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { closeSync, constants as fsConstants, fsyncSync, lstatSync, openSync, realpathSync } from "node:fs"; +import { access, lstat, mkdir, open, readFile, readdir, rename, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { execFile } from "node:child_process"; +import http from "node:http"; + +import { buildSafeEnvironment } from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const HOST = "127.0.0.1"; +const BACKEND_PORT = 8791; +const FRONTEND_PORT = 8792; +const HEX64 = /^[0-9a-f]{64}$/; +const OWNERSHIP_DIGEST = "ownership.sha256"; + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (lstatSync(resolved).isFile()) return resolved; + } catch {} + } + throw new Error(`required executable not found: ${name}`); +} +function resolveExecutables(repositoryRoot) { + const repo = realpathSync(repositoryRoot); + const thtPath = join(repo, "harness", ".venv", "bin", "tht"); + if (!lstatSync(thtPath).isFile()) throw new Error("required executable not found: tht"); + return { gitPath: resolveSystemExecutable("git"), pythonPath: resolveSystemExecutable("python3"), thtPath: realpathSync(thtPath) }; +} + +function nowIso() { return new Date().toISOString(); } +function fixedManualRoot(repositoryRoot = defaultRepositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p11"); } +function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); } +function noSymlinkExisting(repo, target) { + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("root leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!lstatSync(cursor, { throwIfNoEntry: false })) break; + if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink"); + } +} +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + if (handle) await handle.close().catch(() => {}); + await rm(staging, { force: true }).catch(() => {}); + throw error; + } +} +function ownershipDigest(bytes) { return createHash("sha256").update(bytes).digest("hex"); } +async function writeManualOwnership(root, value) { + const body = `${JSON.stringify(value, null, 2)}\n`; + await atomicWrite(join(root, "ownership.json"), body); + await atomicWrite(join(root, OWNERSHIP_DIGEST), `${ownershipDigest(body)}\n`); +} +async function git(executable, argv, options = {}) { + const result = await execFileAsync(executable, argv, { cwd: options.cwd, env: options.env, timeout: options.timeoutMs ?? 30_000, maxBuffer: 8 * 1024 * 1024, encoding: "utf8" }); + return { stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; +} +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } +function baseWorkspace(id, evidenceSource) { + return { + workspace: { schema_version: 3, id, name: `P1.1 ${id}`, description: `Catalog entry for ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "postgres", schema: "public", supported_transports: ["postgres_direct"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } }, + }; +} +function descriptors() { + return [ + baseWorkspace("p11-filesystem", { type: "filesystem", uri: "p11-filesystem/evidence", patterns: ["**/*.md"], max_bytes: 10485760 }), + baseWorkspace("p11-http", { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", connect_timeout_ms: 1250, read_timeout_ms: 30001, max_bytes: 12345, max_redirects: 2, allow_private_hosts: false, max_cache_bytes: 67890 }), + baseWorkspace("p11-s3", { type: "s3", uri: "s3://p11-evidence/published/", endpoint_url: "https://s3.example.test/", region: "eu-west-1", credentials: "static_files", trusted_endpoint: true, allow_private_endpoint: false, allow_insecure_endpoint: false, max_bytes: 12345, max_objects: 33, max_pages: 4, page_size: 5 }), + ]; +} +function catalog(entries) { + return { schema_version: 1, workspaces: entries.map(({ workspace }) => ({ id: workspace.id, name: workspace.name, description: workspace.description })) }; +} +function quote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; } +function requestFixtures(items) { + const fixtures = { "status.json": { method: "GET", path: "/workspace-registry/status" }, "pull.json": { method: "POST", path: "/workspace-registry/pull" } }; + for (const workspace of items) { + const id = workspace.workspace.id; + fixtures[`validate-${id}.json`] = { workspace }; + fixtures[`publish-${id}.json`] = { action: "create", workspace }; + fixtures[`read-${id}.json`] = { method: "GET", path: `/workspaces/${id}` }; + fixtures[`export-${id}.json`] = { method: "GET", path: `/workspaces/${id}/export` }; + } + fixtures["negative-invalid-uri.json"] = { workspace: { ...items[0], evidence: { ...items[0].evidence, source: { ...items[0].evidence.source, uri: "/etc/passwd" } } } }; + fixtures["negative-secret-field.json"] = { workspace: { ...items[2], evidence: { ...items[2].evidence, source: { ...items[2].evidence.source, access_key: "CANARY-MUST-BE-REJECTED" } } } }; + return fixtures; +} +function curlGet(url, output) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; } +function curlPost(url, output, body) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(body)} --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; } +function curlPostEmpty(url, output) { return `#!/usr/bin/env bash\nset -euo pipefail\ncurl --fail-with-body --silent --show-error --request POST --output ${quote(output)} --write-out 'HTTP %{http_code}\\n' ${quote(url)}\n`; } +function publishCurl(root, id, previousResponse) { + const descriptor = join(root, "requests", `publish-${id}.json`); + const response = join(root, "responses", `publish-${id}.json`); + return `#!/usr/bin/env bash\nset -euo pipefail\nbase_commit=$(node -e 'const fs=require("node:fs");const value=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));console.log(value.head ?? value.revision?.commit ?? "");' ${quote(previousResponse)})\nnode -e 'const fs=require("node:fs");const body=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));body.baseCommit=process.argv[2];fs.writeFileSync(process.argv[1],JSON.stringify(body,null,2)+"\\n");' ${quote(descriptor)} "$base_commit"\ncurl --fail-with-body --silent --show-error --request POST --header 'content-type: application/json' --data-binary @${quote(descriptor)} --output ${quote(response)} --write-out 'HTTP %{http_code}\\n' 'http://${HOST}:${BACKEND_PORT}/workspaces/publish'\n`; } +function renderCommand(repo, root, observation) { + const readResponse = join(root, "responses", "read-p11-filesystem.json"); + const output = join(root, "rendered", `runtime-${observation}.yaml`); + return `#!/usr/bin/env bash\nset -euo pipefail\nread_snapshot=$(node -e 'const fs=require("node:fs");const read=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));const path=read.revision.snapshotPath;const manifest=JSON.parse(fs.readFileSync(require("node:path").join(require("node:path").dirname(path),"snapshot.json"),"utf8"));const name=require("node:path").basename(path);console.log(JSON.stringify({snapshot:path,digest:manifest.files[name]}));' ${quote(readResponse)})\nsnapshot=$(node -e 'const value=JSON.parse(process.argv[1]);console.log(value.snapshot)' "$read_snapshot")\ndigest=$(node -e 'const value=JSON.parse(process.argv[1]);console.log(value.digest)' "$read_snapshot")\nnode ${quote(join(repo, "backend", "scripts", "p11-render-snapshot.mjs"))} --ownership ${quote(join(root, "ownership.json"))} --snapshot "$snapshot" --output ${quote(output)} --snapshot-sha256 "$digest"\n`; +} +function guide(root) { + return `# P1.1 manual acceptance guide + +1. Inspect ${join(root, "ownership.json")}, ${join(root, "author", "thoth-workspaces.yaml")}, nested workspace directories, evidence tree, and fixture secret paths without printing secret bytes. +2. Run ./scripts/p11-manual-acceptance.sh serve and confirm only ${HOST}:${BACKEND_PORT} and ${HOST}:${FRONTEND_PORT} are listening for this lab. +3. Run commands/http-01-status.sh and inspect responses/status.json plus GET /workspaces for configuration_required slots. +4. Run the validate and publish scripts once per slot in numeric order. +5. Inspect Git object IDs for thoth-workspaces.yaml, /workspace.yaml, /evidence, and workspace-docs/. +6. Retry create/update/delete and verify refusal plus unchanged object IDs. +7. In ${join(root, "author")}, edit p11-filesystem/workspace.yaml and thoth-workspaces.yaml together, commit, push, then run commands/http-08-pull.sh and verify the API activated curator bytes without rewriting the descriptor. +8. Make an evidence-only commit under p11-filesystem/evidence, push, pull, and inspect the new revision commit with unchanged descriptor blob. +9. In the UI at http://${HOST}:${FRONTEND_PORT}, confirm ready workspaces are read-only and bootstrap-only slots are editable before creation. +10. Export/import only under bootstrap rules. +11. Run commands/render-1.sh and commands/render-2.sh, diff rendered/runtime-1.yaml rendered/runtime-2.yaml, then run tht config check -c on both outputs. +12. Run the negative validate scripts and a bounded secret scan outside fixture-secrets. +13. Run ./scripts/p11-manual-acceptance.sh stop, verify cleanup of both listeners, write VERDICT.md yourself, and run cleanup only when evidence is no longer needed. +`; +} +function ownershipValue(root, repositoryRoot, nonce, extras = {}) { + return { + schemaVersion: 1, + kind: "p11-manual-acceptance", + nonce, + repositoryRoot, + root, + createdAt: nowIso(), + status: "PENDING", + listeners: { + backend: { host: HOST, port: BACKEND_PORT }, + frontend: { host: HOST, port: FRONTEND_PORT }, + }, + resources: [root, join(root, "remote.git"), join(root, "author"), join(root, "fixture-secrets")], + ...extras, + }; +} +export async function readManualOwnership({ repositoryRoot = defaultRepositoryRoot } = {}) { + const repo = realpathSync(repositoryRoot); + const root = fixedManualRoot(repo); + noSymlinkExisting(repo, root); + const rootEntry = await lstat(root); + const ownershipPath = join(root, "ownership.json"); + const digestPath = join(root, OWNERSHIP_DIGEST); + const ownershipEntry = await lstat(ownershipPath); + const digestEntry = await lstat(digestPath); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink() || !digestEntry.isFile() || digestEntry.isSymbolicLink()) throw new Error("manual ownership is unsafe"); + const ownershipBytes = await readFile(ownershipPath, "utf8"); + const recordedDigest = (await readFile(digestPath, "utf8")).trim(); + if (!HEX64.test(recordedDigest) || recordedDigest !== ownershipDigest(ownershipBytes)) throw new Error("manual ownership digest mismatch"); + const value = JSON.parse(ownershipBytes); + if (value?.schemaVersion !== 1 || value.kind !== "p11-manual-acceptance" || !HEX64.test(value.nonce ?? "") || value.repositoryRoot !== repo || value.root !== root) { + throw new Error("manual ownership identity mismatch"); + } + return value; +} +async function ensureRootAbsent(root) { + try { await lstat(root); throw new Error("manual acceptance root already exists"); } catch (error) { if (error.code !== "ENOENT") throw error; } +} +async function waitForHttp(url, timeoutMs = 15_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + await new Promise((resolvePromise, reject) => { + const request = http.get(url, (response) => { response.resume(); response.statusCode && response.statusCode < 500 ? resolvePromise() : reject(new Error("not ready")); }); + request.on("error", reject); + }); + return; + } catch { + await new Promise((resolvePromise) => setTimeout(resolvePromise, 250)); + } + } + throw new Error(`timed out waiting for ${url}`); +} +function live(pid) { try { process.kill(pid, 0); return true; } catch { return false; } } +async function writeCommands(repo, root) { + const commands = [ + ["http-01-status.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspace-registry/status`, join(root, "responses", "status.json"))], + ["http-02-validate-p11-filesystem.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-filesystem.json"), join(root, "requests", "validate-p11-filesystem.json"))], + ["http-03-validate-p11-http.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-http.json"), join(root, "requests", "validate-p11-http.json"))], + ["http-04-validate-p11-s3.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "validate-p11-s3.json"), join(root, "requests", "validate-p11-s3.json"))], + ["http-05-publish-p11-filesystem.sh", publishCurl(root, "p11-filesystem", join(root, "responses", "status.json"))], + ["http-06-publish-p11-http.sh", publishCurl(root, "p11-http", join(root, "responses", "publish-p11-filesystem.json"))], + ["http-07-publish-p11-s3.sh", publishCurl(root, "p11-s3", join(root, "responses", "publish-p11-http.json"))], + ["http-08-pull.sh", curlPostEmpty(`http://${HOST}:${BACKEND_PORT}/workspace-registry/pull`, join(root, "responses", "pull.json"))], + ["http-09-read-p11-filesystem.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspaces/p11-filesystem`, join(root, "responses", "read-p11-filesystem.json"))], + ["http-10-export-p11-filesystem.sh", curlGet(`http://${HOST}:${BACKEND_PORT}/workspaces/p11-filesystem/export`, join(root, "exports", "raw", "p11-filesystem.zip"))], + ["http-11-negative-invalid-uri.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "negative-invalid-uri.json"), join(root, "requests", "negative-invalid-uri.json"))], + ["http-12-negative-secret-field.sh", curlPost(`http://${HOST}:${BACKEND_PORT}/workspaces/validate`, join(root, "responses", "negative-secret-field.json"), join(root, "requests", "negative-secret-field.json"))], + ["render-1.sh", renderCommand(repo, root, 1)], + ["render-2.sh", renderCommand(repo, root, 2)], + ]; + for (const [name, body] of commands) { + const path = join(root, "commands", name); + await atomicWrite(path, body, 0o700); + } +} +export async function prepareManual({ repositoryRoot = defaultRepositoryRoot } = {}) { + const repo = realpathSync(repositoryRoot); + const root = fixedManualRoot(repo); + noSymlinkExisting(repo, root); + await ensureRootAbsent(root); + await mkdir(join(repo, ".artifacts", "manual-acceptance"), { recursive: true, mode: 0o700 }); + await mkdir(root, { mode: 0o700 }); + const executables = resolveExecutables(repo); + const nonce = randomBytes(32).toString("hex"); + await writeManualOwnership(root, ownershipValue(root, repo, nonce)); + for (const path of ["fixture-secrets", "requests", "responses", "commands", "rendered", "logs", "exports/raw", "exports/extracted", "installation/registry", "installation/data", "installation/runtime"]) { + await mkdir(join(root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); + } + const env = buildSafeEnvironment({ ambient: process.env, fixture: { PATH: dirname(executables.gitPath) } }); + await git(executables.gitPath, ["init", "--bare", "--initial-branch=main", join(root, "remote.git")], { cwd: root, env }); + await git(executables.gitPath, ["clone", join(root, "remote.git"), join(root, "author")], { cwd: root, env }); + await git(executables.gitPath, ["config", "user.name", "P1 Fixture Curator"], { cwd: join(root, "author"), env }); + await git(executables.gitPath, ["config", "user.email", "p1-curator@example.invalid"], { cwd: join(root, "author"), env }); + const items = descriptors(); + await atomicWrite(join(root, "author", "thoth-workspaces.yaml"), `${JSON.stringify(catalog(items), null, 2)}\n`, 0o644); + await mkdir(join(root, "author", "p11-filesystem", "evidence", "domain"), { recursive: true }); + await atomicWrite(join(root, "author", "p11-filesystem", "evidence", "guide.md"), "# P1.1 curated Evidence\n", 0o644); + await atomicWrite(join(root, "author", "p11-filesystem", "evidence", "domain", "table.md"), "# Curated table\n", 0o644); + await git(executables.gitPath, ["add", "thoth-workspaces.yaml"], { cwd: join(root, "author"), env }); + await git(executables.gitPath, ["add", "-A", "p11-filesystem/evidence"], { cwd: join(root, "author"), env }); + await git(executables.gitPath, ["commit", "-m", "Bootstrap curated P1 content"], { cwd: join(root, "author"), env }); + await git(executables.gitPath, ["push", "origin", "main"], { cwd: join(root, "author"), env }); + const secrets = { + dwh: join(root, "fixture-secrets", "dwh-password"), + signed: join(root, "fixture-secrets", "evidence-signed-urls.json"), + access: join(root, "fixture-secrets", "evidence-access"), + secret: join(root, "fixture-secrets", "evidence-secret"), + session: join(root, "fixture-secrets", "evidence-session"), + }; + await atomicWrite(secrets.dwh, "manual-dwh-secret", 0o600); + await atomicWrite(secrets.signed, JSON.stringify(["https://evidence.example.test/guide.md?token=manual"]), 0o600); + await atomicWrite(secrets.access, "manual-access", 0o600); + await atomicWrite(secrets.secret, "manual-secret", 0o600); + await atomicWrite(secrets.session, "manual-session", 0o600); + const bindings = {}; + for (const workspace of items) { + const prefix = `THT_WS_${namespace(workspace.workspace.id)}`; + Object.assign(bindings, { + [`${prefix}_DWH_TRANSPORT`]: "postgres_direct", + [`${prefix}_DWH_HOST`]: "dwh.invalid", + [`${prefix}_DWH_PORT`]: "5432", + [`${prefix}_DWH_USER`]: "reader", + [`${prefix}_DWH_PASSWORD_FILE`]: secrets.dwh, + }); + } + Object.assign(bindings, { + THT_WS_P11_HTTP_EVIDENCE_SIGNED_URLS_FILE: secrets.signed, + THT_WS_P11_S3_EVIDENCE_ACCESS_KEY_FILE: secrets.access, + THT_WS_P11_S3_EVIDENCE_SECRET_KEY_FILE: secrets.secret, + THT_WS_P11_S3_EVIDENCE_SESSION_TOKEN_FILE: secrets.session, + }); + await atomicWrite(join(root, "installation", "bindings.env"), `${Object.entries(bindings).map(([key, value]) => `${key}=${value}`).join("\n")}\n`); + await atomicWrite(join(root, "installation", "runtime", "base.yaml"), "{}\n"); + for (const [name, value] of Object.entries(requestFixtures(items))) await atomicWrite(join(root, "requests", name), `${JSON.stringify(value, null, 2)}\n`, 0o600); + await writeCommands(repo, root); + await atomicWrite(join(root, "GUIDE.md"), guide(root), 0o600); + await atomicWrite(join(root, "logs", "backend.log"), "", 0o600); + const current = await readManualOwnership({ repositoryRoot: repo }); + current.status = "PENDING"; + current.requestFixtures = Object.keys(requestFixtures(items)); + current.commandScripts = (await readdir(join(root, "commands"))).sort(); + await writeManualOwnership(root, current); + return root; +} +export async function serveManual({ repositoryRoot = defaultRepositoryRoot } = {}) { + const repo = realpathSync(repositoryRoot); + const root = fixedManualRoot(repo); + const owned = await readManualOwnership({ repositoryRoot: repo }); + if (owned.status === "RUNNING") throw new Error("manual acceptance is already serving"); + await access(join(repo, "backend", "dist", "server.js")); + await access(join(repo, "frontend", "dist", "index.html")); + const executables = resolveExecutables(repo); + const logHandle = await open(join(root, "logs", "backend.log"), fsConstants.O_WRONLY | fsConstants.O_APPEND); + const homeDir = join(root, "installation", "runtime", "home"); + const tmpDir = join(root, "installation", "runtime", "tmp"); + await mkdir(homeDir, { recursive: true, mode: 0o700 }); + await mkdir(tmpDir, { recursive: true, mode: 0o700 }); + const fixtureEnv = { + PATH: `${dirname(executables.gitPath)}:${dirname(executables.pythonPath)}:${dirname(executables.thtPath)}:/usr/bin:/bin`, + HOME: homeDir, + TMPDIR: tmpDir, + HOST, + PORT: String(BACKEND_PORT), + AUTH_MODE: "none", + THT_BIN: executables.thtPath, + THT_HARNESS_DIR: join(repo, "harness"), + THT_DATA_ROOT: join(root, "installation", "data"), + SETTINGS_FILE: join(root, "installation", "data", "settings.json"), + MAINTENANCE_STATE_FILE: join(root, "installation", "data", "maintenance.json"), + THT_WORKSPACE_REGISTRY_ROOT: join(root, "installation", "registry"), + THT_WORKSPACE_GIT_REMOTE: join(root, "remote.git"), + THT_WORKSPACE_GIT_BRANCH: "main", + THT_WORKSPACE_GIT_AUTHOR_NAME: "P1 API Publisher", + THT_WORKSPACE_GIT_AUTHOR_EMAIL: "p1-api@example.invalid", + THT_WORKSPACE_INSTALLATION_ID: "p11-manual-acceptance", + THT_WORKSPACE_SECRET_ROOTS: join(root, "fixture-secrets"), + THT_HOME: join(root, "installation", "runtime", "tht-home"), + PYTHONDONTWRITEBYTECODE: "1", + PYTHONNOUSERSITE: "1", + }; + const bindingEnv = Object.fromEntries((await readFile(join(root, "installation", "bindings.env"), "utf8")).trim().split(/\n+/).map((line) => line.split(/=(.+)/))); + const env = buildSafeEnvironment({ ambient: process.env, fixture: { ...fixtureEnv, ...bindingEnv } }); + const backend = spawn(process.execPath, [join(repo, "backend", "dist", "server.js")], { cwd: repo, env, stdio: ["ignore", logHandle.fd, logHandle.fd], detached: true }); + const frontend = spawn(executables.pythonPath, ["-m", "http.server", String(FRONTEND_PORT), "--bind", HOST, "--directory", join(repo, "frontend", "dist")], { cwd: repo, env, stdio: ["ignore", "ignore", "ignore"], detached: true }); + backend.unref(); frontend.unref(); + await waitForHttp(`http://${HOST}:${BACKEND_PORT}/health`); + await waitForHttp(`http://${HOST}:${FRONTEND_PORT}/`); + await logHandle.close(); + owned.status = "RUNNING"; + owned.backend = { pid: backend.pid, port: BACKEND_PORT, command: [process.execPath, join(repo, "backend", "dist", "server.js")] }; + owned.frontend = { pid: frontend.pid, port: FRONTEND_PORT, command: [executables.pythonPath, "-m", "http.server", String(FRONTEND_PORT)] }; + await writeManualOwnership(root, owned); + return owned; +} +async function processCommandMatches(pid, expectedCommand) { + if (!Array.isArray(expectedCommand) || expectedCommand.length === 0) return false; + let output; + try { + const { stdout } = await execFileAsync("ps", ["-p", String(pid), "-o", "command="], { encoding: "utf8" }); + output = stdout.trim(); + } catch { + return false; + } + if (output.length === 0) return false; + // The recorded command is the argv array used to spawn the process; verify every token appears + // in the current command line in order, so a reused PID with unrelated command is refused. + let cursor = 0; + for (const token of expectedCommand) { + if (token.length === 0) continue; + const index = output.indexOf(token, cursor); + if (index < 0) return false; + cursor = index + token.length; + } + return true; +} + +export async function stopManual({ repositoryRoot = defaultRepositoryRoot } = {}) { + const repo = realpathSync(repositoryRoot); + const root = fixedManualRoot(repo); + const owned = await readManualOwnership({ repositoryRoot: repo }); + if (owned.status !== "RUNNING" || !owned.backend?.pid || !owned.frontend?.pid) throw new Error("manual acceptance is not running"); + for (const pid of [owned.backend.pid, owned.frontend.pid]) { + try { process.kill(-pid, "SIGTERM"); } catch (error) { if (error?.code !== "ESRCH") throw error; } + } + const deadline = Date.now() + 15_000; + while (Date.now() < deadline && (live(owned.backend.pid) || live(owned.frontend.pid))) await new Promise((resolvePromise) => setTimeout(resolvePromise, 250)); + owned.status = "STOPPED"; + await writeManualOwnership(root, owned); + return owned; +} + +export async function cleanupManual({ repositoryRoot = defaultRepositoryRoot } = {}) { + const repo = realpathSync(repositoryRoot); + const root = fixedManualRoot(repo); + const owned = await readManualOwnership({ repositoryRoot: repo }); + if (owned.status === "RUNNING") throw new Error("manual acceptance is still live"); + if (owned.backend?.pid && live(owned.backend.pid)) throw new Error("backend process is still live"); + if (owned.frontend?.pid && live(owned.frontend.pid)) throw new Error("frontend process is still live"); + const parent = dirname(root); + const tombstone = join(parent, `.deleting-p11-${owned.nonce.slice(0, 16)}`); + await rename(root, tombstone); + await rm(tombstone, { recursive: true, force: false }); +} +export async function main(argv = process.argv.slice(2)) { + if (argv.length !== 1 || !["prepare", "serve", "stop", "cleanup"].includes(argv[0])) throw new Error("usage: p11-manual-acceptance.mjs prepare|serve|stop|cleanup"); + switch (argv[0]) { + case "prepare": await prepareManual(); break; + case "serve": await serveManual(); break; + case "stop": await stopManual(); break; + case "cleanup": await cleanupManual(); break; + } +} +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { await main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; } +} diff --git a/backend/scripts/p11-manual-acceptance.test.mjs b/backend/scripts/p11-manual-acceptance.test.mjs new file mode 100644 index 00000000..12d5fe22 --- /dev/null +++ b/backend/scripts/p11-manual-acceptance.test.mjs @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { access, lstat, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; + +import { + cleanupManual, + prepareManual, + readManualOwnership, + serveManual, + stopManual, +} from "./p11-manual-acceptance.mjs"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const fixedRoot = join(repoRoot, ".artifacts", "manual-acceptance", "p11"); + +async function safeCleanup() { + try { + const owned = await readManualOwnership({ repositoryRoot: repoRoot }); + if (owned.status === "RUNNING") await stopManual({ repositoryRoot: repoRoot }).catch(() => {}); + await cleanupManual({ repositoryRoot: repoRoot }).catch(() => {}); + } catch { + await rm(fixedRoot, { recursive: true, force: true }).catch(() => {}); + } +} + +test.beforeEach(async () => { + await safeCleanup(); +}); + +test.afterEach(async () => { + await safeCleanup(); +}); + +test("prepare creates an independent pending lab without verdict", { concurrency: false }, async () => { + const root = await prepareManual({ repositoryRoot: repoRoot }); + assert.equal(root, fixedRoot); + const owned = await readManualOwnership({ repositoryRoot: repoRoot }); + assert.equal(owned.kind, "p11-manual-acceptance"); + assert.equal(owned.status, "PENDING"); + await access(join(root, "GUIDE.md")); + await access(join(root, "author", "thoth-workspaces.yaml")); + await access(join(root, "author", "p11-filesystem", "evidence", "guide.md")); + await access(join(root, "requests", "validate-p11-filesystem.json")); + await access(join(root, "commands", "http-01-status.sh")); + await access(join(root, "commands", "render-1.sh")); + await assert.rejects(access(join(root, "VERDICT.md"))); + const guide = await readFile(join(root, "GUIDE.md"), "utf8"); + assert.match(guide, /VERDICT\.md/); + assert.match(guide, /read-only/); +}); + +test("serve, stop, and cleanup manage the owned backend and frontend listeners", { concurrency: false }, async () => { + await prepareManual({ repositoryRoot: repoRoot }); + const running = await serveManual({ repositoryRoot: repoRoot }); + assert.equal(running.status, "RUNNING"); + assert.equal(typeof running.backend.pid, "number"); + assert.equal(typeof running.frontend.pid, "number"); + const status = await fetch("http://127.0.0.1:8791/workspace-registry/status"); + assert.equal(status.status, 200); + const frontend = await fetch("http://127.0.0.1:8792/"); + assert.equal(frontend.status, 200); + await assert.rejects(cleanupManual({ repositoryRoot: repoRoot }), /still live/); + const stopped = await stopManual({ repositoryRoot: repoRoot }); + assert.equal(stopped.status, "STOPPED"); + await cleanupManual({ repositoryRoot: repoRoot }); + await assert.rejects(lstat(fixedRoot)); +}); + +test("stop fails closed when ownership is tampered", { concurrency: false }, async () => { + await prepareManual({ repositoryRoot: repoRoot }); + const running = await serveManual({ repositoryRoot: repoRoot }); + const ownershipPath = join(fixedRoot, "ownership.json"); + const digestPath = join(fixedRoot, "ownership.sha256"); + const original = JSON.parse(await readFile(ownershipPath, "utf8")); + const tampered = { ...original, backend: { ...original.backend, pid: original.backend.pid + 1 } }; + await rm(ownershipPath); + await readFile(join(fixedRoot, "logs", "backend.log")); + await import("node:fs/promises").then(({ writeFile }) => writeFile(ownershipPath, `${JSON.stringify(tampered, null, 2)} +`)); + await assert.rejects(stopManual({ repositoryRoot: repoRoot }), /manual ownership digest mismatch/); + const restored = `${JSON.stringify(running, null, 2)} +`; + const restoredDigest = `${createHash("sha256").update(restored).digest("hex")} +`; + await import("node:fs/promises").then(({ writeFile }) => Promise.all([writeFile(ownershipPath, restored), writeFile(digestPath, restoredDigest)])); + await stopManual({ repositoryRoot: repoRoot }); +}); diff --git a/backend/scripts/p11-render-snapshot.mjs b/backend/scripts/p11-render-snapshot.mjs new file mode 100644 index 00000000..16b32e43 --- /dev/null +++ b/backend/scripts/p11-render-snapshot.mjs @@ -0,0 +1,190 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { constants, lstatSync, realpathSync } from "node:fs"; +import { lstat, mkdir, open, readFile, realpath } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { ThtRunner } from "../dist/tht/tht-runner.js"; + +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; + +function fixedRoot(repositoryRoot) { return join(realpathSync(repositoryRoot), ".artifacts", "manual-acceptance", "p11"); } +function below(parent, child) { const rel = relative(parent, child); return rel !== "" && !rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel); } +function assertNoSymlinks(root, path, allowMissingLeaf = false) { + const rel = relative(root, path); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path is outside owned root"); + let cursor = root; + const parts = rel.split(sep).filter(Boolean); + for (const [index, part] of parts.entries()) { + cursor = join(cursor, part); + try { if (lstatSync(cursor).isSymbolicLink()) throw new Error("owned path contains a symlink"); } + catch (error) { + if (allowMissingLeaf && error?.code === "ENOENT" && index === parts.length - 1) return; + throw error; + } + } +} +async function ownership(repositoryRoot, ownershipPath) { + const root = fixedRoot(repositoryRoot); + const expected = join(root, "ownership.json"); + if (resolve(ownershipPath) !== expected) throw new Error("ownership path is not owned"); + const rootEntry = await lstat(root); const ownershipEntry = await lstat(expected); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink() || !ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership is unsafe"); + if (await realpath(root) !== root) throw new Error("ownership root is not canonical"); + let value; try { value = JSON.parse(await readFile(expected, "utf8")); } catch { throw new Error("ownership is malformed"); } + if (value?.schemaVersion !== 1 || value.kind !== "p11-manual-acceptance" || !HEX64.test(value.nonce ?? "") || value.root !== root || value.repositoryRoot !== realpathSync(repositoryRoot)) { + throw new Error("ownership identity mismatch"); + } + return { root, value }; +} +const ANCHORED_PUBLISH_SOURCE=String.raw`import os,secrets,stat,sys +parent,name,expected_dev,expected_ino=sys.argv[1:] +pfd=fd=None;stage=".render-stage-"+secrets.token_hex(16);published=False +def fail(): raise RuntimeError("anchored publication refused") +try: + pfd=os.open(parent,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + identity=os.fstat(pfd) + if (identity.st_dev,identity.st_ino)!=(int(expected_dev),int(expected_ino)): fail() + try: os.stat(name,dir_fd=pfd,follow_symlinks=False); fail() + except FileNotFoundError: pass + fd=os.open(stage,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=pfd) + data=sys.stdin.buffer.read(33554433) + if len(data)>33554432: fail() + view=memoryview(data) + while view: + written=os.write(fd,view) + if written<=0: fail() + view=view[written:] + os.fsync(fd);os.close(fd);fd=None;os.rename(stage,name,src_dir_fd=pfd,dst_dir_fd=pfd);published=True;os.fsync(pfd) + current=os.stat(parent,follow_symlinks=False) + if not stat.S_ISDIR(current.st_mode) or (current.st_dev,current.st_ino)!=(identity.st_dev,identity.st_ino): fail() +except Exception: + if published: + try: os.unlink(name,dir_fd=pfd);os.fsync(pfd) + except Exception: pass + print("anchored output publication refused (details redacted)",file=sys.stderr);raise SystemExit(1) +finally: + if fd is not None: os.close(fd) + if pfd is not None: + try: os.unlink(stage,dir_fd=pfd) + except FileNotFoundError: pass + os.close(pfd) +`; +async function atomicCopy(source, output) { + const parent = dirname(output); + const entry = await lstat(parent); + if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("rendered parent identity is unsafe"); + const bytes = await readFile(source); + const result = spawnSync("python3", ["-c", ANCHORED_PUBLISH_SOURCE, parent, basename(output), String(entry.dev), String(entry.ino)], { input: bytes, encoding: "utf8", maxBuffer: 1024 * 1024 }); + if (result.error || result.status !== 0) throw new Error("anchored output publication refused; rendered parent identity changed or output is unsafe"); +} +function sameEntry(actual, expected) { return actual.dev === expected.dev && actual.ino === expected.ino; } +async function readBounded(path, max, label) { + let handle; + try { + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = await handle.stat(); const pathEntry = await lstat(path); + if (!before.isFile() || pathEntry.isSymbolicLink() || !pathEntry.isFile() || !sameEntry(before, pathEntry)) throw new Error(`${label} is unsafe`); + if (before.size < 1 || before.size > max) throw new Error(`${label} is unbounded`); + const bytes = Buffer.alloc(before.size); let offset = 0; + while (offset < bytes.length) { + const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset); + if (bytesRead < 1) throw new Error(`${label} changed while reading`); + offset += bytesRead; + } + const after = await handle.stat(); + if (!sameEntry(before, after) || after.size !== before.size) throw new Error(`${label} changed while reading`); + return bytes; + } finally { + if (handle) await handle.close().catch(() => {}); + } +} +async function readSnapshotManifest(root, manifestPath, commit, yamlName, expectedDigest) { + let manifestEntry; + try { assertNoSymlinks(root, manifestPath); manifestEntry = await lstat(manifestPath); } + catch (error) { if (error?.code === "ENOENT") throw new Error("snapshot manifest is missing or unbounded"); throw error; } + if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink() || await realpath(manifestPath) !== manifestPath) throw new Error("snapshot manifest is unsafe"); + const bytes = await readBounded(manifestPath, 1024 * 1024, "snapshot manifest"); + let manifest; try { manifest = JSON.parse(bytes.toString("utf8")); } catch { throw new Error("snapshot manifest is malformed"); } + const files = manifest?.files; + if (manifest?.head !== commit || !files || typeof files !== "object" || Array.isArray(files)) throw new Error("snapshot manifest identity is unsafe"); + if (!HEX64.test(files[yamlName] ?? "") || files[yamlName] !== expectedDigest) throw new Error("snapshot manifest digest is unsafe"); + return manifest; +} + +export async function renderOwnedSnapshot({ repositoryRoot = defaultRepositoryRoot, ownershipPath, snapshotPath, outputPath, snapshotSha256, env = process.env, beforePublish }) { + const repo = realpathSync(repositoryRoot); + const { root } = await ownership(repo, resolve(repo, ownershipPath)); + const snapshot = resolve(repo, snapshotPath); + const output = resolve(repo, outputPath); + const snapshotsRoot = join(root, "installation", "registry", "snapshots"); + const renderedRoot = join(root, "rendered"); + if (!isAbsolute(snapshotPath) || !below(snapshotsRoot, snapshot)) throw new Error("snapshot is not an owned absolute path"); + const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(relative(snapshotsRoot, snapshot).split(sep).join("/")); + if (!match || !HEX40.test(match[1])) throw new Error("snapshot is not commit addressed"); + if (!HEX64.test(snapshotSha256 ?? "")) throw new Error("snapshot digest identity is unsafe"); + assertNoSymlinks(root, snapshot); + const snapshotEntry = await lstat(snapshot); + if (!snapshotEntry.isFile() || snapshotEntry.isSymbolicLink() || await realpath(snapshot) !== snapshot) throw new Error("snapshot is unsafe"); + const yamlName = `${match[2]}.yaml`; + await readSnapshotManifest(root, join(snapshotsRoot, match[1], "snapshot.json"), match[1], yamlName, snapshotSha256); + const snapshotBytes = await readBounded(snapshot, 1024 * 1024, "snapshot"); + if (createHash("sha256").update(snapshotBytes).digest("hex") !== snapshotSha256) throw new Error("snapshot bytes changed"); + if (!below(renderedRoot, output) || dirname(output) !== renderedRoot || !output.endsWith(".yaml")) throw new Error("output is not an owned rendered path"); + assertNoSymlinks(root, dirname(output)); + try { if ((await lstat(output)).isSymbolicLink()) throw new Error("output is unsafe"); } catch (error) { if (error.code !== "ENOENT") throw error; } + await mkdir(join(snapshotsRoot, "runtime"), { recursive: true, mode: 0o700 }); + const bindingEnv = Object.fromEntries((await readFile(join(root, "installation", "bindings.env"), "utf8")).trim().split(/\n+/).filter(Boolean).map((line) => line.split(/=(.+)/))); + const effectiveEnv = { ...bindingEnv, ...env }; + const prior = {}; + for (const [key, value] of Object.entries(effectiveEnv)) { prior[key] = process.env[key]; if (value === undefined) delete process.env[key]; else process.env[key] = value; } + const runner = new ThtRunner({ + thtBin: join(repo, "harness", ".venv", "bin", "tht"), + harnessDir: join(repo, "harness"), + configPath: join(root, "installation", "runtime", "base.yaml"), + dataRoot: join(root, "installation", "data"), + runtimeSnapshotRoot: join(snapshotsRoot, "runtime"), + secretRoots: [join(root, "fixture-secrets")], + semanticRuntime: { internalQdrantUrl: "http://qdrant:6333", internalEmbeddingUrl: "http://embedding:11434", internalEmbeddingModel: "qwen3-embedding:0.6b", internalEmbeddingDimensions: 1024 }, + }); + let lease; + try { + lease = runner.acquireWorkspaceRuntime(snapshot); + const verifySnapshot = async () => { + const current = await readBounded(snapshot, 1024 * 1024, "snapshot"); + if (createHash("sha256").update(current).digest("hex") !== snapshotSha256) throw new Error("snapshot content changed during rendering"); + }; + await verifySnapshot(); + if (beforePublish) await beforePublish({ output, renderedRoot }); + await verifySnapshot(); + await atomicCopy(lease.path, output); + } finally { + if (lease) lease.release(); + for (const key of Object.keys(env)) { if (prior[key] === undefined) delete process.env[key]; else process.env[key] = prior[key]; } + } + return output; +} +function parseArgs(argv) { + if (argv.length !== 8) throw new Error("usage: p11-render-snapshot.mjs --ownership PATH --snapshot ABSOLUTE_PATH --output PATH --snapshot-sha256 HEX"); + const result = {}; + for (let index = 0; index < argv.length; index += 2) { + if (!["--ownership", "--snapshot", "--output", "--snapshot-sha256"].includes(argv[index]) || result[argv[index]]) throw new Error("invalid arguments"); + result[argv[index]] = argv[index + 1]; + } + return result; +} +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { + const args = parseArgs(process.argv.slice(2)); + await renderOwnedSnapshot({ ownershipPath: args["--ownership"], snapshotPath: args["--snapshot"], outputPath: args["--output"], snapshotSha256: args["--snapshot-sha256"] }); + console.log(`rendered ${resolve(args["--output"])}`); + } catch (error) { + console.error(`p11 render refused: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/backend/scripts/p11-render-snapshot.test.mjs b/backend/scripts/p11-render-snapshot.test.mjs new file mode 100644 index 00000000..efb3c8cc --- /dev/null +++ b/backend/scripts/p11-render-snapshot.test.mjs @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { access, readFile, rm } from "node:fs/promises"; +import { join, dirname, resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { renderOwnedSnapshot } from "./p11-render-snapshot.mjs"; +import { cleanupManual, prepareManual, readManualOwnership, serveManual, stopManual } from "./p11-manual-acceptance.mjs"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); +const fixedRoot = join(repoRoot, ".artifacts", "manual-acceptance", "p11"); + +async function safeCleanup() { + try { + const owned = await readManualOwnership({ repositoryRoot: repoRoot }); + if (owned.status === "RUNNING") await stopManual({ repositoryRoot: repoRoot }).catch(() => {}); + await cleanupManual({ repositoryRoot: repoRoot }).catch(() => {}); + } catch { + await rm(fixedRoot, { recursive: true, force: true }).catch(() => {}); + } +} + +test.beforeEach(async () => { await safeCleanup(); }); +test.afterEach(async () => { await safeCleanup(); }); + +test("renderer rejects unowned ownership and out-of-root snapshot paths", { concurrency: false }, async () => { + await prepareManual({ repositoryRoot: repoRoot }); + const outside = join(repoRoot, "outside.yaml"); + await import("node:fs/promises").then(({ writeFile }) => writeFile(outside, "x")); + await assert.rejects(renderOwnedSnapshot({ + repositoryRoot: repoRoot, + ownershipPath: join(repoRoot, "ownership.json"), + snapshotPath: outside, + outputPath: join(fixedRoot, "rendered", "bad.yaml"), + snapshotSha256: "a".repeat(64), + })); + await rm(outside, { force: true }); +}); + +test("renderer copies an owned runtime lease deterministically", { concurrency: false }, async () => { + await prepareManual({ repositoryRoot: repoRoot }); + await serveManual({ repositoryRoot: repoRoot }); + const validateRequest = JSON.parse(await readFile(join(fixedRoot, "requests", "validate-p11-filesystem.json"), "utf8")); + const status = await fetch("http://127.0.0.1:8791/workspace-registry/status"); + const statusBody = await status.json(); + const publish = await fetch("http://127.0.0.1:8791/workspaces/publish", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ action: "create", workspace: validateRequest.workspace, baseCommit: statusBody.head }), + }); + assert.equal(publish.status, 200); + const readResponse = await fetch("http://127.0.0.1:8791/workspaces/p11-filesystem"); + const readBody = await readResponse.json(); + const snapshotPath = readBody.revision.snapshotPath; + const manifest = JSON.parse(await readFile(join(dirname(snapshotPath), "snapshot.json"), "utf8")); + const digest = manifest.files["p11-filesystem.yaml"]; + const one = join(fixedRoot, "rendered", "one.yaml"); + const two = join(fixedRoot, "rendered", "two.yaml"); + await renderOwnedSnapshot({ repositoryRoot: repoRoot, ownershipPath: join(fixedRoot, "ownership.json"), snapshotPath, outputPath: one, snapshotSha256: digest }); + await renderOwnedSnapshot({ repositoryRoot: repoRoot, ownershipPath: join(fixedRoot, "ownership.json"), snapshotPath, outputPath: two, snapshotSha256: digest }); + assert.equal(await readFile(one, "utf8"), await readFile(two, "utf8")); + await access(one); + await access(two); +}); diff --git a/backend/scripts/p2-acceptance.mjs b/backend/scripts/p2-acceptance.mjs new file mode 100644 index 00000000..d62d6542 --- /dev/null +++ b/backend/scripts/p2-acceptance.mjs @@ -0,0 +1,1352 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from "node:crypto"; +import { execFile, execFileSync } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { createServer } from "node:http"; +import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs"; +import { access, lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import net from "node:net"; +import process from "node:process"; + +import { stringify as yamlStringify } from "yaml"; + +import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const RUN_ID = /^p2-[0-9a-f]{32}$/; +const HEX32 = /^[0-9a-f]{32}$/; +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const COMMAND = /^[a-z0-9][a-z0-9-]*$/; +const CHECK_RESULT_STATUS = new Set(["PASS", "FAIL"]); +const CHECK_IDS = Object.freeze([ + "preflight", + "clean_state", + "ownership", + "inspect_identity", + "dwh_processing", + "schema_review", + "schema_index", + "evidence_processing", + "negative_cases", + "secret_scan", + "cleanup_confinement", +]); +const TOPOLOGY = [ + "remote.git", + "author", + "installation", + "installation/data", + "installation/data/sessions", + "installation/registry", + "installation/pi-state", + "fixture-secrets", + "fixtures", + "fixtures/logs", + "logs", +]; +const MAX_REPORT_JSON_BYTES = 64 * 1024; +const MAX_REPORT_MD_BYTES = 32 * 1024; +const MAX_STDIO_BYTES = 512 * 1024; +const MAX_SECRET_SCAN_VIRTUAL_BYTES = 256 * 1024; + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function assert(condition, message) { if (!condition) throw new Error(message); } +function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function canonicalRoot(repositoryRoot = defaultRepositoryRoot) { + return realpathSync(repositoryRoot); +} + +export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p2-integration"); +} + +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} + +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!existsSync(cursor)) break; + const entry = lstatSync(cursor); + if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink"); + } +} + +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + if (handle) await handle.close().catch(() => {}); + await rm(staging, { force: true }).catch(() => {}); + throw error; + } +} + +function initialResources(run) { + return [ + run.root, + join(run.root, "remote.git"), + join(run.root, "author"), + join(run.root, "installation"), + join(run.root, "installation", "registry"), + join(run.root, "installation", "data"), + join(run.root, "fixture-secrets"), + ]; +} + +function ownershipValue(run) { + return { + schemaVersion: 1, + kind: "p2-acceptance", + runId: run.runId, + runNonce: run.nonce, + root: run.root, + repositoryRoot: run.repositoryRoot, + startedAt: run.startedAt, + pid: run.pid, + resources: initialResources(run), + }; +} + +async function writeOwnership(run) { + await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run), null, 2)}\n`); +} + +export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) { + const repo = canonicalRoot(repositoryRoot); + const base = canonicalIntegrationBase(repo); + validateNoSymlinkAncestors(repo, base); + await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + const id = runId ?? `p2-${randomBytes(16).toString("hex")}`; + const root = validateRunRoot(repo, join(base, id), id); + const run = { + repositoryRoot: repo, + root, + runId: id, + nonce: nonce ?? randomBytes(32).toString("hex"), + startedAt: now ?? nowIso(), + pid: pid ?? process.pid, + }; + if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity"); + await mkdir(root, { mode: 0o700 }); + await writeOwnership(run); + return run; +} + +function strictOwnership(value, run, expectedNonce) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed"); + if (value.schemaVersion !== 1 || value.kind !== "p2-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce + || value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid + || !ISO_UTC.test(value.startedAt ?? "") + || JSON.stringify(value.resources) !== JSON.stringify(initialResources(run))) throw new Error("ownership identity mismatch"); + return value; +} + +export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const repo = canonicalRoot(repositoryRoot); + const id = basename(resolve(runRoot)); + const lexical = validateRunRoot(repo, runRoot, id); + const rootEntry = await lstat(lexical); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory"); + const ownershipPath = join(lexical, "ownership.json"); + const ownershipEntry = await lstat(ownershipPath); + if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe"); + let value; + try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); } + return strictOwnership(value, { repositoryRoot: repo, root: lexical, runId: id }, expectedNonce); +} + +export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }); + const base = canonicalIntegrationBase(repositoryRoot); + const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`); + await rename(runRoot, tombstone); + await rm(tombstone, { recursive: true, force: false }); +} + +async function finalizeOwnedRun({ run, success, keep }) { + if (!success || keep) return false; + await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + return true; +} + +function safeArtifactPath(path) { + if (typeof path !== "string" || path.length === 0 || path.length > 255 || path.startsWith("/") || path.includes("..") || path.includes("\\") || /[\0\r\n]/.test(path)) { + throw new Error("report artifact path is invalid"); + } + return path; +} + +function hasExactCheckIds(checks) { + return checks.length === CHECK_IDS.length && checks.every(({ id }, index) => id === CHECK_IDS[index]); +} + +export function validateReport(report) { + if (!report || report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "") + || !ISO_UTC.test(report.finishedAt ?? "") || typeof report.command !== "string" + || !Array.isArray(report.checks) || !hasExactCheckIds(report.checks)) throw new Error("report is invalid"); + const ids = new Set(); + const artifactPaths = new Set(); + for (const check of report.checks) { + if (!check || !/^[a-z0-9_]+$/.test(check.id ?? "") || ids.has(check.id) || !CHECK_RESULT_STATUS.has(check.status) + || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "") + || !Array.isArray(check.commands) || check.commands.some((name) => !COMMAND.test(name)) + || !Array.isArray(check.artifacts)) throw new Error("report check is invalid"); + ids.add(check.id); + for (const artifact of check.artifacts) { + safeArtifactPath(artifact.path); + if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report check is invalid"); + if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated"); + artifactPaths.add(artifact.path); + } + } + if (report.overall !== deriveOverall(report.checks)) throw new Error("report overall is not derived"); + return report; +} + +function renderReportMarkdown(report) { + validateReport(report); + const rows = report.checks.map((check) => `| ${check.id} | ${check.status} |`).join("\n"); + return [ + "# P2 acceptance report", + "", + `Run: \`${report.runId}\``, + "", + "| Check | Status |", + "|---|---|", + rows, + "", + `P2 automated integration: ${report.overall}`, + "P2 manual acceptance: PENDING", + "", + ].join("\n"); +} + +async function walkFiles(root) { + const files = []; + async function visit(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const rel = relative(root, path).split(sep).join("/"); + if (entry.isSymbolicLink()) throw new Error(`unsafe file tree: ${rel}`); + if (entry.isDirectory()) await visit(path); + else if (entry.isFile()) files.push({ path, rel }); + } + } + if (existsSync(root)) await visit(root); + files.sort((a, b) => a.rel.localeCompare(b.rel)); + return files; +} + +async function snapshotDigest(root, excludedPrefixes = []) { + const result = {}; + for (const file of await walkFiles(root)) { + if (excludedPrefixes.some((prefix) => file.rel === prefix || file.rel.startsWith(`${prefix}/`))) continue; + result[file.rel] = sha256(await readFile(file.path)); + } + return result; +} + +async function fileArtifact(root, relativePath) { + const bytes = await readFile(join(root, relativePath)); + return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) }; +} + +async function writeJson(path, value) { + await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function writeReportFiles({ run, report }) { + validateReport(report); + const reportJsonPath = join(run.root, "report.json"); + const reportMdPath = join(run.root, "report.md"); + const reportMd = renderReportMarkdown(report); + if (Buffer.byteLength(JSON.stringify(report)) > MAX_REPORT_JSON_BYTES) throw new Error("report.json exceeds bound"); + if (Buffer.byteLength(reportMd) > MAX_REPORT_MD_BYTES) throw new Error("report.md exceeds bound"); + await writeJson(reportJsonPath, report); + await atomicWrite(reportMdPath, reportMd, 0o600); + return { + reportJson: await fileArtifact(run.root, "report.json"), + reportMd: await fileArtifact(run.root, "report.md"), + }; +} + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (statSync(resolved).isFile()) return resolved; + } catch {} + } + throw new Error(`required executable not found: ${name}`); +} + +function scalarSecretBytes(value) { + if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid"); + return Buffer.from(value); +} + +async function manifestFiles(root, paths) { + const files = []; + const visit = async (absolute, rel) => { + const entry = await lstat(absolute); + if (entry.isSymbolicLink()) throw new Error(`provenance path is a symlink: ${rel}`); + if (entry.isDirectory()) { + for (const child of (await readdir(absolute, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + await visit(join(absolute, child.name), rel ? `${rel}/${child.name}` : child.name); + } + } else if (entry.isFile()) { + const bytes = await readFile(absolute); + files.push({ path: rel, bytes: bytes.length, sha256: sha256(bytes) }); + } else throw new Error(`provenance path is not a regular file: ${rel}`); + }; + for (const path of paths) await visit(join(root, path), path); + files.sort((a, b) => a.path.localeCompare(b.path)); + return { files, manifestSha256: sha256(JSON.stringify(files)) }; +} + +async function collectRepositoryProvenance({ repositoryRoot, gitPath = resolveSystemExecutable("git") }) { + const repo = canonicalRoot(repositoryRoot); + const safeEnv = buildSafeEnvironment({ ambient: {}, fixture: { PATH: `${dirname(gitPath)}:/usr/bin:/bin`, HOME: repo, TMPDIR: join(repo, ".artifacts") } }); + const run = async (argv) => await execFileAsync(gitPath, ["-C", repo, ...argv], { env: safeEnv, maxBuffer: MAX_STDIO_BYTES }); + const beforeHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const beforeTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const beforeStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (!HEX40.test(beforeHead) || !HEX40.test(beforeTree) || beforeStatus !== "") throw new Error("repository is not clean at exact HEAD"); + const backendRoot = join(repo, "backend"); + const backendSource = await manifestFiles(backendRoot, [ + "src", + "scripts/p2-acceptance.mjs", + "package.json", + "package-lock.json", + "tsconfig.json", + ]); + const backendDist = existsSync(join(backendRoot, "dist")) ? await manifestFiles(backendRoot, ["dist"]) : { files: [], manifestSha256: sha256("[]") }; + const afterHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const afterTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const afterStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (afterHead !== beforeHead || afterTree !== beforeTree || afterStatus !== beforeStatus) throw new Error("repository provenance changed during binding"); + return { schemaVersion: 1, head: beforeHead, tree: beforeTree, clean: true, backendSource, backendDist }; +} + +async function createTopology(run) { + for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); +} + +async function allocatePort() { + const server = net.createServer(); + await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).on("error", reject)); + const port = server.address().port; + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + return port; +} + +function installationProjectName(installationPath) { + return `thothii-${sha256(installationPath).slice(0, 12)}`; +} + +function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) { + return { + workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + diagnostics: { + dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}), + }; +} + +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } + +function descriptorYaml(obj) { + return yamlStringify(obj, { lineWidth: 0, sortMapEntries: false }); +} + +async function setupSecrets(ctx) { + const secretDir = join(ctx.run.root, "fixture-secrets"); + const values = { + dwhToken: `P2-DWH-${randomBytes(16).toString("hex")}`, + signedToken: `P2-SIGNED-${randomBytes(16).toString("hex")}`, + bundle: `P2-BUNDLE-${randomBytes(16).toString("hex")}`, + }; + ctx.forbiddenValues = Object.values(values); + ctx.secretValues = values; + const paths = { + dwh: join(secretDir, "p2-dwh-api-key"), + filesystemDwh: join(secretDir, "p2-filesystem-api-key"), + signed: join(secretDir, "p2-dwh-evidence-signed-urls.json"), + bundle: join(secretDir, "thothii.secrets"), + }; + await atomicWrite(paths.dwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.filesystemDwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.bundle, scalarSecretBytes(values.bundle)); + ctx.secretPaths = paths; +} + +async function setupFixtures(ctx) { + ctx.fixturePorts = { + dwh: await allocatePort(), + evidence: await allocatePort(), + embedding: await allocatePort(), + qdrant: await allocatePort(), + }; + const dwhBaseUrl = `http://host.docker.internal:${ctx.fixturePorts.dwh}`; + const evidenceProvenance = `http://host.docker.internal:${ctx.fixturePorts.evidence}/p2-dwh/guide.md`; + ctx.workspaceObjects = { + dwh: baseWorkspace("p2-dwh", { + dwhBaseUrl, + evidenceSource: { + type: "http", + uris: [evidenceProvenance], + authentication: "signed_urls_file", + connect_timeout_ms: 1250, + read_timeout_ms: 30001, + max_bytes: 65536, + max_redirects: 2, + allow_private_hosts: true, + max_cache_bytes: 65536, + }, + }), + filesystem: baseWorkspace("p2-filesystem", { + dwhBaseUrl, + evidenceSource: { + type: "filesystem", + uri: "p2-filesystem/evidence", + patterns: ["**/*.md"], + max_bytes: 1048576, + }, + }), + }; + const signedUrl = `${evidenceProvenance}?token=${ctx.secretValues.signedToken}`; + await atomicWrite(ctx.secretPaths.signed, `${JSON.stringify([signedUrl], null, 2)}\n`); + + ctx.evidenceState = { + content: "# P2 Evidence\n\nFirst generation.\n", + token: ctx.secretValues.signedToken, + }; + ctx.dwhState = { + tables: { + patients: { + comment: "Patients", + rows: [ + { patient_id: "p1", name: "Alice" }, + { patient_id: "p2", name: "Bob" }, + ], + }, + visits: { + comment: "Visits", + rows: [ + { id: "v1", patient_id: "p1", note: "checkup" }, + { id: "v2", patient_id: "p2", note: "xray" }, + ], + }, + labs: { + comment: "Labs", + rows: [ + { id: "l1", patient_id: "p1", code: "hemoglobin" }, + { id: "l2", patient_id: "p2", code: "glucose" }, + ], + }, + }, + token: ctx.secretValues.dwhToken, + }; +} + +function inferColumnType(value) { + return typeof value === "number" ? "integer" : "text"; +} + +function topValues(rows, column, limit) { + const counts = new Map(); + for (const row of rows) { + const value = row[column]; + if (value === undefined || value === null || value === "") continue; + counts.set(String(value), (counts.get(String(value)) ?? 0) + 1); + } + return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, limit).map(([value]) => ({ value })); +} + +async function startHttpServer({ port, handler }) { + const server = createServer(async (req, res) => { + try { + await handler(req, res); + } catch { + res.statusCode = 500; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "fixture failed" })); + } + }); + await new Promise((resolve, reject) => server.listen(port, "127.0.0.1", () => resolve()).on("error", reject)); + return server; +} + +async function startServers(ctx) { + const dwhServer = await startHttpServer({ + port: ctx.fixturePorts.dwh, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const json = body.length === 0 ? {} : JSON.parse(body); + if (req.headers["x-api-key"] !== ctx.dwhState.token) { + res.statusCode = 401; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ message: "unauthorized" })); + return; + } + const send = (payload) => { + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(payload)); + }; + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.dwh}`); + if (req.method !== "POST" || !url.pathname.startsWith("/rpc/")) { + res.statusCode = 404; + res.end(JSON.stringify({ message: "not found" })); + return; + } + const fn = url.pathname.slice("/rpc/".length); + const schemaName = json.schema_name ?? "dw"; + if (schemaName !== "dw") { + send([]); + return; + } + if (fn === "ping") { + send({ db_connected: true, schema_accessible: true, database: "warehouse", schema: "dw" }); + return; + } + const table = typeof json.table_name === "string" ? json.table_name : ""; + const tableData = ctx.dwhState.tables[table]; + if (fn === "list_tables") { + send(Object.entries(ctx.dwhState.tables).map(([name, info]) => ({ table: name, type: "TABLE", comment: info.comment, rows: info.rows.length }))); + return; + } + if (!tableData) { + send([]); + return; + } + if (fn === "table_columns") { + const first = tableData.rows[0] ?? {}; + send(Object.keys(first).map((column) => ({ + column, + type: inferColumnType(first[column]), + nullable: false, + // Only the referenced table marks `patient_id` as primary, so the SQL miner sees a + // PK/non-PK pair while the same-name heuristic still discovers joins from the others. + pk: column === "id" || (table === "patients" && column === "patient_id"), + default: null, + }))); + return; + } + if (fn === "table_comments") { + send(Object.keys(tableData.rows[0] ?? {}).map((column) => ({ object: "COLUMN", name: column, comment: `${table}.${column}` }))); + return; + } + if (fn === "table_foreign_keys") { + send([]); + return; + } + if (fn === "top_values") { + send(topValues(tableData.rows, json.column_name, Number(json.max_values ?? 10))); + return; + } + if (fn === "column_stats") { + send({}); + return; + } + if (fn === "run_query") { + send([]); + return; + } + if (fn === "explain_query") { + send([{ line: "Seq Scan" }]); + return; + } + res.statusCode = 404; + res.end(JSON.stringify({ message: "unknown rpc" })); + }, + }); + const evidenceServer = await startHttpServer({ + port: ctx.fixturePorts.evidence, + handler: async (req, res) => { + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.evidence}`); + if (url.pathname !== "/p2-dwh/guide.md" || url.searchParams.get("token") !== ctx.evidenceState.token) { + res.statusCode = 403; + res.end("forbidden"); + return; + } + res.statusCode = 200; + res.setHeader("content-type", "text/markdown; charset=utf-8"); + res.end(ctx.evidenceState.content); + }, + }); + const embeddingServer = await startHttpServer({ + port: ctx.fixturePorts.embedding, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.embedding}`); + if (req.method !== "POST" || url.pathname !== "/api/embed") { + res.statusCode = 404; + res.end(JSON.stringify({ error: "not found" })); + return; + } + const payload = JSON.parse(body || "{}"); + const inputs = Array.isArray(payload.input) ? payload.input : []; + const embeddings = inputs.map((text) => { + const seed = sha256(String(text)); + return Array.from({ length: 1024 }, (_, index) => { + const offset = (index * 2) % seed.length; + const value = Number.parseInt(seed.slice(offset, offset + 2), 16); + return (value / 255) - 0.5; + }); + }); + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ model: payload.model, embeddings })); + }, + }); + ctx.servers = [dwhServer, evidenceServer, embeddingServer]; +} + +async function stopServers(ctx) { + for (const server of ctx.servers ?? []) { + await new Promise((resolve) => server.close(() => resolve())); + } + ctx.servers = []; +} + +async function git(ctx, args, cwd = join(ctx.run.root, "author")) { + return await runCommand({ executable: ctx.executables.gitPath, argv: args, cwd, env: ctx.execEnv }); +} + +async function initializeGitAndRegistry(ctx) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], ctx.run.root); + await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], ctx.run.root); + await git(ctx, ["config", "user.name", "P2 Fixture Curator"], author); + await git(ctx, ["config", "user.email", "p2-curator@example.invalid"], author); + + const writeWorkspaces = async () => { + const catalog = { + schema_version: 1, + workspaces: [ + { id: "p2-dwh", name: ctx.workspaceObjects.dwh.workspace.name }, + { id: "p2-filesystem", name: ctx.workspaceObjects.filesystem.workspace.name }, + ], + }; + await writeFile(join(author, "thoth-workspaces.yaml"), yamlStringify(catalog, { lineWidth: 0, sortMapEntries: false })); + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId), { recursive: true }); + const yaml = descriptorYaml(workspace); + await writeFile(join(author, pathId, "workspace.yaml"), yaml); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + await mkdir(join(author, "workspace-docs", pathId), { recursive: true }); + await writeFile(join(author, "workspace-docs", pathId, "contract.env.example"), docs.envExample); + await writeFile(join(author, "workspace-docs", pathId, "README.md"), docs.markdown); + } + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "evidence", "guide.md"), "# P2 Filesystem Evidence\n\nCommitted fixture.\n"); + }; + + await writeWorkspaces(); + await git(ctx, ["add", "."], author); + await git(ctx, ["commit", "-m", "Bootstrap P2 fixtures"], author); + await git(ctx, ["push", "origin", "main"], author); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); + const registry = new ctx.workspaceModules.WorkspaceRegistry({ + root: join(ctx.run.root, "installation", "registry"), + remoteUrl: join(ctx.run.root, "remote.git"), + branch: "main", + gitAuthorName: "P2 Acceptance", + gitAuthorEmail: "p2-acceptance@example.invalid", + installationId: "p2-acceptance", + secretRoots: [join(ctx.run.root, "fixture-secrets")], + maxImportBytes: 16 * 1024 * 1024, + maxImportEntries: 1024, + }); + await registry.bootstrap(); + ctx.registry = registry; +} + +async function mutateWorkspaceDescriptor(ctx, workspaceId, mutator, commitMessage) { + const author = join(ctx.run.root, "author"); + const workspace = structuredClone(ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"]); + mutator(workspace); + ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"] = workspace; + await writeFile(join(author, workspaceId, "workspace.yaml"), descriptorYaml(workspace)); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + await writeFile(join(author, "workspace-docs", workspaceId, "contract.env.example"), docs.envExample); + await writeFile(join(author, "workspace-docs", workspaceId, "README.md"), docs.markdown); + await git(ctx, ["add", `${workspaceId}/workspace.yaml`, `workspace-docs/${workspaceId}/contract.env.example`, `workspace-docs/${workspaceId}/README.md`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + +async function writeInstallationFiles(ctx) { + const installationDir = join(ctx.run.root, "installation"); + const operatorEnvPath = join(installationDir, "operator.env"); + const bindingsEnvPath = join(installationDir, "workspace-bindings.env"); + const connectorOverridePath = join(installationDir, "connector-secrets.override.yaml"); + const fixtureOverridePath = join(installationDir, "fixture.override.yaml"); + const installationPath = join(installationDir, "thothii-installation.yaml"); + ctx.installationPath = installationPath; + ctx.composeProject = installationProjectName(installationPath); + const qdrantPort = ctx.fixturePorts.qdrant; + const bindings = [ + `THT_WS_P2_DWH_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_DWH_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_DWH_DWH_API_KEY_FILE=/run/secrets/p2-dwh-api-key`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_FILE=/run/secrets/p2-dwh-evidence-signed-urls`, + `THT_WS_P2_FILESYSTEM_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_FILESYSTEM_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_FILE=/run/secrets/p2-filesystem-api-key`, + ].join("\n") + "\n"; + await atomicWrite(bindingsEnvPath, bindings); + const operatorEnv = [ + `THT_DATA_ROOT=${join(ctx.run.root, "installation", "data")}`, + `THT_WORKSPACE_REGISTRY_ROOT=${join(ctx.run.root, "installation", "registry")}`, + `THT_PI_STATE_ROOT=${join(ctx.run.root, "installation", "pi-state")}`, + `PI_AUTH_FILE=${join(ctx.run.root, "installation", "pi-auth.json")}`, + `THT_SECRETS_FILE=${ctx.secretPaths.bundle}`, + `THT_WORKSPACE_BINDINGS_ENV_FILE=${bindingsEnvPath}`, + `THT_WORKSPACE_GIT_REMOTE=${join(ctx.run.root, "remote.git")}`, + `THT_WORKSPACE_GIT_BRANCH=main`, + `THT_WORKSPACE_GIT_AUTHOR_NAME=P2 Acceptance`, + `THT_WORKSPACE_GIT_AUTHOR_EMAIL=p2-acceptance@example.invalid`, + `THT_WORKSPACE_INSTALLATION_ID=p2-acceptance`, + `THT_DB_NAME=warehouse`, + `THT_DWH_REST_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_LLM_URL=http://127.0.0.1:9`, + `THOTH_SERVER_BIND=127.0.0.1`, + `THOTH_HTTP_PORT=18080`, + `THOTH_CORE_HTTP_PORT=18787`, + `THT_WS_P2_DWH_DWH_API_KEY_SOURCE=${ctx.secretPaths.dwh}`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_SOURCE=${ctx.secretPaths.signed}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_SOURCE=${ctx.secretPaths.filesystemDwh}`, + `THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST=host.docker.internal`, + ].join("\n") + "\n"; + await atomicWrite(operatorEnvPath, operatorEnv); + await atomicWrite(join(ctx.run.root, "installation", "pi-auth.json"), JSON.stringify({ fixture: true })); + const embeddingStubPath = join(installationDir, "embedding-stub.py"); + await atomicWrite(embeddingStubPath, EMBEDDING_STUB_SOURCE); + const override = { + services: { + core: { + image: ctx.coreImageTag, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + "workspace-maintenance": { + image: ctx.coreImageTag, + environment: { + THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST: "host.docker.internal", + }, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + qdrant: { + ports: [`127.0.0.1:${qdrantPort}:6333`], + restart: "no", + }, + // Deterministic Ollama-compatible embedding fixture on the internal allowlisted host + // name `embedding` (http://embedding:11434). Replaces the real Ollama service entirely. + embedding: { + image: ctx.coreImageTag, + entrypoint: ["python3", "/stub.py"], + volumes: [ + { type: "bind", source: embeddingStubPath, target: "/stub.py", read_only: true }, + ], + healthcheck: { disable: true }, + }, + }, + }; + await atomicWrite(fixtureOverridePath, yamlStringify(override, { lineWidth: 0, sortMapEntries: false })); + const generated = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "generate-connector-secrets-override.sh"), + argv: [ + "--bindings-env", bindingsEnvPath, + "--operator-env", operatorEnvPath, + "--output", connectorOverridePath, + "--service", "workspace-maintenance", + "--role", "all", + ], + env: ctx.execEnv, + }); + if (generated.exitCode !== 0) throw new Error(`connector override generation failed: ${generated.stderr || generated.stdout}`); + const installation = { + profile: "server", + projectDirectory: ctx.repositoryRoot, + envFile: operatorEnvPath, + overrides: [ + join(ctx.repositoryRoot, "deploy", "compose.server.yaml"), + fixtureOverridePath, + connectorOverridePath, + ], + }; + await atomicWrite(installationPath, yamlStringify(installation, { lineWidth: 0, sortMapEntries: false })); + ctx.installation = installation; +} + +function thothctlBinaryPath(repositoryRoot) { + const platform = { darwin: "darwin", linux: "linux", win32: "windows" }[process.platform] ?? "linux"; + const arch = { x64: "amd64", arm64: "arm64" }[process.arch] ?? "amd64"; + const suffix = platform === "windows" ? ".exe" : ""; + const candidates = [ + join(repositoryRoot, "dist", "thothctl", `thothctl-${platform}-${arch}${suffix}`), + join(repositoryRoot, "tools", "thothctl", "bin", `thothctl${suffix}`), + ]; + for (const candidate of candidates) if (existsSync(candidate)) return candidate; + throw new Error("built thothctl binary is unavailable"); +} + +async function runCommand({ executable, argv = [], cwd, env, input, maxOutputBytes = MAX_STDIO_BYTES }) { + const result = await execFileAsync(executable, argv, { + cwd, + env, + encoding: "utf8", + maxBuffer: maxOutputBytes, + ...(input === undefined ? {} : { input }), + }).then( + ({ stdout, stderr }) => ({ exitCode: 0, stdout, stderr }), + (error) => ({ exitCode: error.code ?? 1, stdout: error.stdout ?? "", stderr: error.stderr ?? error.message ?? "" }), + ); + return result; +} + +async function buildCoreImage(ctx) { + const tag = `thothii-core:p2-${ctx.run.runId.slice(3, 15)}`; + ctx.coreImageTag = tag; + const build = await runCommand({ + executable: ctx.executables.dockerPath, + argv: ["build", "-f", join(ctx.repositoryRoot, "docker", "core.Dockerfile"), "-t", tag, ctx.repositoryRoot], + env: { ...ctx.execEnv, DOCKER_BUILDKIT: "1" }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (build.exitCode !== 0) throw new Error(`core image build failed: ${build.stderr || build.stdout}`); +} + +async function buildThothctl(ctx) { + const command = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "build-thothctl.sh"), + argv: [], + env: { ...ctx.execEnv, THT_THOTHCTL_OUTPUT_DIRECTORY: join(ctx.repositoryRoot, "dist", "thothctl") }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (command.exitCode !== 0) throw new Error(`build-thothctl failed: ${command.stderr || command.stdout}`); + ctx.thothctlPath = thothctlBinaryPath(ctx.repositoryRoot); +} + +function composeBaseArgs(ctx) { + const args = [ + "compose", + "--project-name", ctx.composeProject, + "--project-directory", ctx.installation.projectDirectory, + "--env-file", ctx.installation.envFile, + "-f", join(ctx.repositoryRoot, "compose.yaml"), + ]; + for (const override of ctx.installation.overrides) args.push("-f", override); + return args; +} + +async function dockerCompose(ctx, commandArgs, { allowFailure = false, maxOutputBytes = 2 * 1024 * 1024 } = {}) { + const result = await runCommand({ + executable: ctx.executables.dockerPath, + argv: [...composeBaseArgs(ctx), ...commandArgs], + env: ctx.execEnv, + maxOutputBytes, + }); + if (!allowFailure && result.exitCode !== 0) throw new Error(`docker compose ${commandArgs.join(" ")} failed: ${result.stderr || result.stdout}`); + return result; +} + +async function startQdrant(ctx) { + await dockerCompose(ctx, ["up", "-d", "qdrant", "embedding"]); + for (let attempt = 0; attempt < 60; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}/collections`); + if (response.ok) return; + } catch {} + await sleep(1000); + } + throw new Error("qdrant did not become ready"); +} + +async function qdrantJson(ctx, method, path, body) { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}${path}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const payload = response.status === 204 ? {} : await response.json().catch(() => ({})); + if (!response.ok) throw new Error(`qdrant request failed: ${method} ${path} ${response.status}`); + return payload; +} + +async function preprovisionCollection(ctx, workspaceId) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}`, { + vectors: { size: 1024, distance: "Cosine" }, + }); + for (const field of ["content_hash", "document_id", "kind", "record_key", "record_kind", "vector_generation", "workspace_id", "workspace_revision"]) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}/index`, { field_name: field, field_schema: "keyword" }); + } +} + +async function listCollections(ctx) { + const payload = await qdrantJson(ctx, "GET", "/collections"); + const collections = payload.result?.collections ?? []; + return collections.map((item) => item.name).sort(); +} + +async function dumpQdrantPayloads(ctx, workspaceId) { + const response = await qdrantJson(ctx, "POST", `/collections/${workspaceId}/points/scroll`, { limit: 128, with_payload: true, with_vector: false }); + return JSON.stringify(response.result?.points ?? []); +} + +async function runThothctlJson(ctx, label, workspaceArgs, expectedExitCode) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.json`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs, "--json"], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + if (expectedExitCode !== undefined && result.exitCode !== expectedExitCode) { + throw new Error(`${label} exit ${result.exitCode} != ${expectedExitCode}`); + } + let payload; + try { payload = JSON.parse(result.stdout); } catch (error) { throw new Error(`${label} returned non-JSON stdout`); } + return { result, payload, artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath))] }; +} + +async function loadWorkspaceSnapshot(ctx, workspaceId) { + const active = JSON.parse(await readFile(join(ctx.run.root, "installation", "registry", "state", "active.json"), "utf8")); + const revision = active.revisions.find((entry) => entry.id === workspaceId); + const snapshotPath = revision.snapshotPath; + const contents = await readFile(snapshotPath, "utf8"); + return { active, revision, contents }; +} + +async function assertNoCoreFrontendRunning(ctx) { + const ps = await dockerCompose(ctx, ["ps", "--status", "running", "--format", "json"], { allowFailure: true }); + if (ps.exitCode !== 0) return []; + const lines = ps.stdout.trim() === "" ? [] : ps.stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); + const services = lines.map((item) => item.Service); + if (services.includes("core") || services.includes("frontend") || services.includes("workspace-maintenance")) { + throw new Error("core/frontend/maintenance is unexpectedly running"); + } + return services; +} + +function sameSet(left, right) { + return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort()); +} + +const EMBEDDING_STUB_SOURCE = String.raw`import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +class _Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(length)) + inputs = payload.get("input", []) + if isinstance(inputs, str): + inputs = [inputs] + embeddings = [[0.01] * 1024 for _ in inputs] + body = json.dumps({"model": payload.get("model", "qwen3-embedding:0.6b"), "embeddings": embeddings}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + +HTTPServer(("0.0.0.0", 11434), _Handler).serve_forever() +`; + +function realUserHome() { + try { + const output = execFileSync("bash", ["-lc", 'printf "%s" ~'], { encoding: "utf8" }).trim(); + return output.length > 0 ? output : undefined; + } catch { + return undefined; + } +} + +async function setupRealContext({ repositoryRoot = defaultRepositoryRoot, env = process.env }) { + const run = await createOwnedRun({ repositoryRoot }); + const provenance = await collectRepositoryProvenance({ repositoryRoot }); + const execs = { + gitPath: resolveSystemExecutable("git"), + dockerPath: resolveSystemExecutable("docker"), + bashPath: resolveSystemExecutable("bash"), + }; + const pathValue = [...new Set([dirname(execs.gitPath), dirname(execs.dockerPath), "/usr/bin", "/bin", "/opt/homebrew/bin", "/usr/local/bin"])].join(":"); + // Docker CLI plugins (buildx) live under the real user's ~/.docker; the wrapper runs with a + // scrubbed environment, so derive the real home from the passwd entry and expose DOCKER_CONFIG. + const realHome = env.P2_REAL_HOME ?? realUserHome(); + const execEnv = buildSafeEnvironment({ ambient: env, fixture: { + PATH: pathValue, + HOME: run.root, + TMPDIR: join(run.root, "tmp"), + ...(realHome ? { DOCKER_CONFIG: join(realHome, ".docker") } : {}), + } }); + const workspaceModules = await import("../dist/workspaces/registry.js").then(async (registryModule) => ({ + WorkspaceRegistry: registryModule.WorkspaceRegistry, + ...(await import("../dist/workspaces/schema.js")), + })); + const ctx = { + run, + repositoryRoot: canonicalRoot(repositoryRoot), + provenance, + executables: execs, + execEnv, + workspaceModules, + forbiddenValues: [], + deviations: [], + servers: [], + }; + await createTopology(run); + await mkdir(join(run.root, "tmp"), { recursive: true, mode: 0o700 }); + await setupSecrets(ctx); + await setupFixtures(ctx); + return ctx; +} + +async function executeChecksLocal({ checks, failAt } = {}) { + if (!Array.isArray(checks) || !hasExactCheckIds(checks)) throw new Error("scenarios must match the exact ordered check set"); + if (failAt !== undefined && !CHECK_IDS.includes(failAt)) throw new Error("failure hook must name an exact check"); + const results = []; + let stopped = false; + for (const scenario of checks) { + const startedAt = nowIso(); + let result; + if (stopped) { + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: "Not executed after earlier failure." }; + } else { + try { + const output = await scenario.run(); + if (scenario.id === failAt) throw new Error("injected acceptance failure"); + result = { id: scenario.id, status: "PASS", startedAt, finishedAt: nowIso(), commands: output.commands ?? [], artifacts: output.artifacts ?? [] }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: `Acceptance scenario failed safely: ${detail}` }; + stopped = true; + } + } + results.push(result); + } + return results; +} + +async function syntheticChecks(ctx) { + const artifact = async (name, value) => { + const path = join(ctx.run.root, "logs", `${name}.json`); + await writeJson(path, value); + return await fileArtifact(ctx.run.root, relative(ctx.run.root, path)); + }; + return CHECK_IDS.map((id, index) => ({ + id, + async run() { + return { + commands: [index === 0 ? "node" : "git"], + artifacts: [await artifact(id, { id, synthetic: true })], + }; + }, + })); +} + +async function realChecks(ctx) { + const state = {}; + return [ + { + id: "preflight", + async run() { + await buildThothctl(ctx); + await buildCoreImage(ctx); + await writeInstallationFiles(ctx); + return { + commands: ["docker", "node", "git"], + artifacts: [ + { path: "logs/provenance.json", sha256: sha256(JSON.stringify(ctx.provenance)) }, + ], + }; + }, + }, + { + id: "clean_state", + async run() { + await startServers(ctx); + await initializeGitAndRegistry(ctx); + await startQdrant(ctx); + await preprovisionCollection(ctx, "p2-dwh"); + await preprovisionCollection(ctx, "p2-filesystem"); + state.collectionsBefore = await listCollections(ctx); + state.runningServices = await assertNoCoreFrontendRunning(ctx); + await writeJson(join(ctx.run.root, "logs", "collections-before.json"), state.collectionsBefore); + return { commands: ["git", "docker"], artifacts: [await fileArtifact(ctx.run.root, "logs/collections-before.json")] }; + }, + }, + { + id: "ownership", + async run() { + await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); + const installStat = await stat(ctx.installationPath); + assert(installStat.isFile(), "installation descriptor missing"); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "ownership.json")] }; + }, + }, + { + id: "inspect_identity", + async run() { + const response = await runThothctlJson(ctx, "inspect-p2-dwh", ["workspace", "inspect", "--workspace", "p2-dwh"], 0); + const snapshot = await loadWorkspaceSnapshot(ctx, "p2-dwh"); + assert(response.payload.workspaceId === "p2-dwh", "inspect workspace id mismatch"); + assert(response.payload.workspaceRevision === snapshot.active.head, "inspect revision mismatch"); + assert(`sha256:${sha256(snapshot.contents)}` === response.payload.descriptorBlob, "inspect descriptor mismatch"); + state.inspect = response.payload; + return { commands: ["thothctl"], artifacts: response.artifacts }; + }, + }, + { + id: "dwh_processing", + async run() { + const first = await runThothctlJson(ctx, "preprocess-dwh-first", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh"], 0); + assert(first.payload.status === "succeeded" && first.payload.code === "ok", "dwh first run failed"); + const rerun = await runThothctlJson(ctx, "preprocess-dwh-rerun", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh"], 0); + const resume = await runThothctlJson(ctx, "preprocess-dwh-resume", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh", "--resume", first.payload.runId], 0); + assert(["unchanged", "succeeded"].includes(rerun.payload.status), "dwh rerun not idempotent"); + assert(["unchanged", "succeeded"].includes(resume.payload.status), "dwh resume failed"); + state.dwhRunId = first.payload.runId; + return { commands: ["thothctl"], artifacts: [...first.artifacts, ...rerun.artifacts, ...resume.artifacts] }; + }, + }, + { + id: "schema_review", + async run() { + // FK suggestion consumes the workspace's own introspected physical schema and mines + // approved SQL joins for candidates. + await runThothctlJson(ctx, "preprocess-dwh-filesystem", ["workspace", "preprocess", "dwh", "--workspace", "p2-filesystem"], 0); + const sqlPath = join(ctx.run.root, "fixtures", "p2-filesystem.sql"); + await atomicWrite(sqlPath, "SELECT v.id FROM dw.visits v JOIN dw.patients p ON v.patient_id = p.patient_id\n"); + const suggest = await runThothctlJson(ctx, "schema-suggest-filesystem", ["workspace", "schema", "suggest-fks", "--workspace", "p2-filesystem", "--from-sql", sqlPath], 3); + assert(suggest.payload.code === "manual_review_required", "suggest did not block"); + assert(typeof suggest.payload.suggestedFksYaml === "string" && suggest.payload.suggestedFksYaml.length > 0, "suggested FK YAML missing"); + const digest = suggest.payload.artifactIdentities?.[0]?.digest; + assert(/^sha256:[0-9a-f]{64}$/.test(digest ?? ""), "candidate digest missing"); + const candidatePath = join(ctx.run.root, "fixtures", "p2-filesystem.candidates.yaml"); + await atomicWrite(candidatePath, suggest.payload.suggestedFksYaml); + assert(`sha256:${sha256(suggest.payload.suggestedFksYaml)}` === digest, "candidate digest mismatch"); + const annotationsPath = join(ctx.run.root, "fixtures", "p2-filesystem.annotations.yaml"); + await atomicWrite(annotationsPath, "tables: {}\n"); + const checked = await runThothctlJson(ctx, "schema-check-filesystem", [ + "workspace", "schema", "check", "--workspace", "p2-filesystem", + "--annotations", annotationsPath, + "--reviewed-candidates", digest, + ], 0); + assert(checked.payload.status === "succeeded", "schema check failed"); + state.filesystemCandidateDigest = digest; + return { commands: ["thothctl"], artifacts: [...suggest.artifacts, ...checked.artifacts, await fileArtifact(ctx.run.root, "fixtures/p2-filesystem.candidates.yaml"), await fileArtifact(ctx.run.root, "fixtures/p2-filesystem.annotations.yaml"), await fileArtifact(ctx.run.root, "fixtures/p2-filesystem.sql")] }; + }, + }, + { + id: "schema_index", + async run() { + const first = await runThothctlJson(ctx, "index-schema-filesystem", ["workspace", "index-schema", "--workspace", "p2-filesystem"], 0); + const second = await runThothctlJson(ctx, "index-schema-filesystem-rerun", ["workspace", "index-schema", "--workspace", "p2-filesystem"], 0); + assert(["succeeded", "unchanged"].includes(first.payload.status), "index schema first failed"); + assert(["unchanged", "succeeded"].includes(second.payload.status), "index schema rerun failed"); + return { commands: ["thothctl"], artifacts: [...first.artifacts, ...second.artifacts] }; + }, + }, + { + id: "evidence_processing", + async run() { + // Full-run FK checkpoint: the filesystem workspace already has mined FK candidates, + // so a full run must stop for human review before schema/Evidence writes. + const full = await runThothctlJson(ctx, "preprocess-run-fs-blocked", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem"], 3); + assert(full.payload.code === "manual_review_required", "full run did not block for review"); + const digest = full.payload.artifactIdentities?.[0]?.digest; + assert(/^sha256:[0-9a-f]{64}$/.test(digest ?? ""), "full run digest missing"); + const reviewPath = join(ctx.run.root, "fixtures", "p2-filesystem.full-annotations.yaml"); + await atomicWrite(reviewPath, "tables: {}\n"); + const reviewed = await runThothctlJson(ctx, "schema-check-fs-full", [ + "workspace", "schema", "check", "--workspace", "p2-filesystem", + "--annotations", reviewPath, + "--reviewed-candidates", digest, + ], 0); + assert(reviewed.payload.status === "succeeded", "full-run review failed"); + // Resume continues through index-schema and stops at filesystem Evidence materialization. + const resumed = await runThothctlJson(ctx, "preprocess-run-fs-resume", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem", "--resume", full.payload.runId], 3); + assert(resumed.payload.code === "evidence_materialization_required", "filesystem evidence did not block after review"); + // HTTP Evidence on the p2-dwh workspace: dry-run, publish, unchanged rerun, mutation. + const dryRun = await runThothctlJson(ctx, "preprocess-evidence-dry-run", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh", "--dry-run"], 0); + const publish = await runThothctlJson(ctx, "preprocess-evidence-publish", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + const rerun = await runThothctlJson(ctx, "preprocess-evidence-rerun", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + ctx.evidenceState.content = "# P2 Evidence\n\nSecond generation.\n"; + const mutated = await runThothctlJson(ctx, "preprocess-evidence-mutated", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + state.fullRunId = full.payload.runId; + return { commands: ["thothctl"], artifacts: [ + ...full.artifacts, ...reviewed.artifacts, ...resumed.artifacts, ...dryRun.artifacts, + ...publish.artifacts, ...rerun.artifacts, ...mutated.artifacts, + ] }; + }, + }, + { + id: "negative_cases", + async run() { + const missing = await runThothctlJson(ctx, "negative-missing-workspace", ["workspace", "inspect", "--workspace", "missing-workspace"], 1); + const resumeMismatch = await runThothctlJson(ctx, "negative-resume-mismatch", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh", "--resume", "0".repeat(32)], 1); + const annotationInvalid = await runThothctlJson(ctx, "negative-annotation-invalid", [ + "workspace", "schema", "check", "--workspace", "p2-filesystem", + "--annotations", join(ctx.run.root, "fixtures", "p2-filesystem.annotations.yaml"), + "--reviewed-candidates", `sha256:${"0".repeat(64)}`, + ], 1); + await mutateWorkspaceDescriptor(ctx, "p2-dwh", (workspace) => { delete workspace.evidence; }, "Remove P2 Evidence"); + const noEvidence = await runThothctlJson(ctx, "negative-no-evidence-run", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + assert(["succeeded", "unchanged"].includes(noEvidence.payload.status), "no-evidence evidence did not skip"); + assert(Array.isArray(noEvidence.payload.warnings) && noEvidence.payload.warnings.length > 0, "no-evidence warning missing"); + const conflict = await runThothctlJson(ctx, "negative-revision-conflict", ["workspace", "preprocess", "run", "--workspace", "p2-dwh", "--resume", state.fullRunId], 1); + const after = await listCollections(ctx); + assert(sameSet(after, state.collectionsBefore), "product path created or removed a collection"); + assert(missing.payload.code === "workspace_not_activatable" || missing.payload.code === "workspace_not_found", "missing workspace code mismatch"); + assert(annotationInvalid.payload.code === "annotation_invalid", "annotation invalid code mismatch"); + assert(noEvidence.payload.warnings?.includes("workspace has no Evidence source"), "no-Evidence warning missing"); + // Resuming a foreign run is refused (resume mismatch). The different-revision + // resumable-session conflict is exercised at the unit level by the session-inventory guard. + assert(["preprocessing_conflict", "preprocessing_resume_mismatch"].includes(conflict.payload.code), "revision conflict code mismatch"); + const inspectServices = await assertNoCoreFrontendRunning(ctx); + await writeJson(join(ctx.run.root, "logs", "services-after.json"), inspectServices); + return { commands: ["thothctl", "docker"], artifacts: [ + ...missing.artifacts, ...resumeMismatch.artifacts, ...annotationInvalid.artifacts, + ...noEvidence.artifacts, ...conflict.artifacts, await fileArtifact(ctx.run.root, "logs/services-after.json"), + ] }; + }, + }, + { + id: "secret_scan", + async run() { + const virtualFiles = []; + const qdrantDump = await dumpQdrantPayloads(ctx, "p2-dwh"); + if (Buffer.byteLength(qdrantDump) <= MAX_SECRET_SCAN_VIRTUAL_BYTES) virtualFiles.push({ path: "virtual/qdrant-p2-dwh.json", bytes: qdrantDump }); + const findings = await scanSecrets({ + runRoot: ctx.run.root, + forbiddenValues: ctx.forbiddenValues, + virtualFiles, + expectedGitRepositories: ["remote.git", "author"], + }); + await writeJson(join(ctx.run.root, "logs", "secret-scan.json"), findings); + if (findings.length > 0) throw new Error(`secret scan found ${findings.length} leak(s)`); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "logs/secret-scan.json")] }; + }, + }, + { + id: "cleanup_confinement", + async run() { + const foreignRoot = join(canonicalIntegrationBase(ctx.repositoryRoot), `p2-${"f".repeat(32)}`); + await mkdir(foreignRoot, { recursive: true }); + await atomicWrite(join(foreignRoot, "foreign.txt"), "foreign"); + assert(readFileSync(join(foreignRoot, "foreign.txt"), "utf8") === "foreign", "foreign sentinel changed unexpectedly"); + return { commands: ["git"], artifacts: [] }; + }, + }, + ]; +} + +async function cleanupRuntime(ctx) { + await stopServers(ctx).catch(() => {}); + if (ctx.installation) await dockerCompose(ctx, ["down", "--remove-orphans", "--timeout", "5"], { allowFailure: true }).catch(() => {}); + if (ctx.coreImageTag) await runCommand({ executable: ctx.executables.dockerPath, argv: ["image", "rm", "-f", ctx.coreImageTag], env: ctx.execEnv, maxOutputBytes: MAX_STDIO_BYTES }).catch(() => {}); +} + +export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) { + const synthetic = env.P2_ACCEPTANCE_SYNTHETIC === "1"; + const failAt = env.P2_ACCEPTANCE_FAIL_AT; + const ctx = synthetic + ? { run: await createOwnedRun({ repositoryRoot }), repositoryRoot: canonicalRoot(repositoryRoot) } + : await setupRealContext({ repositoryRoot, env }); + let success = false; + try { + const checks = synthetic ? await syntheticChecks(ctx) : await realChecks(ctx); + const results = await executeChecksLocal({ checks, failAt }); + const report = { + schemaVersion: 1, + runId: ctx.run.runId, + startedAt: ctx.run.startedAt, + finishedAt: nowIso(), + command: "p2-acceptance integration --keep", + overall: deriveOverall(results), + checks: results, + }; + await writeReportFiles({ run: ctx.run, report }); + success = report.overall === "PASS"; + if (announce) await announce({ report, runRoot: ctx.run.root }); + return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) }; + } finally { + if (!synthetic) await cleanupRuntime(ctx).catch(() => {}); + } +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) { + throw new Error("usage: p2-acceptance.mjs integration [--keep]"); + } + const result = await runIntegration({ keep: argv.includes("--keep"), env }); + return result.exitCode; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { + const code = await main(); + process.exitCode = code; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +export { CHECK_IDS }; diff --git a/backend/scripts/p2-acceptance.test.mjs b/backend/scripts/p2-acceptance.test.mjs new file mode 100644 index 00000000..6a6a2eee --- /dev/null +++ b/backend/scripts/p2-acceptance.test.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_IDS, + canonicalIntegrationBase, + cleanupOwnedRun, + createOwnedRun, + readAndValidateOwnership, + runIntegration, + validateReport, + validateRunRoot, +} from "./p2-acceptance.mjs"; + +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p2-acceptance-repo-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p11-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true }); + return root; +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("run roots are only canonical direct p2 integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p2-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [ + base, + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(repositoryRoot, ".artifacts", "p1-integration", id), + join(repositoryRoot, ".artifacts", "p11-integration", id), + join(base, id, "nested"), + join(base, "foreign"), + ]) { + assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + } + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p2-${"A".repeat(32)}`), `p2-${"A".repeat(32)}`)); +}); + +test("cleanup refuses p1, p11, manual, sibling, and wrong-nonce roots", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + for (const bad of [ + join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`), + join(repositoryRoot, ".artifacts", "p11-integration", `p11-${"c".repeat(32)}`), + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(canonicalIntegrationBase(repositoryRoot), `p2-${"d".repeat(32)}`), + ]) { + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce })); + } + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) })); +}); + +test("cleanup removes exactly one owned p2 root", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const sibling = join(canonicalIntegrationBase(repositoryRoot), `p2-${"e".repeat(32)}`); + await mkdir(sibling); + await writeFile(join(sibling, "sentinel"), "foreign"); + await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await assert.rejects(readFile(join(run.root, "ownership.json"))); + assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign"); +}); + +function resultFor(id) { + return { + id, + status: "PASS", + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:01.000Z", + commands: ["node"], + artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }], + }; +} + +test("report validation requires exact p2 identity, check order, and unique artifacts", () => { + const report = { + schemaVersion: 1, + runId: `p2-${"f".repeat(32)}`, + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:10.000Z", + command: "p2-acceptance integration --keep", + overall: "PASS", + checks: CHECK_IDS.map(resultFor), + }; + assert.doesNotThrow(() => validateReport(report)); + const invalid = structuredClone(report); + invalid.runId = `p11-${"f".repeat(32)}`; + assert.throws(() => validateReport(invalid)); + const duplicate = structuredClone(report); + duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path; + assert.throws(() => validateReport(duplicate), /duplicated/); + const reordered = structuredClone(report); + reordered.checks.reverse(); + reordered.overall = "FAIL"; + assert.throws(() => validateReport(reordered)); +}); + +test("public wrapper uses a strict empty environment", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p2-acceptance.sh"), "utf8"); + assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/); + assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/); + assert.doesNotMatch(wrapper, /P2_ACCEPTANCE_FAIL_AT/); +}); + +test("synthetic integration cleans up successful non-kept runs", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: false, env: { P2_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, false); + await assert.rejects(readFile(join(result.runRoot, "ownership.json"))); +}); + +test("synthetic integration retains kept runs with bounded reports", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: true, env: { P2_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "PASS"); + const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8"); + assert.match(reportMd, /P2 automated integration: PASS/); + assert.match(reportMd, /P2 manual acceptance: PENDING/); + const reportJsonStat = await stat(join(result.runRoot, "report.json")); + const reportMdStat = await stat(join(result.runRoot, "report.md")); + assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`); + assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`); +}); + +test("synthetic injected failure retains the owned run and records a single failed report", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ + repositoryRoot, + keep: false, + env: { P2_ACCEPTANCE_SYNTHETIC: "1", P2_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] }, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "FAIL"); + const failed = report.checks.find((check) => check.id === CHECK_IDS[2]); + assert.equal(failed.status, "FAIL"); + const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8"); + assert.match(roots, /p2-acceptance/); +}); diff --git a/backend/scripts/p2p6-acceptance.mjs b/backend/scripts/p2p6-acceptance.mjs new file mode 100644 index 00000000..2f3b9f2d --- /dev/null +++ b/backend/scripts/p2p6-acceptance.mjs @@ -0,0 +1,1425 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from "node:crypto"; +import { execFile, execFileSync } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { createServer } from "node:http"; +import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, rmSync, statSync, symlinkSync } from "node:fs"; +import { access, lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import net from "node:net"; +import process from "node:process"; + +import { stringify as yamlStringify } from "yaml"; + +import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const RUN_ID = /^p2p6-[0-9a-f]{32}$/; +const HEX32 = /^[0-9a-f]{32}$/; +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const COMMAND = /^[a-z0-9][a-z0-9-]*$/; +const CHECK_RESULT_STATUS = new Set(["PASS", "FAIL"]); +const CHECK_IDS = Object.freeze([ + "preflight", + "clean_state", + "ownership", + "activation_materialization", + "dwh_chain", + "fk_schema_evidence_chain", + "revision_isolation", + "second_installation", + "unsafe_tree_refused", + "bound_refused", + "secret_scan", + "cleanup_confinement", +]); +const TOPOLOGY = [ + "remote.git", + "author", + "installation", + "installation/data", + "installation/data/sessions", + "installation/registry", + "installation/pi-state", + "fixture-secrets", + "fixtures", + "fixtures/logs", + "logs", +]; +const MAX_REPORT_JSON_BYTES = 64 * 1024; +const MAX_REPORT_MD_BYTES = 32 * 1024; +const MAX_STDIO_BYTES = 512 * 1024; +const MAX_SECRET_SCAN_VIRTUAL_BYTES = 256 * 1024; + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function assert(condition, message) { if (!condition) throw new Error(message); } +function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function canonicalRoot(repositoryRoot = defaultRepositoryRoot) { + return realpathSync(repositoryRoot); +} + +export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p2p6-integration"); +} + +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} + +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!existsSync(cursor)) break; + const entry = lstatSync(cursor); + if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink"); + } +} + +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + if (handle) await handle.close().catch(() => {}); + await rm(staging, { force: true }).catch(() => {}); + throw error; + } +} + +function initialResources(run) { + return [ + run.root, + join(run.root, "remote.git"), + join(run.root, "author"), + join(run.root, "installation"), + join(run.root, "installation", "registry"), + join(run.root, "installation", "data"), + join(run.root, "fixture-secrets"), + ]; +} + +function ownershipValue(run) { + return { + schemaVersion: 1, + kind: "p2p6-acceptance", + runId: run.runId, + runNonce: run.nonce, + root: run.root, + repositoryRoot: run.repositoryRoot, + startedAt: run.startedAt, + pid: run.pid, + resources: initialResources(run), + }; +} + +async function writeOwnership(run) { + await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run), null, 2)}\n`); +} + +export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) { + const repo = canonicalRoot(repositoryRoot); + const base = canonicalIntegrationBase(repo); + validateNoSymlinkAncestors(repo, base); + await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + const id = runId ?? `p2p6-${randomBytes(16).toString("hex")}`; + const root = validateRunRoot(repo, join(base, id), id); + const run = { + repositoryRoot: repo, + root, + runId: id, + nonce: nonce ?? randomBytes(32).toString("hex"), + startedAt: now ?? nowIso(), + pid: pid ?? process.pid, + }; + if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity"); + await mkdir(root, { mode: 0o700 }); + await writeOwnership(run); + return run; +} + +function strictOwnership(value, run, expectedNonce) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed"); + if (value.schemaVersion !== 1 || value.kind !== "p2p6-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce + || value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid + || !ISO_UTC.test(value.startedAt ?? "") + || JSON.stringify(value.resources) !== JSON.stringify(initialResources(run))) throw new Error("ownership identity mismatch"); + return value; +} + +export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const repo = canonicalRoot(repositoryRoot); + const id = basename(resolve(runRoot)); + const lexical = validateRunRoot(repo, runRoot, id); + const rootEntry = await lstat(lexical); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory"); + const ownershipPath = join(lexical, "ownership.json"); + const ownershipEntry = await lstat(ownershipPath); + if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe"); + let value; + try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); } + return strictOwnership(value, { repositoryRoot: repo, root: lexical, runId: id }, expectedNonce); +} + +export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }); + const base = canonicalIntegrationBase(repositoryRoot); + const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`); + await rename(runRoot, tombstone); + await rm(tombstone, { recursive: true, force: false }); +} + +async function finalizeOwnedRun({ run, success, keep }) { + if (!success || keep) return false; + await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + return true; +} + +function safeArtifactPath(path) { + if (typeof path !== "string" || path.length === 0 || path.length > 255 || path.startsWith("/") || path.includes("..") || path.includes("\\") || /[\0\r\n]/.test(path)) { + throw new Error("report artifact path is invalid"); + } + return path; +} + +function hasExactCheckIds(checks) { + return checks.length === CHECK_IDS.length && checks.every(({ id }, index) => id === CHECK_IDS[index]); +} + +export function validateReport(report) { + if (!report || report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "") + || !ISO_UTC.test(report.finishedAt ?? "") || typeof report.command !== "string" + || !Array.isArray(report.checks) || !hasExactCheckIds(report.checks)) throw new Error("report is invalid"); + const ids = new Set(); + const artifactPaths = new Set(); + for (const check of report.checks) { + if (!check || !/^[a-z0-9_]+$/.test(check.id ?? "") || ids.has(check.id) || !CHECK_RESULT_STATUS.has(check.status) + || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "") + || !Array.isArray(check.commands) || check.commands.some((name) => !COMMAND.test(name)) + || !Array.isArray(check.artifacts)) throw new Error("report check is invalid"); + ids.add(check.id); + for (const artifact of check.artifacts) { + safeArtifactPath(artifact.path); + if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report check is invalid"); + if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated"); + artifactPaths.add(artifact.path); + } + } + if (report.overall !== deriveOverall(report.checks)) throw new Error("report overall is not derived"); + return report; +} + +function renderReportMarkdown(report) { + validateReport(report); + const rows = report.checks.map((check) => `| ${check.id} | ${check.status} |`).join("\n"); + return [ + "# P2P6 acceptance report", + "", + `Run: \`${report.runId}\``, + "", + "| Check | Status |", + "|---|---|", + rows, + "", + `P2P6 automated integration: ${report.overall}`, + "P2P6 manual acceptance: PENDING", + "", + ].join("\n"); +} + +async function walkFiles(root) { + const files = []; + async function visit(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const rel = relative(root, path).split(sep).join("/"); + if (entry.isSymbolicLink()) throw new Error(`unsafe file tree: ${rel}`); + if (entry.isDirectory()) await visit(path); + else if (entry.isFile()) files.push({ path, rel }); + } + } + if (existsSync(root)) await visit(root); + files.sort((a, b) => a.rel.localeCompare(b.rel)); + return files; +} + +async function snapshotDigest(root, excludedPrefixes = []) { + const result = {}; + for (const file of await walkFiles(root)) { + if (excludedPrefixes.some((prefix) => file.rel === prefix || file.rel.startsWith(`${prefix}/`))) continue; + result[file.rel] = sha256(await readFile(file.path)); + } + return result; +} + +async function fileArtifact(root, relativePath) { + const bytes = await readFile(join(root, relativePath)); + return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) }; +} + +async function writeJson(path, value) { + await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function writeReportFiles({ run, report }) { + validateReport(report); + const reportJsonPath = join(run.root, "report.json"); + const reportMdPath = join(run.root, "report.md"); + const reportMd = renderReportMarkdown(report); + if (Buffer.byteLength(JSON.stringify(report)) > MAX_REPORT_JSON_BYTES) throw new Error("report.json exceeds bound"); + if (Buffer.byteLength(reportMd) > MAX_REPORT_MD_BYTES) throw new Error("report.md exceeds bound"); + await writeJson(reportJsonPath, report); + await atomicWrite(reportMdPath, reportMd, 0o600); + return { + reportJson: await fileArtifact(run.root, "report.json"), + reportMd: await fileArtifact(run.root, "report.md"), + }; +} + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (statSync(resolved).isFile()) return resolved; + } catch {} + } + throw new Error(`required executable not found: ${name}`); +} + +function scalarSecretBytes(value) { + if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid"); + return Buffer.from(value); +} + +async function manifestFiles(root, paths) { + const files = []; + const visit = async (absolute, rel) => { + const entry = await lstat(absolute); + if (entry.isSymbolicLink()) throw new Error(`provenance path is a symlink: ${rel}`); + if (entry.isDirectory()) { + for (const child of (await readdir(absolute, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + await visit(join(absolute, child.name), rel ? `${rel}/${child.name}` : child.name); + } + } else if (entry.isFile()) { + const bytes = await readFile(absolute); + files.push({ path: rel, bytes: bytes.length, sha256: sha256(bytes) }); + } else throw new Error(`provenance path is not a regular file: ${rel}`); + }; + for (const path of paths) await visit(join(root, path), path); + files.sort((a, b) => a.path.localeCompare(b.path)); + return { files, manifestSha256: sha256(JSON.stringify(files)) }; +} + +async function collectRepositoryProvenance({ repositoryRoot, gitPath = resolveSystemExecutable("git") }) { + const repo = canonicalRoot(repositoryRoot); + const safeEnv = buildSafeEnvironment({ ambient: {}, fixture: { PATH: `${dirname(gitPath)}:/usr/bin:/bin`, HOME: repo, TMPDIR: join(repo, ".artifacts") } }); + const run = async (argv) => await execFileAsync(gitPath, ["-C", repo, ...argv], { env: safeEnv, maxBuffer: MAX_STDIO_BYTES }); + const beforeHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const beforeTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const beforeStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (!HEX40.test(beforeHead) || !HEX40.test(beforeTree) || beforeStatus !== "") throw new Error("repository is not clean at exact HEAD"); + const backendRoot = join(repo, "backend"); + const backendSource = await manifestFiles(backendRoot, [ + "src", + "scripts/p2p6-acceptance.mjs", + "package.json", + "package-lock.json", + "tsconfig.json", + ]); + const backendDist = existsSync(join(backendRoot, "dist")) ? await manifestFiles(backendRoot, ["dist"]) : { files: [], manifestSha256: sha256("[]") }; + const afterHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const afterTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const afterStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (afterHead !== beforeHead || afterTree !== beforeTree || afterStatus !== beforeStatus) throw new Error("repository provenance changed during binding"); + return { schemaVersion: 1, head: beforeHead, tree: beforeTree, clean: true, backendSource, backendDist }; +} + +async function createTopology(run) { + for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); +} + +async function allocatePort() { + const server = net.createServer(); + await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).on("error", reject)); + const port = server.address().port; + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + return port; +} + +function installationProjectName(installationPath) { + return `thothii-${sha256(installationPath).slice(0, 12)}`; +} + +function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) { + return { + workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + diagnostics: { + dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}), + }; +} + +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } + +function descriptorYaml(obj) { + return yamlStringify(obj, { lineWidth: 0, sortMapEntries: false }); +} + +async function setupSecrets(ctx) { + const secretDir = join(ctx.run.root, "fixture-secrets"); + const values = { + dwhToken: `P2-DWH-${randomBytes(16).toString("hex")}`, + signedToken: `P2-SIGNED-${randomBytes(16).toString("hex")}`, + bundle: `P2-BUNDLE-${randomBytes(16).toString("hex")}`, + }; + ctx.forbiddenValues = Object.values(values); + ctx.secretValues = values; + const paths = { + dwh: join(secretDir, "p2-dwh-api-key"), + filesystemDwh: join(secretDir, "p2-filesystem-api-key"), + signed: join(secretDir, "p2-dwh-evidence-signed-urls.json"), + bundle: join(secretDir, "thothii.secrets"), + }; + await atomicWrite(paths.dwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.filesystemDwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.bundle, scalarSecretBytes(values.bundle)); + ctx.secretPaths = paths; +} + +async function setupFixtures(ctx) { + ctx.fixturePorts = { + dwh: await allocatePort(), + evidence: await allocatePort(), + embedding: await allocatePort(), + qdrant: await allocatePort(), + }; + const dwhBaseUrl = `http://host.docker.internal:${ctx.fixturePorts.dwh}`; + const evidenceProvenance = `http://host.docker.internal:${ctx.fixturePorts.evidence}/p2-dwh/guide.md`; + ctx.workspaceObjects = { + dwh: baseWorkspace("p2-dwh", { + dwhBaseUrl, + evidenceSource: { + type: "http", + uris: [evidenceProvenance], + authentication: "signed_urls_file", + connect_timeout_ms: 1250, + read_timeout_ms: 30001, + max_bytes: 65536, + max_redirects: 2, + allow_private_hosts: true, + max_cache_bytes: 65536, + }, + }), + filesystem: baseWorkspace("p2-filesystem", { + dwhBaseUrl, + evidenceSource: { + type: "filesystem", + uri: "p2-filesystem/evidence", + patterns: ["**/*.md"], + max_bytes: 1048576, + }, + }), + }; + const signedUrl = `${evidenceProvenance}?token=${ctx.secretValues.signedToken}`; + await atomicWrite(ctx.secretPaths.signed, `${JSON.stringify([signedUrl], null, 2)}\n`); + + ctx.curatedAnnotations = { + "p2-dwh": "tables: {}\n", + "p2-filesystem": "tables: {}\n", + }; + ctx.evidenceState = { + content: "# P2 Evidence\n\nFirst generation.\n", + token: ctx.secretValues.signedToken, + }; + ctx.dwhState = { + tables: { + patients: { + comment: "Patients", + rows: [ + { patient_id: "p1", name: "Alice" }, + { patient_id: "p2", name: "Bob" }, + ], + }, + visits: { + comment: "Visits", + rows: [ + { id: "v1", patient_id: "p1", note: "checkup" }, + { id: "v2", patient_id: "p2", note: "xray" }, + ], + }, + labs: { + comment: "Labs", + rows: [ + { id: "l1", patient_id: "p1", code: "hemoglobin" }, + { id: "l2", patient_id: "p2", code: "glucose" }, + ], + }, + }, + token: ctx.secretValues.dwhToken, + }; +} + +function inferColumnType(value) { + return typeof value === "number" ? "integer" : "text"; +} + +function topValues(rows, column, limit) { + const counts = new Map(); + for (const row of rows) { + const value = row[column]; + if (value === undefined || value === null || value === "") continue; + counts.set(String(value), (counts.get(String(value)) ?? 0) + 1); + } + return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, limit).map(([value]) => ({ value })); +} + +async function startHttpServer({ port, handler }) { + const server = createServer(async (req, res) => { + try { + await handler(req, res); + } catch { + res.statusCode = 500; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "fixture failed" })); + } + }); + await new Promise((resolve, reject) => server.listen(port, "127.0.0.1", () => resolve()).on("error", reject)); + return server; +} + +async function startServers(ctx) { + const dwhServer = await startHttpServer({ + port: ctx.fixturePorts.dwh, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const json = body.length === 0 ? {} : JSON.parse(body); + if (req.headers["x-api-key"] !== ctx.dwhState.token) { + res.statusCode = 401; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ message: "unauthorized" })); + return; + } + const send = (payload) => { + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(payload)); + }; + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.dwh}`); + if (req.method !== "POST" || !url.pathname.startsWith("/rpc/")) { + res.statusCode = 404; + res.end(JSON.stringify({ message: "not found" })); + return; + } + const fn = url.pathname.slice("/rpc/".length); + const schemaName = json.schema_name ?? "dw"; + if (schemaName !== "dw") { + send([]); + return; + } + if (fn === "ping") { + send({ db_connected: true, schema_accessible: true, database: "warehouse", schema: "dw" }); + return; + } + const table = typeof json.table_name === "string" ? json.table_name : ""; + const tableData = ctx.dwhState.tables[table]; + if (fn === "list_tables") { + send(Object.entries(ctx.dwhState.tables).map(([name, info]) => ({ table: name, type: "TABLE", comment: info.comment, rows: info.rows.length }))); + return; + } + if (!tableData) { + send([]); + return; + } + if (fn === "table_columns") { + const first = tableData.rows[0] ?? {}; + send(Object.keys(first).map((column) => ({ + column, + type: inferColumnType(first[column]), + nullable: false, + // Only the referenced table marks `patient_id` as primary, so the SQL miner sees a + // PK/non-PK pair while the same-name heuristic still discovers joins from the others. + pk: column === "id" || (table === "patients" && column === "patient_id"), + default: null, + }))); + return; + } + if (fn === "table_comments") { + send(Object.keys(tableData.rows[0] ?? {}).map((column) => ({ object: "COLUMN", name: column, comment: `${table}.${column}` }))); + return; + } + if (fn === "table_foreign_keys") { + send([]); + return; + } + if (fn === "top_values") { + send(topValues(tableData.rows, json.column_name, Number(json.max_values ?? 10))); + return; + } + if (fn === "column_stats") { + send({}); + return; + } + if (fn === "run_query") { + send([]); + return; + } + if (fn === "explain_query") { + send([{ line: "Seq Scan" }]); + return; + } + res.statusCode = 404; + res.end(JSON.stringify({ message: "unknown rpc" })); + }, + }); + const evidenceServer = await startHttpServer({ + port: ctx.fixturePorts.evidence, + handler: async (req, res) => { + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.evidence}`); + if (url.pathname !== "/p2-dwh/guide.md" || url.searchParams.get("token") !== ctx.evidenceState.token) { + res.statusCode = 403; + res.end("forbidden"); + return; + } + res.statusCode = 200; + res.setHeader("content-type", "text/markdown; charset=utf-8"); + res.end(ctx.evidenceState.content); + }, + }); + const embeddingServer = await startHttpServer({ + port: ctx.fixturePorts.embedding, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.embedding}`); + if (req.method !== "POST" || url.pathname !== "/api/embed") { + res.statusCode = 404; + res.end(JSON.stringify({ error: "not found" })); + return; + } + const payload = JSON.parse(body || "{}"); + const inputs = Array.isArray(payload.input) ? payload.input : []; + const embeddings = inputs.map((text) => { + const seed = sha256(String(text)); + return Array.from({ length: 1024 }, (_, index) => { + const offset = (index * 2) % seed.length; + const value = Number.parseInt(seed.slice(offset, offset + 2), 16); + return (value / 255) - 0.5; + }); + }); + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ model: payload.model, embeddings })); + }, + }); + ctx.servers = [dwhServer, evidenceServer, embeddingServer]; +} + +async function stopServers(ctx) { + for (const server of ctx.servers ?? []) { + await new Promise((resolve) => server.close(() => resolve())); + } + ctx.servers = []; +} + +async function git(ctx, args, cwd = join(ctx.run.root, "author")) { + return await runCommand({ executable: ctx.executables.gitPath, argv: args, cwd, env: ctx.execEnv }); +} + +async function initializeGitAndRegistry(ctx) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], ctx.run.root); + await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], ctx.run.root); + await git(ctx, ["config", "user.name", "P2 Fixture Curator"], author); + await git(ctx, ["config", "user.email", "p2p6-curator@example.invalid"], author); + + const writeWorkspaces = async () => { + const catalog = { + schema_version: 1, + workspaces: [ + { id: "p2-dwh", name: ctx.workspaceObjects.dwh.workspace.name }, + { id: "p2-filesystem", name: ctx.workspaceObjects.filesystem.workspace.name }, + ], + }; + await writeFile(join(author, "thoth-workspaces.yaml"), yamlStringify(catalog, { lineWidth: 0, sortMapEntries: false })); + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId), { recursive: true }); + const yaml = descriptorYaml(workspace); + await writeFile(join(author, pathId, "workspace.yaml"), yaml); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + await mkdir(join(author, "workspace-docs", pathId), { recursive: true }); + await writeFile(join(author, "workspace-docs", pathId, "contract.env.example"), docs.envExample); + await writeFile(join(author, "workspace-docs", pathId, "README.md"), docs.markdown); + } + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId, "schema"), { recursive: true }); + await writeFile(join(author, pathId, "schema", "annotations.yaml"), ctx.curatedAnnotations[pathId]); + } + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "evidence", "guide.md"), "# P2 Filesystem Evidence\n\nCommitted fixture.\n"); + }; + + await writeWorkspaces(); + await git(ctx, ["add", "."], author); + await git(ctx, ["commit", "-m", "Bootstrap P2 fixtures"], author); + await git(ctx, ["push", "origin", "main"], author); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); + const registry = new ctx.workspaceModules.WorkspaceRegistry({ + root: join(ctx.run.root, "installation", "registry"), + remoteUrl: join(ctx.run.root, "remote.git"), + branch: "main", + gitAuthorName: "P2 Acceptance", + gitAuthorEmail: "p2p6-acceptance@example.invalid", + installationId: "p2p6-acceptance", + secretRoots: [join(ctx.run.root, "fixture-secrets")], + maxImportBytes: 16 * 1024 * 1024, + maxImportEntries: 1024, + dataRoot: join(ctx.run.root, "installation", "data"), + }); + await registry.bootstrap(); + ctx.registry = registry; +} + +async function mutateWorkspaceDescriptor(ctx, workspaceId, mutator, commitMessage) { + const author = join(ctx.run.root, "author"); + // The registry may have produced docs-only follow-up commits on the remote; the curator + // always rebases onto the latest remote head before committing so the push stays fast-forward. + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const workspace = structuredClone(ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"]); + mutator(workspace); + ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"] = workspace; + await writeFile(join(author, workspaceId, "workspace.yaml"), descriptorYaml(workspace)); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + const docsDir = join(author, "workspace-docs", workspaceId); + await mkdir(docsDir, { recursive: true, mode: 0o700 }); + await writeFile(join(docsDir, "contract.env.example"), docs.envExample); + await writeFile(join(docsDir, "README.md"), docs.markdown); + await git(ctx, ["add", `${workspaceId}/workspace.yaml`, `workspace-docs/${workspaceId}/contract.env.example`, `workspace-docs/${workspaceId}/README.md`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + + +async function mutateWorkspaceAnnotations(ctx, workspaceId, contents, commitMessage) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const annotationsPath = join(author, workspaceId, "schema", "annotations.yaml"); + await mkdir(dirname(annotationsPath), { recursive: true }); + await writeFile(annotationsPath, contents); + ctx.curatedAnnotations[workspaceId] = contents; + await git(ctx, ["add", `${workspaceId}/schema/annotations.yaml`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + +async function writeEvidenceAndPush(ctx, workspaceId, files, commitMessage) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const evidenceDir = join(author, workspaceId, "evidence"); + await rm(evidenceDir, { recursive: true, force: true }); + await mkdir(evidenceDir, { recursive: true }); + for (const [name, contents] of Object.entries(files)) { + const target = join(evidenceDir, name); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, contents); + } + await git(ctx, ["add", `${workspaceId}/evidence`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + return (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + +async function writeInstallationFiles(ctx) { + const installationDir = join(ctx.run.root, "installation"); + const operatorEnvPath = join(installationDir, "operator.env"); + const bindingsEnvPath = join(installationDir, "workspace-bindings.env"); + const connectorOverridePath = join(installationDir, "connector-secrets.override.yaml"); + const fixtureOverridePath = join(installationDir, "fixture.override.yaml"); + const installationPath = join(installationDir, "thothii-installation.yaml"); + ctx.installationPath = installationPath; + ctx.composeProject = installationProjectName(installationPath); + const qdrantPort = ctx.fixturePorts.qdrant; + const bindings = [ + `THT_WS_P2_DWH_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_DWH_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_DWH_DWH_API_KEY_FILE=/run/secrets/p2-dwh-api-key`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_FILE=/run/secrets/p2-dwh-evidence-signed-urls`, + `THT_WS_P2_FILESYSTEM_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_FILESYSTEM_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_FILE=/run/secrets/p2-filesystem-api-key`, + ].join("\n") + "\n"; + await atomicWrite(bindingsEnvPath, bindings); + const operatorEnv = [ + `THT_DATA_ROOT=${join(ctx.run.root, "installation", "data")}`, + `THT_WORKSPACE_REGISTRY_ROOT=${join(ctx.run.root, "installation", "registry")}`, + `THT_PI_STATE_ROOT=${join(ctx.run.root, "installation", "pi-state")}`, + `PI_AUTH_FILE=${join(ctx.run.root, "installation", "pi-auth.json")}`, + `THT_SECRETS_FILE=${ctx.secretPaths.bundle}`, + `THT_WORKSPACE_BINDINGS_ENV_FILE=${bindingsEnvPath}`, + `THT_WORKSPACE_GIT_REMOTE=${join(ctx.run.root, "remote.git")}`, + `THT_WORKSPACE_GIT_BRANCH=main`, + `THT_WORKSPACE_GIT_AUTHOR_NAME=P2 Acceptance`, + `THT_WORKSPACE_GIT_AUTHOR_EMAIL=p2p6-acceptance@example.invalid`, + `THT_WORKSPACE_INSTALLATION_ID=p2p6-acceptance`, + `THT_DB_NAME=warehouse`, + `THT_DWH_REST_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_LLM_URL=http://127.0.0.1:9`, + `THOTH_SERVER_BIND=127.0.0.1`, + `THOTH_HTTP_PORT=18080`, + `THOTH_CORE_HTTP_PORT=18787`, + `THT_WS_P2_DWH_DWH_API_KEY_SOURCE=${ctx.secretPaths.dwh}`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_SOURCE=${ctx.secretPaths.signed}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_SOURCE=${ctx.secretPaths.filesystemDwh}`, + `THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST=host.docker.internal`, + ].join("\n") + "\n"; + await atomicWrite(operatorEnvPath, operatorEnv); + await atomicWrite(join(ctx.run.root, "installation", "pi-auth.json"), JSON.stringify({ fixture: true })); + const embeddingStubPath = join(installationDir, "embedding-stub.py"); + await atomicWrite(embeddingStubPath, EMBEDDING_STUB_SOURCE); + const override = { + services: { + core: { + image: ctx.coreImageTag, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + "workspace-maintenance": { + image: ctx.coreImageTag, + environment: { + THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST: "host.docker.internal", + }, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + qdrant: { + ports: [`127.0.0.1:${qdrantPort}:6333`], + restart: "no", + }, + // Deterministic Ollama-compatible embedding fixture on the internal allowlisted host + // name `embedding` (http://embedding:11434). Replaces the real Ollama service entirely. + embedding: { + image: ctx.coreImageTag, + entrypoint: ["python3", "/stub.py"], + volumes: [ + { type: "bind", source: embeddingStubPath, target: "/stub.py", read_only: true }, + ], + healthcheck: { disable: true }, + }, + }, + }; + await atomicWrite(fixtureOverridePath, yamlStringify(override, { lineWidth: 0, sortMapEntries: false })); + const generated = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "generate-connector-secrets-override.sh"), + argv: [ + "--bindings-env", bindingsEnvPath, + "--operator-env", operatorEnvPath, + "--output", connectorOverridePath, + "--service", "workspace-maintenance", + "--role", "all", + ], + env: ctx.execEnv, + }); + if (generated.exitCode !== 0) throw new Error(`connector override generation failed: ${generated.stderr || generated.stdout}`); + const installation = { + profile: "server", + projectDirectory: ctx.repositoryRoot, + envFile: operatorEnvPath, + overrides: [ + join(ctx.repositoryRoot, "deploy", "compose.server.yaml"), + fixtureOverridePath, + connectorOverridePath, + ], + }; + await atomicWrite(installationPath, yamlStringify(installation, { lineWidth: 0, sortMapEntries: false })); + ctx.installation = installation; +} + +function thothctlBinaryPath(repositoryRoot) { + const platform = { darwin: "darwin", linux: "linux", win32: "windows" }[process.platform] ?? "linux"; + const arch = { x64: "amd64", arm64: "arm64" }[process.arch] ?? "amd64"; + const suffix = platform === "windows" ? ".exe" : ""; + const candidates = [ + join(repositoryRoot, "dist", "thothctl", `thothctl-${platform}-${arch}${suffix}`), + join(repositoryRoot, "tools", "thothctl", "bin", `thothctl${suffix}`), + ]; + for (const candidate of candidates) if (existsSync(candidate)) return candidate; + throw new Error("built thothctl binary is unavailable"); +} + +async function runCommand({ executable, argv = [], cwd, env, input, maxOutputBytes = MAX_STDIO_BYTES }) { + const result = await execFileAsync(executable, argv, { + cwd, + env, + encoding: "utf8", + maxBuffer: maxOutputBytes, + ...(input === undefined ? {} : { input }), + }).then( + ({ stdout, stderr }) => ({ exitCode: 0, stdout, stderr }), + (error) => ({ exitCode: error.code ?? 1, stdout: error.stdout ?? "", stderr: error.stderr ?? error.message ?? "" }), + ); + return result; +} + +async function buildCoreImage(ctx) { + const tag = `thothii-core:p2-${ctx.run.runId.slice(3, 15)}`; + ctx.coreImageTag = tag; + const build = await runCommand({ + executable: ctx.executables.dockerPath, + argv: ["build", "-f", join(ctx.repositoryRoot, "docker", "core.Dockerfile"), "-t", tag, ctx.repositoryRoot], + env: { ...ctx.execEnv, DOCKER_BUILDKIT: "1" }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (build.exitCode !== 0) throw new Error(`core image build failed: ${build.stderr || build.stdout}`); +} + +async function buildThothctl(ctx) { + const command = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "build-thothctl.sh"), + argv: [], + env: { ...ctx.execEnv, THT_THOTHCTL_OUTPUT_DIRECTORY: join(ctx.repositoryRoot, "dist", "thothctl") }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (command.exitCode !== 0) throw new Error(`build-thothctl failed: ${command.stderr || command.stdout}`); + ctx.thothctlPath = thothctlBinaryPath(ctx.repositoryRoot); +} + +function composeBaseArgs(ctx) { + const args = [ + "compose", + "--project-name", ctx.composeProject, + "--project-directory", ctx.installation.projectDirectory, + "--env-file", ctx.installation.envFile, + "-f", join(ctx.repositoryRoot, "compose.yaml"), + ]; + for (const override of ctx.installation.overrides) args.push("-f", override); + return args; +} + +async function dockerCompose(ctx, commandArgs, { allowFailure = false, maxOutputBytes = 2 * 1024 * 1024 } = {}) { + const result = await runCommand({ + executable: ctx.executables.dockerPath, + argv: [...composeBaseArgs(ctx), ...commandArgs], + env: ctx.execEnv, + maxOutputBytes, + }); + if (!allowFailure && result.exitCode !== 0) throw new Error(`docker compose ${commandArgs.join(" ")} failed: ${result.stderr || result.stdout}`); + return result; +} + +async function startQdrant(ctx) { + await dockerCompose(ctx, ["up", "-d", "qdrant", "embedding"]); + for (let attempt = 0; attempt < 60; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}/collections`); + if (response.ok) return; + } catch {} + await sleep(1000); + } + throw new Error("qdrant did not become ready"); +} + +async function qdrantJson(ctx, method, path, body) { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}${path}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const payload = response.status === 204 ? {} : await response.json().catch(() => ({})); + if (!response.ok) throw new Error(`qdrant request failed: ${method} ${path} ${response.status}`); + return payload; +} + +async function preprovisionCollection(ctx, workspaceId) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}`, { + vectors: { size: 1024, distance: "Cosine" }, + }); + for (const field of ["content_hash", "document_id", "kind", "record_key", "record_kind", "vector_generation", "workspace_id", "workspace_revision"]) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}/index`, { field_name: field, field_schema: "keyword" }); + } +} + +async function listCollections(ctx) { + const payload = await qdrantJson(ctx, "GET", "/collections"); + const collections = payload.result?.collections ?? []; + return collections.map((item) => item.name).sort(); +} + +async function dumpQdrantPayloads(ctx, workspaceId) { + const response = await qdrantJson(ctx, "POST", `/collections/${workspaceId}/points/scroll`, { limit: 128, with_payload: true, with_vector: false }); + return JSON.stringify(response.result?.points ?? []); +} + +async function runThothctlJson(ctx, label, workspaceArgs, expectedExitCode) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.json`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs, "--json"], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + if (expectedExitCode !== undefined && result.exitCode !== expectedExitCode) { + throw new Error(`${label} exit ${result.exitCode} != ${expectedExitCode}`); + } + let payload; + try { payload = JSON.parse(result.stdout); } catch (error) { throw new Error(`${label} returned non-JSON stdout`); } + return { result, payload, artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath))] }; +} + +async function runThothctlRaw(ctx, label, workspaceArgs) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.txt`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + return { + result, + artifacts: [ + await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), + await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath)), + ], + }; +} + +async function loadWorkspaceSnapshot(ctx, workspaceId) { + const active = JSON.parse(await readFile(join(ctx.run.root, "installation", "registry", "state", "active.json"), "utf8")); + const revision = active.revisions.find((entry) => entry.id === workspaceId); + const snapshotPath = revision.snapshotPath; + const contents = await readFile(snapshotPath, "utf8"); + return { active, revision, contents }; +} + +async function assertNoCoreFrontendRunning(ctx) { + const ps = await dockerCompose(ctx, ["ps", "--status", "running", "--format", "json"], { allowFailure: true }); + if (ps.exitCode !== 0) return []; + const lines = ps.stdout.trim() === "" ? [] : ps.stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); + const services = lines.map((item) => item.Service); + if (services.includes("core") || services.includes("frontend") || services.includes("workspace-maintenance")) { + throw new Error("core/frontend/maintenance is unexpectedly running"); + } + return services; +} + +function sameSet(left, right) { + return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort()); +} + +const EMBEDDING_STUB_SOURCE = String.raw`import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +class _Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(length)) + inputs = payload.get("input", []) + if isinstance(inputs, str): + inputs = [inputs] + embeddings = [[0.01] * 1024 for _ in inputs] + body = json.dumps({"model": payload.get("model", "qwen3-embedding:0.6b"), "embeddings": embeddings}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + +HTTPServer(("0.0.0.0", 11434), _Handler).serve_forever() +`; + +function realUserHome() { + try { + const output = execFileSync("bash", ["-lc", 'printf "%s" ~'], { encoding: "utf8" }).trim(); + return output.length > 0 ? output : undefined; + } catch { + return undefined; + } +} + +async function setupRealContext({ repositoryRoot = defaultRepositoryRoot, env = process.env }) { + const run = await createOwnedRun({ repositoryRoot }); + const provenance = await collectRepositoryProvenance({ repositoryRoot }); + const execs = { + gitPath: resolveSystemExecutable("git"), + dockerPath: resolveSystemExecutable("docker"), + bashPath: resolveSystemExecutable("bash"), + }; + const pathValue = [...new Set([dirname(execs.gitPath), dirname(execs.dockerPath), "/usr/bin", "/bin", "/opt/homebrew/bin", "/usr/local/bin"])].join(":"); + // Docker CLI plugins (buildx) live under the real user's ~/.docker; the wrapper runs with a + // scrubbed environment, so derive the real home from the passwd entry and expose DOCKER_CONFIG. + const realHome = env.P2P6_REAL_HOME ?? realUserHome(); + const execEnv = buildSafeEnvironment({ ambient: env, fixture: { + PATH: pathValue, + HOME: run.root, + TMPDIR: join(run.root, "tmp"), + ...(realHome ? { DOCKER_CONFIG: join(realHome, ".docker") } : {}), + } }); + const workspaceModules = await import("../dist/workspaces/registry.js").then(async (registryModule) => ({ + WorkspaceRegistry: registryModule.WorkspaceRegistry, + ...(await import("../dist/workspaces/schema.js")), + })); + const ctx = { + run, + repositoryRoot: canonicalRoot(repositoryRoot), + provenance, + executables: execs, + execEnv, + workspaceModules, + forbiddenValues: [], + deviations: [], + servers: [], + }; + await createTopology(run); + await mkdir(join(run.root, "tmp"), { recursive: true, mode: 0o700 }); + await setupSecrets(ctx); + await setupFixtures(ctx); + return ctx; +} + +async function executeChecksLocal({ checks, failAt } = {}) { + if (!Array.isArray(checks) || !hasExactCheckIds(checks)) throw new Error("scenarios must match the exact ordered check set"); + if (failAt !== undefined && !CHECK_IDS.includes(failAt)) throw new Error("failure hook must name an exact check"); + const results = []; + let stopped = false; + for (const scenario of checks) { + const startedAt = nowIso(); + let result; + if (stopped) { + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: "Not executed after earlier failure." }; + } else { + try { + const output = await scenario.run(); + if (scenario.id === failAt) throw new Error("injected acceptance failure"); + result = { id: scenario.id, status: "PASS", startedAt, finishedAt: nowIso(), commands: output.commands ?? [], artifacts: output.artifacts ?? [] }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: `Acceptance scenario failed safely: ${detail}` }; + stopped = true; + } + } + results.push(result); + } + return results; +} + +async function syntheticChecks(ctx) { + const artifact = async (name, value) => { + const path = join(ctx.run.root, "logs", `${name}.json`); + await writeJson(path, value); + return await fileArtifact(ctx.run.root, relative(ctx.run.root, path)); + }; + return CHECK_IDS.map((id, index) => ({ + id, + async run() { + return { + commands: [index === 0 ? "node" : "git"], + artifacts: [await artifact(id, { id, synthetic: true })], + }; + }, + })); +} + +async function realChecks(ctx) { + const state = {}; + const evidenceRoot = (workspaceId, commit) => join(ctx.run.root, "installation", "registry", "snapshots", commit, workspaceId, "evidence"); + const evidenceManifestPath = (workspaceId, commit) => join(ctx.run.root, "installation", "registry", "snapshots", commit, workspaceId, "evidence.manifest.json"); + const activeCommit = async (workspaceId) => (await loadWorkspaceSnapshot(ctx, workspaceId)).revision.commit; + const fileArtifactFrom = (path) => fileArtifact(ctx.run.root, relative(ctx.run.root, path)); + + return [ + { + id: "preflight", + async run() { + await buildThothctl(ctx); + await buildCoreImage(ctx); + await writeInstallationFiles(ctx); + return { + commands: ["docker", "node", "git"], + artifacts: [{ path: "logs/provenance.json", sha256: sha256(JSON.stringify(ctx.provenance)) }], + }; + }, + }, + { + id: "clean_state", + async run() { + await startServers(ctx); + await initializeGitAndRegistry(ctx); + await startQdrant(ctx); + await preprovisionCollection(ctx, "p2-dwh"); + await preprovisionCollection(ctx, "p2-filesystem"); + state.collectionsBefore = await listCollections(ctx); + state.runningServices = await assertNoCoreFrontendRunning(ctx); + state.initialCommit = await activeCommit("p2-filesystem"); + await writeJson(join(ctx.run.root, "logs", "collections-before.json"), state.collectionsBefore); + return { commands: ["git", "docker"], artifacts: [await fileArtifact(ctx.run.root, "logs/collections-before.json")] }; + }, + }, + { + id: "ownership", + async run() { + await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); + const installStat = await stat(ctx.installationPath); + assert(installStat.isFile(), "installation descriptor missing"); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "ownership.json")] }; + }, + }, + { + id: "activation_materialization", + async run() { + const commit = state.initialCommit; + const guide = join(evidenceRoot("p2-filesystem", commit), "guide.md"); + assert(readFileSync(guide, "utf8") === "# P2 Filesystem Evidence\n\nCommitted fixture.\n", "materialized Evidence content mismatch"); + const manifest = JSON.parse(readFileSync(evidenceManifestPath("p2-filesystem", commit), "utf8")); + assert(manifest.workspace === "p2-filesystem" && manifest.commit === commit && manifest.entryCount === 1, "Evidence manifest mismatch"); + const snapshot = JSON.parse(readFileSync(join(ctx.run.root, "installation", "registry", "snapshots", commit, "snapshot.json"), "utf8")); + assert(/^[0-9a-f]{64}$/.test(snapshot.files["p2-filesystem/evidence.manifest.json"] ?? ""), "snapshot manifest lacks Evidence manifest digest"); + return { commands: [], artifacts: [await fileArtifactFrom(guide), await fileArtifactFrom(evidenceManifestPath("p2-filesystem", commit))] }; + }, + }, + { + id: "dwh_chain", + async run() { + const first = await runThothctlJson(ctx, "dwh-first", ["workspace", "preprocess", "dwh", "--workspace", "p2-filesystem"], 0); + assert(first.payload.status === "succeeded" && first.payload.code === "ok", "DWH first run failed"); + const rerun = await runThothctlJson(ctx, "dwh-rerun", ["workspace", "preprocess", "dwh", "--workspace", "p2-filesystem"], 0); + assert(["unchanged", "succeeded"].includes(rerun.payload.status), "DWH rerun not idempotent"); + const resume = await runThothctlJson(ctx, "dwh-resume", ["workspace", "preprocess", "dwh", "--workspace", "p2-filesystem", "--resume", first.payload.runId], 0); + assert(["unchanged", "succeeded"].includes(resume.payload.status), "DWH resume failed"); + return { commands: ["thothctl"], artifacts: [...first.artifacts, ...rerun.artifacts, ...resume.artifacts] }; + }, + }, + { + id: "fk_schema_evidence_chain", + async run() { + // A fresh full run introspects the DWH, mines FK candidates, and blocks for review (P5). + const full = await runThothctlJson(ctx, "run-blocked", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem"], 3); + assert(full.payload.code === "manual_review_required", `full run did not block: ${full.payload.code}`); + state.runId = full.payload.runId; + assert(/^[0-9a-f]{32}$/.test(state.runId ?? ""), "run id missing"); + + // The curated empty annotation set (no approved FKs) is the human review; accept it (P5). + const accepted = await runThothctlJson(ctx, "schema-accept", ["workspace", "schema", "accept", "--workspace", "p2-filesystem", "--run", state.runId, "--yes"], 0); + assert(accepted.payload.status === "succeeded" && accepted.payload.code === "ok", "schema accept failed"); + + // Resume completes schema indexing and the now-materialized filesystem Evidence (P6). + const resumed = await runThothctlJson(ctx, "run-resume", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem", "--resume", state.runId], 0); + assert(resumed.payload.status === "succeeded" && resumed.payload.code === "ok", `full run did not succeed: ${resumed.payload.code}`); + const rerun = await runThothctlJson(ctx, "run-resume-again", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem", "--resume", state.runId], 0); + assert(["unchanged", "succeeded"].includes(rerun.payload.status), "full run rerun not idempotent"); + return { commands: ["thothctl"], artifacts: [...full.artifacts, ...accepted.artifacts, ...resumed.artifacts, ...rerun.artifacts] }; + }, + }, + { + id: "revision_isolation", + async run() { + await writeEvidenceAndPush(ctx, "p2-filesystem", { "guide.md": "# P2 Filesystem Evidence\n\nSecond generation.\n" }, "Evidence content v2"); + await ctx.registry.pull(); + const revisionB = await activeCommit("p2-filesystem"); + assert(revisionB !== state.initialCommit, "Evidence commit did not change the revision"); + assert(readFileSync(join(evidenceRoot("p2-filesystem", revisionB), "guide.md"), "utf8").includes("Second generation"), "revision B Evidence not materialized"); + state.revisionB = revisionB; + + // A run pinned to the old revision must not be silently resumed after the revision change. + const stale = await runThothctlJson(ctx, "stale-resume", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem", "--resume", state.runId], 1); + assert(["preprocessing_resume_mismatch", "preprocessing_conflict"].includes(stale.payload.code), `stale resume code mismatch: ${stale.payload.code}`); + return { commands: ["thothctl", "git"], artifacts: [...stale.artifacts, await fileArtifactFrom(join(evidenceRoot("p2-filesystem", revisionB), "guide.md"))] }; + }, + }, + { + id: "second_installation", + async run() { + const secondRoot = join(ctx.run.root, "installation2", "registry"); + await mkdir(secondRoot, { recursive: true, mode: 0o700 }); + const second = new ctx.workspaceModules.WorkspaceRegistry({ + root: secondRoot, + remoteUrl: join(ctx.run.root, "remote.git"), + branch: "main", + gitAuthorName: "P2P6 Acceptance", + gitAuthorEmail: "p2p6-acceptance@example.invalid", + installationId: "p2p6-acceptance-second", + secretRoots: [join(ctx.run.root, "fixture-secrets")], + maxImportBytes: 16 * 1024 * 1024, + maxImportEntries: 1024, + }); + await second.bootstrap(); + const active = await second.read("p2-filesystem"); + assert(active.workspace.workspace.id === "p2-filesystem", "second installation read failed"); + const secondGuide = join(secondRoot, "snapshots", state.revisionB, "p2-filesystem", "evidence", "guide.md"); + assert(readFileSync(secondGuide, "utf8").includes("Second generation"), "second installation did not materialize its own Evidence"); + return { commands: ["git"], artifacts: [await fileArtifactFrom(secondGuide)] }; + }, + }, + { + id: "unsafe_tree_refused", + async run() { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + await rm(join(author, "p2-filesystem", "evidence"), { recursive: true, force: true }); + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "outside.md"), "# outside\n"); + symlinkSync("../outside.md", join(author, "p2-filesystem", "evidence", "link.md")); + await git(ctx, ["add", "p2-filesystem/evidence", "p2-filesystem/outside.md"], author); + await git(ctx, ["commit", "-m", "Unsafe Evidence symlink"], author); + await git(ctx, ["push", "origin", "main"], author); + let refused = false; + try { await ctx.registry.pull(); } catch (error) { refused = String(error?.code ?? error) === "workspace_invalid"; } + assert(refused, "unsafe Evidence tree did not fail closed"); + assert(await activeCommit("p2-filesystem") === state.revisionB, "unsafe pull changed the active revision"); + return { commands: ["git"], artifacts: [] }; + }, + }, + { + id: "bound_refused", + async run() { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + await rm(join(author, "p2-filesystem", "evidence"), { recursive: true, force: true }); + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "evidence", "big.md"), `# big\n${"x".repeat(9 * 1024 * 1024)}`); + await git(ctx, ["add", "p2-filesystem/evidence"], author); + await git(ctx, ["commit", "-m", "Oversized Evidence file"], author); + await git(ctx, ["push", "origin", "main"], author); + let refused = false; + try { await ctx.registry.pull(); } catch (error) { refused = String(error?.code ?? error) === "workspace_invalid"; } + assert(refused, "oversized Evidence file did not fail closed"); + assert(await activeCommit("p2-filesystem") === state.revisionB, "bound-violating pull changed the active revision"); + return { commands: ["git"], artifacts: [] }; + }, + }, + { + id: "secret_scan", + async run() { + const virtualFiles = []; + const qdrantDump = await dumpQdrantPayloads(ctx, "p2-filesystem"); + if (Buffer.byteLength(qdrantDump) <= MAX_SECRET_SCAN_VIRTUAL_BYTES) virtualFiles.push({ path: "virtual/qdrant-p2-filesystem.json", bytes: qdrantDump }); + const findings = await scanSecrets({ + runRoot: ctx.run.root, + forbiddenValues: ctx.forbiddenValues, + virtualFiles, + expectedGitRepositories: ["remote.git", "author"], + }); + await writeJson(join(ctx.run.root, "logs", "secret-scan.json"), findings); + if (findings.length > 0) throw new Error(`secret scan found ${findings.length} leak(s)`); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "logs/secret-scan.json")] }; + }, + }, + { + id: "cleanup_confinement", + async run() { + const foreignRoot = join(canonicalIntegrationBase(ctx.repositoryRoot), `p2p6-${"f".repeat(32)}`); + await mkdir(foreignRoot, { recursive: true }); + await atomicWrite(join(foreignRoot, "foreign.txt"), "foreign"); + assert(readFileSync(join(foreignRoot, "foreign.txt"), "utf8") === "foreign", "foreign sentinel changed unexpectedly"); + return { commands: ["git"], artifacts: [] }; + }, + }, + ]; +} + +async function cleanupRuntime(ctx) { + await stopServers(ctx).catch(() => {}); + if (ctx.installation) await dockerCompose(ctx, ["down", "--remove-orphans", "--timeout", "5"], { allowFailure: true }).catch(() => {}); + if (ctx.coreImageTag) await runCommand({ executable: ctx.executables.dockerPath, argv: ["image", "rm", "-f", ctx.coreImageTag], env: ctx.execEnv, maxOutputBytes: MAX_STDIO_BYTES }).catch(() => {}); +} + +export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) { + const synthetic = env.P2P6_ACCEPTANCE_SYNTHETIC === "1"; + const failAt = env.P2P6_ACCEPTANCE_FAIL_AT; + const ctx = synthetic + ? { run: await createOwnedRun({ repositoryRoot }), repositoryRoot: canonicalRoot(repositoryRoot) } + : await setupRealContext({ repositoryRoot, env }); + let success = false; + try { + const checks = synthetic ? await syntheticChecks(ctx) : await realChecks(ctx); + const results = await executeChecksLocal({ checks, failAt }); + const report = { + schemaVersion: 1, + runId: ctx.run.runId, + startedAt: ctx.run.startedAt, + finishedAt: nowIso(), + command: "p2p6-acceptance integration --keep", + overall: deriveOverall(results), + checks: results, + }; + await writeReportFiles({ run: ctx.run, report }); + success = report.overall === "PASS"; + if (announce) await announce({ report, runRoot: ctx.run.root }); + return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) }; + } finally { + if (!synthetic) await cleanupRuntime(ctx).catch(() => {}); + } +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) { + throw new Error("usage: p2p6-acceptance.mjs integration [--keep]"); + } + const result = await runIntegration({ keep: argv.includes("--keep"), env }); + return result.exitCode; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { + const code = await main(); + process.exitCode = code; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +export { CHECK_IDS }; diff --git a/backend/scripts/p2p6-acceptance.test.mjs b/backend/scripts/p2p6-acceptance.test.mjs new file mode 100644 index 00000000..38b1f1e3 --- /dev/null +++ b/backend/scripts/p2p6-acceptance.test.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_IDS, + canonicalIntegrationBase, + cleanupOwnedRun, + createOwnedRun, + readAndValidateOwnership, + runIntegration, + validateReport, + validateRunRoot, +} from "./p2p6-acceptance.mjs"; + +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p2p6-acceptance-repo-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p2p6-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true }); + return root; +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("run roots are only canonical direct p2p6 integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p2p6-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [ + base, + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(repositoryRoot, ".artifacts", "p1-integration", id), + join(repositoryRoot, ".artifacts", "p2-integration", id), + join(base, id, "nested"), + join(base, "foreign"), + ]) { + assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + } + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p2p6-${"A".repeat(32)}`), `p2p6-${"A".repeat(32)}`)); +}); + +test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + for (const bad of [ + join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`), + join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`), + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(canonicalIntegrationBase(repositoryRoot), `p2p6-${"d".repeat(32)}`), + ]) { + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce })); + } + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) })); +}); + +test("cleanup removes exactly one owned p2p6 root", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const sibling = join(canonicalIntegrationBase(repositoryRoot), `p2p6-${"e".repeat(32)}`); + await mkdir(sibling); + await writeFile(join(sibling, "sentinel"), "foreign"); + await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await assert.rejects(readFile(join(run.root, "ownership.json"))); + assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign"); +}); + +function resultFor(id) { + return { + id, + status: "PASS", + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:01.000Z", + commands: ["node"], + artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }], + }; +} + +test("report validation requires exact p2p6 identity, check order, and unique artifacts", () => { + const report = { + schemaVersion: 1, + runId: `p2p6-${"f".repeat(32)}`, + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:10.000Z", + command: "p2p6-acceptance integration --keep", + overall: "PASS", + checks: CHECK_IDS.map(resultFor), + }; + assert.doesNotThrow(() => validateReport(report)); + const invalid = structuredClone(report); + invalid.runId = `p2-${"f".repeat(32)}`; + assert.throws(() => validateReport(invalid)); + const duplicate = structuredClone(report); + duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path; + assert.throws(() => validateReport(duplicate), /duplicated/); + const reordered = structuredClone(report); + reordered.checks.reverse(); + reordered.overall = "FAIL"; + assert.throws(() => validateReport(reordered)); +}); + +test("public wrapper uses a strict empty environment", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p2p6-acceptance.sh"), "utf8"); + assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/); + assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/); + assert.doesNotMatch(wrapper, /P2P6_ACCEPTANCE_FAIL_AT/); +}); + +test("synthetic integration cleans up successful non-kept runs", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: false, env: { P2P6_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, false); + await assert.rejects(readFile(join(result.runRoot, "ownership.json"))); +}); + +test("synthetic integration retains kept runs with bounded reports", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: true, env: { P2P6_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "PASS"); + const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8"); + assert.match(reportMd, /P2P6 automated integration: PASS/); + assert.match(reportMd, /P2P6 manual acceptance: PENDING/); + const reportJsonStat = await stat(join(result.runRoot, "report.json")); + const reportMdStat = await stat(join(result.runRoot, "report.md")); + assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`); + assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`); +}); + +test("synthetic injected failure retains the owned run and records a single failed report", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ + repositoryRoot, + keep: false, + env: { P2P6_ACCEPTANCE_SYNTHETIC: "1", P2P6_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] }, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "FAIL"); + const failed = report.checks.find((check) => check.id === CHECK_IDS[2]); + assert.equal(failed.status, "FAIL"); + const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8"); + assert.match(roots, /p2p6-acceptance/); +}); diff --git a/backend/scripts/p3-acceptance.mjs b/backend/scripts/p3-acceptance.mjs new file mode 100644 index 00000000..b851af05 --- /dev/null +++ b/backend/scripts/p3-acceptance.mjs @@ -0,0 +1,1433 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from "node:crypto"; +import { execFile, execFileSync } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { createServer } from "node:http"; +import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs"; +import { access, lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import net from "node:net"; +import process from "node:process"; + +import { stringify as yamlStringify } from "yaml"; + +import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const RUN_ID = /^p3-[0-9a-f]{32}$/; +const HEX32 = /^[0-9a-f]{32}$/; +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const COMMAND = /^[a-z0-9][a-z0-9-]*$/; +const CHECK_RESULT_STATUS = new Set(["PASS", "FAIL"]); +const CHECK_IDS = Object.freeze([ + "preflight", + "clean_state", + "ownership", + "effective_config_identity", + "dwh_processing", + "schema_review", + "schema_index", + "evidence_processing", + "content_only_reuse", + "dwh_change_fail_closed", + "memory_root", + "revision_scoped_records", + "negative_cases", + "secret_scan", + "cleanup_confinement", +]); +const TOPOLOGY = [ + "remote.git", + "author", + "installation", + "installation/data", + "installation/data/sessions", + "installation/registry", + "installation/pi-state", + "fixture-secrets", + "fixtures", + "fixtures/logs", + "logs", +]; +const MAX_REPORT_JSON_BYTES = 64 * 1024; +const MAX_REPORT_MD_BYTES = 32 * 1024; +const MAX_STDIO_BYTES = 512 * 1024; +const MAX_SECRET_SCAN_VIRTUAL_BYTES = 256 * 1024; + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function assert(condition, message) { if (!condition) throw new Error(message); } +function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function canonicalRoot(repositoryRoot = defaultRepositoryRoot) { + return realpathSync(repositoryRoot); +} + +export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p3-integration"); +} + +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} + +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!existsSync(cursor)) break; + const entry = lstatSync(cursor); + if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink"); + } +} + +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + if (handle) await handle.close().catch(() => {}); + await rm(staging, { force: true }).catch(() => {}); + throw error; + } +} + +function initialResources(run) { + return [ + run.root, + join(run.root, "remote.git"), + join(run.root, "author"), + join(run.root, "installation"), + join(run.root, "installation", "registry"), + join(run.root, "installation", "data"), + join(run.root, "fixture-secrets"), + ]; +} + +function ownershipValue(run) { + return { + schemaVersion: 1, + kind: "p3-acceptance", + runId: run.runId, + runNonce: run.nonce, + root: run.root, + repositoryRoot: run.repositoryRoot, + startedAt: run.startedAt, + pid: run.pid, + resources: initialResources(run), + }; +} + +async function writeOwnership(run) { + await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run), null, 2)}\n`); +} + +export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) { + const repo = canonicalRoot(repositoryRoot); + const base = canonicalIntegrationBase(repo); + validateNoSymlinkAncestors(repo, base); + await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + const id = runId ?? `p3-${randomBytes(16).toString("hex")}`; + const root = validateRunRoot(repo, join(base, id), id); + const run = { + repositoryRoot: repo, + root, + runId: id, + nonce: nonce ?? randomBytes(32).toString("hex"), + startedAt: now ?? nowIso(), + pid: pid ?? process.pid, + }; + if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity"); + await mkdir(root, { mode: 0o700 }); + await writeOwnership(run); + return run; +} + +function strictOwnership(value, run, expectedNonce) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed"); + if (value.schemaVersion !== 1 || value.kind !== "p3-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce + || value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid + || !ISO_UTC.test(value.startedAt ?? "") + || JSON.stringify(value.resources) !== JSON.stringify(initialResources(run))) throw new Error("ownership identity mismatch"); + return value; +} + +export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const repo = canonicalRoot(repositoryRoot); + const id = basename(resolve(runRoot)); + const lexical = validateRunRoot(repo, runRoot, id); + const rootEntry = await lstat(lexical); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory"); + const ownershipPath = join(lexical, "ownership.json"); + const ownershipEntry = await lstat(ownershipPath); + if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe"); + let value; + try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); } + return strictOwnership(value, { repositoryRoot: repo, root: lexical, runId: id }, expectedNonce); +} + +export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }); + const base = canonicalIntegrationBase(repositoryRoot); + const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`); + await rename(runRoot, tombstone); + await rm(tombstone, { recursive: true, force: false }); +} + +async function finalizeOwnedRun({ run, success, keep }) { + if (!success || keep) return false; + await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + return true; +} + +function safeArtifactPath(path) { + if (typeof path !== "string" || path.length === 0 || path.length > 255 || path.startsWith("/") || path.includes("..") || path.includes("\\") || /[\0\r\n]/.test(path)) { + throw new Error("report artifact path is invalid"); + } + return path; +} + +function hasExactCheckIds(checks) { + return checks.length === CHECK_IDS.length && checks.every(({ id }, index) => id === CHECK_IDS[index]); +} + +export function validateReport(report) { + if (!report || report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "") + || !ISO_UTC.test(report.finishedAt ?? "") || typeof report.command !== "string" + || !Array.isArray(report.checks) || !hasExactCheckIds(report.checks)) throw new Error("report is invalid"); + const ids = new Set(); + const artifactPaths = new Set(); + for (const check of report.checks) { + if (!check || !/^[a-z0-9_]+$/.test(check.id ?? "") || ids.has(check.id) || !CHECK_RESULT_STATUS.has(check.status) + || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "") + || !Array.isArray(check.commands) || check.commands.some((name) => !COMMAND.test(name)) + || !Array.isArray(check.artifacts)) throw new Error("report check is invalid"); + ids.add(check.id); + for (const artifact of check.artifacts) { + safeArtifactPath(artifact.path); + if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report check is invalid"); + if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated"); + artifactPaths.add(artifact.path); + } + } + if (report.overall !== deriveOverall(report.checks)) throw new Error("report overall is not derived"); + return report; +} + +function renderReportMarkdown(report) { + validateReport(report); + const rows = report.checks.map((check) => `| ${check.id} | ${check.status} |`).join("\n"); + return [ + "# P2 acceptance report", + "", + `Run: \`${report.runId}\``, + "", + "| Check | Status |", + "|---|---|", + rows, + "", + `P3 automated integration: ${report.overall}`, + "P3 manual acceptance: PENDING", + "", + ].join("\n"); +} + +async function walkFiles(root) { + const files = []; + async function visit(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const rel = relative(root, path).split(sep).join("/"); + if (entry.isSymbolicLink()) throw new Error(`unsafe file tree: ${rel}`); + if (entry.isDirectory()) await visit(path); + else if (entry.isFile()) files.push({ path, rel }); + } + } + if (existsSync(root)) await visit(root); + files.sort((a, b) => a.rel.localeCompare(b.rel)); + return files; +} + +async function snapshotDigest(root, excludedPrefixes = []) { + const result = {}; + for (const file of await walkFiles(root)) { + if (excludedPrefixes.some((prefix) => file.rel === prefix || file.rel.startsWith(`${prefix}/`))) continue; + result[file.rel] = sha256(await readFile(file.path)); + } + return result; +} + +async function fileArtifact(root, relativePath) { + const bytes = await readFile(join(root, relativePath)); + return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) }; +} + +async function writeJson(path, value) { + await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function writeReportFiles({ run, report }) { + validateReport(report); + const reportJsonPath = join(run.root, "report.json"); + const reportMdPath = join(run.root, "report.md"); + const reportMd = renderReportMarkdown(report); + if (Buffer.byteLength(JSON.stringify(report)) > MAX_REPORT_JSON_BYTES) throw new Error("report.json exceeds bound"); + if (Buffer.byteLength(reportMd) > MAX_REPORT_MD_BYTES) throw new Error("report.md exceeds bound"); + await writeJson(reportJsonPath, report); + await atomicWrite(reportMdPath, reportMd, 0o600); + return { + reportJson: await fileArtifact(run.root, "report.json"), + reportMd: await fileArtifact(run.root, "report.md"), + }; +} + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (statSync(resolved).isFile()) return resolved; + } catch {} + } + throw new Error(`required executable not found: ${name}`); +} + +function scalarSecretBytes(value) { + if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid"); + return Buffer.from(value); +} + +async function manifestFiles(root, paths) { + const files = []; + const visit = async (absolute, rel) => { + const entry = await lstat(absolute); + if (entry.isSymbolicLink()) throw new Error(`provenance path is a symlink: ${rel}`); + if (entry.isDirectory()) { + for (const child of (await readdir(absolute, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + await visit(join(absolute, child.name), rel ? `${rel}/${child.name}` : child.name); + } + } else if (entry.isFile()) { + const bytes = await readFile(absolute); + files.push({ path: rel, bytes: bytes.length, sha256: sha256(bytes) }); + } else throw new Error(`provenance path is not a regular file: ${rel}`); + }; + for (const path of paths) await visit(join(root, path), path); + files.sort((a, b) => a.path.localeCompare(b.path)); + return { files, manifestSha256: sha256(JSON.stringify(files)) }; +} + +async function collectRepositoryProvenance({ repositoryRoot, gitPath = resolveSystemExecutable("git") }) { + const repo = canonicalRoot(repositoryRoot); + const safeEnv = buildSafeEnvironment({ ambient: {}, fixture: { PATH: `${dirname(gitPath)}:/usr/bin:/bin`, HOME: repo, TMPDIR: join(repo, ".artifacts") } }); + const run = async (argv) => await execFileAsync(gitPath, ["-C", repo, ...argv], { env: safeEnv, maxBuffer: MAX_STDIO_BYTES }); + const beforeHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const beforeTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const beforeStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (!HEX40.test(beforeHead) || !HEX40.test(beforeTree) || beforeStatus !== "") throw new Error("repository is not clean at exact HEAD"); + const backendRoot = join(repo, "backend"); + const backendSource = await manifestFiles(backendRoot, [ + "src", + "scripts/p3-acceptance.mjs", + "package.json", + "package-lock.json", + "tsconfig.json", + ]); + const backendDist = existsSync(join(backendRoot, "dist")) ? await manifestFiles(backendRoot, ["dist"]) : { files: [], manifestSha256: sha256("[]") }; + const afterHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const afterTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const afterStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (afterHead !== beforeHead || afterTree !== beforeTree || afterStatus !== beforeStatus) throw new Error("repository provenance changed during binding"); + return { schemaVersion: 1, head: beforeHead, tree: beforeTree, clean: true, backendSource, backendDist }; +} + +async function createTopology(run) { + for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); +} + +async function allocatePort() { + const server = net.createServer(); + await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).on("error", reject)); + const port = server.address().port; + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + return port; +} + +function installationProjectName(installationPath) { + return `thothii-${sha256(installationPath).slice(0, 12)}`; +} + +function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) { + return { + workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + diagnostics: { + dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}), + }; +} + +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } + +function descriptorYaml(obj) { + return yamlStringify(obj, { lineWidth: 0, sortMapEntries: false }); +} + +async function setupSecrets(ctx) { + const secretDir = join(ctx.run.root, "fixture-secrets"); + const values = { + dwhToken: `P2-DWH-${randomBytes(16).toString("hex")}`, + signedToken: `P2-SIGNED-${randomBytes(16).toString("hex")}`, + bundle: `P2-BUNDLE-${randomBytes(16).toString("hex")}`, + }; + ctx.forbiddenValues = Object.values(values); + ctx.secretValues = values; + const paths = { + dwh: join(secretDir, "p2-dwh-api-key"), + filesystemDwh: join(secretDir, "p2-filesystem-api-key"), + signed: join(secretDir, "p2-dwh-evidence-signed-urls.json"), + bundle: join(secretDir, "thothii.secrets"), + }; + await atomicWrite(paths.dwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.filesystemDwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.bundle, scalarSecretBytes(values.bundle)); + ctx.secretPaths = paths; +} + +async function setupFixtures(ctx) { + ctx.fixturePorts = { + dwh: await allocatePort(), + evidence: await allocatePort(), + embedding: await allocatePort(), + qdrant: await allocatePort(), + }; + const dwhBaseUrl = `http://host.docker.internal:${ctx.fixturePorts.dwh}`; + const evidenceProvenance = `http://host.docker.internal:${ctx.fixturePorts.evidence}/p2-dwh/guide.md`; + ctx.workspaceObjects = { + dwh: baseWorkspace("p2-dwh", { + dwhBaseUrl, + evidenceSource: { + type: "http", + uris: [evidenceProvenance], + authentication: "signed_urls_file", + connect_timeout_ms: 1250, + read_timeout_ms: 30001, + max_bytes: 65536, + max_redirects: 2, + allow_private_hosts: true, + max_cache_bytes: 65536, + }, + }), + filesystem: baseWorkspace("p2-filesystem", { + dwhBaseUrl, + evidenceSource: { + type: "filesystem", + uri: "p2-filesystem/evidence", + patterns: ["**/*.md"], + max_bytes: 1048576, + }, + }), + }; + const signedUrl = `${evidenceProvenance}?token=${ctx.secretValues.signedToken}`; + await atomicWrite(ctx.secretPaths.signed, `${JSON.stringify([signedUrl], null, 2)}\n`); + + ctx.evidenceState = { + content: "# P2 Evidence\n\nFirst generation.\n", + token: ctx.secretValues.signedToken, + }; + ctx.dwhState = { + tables: { + patients: { + comment: "Patients", + rows: [ + { patient_id: "p1", name: "Alice" }, + { patient_id: "p2", name: "Bob" }, + ], + }, + visits: { + comment: "Visits", + rows: [ + { id: "v1", patient_id: "p1", note: "checkup" }, + { id: "v2", patient_id: "p2", note: "xray" }, + ], + }, + labs: { + comment: "Labs", + rows: [ + { id: "l1", patient_id: "p1", code: "hemoglobin" }, + { id: "l2", patient_id: "p2", code: "glucose" }, + ], + }, + }, + token: ctx.secretValues.dwhToken, + }; +} + +function inferColumnType(value) { + return typeof value === "number" ? "integer" : "text"; +} + +function topValues(rows, column, limit) { + const counts = new Map(); + for (const row of rows) { + const value = row[column]; + if (value === undefined || value === null || value === "") continue; + counts.set(String(value), (counts.get(String(value)) ?? 0) + 1); + } + return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, limit).map(([value]) => ({ value })); +} + +async function startHttpServer({ port, handler }) { + const server = createServer(async (req, res) => { + try { + await handler(req, res); + } catch { + res.statusCode = 500; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "fixture failed" })); + } + }); + await new Promise((resolve, reject) => server.listen(port, "127.0.0.1", () => resolve()).on("error", reject)); + return server; +} + +async function startServers(ctx) { + const dwhServer = await startHttpServer({ + port: ctx.fixturePorts.dwh, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const json = body.length === 0 ? {} : JSON.parse(body); + if (req.headers["x-api-key"] !== ctx.dwhState.token) { + res.statusCode = 401; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ message: "unauthorized" })); + return; + } + const send = (payload) => { + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(payload)); + }; + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.dwh}`); + if (req.method !== "POST" || !url.pathname.startsWith("/rpc/")) { + res.statusCode = 404; + res.end(JSON.stringify({ message: "not found" })); + return; + } + const fn = url.pathname.slice("/rpc/".length); + const schemaName = json.schema_name ?? "dw"; + if (schemaName !== "dw") { + send([]); + return; + } + if (fn === "ping") { + send({ db_connected: true, schema_accessible: true, database: "warehouse", schema: "dw" }); + return; + } + const table = typeof json.table_name === "string" ? json.table_name : ""; + const tableData = ctx.dwhState.tables[table]; + if (fn === "list_tables") { + send(Object.entries(ctx.dwhState.tables).map(([name, info]) => ({ table: name, type: "TABLE", comment: info.comment, rows: info.rows.length }))); + return; + } + if (!tableData) { + send([]); + return; + } + if (fn === "table_columns") { + const first = tableData.rows[0] ?? {}; + send(Object.keys(first).map((column) => ({ + column, + type: inferColumnType(first[column]), + nullable: false, + // Only the referenced table marks `patient_id` as primary, so the SQL miner sees a + // PK/non-PK pair while the same-name heuristic still discovers joins from the others. + pk: column === "id" || (table === "patients" && column === "patient_id"), + default: null, + }))); + return; + } + if (fn === "table_comments") { + send(Object.keys(tableData.rows[0] ?? {}).map((column) => ({ object: "COLUMN", name: column, comment: `${table}.${column}` }))); + return; + } + if (fn === "table_foreign_keys") { + send([]); + return; + } + if (fn === "top_values") { + send(topValues(tableData.rows, json.column_name, Number(json.max_values ?? 10))); + return; + } + if (fn === "column_stats") { + send({}); + return; + } + if (fn === "run_query") { + send([]); + return; + } + if (fn === "explain_query") { + send([{ line: "Seq Scan" }]); + return; + } + res.statusCode = 404; + res.end(JSON.stringify({ message: "unknown rpc" })); + }, + }); + const evidenceServer = await startHttpServer({ + port: ctx.fixturePorts.evidence, + handler: async (req, res) => { + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.evidence}`); + if (url.pathname !== "/p2-dwh/guide.md" || url.searchParams.get("token") !== ctx.evidenceState.token) { + res.statusCode = 403; + res.end("forbidden"); + return; + } + res.statusCode = 200; + res.setHeader("content-type", "text/markdown; charset=utf-8"); + res.end(ctx.evidenceState.content); + }, + }); + const embeddingServer = await startHttpServer({ + port: ctx.fixturePorts.embedding, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.embedding}`); + if (req.method !== "POST" || url.pathname !== "/api/embed") { + res.statusCode = 404; + res.end(JSON.stringify({ error: "not found" })); + return; + } + const payload = JSON.parse(body || "{}"); + const inputs = Array.isArray(payload.input) ? payload.input : []; + const embeddings = inputs.map((text) => { + const seed = sha256(String(text)); + return Array.from({ length: 1024 }, (_, index) => { + const offset = (index * 2) % seed.length; + const value = Number.parseInt(seed.slice(offset, offset + 2), 16); + return (value / 255) - 0.5; + }); + }); + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ model: payload.model, embeddings })); + }, + }); + ctx.servers = [dwhServer, evidenceServer, embeddingServer]; +} + +async function stopServers(ctx) { + for (const server of ctx.servers ?? []) { + await new Promise((resolve) => server.close(() => resolve())); + } + ctx.servers = []; +} + +async function git(ctx, args, cwd = join(ctx.run.root, "author")) { + return await runCommand({ executable: ctx.executables.gitPath, argv: args, cwd, env: ctx.execEnv }); +} + +async function initializeGitAndRegistry(ctx) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], ctx.run.root); + await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], ctx.run.root); + await git(ctx, ["config", "user.name", "P2 Fixture Curator"], author); + await git(ctx, ["config", "user.email", "p3-curator@example.invalid"], author); + + const writeWorkspaces = async () => { + const catalog = { + schema_version: 1, + workspaces: [ + { id: "p2-dwh", name: ctx.workspaceObjects.dwh.workspace.name }, + { id: "p2-filesystem", name: ctx.workspaceObjects.filesystem.workspace.name }, + ], + }; + await writeFile(join(author, "thoth-workspaces.yaml"), yamlStringify(catalog, { lineWidth: 0, sortMapEntries: false })); + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId), { recursive: true }); + const yaml = descriptorYaml(workspace); + await writeFile(join(author, pathId, "workspace.yaml"), yaml); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + await mkdir(join(author, "workspace-docs", pathId), { recursive: true }); + await writeFile(join(author, "workspace-docs", pathId, "contract.env.example"), docs.envExample); + await writeFile(join(author, "workspace-docs", pathId, "README.md"), docs.markdown); + } + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "evidence", "guide.md"), "# P2 Filesystem Evidence\n\nCommitted fixture.\n"); + }; + + await writeWorkspaces(); + await git(ctx, ["add", "."], author); + await git(ctx, ["commit", "-m", "Bootstrap P2 fixtures"], author); + await git(ctx, ["push", "origin", "main"], author); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); + const registry = new ctx.workspaceModules.WorkspaceRegistry({ + root: join(ctx.run.root, "installation", "registry"), + remoteUrl: join(ctx.run.root, "remote.git"), + branch: "main", + gitAuthorName: "P2 Acceptance", + gitAuthorEmail: "p3-acceptance@example.invalid", + installationId: "p3-acceptance", + secretRoots: [join(ctx.run.root, "fixture-secrets")], + maxImportBytes: 16 * 1024 * 1024, + maxImportEntries: 1024, + }); + await registry.bootstrap(); + ctx.registry = registry; +} + +async function mutateWorkspaceDescriptor(ctx, workspaceId, mutator, commitMessage) { + const author = join(ctx.run.root, "author"); + // The registry may have produced docs-only follow-up commits on the remote; the curator + // always rebases onto the latest remote head before committing so the push stays fast-forward. + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const workspace = structuredClone(ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"]); + mutator(workspace); + ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"] = workspace; + await writeFile(join(author, workspaceId, "workspace.yaml"), descriptorYaml(workspace)); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + const docsDir = join(author, "workspace-docs", workspaceId); + await mkdir(docsDir, { recursive: true, mode: 0o700 }); + await writeFile(join(docsDir, "contract.env.example"), docs.envExample); + await writeFile(join(docsDir, "README.md"), docs.markdown); + await git(ctx, ["add", `${workspaceId}/workspace.yaml`, `workspace-docs/${workspaceId}/contract.env.example`, `workspace-docs/${workspaceId}/README.md`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + +async function writeInstallationFiles(ctx) { + const installationDir = join(ctx.run.root, "installation"); + const operatorEnvPath = join(installationDir, "operator.env"); + const bindingsEnvPath = join(installationDir, "workspace-bindings.env"); + const connectorOverridePath = join(installationDir, "connector-secrets.override.yaml"); + const fixtureOverridePath = join(installationDir, "fixture.override.yaml"); + const installationPath = join(installationDir, "thothii-installation.yaml"); + ctx.installationPath = installationPath; + ctx.composeProject = installationProjectName(installationPath); + const qdrantPort = ctx.fixturePorts.qdrant; + const bindings = [ + `THT_WS_P2_DWH_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_DWH_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_DWH_DWH_API_KEY_FILE=/run/secrets/p2-dwh-api-key`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_FILE=/run/secrets/p2-dwh-evidence-signed-urls`, + `THT_WS_P2_FILESYSTEM_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_FILESYSTEM_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_FILE=/run/secrets/p2-filesystem-api-key`, + ].join("\n") + "\n"; + await atomicWrite(bindingsEnvPath, bindings); + const operatorEnv = [ + `THT_DATA_ROOT=${join(ctx.run.root, "installation", "data")}`, + `THT_WORKSPACE_REGISTRY_ROOT=${join(ctx.run.root, "installation", "registry")}`, + `THT_PI_STATE_ROOT=${join(ctx.run.root, "installation", "pi-state")}`, + `PI_AUTH_FILE=${join(ctx.run.root, "installation", "pi-auth.json")}`, + `THT_SECRETS_FILE=${ctx.secretPaths.bundle}`, + `THT_WORKSPACE_BINDINGS_ENV_FILE=${bindingsEnvPath}`, + `THT_WORKSPACE_GIT_REMOTE=${join(ctx.run.root, "remote.git")}`, + `THT_WORKSPACE_GIT_BRANCH=main`, + `THT_WORKSPACE_GIT_AUTHOR_NAME=P2 Acceptance`, + `THT_WORKSPACE_GIT_AUTHOR_EMAIL=p3-acceptance@example.invalid`, + `THT_WORKSPACE_INSTALLATION_ID=p3-acceptance`, + `THT_DB_NAME=warehouse`, + `THT_DWH_REST_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_LLM_URL=http://127.0.0.1:9`, + `THOTH_SERVER_BIND=127.0.0.1`, + `THOTH_HTTP_PORT=18080`, + `THOTH_CORE_HTTP_PORT=18787`, + `THT_WS_P2_DWH_DWH_API_KEY_SOURCE=${ctx.secretPaths.dwh}`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_SOURCE=${ctx.secretPaths.signed}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_SOURCE=${ctx.secretPaths.filesystemDwh}`, + `THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST=host.docker.internal`, + ].join("\n") + "\n"; + await atomicWrite(operatorEnvPath, operatorEnv); + await atomicWrite(join(ctx.run.root, "installation", "pi-auth.json"), JSON.stringify({ fixture: true })); + const embeddingStubPath = join(installationDir, "embedding-stub.py"); + await atomicWrite(embeddingStubPath, EMBEDDING_STUB_SOURCE); + const override = { + services: { + core: { + image: ctx.coreImageTag, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + "workspace-maintenance": { + image: ctx.coreImageTag, + environment: { + THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST: "host.docker.internal", + }, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + qdrant: { + ports: [`127.0.0.1:${qdrantPort}:6333`], + restart: "no", + }, + // Deterministic Ollama-compatible embedding fixture on the internal allowlisted host + // name `embedding` (http://embedding:11434). Replaces the real Ollama service entirely. + embedding: { + image: ctx.coreImageTag, + entrypoint: ["python3", "/stub.py"], + volumes: [ + { type: "bind", source: embeddingStubPath, target: "/stub.py", read_only: true }, + ], + healthcheck: { disable: true }, + }, + }, + }; + await atomicWrite(fixtureOverridePath, yamlStringify(override, { lineWidth: 0, sortMapEntries: false })); + const generated = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "generate-connector-secrets-override.sh"), + argv: [ + "--bindings-env", bindingsEnvPath, + "--operator-env", operatorEnvPath, + "--output", connectorOverridePath, + "--service", "workspace-maintenance", + "--role", "all", + ], + env: ctx.execEnv, + }); + if (generated.exitCode !== 0) throw new Error(`connector override generation failed: ${generated.stderr || generated.stdout}`); + const installation = { + profile: "server", + projectDirectory: ctx.repositoryRoot, + envFile: operatorEnvPath, + overrides: [ + join(ctx.repositoryRoot, "deploy", "compose.server.yaml"), + fixtureOverridePath, + connectorOverridePath, + ], + }; + await atomicWrite(installationPath, yamlStringify(installation, { lineWidth: 0, sortMapEntries: false })); + ctx.installation = installation; +} + +function thothctlBinaryPath(repositoryRoot) { + const platform = { darwin: "darwin", linux: "linux", win32: "windows" }[process.platform] ?? "linux"; + const arch = { x64: "amd64", arm64: "arm64" }[process.arch] ?? "amd64"; + const suffix = platform === "windows" ? ".exe" : ""; + const candidates = [ + join(repositoryRoot, "dist", "thothctl", `thothctl-${platform}-${arch}${suffix}`), + join(repositoryRoot, "tools", "thothctl", "bin", `thothctl${suffix}`), + ]; + for (const candidate of candidates) if (existsSync(candidate)) return candidate; + throw new Error("built thothctl binary is unavailable"); +} + +async function runCommand({ executable, argv = [], cwd, env, input, maxOutputBytes = MAX_STDIO_BYTES }) { + const result = await execFileAsync(executable, argv, { + cwd, + env, + encoding: "utf8", + maxBuffer: maxOutputBytes, + ...(input === undefined ? {} : { input }), + }).then( + ({ stdout, stderr }) => ({ exitCode: 0, stdout, stderr }), + (error) => ({ exitCode: error.code ?? 1, stdout: error.stdout ?? "", stderr: error.stderr ?? error.message ?? "" }), + ); + return result; +} + +async function buildCoreImage(ctx) { + const tag = `thothii-core:p2-${ctx.run.runId.slice(3, 15)}`; + ctx.coreImageTag = tag; + const build = await runCommand({ + executable: ctx.executables.dockerPath, + argv: ["build", "-f", join(ctx.repositoryRoot, "docker", "core.Dockerfile"), "-t", tag, ctx.repositoryRoot], + env: { ...ctx.execEnv, DOCKER_BUILDKIT: "1" }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (build.exitCode !== 0) throw new Error(`core image build failed: ${build.stderr || build.stdout}`); +} + +async function buildThothctl(ctx) { + const command = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "build-thothctl.sh"), + argv: [], + env: { ...ctx.execEnv, THT_THOTHCTL_OUTPUT_DIRECTORY: join(ctx.repositoryRoot, "dist", "thothctl") }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (command.exitCode !== 0) throw new Error(`build-thothctl failed: ${command.stderr || command.stdout}`); + ctx.thothctlPath = thothctlBinaryPath(ctx.repositoryRoot); +} + +function composeBaseArgs(ctx) { + const args = [ + "compose", + "--project-name", ctx.composeProject, + "--project-directory", ctx.installation.projectDirectory, + "--env-file", ctx.installation.envFile, + "-f", join(ctx.repositoryRoot, "compose.yaml"), + ]; + for (const override of ctx.installation.overrides) args.push("-f", override); + return args; +} + +async function dockerCompose(ctx, commandArgs, { allowFailure = false, maxOutputBytes = 2 * 1024 * 1024 } = {}) { + const result = await runCommand({ + executable: ctx.executables.dockerPath, + argv: [...composeBaseArgs(ctx), ...commandArgs], + env: ctx.execEnv, + maxOutputBytes, + }); + if (!allowFailure && result.exitCode !== 0) throw new Error(`docker compose ${commandArgs.join(" ")} failed: ${result.stderr || result.stdout}`); + return result; +} + +async function startQdrant(ctx) { + await dockerCompose(ctx, ["up", "-d", "qdrant", "embedding"]); + for (let attempt = 0; attempt < 60; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}/collections`); + if (response.ok) return; + } catch {} + await sleep(1000); + } + throw new Error("qdrant did not become ready"); +} + +async function qdrantJson(ctx, method, path, body) { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}${path}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const payload = response.status === 204 ? {} : await response.json().catch(() => ({})); + if (!response.ok) throw new Error(`qdrant request failed: ${method} ${path} ${response.status}`); + return payload; +} + +async function preprovisionCollection(ctx, workspaceId) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}`, { + vectors: { size: 1024, distance: "Cosine" }, + }); + for (const field of ["content_hash", "document_id", "kind", "record_key", "record_kind", "vector_generation", "workspace_id", "workspace_revision"]) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}/index`, { field_name: field, field_schema: "keyword" }); + } +} + +async function listCollections(ctx) { + const payload = await qdrantJson(ctx, "GET", "/collections"); + const collections = payload.result?.collections ?? []; + return collections.map((item) => item.name).sort(); +} + +async function dumpQdrantPayloads(ctx, workspaceId) { + const response = await qdrantJson(ctx, "POST", `/collections/${workspaceId}/points/scroll`, { limit: 128, with_payload: true, with_vector: false }); + return JSON.stringify(response.result?.points ?? []); +} + +async function runThothctlJson(ctx, label, workspaceArgs, expectedExitCode) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.json`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs, "--json"], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + if (expectedExitCode !== undefined && result.exitCode !== expectedExitCode) { + throw new Error(`${label} exit ${result.exitCode} != ${expectedExitCode}`); + } + let payload; + try { payload = JSON.parse(result.stdout); } catch (error) { throw new Error(`${label} returned non-JSON stdout`); } + return { result, payload, artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath))] }; +} + +async function loadWorkspaceSnapshot(ctx, workspaceId) { + const active = JSON.parse(await readFile(join(ctx.run.root, "installation", "registry", "state", "active.json"), "utf8")); + const revision = active.revisions.find((entry) => entry.id === workspaceId); + const snapshotPath = revision.snapshotPath; + const contents = await readFile(snapshotPath, "utf8"); + return { active, revision, contents }; +} + +async function assertNoCoreFrontendRunning(ctx) { + const ps = await dockerCompose(ctx, ["ps", "--status", "running", "--format", "json"], { allowFailure: true }); + if (ps.exitCode !== 0) return []; + const lines = ps.stdout.trim() === "" ? [] : ps.stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); + const services = lines.map((item) => item.Service); + if (services.includes("core") || services.includes("frontend") || services.includes("workspace-maintenance")) { + throw new Error("core/frontend/maintenance is unexpectedly running"); + } + return services; +} + +function sameSet(left, right) { + return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort()); +} + +const EMBEDDING_STUB_SOURCE = String.raw`import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +class _Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(length)) + inputs = payload.get("input", []) + if isinstance(inputs, str): + inputs = [inputs] + embeddings = [[0.01] * 1024 for _ in inputs] + body = json.dumps({"model": payload.get("model", "qwen3-embedding:0.6b"), "embeddings": embeddings}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + +HTTPServer(("0.0.0.0", 11434), _Handler).serve_forever() +`; + +function realUserHome() { + try { + const output = execFileSync("bash", ["-lc", 'printf "%s" ~'], { encoding: "utf8" }).trim(); + return output.length > 0 ? output : undefined; + } catch { + return undefined; + } +} + +async function setupRealContext({ repositoryRoot = defaultRepositoryRoot, env = process.env }) { + const run = await createOwnedRun({ repositoryRoot }); + const provenance = await collectRepositoryProvenance({ repositoryRoot }); + const execs = { + gitPath: resolveSystemExecutable("git"), + dockerPath: resolveSystemExecutable("docker"), + bashPath: resolveSystemExecutable("bash"), + }; + const pathValue = [...new Set([dirname(execs.gitPath), dirname(execs.dockerPath), "/usr/bin", "/bin", "/opt/homebrew/bin", "/usr/local/bin"])].join(":"); + // Docker CLI plugins (buildx) live under the real user's ~/.docker; the wrapper runs with a + // scrubbed environment, so derive the real home from the passwd entry and expose DOCKER_CONFIG. + const realHome = env.P3_REAL_HOME ?? realUserHome(); + const execEnv = buildSafeEnvironment({ ambient: env, fixture: { + PATH: pathValue, + HOME: run.root, + TMPDIR: join(run.root, "tmp"), + ...(realHome ? { DOCKER_CONFIG: join(realHome, ".docker") } : {}), + } }); + const workspaceModules = await import("../dist/workspaces/registry.js").then(async (registryModule) => ({ + WorkspaceRegistry: registryModule.WorkspaceRegistry, + ...(await import("../dist/workspaces/schema.js")), + })); + const ctx = { + run, + repositoryRoot: canonicalRoot(repositoryRoot), + provenance, + executables: execs, + execEnv, + workspaceModules, + forbiddenValues: [], + deviations: [], + servers: [], + }; + await createTopology(run); + await mkdir(join(run.root, "tmp"), { recursive: true, mode: 0o700 }); + await setupSecrets(ctx); + await setupFixtures(ctx); + return ctx; +} + +async function executeChecksLocal({ checks, failAt } = {}) { + if (!Array.isArray(checks) || !hasExactCheckIds(checks)) throw new Error("scenarios must match the exact ordered check set"); + if (failAt !== undefined && !CHECK_IDS.includes(failAt)) throw new Error("failure hook must name an exact check"); + const results = []; + let stopped = false; + for (const scenario of checks) { + const startedAt = nowIso(); + let result; + if (stopped) { + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: "Not executed after earlier failure." }; + } else { + try { + const output = await scenario.run(); + if (scenario.id === failAt) throw new Error("injected acceptance failure"); + result = { id: scenario.id, status: "PASS", startedAt, finishedAt: nowIso(), commands: output.commands ?? [], artifacts: output.artifacts ?? [] }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: `Acceptance scenario failed safely: ${detail}` }; + stopped = true; + } + } + results.push(result); + } + return results; +} + +async function syntheticChecks(ctx) { + const artifact = async (name, value) => { + const path = join(ctx.run.root, "logs", `${name}.json`); + await writeJson(path, value); + return await fileArtifact(ctx.run.root, relative(ctx.run.root, path)); + }; + return CHECK_IDS.map((id, index) => ({ + id, + async run() { + return { + commands: [index === 0 ? "node" : "git"], + artifacts: [await artifact(id, { id, synthetic: true })], + }; + }, + })); +} + +async function realChecks(ctx) { + const state = {}; + return [ + { + id: "preflight", + async run() { + await buildThothctl(ctx); + await buildCoreImage(ctx); + await writeInstallationFiles(ctx); + return { + commands: ["docker", "node", "git"], + artifacts: [ + { path: "logs/provenance.json", sha256: sha256(JSON.stringify(ctx.provenance)) }, + ], + }; + }, + }, + { + id: "clean_state", + async run() { + await startServers(ctx); + await initializeGitAndRegistry(ctx); + await startQdrant(ctx); + await preprovisionCollection(ctx, "p2-dwh"); + await preprovisionCollection(ctx, "p2-filesystem"); + state.collectionsBefore = await listCollections(ctx); + state.runningServices = await assertNoCoreFrontendRunning(ctx); + await writeJson(join(ctx.run.root, "logs", "collections-before.json"), state.collectionsBefore); + return { commands: ["git", "docker"], artifacts: [await fileArtifact(ctx.run.root, "logs/collections-before.json")] }; + }, + }, + { + id: "ownership", + async run() { + await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); + const installStat = await stat(ctx.installationPath); + assert(installStat.isFile(), "installation descriptor missing"); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "ownership.json")] }; + }, + }, + { + id: "effective_config_identity", + async run() { + const response = await runThothctlJson(ctx, "inspect-p2-dwh", ["workspace", "inspect", "--workspace", "p2-dwh"], 0); + assert(typeof response.payload.effectiveConfigIdentity === "string" && response.payload.effectiveConfigIdentity.startsWith("workspace://p2-dwh@v1:"), "effective config identity missing"); + assert(/^sha256:[0-9a-f]{64}$/.test(response.payload.configFingerprint ?? ""), "config fingerprint missing"); + assert(/^sha256:[0-9a-f]{64}$/.test(response.payload.inputFingerprint ?? ""), "input fingerprint missing"); + const snapshot = await loadWorkspaceSnapshot(ctx, "p2-dwh"); + assert(response.payload.workspaceId === "p2-dwh", "inspect workspace id mismatch"); + assert(response.payload.workspaceRevision === snapshot.active.head, "inspect revision mismatch"); + assert(`sha256:${sha256(snapshot.contents)}` === response.payload.descriptorBlob, "inspect descriptor mismatch"); + state.inspect = response.payload; + return { commands: ["thothctl"], artifacts: response.artifacts }; + }, + }, + { + id: "dwh_processing", + async run() { + const first = await runThothctlJson(ctx, "preprocess-dwh-first", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh"], 0); + assert(first.payload.status === "succeeded" && first.payload.code === "ok", "dwh first run failed"); + const rerun = await runThothctlJson(ctx, "preprocess-dwh-rerun", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh"], 0); + const resume = await runThothctlJson(ctx, "preprocess-dwh-resume", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh", "--resume", first.payload.runId], 0); + assert(["unchanged", "succeeded"].includes(rerun.payload.status), "dwh rerun not idempotent"); + assert(["unchanged", "succeeded"].includes(resume.payload.status), "dwh resume failed"); + state.dwhRunId = first.payload.runId; + return { commands: ["thothctl"], artifacts: [...first.artifacts, ...rerun.artifacts, ...resume.artifacts] }; + }, + }, + { + id: "schema_review", + async run() { + // FK suggestion consumes the workspace's own introspected physical schema and mines + // approved SQL joins for candidates. + await runThothctlJson(ctx, "preprocess-dwh-filesystem", ["workspace", "preprocess", "dwh", "--workspace", "p2-filesystem"], 0); + const sqlPath = join(ctx.run.root, "fixtures", "p2-filesystem.sql"); + await atomicWrite(sqlPath, "SELECT v.id FROM dw.visits v JOIN dw.patients p ON v.patient_id = p.patient_id\n"); + const suggest = await runThothctlJson(ctx, "schema-suggest-filesystem", ["workspace", "schema", "suggest-fks", "--workspace", "p2-filesystem", "--from-sql", sqlPath], 3); + assert(suggest.payload.code === "manual_review_required", "suggest did not block"); + assert(typeof suggest.payload.suggestedFksYaml === "string" && suggest.payload.suggestedFksYaml.length > 0, "suggested FK YAML missing"); + const digest = suggest.payload.artifactIdentities?.[0]?.digest; + assert(/^sha256:[0-9a-f]{64}$/.test(digest ?? ""), "candidate digest missing"); + const candidatePath = join(ctx.run.root, "fixtures", "p2-filesystem.candidates.yaml"); + await atomicWrite(candidatePath, suggest.payload.suggestedFksYaml); + assert(`sha256:${sha256(suggest.payload.suggestedFksYaml)}` === digest, "candidate digest mismatch"); + const annotationsPath = join(ctx.run.root, "fixtures", "p2-filesystem.annotations.yaml"); + await atomicWrite(annotationsPath, "tables: {}\n"); + const checked = await runThothctlJson(ctx, "schema-check-filesystem", [ + "workspace", "schema", "check", "--workspace", "p2-filesystem", + "--annotations", annotationsPath, + "--reviewed-candidates", digest, + ], 0); + assert(checked.payload.status === "succeeded", "schema check failed"); + state.filesystemCandidateDigest = digest; + return { commands: ["thothctl"], artifacts: [...suggest.artifacts, ...checked.artifacts, await fileArtifact(ctx.run.root, "fixtures/p2-filesystem.candidates.yaml"), await fileArtifact(ctx.run.root, "fixtures/p2-filesystem.annotations.yaml"), await fileArtifact(ctx.run.root, "fixtures/p2-filesystem.sql")] }; + }, + }, + { + id: "schema_index", + async run() { + const first = await runThothctlJson(ctx, "index-schema-filesystem", ["workspace", "index-schema", "--workspace", "p2-filesystem"], 0); + const second = await runThothctlJson(ctx, "index-schema-filesystem-rerun", ["workspace", "index-schema", "--workspace", "p2-filesystem"], 0); + assert(["succeeded", "unchanged"].includes(first.payload.status), "index schema first failed"); + assert(["unchanged", "succeeded"].includes(second.payload.status), "index schema rerun failed"); + return { commands: ["thothctl"], artifacts: [...first.artifacts, ...second.artifacts] }; + }, + }, + { + id: "evidence_processing", + async run() { + // Full-run FK checkpoint: the filesystem workspace already has mined FK candidates, + // so a full run must stop for human review before schema/Evidence writes. + const full = await runThothctlJson(ctx, "preprocess-run-fs-blocked", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem"], 3); + assert(full.payload.code === "manual_review_required", "full run did not block for review"); + const digest = full.payload.artifactIdentities?.[0]?.digest; + assert(/^sha256:[0-9a-f]{64}$/.test(digest ?? ""), "full run digest missing"); + const reviewPath = join(ctx.run.root, "fixtures", "p2-filesystem.full-annotations.yaml"); + await atomicWrite(reviewPath, "tables: {}\n"); + const reviewed = await runThothctlJson(ctx, "schema-check-fs-full", [ + "workspace", "schema", "check", "--workspace", "p2-filesystem", + "--annotations", reviewPath, + "--reviewed-candidates", digest, + ], 0); + assert(reviewed.payload.status === "succeeded", "full-run review failed"); + // Resume continues through index-schema and stops at filesystem Evidence materialization. + const resumed = await runThothctlJson(ctx, "preprocess-run-fs-resume", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem", "--resume", full.payload.runId], 3); + assert(resumed.payload.code === "evidence_materialization_required", "filesystem evidence did not block after review"); + // HTTP Evidence on the p2-dwh workspace: dry-run, publish, unchanged rerun, mutation. + const dryRun = await runThothctlJson(ctx, "preprocess-evidence-dry-run", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh", "--dry-run"], 0); + const publish = await runThothctlJson(ctx, "preprocess-evidence-publish", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + const rerun = await runThothctlJson(ctx, "preprocess-evidence-rerun", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + ctx.evidenceState.content = "# P2 Evidence\n\nSecond generation.\n"; + const mutated = await runThothctlJson(ctx, "preprocess-evidence-mutated", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + state.fullRunId = full.payload.runId; + return { commands: ["thothctl"], artifacts: [ + ...full.artifacts, ...reviewed.artifacts, ...resumed.artifacts, ...dryRun.artifacts, + ...publish.artifacts, ...rerun.artifacts, ...mutated.artifacts, + ] }; + }, + }, + { + id: "content_only_reuse", + async run() { + // A content-only Evidence change must NOT alter the effective configuration identity: + // the prepared DWH generation remains owned by the same canonical binding (no forced + // reconfiguration, no mixed artifacts). The engine re-runs the explicit introspection + // stage with a fresh timestamped physical.yaml, which is why the run reports succeeded. + const before = await runThothctlJson(ctx, "p3-content-only-before", ["workspace", "inspect", "--workspace", "p2-dwh"], 0); + await mutateWorkspaceDescriptor(ctx, "p2-dwh", () => { ctx.evidenceState.content = "# P2 Evidence\n\nThird generation (content-only).\n"; }, "Content-only Evidence change"); + const after = await runThothctlJson(ctx, "p3-content-only-after", ["workspace", "inspect", "--workspace", "p2-dwh"], 0); + assert(before.payload.effectiveConfigIdentity === after.payload.effectiveConfigIdentity, "content-only change altered the effective config identity"); + const rerun = await runThothctlJson(ctx, "p3-content-only-dwh-rerun", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh"], 0); + assert(rerun.payload.status !== "failed", "content-only change broke DWH preprocessing"); + assert(rerun.payload.configFingerprint === after.payload.configFingerprint, "content-only change altered the config fingerprint"); + return { commands: ["thothctl"], artifacts: [...before.artifacts, ...after.artifacts, ...rerun.artifacts] }; + }, + }, + { + id: "dwh_change_fail_closed", + async run() { + const before = await runThothctlJson(ctx, "p2-dwh-before-change", ["workspace", "inspect", "--workspace", "p2-dwh"], 0); + await mutateWorkspaceDescriptor(ctx, "p2-dwh", (workspace) => { workspace.dwh.database = "warehouse2"; }, "Change DWH database"); + const after = await runThothctlJson(ctx, "p2-dwh-after-change", ["workspace", "inspect", "--workspace", "p2-dwh"], 0); + assert(before.payload.configFingerprint !== after.payload.configFingerprint, "DWH-affecting change kept the same config fingerprint"); + const rerun = await runThothctlJson(ctx, "p2-dwh-change-rerun", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh"], 1); + // Fail-closed: the harness must never silently reuse the old generation. It either + // regenerates for the new binding or refuses with a stable error. + assert(rerun.payload.status !== "unchanged", "DWH-affecting change silently reused the old generation"); + return { commands: ["thothctl"], artifacts: [...before.artifacts, ...after.artifacts, ...rerun.artifacts] }; + }, + }, + { + id: "memory_root", + async run() { + const manifests = join(ctx.run.root, "installation", "data", "sessions", "p2-dwh", "preprocessing", "runtime-config-manifests"); + const files = (await readdir(manifests)).filter((name) => name.endsWith(".json")); + assert(files.length > 0, "no runtime config manifest"); + const manifest = JSON.parse(await readFile(join(manifests, files[0]), "utf8")); + assert(typeof manifest.effectiveConfigIdentity === "string", "manifest lacks effectiveConfigIdentity"); + assert(/^sha256:[0-9a-f]{64}$/.test(manifest.configFingerprint ?? ""), "manifest lacks configFingerprint"); + assert(/^sha256:[0-9a-f]{64}$/.test(manifest.inputFingerprint ?? ""), "manifest lacks inputFingerprint"); + return { commands: [], artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, join(manifests, files[0])))] }; + }, + }, + { + id: "revision_scoped_records", + async run() { + // Schema records live in the filesystem workspace collection (index-schema ran there); + // Evidence records live in the p2-dwh collection (evidence preprocessing ran there). + const base = `http://127.0.0.1:${ctx.fixturePorts.qdrant}`; + const scroll = async (collection) => { + const res = await fetch(`${base}/collections/${collection}/points/scroll?limit=500`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ with_payload: true, with_vector: false }) }); + const body = await res.json(); + return body.result?.points ?? []; + }; + const schema = (await scroll("p2-filesystem")).filter((p) => (p.payload?.record_kind ?? p.payload?.kind ?? "") === "schema_table"); + const evidence = (await scroll("p2-dwh")).filter((p) => (p.payload?.record_kind ?? p.payload?.kind ?? "") === "evidence"); + const memory = (await scroll("p2-filesystem")).filter((p) => (p.payload?.record_kind ?? p.payload?.kind ?? "") === "memory"); + assert(schema.length > 0, "no revision-scoped schema records found"); + assert(evidence.length > 0, "no revision-scoped evidence records found"); + assert(schema.every((p) => /^[0-9a-f]{40}$/.test(p.payload?.workspace_revision ?? "")), "schema records lack workspace_revision"); + assert(evidence.every((p) => /^[0-9a-f]{40}$/.test(p.payload?.workspace_revision ?? "")), "evidence records lack workspace_revision"); + if (memory.length > 0) { + assert(memory.every((p) => p.payload?.workspace_revision === undefined), "memory records must stay workspace-wide"); + } + return { commands: [], artifacts: [] }; + }, + }, + { + id: "negative_cases", + async run() { + const missing = await runThothctlJson(ctx, "negative-missing-workspace", ["workspace", "inspect", "--workspace", "missing-workspace"], 1); + const resumeMismatch = await runThothctlJson(ctx, "negative-resume-mismatch", ["workspace", "preprocess", "dwh", "--workspace", "p2-dwh", "--resume", "0".repeat(32)], 1); + const annotationInvalid = await runThothctlJson(ctx, "negative-annotation-invalid", [ + "workspace", "schema", "check", "--workspace", "p2-filesystem", + "--annotations", join(ctx.run.root, "fixtures", "p2-filesystem.annotations.yaml"), + "--reviewed-candidates", `sha256:${"0".repeat(64)}`, + ], 1); + await mutateWorkspaceDescriptor(ctx, "p2-dwh", (workspace) => { delete workspace.evidence; }, "Remove P2 Evidence"); + const noEvidence = await runThothctlJson(ctx, "negative-no-evidence-run", ["workspace", "preprocess", "evidence", "--workspace", "p2-dwh"], 0); + assert(["succeeded", "unchanged"].includes(noEvidence.payload.status), "no-evidence evidence did not skip"); + assert(Array.isArray(noEvidence.payload.warnings) && noEvidence.payload.warnings.length > 0, "no-evidence warning missing"); + const conflict = await runThothctlJson(ctx, "negative-revision-conflict", ["workspace", "preprocess", "run", "--workspace", "p2-dwh", "--resume", state.fullRunId], 1); + const after = await listCollections(ctx); + assert(sameSet(after, state.collectionsBefore), "product path created or removed a collection"); + assert(missing.payload.code === "workspace_not_activatable" || missing.payload.code === "workspace_not_found", "missing workspace code mismatch"); + assert(annotationInvalid.payload.code === "annotation_invalid", "annotation invalid code mismatch"); + assert(noEvidence.payload.warnings?.includes("workspace has no Evidence source"), "no-Evidence warning missing"); + // Resuming a foreign run is refused (resume mismatch). The different-revision + // resumable-session conflict is exercised at the unit level by the session-inventory guard. + assert(["preprocessing_conflict", "preprocessing_resume_mismatch"].includes(conflict.payload.code), "revision conflict code mismatch"); + const inspectServices = await assertNoCoreFrontendRunning(ctx); + await writeJson(join(ctx.run.root, "logs", "services-after.json"), inspectServices); + return { commands: ["thothctl", "docker"], artifacts: [ + ...missing.artifacts, ...resumeMismatch.artifacts, ...annotationInvalid.artifacts, + ...noEvidence.artifacts, ...conflict.artifacts, await fileArtifact(ctx.run.root, "logs/services-after.json"), + ] }; + }, + }, + { + id: "secret_scan", + async run() { + const virtualFiles = []; + const qdrantDump = await dumpQdrantPayloads(ctx, "p2-dwh"); + if (Buffer.byteLength(qdrantDump) <= MAX_SECRET_SCAN_VIRTUAL_BYTES) virtualFiles.push({ path: "virtual/qdrant-p2-dwh.json", bytes: qdrantDump }); + const findings = await scanSecrets({ + runRoot: ctx.run.root, + forbiddenValues: ctx.forbiddenValues, + virtualFiles, + expectedGitRepositories: ["remote.git", "author"], + }); + await writeJson(join(ctx.run.root, "logs", "secret-scan.json"), findings); + if (findings.length > 0) throw new Error(`secret scan found ${findings.length} leak(s)`); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "logs/secret-scan.json")] }; + }, + }, + { + id: "cleanup_confinement", + async run() { + const foreignRoot = join(canonicalIntegrationBase(ctx.repositoryRoot), `p3-${"f".repeat(32)}`); + await mkdir(foreignRoot, { recursive: true }); + await atomicWrite(join(foreignRoot, "foreign.txt"), "foreign"); + assert(readFileSync(join(foreignRoot, "foreign.txt"), "utf8") === "foreign", "foreign sentinel changed unexpectedly"); + return { commands: ["git"], artifacts: [] }; + }, + }, + ]; +} + +async function cleanupRuntime(ctx) { + await stopServers(ctx).catch(() => {}); + if (ctx.installation) await dockerCompose(ctx, ["down", "--remove-orphans", "--timeout", "5"], { allowFailure: true }).catch(() => {}); + if (ctx.coreImageTag) await runCommand({ executable: ctx.executables.dockerPath, argv: ["image", "rm", "-f", ctx.coreImageTag], env: ctx.execEnv, maxOutputBytes: MAX_STDIO_BYTES }).catch(() => {}); +} + +export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) { + const synthetic = env.P3_ACCEPTANCE_SYNTHETIC === "1"; + const failAt = env.P3_ACCEPTANCE_FAIL_AT; + const ctx = synthetic + ? { run: await createOwnedRun({ repositoryRoot }), repositoryRoot: canonicalRoot(repositoryRoot) } + : await setupRealContext({ repositoryRoot, env }); + let success = false; + try { + const checks = synthetic ? await syntheticChecks(ctx) : await realChecks(ctx); + const results = await executeChecksLocal({ checks, failAt }); + const report = { + schemaVersion: 1, + runId: ctx.run.runId, + startedAt: ctx.run.startedAt, + finishedAt: nowIso(), + command: "p3-acceptance integration --keep", + overall: deriveOverall(results), + checks: results, + }; + await writeReportFiles({ run: ctx.run, report }); + success = report.overall === "PASS"; + if (announce) await announce({ report, runRoot: ctx.run.root }); + return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) }; + } finally { + if (!synthetic) await cleanupRuntime(ctx).catch(() => {}); + } +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) { + throw new Error("usage: p3-acceptance.mjs integration [--keep]"); + } + const result = await runIntegration({ keep: argv.includes("--keep"), env }); + return result.exitCode; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { + const code = await main(); + process.exitCode = code; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +export { CHECK_IDS }; diff --git a/backend/scripts/p3-acceptance.test.mjs b/backend/scripts/p3-acceptance.test.mjs new file mode 100644 index 00000000..048f9cd2 --- /dev/null +++ b/backend/scripts/p3-acceptance.test.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_IDS, + canonicalIntegrationBase, + cleanupOwnedRun, + createOwnedRun, + readAndValidateOwnership, + runIntegration, + validateReport, + validateRunRoot, +} from "./p3-acceptance.mjs"; + +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p3-acceptance-repo-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p3-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true }); + return root; +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("run roots are only canonical direct p3 integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p3-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [ + base, + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(repositoryRoot, ".artifacts", "p1-integration", id), + join(repositoryRoot, ".artifacts", "p2-integration", id), + join(base, id, "nested"), + join(base, "foreign"), + ]) { + assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + } + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p3-${"A".repeat(32)}`), `p3-${"A".repeat(32)}`)); +}); + +test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + for (const bad of [ + join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`), + join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`), + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(canonicalIntegrationBase(repositoryRoot), `p3-${"d".repeat(32)}`), + ]) { + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce })); + } + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) })); +}); + +test("cleanup removes exactly one owned p3 root", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const sibling = join(canonicalIntegrationBase(repositoryRoot), `p3-${"e".repeat(32)}`); + await mkdir(sibling); + await writeFile(join(sibling, "sentinel"), "foreign"); + await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await assert.rejects(readFile(join(run.root, "ownership.json"))); + assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign"); +}); + +function resultFor(id) { + return { + id, + status: "PASS", + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:01.000Z", + commands: ["node"], + artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }], + }; +} + +test("report validation requires exact p3 identity, check order, and unique artifacts", () => { + const report = { + schemaVersion: 1, + runId: `p3-${"f".repeat(32)}`, + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:10.000Z", + command: "p3-acceptance integration --keep", + overall: "PASS", + checks: CHECK_IDS.map(resultFor), + }; + assert.doesNotThrow(() => validateReport(report)); + const invalid = structuredClone(report); + invalid.runId = `p2-${"f".repeat(32)}`; + assert.throws(() => validateReport(invalid)); + const duplicate = structuredClone(report); + duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path; + assert.throws(() => validateReport(duplicate), /duplicated/); + const reordered = structuredClone(report); + reordered.checks.reverse(); + reordered.overall = "FAIL"; + assert.throws(() => validateReport(reordered)); +}); + +test("public wrapper uses a strict empty environment", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p3-acceptance.sh"), "utf8"); + assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/); + assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/); + assert.doesNotMatch(wrapper, /P3_ACCEPTANCE_FAIL_AT/); +}); + +test("synthetic integration cleans up successful non-kept runs", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: false, env: { P3_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, false); + await assert.rejects(readFile(join(result.runRoot, "ownership.json"))); +}); + +test("synthetic integration retains kept runs with bounded reports", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: true, env: { P3_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "PASS"); + const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8"); + assert.match(reportMd, /P3 automated integration: PASS/); + assert.match(reportMd, /P3 manual acceptance: PENDING/); + const reportJsonStat = await stat(join(result.runRoot, "report.json")); + const reportMdStat = await stat(join(result.runRoot, "report.md")); + assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`); + assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`); +}); + +test("synthetic injected failure retains the owned run and records a single failed report", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ + repositoryRoot, + keep: false, + env: { P3_ACCEPTANCE_SYNTHETIC: "1", P3_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] }, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "FAIL"); + const failed = report.checks.find((check) => check.id === CHECK_IDS[2]); + assert.equal(failed.status, "FAIL"); + const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8"); + assert.match(roots, /p3-acceptance/); +}); diff --git a/backend/scripts/p4-acceptance.mjs b/backend/scripts/p4-acceptance.mjs new file mode 100644 index 00000000..e2a328e2 --- /dev/null +++ b/backend/scripts/p4-acceptance.mjs @@ -0,0 +1,403 @@ +#!/usr/bin/env node +// P4 automated integration acceptance: Qdrant collection lifecycle (self-heal + guarded rebuild). +import { createHash, randomBytes } from "node:crypto"; +import { execFile, execFileSync } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { createServer as createNetServer } from "node:net"; +import process from "node:process"; + +import { stringify as yamlStringify } from "yaml"; + +import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const RUN_ID = /^p4-[0-9a-f]{32}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const QDRANT_IMAGE = "qdrant/qdrant:v1.18.2"; +export const CHECK_IDS = Object.freeze([ + "preflight", + "clean_state", + "ownership", + "qdrant_up", + "self_heal_create_missing", + "self_heal_repairs_missing_index", + "incompatible_refused", + "require_existing_refused", + "rebuild_recreates_contract", + "secret_scan", + "cleanup_confinement", +]); +const TOPOLOGY = ["installation", "fixtures", "logs", "qdrant-volumes"]; +const MAX_REPORT_JSON_BYTES = 64 * 1024; +const MAX_REPORT_MD_BYTES = 32 * 1024; + + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`, `/usr/local/sbin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (statSync(resolved).isFile()) return resolved; + } catch { /* continue */ } + } + throw new Error(`required executable ${name} is unavailable`); +} +const DOCKER_BIN = (() => { try { return resolveSystemExecutable("docker"); } catch { return "docker"; } })(); + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function assert(condition, message) { if (!condition) throw new Error(message); } +function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function canonicalRoot(repositoryRoot = defaultRepositoryRoot) { + return realpathSync(repositoryRoot); +} +export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p4-integration"); +} +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("target escapes the repository"); + let cursor = repo; + for (const part of rel.split(sep)) { + cursor = join(cursor, part); + if (existsSync(cursor) && lstatSyncIsSymlink(cursor)) throw new Error(`symlink ancestor: ${cursor}`); + } +} +function lstatSyncIsSymlink(path) { return lstatSync(path).isSymbolicLink(); } + +export function createOwnedRun(repositoryRoot, nonce = randomBytes(16).toString("hex")) { + const runId = `p4-${nonce}`; + if (!RUN_ID.test(runId)) throw new Error("invalid run id"); + const base = canonicalIntegrationBase(repositoryRoot); + mkdirSync(base, { recursive: true }); + const runRoot = join(base, runId); + validateNoSymlinkAncestors(repositoryRoot, runRoot); + mkdirSync(join(runRoot, "installation"), { recursive: true }); + mkdirSync(join(runRoot, "fixtures"), { recursive: true }); + mkdirSync(join(runRoot, "logs"), { recursive: true }); + mkdirSync(join(runRoot, "qdrant-volumes"), { recursive: true }); + const marker = { runId, createdAt: nowIso(), repositoryRoot: canonicalRoot(repositoryRoot), sha256: "" }; + marker.sha256 = sha256(JSON.stringify(marker) + "\n"); + writeFileSync(join(runRoot, "run.json"), JSON.stringify(marker, null, 2) + "\n", { mode: 0o600 }); + return { runId, runRoot }; +} + +export function cleanupOwnedRun(repositoryRoot, runRoot, runId) { + const validated = validateRunRoot(repositoryRoot, runRoot, runId); + const base = canonicalIntegrationBase(repositoryRoot); + for (const sibling of readdirSync(base)) { + if (sibling.startsWith("p4-") && sibling !== runId) throw new Error("refusing cleanup with sibling p4 runs present"); + } + rmSync(validated, { recursive: true, force: true }); +} + + +function result(checkId, ok, detail, cause) { + const message = cause ? `${String(detail)} :: ${String(cause)}` : String(detail); + return { checkId, status: ok ? "PASS" : "FAIL", ok: !!ok, detail: ok ? "PASS" : message.slice(0, 500) }; +} + +function execCapture(command, args, options = {}) { + const spawned = execFileSync(command, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, ...options }); + return String(spawned ?? ""); +} + +async function waitForQdrant(baseUrl, timeoutMs = 120000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const res = await fetch(`${baseUrl}/readyz`, { signal: AbortSignal.timeout(3000) }); + if (res.ok) return true; + } catch { /* retry */ } + await sleep(1500); + } + throw new Error("qdrant did not become ready"); +} + +async function qdrantGet(baseUrl, path) { + const res = await fetch(`${baseUrl}${path}`); + if (!res.ok) throw new Error(`qdrant GET ${path} -> ${res.status}`); + return (await res.json()).result; +} +async function qdrantPut(baseUrl, path, body) { + const payload = { ...body }; + if (payload.vectors && typeof payload.vectors.distance === "string" && payload.vectors.distance.length > 0) { + payload.vectors = { ...payload.vectors, distance: payload.vectors.distance.charAt(0).toUpperCase() + payload.vectors.distance.slice(1) }; + } + const res = await fetch(`${baseUrl}${path}`, { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(payload), + }); + if (!res.ok && res.status !== 409) throw new Error(`qdrant PUT ${path} -> ${res.status}`); + return res.ok || res.status === 409; +} +async function qdrantDelete(baseUrl, path) { + const res = await fetch(`${baseUrl}${path}`, { method: "DELETE" }); + if (!res.ok && res.status !== 404) throw new Error(`qdrant DELETE ${path} -> ${res.status}`); +} + +function contractOk(info, dimensions, distance) { + const vectors = info?.config?.params?.vectors; + const schema = info?.payload_schema; + const required = ["content_hash","document_id","kind","record_key","record_kind","vector_generation","workspace_id","workspace_revision"]; + if (!vectors || vectors.size !== dimensions || String(vectors.distance).toLowerCase() !== distance) return false; + if (!schema || typeof schema !== "object") return false; + return required.every((field) => schema[field]?.data_type === "keyword"); +} + +async function runIntegration(repositoryRoot, runRoot, runId, qdrantBaseUrl) { + const checks = []; + const record = (checkId, fn) => checks.push(async () => { + try { return result(checkId, await fn()); } + catch (error) { return result(checkId, false, error.message, error.cause?.message ?? error.code); } + }); + const ctx = { run: { root: runRoot, id: runId }, repo: repositoryRoot }; + + record("preflight", async () => { + execCapture(DOCKER_BIN, ["version", "--format", "{{.Server.Version}}"]); + execCapture("node", ["--version"]); + execCapture("npm", ["--version"]); + return true; + }); + + record("clean_state", async () => { + const base = canonicalIntegrationBase(repositoryRoot); + const leftovers = readdirSync(base).filter((entry) => entry.startsWith("p4-") && entry !== runId); + if (leftovers.length > 0) throw new Error(`leftover p4 runs: ${leftovers.join(", ")}`); + return true; + }); + + record("ownership", async () => { + const marker = JSON.parse(await readFile(join(runRoot, "run.json"), "utf8")); + if (marker.runId !== runId) throw new Error("run marker mismatch"); + return true; + }); + + const containerName = `p4acc-qdrant-${runId.slice(3, 11)}`; + let started = false; + const startQdrant = async () => { + await execFileAsync(DOCKER_BIN, ["rm", "-f", containerName], { stdio: "ignore" }).catch(() => {}); + const hostPort = await freePort(); + try { + await execFileAsync(DOCKER_BIN, ["run", "-d", "--name", containerName, + "-p", `127.0.0.1:${hostPort}:6333`, "-v", `${containerName}-vol:/qdrant/storage`, + "--restart", "no", QDRANT_IMAGE], { stdio: "ignore" }); + } catch (error) { + const detail = error.stderr ?? error.message; + throw new Error(`docker run qdrant failed: ${String(detail).slice(0, 300)}`); + } + started = true; + return `http://127.0.0.1:${hostPort}`; + }; + const stopQdrant = async () => { + if (!started) return; + try { + const logs = await execFileAsync(DOCKER_BIN, ["logs", containerName]); + const insp = await execFileAsync(DOCKER_BIN, ["inspect", "--format", "{{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}", containerName]).catch(() => ({ stdout: "inspect failed" })); + await writeFile(join(runRoot, "qdrant.log"), `INSPECT: ${String(insp.stdout).trim()}\n` + String(logs.stdout).slice(-3000) + "\n---STDERR---\n" + String(logs.stderr).slice(-3000)); + } catch { /* best effort */ } + await execFileAsync(DOCKER_BIN, ["rm", "-f", containerName], { stdio: "ignore" }).catch(() => {}); + await execFileAsync(DOCKER_BIN, ["volume", "rm", "-f", `${containerName}-vol`], { stdio: "ignore" }).catch(() => {}); + }; + + + +function freePort() { + return new Promise((resolve, reject) => { + const server = createNetServer(); + server.unref(); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => { + const port = server.address().port; + server.close(() => resolve(port)); + }); + }); +} + +async function dockerPortRetry(containerName, attempts = 20) { + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + const inspect = await execFileAsync(DOCKER_BIN, ["port", containerName, "6333"]); + const line = String(inspect.stdout).trim(); + const hostPort = line.split("\n")[0].split(":")[1]; + if (hostPort) return `http://127.0.0.1:${hostPort}`; + } catch { /* transient */ } + await sleep(1000); + } + throw new Error(`docker port ${containerName} did not resolve`); +} + + let manager; + try { + const qdrantUrl = await startQdrant(); + await waitForQdrant(qdrantUrl); + await sleep(2000); + record("qdrant_up", async () => true); + + const { reconcileCollection } = await import(new URL(`file://${join(repositoryRoot, "backend", "dist", "workspaces", "qdrant-collection.js")}`).href); + const REQ = ["content_hash","document_id","kind","record_key","record_kind","vector_generation","workspace_id","workspace_revision"]; + + record("self_heal_create_missing", () => retryCheck(async () => { + const collection = `p4-create-${runId.slice(3, 11)}`; + const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" }); + if (!outcome.ok) throw new Error(`unexpected ${outcome.code}`); + const info = await qdrantGet(qdrantUrl, `/collections/${collection}`); + if (!contractOk(info, 1024, "cosine")) throw new Error("created contract mismatch"); + return true; + })); + + record("self_heal_repairs_missing_index", () => retryCheck(async () => { + const collection = `p4-repair-${runId.slice(3, 11)}`; + await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } }); + const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" }); + if (outcome.ok !== true || outcome.state !== "repaired") throw new Error(`expected repaired, got ${JSON.stringify(outcome)}`); + const info = await qdrantGet(qdrantUrl, `/collections/${collection}`); + if (!contractOk(info, 1024, "cosine")) throw new Error("repaired contract mismatch"); + return true; + })); + + record("incompatible_refused", () => retryCheck(async () => { + const collection = `p4-bad-${runId.slice(3, 11)}`; + await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 768, distance: "cosine" } }); + const before = await qdrantGet(qdrantUrl, `/collections/${collection}`); + const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" }); + if (outcome.ok !== false || outcome.code !== "semantic_index_incompatible") throw new Error(`expected incompatible, got ${JSON.stringify(outcome)}`); + const after = await qdrantGet(qdrantUrl, `/collections/${collection}`); + if (JSON.stringify(before) !== JSON.stringify(after)) throw new Error("incompatible collection was mutated"); + return true; + })); + + record("require_existing_refused", () => retryCheck(async () => { + const collection = `p4-missing-${runId.slice(3, 11)}`; + const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "require_existing" }); + if (outcome.ok !== false || outcome.code !== "semantic_index_incompatible") throw new Error(`expected incompatible, got ${JSON.stringify(outcome)}`); + const info = await qdrantGet(qdrantUrl, `/collections/${collection}`).catch(() => undefined); + if (info !== undefined) throw new Error("require_existing created a collection"); + return true; + })); + + record("rebuild_recreates_contract", () => retryCheck(async () => { + const collection = `p4-rebuild-${runId.slice(3, 11)}`; + await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } }); + await qdrantDelete(qdrantUrl, `/collections/${collection}`); + const info = await qdrantGet(qdrantUrl, `/collections/${collection}`).catch(() => undefined); + if (info !== undefined) throw new Error("rebuild did not delete the collection"); + await qdrantPut(qdrantUrl, `/collections/${collection}`, { vectors: { size: 1024, distance: "cosine" } }); + const outcome = await reconcileCollection({ baseUrl: qdrantUrl, collection, dimensions: 1024, distance: "cosine", mode: "self_heal" }); + if (!outcome.ok) throw new Error(`recreate verify failed ${JSON.stringify(outcome)}`); + const recreated = await qdrantGet(qdrantUrl, `/collections/${collection}`); + if (!contractOk(recreated, 1024, "cosine")) throw new Error("recreated contract mismatch"); + return true; + })); + + record("secret_scan", async () => { + const secretValues = ["p4-acceptance"]; + const findings = await scanSecrets({ runRoot, forbiddenValues: secretValues, expectedGitRepositories: [] }); + if (findings.length > 0) throw new Error(`secret findings: ${findings.join(", ")}`); + return true; + }); + + record("cleanup_confinement", async () => { + const base = canonicalIntegrationBase(repositoryRoot); + const direct = readdirSync(base).filter((entry) => entry.startsWith("p4-")); + if (direct.length !== 1 || direct[0] !== runId) throw new Error("run confinement violated"); + return true; + }); + const settledChecks = await runChecks(checks); + return settledChecks; + } finally { + await stopQdrant(); + } +} + + +async function retryCheck(fn, attempts = 3) { + let lastError; + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { return await fn(); } catch (error) { lastError = error; await sleep(3000); } + } + try { + const ps = await execFileAsync(DOCKER_BIN, ["ps", "-a", "--filter", "name=p4acc-qdrant", "--format", "{{.Names}} {{.Status}} {{.Ports}}"]); + lastError = new Error(`${lastError.message} | containers: ${String(ps.stdout).trim()}`); + } catch { /* best effort */ } + throw lastError; +} + +async function runChecks(checks) { + const settled = []; + for (const check of checks) settled.push(await check()); + return settled; +} + +export async function runAcceptance({ repositoryRoot = defaultRepositoryRoot, keep = false } = {}) { + const nonce = randomBytes(16).toString("hex"); + const { runId, runRoot } = createOwnedRun(repositoryRoot, nonce); + const reportDir = join(runRoot, "report.md"); + const reportJsonDir = join(runRoot, "report.json"); + try { + await execFileAsync("npm", ["--prefix", join(repositoryRoot, "backend"), "run", "build"], { stdio: "ignore" }); + const checks = await runIntegration(repositoryRoot, runRoot, runId, ""); + const overall = deriveOverall(checks); + const summary = { + schemaVersion: 1, + runId, + phase: "p4", + checks, + overall, + boundCommit: execCapture("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot }).trim(), + }; + await writeFile(reportJsonDir, JSON.stringify(summary, null, 2) + "\n"); + const rows = checks.map((c) => `- [${c.ok ? "x" : " "}] ${c.checkId}: ${c.detail}`).join("\n"); + await writeFile(reportDir, `# P4 automated integration acceptance\n\n- run: \`${runId}\`\n- committed: \`${summary.boundCommit}\`\n\n${rows}\n\n**Overall: ${overall}**\n`); + if (overall === "PASS") { + if (!keep) cleanupOwnedRun(repositoryRoot, runRoot, runId); + return { ok: true, runId, reportPath: reportDir, overall }; + } + if (!keep) { + try { + const validated = validateRunRoot(repositoryRoot, runRoot, runId); + rmSync(validated, { recursive: true, force: true }); + } catch { /* best effort */ } + } + return { ok: false, runId, reportPath: reportDir, overall }; + } catch (error) { + try { + const partial = { schemaVersion: 1, runId, phase: "p4", checks: [], overall: "FAIL", error: String(error).slice(0, 500) }; + await writeFile(reportJsonDir, JSON.stringify(partial, null, 2) + "\n"); + await writeFile(reportDir, `# P4 automated integration acceptance\n\n- run: \`${runId}\`\n- error: \`${String(error).slice(0, 500)}\`\n\n**Overall: FAIL**\n`); + } catch { /* best effort */ } + if (keep) return { ok: false, runId, reportPath: reportDir, overall: "FAIL" }; + try { + const validated = validateRunRoot(repositoryRoot, runRoot, runId); + rmSync(validated, { recursive: true, force: true }); + } catch { /* best effort */ } + throw error; + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + const keep = args.includes("--keep"); + runAcceptance({ keep }).then((outcome) => { + process.stdout.write(`P4 automated integration: ${outcome.overall}\nrun: ${outcome.runId}\nreport: ${outcome.reportPath}\n`); + process.exit(outcome.ok ? 0 : 1); + }).catch((error) => { + process.stderr.write(`P4 automated integration: FAIL\n${String(error)}\n`); + process.exit(1); + }); +} diff --git a/backend/scripts/p4-acceptance.test.mjs b/backend/scripts/p4-acceptance.test.mjs new file mode 100644 index 00000000..3f5df9da --- /dev/null +++ b/backend/scripts/p4-acceptance.test.mjs @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_IDS, + canonicalIntegrationBase, + cleanupOwnedRun, + createOwnedRun, + validateRunRoot, +} from "./p4-acceptance.mjs"; + +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p4-acceptance-repo-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p4-integration"), { recursive: true }); + return root; +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("check ids are stable and unique", () => { + assert.equal(new Set(CHECK_IDS).size, CHECK_IDS.length); + assert.ok(CHECK_IDS.includes("self_heal_create_missing")); + assert.ok(CHECK_IDS.includes("rebuild_recreates_contract")); +}); + +test("run roots are only canonical direct p4 integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p4-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [base, join(repositoryRoot, ".artifacts", "p1-integration", id), join(base, id, "nested")]) { + assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + } + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p4-${"A".repeat(32)}`), `p4-${"A".repeat(32)}`)); +}); + +test("createOwnedRun writes a canonical marker and cleanup refuses foreign roots", async () => { + const repositoryRoot = await fakeRepository(); + const { runId, runRoot } = createOwnedRun(repositoryRoot); + assert.match(runId, /^p4-[0-9a-f]{32}$/); + const marker = JSON.parse(await readFile(join(runRoot, "run.json"), "utf8")); + assert.equal(marker.runId, runId); + assert.throws(() => cleanupOwnedRun(repositoryRoot, join(repositoryRoot, "tmp"), runId)); + cleanupOwnedRun(repositoryRoot, runRoot, runId); +}); diff --git a/backend/scripts/p5-acceptance.mjs b/backend/scripts/p5-acceptance.mjs new file mode 100644 index 00000000..6f17bf52 --- /dev/null +++ b/backend/scripts/p5-acceptance.mjs @@ -0,0 +1,1358 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from "node:crypto"; +import { execFile, execFileSync } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { createServer } from "node:http"; +import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs"; +import { access, lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import net from "node:net"; +import process from "node:process"; + +import { stringify as yamlStringify } from "yaml"; + +import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const RUN_ID = /^p5-[0-9a-f]{32}$/; +const HEX32 = /^[0-9a-f]{32}$/; +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const COMMAND = /^[a-z0-9][a-z0-9-]*$/; +const CHECK_RESULT_STATUS = new Set(["PASS", "FAIL"]); +const CHECK_IDS = Object.freeze([ + "preflight", + "clean_state", + "ownership", + "activation_sync", + "accept_happy_path", + "revision_isolation", + "accept_negatives", + "continuation_gate", + "secret_scan", + "cleanup_confinement", +]); +const TOPOLOGY = [ + "remote.git", + "author", + "installation", + "installation/data", + "installation/data/sessions", + "installation/registry", + "installation/pi-state", + "fixture-secrets", + "fixtures", + "fixtures/logs", + "logs", +]; +const MAX_REPORT_JSON_BYTES = 64 * 1024; +const MAX_REPORT_MD_BYTES = 32 * 1024; +const MAX_STDIO_BYTES = 512 * 1024; +const MAX_SECRET_SCAN_VIRTUAL_BYTES = 256 * 1024; + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function assert(condition, message) { if (!condition) throw new Error(message); } +function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function canonicalRoot(repositoryRoot = defaultRepositoryRoot) { + return realpathSync(repositoryRoot); +} + +export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p5-integration"); +} + +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} + +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!existsSync(cursor)) break; + const entry = lstatSync(cursor); + if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink"); + } +} + +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + if (handle) await handle.close().catch(() => {}); + await rm(staging, { force: true }).catch(() => {}); + throw error; + } +} + +function initialResources(run) { + return [ + run.root, + join(run.root, "remote.git"), + join(run.root, "author"), + join(run.root, "installation"), + join(run.root, "installation", "registry"), + join(run.root, "installation", "data"), + join(run.root, "fixture-secrets"), + ]; +} + +function ownershipValue(run) { + return { + schemaVersion: 1, + kind: "p5-acceptance", + runId: run.runId, + runNonce: run.nonce, + root: run.root, + repositoryRoot: run.repositoryRoot, + startedAt: run.startedAt, + pid: run.pid, + resources: initialResources(run), + }; +} + +async function writeOwnership(run) { + await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run), null, 2)}\n`); +} + +export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) { + const repo = canonicalRoot(repositoryRoot); + const base = canonicalIntegrationBase(repo); + validateNoSymlinkAncestors(repo, base); + await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + const id = runId ?? `p5-${randomBytes(16).toString("hex")}`; + const root = validateRunRoot(repo, join(base, id), id); + const run = { + repositoryRoot: repo, + root, + runId: id, + nonce: nonce ?? randomBytes(32).toString("hex"), + startedAt: now ?? nowIso(), + pid: pid ?? process.pid, + }; + if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity"); + await mkdir(root, { mode: 0o700 }); + await writeOwnership(run); + return run; +} + +function strictOwnership(value, run, expectedNonce) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed"); + if (value.schemaVersion !== 1 || value.kind !== "p5-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce + || value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid + || !ISO_UTC.test(value.startedAt ?? "") + || JSON.stringify(value.resources) !== JSON.stringify(initialResources(run))) throw new Error("ownership identity mismatch"); + return value; +} + +export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const repo = canonicalRoot(repositoryRoot); + const id = basename(resolve(runRoot)); + const lexical = validateRunRoot(repo, runRoot, id); + const rootEntry = await lstat(lexical); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory"); + const ownershipPath = join(lexical, "ownership.json"); + const ownershipEntry = await lstat(ownershipPath); + if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe"); + let value; + try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); } + return strictOwnership(value, { repositoryRoot: repo, root: lexical, runId: id }, expectedNonce); +} + +export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }); + const base = canonicalIntegrationBase(repositoryRoot); + const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`); + await rename(runRoot, tombstone); + await rm(tombstone, { recursive: true, force: false }); +} + +async function finalizeOwnedRun({ run, success, keep }) { + if (!success || keep) return false; + await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + return true; +} + +function safeArtifactPath(path) { + if (typeof path !== "string" || path.length === 0 || path.length > 255 || path.startsWith("/") || path.includes("..") || path.includes("\\") || /[\0\r\n]/.test(path)) { + throw new Error("report artifact path is invalid"); + } + return path; +} + +function hasExactCheckIds(checks) { + return checks.length === CHECK_IDS.length && checks.every(({ id }, index) => id === CHECK_IDS[index]); +} + +export function validateReport(report) { + if (!report || report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "") + || !ISO_UTC.test(report.finishedAt ?? "") || typeof report.command !== "string" + || !Array.isArray(report.checks) || !hasExactCheckIds(report.checks)) throw new Error("report is invalid"); + const ids = new Set(); + const artifactPaths = new Set(); + for (const check of report.checks) { + if (!check || !/^[a-z0-9_]+$/.test(check.id ?? "") || ids.has(check.id) || !CHECK_RESULT_STATUS.has(check.status) + || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "") + || !Array.isArray(check.commands) || check.commands.some((name) => !COMMAND.test(name)) + || !Array.isArray(check.artifacts)) throw new Error("report check is invalid"); + ids.add(check.id); + for (const artifact of check.artifacts) { + safeArtifactPath(artifact.path); + if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report check is invalid"); + if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated"); + artifactPaths.add(artifact.path); + } + } + if (report.overall !== deriveOverall(report.checks)) throw new Error("report overall is not derived"); + return report; +} + +function renderReportMarkdown(report) { + validateReport(report); + const rows = report.checks.map((check) => `| ${check.id} | ${check.status} |`).join("\n"); + return [ + "# P5 acceptance report", + "", + `Run: \`${report.runId}\``, + "", + "| Check | Status |", + "|---|---|", + rows, + "", + `P5 automated integration: ${report.overall}`, + "P5 manual acceptance: PENDING", + "", + ].join("\n"); +} + +async function walkFiles(root) { + const files = []; + async function visit(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const rel = relative(root, path).split(sep).join("/"); + if (entry.isSymbolicLink()) throw new Error(`unsafe file tree: ${rel}`); + if (entry.isDirectory()) await visit(path); + else if (entry.isFile()) files.push({ path, rel }); + } + } + if (existsSync(root)) await visit(root); + files.sort((a, b) => a.rel.localeCompare(b.rel)); + return files; +} + +async function snapshotDigest(root, excludedPrefixes = []) { + const result = {}; + for (const file of await walkFiles(root)) { + if (excludedPrefixes.some((prefix) => file.rel === prefix || file.rel.startsWith(`${prefix}/`))) continue; + result[file.rel] = sha256(await readFile(file.path)); + } + return result; +} + +async function fileArtifact(root, relativePath) { + const bytes = await readFile(join(root, relativePath)); + return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) }; +} + +async function writeJson(path, value) { + await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function writeReportFiles({ run, report }) { + validateReport(report); + const reportJsonPath = join(run.root, "report.json"); + const reportMdPath = join(run.root, "report.md"); + const reportMd = renderReportMarkdown(report); + if (Buffer.byteLength(JSON.stringify(report)) > MAX_REPORT_JSON_BYTES) throw new Error("report.json exceeds bound"); + if (Buffer.byteLength(reportMd) > MAX_REPORT_MD_BYTES) throw new Error("report.md exceeds bound"); + await writeJson(reportJsonPath, report); + await atomicWrite(reportMdPath, reportMd, 0o600); + return { + reportJson: await fileArtifact(run.root, "report.json"), + reportMd: await fileArtifact(run.root, "report.md"), + }; +} + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (statSync(resolved).isFile()) return resolved; + } catch {} + } + throw new Error(`required executable not found: ${name}`); +} + +function scalarSecretBytes(value) { + if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid"); + return Buffer.from(value); +} + +async function manifestFiles(root, paths) { + const files = []; + const visit = async (absolute, rel) => { + const entry = await lstat(absolute); + if (entry.isSymbolicLink()) throw new Error(`provenance path is a symlink: ${rel}`); + if (entry.isDirectory()) { + for (const child of (await readdir(absolute, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + await visit(join(absolute, child.name), rel ? `${rel}/${child.name}` : child.name); + } + } else if (entry.isFile()) { + const bytes = await readFile(absolute); + files.push({ path: rel, bytes: bytes.length, sha256: sha256(bytes) }); + } else throw new Error(`provenance path is not a regular file: ${rel}`); + }; + for (const path of paths) await visit(join(root, path), path); + files.sort((a, b) => a.path.localeCompare(b.path)); + return { files, manifestSha256: sha256(JSON.stringify(files)) }; +} + +async function collectRepositoryProvenance({ repositoryRoot, gitPath = resolveSystemExecutable("git") }) { + const repo = canonicalRoot(repositoryRoot); + const safeEnv = buildSafeEnvironment({ ambient: {}, fixture: { PATH: `${dirname(gitPath)}:/usr/bin:/bin`, HOME: repo, TMPDIR: join(repo, ".artifacts") } }); + const run = async (argv) => await execFileAsync(gitPath, ["-C", repo, ...argv], { env: safeEnv, maxBuffer: MAX_STDIO_BYTES }); + const beforeHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const beforeTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const beforeStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (!HEX40.test(beforeHead) || !HEX40.test(beforeTree) || beforeStatus !== "") throw new Error("repository is not clean at exact HEAD"); + const backendRoot = join(repo, "backend"); + const backendSource = await manifestFiles(backendRoot, [ + "src", + "scripts/p5-acceptance.mjs", + "package.json", + "package-lock.json", + "tsconfig.json", + ]); + const backendDist = existsSync(join(backendRoot, "dist")) ? await manifestFiles(backendRoot, ["dist"]) : { files: [], manifestSha256: sha256("[]") }; + const afterHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const afterTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const afterStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (afterHead !== beforeHead || afterTree !== beforeTree || afterStatus !== beforeStatus) throw new Error("repository provenance changed during binding"); + return { schemaVersion: 1, head: beforeHead, tree: beforeTree, clean: true, backendSource, backendDist }; +} + +async function createTopology(run) { + for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); +} + +async function allocatePort() { + const server = net.createServer(); + await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).on("error", reject)); + const port = server.address().port; + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + return port; +} + +function installationProjectName(installationPath) { + return `thothii-${sha256(installationPath).slice(0, 12)}`; +} + +function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) { + return { + workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + diagnostics: { + dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}), + }; +} + +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } + +function descriptorYaml(obj) { + return yamlStringify(obj, { lineWidth: 0, sortMapEntries: false }); +} + +async function setupSecrets(ctx) { + const secretDir = join(ctx.run.root, "fixture-secrets"); + const values = { + dwhToken: `P2-DWH-${randomBytes(16).toString("hex")}`, + signedToken: `P2-SIGNED-${randomBytes(16).toString("hex")}`, + bundle: `P2-BUNDLE-${randomBytes(16).toString("hex")}`, + }; + ctx.forbiddenValues = Object.values(values); + ctx.secretValues = values; + const paths = { + dwh: join(secretDir, "p2-dwh-api-key"), + filesystemDwh: join(secretDir, "p2-filesystem-api-key"), + signed: join(secretDir, "p2-dwh-evidence-signed-urls.json"), + bundle: join(secretDir, "thothii.secrets"), + }; + await atomicWrite(paths.dwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.filesystemDwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.bundle, scalarSecretBytes(values.bundle)); + ctx.secretPaths = paths; +} + +async function setupFixtures(ctx) { + ctx.fixturePorts = { + dwh: await allocatePort(), + evidence: await allocatePort(), + embedding: await allocatePort(), + qdrant: await allocatePort(), + }; + const dwhBaseUrl = `http://host.docker.internal:${ctx.fixturePorts.dwh}`; + const evidenceProvenance = `http://host.docker.internal:${ctx.fixturePorts.evidence}/p2-dwh/guide.md`; + ctx.workspaceObjects = { + dwh: baseWorkspace("p2-dwh", { + dwhBaseUrl, + evidenceSource: { + type: "http", + uris: [evidenceProvenance], + authentication: "signed_urls_file", + connect_timeout_ms: 1250, + read_timeout_ms: 30001, + max_bytes: 65536, + max_redirects: 2, + allow_private_hosts: true, + max_cache_bytes: 65536, + }, + }), + filesystem: baseWorkspace("p2-filesystem", { + dwhBaseUrl, + evidenceSource: { + type: "filesystem", + uri: "p2-filesystem/evidence", + patterns: ["**/*.md"], + max_bytes: 1048576, + }, + }), + }; + const signedUrl = `${evidenceProvenance}?token=${ctx.secretValues.signedToken}`; + await atomicWrite(ctx.secretPaths.signed, `${JSON.stringify([signedUrl], null, 2)}\n`); + + ctx.curatedAnnotations = { + "p2-dwh": "tables: {}\n", + "p2-filesystem": "tables: {}\n", + }; + ctx.evidenceState = { + content: "# P2 Evidence\n\nFirst generation.\n", + token: ctx.secretValues.signedToken, + }; + ctx.dwhState = { + tables: { + patients: { + comment: "Patients", + rows: [ + { patient_id: "p1", name: "Alice" }, + { patient_id: "p2", name: "Bob" }, + ], + }, + visits: { + comment: "Visits", + rows: [ + { id: "v1", patient_id: "p1", note: "checkup" }, + { id: "v2", patient_id: "p2", note: "xray" }, + ], + }, + labs: { + comment: "Labs", + rows: [ + { id: "l1", patient_id: "p1", code: "hemoglobin" }, + { id: "l2", patient_id: "p2", code: "glucose" }, + ], + }, + }, + token: ctx.secretValues.dwhToken, + }; +} + +function inferColumnType(value) { + return typeof value === "number" ? "integer" : "text"; +} + +function topValues(rows, column, limit) { + const counts = new Map(); + for (const row of rows) { + const value = row[column]; + if (value === undefined || value === null || value === "") continue; + counts.set(String(value), (counts.get(String(value)) ?? 0) + 1); + } + return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, limit).map(([value]) => ({ value })); +} + +async function startHttpServer({ port, handler }) { + const server = createServer(async (req, res) => { + try { + await handler(req, res); + } catch { + res.statusCode = 500; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "fixture failed" })); + } + }); + await new Promise((resolve, reject) => server.listen(port, "127.0.0.1", () => resolve()).on("error", reject)); + return server; +} + +async function startServers(ctx) { + const dwhServer = await startHttpServer({ + port: ctx.fixturePorts.dwh, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const json = body.length === 0 ? {} : JSON.parse(body); + if (req.headers["x-api-key"] !== ctx.dwhState.token) { + res.statusCode = 401; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ message: "unauthorized" })); + return; + } + const send = (payload) => { + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(payload)); + }; + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.dwh}`); + if (req.method !== "POST" || !url.pathname.startsWith("/rpc/")) { + res.statusCode = 404; + res.end(JSON.stringify({ message: "not found" })); + return; + } + const fn = url.pathname.slice("/rpc/".length); + const schemaName = json.schema_name ?? "dw"; + if (schemaName !== "dw") { + send([]); + return; + } + if (fn === "ping") { + send({ db_connected: true, schema_accessible: true, database: "warehouse", schema: "dw" }); + return; + } + const table = typeof json.table_name === "string" ? json.table_name : ""; + const tableData = ctx.dwhState.tables[table]; + if (fn === "list_tables") { + send(Object.entries(ctx.dwhState.tables).map(([name, info]) => ({ table: name, type: "TABLE", comment: info.comment, rows: info.rows.length }))); + return; + } + if (!tableData) { + send([]); + return; + } + if (fn === "table_columns") { + const first = tableData.rows[0] ?? {}; + send(Object.keys(first).map((column) => ({ + column, + type: inferColumnType(first[column]), + nullable: false, + // Only the referenced table marks `patient_id` as primary, so the SQL miner sees a + // PK/non-PK pair while the same-name heuristic still discovers joins from the others. + pk: column === "id" || (table === "patients" && column === "patient_id"), + default: null, + }))); + return; + } + if (fn === "table_comments") { + send(Object.keys(tableData.rows[0] ?? {}).map((column) => ({ object: "COLUMN", name: column, comment: `${table}.${column}` }))); + return; + } + if (fn === "table_foreign_keys") { + send([]); + return; + } + if (fn === "top_values") { + send(topValues(tableData.rows, json.column_name, Number(json.max_values ?? 10))); + return; + } + if (fn === "column_stats") { + send({}); + return; + } + if (fn === "run_query") { + send([]); + return; + } + if (fn === "explain_query") { + send([{ line: "Seq Scan" }]); + return; + } + res.statusCode = 404; + res.end(JSON.stringify({ message: "unknown rpc" })); + }, + }); + const evidenceServer = await startHttpServer({ + port: ctx.fixturePorts.evidence, + handler: async (req, res) => { + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.evidence}`); + if (url.pathname !== "/p2-dwh/guide.md" || url.searchParams.get("token") !== ctx.evidenceState.token) { + res.statusCode = 403; + res.end("forbidden"); + return; + } + res.statusCode = 200; + res.setHeader("content-type", "text/markdown; charset=utf-8"); + res.end(ctx.evidenceState.content); + }, + }); + const embeddingServer = await startHttpServer({ + port: ctx.fixturePorts.embedding, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.embedding}`); + if (req.method !== "POST" || url.pathname !== "/api/embed") { + res.statusCode = 404; + res.end(JSON.stringify({ error: "not found" })); + return; + } + const payload = JSON.parse(body || "{}"); + const inputs = Array.isArray(payload.input) ? payload.input : []; + const embeddings = inputs.map((text) => { + const seed = sha256(String(text)); + return Array.from({ length: 1024 }, (_, index) => { + const offset = (index * 2) % seed.length; + const value = Number.parseInt(seed.slice(offset, offset + 2), 16); + return (value / 255) - 0.5; + }); + }); + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ model: payload.model, embeddings })); + }, + }); + ctx.servers = [dwhServer, evidenceServer, embeddingServer]; +} + +async function stopServers(ctx) { + for (const server of ctx.servers ?? []) { + await new Promise((resolve) => server.close(() => resolve())); + } + ctx.servers = []; +} + +async function git(ctx, args, cwd = join(ctx.run.root, "author")) { + return await runCommand({ executable: ctx.executables.gitPath, argv: args, cwd, env: ctx.execEnv }); +} + +async function initializeGitAndRegistry(ctx) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], ctx.run.root); + await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], ctx.run.root); + await git(ctx, ["config", "user.name", "P2 Fixture Curator"], author); + await git(ctx, ["config", "user.email", "p5-curator@example.invalid"], author); + + const writeWorkspaces = async () => { + const catalog = { + schema_version: 1, + workspaces: [ + { id: "p2-dwh", name: ctx.workspaceObjects.dwh.workspace.name }, + { id: "p2-filesystem", name: ctx.workspaceObjects.filesystem.workspace.name }, + ], + }; + await writeFile(join(author, "thoth-workspaces.yaml"), yamlStringify(catalog, { lineWidth: 0, sortMapEntries: false })); + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId), { recursive: true }); + const yaml = descriptorYaml(workspace); + await writeFile(join(author, pathId, "workspace.yaml"), yaml); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + await mkdir(join(author, "workspace-docs", pathId), { recursive: true }); + await writeFile(join(author, "workspace-docs", pathId, "contract.env.example"), docs.envExample); + await writeFile(join(author, "workspace-docs", pathId, "README.md"), docs.markdown); + } + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId, "schema"), { recursive: true }); + await writeFile(join(author, pathId, "schema", "annotations.yaml"), ctx.curatedAnnotations[pathId]); + } + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "evidence", "guide.md"), "# P2 Filesystem Evidence\n\nCommitted fixture.\n"); + }; + + await writeWorkspaces(); + await git(ctx, ["add", "."], author); + await git(ctx, ["commit", "-m", "Bootstrap P2 fixtures"], author); + await git(ctx, ["push", "origin", "main"], author); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); + const registry = new ctx.workspaceModules.WorkspaceRegistry({ + root: join(ctx.run.root, "installation", "registry"), + remoteUrl: join(ctx.run.root, "remote.git"), + branch: "main", + gitAuthorName: "P2 Acceptance", + gitAuthorEmail: "p5-acceptance@example.invalid", + installationId: "p5-acceptance", + secretRoots: [join(ctx.run.root, "fixture-secrets")], + maxImportBytes: 16 * 1024 * 1024, + maxImportEntries: 1024, + dataRoot: join(ctx.run.root, "installation", "data"), + }); + await registry.bootstrap(); + ctx.registry = registry; +} + +async function mutateWorkspaceDescriptor(ctx, workspaceId, mutator, commitMessage) { + const author = join(ctx.run.root, "author"); + // The registry may have produced docs-only follow-up commits on the remote; the curator + // always rebases onto the latest remote head before committing so the push stays fast-forward. + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const workspace = structuredClone(ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"]); + mutator(workspace); + ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"] = workspace; + await writeFile(join(author, workspaceId, "workspace.yaml"), descriptorYaml(workspace)); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + const docsDir = join(author, "workspace-docs", workspaceId); + await mkdir(docsDir, { recursive: true, mode: 0o700 }); + await writeFile(join(docsDir, "contract.env.example"), docs.envExample); + await writeFile(join(docsDir, "README.md"), docs.markdown); + await git(ctx, ["add", `${workspaceId}/workspace.yaml`, `workspace-docs/${workspaceId}/contract.env.example`, `workspace-docs/${workspaceId}/README.md`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + + +async function mutateWorkspaceAnnotations(ctx, workspaceId, contents, commitMessage) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const annotationsPath = join(author, workspaceId, "schema", "annotations.yaml"); + await mkdir(dirname(annotationsPath), { recursive: true }); + await writeFile(annotationsPath, contents); + ctx.curatedAnnotations[workspaceId] = contents; + await git(ctx, ["add", `${workspaceId}/schema/annotations.yaml`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + +async function writeInstallationFiles(ctx) { + const installationDir = join(ctx.run.root, "installation"); + const operatorEnvPath = join(installationDir, "operator.env"); + const bindingsEnvPath = join(installationDir, "workspace-bindings.env"); + const connectorOverridePath = join(installationDir, "connector-secrets.override.yaml"); + const fixtureOverridePath = join(installationDir, "fixture.override.yaml"); + const installationPath = join(installationDir, "thothii-installation.yaml"); + ctx.installationPath = installationPath; + ctx.composeProject = installationProjectName(installationPath); + const qdrantPort = ctx.fixturePorts.qdrant; + const bindings = [ + `THT_WS_P2_DWH_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_DWH_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_DWH_DWH_API_KEY_FILE=/run/secrets/p2-dwh-api-key`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_FILE=/run/secrets/p2-dwh-evidence-signed-urls`, + `THT_WS_P2_FILESYSTEM_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_FILESYSTEM_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_FILE=/run/secrets/p2-filesystem-api-key`, + ].join("\n") + "\n"; + await atomicWrite(bindingsEnvPath, bindings); + const operatorEnv = [ + `THT_DATA_ROOT=${join(ctx.run.root, "installation", "data")}`, + `THT_WORKSPACE_REGISTRY_ROOT=${join(ctx.run.root, "installation", "registry")}`, + `THT_PI_STATE_ROOT=${join(ctx.run.root, "installation", "pi-state")}`, + `PI_AUTH_FILE=${join(ctx.run.root, "installation", "pi-auth.json")}`, + `THT_SECRETS_FILE=${ctx.secretPaths.bundle}`, + `THT_WORKSPACE_BINDINGS_ENV_FILE=${bindingsEnvPath}`, + `THT_WORKSPACE_GIT_REMOTE=${join(ctx.run.root, "remote.git")}`, + `THT_WORKSPACE_GIT_BRANCH=main`, + `THT_WORKSPACE_GIT_AUTHOR_NAME=P2 Acceptance`, + `THT_WORKSPACE_GIT_AUTHOR_EMAIL=p5-acceptance@example.invalid`, + `THT_WORKSPACE_INSTALLATION_ID=p5-acceptance`, + `THT_DB_NAME=warehouse`, + `THT_DWH_REST_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_LLM_URL=http://127.0.0.1:9`, + `THOTH_SERVER_BIND=127.0.0.1`, + `THOTH_HTTP_PORT=18080`, + `THOTH_CORE_HTTP_PORT=18787`, + `THT_WS_P2_DWH_DWH_API_KEY_SOURCE=${ctx.secretPaths.dwh}`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_SOURCE=${ctx.secretPaths.signed}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_SOURCE=${ctx.secretPaths.filesystemDwh}`, + `THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST=host.docker.internal`, + ].join("\n") + "\n"; + await atomicWrite(operatorEnvPath, operatorEnv); + await atomicWrite(join(ctx.run.root, "installation", "pi-auth.json"), JSON.stringify({ fixture: true })); + const embeddingStubPath = join(installationDir, "embedding-stub.py"); + await atomicWrite(embeddingStubPath, EMBEDDING_STUB_SOURCE); + const override = { + services: { + core: { + image: ctx.coreImageTag, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + "workspace-maintenance": { + image: ctx.coreImageTag, + environment: { + THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST: "host.docker.internal", + }, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + qdrant: { + ports: [`127.0.0.1:${qdrantPort}:6333`], + restart: "no", + }, + // Deterministic Ollama-compatible embedding fixture on the internal allowlisted host + // name `embedding` (http://embedding:11434). Replaces the real Ollama service entirely. + embedding: { + image: ctx.coreImageTag, + entrypoint: ["python3", "/stub.py"], + volumes: [ + { type: "bind", source: embeddingStubPath, target: "/stub.py", read_only: true }, + ], + healthcheck: { disable: true }, + }, + }, + }; + await atomicWrite(fixtureOverridePath, yamlStringify(override, { lineWidth: 0, sortMapEntries: false })); + const generated = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "generate-connector-secrets-override.sh"), + argv: [ + "--bindings-env", bindingsEnvPath, + "--operator-env", operatorEnvPath, + "--output", connectorOverridePath, + "--service", "workspace-maintenance", + "--role", "all", + ], + env: ctx.execEnv, + }); + if (generated.exitCode !== 0) throw new Error(`connector override generation failed: ${generated.stderr || generated.stdout}`); + const installation = { + profile: "server", + projectDirectory: ctx.repositoryRoot, + envFile: operatorEnvPath, + overrides: [ + join(ctx.repositoryRoot, "deploy", "compose.server.yaml"), + fixtureOverridePath, + connectorOverridePath, + ], + }; + await atomicWrite(installationPath, yamlStringify(installation, { lineWidth: 0, sortMapEntries: false })); + ctx.installation = installation; +} + +function thothctlBinaryPath(repositoryRoot) { + const platform = { darwin: "darwin", linux: "linux", win32: "windows" }[process.platform] ?? "linux"; + const arch = { x64: "amd64", arm64: "arm64" }[process.arch] ?? "amd64"; + const suffix = platform === "windows" ? ".exe" : ""; + const candidates = [ + join(repositoryRoot, "dist", "thothctl", `thothctl-${platform}-${arch}${suffix}`), + join(repositoryRoot, "tools", "thothctl", "bin", `thothctl${suffix}`), + ]; + for (const candidate of candidates) if (existsSync(candidate)) return candidate; + throw new Error("built thothctl binary is unavailable"); +} + +async function runCommand({ executable, argv = [], cwd, env, input, maxOutputBytes = MAX_STDIO_BYTES }) { + const result = await execFileAsync(executable, argv, { + cwd, + env, + encoding: "utf8", + maxBuffer: maxOutputBytes, + ...(input === undefined ? {} : { input }), + }).then( + ({ stdout, stderr }) => ({ exitCode: 0, stdout, stderr }), + (error) => ({ exitCode: error.code ?? 1, stdout: error.stdout ?? "", stderr: error.stderr ?? error.message ?? "" }), + ); + return result; +} + +async function buildCoreImage(ctx) { + const tag = `thothii-core:p2-${ctx.run.runId.slice(3, 15)}`; + ctx.coreImageTag = tag; + const build = await runCommand({ + executable: ctx.executables.dockerPath, + argv: ["build", "-f", join(ctx.repositoryRoot, "docker", "core.Dockerfile"), "-t", tag, ctx.repositoryRoot], + env: { ...ctx.execEnv, DOCKER_BUILDKIT: "1" }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (build.exitCode !== 0) throw new Error(`core image build failed: ${build.stderr || build.stdout}`); +} + +async function buildThothctl(ctx) { + const command = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "build-thothctl.sh"), + argv: [], + env: { ...ctx.execEnv, THT_THOTHCTL_OUTPUT_DIRECTORY: join(ctx.repositoryRoot, "dist", "thothctl") }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (command.exitCode !== 0) throw new Error(`build-thothctl failed: ${command.stderr || command.stdout}`); + ctx.thothctlPath = thothctlBinaryPath(ctx.repositoryRoot); +} + +function composeBaseArgs(ctx) { + const args = [ + "compose", + "--project-name", ctx.composeProject, + "--project-directory", ctx.installation.projectDirectory, + "--env-file", ctx.installation.envFile, + "-f", join(ctx.repositoryRoot, "compose.yaml"), + ]; + for (const override of ctx.installation.overrides) args.push("-f", override); + return args; +} + +async function dockerCompose(ctx, commandArgs, { allowFailure = false, maxOutputBytes = 2 * 1024 * 1024 } = {}) { + const result = await runCommand({ + executable: ctx.executables.dockerPath, + argv: [...composeBaseArgs(ctx), ...commandArgs], + env: ctx.execEnv, + maxOutputBytes, + }); + if (!allowFailure && result.exitCode !== 0) throw new Error(`docker compose ${commandArgs.join(" ")} failed: ${result.stderr || result.stdout}`); + return result; +} + +async function startQdrant(ctx) { + await dockerCompose(ctx, ["up", "-d", "qdrant", "embedding"]); + for (let attempt = 0; attempt < 60; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}/collections`); + if (response.ok) return; + } catch {} + await sleep(1000); + } + throw new Error("qdrant did not become ready"); +} + +async function qdrantJson(ctx, method, path, body) { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}${path}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const payload = response.status === 204 ? {} : await response.json().catch(() => ({})); + if (!response.ok) throw new Error(`qdrant request failed: ${method} ${path} ${response.status}`); + return payload; +} + +async function preprovisionCollection(ctx, workspaceId) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}`, { + vectors: { size: 1024, distance: "Cosine" }, + }); + for (const field of ["content_hash", "document_id", "kind", "record_key", "record_kind", "vector_generation", "workspace_id", "workspace_revision"]) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}/index`, { field_name: field, field_schema: "keyword" }); + } +} + +async function listCollections(ctx) { + const payload = await qdrantJson(ctx, "GET", "/collections"); + const collections = payload.result?.collections ?? []; + return collections.map((item) => item.name).sort(); +} + +async function dumpQdrantPayloads(ctx, workspaceId) { + const response = await qdrantJson(ctx, "POST", `/collections/${workspaceId}/points/scroll`, { limit: 128, with_payload: true, with_vector: false }); + return JSON.stringify(response.result?.points ?? []); +} + +async function runThothctlJson(ctx, label, workspaceArgs, expectedExitCode) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.json`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs, "--json"], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + if (expectedExitCode !== undefined && result.exitCode !== expectedExitCode) { + throw new Error(`${label} exit ${result.exitCode} != ${expectedExitCode}`); + } + let payload; + try { payload = JSON.parse(result.stdout); } catch (error) { throw new Error(`${label} returned non-JSON stdout`); } + return { result, payload, artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath))] }; +} + +async function runThothctlRaw(ctx, label, workspaceArgs) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.txt`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + return { + result, + artifacts: [ + await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), + await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath)), + ], + }; +} + +async function loadWorkspaceSnapshot(ctx, workspaceId) { + const active = JSON.parse(await readFile(join(ctx.run.root, "installation", "registry", "state", "active.json"), "utf8")); + const revision = active.revisions.find((entry) => entry.id === workspaceId); + const snapshotPath = revision.snapshotPath; + const contents = await readFile(snapshotPath, "utf8"); + return { active, revision, contents }; +} + +async function assertNoCoreFrontendRunning(ctx) { + const ps = await dockerCompose(ctx, ["ps", "--status", "running", "--format", "json"], { allowFailure: true }); + if (ps.exitCode !== 0) return []; + const lines = ps.stdout.trim() === "" ? [] : ps.stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); + const services = lines.map((item) => item.Service); + if (services.includes("core") || services.includes("frontend") || services.includes("workspace-maintenance")) { + throw new Error("core/frontend/maintenance is unexpectedly running"); + } + return services; +} + +function sameSet(left, right) { + return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort()); +} + +const EMBEDDING_STUB_SOURCE = String.raw`import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +class _Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(length)) + inputs = payload.get("input", []) + if isinstance(inputs, str): + inputs = [inputs] + embeddings = [[0.01] * 1024 for _ in inputs] + body = json.dumps({"model": payload.get("model", "qwen3-embedding:0.6b"), "embeddings": embeddings}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + +HTTPServer(("0.0.0.0", 11434), _Handler).serve_forever() +`; + +function realUserHome() { + try { + const output = execFileSync("bash", ["-lc", 'printf "%s" ~'], { encoding: "utf8" }).trim(); + return output.length > 0 ? output : undefined; + } catch { + return undefined; + } +} + +async function setupRealContext({ repositoryRoot = defaultRepositoryRoot, env = process.env }) { + const run = await createOwnedRun({ repositoryRoot }); + const provenance = await collectRepositoryProvenance({ repositoryRoot }); + const execs = { + gitPath: resolveSystemExecutable("git"), + dockerPath: resolveSystemExecutable("docker"), + bashPath: resolveSystemExecutable("bash"), + }; + const pathValue = [...new Set([dirname(execs.gitPath), dirname(execs.dockerPath), "/usr/bin", "/bin", "/opt/homebrew/bin", "/usr/local/bin"])].join(":"); + // Docker CLI plugins (buildx) live under the real user's ~/.docker; the wrapper runs with a + // scrubbed environment, so derive the real home from the passwd entry and expose DOCKER_CONFIG. + const realHome = env.P5_REAL_HOME ?? realUserHome(); + const execEnv = buildSafeEnvironment({ ambient: env, fixture: { + PATH: pathValue, + HOME: run.root, + TMPDIR: join(run.root, "tmp"), + ...(realHome ? { DOCKER_CONFIG: join(realHome, ".docker") } : {}), + } }); + const workspaceModules = await import("../dist/workspaces/registry.js").then(async (registryModule) => ({ + WorkspaceRegistry: registryModule.WorkspaceRegistry, + ...(await import("../dist/workspaces/schema.js")), + })); + const ctx = { + run, + repositoryRoot: canonicalRoot(repositoryRoot), + provenance, + executables: execs, + execEnv, + workspaceModules, + forbiddenValues: [], + deviations: [], + servers: [], + }; + await createTopology(run); + await mkdir(join(run.root, "tmp"), { recursive: true, mode: 0o700 }); + await setupSecrets(ctx); + await setupFixtures(ctx); + return ctx; +} + +async function executeChecksLocal({ checks, failAt } = {}) { + if (!Array.isArray(checks) || !hasExactCheckIds(checks)) throw new Error("scenarios must match the exact ordered check set"); + if (failAt !== undefined && !CHECK_IDS.includes(failAt)) throw new Error("failure hook must name an exact check"); + const results = []; + let stopped = false; + for (const scenario of checks) { + const startedAt = nowIso(); + let result; + if (stopped) { + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: "Not executed after earlier failure." }; + } else { + try { + const output = await scenario.run(); + if (scenario.id === failAt) throw new Error("injected acceptance failure"); + result = { id: scenario.id, status: "PASS", startedAt, finishedAt: nowIso(), commands: output.commands ?? [], artifacts: output.artifacts ?? [] }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: `Acceptance scenario failed safely: ${detail}` }; + stopped = true; + } + } + results.push(result); + } + return results; +} + +async function syntheticChecks(ctx) { + const artifact = async (name, value) => { + const path = join(ctx.run.root, "logs", `${name}.json`); + await writeJson(path, value); + return await fileArtifact(ctx.run.root, relative(ctx.run.root, path)); + }; + return CHECK_IDS.map((id, index) => ({ + id, + async run() { + return { + commands: [index === 0 ? "node" : "git"], + artifacts: [await artifact(id, { id, synthetic: true })], + }; + }, + })); +} + +async function realChecks(ctx) { + const state = {}; + const syncedRoot = (workspaceId, commit) => join(ctx.run.root, "installation", "data", "sessions", workspaceId, "revisions", commit, "artifacts"); + const syncedAnnotations = (workspaceId, commit) => join(syncedRoot(workspaceId, commit), "mschema", "annotations.yaml"); + const syncedManifest = (workspaceId, commit) => join(syncedRoot(workspaceId, commit), "mschema", "annotations.ownership.json"); + const activeCommit = async (workspaceId) => (await loadWorkspaceSnapshot(ctx, workspaceId)).revision.commit; + + return [ + { + id: "preflight", + async run() { + await buildThothctl(ctx); + await buildCoreImage(ctx); + await writeInstallationFiles(ctx); + return { + commands: ["docker", "node", "git"], + artifacts: [{ path: "logs/provenance.json", sha256: sha256(JSON.stringify(ctx.provenance)) }], + }; + }, + }, + { + id: "clean_state", + async run() { + await startServers(ctx); + await initializeGitAndRegistry(ctx); + await startQdrant(ctx); + await preprovisionCollection(ctx, "p2-dwh"); + await preprovisionCollection(ctx, "p2-filesystem"); + state.collectionsBefore = await listCollections(ctx); + state.runningServices = await assertNoCoreFrontendRunning(ctx); + state.initialCommit = await activeCommit("p2-filesystem"); + await writeJson(join(ctx.run.root, "logs", "collections-before.json"), state.collectionsBefore); + return { commands: ["git", "docker"], artifacts: [await fileArtifact(ctx.run.root, "logs/collections-before.json")] }; + }, + }, + { + id: "ownership", + async run() { + await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); + const installStat = await stat(ctx.installationPath); + assert(installStat.isFile(), "installation descriptor missing"); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "ownership.json")] }; + }, + }, + { + id: "activation_sync", + async run() { + // The initial curated blob must have been synchronized on activation with a verified + // ownership manifest at the exact revision-qualified runtime root. + const commit = state.initialCommit; + const annotationsPath = syncedAnnotations("p2-filesystem", commit); + const manifestPath = syncedManifest("p2-filesystem", commit); + assert(readFileSync(annotationsPath, "utf8") === "tables: {}\n", "synced annotations content mismatch"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + assert(manifest.workspace === "p2-filesystem", "ownership manifest workspace mismatch"); + assert(manifest.commit === commit, "ownership manifest commit mismatch"); + assert(/^[0-9a-f]{40}$/.test(manifest.blobId ?? ""), "ownership manifest blobId missing"); + assert(manifest.contentDigest === `sha256:${sha256("tables: {}\n")}`, "ownership manifest digest mismatch"); + assert(manifest.destination === annotationsPath, "ownership manifest destination mismatch"); + return { commands: [], artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, annotationsPath)), await fileArtifact(ctx.run.root, relative(ctx.run.root, manifestPath))] }; + }, + }, + { + id: "accept_happy_path", + async run() { + // 1) a fresh full run introspects the DWH, mines FK candidates, and blocks for review. + const full = await runThothctlJson(ctx, "preprocess-run-fs-blocked", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem"], 3); + assert(full.payload.code === "manual_review_required", `full run did not block: ${full.payload.code}`); + const digest = full.payload.artifactIdentities?.[0]?.digest; + assert(/^sha256:[0-9a-f]{64}$/.test(digest ?? ""), "candidate digest missing"); + state.runId = full.payload.runId; + assert(/^[0-9a-f]{32}$/.test(state.runId ?? ""), "run id missing"); + + // 2) the operator records the human review with the explicit accept command. + const accepted = await runThothctlJson(ctx, "schema-accept-filesystem", ["workspace", "schema", "accept", "--workspace", "p2-filesystem", "--run", state.runId, "--yes"], 0); + assert(accepted.payload.status === "succeeded" && accepted.payload.code === "ok", "schema accept failed"); + assert(Array.isArray(accepted.payload.artifactIdentities), "accept artifact identities missing"); + + // 3) same-revision resume continues through the FK gate and stops at filesystem Evidence. + const resumed = await runThothctlJson(ctx, "preprocess-run-fs-resume", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem", "--resume", state.runId], 3); + assert(resumed.payload.code === "evidence_materialization_required", `resume code mismatch: ${resumed.payload.code}`); + state.acceptedCommit = state.initialCommit; + return { commands: ["thothctl"], artifacts: [...full.artifacts, ...accepted.artifacts, ...resumed.artifacts] }; + }, + }, + { + id: "revision_isolation", + async run() { + // A new annotations revision writes a distinct immutable runtime root. + await mutateWorkspaceAnnotations(ctx, "p2-filesystem", "tables: {}\n# revision B\n", "Curate reviewed FK annotations (revision B)"); + const revisionB = await activeCommit("p2-filesystem"); + assert(revisionB !== state.initialCommit, "annotations commit did not change the revision"); + assert(existsSync(syncedAnnotations("p2-filesystem", revisionB)), "revision B annotations not synced"); + assert(syncedAnnotations("p2-filesystem", revisionB) !== syncedAnnotations("p2-filesystem", state.initialCommit), "revision roots are not isolated"); + state.revisionB = revisionB; + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, syncedAnnotations("p2-filesystem", revisionB)))] }; + }, + }, + { + id: "accept_negatives", + async run() { + // Grammar: --yes is required (exit 2, host-side, no JSON result). + const missingYes = await runThothctlRaw(ctx, "neg-missing-yes", ["workspace", "schema", "accept", "--workspace", "p2-filesystem", "--run", state.runId]); + assert(missingYes.result.exitCode === 2, `missing --yes exit ${missingYes.result.exitCode} != 2`); + + // Unknown run fails closed without recording a review. + const unknown = await runThothctlJson(ctx, "neg-unknown-run", ["workspace", "schema", "accept", "--workspace", "p2-filesystem", "--run", "e".repeat(32), "--yes"], 1); + assert(unknown.payload.code === "annotation_invalid", "unknown run code mismatch"); + + return { commands: ["thothctl"], artifacts: [...missingYes.artifacts, ...unknown.artifacts] }; + }, + }, + { + id: "continuation_gate", + async run() { + // A run accepted at revision A must not be silently resumed after the annotations revision + // changed to B: the pinned job no longer matches the active revision. + const stale = await runThothctlJson(ctx, "stale-resume-after-revision-change", ["workspace", "preprocess", "run", "--workspace", "p2-filesystem", "--resume", state.runId], 1); + assert(["preprocessing_resume_mismatch", "preprocessing_conflict"].includes(stale.payload.code), `stale resume code mismatch: ${stale.payload.code}`); + return { commands: ["thothctl"], artifacts: [...stale.artifacts] }; + }, + }, + { + id: "secret_scan", + async run() { + const virtualFiles = []; + const qdrantDump = await dumpQdrantPayloads(ctx, "p2-filesystem"); + if (Buffer.byteLength(qdrantDump) <= MAX_SECRET_SCAN_VIRTUAL_BYTES) virtualFiles.push({ path: "virtual/qdrant-p2-filesystem.json", bytes: qdrantDump }); + const findings = await scanSecrets({ + runRoot: ctx.run.root, + forbiddenValues: ctx.forbiddenValues, + virtualFiles, + expectedGitRepositories: ["remote.git", "author"], + }); + await writeJson(join(ctx.run.root, "logs", "secret-scan.json"), findings); + if (findings.length > 0) throw new Error(`secret scan found ${findings.length} leak(s)`); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "logs/secret-scan.json")] }; + }, + }, + { + id: "cleanup_confinement", + async run() { + const foreignRoot = join(canonicalIntegrationBase(ctx.repositoryRoot), `p5-${"f".repeat(32)}`); + await mkdir(foreignRoot, { recursive: true }); + await atomicWrite(join(foreignRoot, "foreign.txt"), "foreign"); + assert(readFileSync(join(foreignRoot, "foreign.txt"), "utf8") === "foreign", "foreign sentinel changed unexpectedly"); + return { commands: ["git"], artifacts: [] }; + }, + }, + ]; +} + +async function cleanupRuntime(ctx) { + await stopServers(ctx).catch(() => {}); + if (ctx.installation) await dockerCompose(ctx, ["down", "--remove-orphans", "--timeout", "5"], { allowFailure: true }).catch(() => {}); + if (ctx.coreImageTag) await runCommand({ executable: ctx.executables.dockerPath, argv: ["image", "rm", "-f", ctx.coreImageTag], env: ctx.execEnv, maxOutputBytes: MAX_STDIO_BYTES }).catch(() => {}); +} + +export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) { + const synthetic = env.P5_ACCEPTANCE_SYNTHETIC === "1"; + const failAt = env.P5_ACCEPTANCE_FAIL_AT; + const ctx = synthetic + ? { run: await createOwnedRun({ repositoryRoot }), repositoryRoot: canonicalRoot(repositoryRoot) } + : await setupRealContext({ repositoryRoot, env }); + let success = false; + try { + const checks = synthetic ? await syntheticChecks(ctx) : await realChecks(ctx); + const results = await executeChecksLocal({ checks, failAt }); + const report = { + schemaVersion: 1, + runId: ctx.run.runId, + startedAt: ctx.run.startedAt, + finishedAt: nowIso(), + command: "p5-acceptance integration --keep", + overall: deriveOverall(results), + checks: results, + }; + await writeReportFiles({ run: ctx.run, report }); + success = report.overall === "PASS"; + if (announce) await announce({ report, runRoot: ctx.run.root }); + return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) }; + } finally { + if (!synthetic) await cleanupRuntime(ctx).catch(() => {}); + } +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) { + throw new Error("usage: p5-acceptance.mjs integration [--keep]"); + } + const result = await runIntegration({ keep: argv.includes("--keep"), env }); + return result.exitCode; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { + const code = await main(); + process.exitCode = code; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +export { CHECK_IDS }; diff --git a/backend/scripts/p5-acceptance.test.mjs b/backend/scripts/p5-acceptance.test.mjs new file mode 100644 index 00000000..93959c74 --- /dev/null +++ b/backend/scripts/p5-acceptance.test.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_IDS, + canonicalIntegrationBase, + cleanupOwnedRun, + createOwnedRun, + readAndValidateOwnership, + runIntegration, + validateReport, + validateRunRoot, +} from "./p5-acceptance.mjs"; + +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p5-acceptance-repo-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p5-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true }); + return root; +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("run roots are only canonical direct p5 integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p5-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [ + base, + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(repositoryRoot, ".artifacts", "p1-integration", id), + join(repositoryRoot, ".artifacts", "p2-integration", id), + join(base, id, "nested"), + join(base, "foreign"), + ]) { + assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + } + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p5-${"A".repeat(32)}`), `p5-${"A".repeat(32)}`)); +}); + +test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + for (const bad of [ + join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`), + join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`), + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(canonicalIntegrationBase(repositoryRoot), `p5-${"d".repeat(32)}`), + ]) { + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce })); + } + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) })); +}); + +test("cleanup removes exactly one owned p5 root", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const sibling = join(canonicalIntegrationBase(repositoryRoot), `p5-${"e".repeat(32)}`); + await mkdir(sibling); + await writeFile(join(sibling, "sentinel"), "foreign"); + await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await assert.rejects(readFile(join(run.root, "ownership.json"))); + assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign"); +}); + +function resultFor(id) { + return { + id, + status: "PASS", + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:01.000Z", + commands: ["node"], + artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }], + }; +} + +test("report validation requires exact p5 identity, check order, and unique artifacts", () => { + const report = { + schemaVersion: 1, + runId: `p5-${"f".repeat(32)}`, + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:10.000Z", + command: "p5-acceptance integration --keep", + overall: "PASS", + checks: CHECK_IDS.map(resultFor), + }; + assert.doesNotThrow(() => validateReport(report)); + const invalid = structuredClone(report); + invalid.runId = `p2-${"f".repeat(32)}`; + assert.throws(() => validateReport(invalid)); + const duplicate = structuredClone(report); + duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path; + assert.throws(() => validateReport(duplicate), /duplicated/); + const reordered = structuredClone(report); + reordered.checks.reverse(); + reordered.overall = "FAIL"; + assert.throws(() => validateReport(reordered)); +}); + +test("public wrapper uses a strict empty environment", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p5-acceptance.sh"), "utf8"); + assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/); + assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/); + assert.doesNotMatch(wrapper, /P5_ACCEPTANCE_FAIL_AT/); +}); + +test("synthetic integration cleans up successful non-kept runs", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: false, env: { P5_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, false); + await assert.rejects(readFile(join(result.runRoot, "ownership.json"))); +}); + +test("synthetic integration retains kept runs with bounded reports", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: true, env: { P5_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "PASS"); + const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8"); + assert.match(reportMd, /P5 automated integration: PASS/); + assert.match(reportMd, /P5 manual acceptance: PENDING/); + const reportJsonStat = await stat(join(result.runRoot, "report.json")); + const reportMdStat = await stat(join(result.runRoot, "report.md")); + assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`); + assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`); +}); + +test("synthetic injected failure retains the owned run and records a single failed report", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ + repositoryRoot, + keep: false, + env: { P5_ACCEPTANCE_SYNTHETIC: "1", P5_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] }, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "FAIL"); + const failed = report.checks.find((check) => check.id === CHECK_IDS[2]); + assert.equal(failed.status, "FAIL"); + const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8"); + assert.match(roots, /p5-acceptance/); +}); diff --git a/backend/scripts/p6-acceptance.mjs b/backend/scripts/p6-acceptance.mjs new file mode 100644 index 00000000..c11bf50a --- /dev/null +++ b/backend/scripts/p6-acceptance.mjs @@ -0,0 +1,1389 @@ +#!/usr/bin/env node +import { createHash, randomBytes } from "node:crypto"; +import { execFile, execFileSync } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { createServer } from "node:http"; +import { closeSync, constants as fsConstants, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, rmSync, statSync, symlinkSync } from "node:fs"; +import { access, lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import net from "node:net"; +import process from "node:process"; + +import { stringify as yamlStringify } from "yaml"; + +import { buildSafeEnvironment, deriveOverall, scanSecrets } from "./p1-acceptance.mjs"; + +const execFileAsync = promisify(execFile); +const modulePath = fileURLToPath(import.meta.url); +const defaultRepositoryRoot = realpathSync(resolve(dirname(modulePath), "../..")); +const RUN_ID = /^p6-[0-9a-f]{32}$/; +const HEX32 = /^[0-9a-f]{32}$/; +const HEX40 = /^[0-9a-f]{40}$/; +const HEX64 = /^[0-9a-f]{64}$/; +const ISO_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const COMMAND = /^[a-z0-9][a-z0-9-]*$/; +const CHECK_RESULT_STATUS = new Set(["PASS", "FAIL"]); +const CHECK_IDS = Object.freeze([ + "preflight", + "clean_state", + "ownership", + "activation_materialization", + "evidence_preprocess", + "revision_isolation", + "unsafe_tree_refused", + "bound_refused", + "secret_scan", + "cleanup_confinement", +]); +const TOPOLOGY = [ + "remote.git", + "author", + "installation", + "installation/data", + "installation/data/sessions", + "installation/registry", + "installation/pi-state", + "fixture-secrets", + "fixtures", + "fixtures/logs", + "logs", +]; +const MAX_REPORT_JSON_BYTES = 64 * 1024; +const MAX_REPORT_MD_BYTES = 32 * 1024; +const MAX_STDIO_BYTES = 512 * 1024; +const MAX_SECRET_SCAN_VIRTUAL_BYTES = 256 * 1024; + +function nowIso() { return new Date().toISOString(); } +function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function assert(condition, message) { if (!condition) throw new Error(message); } +function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } + +function canonicalRoot(repositoryRoot = defaultRepositoryRoot) { + return realpathSync(repositoryRoot); +} + +export function canonicalIntegrationBase(repositoryRoot = defaultRepositoryRoot) { + return join(canonicalRoot(repositoryRoot), ".artifacts", "p6-integration"); +} + +export function validateRunRoot(repositoryRoot, runRoot, runId) { + if (!RUN_ID.test(runId)) throw new Error("invalid owned run id"); + const base = canonicalIntegrationBase(repositoryRoot); + const lexical = resolve(runRoot); + if (dirname(lexical) !== base || basename(lexical) !== runId) throw new Error("run root is not a direct integration child"); + return lexical; +} + +function validateNoSymlinkAncestors(repositoryRoot, target) { + const repo = canonicalRoot(repositoryRoot); + const rel = relative(repo, target); + if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("path leaves repository"); + let cursor = repo; + for (const part of rel.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + if (!existsSync(cursor)) break; + const entry = lstatSync(cursor); + if (entry.isSymbolicLink()) throw new Error("owned path ancestor is a symlink"); + } +} + +async function atomicWrite(path, bytes, mode = 0o600) { + await mkdir(dirname(path), { recursive: true }); + const staging = join(dirname(path), `.${basename(path)}.${randomBytes(12).toString("hex")}.tmp`); + let handle; + try { + handle = await open(staging, "wx", mode); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(staging, path); + const directory = openSync(dirname(path), fsConstants.O_RDONLY); + try { fsyncSync(directory); } finally { closeSync(directory); } + } catch (error) { + if (handle) await handle.close().catch(() => {}); + await rm(staging, { force: true }).catch(() => {}); + throw error; + } +} + +function initialResources(run) { + return [ + run.root, + join(run.root, "remote.git"), + join(run.root, "author"), + join(run.root, "installation"), + join(run.root, "installation", "registry"), + join(run.root, "installation", "data"), + join(run.root, "fixture-secrets"), + ]; +} + +function ownershipValue(run) { + return { + schemaVersion: 1, + kind: "p6-acceptance", + runId: run.runId, + runNonce: run.nonce, + root: run.root, + repositoryRoot: run.repositoryRoot, + startedAt: run.startedAt, + pid: run.pid, + resources: initialResources(run), + }; +} + +async function writeOwnership(run) { + await atomicWrite(join(run.root, "ownership.json"), `${JSON.stringify(ownershipValue(run), null, 2)}\n`); +} + +export async function createOwnedRun({ repositoryRoot = defaultRepositoryRoot, runId, nonce, now, pid } = {}) { + const repo = canonicalRoot(repositoryRoot); + const base = canonicalIntegrationBase(repo); + validateNoSymlinkAncestors(repo, base); + await mkdir(join(repo, ".artifacts"), { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + await mkdir(base, { mode: 0o700 }).catch((error) => { if (error.code !== "EEXIST") throw error; }); + const id = runId ?? `p6-${randomBytes(16).toString("hex")}`; + const root = validateRunRoot(repo, join(base, id), id); + const run = { + repositoryRoot: repo, + root, + runId: id, + nonce: nonce ?? randomBytes(32).toString("hex"), + startedAt: now ?? nowIso(), + pid: pid ?? process.pid, + }; + if (!HEX64.test(run.nonce) || !ISO_UTC.test(run.startedAt)) throw new Error("invalid ownership identity"); + await mkdir(root, { mode: 0o700 }); + await writeOwnership(run); + return run; +} + +function strictOwnership(value, run, expectedNonce) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("ownership is malformed"); + if (value.schemaVersion !== 1 || value.kind !== "p6-acceptance" || value.runId !== run.runId || value.runNonce !== expectedNonce + || value.root !== run.root || value.repositoryRoot !== run.repositoryRoot || value.pid !== process.pid + || !ISO_UTC.test(value.startedAt ?? "") + || JSON.stringify(value.resources) !== JSON.stringify(initialResources(run))) throw new Error("ownership identity mismatch"); + return value; +} + +export async function readAndValidateOwnership({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const repo = canonicalRoot(repositoryRoot); + const id = basename(resolve(runRoot)); + const lexical = validateRunRoot(repo, runRoot, id); + const rootEntry = await lstat(lexical); + if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) throw new Error("owned run root is not a directory"); + const ownershipPath = join(lexical, "ownership.json"); + const ownershipEntry = await lstat(ownershipPath); + if (!ownershipEntry.isFile() || ownershipEntry.isSymbolicLink()) throw new Error("ownership file is unsafe"); + let value; + try { value = JSON.parse(await readFile(ownershipPath, "utf8")); } catch { throw new Error("ownership is malformed"); } + return strictOwnership(value, { repositoryRoot: repo, root: lexical, runId: id }, expectedNonce); +} + +export async function cleanupOwnedRun({ repositoryRoot = defaultRepositoryRoot, runRoot, expectedNonce }) { + const value = await readAndValidateOwnership({ repositoryRoot, runRoot, expectedNonce }); + const base = canonicalIntegrationBase(repositoryRoot); + const tombstone = join(base, `.deleting-${value.runId}-${expectedNonce.slice(0, 16)}`); + await rename(runRoot, tombstone); + await rm(tombstone, { recursive: true, force: false }); +} + +async function finalizeOwnedRun({ run, success, keep }) { + if (!success || keep) return false; + await cleanupOwnedRun({ repositoryRoot: run.repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + return true; +} + +function safeArtifactPath(path) { + if (typeof path !== "string" || path.length === 0 || path.length > 255 || path.startsWith("/") || path.includes("..") || path.includes("\\") || /[\0\r\n]/.test(path)) { + throw new Error("report artifact path is invalid"); + } + return path; +} + +function hasExactCheckIds(checks) { + return checks.length === CHECK_IDS.length && checks.every(({ id }, index) => id === CHECK_IDS[index]); +} + +export function validateReport(report) { + if (!report || report.schemaVersion !== 1 || !RUN_ID.test(report.runId ?? "") || !ISO_UTC.test(report.startedAt ?? "") + || !ISO_UTC.test(report.finishedAt ?? "") || typeof report.command !== "string" + || !Array.isArray(report.checks) || !hasExactCheckIds(report.checks)) throw new Error("report is invalid"); + const ids = new Set(); + const artifactPaths = new Set(); + for (const check of report.checks) { + if (!check || !/^[a-z0-9_]+$/.test(check.id ?? "") || ids.has(check.id) || !CHECK_RESULT_STATUS.has(check.status) + || !ISO_UTC.test(check.startedAt ?? "") || !ISO_UTC.test(check.finishedAt ?? "") + || !Array.isArray(check.commands) || check.commands.some((name) => !COMMAND.test(name)) + || !Array.isArray(check.artifacts)) throw new Error("report check is invalid"); + ids.add(check.id); + for (const artifact of check.artifacts) { + safeArtifactPath(artifact.path); + if (!HEX64.test(artifact.sha256 ?? "")) throw new Error("report check is invalid"); + if (artifactPaths.has(artifact.path)) throw new Error("report artifact path is duplicated"); + artifactPaths.add(artifact.path); + } + } + if (report.overall !== deriveOverall(report.checks)) throw new Error("report overall is not derived"); + return report; +} + +function renderReportMarkdown(report) { + validateReport(report); + const rows = report.checks.map((check) => `| ${check.id} | ${check.status} |`).join("\n"); + return [ + "# P6 acceptance report", + "", + `Run: \`${report.runId}\``, + "", + "| Check | Status |", + "|---|---|", + rows, + "", + `P6 automated integration: ${report.overall}`, + "P6 manual acceptance: PENDING", + "", + ].join("\n"); +} + +async function walkFiles(root) { + const files = []; + async function visit(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const rel = relative(root, path).split(sep).join("/"); + if (entry.isSymbolicLink()) throw new Error(`unsafe file tree: ${rel}`); + if (entry.isDirectory()) await visit(path); + else if (entry.isFile()) files.push({ path, rel }); + } + } + if (existsSync(root)) await visit(root); + files.sort((a, b) => a.rel.localeCompare(b.rel)); + return files; +} + +async function snapshotDigest(root, excludedPrefixes = []) { + const result = {}; + for (const file of await walkFiles(root)) { + if (excludedPrefixes.some((prefix) => file.rel === prefix || file.rel.startsWith(`${prefix}/`))) continue; + result[file.rel] = sha256(await readFile(file.path)); + } + return result; +} + +async function fileArtifact(root, relativePath) { + const bytes = await readFile(join(root, relativePath)); + return { path: relativePath.split(sep).join("/"), sha256: sha256(bytes) }; +} + +async function writeJson(path, value) { + await atomicWrite(path, `${JSON.stringify(value, null, 2)}\n`); +} + +async function writeReportFiles({ run, report }) { + validateReport(report); + const reportJsonPath = join(run.root, "report.json"); + const reportMdPath = join(run.root, "report.md"); + const reportMd = renderReportMarkdown(report); + if (Buffer.byteLength(JSON.stringify(report)) > MAX_REPORT_JSON_BYTES) throw new Error("report.json exceeds bound"); + if (Buffer.byteLength(reportMd) > MAX_REPORT_MD_BYTES) throw new Error("report.md exceeds bound"); + await writeJson(reportJsonPath, report); + await atomicWrite(reportMdPath, reportMd, 0o600); + return { + reportJson: await fileArtifact(run.root, "report.json"), + reportMd: await fileArtifact(run.root, "report.md"), + }; +} + +function resolveSystemExecutable(name) { + for (const candidate of [`/usr/bin/${name}`, `/bin/${name}`, `/opt/homebrew/bin/${name}`, `/usr/local/bin/${name}`]) { + try { + const resolved = realpathSync(candidate); + if (statSync(resolved).isFile()) return resolved; + } catch {} + } + throw new Error(`required executable not found: ${name}`); +} + +function scalarSecretBytes(value) { + if (typeof value !== "string" || value.length === 0 || /\s|\0/.test(value)) throw new Error("scalar fixture secret is invalid"); + return Buffer.from(value); +} + +async function manifestFiles(root, paths) { + const files = []; + const visit = async (absolute, rel) => { + const entry = await lstat(absolute); + if (entry.isSymbolicLink()) throw new Error(`provenance path is a symlink: ${rel}`); + if (entry.isDirectory()) { + for (const child of (await readdir(absolute, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) { + await visit(join(absolute, child.name), rel ? `${rel}/${child.name}` : child.name); + } + } else if (entry.isFile()) { + const bytes = await readFile(absolute); + files.push({ path: rel, bytes: bytes.length, sha256: sha256(bytes) }); + } else throw new Error(`provenance path is not a regular file: ${rel}`); + }; + for (const path of paths) await visit(join(root, path), path); + files.sort((a, b) => a.path.localeCompare(b.path)); + return { files, manifestSha256: sha256(JSON.stringify(files)) }; +} + +async function collectRepositoryProvenance({ repositoryRoot, gitPath = resolveSystemExecutable("git") }) { + const repo = canonicalRoot(repositoryRoot); + const safeEnv = buildSafeEnvironment({ ambient: {}, fixture: { PATH: `${dirname(gitPath)}:/usr/bin:/bin`, HOME: repo, TMPDIR: join(repo, ".artifacts") } }); + const run = async (argv) => await execFileAsync(gitPath, ["-C", repo, ...argv], { env: safeEnv, maxBuffer: MAX_STDIO_BYTES }); + const beforeHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const beforeTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const beforeStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (!HEX40.test(beforeHead) || !HEX40.test(beforeTree) || beforeStatus !== "") throw new Error("repository is not clean at exact HEAD"); + const backendRoot = join(repo, "backend"); + const backendSource = await manifestFiles(backendRoot, [ + "src", + "scripts/p6-acceptance.mjs", + "package.json", + "package-lock.json", + "tsconfig.json", + ]); + const backendDist = existsSync(join(backendRoot, "dist")) ? await manifestFiles(backendRoot, ["dist"]) : { files: [], manifestSha256: sha256("[]") }; + const afterHead = (await run(["rev-parse", "HEAD"]).catch((error) => { throw error; })).stdout.trim(); + const afterTree = (await run(["rev-parse", "HEAD^{tree}"])).stdout.trim(); + const afterStatus = (await run(["status", "--porcelain=v1", "--untracked-files=no"])).stdout; + if (afterHead !== beforeHead || afterTree !== beforeTree || afterStatus !== beforeStatus) throw new Error("repository provenance changed during binding"); + return { schemaVersion: 1, head: beforeHead, tree: beforeTree, clean: true, backendSource, backendDist }; +} + +async function createTopology(run) { + for (const path of TOPOLOGY) await mkdir(join(run.root, path), { recursive: true, mode: path === "fixture-secrets" ? 0o700 : 0o755 }); +} + +async function allocatePort() { + const server = net.createServer(); + await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).on("error", reject)); + const port = server.address().port; + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + return port; +} + +function installationProjectName(installationPath) { + return `thothii-${sha256(installationPath).slice(0, 12)}`; +} + +function baseWorkspace(id, { dwhBaseUrl, evidenceSource }) { + return { + workspace: { schema_version: 3, id, name: `P2 ${id}`, language: "en" }, + dwh: { engine: "postgres", database: "warehouse", schema: "dw", supported_transports: ["rest_api"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + diagnostics: { + dwh_rest: { method: "POST", path: "/rpc/ping", auth: "x-api-key", response: { database: "database", schema: "schema" } }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + ...(evidenceSource ? { evidence: { source: evidenceSource, policy: { max_chunk_chars: 4000, retain_published_generations: 3 } } } : {}), + }; +} + +function namespace(id) { return id.toUpperCase().replaceAll("-", "_"); } + +function descriptorYaml(obj) { + return yamlStringify(obj, { lineWidth: 0, sortMapEntries: false }); +} + +async function setupSecrets(ctx) { + const secretDir = join(ctx.run.root, "fixture-secrets"); + const values = { + dwhToken: `P2-DWH-${randomBytes(16).toString("hex")}`, + signedToken: `P2-SIGNED-${randomBytes(16).toString("hex")}`, + bundle: `P2-BUNDLE-${randomBytes(16).toString("hex")}`, + }; + ctx.forbiddenValues = Object.values(values); + ctx.secretValues = values; + const paths = { + dwh: join(secretDir, "p2-dwh-api-key"), + filesystemDwh: join(secretDir, "p2-filesystem-api-key"), + signed: join(secretDir, "p2-dwh-evidence-signed-urls.json"), + bundle: join(secretDir, "thothii.secrets"), + }; + await atomicWrite(paths.dwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.filesystemDwh, scalarSecretBytes(values.dwhToken)); + await atomicWrite(paths.bundle, scalarSecretBytes(values.bundle)); + ctx.secretPaths = paths; +} + +async function setupFixtures(ctx) { + ctx.fixturePorts = { + dwh: await allocatePort(), + evidence: await allocatePort(), + embedding: await allocatePort(), + qdrant: await allocatePort(), + }; + const dwhBaseUrl = `http://host.docker.internal:${ctx.fixturePorts.dwh}`; + const evidenceProvenance = `http://host.docker.internal:${ctx.fixturePorts.evidence}/p2-dwh/guide.md`; + ctx.workspaceObjects = { + dwh: baseWorkspace("p2-dwh", { + dwhBaseUrl, + evidenceSource: { + type: "http", + uris: [evidenceProvenance], + authentication: "signed_urls_file", + connect_timeout_ms: 1250, + read_timeout_ms: 30001, + max_bytes: 65536, + max_redirects: 2, + allow_private_hosts: true, + max_cache_bytes: 65536, + }, + }), + filesystem: baseWorkspace("p2-filesystem", { + dwhBaseUrl, + evidenceSource: { + type: "filesystem", + uri: "p2-filesystem/evidence", + patterns: ["**/*.md"], + max_bytes: 1048576, + }, + }), + }; + const signedUrl = `${evidenceProvenance}?token=${ctx.secretValues.signedToken}`; + await atomicWrite(ctx.secretPaths.signed, `${JSON.stringify([signedUrl], null, 2)}\n`); + + ctx.curatedAnnotations = { + "p2-dwh": "tables: {}\n", + "p2-filesystem": "tables: {}\n", + }; + ctx.evidenceState = { + content: "# P2 Evidence\n\nFirst generation.\n", + token: ctx.secretValues.signedToken, + }; + ctx.dwhState = { + tables: { + patients: { + comment: "Patients", + rows: [ + { patient_id: "p1", name: "Alice" }, + { patient_id: "p2", name: "Bob" }, + ], + }, + visits: { + comment: "Visits", + rows: [ + { id: "v1", patient_id: "p1", note: "checkup" }, + { id: "v2", patient_id: "p2", note: "xray" }, + ], + }, + labs: { + comment: "Labs", + rows: [ + { id: "l1", patient_id: "p1", code: "hemoglobin" }, + { id: "l2", patient_id: "p2", code: "glucose" }, + ], + }, + }, + token: ctx.secretValues.dwhToken, + }; +} + +function inferColumnType(value) { + return typeof value === "number" ? "integer" : "text"; +} + +function topValues(rows, column, limit) { + const counts = new Map(); + for (const row of rows) { + const value = row[column]; + if (value === undefined || value === null || value === "") continue; + counts.set(String(value), (counts.get(String(value)) ?? 0) + 1); + } + return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, limit).map(([value]) => ({ value })); +} + +async function startHttpServer({ port, handler }) { + const server = createServer(async (req, res) => { + try { + await handler(req, res); + } catch { + res.statusCode = 500; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ error: "fixture failed" })); + } + }); + await new Promise((resolve, reject) => server.listen(port, "127.0.0.1", () => resolve()).on("error", reject)); + return server; +} + +async function startServers(ctx) { + const dwhServer = await startHttpServer({ + port: ctx.fixturePorts.dwh, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const json = body.length === 0 ? {} : JSON.parse(body); + if (req.headers["x-api-key"] !== ctx.dwhState.token) { + res.statusCode = 401; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ message: "unauthorized" })); + return; + } + const send = (payload) => { + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(payload)); + }; + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.dwh}`); + if (req.method !== "POST" || !url.pathname.startsWith("/rpc/")) { + res.statusCode = 404; + res.end(JSON.stringify({ message: "not found" })); + return; + } + const fn = url.pathname.slice("/rpc/".length); + const schemaName = json.schema_name ?? "dw"; + if (schemaName !== "dw") { + send([]); + return; + } + if (fn === "ping") { + send({ db_connected: true, schema_accessible: true, database: "warehouse", schema: "dw" }); + return; + } + const table = typeof json.table_name === "string" ? json.table_name : ""; + const tableData = ctx.dwhState.tables[table]; + if (fn === "list_tables") { + send(Object.entries(ctx.dwhState.tables).map(([name, info]) => ({ table: name, type: "TABLE", comment: info.comment, rows: info.rows.length }))); + return; + } + if (!tableData) { + send([]); + return; + } + if (fn === "table_columns") { + const first = tableData.rows[0] ?? {}; + send(Object.keys(first).map((column) => ({ + column, + type: inferColumnType(first[column]), + nullable: false, + // Only the referenced table marks `patient_id` as primary, so the SQL miner sees a + // PK/non-PK pair while the same-name heuristic still discovers joins from the others. + pk: column === "id" || (table === "patients" && column === "patient_id"), + default: null, + }))); + return; + } + if (fn === "table_comments") { + send(Object.keys(tableData.rows[0] ?? {}).map((column) => ({ object: "COLUMN", name: column, comment: `${table}.${column}` }))); + return; + } + if (fn === "table_foreign_keys") { + send([]); + return; + } + if (fn === "top_values") { + send(topValues(tableData.rows, json.column_name, Number(json.max_values ?? 10))); + return; + } + if (fn === "column_stats") { + send({}); + return; + } + if (fn === "run_query") { + send([]); + return; + } + if (fn === "explain_query") { + send([{ line: "Seq Scan" }]); + return; + } + res.statusCode = 404; + res.end(JSON.stringify({ message: "unknown rpc" })); + }, + }); + const evidenceServer = await startHttpServer({ + port: ctx.fixturePorts.evidence, + handler: async (req, res) => { + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.evidence}`); + if (url.pathname !== "/p2-dwh/guide.md" || url.searchParams.get("token") !== ctx.evidenceState.token) { + res.statusCode = 403; + res.end("forbidden"); + return; + } + res.statusCode = 200; + res.setHeader("content-type", "text/markdown; charset=utf-8"); + res.end(ctx.evidenceState.content); + }, + }); + const embeddingServer = await startHttpServer({ + port: ctx.fixturePorts.embedding, + handler: async (req, res) => { + const body = await new Promise((resolve) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const url = new URL(req.url, `http://127.0.0.1:${ctx.fixturePorts.embedding}`); + if (req.method !== "POST" || url.pathname !== "/api/embed") { + res.statusCode = 404; + res.end(JSON.stringify({ error: "not found" })); + return; + } + const payload = JSON.parse(body || "{}"); + const inputs = Array.isArray(payload.input) ? payload.input : []; + const embeddings = inputs.map((text) => { + const seed = sha256(String(text)); + return Array.from({ length: 1024 }, (_, index) => { + const offset = (index * 2) % seed.length; + const value = Number.parseInt(seed.slice(offset, offset + 2), 16); + return (value / 255) - 0.5; + }); + }); + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ model: payload.model, embeddings })); + }, + }); + ctx.servers = [dwhServer, evidenceServer, embeddingServer]; +} + +async function stopServers(ctx) { + for (const server of ctx.servers ?? []) { + await new Promise((resolve) => server.close(() => resolve())); + } + ctx.servers = []; +} + +async function git(ctx, args, cwd = join(ctx.run.root, "author")) { + return await runCommand({ executable: ctx.executables.gitPath, argv: args, cwd, env: ctx.execEnv }); +} + +async function initializeGitAndRegistry(ctx) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["init", "--bare", "--initial-branch=main", join(ctx.run.root, "remote.git")], ctx.run.root); + await git(ctx, ["clone", join(ctx.run.root, "remote.git"), author], ctx.run.root); + await git(ctx, ["config", "user.name", "P2 Fixture Curator"], author); + await git(ctx, ["config", "user.email", "p6-curator@example.invalid"], author); + + const writeWorkspaces = async () => { + const catalog = { + schema_version: 1, + workspaces: [ + { id: "p2-dwh", name: ctx.workspaceObjects.dwh.workspace.name }, + { id: "p2-filesystem", name: ctx.workspaceObjects.filesystem.workspace.name }, + ], + }; + await writeFile(join(author, "thoth-workspaces.yaml"), yamlStringify(catalog, { lineWidth: 0, sortMapEntries: false })); + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId), { recursive: true }); + const yaml = descriptorYaml(workspace); + await writeFile(join(author, pathId, "workspace.yaml"), yaml); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + await mkdir(join(author, "workspace-docs", pathId), { recursive: true }); + await writeFile(join(author, "workspace-docs", pathId, "contract.env.example"), docs.envExample); + await writeFile(join(author, "workspace-docs", pathId, "README.md"), docs.markdown); + } + for (const [id, workspace] of Object.entries(ctx.workspaceObjects)) { + const pathId = workspace.workspace.id; + await mkdir(join(author, pathId, "schema"), { recursive: true }); + await writeFile(join(author, pathId, "schema", "annotations.yaml"), ctx.curatedAnnotations[pathId]); + } + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "evidence", "guide.md"), "# P2 Filesystem Evidence\n\nCommitted fixture.\n"); + }; + + await writeWorkspaces(); + await git(ctx, ["add", "."], author); + await git(ctx, ["commit", "-m", "Bootstrap P2 fixtures"], author); + await git(ctx, ["push", "origin", "main"], author); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); + const registry = new ctx.workspaceModules.WorkspaceRegistry({ + root: join(ctx.run.root, "installation", "registry"), + remoteUrl: join(ctx.run.root, "remote.git"), + branch: "main", + gitAuthorName: "P2 Acceptance", + gitAuthorEmail: "p6-acceptance@example.invalid", + installationId: "p6-acceptance", + secretRoots: [join(ctx.run.root, "fixture-secrets")], + maxImportBytes: 16 * 1024 * 1024, + maxImportEntries: 1024, + dataRoot: join(ctx.run.root, "installation", "data"), + }); + await registry.bootstrap(); + ctx.registry = registry; +} + +async function mutateWorkspaceDescriptor(ctx, workspaceId, mutator, commitMessage) { + const author = join(ctx.run.root, "author"); + // The registry may have produced docs-only follow-up commits on the remote; the curator + // always rebases onto the latest remote head before committing so the push stays fast-forward. + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const workspace = structuredClone(ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"]); + mutator(workspace); + ctx.workspaceObjects[workspaceId === "p2-dwh" ? "dwh" : "filesystem"] = workspace; + await writeFile(join(author, workspaceId, "workspace.yaml"), descriptorYaml(workspace)); + const docs = ctx.workspaceModules.renderWorkspaceDocs(workspace); + const docsDir = join(author, "workspace-docs", workspaceId); + await mkdir(docsDir, { recursive: true, mode: 0o700 }); + await writeFile(join(docsDir, "contract.env.example"), docs.envExample); + await writeFile(join(docsDir, "README.md"), docs.markdown); + await git(ctx, ["add", `${workspaceId}/workspace.yaml`, `workspace-docs/${workspaceId}/contract.env.example`, `workspace-docs/${workspaceId}/README.md`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + + +async function mutateWorkspaceAnnotations(ctx, workspaceId, contents, commitMessage) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const annotationsPath = join(author, workspaceId, "schema", "annotations.yaml"); + await mkdir(dirname(annotationsPath), { recursive: true }); + await writeFile(annotationsPath, contents); + ctx.curatedAnnotations[workspaceId] = contents; + await git(ctx, ["add", `${workspaceId}/schema/annotations.yaml`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + await ctx.registry.pull(); + ctx.registryCommit = (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + +async function writeEvidenceAndPush(ctx, workspaceId, files, commitMessage) { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + const evidenceDir = join(author, workspaceId, "evidence"); + await rm(evidenceDir, { recursive: true, force: true }); + await mkdir(evidenceDir, { recursive: true }); + for (const [name, contents] of Object.entries(files)) { + const target = join(evidenceDir, name); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, contents); + } + await git(ctx, ["add", `${workspaceId}/evidence`], author); + await git(ctx, ["commit", "-m", commitMessage], author); + await git(ctx, ["push", "origin", "main"], author); + return (await git(ctx, ["rev-parse", "HEAD"], author)).stdout.trim(); +} + +async function writeInstallationFiles(ctx) { + const installationDir = join(ctx.run.root, "installation"); + const operatorEnvPath = join(installationDir, "operator.env"); + const bindingsEnvPath = join(installationDir, "workspace-bindings.env"); + const connectorOverridePath = join(installationDir, "connector-secrets.override.yaml"); + const fixtureOverridePath = join(installationDir, "fixture.override.yaml"); + const installationPath = join(installationDir, "thothii-installation.yaml"); + ctx.installationPath = installationPath; + ctx.composeProject = installationProjectName(installationPath); + const qdrantPort = ctx.fixturePorts.qdrant; + const bindings = [ + `THT_WS_P2_DWH_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_DWH_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_DWH_DWH_API_KEY_FILE=/run/secrets/p2-dwh-api-key`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_FILE=/run/secrets/p2-dwh-evidence-signed-urls`, + `THT_WS_P2_FILESYSTEM_DWH_TRANSPORT=rest_api`, + `THT_WS_P2_FILESYSTEM_DWH_BASE_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_FILE=/run/secrets/p2-filesystem-api-key`, + ].join("\n") + "\n"; + await atomicWrite(bindingsEnvPath, bindings); + const operatorEnv = [ + `THT_DATA_ROOT=${join(ctx.run.root, "installation", "data")}`, + `THT_WORKSPACE_REGISTRY_ROOT=${join(ctx.run.root, "installation", "registry")}`, + `THT_PI_STATE_ROOT=${join(ctx.run.root, "installation", "pi-state")}`, + `PI_AUTH_FILE=${join(ctx.run.root, "installation", "pi-auth.json")}`, + `THT_SECRETS_FILE=${ctx.secretPaths.bundle}`, + `THT_WORKSPACE_BINDINGS_ENV_FILE=${bindingsEnvPath}`, + `THT_WORKSPACE_GIT_REMOTE=${join(ctx.run.root, "remote.git")}`, + `THT_WORKSPACE_GIT_BRANCH=main`, + `THT_WORKSPACE_GIT_AUTHOR_NAME=P2 Acceptance`, + `THT_WORKSPACE_GIT_AUTHOR_EMAIL=p6-acceptance@example.invalid`, + `THT_WORKSPACE_INSTALLATION_ID=p6-acceptance`, + `THT_DB_NAME=warehouse`, + `THT_DWH_REST_URL=http://host.docker.internal:${ctx.fixturePorts.dwh}`, + `THT_LLM_URL=http://127.0.0.1:9`, + `THOTH_SERVER_BIND=127.0.0.1`, + `THOTH_HTTP_PORT=18080`, + `THOTH_CORE_HTTP_PORT=18787`, + `THT_WS_P2_DWH_DWH_API_KEY_SOURCE=${ctx.secretPaths.dwh}`, + `THT_WS_P2_DWH_EVIDENCE_SIGNED_URLS_SOURCE=${ctx.secretPaths.signed}`, + `THT_WS_P2_FILESYSTEM_DWH_API_KEY_SOURCE=${ctx.secretPaths.filesystemDwh}`, + `THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST=host.docker.internal`, + ].join("\n") + "\n"; + await atomicWrite(operatorEnvPath, operatorEnv); + await atomicWrite(join(ctx.run.root, "installation", "pi-auth.json"), JSON.stringify({ fixture: true })); + const embeddingStubPath = join(installationDir, "embedding-stub.py"); + await atomicWrite(embeddingStubPath, EMBEDDING_STUB_SOURCE); + const override = { + services: { + core: { + image: ctx.coreImageTag, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + "workspace-maintenance": { + image: ctx.coreImageTag, + environment: { + THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST: "host.docker.internal", + }, + extra_hosts: ["host.docker.internal:host-gateway"], + }, + qdrant: { + ports: [`127.0.0.1:${qdrantPort}:6333`], + restart: "no", + }, + // Deterministic Ollama-compatible embedding fixture on the internal allowlisted host + // name `embedding` (http://embedding:11434). Replaces the real Ollama service entirely. + embedding: { + image: ctx.coreImageTag, + entrypoint: ["python3", "/stub.py"], + volumes: [ + { type: "bind", source: embeddingStubPath, target: "/stub.py", read_only: true }, + ], + healthcheck: { disable: true }, + }, + }, + }; + await atomicWrite(fixtureOverridePath, yamlStringify(override, { lineWidth: 0, sortMapEntries: false })); + const generated = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "generate-connector-secrets-override.sh"), + argv: [ + "--bindings-env", bindingsEnvPath, + "--operator-env", operatorEnvPath, + "--output", connectorOverridePath, + "--service", "workspace-maintenance", + "--role", "all", + ], + env: ctx.execEnv, + }); + if (generated.exitCode !== 0) throw new Error(`connector override generation failed: ${generated.stderr || generated.stdout}`); + const installation = { + profile: "server", + projectDirectory: ctx.repositoryRoot, + envFile: operatorEnvPath, + overrides: [ + join(ctx.repositoryRoot, "deploy", "compose.server.yaml"), + fixtureOverridePath, + connectorOverridePath, + ], + }; + await atomicWrite(installationPath, yamlStringify(installation, { lineWidth: 0, sortMapEntries: false })); + ctx.installation = installation; +} + +function thothctlBinaryPath(repositoryRoot) { + const platform = { darwin: "darwin", linux: "linux", win32: "windows" }[process.platform] ?? "linux"; + const arch = { x64: "amd64", arm64: "arm64" }[process.arch] ?? "amd64"; + const suffix = platform === "windows" ? ".exe" : ""; + const candidates = [ + join(repositoryRoot, "dist", "thothctl", `thothctl-${platform}-${arch}${suffix}`), + join(repositoryRoot, "tools", "thothctl", "bin", `thothctl${suffix}`), + ]; + for (const candidate of candidates) if (existsSync(candidate)) return candidate; + throw new Error("built thothctl binary is unavailable"); +} + +async function runCommand({ executable, argv = [], cwd, env, input, maxOutputBytes = MAX_STDIO_BYTES }) { + const result = await execFileAsync(executable, argv, { + cwd, + env, + encoding: "utf8", + maxBuffer: maxOutputBytes, + ...(input === undefined ? {} : { input }), + }).then( + ({ stdout, stderr }) => ({ exitCode: 0, stdout, stderr }), + (error) => ({ exitCode: error.code ?? 1, stdout: error.stdout ?? "", stderr: error.stderr ?? error.message ?? "" }), + ); + return result; +} + +async function buildCoreImage(ctx) { + const tag = `thothii-core:p2-${ctx.run.runId.slice(3, 15)}`; + ctx.coreImageTag = tag; + const build = await runCommand({ + executable: ctx.executables.dockerPath, + argv: ["build", "-f", join(ctx.repositoryRoot, "docker", "core.Dockerfile"), "-t", tag, ctx.repositoryRoot], + env: { ...ctx.execEnv, DOCKER_BUILDKIT: "1" }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (build.exitCode !== 0) throw new Error(`core image build failed: ${build.stderr || build.stdout}`); +} + +async function buildThothctl(ctx) { + const command = await runCommand({ + executable: join(ctx.repositoryRoot, "scripts", "build-thothctl.sh"), + argv: [], + env: { ...ctx.execEnv, THT_THOTHCTL_OUTPUT_DIRECTORY: join(ctx.repositoryRoot, "dist", "thothctl") }, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (command.exitCode !== 0) throw new Error(`build-thothctl failed: ${command.stderr || command.stdout}`); + ctx.thothctlPath = thothctlBinaryPath(ctx.repositoryRoot); +} + +function composeBaseArgs(ctx) { + const args = [ + "compose", + "--project-name", ctx.composeProject, + "--project-directory", ctx.installation.projectDirectory, + "--env-file", ctx.installation.envFile, + "-f", join(ctx.repositoryRoot, "compose.yaml"), + ]; + for (const override of ctx.installation.overrides) args.push("-f", override); + return args; +} + +async function dockerCompose(ctx, commandArgs, { allowFailure = false, maxOutputBytes = 2 * 1024 * 1024 } = {}) { + const result = await runCommand({ + executable: ctx.executables.dockerPath, + argv: [...composeBaseArgs(ctx), ...commandArgs], + env: ctx.execEnv, + maxOutputBytes, + }); + if (!allowFailure && result.exitCode !== 0) throw new Error(`docker compose ${commandArgs.join(" ")} failed: ${result.stderr || result.stdout}`); + return result; +} + +async function startQdrant(ctx) { + await dockerCompose(ctx, ["up", "-d", "qdrant", "embedding"]); + for (let attempt = 0; attempt < 60; attempt += 1) { + try { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}/collections`); + if (response.ok) return; + } catch {} + await sleep(1000); + } + throw new Error("qdrant did not become ready"); +} + +async function qdrantJson(ctx, method, path, body) { + const response = await fetch(`http://127.0.0.1:${ctx.fixturePorts.qdrant}${path}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + const payload = response.status === 204 ? {} : await response.json().catch(() => ({})); + if (!response.ok) throw new Error(`qdrant request failed: ${method} ${path} ${response.status}`); + return payload; +} + +async function preprovisionCollection(ctx, workspaceId) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}`, { + vectors: { size: 1024, distance: "Cosine" }, + }); + for (const field of ["content_hash", "document_id", "kind", "record_key", "record_kind", "vector_generation", "workspace_id", "workspace_revision"]) { + await qdrantJson(ctx, "PUT", `/collections/${workspaceId}/index`, { field_name: field, field_schema: "keyword" }); + } +} + +async function listCollections(ctx) { + const payload = await qdrantJson(ctx, "GET", "/collections"); + const collections = payload.result?.collections ?? []; + return collections.map((item) => item.name).sort(); +} + +async function dumpQdrantPayloads(ctx, workspaceId) { + const response = await qdrantJson(ctx, "POST", `/collections/${workspaceId}/points/scroll`, { limit: 128, with_payload: true, with_vector: false }); + return JSON.stringify(response.result?.points ?? []); +} + +async function runThothctlJson(ctx, label, workspaceArgs, expectedExitCode) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.json`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs, "--json"], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + if (expectedExitCode !== undefined && result.exitCode !== expectedExitCode) { + throw new Error(`${label} exit ${result.exitCode} != ${expectedExitCode}`); + } + let payload; + try { payload = JSON.parse(result.stdout); } catch (error) { throw new Error(`${label} returned non-JSON stdout`); } + return { result, payload, artifacts: [await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath))] }; +} + +async function runThothctlRaw(ctx, label, workspaceArgs) { + const stdoutPath = join(ctx.run.root, "logs", `${label}.stdout.txt`); + const stderrPath = join(ctx.run.root, "logs", `${label}.stderr.txt`); + const result = await runCommand({ + executable: ctx.thothctlPath, + argv: ["--installation", ctx.installationPath, ...workspaceArgs], + env: ctx.execEnv, + maxOutputBytes: 2 * 1024 * 1024, + }); + await atomicWrite(stdoutPath, result.stdout || ""); + await atomicWrite(stderrPath, result.stderr || ""); + return { + result, + artifacts: [ + await fileArtifact(ctx.run.root, relative(ctx.run.root, stdoutPath)), + await fileArtifact(ctx.run.root, relative(ctx.run.root, stderrPath)), + ], + }; +} + +async function loadWorkspaceSnapshot(ctx, workspaceId) { + const active = JSON.parse(await readFile(join(ctx.run.root, "installation", "registry", "state", "active.json"), "utf8")); + const revision = active.revisions.find((entry) => entry.id === workspaceId); + const snapshotPath = revision.snapshotPath; + const contents = await readFile(snapshotPath, "utf8"); + return { active, revision, contents }; +} + +async function assertNoCoreFrontendRunning(ctx) { + const ps = await dockerCompose(ctx, ["ps", "--status", "running", "--format", "json"], { allowFailure: true }); + if (ps.exitCode !== 0) return []; + const lines = ps.stdout.trim() === "" ? [] : ps.stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); + const services = lines.map((item) => item.Service); + if (services.includes("core") || services.includes("frontend") || services.includes("workspace-maintenance")) { + throw new Error("core/frontend/maintenance is unexpectedly running"); + } + return services; +} + +function sameSet(left, right) { + return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort()); +} + +const EMBEDDING_STUB_SOURCE = String.raw`import json +from http.server import BaseHTTPRequestHandler, HTTPServer + +class _Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(length)) + inputs = payload.get("input", []) + if isinstance(inputs, str): + inputs = [inputs] + embeddings = [[0.01] * 1024 for _ in inputs] + body = json.dumps({"model": payload.get("model", "qwen3-embedding:0.6b"), "embeddings": embeddings}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + +HTTPServer(("0.0.0.0", 11434), _Handler).serve_forever() +`; + +function realUserHome() { + try { + const output = execFileSync("bash", ["-lc", 'printf "%s" ~'], { encoding: "utf8" }).trim(); + return output.length > 0 ? output : undefined; + } catch { + return undefined; + } +} + +async function setupRealContext({ repositoryRoot = defaultRepositoryRoot, env = process.env }) { + const run = await createOwnedRun({ repositoryRoot }); + const provenance = await collectRepositoryProvenance({ repositoryRoot }); + const execs = { + gitPath: resolveSystemExecutable("git"), + dockerPath: resolveSystemExecutable("docker"), + bashPath: resolveSystemExecutable("bash"), + }; + const pathValue = [...new Set([dirname(execs.gitPath), dirname(execs.dockerPath), "/usr/bin", "/bin", "/opt/homebrew/bin", "/usr/local/bin"])].join(":"); + // Docker CLI plugins (buildx) live under the real user's ~/.docker; the wrapper runs with a + // scrubbed environment, so derive the real home from the passwd entry and expose DOCKER_CONFIG. + const realHome = env.P6_REAL_HOME ?? realUserHome(); + const execEnv = buildSafeEnvironment({ ambient: env, fixture: { + PATH: pathValue, + HOME: run.root, + TMPDIR: join(run.root, "tmp"), + ...(realHome ? { DOCKER_CONFIG: join(realHome, ".docker") } : {}), + } }); + const workspaceModules = await import("../dist/workspaces/registry.js").then(async (registryModule) => ({ + WorkspaceRegistry: registryModule.WorkspaceRegistry, + ...(await import("../dist/workspaces/schema.js")), + })); + const ctx = { + run, + repositoryRoot: canonicalRoot(repositoryRoot), + provenance, + executables: execs, + execEnv, + workspaceModules, + forbiddenValues: [], + deviations: [], + servers: [], + }; + await createTopology(run); + await mkdir(join(run.root, "tmp"), { recursive: true, mode: 0o700 }); + await setupSecrets(ctx); + await setupFixtures(ctx); + return ctx; +} + +async function executeChecksLocal({ checks, failAt } = {}) { + if (!Array.isArray(checks) || !hasExactCheckIds(checks)) throw new Error("scenarios must match the exact ordered check set"); + if (failAt !== undefined && !CHECK_IDS.includes(failAt)) throw new Error("failure hook must name an exact check"); + const results = []; + let stopped = false; + for (const scenario of checks) { + const startedAt = nowIso(); + let result; + if (stopped) { + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: "Not executed after earlier failure." }; + } else { + try { + const output = await scenario.run(); + if (scenario.id === failAt) throw new Error("injected acceptance failure"); + result = { id: scenario.id, status: "PASS", startedAt, finishedAt: nowIso(), commands: output.commands ?? [], artifacts: output.artifacts ?? [] }; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + result = { id: scenario.id, status: "FAIL", startedAt, finishedAt: nowIso(), commands: [], artifacts: [], error: `Acceptance scenario failed safely: ${detail}` }; + stopped = true; + } + } + results.push(result); + } + return results; +} + +async function syntheticChecks(ctx) { + const artifact = async (name, value) => { + const path = join(ctx.run.root, "logs", `${name}.json`); + await writeJson(path, value); + return await fileArtifact(ctx.run.root, relative(ctx.run.root, path)); + }; + return CHECK_IDS.map((id, index) => ({ + id, + async run() { + return { + commands: [index === 0 ? "node" : "git"], + artifacts: [await artifact(id, { id, synthetic: true })], + }; + }, + })); +} + +async function realChecks(ctx) { + const state = {}; + const evidenceRoot = (workspaceId, commit) => join(ctx.run.root, "installation", "registry", "snapshots", commit, workspaceId, "evidence"); + const evidenceManifestPath = (workspaceId, commit) => join(ctx.run.root, "installation", "registry", "snapshots", commit, workspaceId, "evidence.manifest.json"); + const activeCommit = async (workspaceId) => (await loadWorkspaceSnapshot(ctx, workspaceId)).revision.commit; + const fileArtifactFrom = (path) => fileArtifact(ctx.run.root, relative(ctx.run.root, path)); + + return [ + { + id: "preflight", + async run() { + await buildThothctl(ctx); + await buildCoreImage(ctx); + await writeInstallationFiles(ctx); + return { + commands: ["docker", "node", "git"], + artifacts: [{ path: "logs/provenance.json", sha256: sha256(JSON.stringify(ctx.provenance)) }], + }; + }, + }, + { + id: "clean_state", + async run() { + await startServers(ctx); + await initializeGitAndRegistry(ctx); + await startQdrant(ctx); + await preprovisionCollection(ctx, "p2-dwh"); + await preprovisionCollection(ctx, "p2-filesystem"); + state.collectionsBefore = await listCollections(ctx); + state.runningServices = await assertNoCoreFrontendRunning(ctx); + state.initialCommit = await activeCommit("p2-filesystem"); + await writeJson(join(ctx.run.root, "logs", "collections-before.json"), state.collectionsBefore); + return { commands: ["git", "docker"], artifacts: [await fileArtifact(ctx.run.root, "logs/collections-before.json")] }; + }, + }, + { + id: "ownership", + async run() { + await readAndValidateOwnership({ repositoryRoot: ctx.repositoryRoot, runRoot: ctx.run.root, expectedNonce: ctx.run.nonce }); + const installStat = await stat(ctx.installationPath); + assert(installStat.isFile(), "installation descriptor missing"); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "ownership.json")] }; + }, + }, + { + id: "activation_materialization", + async run() { + const commit = state.initialCommit; + const guide = join(evidenceRoot("p2-filesystem", commit), "guide.md"); + assert(readFileSync(guide, "utf8") === "# P2 Filesystem Evidence\n\nCommitted fixture.\n", "materialized Evidence content mismatch"); + const manifest = JSON.parse(readFileSync(evidenceManifestPath("p2-filesystem", commit), "utf8")); + assert(manifest.workspace === "p2-filesystem", "Evidence manifest workspace mismatch"); + assert(manifest.commit === commit, "Evidence manifest commit mismatch"); + assert(manifest.entryCount === 1, "Evidence manifest entry count mismatch"); + const snapshot = JSON.parse(readFileSync(join(ctx.run.root, "installation", "registry", "snapshots", commit, "snapshot.json"), "utf8")); + assert(/^[0-9a-f]{64}$/.test(snapshot.files["p2-filesystem/evidence.manifest.json"] ?? ""), "snapshot manifest lacks the Evidence manifest digest"); + return { commands: [], artifacts: [await fileArtifactFrom(guide), await fileArtifactFrom(evidenceManifestPath("p2-filesystem", commit))] }; + }, + }, + { + id: "evidence_preprocess", + async run() { + const dryRun = await runThothctlJson(ctx, "evidence-dry-run", ["workspace", "preprocess", "evidence", "--workspace", "p2-filesystem", "--dry-run"], 0); + assert(dryRun.payload.status === "dry_run", "Evidence dry-run failed"); + const first = await runThothctlJson(ctx, "evidence-run", ["workspace", "preprocess", "evidence", "--workspace", "p2-filesystem"], 0); + assert(first.payload.status === "succeeded" && first.payload.code === "ok", "Evidence run failed"); + const rerun = await runThothctlJson(ctx, "evidence-rerun", ["workspace", "preprocess", "evidence", "--workspace", "p2-filesystem"], 0); + assert(["unchanged", "succeeded"].includes(rerun.payload.status), "Evidence rerun not idempotent"); + + // Evidence records are revision-scoped in Qdrant. + const base = `http://127.0.0.1:${ctx.fixturePorts.qdrant}`; + const scroll = await fetch(`${base}/collections/p2-filesystem/points/scroll?limit=500`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ with_payload: true, with_vector: false }), + }).then((res) => res.json()); + const points = scroll.result?.points ?? []; + const evidence = points.filter((point) => (point.payload?.record_kind ?? point.payload?.kind ?? "") === "evidence"); + assert(evidence.length > 0, "no revision-scoped Evidence records found"); + assert(evidence.every((point) => /^[0-9a-f]{40}$/.test(point.payload?.workspace_revision ?? "")), "Evidence records lack workspace_revision"); + return { commands: ["thothctl"], artifacts: [...dryRun.artifacts, ...first.artifacts, ...rerun.artifacts] }; + }, + }, + { + id: "revision_isolation", + async run() { + await writeEvidenceAndPush(ctx, "p2-filesystem", { "guide.md": "# P2 Filesystem Evidence\n\nSecond generation.\n" }, "Evidence content v2"); + await ctx.registry.pull(); + const revisionB = await activeCommit("p2-filesystem"); + assert(revisionB !== state.initialCommit, "Evidence commit did not change the revision"); + const guideB = join(evidenceRoot("p2-filesystem", revisionB), "guide.md"); + assert(readFileSync(guideB, "utf8").includes("Second generation"), "revision B Evidence not materialized"); + state.revisionB = revisionB; + return { commands: ["git"], artifacts: [await fileArtifactFrom(guideB)] }; + }, + }, + { + id: "unsafe_tree_refused", + async run() { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + await rm(join(author, "p2-filesystem", "evidence"), { recursive: true, force: true }); + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "outside.md"), "# outside\n"); + symlinkSync("../outside.md", join(author, "p2-filesystem", "evidence", "link.md")); + await git(ctx, ["add", "p2-filesystem/evidence", "p2-filesystem/outside.md"], author); + await git(ctx, ["commit", "-m", "Unsafe Evidence symlink"], author); + await git(ctx, ["push", "origin", "main"], author); + let refused = false; + try { await ctx.registry.pull(); } catch (error) { refused = String(error?.code ?? error) === "workspace_invalid"; } + assert(refused, "unsafe Evidence tree did not fail closed"); + assert(await activeCommit("p2-filesystem") === state.revisionB, "unsafe pull changed the active revision"); + return { commands: ["git"], artifacts: [] }; + }, + }, + { + id: "bound_refused", + async run() { + const author = join(ctx.run.root, "author"); + await git(ctx, ["fetch", "origin", "main"], author); + await git(ctx, ["reset", "--hard", "origin/main"], author); + await rm(join(author, "p2-filesystem", "evidence"), { recursive: true, force: true }); + await mkdir(join(author, "p2-filesystem", "evidence"), { recursive: true }); + await writeFile(join(author, "p2-filesystem", "evidence", "big.md"), `# big\n${"x".repeat(9 * 1024 * 1024)}`); + await git(ctx, ["add", "p2-filesystem/evidence"], author); + await git(ctx, ["commit", "-m", "Oversized Evidence file"], author); + await git(ctx, ["push", "origin", "main"], author); + let refused = false; + try { await ctx.registry.pull(); } catch (error) { refused = String(error?.code ?? error) === "workspace_invalid"; } + assert(refused, "oversized Evidence file did not fail closed"); + assert(await activeCommit("p2-filesystem") === state.revisionB, "bound-violating pull changed the active revision"); + return { commands: ["git"], artifacts: [] }; + }, + }, + { + id: "secret_scan", + async run() { + const virtualFiles = []; + const qdrantDump = await dumpQdrantPayloads(ctx, "p2-filesystem"); + if (Buffer.byteLength(qdrantDump) <= MAX_SECRET_SCAN_VIRTUAL_BYTES) virtualFiles.push({ path: "virtual/qdrant-p2-filesystem.json", bytes: qdrantDump }); + const findings = await scanSecrets({ + runRoot: ctx.run.root, + forbiddenValues: ctx.forbiddenValues, + virtualFiles, + expectedGitRepositories: ["remote.git", "author"], + }); + await writeJson(join(ctx.run.root, "logs", "secret-scan.json"), findings); + if (findings.length > 0) throw new Error(`secret scan found ${findings.length} leak(s)`); + return { commands: ["git"], artifacts: [await fileArtifact(ctx.run.root, "logs/secret-scan.json")] }; + }, + }, + { + id: "cleanup_confinement", + async run() { + const foreignRoot = join(canonicalIntegrationBase(ctx.repositoryRoot), `p6-${"f".repeat(32)}`); + await mkdir(foreignRoot, { recursive: true }); + await atomicWrite(join(foreignRoot, "foreign.txt"), "foreign"); + assert(readFileSync(join(foreignRoot, "foreign.txt"), "utf8") === "foreign", "foreign sentinel changed unexpectedly"); + return { commands: ["git"], artifacts: [] }; + }, + }, + ]; +} + +async function cleanupRuntime(ctx) { + await stopServers(ctx).catch(() => {}); + if (ctx.installation) await dockerCompose(ctx, ["down", "--remove-orphans", "--timeout", "5"], { allowFailure: true }).catch(() => {}); + if (ctx.coreImageTag) await runCommand({ executable: ctx.executables.dockerPath, argv: ["image", "rm", "-f", ctx.coreImageTag], env: ctx.execEnv, maxOutputBytes: MAX_STDIO_BYTES }).catch(() => {}); +} + +export async function runIntegration({ repositoryRoot = defaultRepositoryRoot, keep = false, env = process.env, announce } = {}) { + const synthetic = env.P6_ACCEPTANCE_SYNTHETIC === "1"; + const failAt = env.P6_ACCEPTANCE_FAIL_AT; + const ctx = synthetic + ? { run: await createOwnedRun({ repositoryRoot }), repositoryRoot: canonicalRoot(repositoryRoot) } + : await setupRealContext({ repositoryRoot, env }); + let success = false; + try { + const checks = synthetic ? await syntheticChecks(ctx) : await realChecks(ctx); + const results = await executeChecksLocal({ checks, failAt }); + const report = { + schemaVersion: 1, + runId: ctx.run.runId, + startedAt: ctx.run.startedAt, + finishedAt: nowIso(), + command: "p6-acceptance integration --keep", + overall: deriveOverall(results), + checks: results, + }; + await writeReportFiles({ run: ctx.run, report }); + success = report.overall === "PASS"; + if (announce) await announce({ report, runRoot: ctx.run.root }); + return { exitCode: success ? 0 : 1, runRoot: ctx.run.root, retained: !(await finalizeOwnedRun({ run: ctx.run, success, keep })) }; + } finally { + if (!synthetic) await cleanupRuntime(ctx).catch(() => {}); + } +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + if (argv.length < 1 || argv[0] !== "integration" || argv.length > 2 || (argv[1] && argv[1] !== "--keep")) { + throw new Error("usage: p6-acceptance.mjs integration [--keep]"); + } + const result = await runIntegration({ keep: argv.includes("--keep"), env }); + return result.exitCode; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === modulePath) { + try { + const code = await main(); + process.exitCode = code; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +export { CHECK_IDS }; diff --git a/backend/scripts/p6-acceptance.test.mjs b/backend/scripts/p6-acceptance.test.mjs new file mode 100644 index 00000000..97e9d45c --- /dev/null +++ b/backend/scripts/p6-acceptance.test.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_IDS, + canonicalIntegrationBase, + cleanupOwnedRun, + createOwnedRun, + readAndValidateOwnership, + runIntegration, + validateReport, + validateRunRoot, +} from "./p6-acceptance.mjs"; + +const roots = []; +async function fakeRepository() { + const root = await mkdtemp(join(tmpdir(), "p6-acceptance-repo-")); + roots.push(root); + await mkdir(join(root, ".artifacts", "p6-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p2-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "p1-integration"), { recursive: true }); + await mkdir(join(root, ".artifacts", "manual-acceptance", "p11"), { recursive: true }); + return root; +} + +test.afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +test("run roots are only canonical direct p6 integration children", async () => { + const repositoryRoot = await fakeRepository(); + const base = canonicalIntegrationBase(repositoryRoot); + const id = `p6-${"a".repeat(32)}`; + assert.equal(validateRunRoot(repositoryRoot, join(base, id), id), join(base, id)); + for (const candidate of [ + base, + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(repositoryRoot, ".artifacts", "p1-integration", id), + join(repositoryRoot, ".artifacts", "p2-integration", id), + join(base, id, "nested"), + join(base, "foreign"), + ]) { + assert.throws(() => validateRunRoot(repositoryRoot, candidate, id)); + } + assert.throws(() => validateRunRoot(repositoryRoot, join(base, `p6-${"A".repeat(32)}`), `p6-${"A".repeat(32)}`)); +}); + +test("cleanup refuses p1, p2, p11, manual, sibling, and wrong-nonce roots", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + await readAndValidateOwnership({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + for (const bad of [ + join(repositoryRoot, ".artifacts", "p1-integration", `p1-${"b".repeat(32)}`), + join(repositoryRoot, ".artifacts", "p2-integration", `p2-${"c".repeat(32)}`), + join(repositoryRoot, ".artifacts", "manual-acceptance", "p11"), + join(canonicalIntegrationBase(repositoryRoot), `p6-${"d".repeat(32)}`), + ]) { + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: bad, expectedNonce: run.nonce })); + } + await assert.rejects(cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: "0".repeat(64) })); +}); + +test("cleanup removes exactly one owned p6 root", async () => { + const repositoryRoot = await fakeRepository(); + const run = await createOwnedRun({ repositoryRoot }); + const sibling = join(canonicalIntegrationBase(repositoryRoot), `p6-${"e".repeat(32)}`); + await mkdir(sibling); + await writeFile(join(sibling, "sentinel"), "foreign"); + await cleanupOwnedRun({ repositoryRoot, runRoot: run.root, expectedNonce: run.nonce }); + await assert.rejects(readFile(join(run.root, "ownership.json"))); + assert.equal(await readFile(join(sibling, "sentinel"), "utf8"), "foreign"); +}); + +function resultFor(id) { + return { + id, + status: "PASS", + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:01.000Z", + commands: ["node"], + artifacts: [{ path: `logs/${id}.json`, sha256: "a".repeat(64) }], + }; +} + +test("report validation requires exact p6 identity, check order, and unique artifacts", () => { + const report = { + schemaVersion: 1, + runId: `p6-${"f".repeat(32)}`, + startedAt: "2026-08-12T00:00:00.000Z", + finishedAt: "2026-08-12T00:00:10.000Z", + command: "p6-acceptance integration --keep", + overall: "PASS", + checks: CHECK_IDS.map(resultFor), + }; + assert.doesNotThrow(() => validateReport(report)); + const invalid = structuredClone(report); + invalid.runId = `p2-${"f".repeat(32)}`; + assert.throws(() => validateReport(invalid)); + const duplicate = structuredClone(report); + duplicate.checks[1].artifacts[0].path = duplicate.checks[0].artifacts[0].path; + assert.throws(() => validateReport(duplicate), /duplicated/); + const reordered = structuredClone(report); + reordered.checks.reverse(); + reordered.overall = "FAIL"; + assert.throws(() => validateReport(reordered)); +}); + +test("public wrapper uses a strict empty environment", async () => { + const wrapper = await readFile(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "p6-acceptance.sh"), "utf8"); + assert.match(wrapper, /safe_env=\(\/usr\/bin\/env -i/); + assert.doesNotMatch(wrapper, /LANG|LC_ALL|TZ/); + assert.doesNotMatch(wrapper, /P6_ACCEPTANCE_FAIL_AT/); +}); + +test("synthetic integration cleans up successful non-kept runs", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: false, env: { P6_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, false); + await assert.rejects(readFile(join(result.runRoot, "ownership.json"))); +}); + +test("synthetic integration retains kept runs with bounded reports", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ repositoryRoot, keep: true, env: { P6_ACCEPTANCE_SYNTHETIC: "1" } }); + assert.equal(result.exitCode, 0); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "PASS"); + const reportMd = await readFile(join(result.runRoot, "report.md"), "utf8"); + assert.match(reportMd, /P6 automated integration: PASS/); + assert.match(reportMd, /P6 manual acceptance: PENDING/); + const reportJsonStat = await stat(join(result.runRoot, "report.json")); + const reportMdStat = await stat(join(result.runRoot, "report.md")); + assert.ok(reportJsonStat.size <= 64 * 1024, `report.json too large: ${reportJsonStat.size}`); + assert.ok(reportMdStat.size <= 32 * 1024, `report.md too large: ${reportMdStat.size}`); +}); + +test("synthetic injected failure retains the owned run and records a single failed report", async () => { + const repositoryRoot = await fakeRepository(); + const result = await runIntegration({ + repositoryRoot, + keep: false, + env: { P6_ACCEPTANCE_SYNTHETIC: "1", P6_ACCEPTANCE_FAIL_AT: CHECK_IDS[2] }, + }); + assert.equal(result.exitCode, 1); + assert.equal(result.retained, true); + const report = JSON.parse(await readFile(join(result.runRoot, "report.json"), "utf8")); + assert.equal(report.overall, "FAIL"); + const failed = report.checks.find((check) => check.id === CHECK_IDS[2]); + assert.equal(failed.status, "FAIL"); + const roots = await readFile(join(result.runRoot, "ownership.json"), "utf8"); + assert.match(roots, /p6-acceptance/); +}); diff --git a/backend/scripts/revision-state-policy.mjs b/backend/scripts/revision-state-policy.mjs new file mode 100644 index 00000000..79af7e6e --- /dev/null +++ b/backend/scripts/revision-state-policy.mjs @@ -0,0 +1,943 @@ +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +import ts from "typescript"; +import { literalBashHeredocBodyRanges } from "./bash-heredoc.mjs"; +import { isMap, isScalar, isSeq, parseAllDocuments } from "yaml"; + + +/** + * Revision-state absence policy by source dialect. + * JS/TS syntax uses the TypeScript parser and YAML structure uses the installed YAML parser. + * Shell active consumers are executable code/expansions and jq filter arguments for bare or + * path-qualified jq, optionally through command or env. Quoted heredoc bodies are literal. + * PowerShell analyzes executable code and nested $() in expandable strings. Python policy is + * batched through the isolated stdlib AST helper. jq filters use a bounded path lexer after + * shell argv/wrapper resolution. Offset-preserving transformations keep AST spans stable. + */ +const revisionIdentifiers = new Set(["revision", "workspaceRevision", "selectedWorkspace"]); + +function unwrapExpression(node) { + let current = node; + while (ts.isParenthesizedExpression(current) || ts.isAsExpression(current) || + ts.isTypeAssertionExpression(current) || ts.isNonNullExpression(current) || + ts.isSatisfiesExpression(current)) { + current = current.expression; + } + return current; +} + +function isRevisionName(value, caseInsensitive) { + if (typeof value !== "string") return false; + if (!caseInsensitive) return revisionIdentifiers.has(value); + const lower = value.toLowerCase(); + return lower === "revision" || lower === "workspacerevision" || lower === "selectedworkspace"; +} + +function isRevisionExpression(node, caseInsensitive = false) { + const unwrapped = unwrapExpression(node); + if (ts.isIdentifier(unwrapped)) { + const normalized = unwrapped.text.startsWith("$") && !unwrapped.text.startsWith("$$") ? unwrapped.text.slice(1) : unwrapped.text; + return isRevisionName(normalized, caseInsensitive); + } + if (ts.isPropertyAccessExpression(unwrapped)) return isRevisionName(unwrapped.name.text, caseInsensitive); + if (ts.isElementAccessExpression(unwrapped) && unwrapped.argumentExpression) { + return isRevisionName(staticStringValue(unwrapped.argumentExpression), caseInsensitive); + } + return false; +} + +function staticStringValue(node) { + const expression = unwrapExpression(node); + if (ts.isStringLiteral(expression) || ts.isNoSubstitutionTemplateLiteral(expression)) return expression.text; + if (ts.isTemplateExpression(expression)) { + let value = expression.head.text; + for (const span of expression.templateSpans) { + const part = staticStringValue(span.expression); + if (part === undefined) return undefined; + value += part + span.literal.text; + } + return value; + } + if (ts.isBinaryExpression(expression) && expression.operatorToken.kind === ts.SyntaxKind.PlusToken) { + const left = staticStringValue(expression.left); + const right = staticStringValue(expression.right); + return left === undefined || right === undefined ? undefined : left + right; + } + return undefined; +} + +function propertyNameText(name, caseInsensitive = false) { + if (!name) return undefined; + let value; + if (ts.isComputedPropertyName(name)) value = staticStringValue(name.expression); + else if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNoSubstitutionTemplateLiteral(name) || ts.isNumericLiteral(name)) value = name.text; + else value = staticStringValue(name); + return caseInsensitive && typeof value === "string" ? value.toLowerCase() : value; +} + +function objectBindingHasState(pattern, caseInsensitive) { + return pattern.elements.some((element) => { + if (element.dotDotDotToken) return false; + return propertyNameText(element.propertyName ?? element.name, caseInsensitive) === "state"; + }); +} + +function objectLiteralHasState(object, caseInsensitive) { + return object.properties.some((property) => + !ts.isSpreadAssignment(property) && propertyNameText(property.name, caseInsensitive) === "state"); +} + +function scriptKindFor(path) { + const lower = path.toLowerCase(); + if (lower.endsWith(".tsx")) return ts.ScriptKind.TSX; + if (lower.endsWith(".jsx")) return ts.ScriptKind.JSX; + if (/\.(?:ts|mts|cts)$/u.test(lower)) return ts.ScriptKind.TS; + if (/\.(?:js|mjs|cjs)$/u.test(lower)) return ts.ScriptKind.JS; + return undefined; +} + +function maskRange(output, source, start, end, keepEnds = false) { + for (let cursor = start; cursor < end; cursor += 1) { + if (source[cursor] === "\n" || source[cursor] === "\r") continue; + if (keepEnds && (cursor === start || cursor === end - 1)) continue; + output[cursor] = " "; + } +} + +function lineEnd(source, start) { + const end = source.indexOf("\n", start); + return end < 0 ? source.length : end; +} + +function quotedEnd(source, start, delimiter, escapes = "\\") { + for (let cursor = start + delimiter.length; cursor < source.length; cursor += 1) { + if (escapes.includes(source[cursor])) { + cursor += 1; + continue; + } + if (source.startsWith(delimiter, cursor)) return cursor + delimiter.length; + } + return source.length; +} + +function balancedEnd(source, openIndex, opener, closer, escapes = "\\`") { + let depth = 1; + for (let cursor = openIndex + 1; cursor < source.length; cursor += 1) { + if (escapes.includes(source[cursor])) { + cursor += 1; + continue; + } + if (source[cursor] === "'" || source[cursor] === '"' || source[cursor] === "`") { + cursor = quotedEnd(source, cursor, source[cursor], escapes) - 1; + continue; + } + if (source[cursor] === opener) depth += 1; + else if (source[cursor] === closer && --depth === 0) return cursor; + } + return source.length - 1; +} + +function restoreMasked(output, offset, masked) { + for (let cursor = 0; cursor < masked.length; cursor += 1) output[offset + cursor] = masked[cursor]; +} + +function exposeDollarSubexpressions(output, source, start, end, dialect) { + for (let cursor = start; cursor + 1 < end; cursor += 1) { + if (!source.startsWith("$(", cursor) || source[cursor - 1] === "`") continue; + const close = balancedEnd(source, cursor + 1, "(", ")"); + output[cursor] = " "; + output[cursor + 1] = "("; + restoreMasked(output, cursor + 2, dialect === "shell" ? maskShellSource(source.slice(cursor + 2, close)) : maskPowerShellSource(source.slice(cursor + 2, close))); + if (close < source.length) output[close] = ")"; + cursor = close; + } +} + + +function shellCommentStart(source, index) { + return source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1])); +} + +function canonicalRevisionName(name) { + const lower = name.toLowerCase(); + if (lower === "revision") return "revision"; + if (lower === "workspacerevision") return "workspaceRevision"; + return "selectedWorkspace"; +} + +function normalizePowerShellVariables(source) { + const output = source.split(""); + const patterns = [ + { expression: /\$\{(?:[A-Za-z_][A-Za-z0-9_]*:)?(revision|workspaceRevision|selectedWorkspace)\}/giu, dollar: false }, + { expression: /\$(?:[A-Za-z_][A-Za-z0-9_]*:)(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: false }, + { expression: /\$(revision|workspaceRevision|selectedWorkspace)\b/giu, dollar: true }, + ]; + for (const { expression, dollar } of patterns) { + for (const match of source.matchAll(expression)) { + const name = canonicalRevisionName(match[1]); + const replacement = `${dollar ? "$" : ""}${name}`.padEnd(match[0].length, " "); + for (let offset = 0; offset < match[0].length; offset += 1) output[match.index + offset] = replacement[offset]; + } + } + let normalized = output.join(""); + normalized = normalized.replace(/\.\s*state\b/giu, (match) => match.replace(/state/iu, "state")); + normalized = normalized.replace(/(["'])state\1/giu, (_match, quote) => `${quote}state${quote}`); + return normalized; +} + +function maskShellSource(source) { + return maskShellFamilySource(source, false); +} + +function maskPowerShellSource(source) { + return normalizePowerShellVariables(maskShellFamilySource(source, true)); +} + +function maskShellFamilySource(source, powershell) { + const output = source.split(""); + let squareDepth = 0; + for (let index = 0; index < source.length; index += 1) { + if (powershell && source.startsWith("<#", index)) { + const close = source.indexOf("#>", index + 2); + const end = close < 0 ? source.length : close + 2; + maskRange(output, source, index, end); + index = end - 1; + continue; + } + if (powershell ? source[index] === "#" : shellCommentStart(source, index)) { + const end = lineEnd(source, index); + maskRange(output, source, index, end); + index = end - 1; + continue; + } + if (powershell && source[index] === "`") { + maskRange(output, source, index, Math.min(index + 2, source.length)); + index += 1; + continue; + } + if (!powershell && source[index] === "`") { + const close = source.indexOf("`", index + 1); + const end = close < 0 ? source.length : close + 1; + maskRange(output, source, index, end); + restoreMasked(output, index + 1, maskShellSource(source.slice(index + 1, close < 0 ? source.length : close))); + index = end - 1; + continue; + } + const quote = source[index]; + if (quote === "'" || quote === '"') { + const escapes = powershell ? "`" : quote === "'" ? "" : "\\"; + const end = quotedEnd(source, index, quote, escapes); + const preserveKey = powershell && squareDepth > 0; + if (!preserveKey) maskRange(output, source, index, end, false); + if (quote === '"') { + exposeDollarSubexpressions(output, source, index + 1, end - 1, powershell ? "powershell" : "shell"); + if (!powershell) { + for (let cursor = index + 1; cursor < end - 1; cursor += 1) { + if (source[cursor] !== "`" || source[cursor - 1] === "\\") continue; + const close = source.indexOf("`", cursor + 1); + if (close < 0 || close >= end) break; + restoreMasked(output, cursor + 1, maskShellSource(source.slice(cursor + 1, close))); + cursor = close; + } + } + } + index = end - 1; + continue; + } + if (source.startsWith("$(", index)) output[index] = " "; + if (source[index] === "[") squareDepth += 1; + else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1; + } + return output.join(""); +} + +function maskUnknownSource(source) { + const output = source.split(""); + let squareDepth = 0; + for (let index = 0; index < source.length; index += 1) { + if (source.startsWith("/*", index)) { + const close = source.indexOf("*/", index + 2); + const end = close < 0 ? source.length : close + 2; + maskRange(output, source, index, end); + index = end - 1; + continue; + } + if (source[index] === "#" || source.startsWith("//", index)) { + const end = lineEnd(source, index); + maskRange(output, source, index, end); + index = end - 1; + continue; + } + const quote = source[index]; + if (quote === "'" || quote === '"' || quote === "`") { + const end = quotedEnd(source, index, quote, "\\"); + let after = end; + while (/[ \t]/u.test(source[after] ?? "")) after += 1; + if (!(squareDepth > 0 || source[after] === ":")) maskRange(output, source, index, end, true); + index = end - 1; + continue; + } + if (source[index] === "[") squareDepth += 1; + else if (source[index] === "]" && squareDepth > 0) squareDepth -= 1; + } + return output.join(""); +} + +function maskQuotedShellHeredocBodies(source, label = "") { + const output = source.split(""); + for (const range of literalBashHeredocBodyRanges(source, label)) maskRange(output, source, range.start, range.end); + return output.join(""); +} + +function shellAssociativeRevisionAccess(source) { + let quote; + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + if (character === "\\") { index += 1; continue; } + if (quote === "'") { if (character === "'") quote = undefined; continue; } + if (character === "'") { quote = "'"; continue; } + if (character === '"') { quote = quote === '"' ? undefined : '"'; continue; } + if (character !== "$" || source[index + 1] !== "{") continue; + const close = source.indexOf("}", index + 2); + if (close < 0) break; + const expansion = source.slice(index, close + 1); + if (/^\$\{[ \t]*(?:revision|workspaceRevision|selectedWorkspace)[ \t]*\[[ \t]*(?:["']state["']|state)[ \t]*\][^}]*\}$/u.test(expansion)) return true; + index = close; + } + return false; +} + +const shellCommandPrefixes = new Set(["if", "then", "elif", "else", "while", "until", "do"]); +const shellCommandClosers = new Set(["fi", "done", "esac"]); +const shellControlCharacters = new Set([";", "|", "&", "(", ")", "{", "}", "`"]); + +function shellQuotedSubstitutionEnd(source, start, depth, budget) { + for (let index = start + 1; index < source.length; index += 1) { + budget.characters += 1; + if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded"); + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === '"') return index + 1; + if (source.startsWith("$(", index) || source.startsWith("<(", index) || source.startsWith(">(", index)) { + index = shellParenthesizedEnd(source, index + 1, depth + 1, budget) - 1; + } else if (source[index] === "`") { + const end = quotedEnd(source, index, "`", "\\"); + if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick"); + index = end - 1; + } + } + throw new Error("revision-state shell substitution has an unclosed quote"); +} + +function shellParenthesizedEnd(source, openIndex, depth, budget) { + if (depth > 64) throw new Error("revision-state shell substitution nesting limit exceeded"); + for (let index = openIndex + 1; index < source.length; index += 1) { + budget.characters += 1; + if (budget.characters > 100_000) throw new Error("revision-state shell substitution size limit exceeded"); + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === "'") { + const end = quotedEnd(source, index, "'", ""); + if (end - 1 <= index || source[end - 1] !== "'") throw new Error("revision-state shell substitution has an unclosed quote"); + index = end - 1; + continue; + } + if (source[index] === '"') { index = shellQuotedSubstitutionEnd(source, index, depth, budget) - 1; continue; } + if (source[index] === "`") { + const end = quotedEnd(source, index, "`", "\\"); + if (end - 1 <= index || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick"); + index = end - 1; + continue; + } + if (source[index] === "#" && (index === openIndex + 1 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) { + index = lineEnd(source, index); + continue; + } + if (source[index] === "(") { index = shellParenthesizedEnd(source, index, depth + 1, budget) - 1; continue; } + if (source[index] === ")") return index + 1; + } + throw new Error("revision-state shell process substitution is unbalanced"); +} + +function shellProcessSubstitutionEnd(source, start) { + if (!(source.startsWith("<(", start) || source.startsWith(">(", start))) return undefined; + return shellParenthesizedEnd(source, start + 1, 1, { characters: 0 }); +} + +function shellRedirectionAt(source, start) { + const match = source.slice(start).match(/^(?:&>>|&>|(?:[0-9]+|\{[A-Za-z_][A-Za-z0-9_]*\})?(?:<<<|<<-|<<|>>|<>|>\||<&|>&|<|>))/u); + if (!match) return undefined; + let end = start + match[0].length; + while (end < source.length && !/\s/u.test(source[end]) && !shellControlCharacters.has(source[end]) && + source[end] !== "<" && source[end] !== ">" && source[end] !== "'" && source[end] !== '"') end += 1; + return { value: source.slice(start, end), end, needsOperand: end === start + match[0].length }; +} + +function shellLexTokens(source) { + const tokens = []; + const push = (value, start, end, type = "word") => { + tokens.push({ value, start, end, type }); + if (tokens.length > 50_000) throw new Error("revision-state shell token limit exceeded"); + }; + for (let index = 0; index < source.length;) { + if (source[index] === "\n" || source[index] === "\r") { push(source[index], index, index + 1, "control"); index += 1; continue; } + if (/\s/u.test(source[index])) { index += 1; continue; } + if (source[index] === "#") { index = lineEnd(source, index); continue; } + const processEnd = shellProcessSubstitutionEnd(source, index); + if (processEnd !== undefined) { + push(source.slice(index, processEnd), index, processEnd); + index = processEnd; + continue; + } + const redirection = shellRedirectionAt(source, index); + if (redirection) { + push(redirection.value, index, redirection.end, "redirection"); + tokens.at(-1).needsOperand = redirection.needsOperand; + index = redirection.end; + continue; + } + if (shellControlCharacters.has(source[index]) || source[index] === "!" && (index === 0 || /\s/u.test(source[index - 1]))) { + const start = index; + let value = source[index++]; + if ((value === ";" || value === "|" || value === "&") && source[index] === value) value += source[index++]; + push(value, start, index, "control"); + continue; + } + const start = index; + let value = ""; + while (index < source.length && !/\s/u.test(source[index]) && !shellControlCharacters.has(source[index]) && source[index] !== "<" && source[index] !== ">") { + const quote = source[index]; + if (quote === "'" || quote === '"') { + const end = quotedEnd(source, index, quote, "\\"); + value += source.slice(index + 1, end - 1); + index = end; + } else if (source[index] === "\\" && index + 1 < source.length) { + value += source[index + 1]; + index += 2; + } else { + value += source[index++]; + } + } + push(value, start, index); + } + return tokens; +} + +function shellCommandWords(source) { + const commands = []; + let words = []; + const finish = () => { if (words.length > 0) commands.push(words); words = []; }; + for (const token of shellLexTokens(source)) { + if (token.type === "control") { + finish(); + continue; + } + if (token.type === "word" && words.length === 0 && shellCommandPrefixes.has(token.value)) continue; + if (token.type === "word" && words.length === 0 && shellCommandClosers.has(token.value)) continue; + words.push(token); + } + finish(); + return commands; +} + +function shellExecutable(word) { + return word?.split("/").pop(); +} + +const shellWrapperSpecs = new Map([ + ["command", { kind: "options", operandOptions: new Set() }], + ["env", { kind: "env", operandOptions: new Set(["-u", "--unset", "-C", "--chdir"]) }], + ["sudo", { kind: "options", operandOptions: new Set(["-u", "--user", "-g", "--group", "-h", "--host", "-p", "--prompt", "-C", "--close-from", "-D", "--chdir"]) }], + ["nice", { kind: "options", operandOptions: new Set(["-n", "--adjustment"]) }], + ["time", { kind: "options", operandOptions: new Set(["-o", "--output", "-f", "--format"]) }], + ["xargs", { kind: "options", operandOptions: new Set(["-I", "--replace", "-n", "--max-args", "-L", "--max-lines", "-P", "--max-procs", "-s", "--max-chars", "-d", "--delimiter"]) }], + ["timeout", { kind: "timeout", operandOptions: new Set(["-k", "--kill-after", "-s", "--signal"]) }], + ["stdbuf", { kind: "stdbuf", operandOptions: new Set(["-i", "--input", "-o", "--output", "-e", "--error"]) }], + ["nohup", { kind: "options", operandOptions: new Set() }], + ["exec", { kind: "options", operandOptions: new Set(["-a"]) }], + ["coproc", { kind: "coproc", operandOptions: new Set() }], +]); + +function skipShellMetadata(words, start) { + let index = start; + while (index < words.length) { + const token = words[index]; + if (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(token.value)) { index += 1; continue; } + if (token.type === "redirection") { index += token.needsOperand ? 2 : 1; continue; } + break; + } + return index; +} + +function skipWrapperOptions(words, start, spec) { + let index = start; + while (index < words.length) { + const word = words[index].value; + if (word === "--") return index + 1; + if (spec.operandOptions.has(word)) { index += 2; continue; } + if (spec.kind === "stdbuf" && /^-(?:i|o|e).+/u.test(word)) { index += 1; continue; } + if (word.startsWith("-")) { index += 1; continue; } + break; + } + return index; +} + +function shellJqArguments(words) { + let index = skipShellMetadata(words, 0); + let wrappers = 0; + while (index < words.length) { + const spec = shellWrapperSpecs.get(shellExecutable(words[index]?.value)); + if (!spec) break; + if (wrappers >= 16) throw new Error("revision-state shell wrapper nesting exceeds policy limit"); + wrappers += 1; + index = skipWrapperOptions(words, index + 1, spec); + if (spec.kind === "env") { + while (/^[A-Za-z_][A-Za-z0-9_]*=/u.test(words[index]?.value ?? "")) index += 1; + } else if (spec.kind === "timeout") { + if (index >= words.length) return undefined; + index += 1; + } else if (spec.kind === "coproc") { + index = skipShellMetadata(words, index); + const current = shellExecutable(words[index]?.value); + if (current !== "jq" && !shellWrapperSpecs.has(current) && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(words[index]?.value ?? "")) { + const afterName = skipShellMetadata(words, index + 1); + const command = shellExecutable(words[afterName]?.value); + if (command === "jq" || shellWrapperSpecs.has(command)) index = afterName; + } + } + index = skipShellMetadata(words, index); + } + return shellExecutable(words[index]?.value) === "jq" ? words.slice(index + 1) : undefined; +} + +const jqOptionOperands = new Map([ + ["--arg", 2], ["--argjson", 2], ["--slurpfile", 2], ["--rawfile", 2], ["--argfile", 2], + ["-L", 1], ["--library-path", 1], ["--indent", 1], + ["-f", 1], ["--from-file", 1], +]); +const jqFileFilterOptions = new Set(["-f", "--from-file"]); + +function withoutShellRedirections(arguments_) { + const semantic = []; + for (let index = 0; index < arguments_.length; index += 1) { + const token = arguments_[index]; + if (token.type === "redirection") { if (token.needsOperand) index += 1; continue; } + semantic.push(token); + } + return semantic; +} + +function jqInvocation(arguments_) { + const semantic = withoutShellRedirections(arguments_); + let fromFile = false; + for (let index = 0; index < semantic.length; index += 1) { + const argument = semantic[index].value; + if (argument === "--") return { filter: fromFile ? undefined : semantic[index + 1], arguments_ }; + const operands = jqOptionOperands.get(argument); + if (operands !== undefined) { + if (jqFileFilterOptions.has(argument)) fromFile = true; + index += operands; + continue; + } + if (argument.startsWith("-")) continue; + return { filter: fromFile ? undefined : semantic[index], arguments_ }; + } + return { filter: undefined, arguments_ }; +} + +function maskShellJqLiteralArguments(source) { + const output = source.split(""); + for (const words of shellCommandWords(source)) { + const arguments_ = shellJqArguments(words); + if (!arguments_) continue; + const invocation = jqInvocation(arguments_); + for (const argument of invocation.arguments_) { + if (argument === invocation.filter) continue; + const raw = source.slice(argument.start, argument.end); + if (!raw.includes("$") && !raw.includes("`")) maskRange(output, source, argument.start, argument.end); + } + } + return output.join(""); +} + +function jqStringEnd(source, start) { + for (let index = start + 1; index < source.length; index += 1) { + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === '"') return index; + } + return source.length; +} + +function jqInterpolationEnd(source, start) { + let depth = 1; + for (let index = start; index < source.length; index += 1) { + if (source[index] === '"') { index = jqStringEnd(source, index); continue; } + if (source[index] === "(") depth += 1; + else if (source[index] === ")" && --depth === 0) return index; + } + return source.length; +} + +function jqTokens(source, budget = { tokens: 0, depth: 0 }) { + if (budget.depth >= 64) throw new Error("jq filter exceeds policy nesting limit"); + budget.depth += 1; + const tokens = []; + for (let index = 0; index < source.length; index += 1) { + budget.tokens += 1; + if (budget.tokens >= 10_000) throw new Error("jq filter exceeds policy token limit"); + if (/\s/u.test(source[index])) continue; + if (source[index] === "#") { index = lineEnd(source, index); continue; } + if (source[index] === '"') { + const end = jqStringEnd(source, index); + const raw = source.slice(index, Math.min(end + 1, source.length)); + let value; + if (!raw.includes("\\(")) { + try { value = JSON.parse(raw); } catch { value = undefined; } + } + tokens.push({ type: "string", value }); + for (let cursor = index + 1; cursor < end; cursor += 1) { + if (source[cursor] === "\\" && source[cursor + 1] === "(") { + const close = jqInterpolationEnd(source, cursor + 2); + tokens.push(...jqTokens(source.slice(cursor + 2, close), budget)); + cursor = close; + } else if (source[cursor] === "\\") cursor += 1; + } + index = end; + continue; + } + const variable = source.slice(index).match(/^\$([A-Za-z_][A-Za-z0-9_]*)/u); + if (variable) { tokens.push({ type: "variable", value: variable[1] }); index += variable[0].length - 1; continue; } + const identifier = source.slice(index).match(/^[A-Za-z_][A-Za-z0-9_]*/u); + if (identifier) { tokens.push({ type: "identifier", value: identifier[0] }); index += identifier[0].length - 1; continue; } + const punctuation = { ".": "dot", "[": "open", "]": "close" }[source[index]]; + tokens.push({ type: punctuation ?? "other", value: source[index] }); + } + budget.depth -= 1; + return tokens; +} + +function jqStaticString(tokens, cursor, depth = 0) { + if (depth >= 64) throw new Error("revision-state jq static-key nesting exceeds policy limit"); + let index = cursor; + let value; + if (tokens[index]?.type === "string" && typeof tokens[index].value === "string") { + value = tokens[index].value; + index += 1; + } else if (tokens[index]?.type === "other" && tokens[index].value === "(") { + const nested = jqStaticString(tokens, index + 1, depth + 1); + if (!nested || tokens[nested.next]?.type !== "other" || tokens[nested.next].value !== ")") return undefined; + value = nested.value; + index = nested.next + 1; + } else return undefined; + while (tokens[index]?.type === "other" && tokens[index].value === "+") { + const right = jqStaticString(tokens, index + 1, depth + 1); + if (!right) return undefined; + value += right.value; + index = right.next; + } + return { value, next: index }; +} + +function jqBracketSegment(tokens, cursor) { + if (tokens[cursor]?.type !== "open") return undefined; + const expression = jqStaticString(tokens, cursor + 1); + return expression && tokens[expression.next]?.type === "close" ? + { value: expression.value, next: expression.next + 1 } : undefined; +} + +function jqPathSegment(tokens, cursor, allowBareBracket = true) { + if (tokens[cursor]?.type === "variable") return { value: tokens[cursor].value, next: cursor + 1 }; + let index = cursor; + if (tokens[index]?.type === "dot") { + index += 1; + if (tokens[index]?.type === "identifier" || tokens[index]?.type === "string") return { value: tokens[index].value, next: index + 1 }; + } + return allowBareBracket ? jqBracketSegment(tokens, index) : undefined; +} + +function jqIdentityPipelineEnd(tokens, cursor) { + let index = cursor; + while (tokens[index]?.type === "other" && tokens[index].value === "(") index += 1; + if (tokens[index]?.type !== "dot") return undefined; + index += 1; + while (tokens[index]?.type === "other" && tokens[index].value === ")") index += 1; + return tokens[index]?.type === "other" && tokens[index].value === "|" ? index + 1 : undefined; +} + +function jqTargetGrammarSupported(tokens) { + for (let index = 0; index < tokens.length; index += 1) { + const token = tokens[index]; + if (token.type === "identifier" && tokens[index - 1]?.type !== "dot") return false; + if (token.type === "open" && !jqBracketSegment(tokens, index)) return false; + if (token.type !== "other") continue; + if (["?", "(", ")", "|"].includes(token.value)) continue; + if (token.value === "+" && (tokens[index - 1]?.type === "string" || tokens[index - 1]?.value === ")") && + (tokens[index + 1]?.type === "string" || tokens[index + 1]?.value === "(")) continue; + return false; + } + return true; +} + +function jqContainsActiveTarget(tokens) { + for (let index = 0; index < tokens.length; index += 1) { + if (tokens[index].type === "variable" && revisionIdentifiers.has(tokens[index].value)) return true; + if (tokens[index].type === "dot" && (tokens[index + 1]?.type === "identifier" || tokens[index + 1]?.type === "string") && + revisionIdentifiers.has(tokens[index + 1].value)) return true; + if (tokens[index].type === "open" && (tokens[index - 1]?.type === "dot" || tokens[index - 1]?.type === "close" || tokens[index - 1]?.type === "identifier")) { + const key = jqStaticString(tokens, index + 1); + if (key && revisionIdentifiers.has(key.value)) return true; + } + } + return false; +} + +function jqRevisionAnalysis(filter) { + const tokens = jqTokens(filter); + let activeTarget = jqContainsActiveTarget(tokens); + for (let index = 0; index < tokens.length; index += 1) { + if (tokens[index].type !== "dot" && tokens[index].type !== "variable") continue; + const segments = []; + let cursor = index; + let pipelineBoundary = false; + while (cursor < tokens.length) { + if (pipelineBoundary && (tokens[cursor]?.type === "open" || tokens[cursor]?.type === "string")) { + segments.length = 0; + break; + } + if (pipelineBoundary && tokens[cursor]?.type === "variable") segments.length = 0; + const segment = jqPathSegment(tokens, cursor, !pipelineBoundary); + if (!segment) break; + pipelineBoundary = false; + segments.push(segment.value); + cursor = segment.next; + while (tokens[cursor]?.type === "other" && tokens[cursor].value === "?") cursor += 1; + while (tokens[cursor]?.type === "other" && tokens[cursor].value === ")") cursor += 1; + if (tokens[cursor]?.type === "other" && tokens[cursor].value === "|") { + cursor += 1; + while (tokens[cursor]?.type === "other" && tokens[cursor].value === "(") cursor += 1; + let identityEnd; + while ((identityEnd = jqIdentityPipelineEnd(tokens, cursor)) !== undefined) cursor = identityEnd; + pipelineBoundary = true; + } + } + if (segments.some((segment) => revisionIdentifiers.has(segment))) activeTarget = true; + for (let position = 0; position + 1 < segments.length; position += 1) { + if (revisionIdentifiers.has(segments[position]) && segments[position + 1] === "state") return "violation"; + } + } + if (!activeTarget) return "safe"; + return jqTargetGrammarSupported(tokens) ? "safe" : "unsupported"; +} + +function shellExecutableSubstitutionBodies(source, arithmeticContext = false) { + const bodies = []; + const addParenthesized = (start, kind) => { + const end = shellParenthesizedEnd(source, start + 1, 1, { characters: 0 }); + bodies.push({ kind, start: start + 2, end: end - 1, source: source.slice(start + 2, end - 1) }); + return end; + }; + const addBacktick = (start) => { + const end = quotedEnd(source, start, "`", "\\"); + if (end - 1 <= start || source[end - 1] !== "`") throw new Error("revision-state shell substitution has an unclosed backtick"); + bodies.push({ kind: "backtick", start: start + 1, end: end - 1, source: source.slice(start + 1, end - 1) }); + return end; + }; + for (let index = 0; index < source.length; index += 1) { + if (source[index] === "\\") { index += 1; continue; } + if (source[index] === "#" && (index === 0 || /[ \t\r\n;|&()]/u.test(source[index - 1]))) { index = lineEnd(source, index); continue; } + if (source[index] === "'") { + const end = quotedEnd(source, index, "'", ""); + if (end - 1 <= index || source[end - 1] !== "'") throw new Error(`revision-state shell policy found an unclosed quote at offset ${index}`); + index = end - 1; + continue; + } + if (source[index] === '"') { + for (let cursor = index + 1; cursor < source.length; cursor += 1) { + if (source[cursor] === "\\") { cursor += 1; continue; } + if (source[cursor] === '"') { index = cursor; break; } + if (source.startsWith("$(", cursor)) { + const end = addParenthesized(cursor, source.startsWith("$((", cursor) ? "arithmetic" : "command"); + cursor = end - 1; + } else if (source[cursor] === "`") { + cursor = addBacktick(cursor) - 1; + } + if (cursor + 1 >= source.length) throw new Error(`revision-state shell policy found an unclosed double quote at offset ${index}`); + } + continue; + } + if (!arithmeticContext && (source.startsWith("<(", index) || source.startsWith(">(", index))) { + index = addParenthesized(index, "process") - 1; + continue; + } + if (source.startsWith("$(", index)) { + const arithmetic = source.startsWith("$((", index); + index = addParenthesized(index, arithmetic ? "arithmetic" : "command") - 1; + continue; + } + if (source[index] === "`") index = addBacktick(index) - 1; + } + return bodies; +} + +function removeBacktickBodyEscapes(source) { + let result = ""; + for (let index = 0; index < source.length; index += 1) { + if (source[index] === "\\" && index + 1 < source.length && ["$", "`", "\\", "\n"].includes(source[index + 1])) { + if (source[index + 1] !== "\n") result += source[index + 1]; + index += 1; + } else { + result += source[index]; + } + } + return result; +} + +function shellJqRevisionAccess(source, budget = { characters: 0 }, depth = 0, arithmeticContext = false) { + if (depth > 32) throw new Error("revision-state executable shell substitution nesting limit exceeded"); + budget.characters += source.length; + if (budget.characters > 500_000) throw new Error("revision-state executable shell substitution size limit exceeded"); + if (!arithmeticContext) { + for (const words of shellCommandWords(source)) { + const arguments_ = shellJqArguments(words); + const filter = arguments_ && jqInvocation(arguments_).filter; + if (filter) { + const analysis = jqRevisionAnalysis(filter.value); + if (analysis === "violation") return true; + if (analysis === "unsupported") throw new Error("revision-state jq target grammar is unsupported"); + } + } + } + for (const body of shellExecutableSubstitutionBodies(source, arithmeticContext)) { + const nestedSource = body.kind === "backtick" ? removeBacktickBodyEscapes(body.source) : body.source; + if (shellJqRevisionAccess(nestedSource, budget, depth + 1, body.kind === "arithmetic")) return true; + } + return false; +} + +function nonJsAnalysisSource(source, label) { + const lower = label.toLowerCase(); + if (lower.endsWith(".sh")) return maskShellSource(maskShellJqLiteralArguments(maskQuotedShellHeredocBodies(source, label))); + if (lower.endsWith(".ps1")) return maskPowerShellSource(source); + return maskUnknownSource(source); +} + +function revisionStateAstNodes(source, label) { + const knownKind = scriptKindFor(label); + const caseInsensitive = label.toLowerCase().endsWith(".ps1"); + const analyzed = knownKind === undefined ? nonJsAnalysisSource(source, label) : source; + const file = ts.createSourceFile(label, analyzed, ts.ScriptTarget.Latest, true, knownKind ?? ts.ScriptKind.TS); + const matches = []; + function visit(node) { + if (ts.isPropertyAccessExpression(node) && node.name.text === "state" && isRevisionExpression(node.expression, caseInsensitive)) { + matches.push(node); + } else if (ts.isElementAccessExpression(node) && isRevisionExpression(node.expression, caseInsensitive) && + node.argumentExpression && propertyNameText(node.argumentExpression, caseInsensitive) === "state") { + matches.push(node); + } else if ((ts.isVariableDeclaration(node) || ts.isParameter(node)) && node.initializer && + isRevisionExpression(node.initializer, caseInsensitive) && ts.isObjectBindingPattern(node.name) && + objectBindingHasState(node.name, caseInsensitive)) { + matches.push(node); + } else if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + isRevisionExpression(node.right, caseInsensitive)) { + const assignmentTarget = unwrapExpression(node.left); + if (ts.isObjectLiteralExpression(assignmentTarget) && objectLiteralHasState(assignmentTarget, caseInsensitive)) matches.push(node); + } else if (ts.isPropertyAssignment(node) && propertyNameText(node.name, caseInsensitive) === "revision" && + ts.isObjectLiteralExpression(node.initializer) && objectLiteralHasState(node.initializer, caseInsensitive)) { + matches.push(node); + } + ts.forEachChild(node, visit); + } + visit(file); + return matches; +} + + +function yamlScalarRevisionAccess(value) { + return /(?:^|[\s;=,(])(?:revision|workspaceRevision|selectedWorkspace)\s*(?:\.\s*state|\[\s*["']?state["']?\s*\])(?:$|[\s;,)])/u.test(value); +} + +function validateYamlRevisionState(source, label) { + const documents = parseAllDocuments(source, { uniqueKeys: true, merge: true }); + for (const document of documents) { + if (document.errors.length > 0) throw new Error(`${label}: revision-state policy cannot parse YAML`); + const walkAst = (node) => { + if (isScalar(node)) { + if (node.type === "PLAIN" && typeof node.value === "string" && yamlScalarRevisionAccess(node.value)) throw new Error(`${label}: forbidden revision-state access`); + return; + } + if (isSeq(node)) { for (const item of node.items) walkAst(item); return; } + if (isMap(node)) { for (const pair of node.items) walkAst(pair.value); } + }; + walkAst(document.contents); + let resolved; + try { resolved = document.toJS({ mapAsMap: true, maxAliasCount: 50 }); } + catch { throw new Error(`${label}: revision-state YAML alias resolution failed`); } + const seen = new WeakSet(); + const walkResolved = (value) => { + if (!value || typeof value !== "object" || seen.has(value)) return; + seen.add(value); + if (value instanceof Map) { + for (const [key, child] of value) { + if (revisionIdentifiers.has(String(key)) && child instanceof Map && child.has("state")) throw new Error(`${label}: forbidden revision-state access`); + walkResolved(child); + } + } else if (Array.isArray(value)) { for (const child of value) walkResolved(child); } + }; + walkResolved(resolved); + } +} + +function validateRevisionState(source, label) { + const lower = label.toLowerCase(); + if (/\.(?:yaml|yml)(?:\.example)?$/u.test(lower)) { + validateYamlRevisionState(source, label); + return; + } + if (lower.endsWith(".sh")) { + const active = maskQuotedShellHeredocBodies(source, label); + try { + if (shellJqRevisionAccess(active) || shellAssociativeRevisionAccess(active)) throw new Error("forbidden revision-state access"); + } catch (error) { + throw new Error(`${label}: ${error instanceof Error ? error.message : String(error)}`); + } + } + if (lower.endsWith(".py") || lower.endsWith(".pyw")) throw new Error(`${label}: revision-state Python input was not batched`); + const matches = revisionStateAstNodes(source, label); + if (matches.length === 0) return; + const historical = 'revision.state !== "operational"'; + const historicalCount = source.split(historical).length - 1; + const match = matches[0]; + if (label === "backend/src/workspaces/registry.ts" && matches.length === 1 && + match.getText() === "revision.state" && match.parent?.getText() === historical && + historicalCount === 1) return; + throw new Error(`${label}: forbidden revision-state access`); +} + + +const pythonHelper = fileURLToPath(new URL("./revision_state_policy.py", import.meta.url)); + +function validatePythonRevisionStates(records) { + if (!Array.isArray(records) || records.length === 0) return; + let stdout; + try { + stdout = execFileSync("python3", ["-I", "-B", pythonHelper], { + input: JSON.stringify(records), encoding: "utf8", timeout: 5_000, maxBuffer: 4 * 1024 * 1024, + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + LANG: "C.UTF-8", + LC_ALL: "C.UTF-8", + PYTHONDONTWRITEBYTECODE: "1", + }, + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (error) { + const detail = error?.stderr?.toString().trim(); + throw new Error(`revision-state helper failed${detail ? `: ${detail}` : ""}`); + } + let result; + try { result = JSON.parse(stdout); } + catch { throw new Error("revision-state helper failed: invalid JSON output"); } + if (!result || !Array.isArray(result.violations) || result.violations.some((label) => typeof label !== "string")) throw new Error("revision-state helper failed: invalid result shape"); + if (result.violations.length > 0) throw new Error(`${result.violations[0]}: forbidden revision-state access`); +} + +export { validatePythonRevisionStates, validateRevisionState }; diff --git a/backend/scripts/revision-state-policy.test.mjs b/backend/scripts/revision-state-policy.test.mjs new file mode 100644 index 00000000..b8be5302 --- /dev/null +++ b/backend/scripts/revision-state-policy.test.mjs @@ -0,0 +1,273 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs"; + +function rejects(source, path) { + assert.throws(() => validateRevisionState(source, path), /revision-state/, source); +} +function passes(source, path) { + assert.doesNotThrow(() => validateRevisionState(source, path)); +} + +test("PowerShell scoped and braced revision variables remain executable", () => { + rejects('${revision}.state', "scripts/direct.ps1"); + rejects('${workspaceRevision}["state"]', "scripts/bracket.ps1"); + rejects('Write-Output "$(${selectedWorkspace}.state)"', "scripts/subexpression.ps1"); + rejects('${script:revision}.state', "scripts/scoped.ps1"); + rejects('${global:workspaceRevision}["state"]', "scripts/global.ps1"); + for (const source of [ + '$REVISION.STATE', + '${Revision}.state', + '$WORKSPACEREVISION["STATE"]', + '${GLOBAL:SELECTEDWORKSPACE}.State', + '$REVISION["ST" + "ATE"]', + '${Revision}[("sT" + "AtE")]', + '$record.REVISION.STATE', + '$record.WORKSPACEREVISION["STATE"]', + '$record["REVISION"].STATE', + ]) rejects(source, "scripts/case.ps1"); + passes('REVISION.STATE; revision.STATE; revision["ST" + "ATE"]; record.REVISION.STATE; record["REVISION"].state', "backend/src/case-sensitive.ts"); +}); + +test("Bash jq command forms and associative revision parameters are active", () => { + for (const source of [ + "value=$(jq -r '.revision.state' snapshot.json)", + "value=$(command jq -r '.workspaceRevision.state' snapshot.json)", + "/usr/bin/jq --arg x y '.selectedWorkspace.state' snapshot.json", + "env -i MODE=x jq -- '.revision.state' snapshot.json", + "env -u MODE /opt/tools/jq -r '.workspaceRevision.state' snapshot.json", + "echo safe\nvalue=`jq -r '.selectedWorkspace.state' snapshot.json`", + "sudo -u nobody /usr/bin/jq -r '.revision.state' snapshot.json", + "nice -n 5 jq -r '.workspaceRevision.state' snapshot.json", + "time jq -r '.selectedWorkspace.state' snapshot.json", + "printf x | xargs -n 1 jq -r '.revision.state'", + "timeout -k 2 5 jq -r '.revision.state' snapshot.json", + `stdbuf -o L jq -r '.["workspaceRevision"].state' snapshot.json`, + `stdbuf -oL jq -r '.["selectedWorkspace"]["state"]' snapshot.json`, + `nohup jq -r '.revision["state"]' snapshot.json`, + "< snapshot.json jq -r '.workspaceRevision.state'", + "sudo MODE=x jq -r '.selectedWorkspace.state' snapshot.json", + String.raw`jq -r '"x \(.revision.state)"' snapshot.json`, + "jq < snapshot.json -r '.revision.state'", + "jq -r < snapshot.json '.workspaceRevision.state'", + "jq --arg note safe < snapshot.json '.selectedWorkspace.state'", + "jq -r '.revision?.state' snapshot.json", + `jq -r '.["workspaceRevision"]?["state"]' snapshot.json`, + `jq -r '.["revision"]?.["state"]' snapshot.json`, + `jq -r '."revision".state' snapshot.json`, + `jq -r '."workspaceRevision"."state"' snapshot.json`, + "jq -r '$revision.state' snapshot.json", + "jq -r '($selectedWorkspace).state' snapshot.json", + `${"env ".repeat(17)}jq -r '.revision.state' snapshot.json`, + "jq/dev/null -r '.workspaceRevision.state' snapshot.json", + "{ jq -r '.selectedWorkspace.state' snapshot.json; }", + "! jq -r '.revision.state' snapshot.json", + "if jq -r '.workspaceRevision.state' snapshot.json; then :; fi", + "if false; then :; elif jq -r '.selectedWorkspace.state' snapshot.json; then :; fi", + "while false; do jq -r '.revision.state' snapshot.json; done", + "until false; do jq -r '.workspaceRevision.state' snapshot.json; done", + "jq -r '.revision | .state' snapshot.json", + "jq -r '(.workspaceRevision | .state)' snapshot.json", + `jq -r '.["revi" + "sion"].state' snapshot.json`, + `jq -r '.["workspace" + "Revision"]["st" + "ate"]' snapshot.json`, + "jq 2>&1 -r '.revision.state' snapshot.json", + "jq 2>&- -r '.workspaceRevision.state' snapshot.json", + "jq 0<&3 -r '.selectedWorkspace.state' snapshot.json", + "jq &>/dev/null -r '.revision.state' snapshot.json", + "jq &>>log -r '.workspaceRevision.state' snapshot.json", + "jq >|output -r '.selectedWorkspace.state' snapshot.json", + "jq {fd}>output -r '.revision.state' snapshot.json", + "exec jq -r '.workspaceRevision.state' snapshot.json", + "coproc jq -r '.selectedWorkspace.state' snapshot.json", + "coproc worker jq -r '.revision.state' snapshot.json", + "coproc worker >out jq -r '.workspaceRevision.state' snapshot.json", + "coproc worker 2>/dev/null jq -r '.selectedWorkspace.state' snapshot.json", + "coproc worker VAR=x jq -r '.revision.state' snapshot.json", + `jq -r '.["revi" + ("sion")].state' snapshot.json`, + "jq -r '.revision | . | .state' snapshot.json", + "jq -r '(.workspaceRevision | (.) | .state)' snapshot.json", + "jq -r '.revision | select(.) | .state' snapshot.json", + "jq -r '.workspaceRevision | {value:.state}' snapshot.json", + "jq -r '.selectedWorkspace | [.state]' snapshot.json", + "jq -r '.revision + .state' snapshot.json", + "jq < <(cat snapshot.json) -r '.revision.state'", + "jq < <(cat <(printf snapshot.json)) -r '.workspaceRevision.state'", + "jq > >(cat >/dev/null) -r '.selectedWorkspace.state' snapshot.json", + `jq < <(printf '%s\n' "$((1 + (2)))") -r '.revision.state'`, + "jq < <(cat snapshot.json -r '.revision.state'", + `${"<(".repeat(65)}echo snapshot${")".repeat(65)} jq -r '.workspaceRevision.state'`, + "cat <(jq -r '.revision.state' snapshot.json)", + "cat snapshot.json > >(jq -r '.workspaceRevision.state')", + `echo "$(jq -r '.selectedWorkspace.state' snapshot.json)"`, + "value=$(jq -r '.revision.state' snapshot.json)", + `echo "\`jq -r '.workspaceRevision.state' snapshot.json\`"`, + `echo "$(cat <(jq -r '.selectedWorkspace.state' snapshot.json))"`, + `${"$(".repeat(33)}jq -r '.revision.state' snapshot.json${")".repeat(33)}`, + `echo "$(( $(jq -r '.revision.state' snapshot.json) + 0 ))"`, + "echo \"$(( `jq -r '.workspaceRevision.state' snapshot.json` + 0 ))\"", + "echo `echo \\`jq -r '.selectedWorkspace.state' snapshot.json\\``", + "echo \"`echo \\`jq -r '.revision.state' snapshot.json\\``\"", + `${"$(( ".repeat(33)}$(jq -r '.workspaceRevision.state' snapshot.json)${" + 0 ))".repeat(33)}`, + 'old=${revision["state"]}', + "old=${workspaceRevision[state]}", + "old=${revision[state]:-missing}", + "old=${workspaceRevision['state']:=missing}", + "old=${selectedWorkspace[state]:1:2}", + ]) rejects(source, "scripts/policy.sh"); + const jqFilters = [ + ".revision?.state", '.["revision"]?.["state"]', '."revision".state', + '."workspaceRevision"."state"', "(.revision).state", "$revision.state", + ".revision | .state", "(.workspaceRevision | .state)", + '.["revi" + "sion"].state', '.["revi" + ("sion")].state', + ".revision | . | .state", "(.workspaceRevision | (.) | .state)", + ".revision | select(.) | .state", ".workspaceRevision | {value:.state}", + '.revision | ["state"]', '(.workspaceRevision | (["state"]))', ".selectedWorkspace | $state", + ]; + for (const filter of jqFilters) { + const compiled = spawnSync("jq", ["-n", "--argjson", "revision", "{}", "--arg", "state", "x", filter], { encoding: "utf8" }); + if (compiled.error?.code !== "ENOENT") assert.equal(compiled.status, 0, `${filter}: ${compiled.stderr}`); + } + passes("cat <<'EOF'\nrevision.state\nEOF\n", "scripts/literal.sh"); + passes("echo '${revision[state]}'\n", "scripts/single-quoted-parameter.sh"); + passes(`echo "<(jq '.revision.state')"\n`, "scripts/literal-process-text.sh"); + passes(`echo "ordinary jq '.workspaceRevision.state' text"\n`, "scripts/literal-jq-text.sh"); + passes(`echo '$(jq -r ".selectedWorkspace.state")'\n`, "scripts/single-quoted-command-text.sh"); + passes(`# profile's harmless note +printf 'ok\n' +`, "scripts/comment-apostrophe.sh"); + passes(`cat <( # profile's harmless note + printf 'snapshot\n' +) +`, "scripts/substitution-comment-apostrophe.sh"); + passes(`echo "$(( 1 + (2 * 3) ))"\n`, "scripts/literal-arithmetic.sh"); + passes(`echo $(( jq + revision + state ))\n`, "scripts/arithmetic-identifiers.sh"); + passes("echo \\`jq -r '.revision.state' snapshot.json\\`\n", "scripts/escaped-literal-backticks.sh"); + passes("echo \"\\`jq -r '.workspaceRevision.state' snapshot.json\\`\"\n", "scripts/double-quoted-literal-backticks.sh"); + passes("echo `printf '%s' '\\`jq -r \".selectedWorkspace.state\" snapshot.json\\`'`\n", "scripts/quoted-nonexecuting-nested-backticks.sh"); + passes("jq --arg note 'revision.state' '.' file\n", "scripts/jq-arg.sh"); + passes(`jq --argjson note '"revision.state"' '.' file +`, "scripts/jq-argjson.sh"); + passes("jq -r '.' revision.state.json\n", "scripts/jq-file.sh"); + passes("jq -r '.revision.id' snapshot.json\n", "scripts/jq-simple-non-state.sh"); + passes("jq -f revision.state.jq snapshot.json\n", "scripts/jq-from-file.sh"); + passes("jq --from-file workspaceRevision.state.jq snapshot.json\n", "scripts/jq-long-from-file.sh"); + passes(`jq -r '"revision.state"' snapshot.json +`, "scripts/jq-string.sh"); + passes(`jq -r '{note:"selectedWorkspace.state"}' snapshot.json +`, "scripts/jq-object.sh"); + passes(`jq -r '.revision | "state"' snapshot.json +`, "scripts/jq-pipe-literal-right.sh"); + passes(`jq -r '"revision" | .state' snapshot.json +`, "scripts/jq-pipe-literal-left.sh"); + passes(`jq -r '.revision | ["state"]' snapshot.json +`, "scripts/jq-pipe-array.sh"); + passes(`jq -r '(.workspaceRevision | (["state"]))' snapshot.json +`, "scripts/jq-pipe-parenthesized-array.sh"); + passes(`jq --arg state x '.selectedWorkspace | $state' snapshot.json +`, "scripts/jq-pipe-variable.sh"); + for (const opener of ["'E'OF", "E'OF'", "E\\OF"]) { + passes(`cat <<${opener} +revision.state +EOF +`, "scripts/partial-quoted-heredoc.sh"); + } + rejects("cat <<'E'OF\nrevision.state\nEOF\nworkspaceRevision.state\n", "scripts/after-heredoc.sh"); + rejects("cat <<'EOF'\nrevision.state\n", "scripts/unclosed-heredoc.sh"); + rejects(`echo "<<'EOF'" +jq -r '.revision.state' snapshot.json +`, "scripts/quoted-opener.sh"); +}); + +test("Python helper resolves active AST expressions and static format bindings", () => { + const rejectsPython = (source) => assert.throws( + () => validatePythonRevisionStates([{ source, label: "backend/scripts/policy.py" }]), + /revision-state/, + ); + for (const source of [ + "old = revision.state", + 'old = workspaceRevision["state"]', + 'old = record["selectedWorkspace"].state', + 'old = f"{revision.state}"', + '"{revision.state}".format(value)', + '"{0.state}".format(revision)', + '"{0[state]}".format(workspaceRevision)', + '"{item.state}".format(item=selectedWorkspace)', + '"{item[state]}".format_map({"item": revision})', + '("{0.state}").format(revision)', + '"{0:{1.state}}".format(value, revision)', + 'old = revision["st" + "ate"]', + 'old = record["revi" + "sion"].state', + 'old = revision[f"state"]', + 'old = record[f"revision"].state', + `old = revision[f"st{'a'}te"]`, + `old = revision[f"{'state'}"]`, + `old = record[f"revi{'sion'}"].state`, + `old = revision[f"{'st' + 'ate'}"]`, + `old = record[f"{'revi' + 'sion'}"].state`, + `old = revision[f"{'state':s}"]`, + '"{0.state}".format(*[revision])', + '"{0[state]}".format(*(revision,))', + '"{1[state]}".format(*[other, workspaceRevision])', + '"{item.state}".format(**{"item": selectedWorkspace})', + '"{item[state]}".format_map({**{"item": revision}})', + '"{.state}".format(revision)', + '"{[state]}".format(revision)', + '"{:{.state}}".format(value, revision)', + '"{.name} {[state]}".format(other, revision)', + '"{item.state}".format(item=revision, **values)', + ]) rejectsPython(source); + validatePythonRevisionStates([ + { source: 'text = "{revision.state}"', label: "backend/scripts/literal.py" }, + { source: 'text = "{{revision.state}}".format(value)', label: "backend/scripts/escaped.py" }, + { source: 'text = "{0.state}".format(other)', label: "backend/scripts/unrelated.py" }, + { source: 'old = revision[f"st{suffix}"]', label: "backend/scripts/dynamic-key.py" }, + { source: 'text = "{.name} {[state]}".format(other, other)', label: "backend/scripts/multi-auto.py" }, + { source: 'text = "{item.state}".format(**values)', label: "backend/scripts/dynamic-map.py" }, + ]); + const hostile = mkdtempSync(join(tmpdir(), "revision-policy-hostile-")); + writeFileSync(join(hostile, "json.py"), "raise RuntimeError('shadowed')\n"); + const previousPythonPath = process.env.PYTHONPATH; + try { + process.env.PYTHONPATH = hostile; + validatePythonRevisionStates([{ source: "value = 1", label: "backend/scripts/isolated.py" }]); + } finally { + if (previousPythonPath === undefined) delete process.env.PYTHONPATH; + else process.env.PYTHONPATH = previousPythonPath; + rmSync(hostile, { recursive: true, force: true }); + } + assert.throws( + () => validatePythonRevisionStates([{ source: 'revision[f"{1:.1000000000f}"]', label: "backend/scripts/oversized.py" }]), + /revision-state helper failed/, + ); + validatePythonRevisionStates([{ source: 'revision[f"{1:04d}"]', label: "backend/scripts/small-format.py" }]); + assert.throws( + () => validatePythonRevisionStates([{ source: "def broken(", label: "backend/scripts/invalid.py" }]), + /revision-state helper failed/, + ); +}); + +test("YAML mappings and only active plain scalar expressions are rejected", () => { + for (const source of [ + "value: { revision: { state: old } }\n", + "value:\n workspaceRevision:\n state: old\n", + 'items:\n - "selectedWorkspace":\n "state": old\n', + "old: selectedWorkspace.state\n", + "url: https://host/x; old: selectedWorkspace.state\n", + "saved: &saved { state: old }\nvalue: { revision: *saved }\n", + "defaults: &defaults { workspaceRevision: { state: old } }\nvalue: { <<: *defaults }\n", + ]) rejects(source, "scripts/policy.yaml"); + rejects("a: &a [x,x,x,x,x,x,x,x,x]\nb: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]\nc: [*b,*b,*b,*b,*b,*b,*b,*b,*b]\n", "scripts/alias-bomb.yaml"); + rejects("value: [\n", "scripts/invalid.yaml"); + for (const source of [ + "value: |\n revision.state\n", + "value: >\n workspaceRevision.state\n", + 'value: "selectedWorkspace.state"\n', + "url: https://host/revision.state\n", + ]) passes(source, "scripts/literal.yaml"); +}); diff --git a/backend/scripts/revision_state_policy.py b/backend/scripts/revision_state_policy.py new file mode 100644 index 00000000..f670040e --- /dev/null +++ b/backend/scripts/revision_state_policy.py @@ -0,0 +1,318 @@ +"""Semantic Python revision-state policy helper. + +Reads one JSON array of ``{"label": str, "source": str}`` records from stdin and +writes ``{"violations": [label, ...]}``. Invalid input or Python source is fatal. +""" + +from __future__ import annotations + +import ast +import json +import re +import string +import sys +from itertools import pairwise +from typing import Any + +TARGETS = frozenset({"revision", "workspaceRevision", "selectedWorkspace"}) +_FORMATTER = string.Formatter() + + +MAX_STATIC_TEXT = 4_096 +MAX_FORMAT_SPEC = 256 +MAX_STATIC_DEPTH = 64 +_UNRESOLVED = object() + + +def _bounded_text(value: str) -> str: + if len(value) > MAX_STATIC_TEXT: + raise ValueError("static text exceeds revision policy limit") + return value + + +def _static_scalar(node: ast.expr, depth: int) -> object: + if depth > MAX_STATIC_DEPTH: + raise ValueError("static expression nesting exceeds revision policy limit") + if isinstance(node, ast.Constant) and type(node.value) in { + str, + int, + float, + complex, + bool, + type(None), + }: + if isinstance(node.value, str): + _bounded_text(node.value) + if isinstance(node.value, int) and node.value.bit_length() > MAX_STATIC_TEXT * 4: + raise ValueError("static integer exceeds revision policy limit") + return node.value + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _static_scalar(node.left, depth + 1) + right = _static_scalar(node.right, depth + 1) + if left is _UNRESOLVED or right is _UNRESOLVED: + return _UNRESOLVED + try: + result = left + right + except TypeError: + return _UNRESOLVED + if type(result) not in {str, int, float, complex, bool}: + return _UNRESOLVED + if isinstance(result, str): + _bounded_text(result) + if isinstance(result, int) and result.bit_length() > MAX_STATIC_TEXT * 4: + raise ValueError("static integer exceeds revision policy limit") + return result + if isinstance(node, ast.JoinedStr): + result = _static_key(node, depth + 1) + return _UNRESOLVED if result is None else result + return _UNRESOLVED + + +def _validate_format_spec(format_spec: str) -> None: + if len(format_spec) > MAX_FORMAT_SPEC: + raise ValueError("static format specification exceeds revision policy limit") + for digits in re.findall(r"[0-9]+", format_spec): + if len(digits) > 6 or int(digits) > MAX_STATIC_TEXT: + raise ValueError("static format width or precision exceeds revision policy limit") + + +def _static_key(node: ast.expr, depth: int = 0) -> str | None: + if depth > MAX_STATIC_DEPTH: + raise ValueError("static key nesting exceeds revision policy limit") + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return _bounded_text(node.value) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + left = _static_key(node.left, depth + 1) + right = _static_key(node.right, depth + 1) + return None if left is None or right is None else _bounded_text(left + right) + if isinstance(node, ast.JoinedStr): + pieces = [] + length = 0 + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + piece = value.value + elif isinstance(value, ast.FormattedValue): + scalar = _static_scalar(value.value, depth + 1) + if scalar is _UNRESOLVED: + return None + format_spec = "" if value.format_spec is None else _static_key(value.format_spec, depth + 1) + if format_spec is None: + return None + _validate_format_spec(format_spec) + try: + if value.conversion == ord("s"): + scalar = str(scalar) + elif value.conversion == ord("r"): + scalar = repr(scalar) + elif value.conversion == ord("a"): + scalar = ascii(scalar) + elif value.conversion != -1: + return None + piece = format(scalar, format_spec) + except (TypeError, ValueError): + return None + else: + return None + length += len(piece) + if length > MAX_STATIC_TEXT: + raise ValueError("static formatted key exceeds revision policy limit") + pieces.append(piece) + return "".join(pieces) + return None + + +def _is_revision_expr(node: ast.expr) -> bool: + if isinstance(node, ast.Name): + return node.id in TARGETS + if isinstance(node, ast.Attribute): + return node.attr in TARGETS + if isinstance(node, ast.Subscript): + return _static_key(node.slice) in TARGETS + return False + + +def _is_state_access(node: ast.AST) -> bool: + if isinstance(node, ast.Attribute): + return node.attr == "state" and _is_revision_expr(node.value) + if isinstance(node, ast.Subscript): + return _static_key(node.slice) == "state" and _is_revision_expr(node.value) + return False + + +def _static_sequence(node: ast.expr) -> list[ast.expr] | None: + if not isinstance(node, (ast.List, ast.Tuple)): + return None + result: list[ast.expr] = [] + for element in node.elts: + if isinstance(element, ast.Starred): + nested = _static_sequence(element.value) + if nested is None: + return None + result.extend(nested) + else: + result.append(element) + return result + + +def _static_mapping(node: ast.expr) -> dict[str, ast.expr] | None: + if not isinstance(node, ast.Dict): + return None + result: dict[str, ast.expr] = {} + for key, value in zip(node.keys, node.values, strict=True): + if key is None: + nested = _static_mapping(value) + if nested is None: + return None + result.update(nested) + elif (name := _static_key(key)) is not None: + result[name] = value + else: + return None + return result + + +def _format_bindings(call: ast.Call, method: str) -> dict[str | int, ast.expr]: + if method == "format": + bindings: dict[str | int, ast.expr] = {} + position = 0 + positional_known = True + for argument in call.args: + if isinstance(argument, ast.Starred): + expanded = _static_sequence(argument.value) + if expanded is None: + positional_known = False + continue + if positional_known: + for value in expanded: + bindings[position] = value + position += 1 + elif positional_known: + bindings[position] = argument + position += 1 + for keyword in call.keywords: + if keyword.arg is not None: + # An explicit keyword remains bound even beside **dynamic; a duplicate is TypeError. + bindings[keyword.arg] = keyword.value + else: + expanded = _static_mapping(keyword.value) + if expanded is not None: + bindings.update(expanded) + return bindings + if len(call.args) != 1 or call.keywords: + return {} + return _static_mapping(call.args[0]) or {} + + +def _field_accesses_state( + field_name: str, bindings: dict[str | int, ast.expr], automatic_index: int | None = None +) -> bool: + root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name) + if root_match is None: + if automatic_index is None or not field_name.startswith((".", "[")): + return False + root: str | int = automatic_index + cursor = 0 + else: + root_text = root_match.group(0) + root = int(root_text) if root_text.isdigit() else root_text + cursor = root_match.end() + steps: list[tuple[bool, str]] = [] + while cursor < len(field_name): + if field_name[cursor] == ".": + match = re.match(r"[A-Za-z_][A-Za-z0-9_]*", field_name[cursor + 1 :]) + if match is None: + return False + steps.append((True, match.group(0))) + cursor += len(match.group(0)) + 1 + elif field_name[cursor] == "[": + close = field_name.find("]", cursor + 1) + if close < 0: + return False + steps.append((False, field_name[cursor + 1 : close])) + cursor = close + 1 + else: + return False + if steps: + first_step = str(steps[0][1]) + if str(root) in TARGETS and first_step == "state": + return True + bound = bindings.get(root) + if bound is not None and _is_revision_expr(bound) and first_step == "state": + return True + names = [str(root), *(str(key) for _is_attr, key in steps)] + return any(left in TARGETS and right == "state" for left, right in pairwise(names)) + + +def _format_call_violation(node: ast.Call) -> bool: + function = node.func + if not isinstance(function, ast.Attribute) or function.attr not in {"format", "format_map"}: + return False + if not isinstance(function.value, ast.Constant) or not isinstance(function.value.value, str): + return False + bindings = _format_bindings(node, function.attr) + numbering: dict[str, int | str | None] = {"next": 0, "mode": None} + visited = 0 + + def analyze_template(template: str) -> bool: + nonlocal visited + visited += 1 + if visited > 1_000: + raise ValueError("format specification nesting exceeds policy limit") + for _literal, field_name, format_spec, _conversion in _FORMATTER.parse(template): + automatic_index = None + if field_name is not None: + root_match = re.match(r"(?:[0-9]+|[A-Za-z_][A-Za-z0-9_]*)", field_name) + automatic = field_name == "" or root_match is None and field_name.startswith((".", "[")) + manual = root_match is not None and root_match.group(0).isdigit() + if automatic: + if numbering["mode"] == "manual": + raise ValueError("cannot switch from manual to automatic field numbering") + numbering["mode"] = "automatic" + automatic_index = int(numbering["next"]) + numbering["next"] = automatic_index + 1 + elif manual: + if numbering["mode"] == "automatic": + raise ValueError("cannot switch from automatic to manual field numbering") + numbering["mode"] = "manual" + if _field_accesses_state(field_name, bindings, automatic_index): + return True + if format_spec and analyze_template(format_spec): + return True + return False + + return analyze_template(function.value.value) + + +def has_revision_state(source: str, label: str = "") -> bool: + tree = ast.parse(source, filename=label, mode="exec") + return any(_is_state_access(node) or (isinstance(node, ast.Call) and _format_call_violation(node)) for node in ast.walk(tree)) + + +def analyze_batch(records: Any) -> list[str]: + if not isinstance(records, list): + raise TypeError("input must be a JSON array") + violations = [] + for record in records: + if not isinstance(record, dict) or set(record) != {"label", "source"}: + raise TypeError("each record must contain exactly label and source") + label, source = record["label"], record["source"] + if not isinstance(label, str) or not isinstance(source, str): + raise TypeError("label and source must be strings") + if has_revision_state(source, label): + violations.append(label) + return violations + + +def main() -> int: + try: + records = json.load(sys.stdin) + json.dump({"violations": analyze_batch(records)}, sys.stdout, ensure_ascii=False) + sys.stdout.write("\n") + return 0 + except Exception as error: # noqa: BLE001 - protocol boundary must fail closed + print(f"python revision-state helper failed: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/scripts/test_revision_state_policy.py b/backend/scripts/test_revision_state_policy.py new file mode 100644 index 00000000..e5645480 --- /dev/null +++ b/backend/scripts/test_revision_state_policy.py @@ -0,0 +1,90 @@ +import importlib.util +import tracemalloc +import unittest +from pathlib import Path +from unittest.mock import patch + +_HELPER = Path(__file__).with_name("revision_state_policy.py") +_SPEC = importlib.util.spec_from_file_location("revision_state_policy", _HELPER) +assert _SPEC is not None and _SPEC.loader is not None +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +analyze_batch = _MODULE.analyze_batch +has_revision_state = _MODULE.has_revision_state + + +class RevisionStatePolicyTests(unittest.TestCase): + def test_ast_access_and_f_strings(self): + for source in ( + "old = revision.state", + 'old = workspaceRevision["state"]', + 'old = record["selectedWorkspace"].state', + 'old = f"{revision.state}"', + 'old = revision["st" + "ate"]', + 'old = record["revi" + "sion"].state', + 'old = revision[f"state"]', + 'old = record[f"revision"].state', + "old = revision[f\"st{'a'}te\"]", + "old = revision[f\"{'state'}\"]", + "old = record[f\"revi{'sion'}\"].state", + "old = revision[f\"{'st' + 'ate'}\"]", + "old = record[f\"{'revi' + 'sion'}\"].state", + "old = revision[f\"{'state':s}\"]", + ): + with self.subTest(source=source): + self.assertTrue(has_revision_state(source)) + + def test_static_format_bindings(self): + for source in ( + '"{0.state}".format(revision)', + '"{0[state]}".format(workspaceRevision)', + '"{item.state}".format(item=selectedWorkspace)', + '"{item[state]}".format_map({"item": revision})', + '("{0.state}").format(revision)', + '"{0:{1.state}}".format(value, revision)', + '"{0.state}".format(*[revision])', + '"{0[state]}".format(*(revision,))', + '"{1[state]}".format(*[other, workspaceRevision])', + '"{item.state}".format(**{"item": selectedWorkspace})', + '"{item[state]}".format(**{"outer": other, **{"item": revision}})', + '"{item.state}".format_map({**{"item": workspaceRevision}})', + '"{.state}".format(revision)', + '"{[state]}".format(revision)', + '"{:{.state}}".format(value, revision)', + '"{.name} {[state]}".format(other, revision)', + '"{item.state}".format(item=revision, **values)', + ): + with self.subTest(source=source): + self.assertTrue(has_revision_state(source)) + self.assertFalse(has_revision_state('"{0.state}".format(other)')) + # Dynamic unpacking is intentionally unresolved rather than guessed. + self.assertFalse(has_revision_state('"{0.state}".format(*values)')) + self.assertFalse(has_revision_state('"{.name} {[state]}".format(other, other)')) + self.assertFalse(has_revision_state('"{item.state}".format(**values)')) + # FormattedValue keys are dynamic and are not treated as static strings. + self.assertFalse(has_revision_state('revision[f"st{suffix}"]')) + + def test_literals_are_not_active(self): + self.assertFalse(has_revision_state('text = "{revision.state}"')) + self.assertFalse(has_revision_state('text = "{{revision.state}}".format(value)')) + + def test_oversized_static_format_fails_before_formatting(self): + tracemalloc.start() + with patch("builtins.format") as format_mock: + with self.assertRaisesRegex(ValueError, "width or precision"): + has_revision_state('revision[f"{1:.1000000000f}"]') + format_mock.assert_not_called() + _current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.assertLess(peak, 1_000_000) + self.assertFalse(has_revision_state('revision[f"{1:04d}"]')) + + def test_batch_contract(self): + self.assertEqual( + analyze_batch([{"label": "one.py", "source": "revision.state"}]), + ["one.py"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/scripts/verify-workspace-descriptor-files.mjs b/backend/scripts/verify-workspace-descriptor-files.mjs new file mode 100755 index 00000000..6dcebc40 --- /dev/null +++ b/backend/scripts/verify-workspace-descriptor-files.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { lstat, readFile, realpath } from "node:fs/promises"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { isMap, isScalar, parseAllDocuments } from "yaml"; +import { extractBashDocuments } from "./bash-heredoc.mjs"; +import { validatePythonRevisionStates, validateRevisionState } from "./revision-state-policy.mjs"; +import { parseWorkspaceYaml } from "../dist/workspaces/schema.js"; + +const scriptPath = fileURLToPath(import.meta.url); +const allowedKinds = new Set(["policy_text", "workspace_descriptor", "deployment_script"]); +// Exact-content trust exceptions. Each digest covers the raw UTF-8 bytes from the +// opener line through the closer line (including physical line endings). These +// blocks are reviewed non-workspace runtime/config generation, not semantic proof. +const reviewedExpandableBlocks = new Map([ + ["scripts/preprocess-smoke.sh", [ + { sha256: "fc530dc721c946644ab6552bbd46b7918d6c5f11f06f3495b6ea1fcda819b38d", rationale: "Generates the reviewed preprocess Compose override." }, + ]], + ["scripts/test-server-pi-state-topology.sh", [ + { sha256: "6f746f7e8442b0a6ea0e216607a6a923d94b24cd8fa17fa2d1dac56e6f14f7ef", rationale: "Generates the isolated server topology test environment." }, + ]], + ["scripts/test-vector-backup-restore-safety.sh", [ + { sha256: "40b8a10a3c06aaa98e324fbf688b7d1f5cead330d7ba7eef98e06256d412a85a", rationale: "Generates the reviewed restore safety manifest." }, + ]], + ["scripts/test-windows-clone-contract.ps1", [ + { sha256: "80f4880576a0679cb58e7b92600e7a90550c93c254553a2d4b299539f9ff0bcf", rationale: "Generates reviewed Windows clone test configuration." }, + { sha256: "6166294bdc8a8bf6436ad402bcbf7cae0f3b67dc6051cecfcca79267a62b082c", rationale: "Same reviewed block in the repository-required CRLF checkout representation." }, + { sha256: "3216201d59400ed7d1ec23e536634b8235a2e78b336e45b4dc598624920f0057", rationale: "Generates reviewed Windows clone test configuration." }, + { sha256: "a4044bb38b27e8120e90d65a0695fe0afd7757c067ae8dd67f170edf569a1de0", rationale: "Same reviewed block in the repository-required CRLF checkout representation." }, + { sha256: "5d0d1a3fc45e99b3aacaf4ee5dd09a6bee1937784375dfe4bcfaa4ae32cfb9de", rationale: "Generates reviewed Windows clone test configuration." }, + { sha256: "b903e5dae953ae1372f1a5276f12a92ed3dd632b897f3afe5e00c646d90a1b42", rationale: "Same reviewed block in the repository-required CRLF checkout representation." }, + ]], + ["scripts/unified-deployment-smoke.sh", [ + { sha256: "ca0c17d9ff8dc0fbe018fc1c5510eb33bc667a936fbe44a9be2d311390576825", rationale: "Generates reviewed Task 13 runtime configuration." }, + { sha256: "31ec00cc315b52da4a3bb6e3fba2d40aef29cdcd090bbc5d14c31f1aebbcfd04", rationale: "Generates reviewed Task 13 runtime configuration." }, + { sha256: "d92822815357ce3424e1a6eb43923df2b37b4fd93a3b5465ee9dfc69559ab0ed", rationale: "Generates reviewed Task 13 runtime configuration." }, + { sha256: "d6b8b7b951936c0452a485e9ee3b18a61251556581d6f7a2ce66f994b5700695", rationale: "Generates reviewed Task 13 runtime configuration." }, + ]], + ["scripts/vector-backup.sh", [ + { sha256: "571899db49dfdcec8107fbe1e0a86a61e7581979d3c4c248c20546843e275bcf", rationale: "Generates the reviewed backup manifest inside the helper command." }, + ]], + ["scripts/vector-restore.sh", [ + { sha256: "f04d872e556a7323583c6e620b25814fb6a8e2568a9a555623978185b473a49d", rationale: "Feeds reviewed parsed manifest values to read loops." }, + { sha256: "c6053ed44abae71ae4821b68f9a513f8070947350e30d89ae0f65bf4a48f66fd", rationale: "Feeds reviewed parsed manifest values to read loops." }, + ]], +]); + +function blockDigest(rawBlock) { + return createHash("sha256").update(rawBlock, "utf8").digest("hex"); +} + +function reviewedExpandableBlock(path, rawBlock) { + const digest = blockDigest(rawBlock); + return (reviewedExpandableBlocks.get(path) ?? []).some((review) => review.sha256 === digest); +} + +function hasAmbiguousExpansion(source, path) { + const powershell = path.endsWith(".ps1"); + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + if (powershell && character === "`") { + index += 1; + continue; + } + if (!powershell && character === "\\") { + index += 1; + continue; + } + if (character === "$" || (!powershell && character === "`")) return true; + } + return false; +} + +function physicalLines(source) { + const rawLines = source.match(/[^\n]*\n|[^\n]+$/gu) ?? []; + if (rawLines.length === 0) rawLines.push(""); + return rawLines.map((raw) => ({ raw, text: raw.replace(/\n$/u, "").replace(/\r$/u, "") })); +} +const prescribedSymbols = [ + "WorkspaceV1", "WorkspaceV2", "DeprecatedV2Descriptor", "LegacyMigrationResult", + "LegacyMigrationOptions", "WorkspaceV2MigrationInput", "migrateLegacyWorkspace", + "writeMigratedWorkspace", "migrateWorkspaceV1ToV2", "migrateWorkspaceV2ToV3", +]; +const migrationMarkers = ["migration_required", "deprecated-v2-descriptor", "migrate-legacy", "migrate-v2-qdrant"]; + +function isPolicyImplementationException(label, category) { + const implementations = new Set([ + "scripts/verify-schema-v3-only.sh", + "scripts/test-verify-schema-v3-only.sh", + "backend/scripts/verify-workspace-descriptor-files.mjs", + "backend/scripts/verify-workspace-descriptor-files.test.mjs", + "backend/scripts/revision-state-policy.mjs", + "backend/scripts/revision-state-policy.test.mjs", + "backend/scripts/bash-heredoc.mjs", + "backend/scripts/revision_state_policy.py", + "backend/scripts/test_revision_state_policy.py", + ]); + if (implementations.has(label)) return true; + if (category === "migration-marker" && new Set([ + "scripts/workspace_descriptor_doc_contract.py", + "scripts/test_workspace_descriptor_doc_contract.py", + "backend/scripts/clean-dist.test.mjs", + ]).has(label)) return true; + return false; +} + + +function validatePolicySource(source, label) { + if (!isPolicyImplementationException(label, "prescribed-symbol")) { + for (const symbol of prescribedSymbols) { + if (source.toLowerCase().includes(symbol.toLowerCase())) throw new Error(`${label}: forbidden prescribed-symbol substring: ${symbol}`); + } + } + if (!isPolicyImplementationException(label, "migration-marker")) { + for (const marker of migrationMarkers) { + if (source.toLowerCase().includes(marker.toLowerCase())) throw new Error(`${label}: forbidden migration-marker substring: ${marker}`); + } + } + if (!isPolicyImplementationException(label, "legacy-workspace")) { + for (const match of source.matchAll(/legacyworkspace/giu)) { + if (match[0] !== "legacyWorkspace") throw new Error(`${label}: forbidden legacy-workspace spelling: ${match[0]}`); + } + } + if (!/\.pyw?$/iu.test(label) && !isPolicyImplementationException(label, "revision-state")) validateRevisionState(source, label); +} + +function documentShape(document) { + const shape = { workspacePresent: false, workspaceMapping: false }; + if (!isMap(document.contents)) return shape; + for (const pair of document.contents.items) { + if (!isScalar(pair.key)) continue; + if (pair.key.value === "workspace") { + shape.workspacePresent = true; + if (isMap(pair.value)) shape.workspaceMapping = true; + } + } + return shape; +} + +function documents(source) { + try { + return parseAllDocuments(source, { uniqueKeys: true }); + } catch (error) { + throw new Error(`YAML parser failed: ${error instanceof Error ? error.message : String(error)}`); + } +} + +function validateWorkspaceSource(source, label, { requireWorkspace, expandable = false, path, rawBlock }) { + const parsed = documents(source); + const shapes = parsed.map(documentShape); + if (requireWorkspace) { + if (!shapes.some((shape) => shape.workspacePresent)) { + throw new Error(`${label}: expected a top-level workspace mapping`); + } + if (!shapes.some((shape) => shape.workspaceMapping)) { + throw new Error(`${label}: top-level workspace must be a mapping`); + } + } else { + if (expandable && hasAmbiguousExpansion(source, path) && !reviewedExpandableBlock(path, rawBlock)) { + throw new Error(`${label}: expandable block interpolation is not in the exact-content reviewed allowlist`); + } + if (shapes.some((shape) => shape.workspaceMapping)) { + throw new Error(`${label}: embedded workspace descriptor is forbidden; use a tracked workspace fixture`); + } + return false; + } + try { + parseWorkspaceYaml(source); + } catch (error) { + throw new Error(`${label}: workspace descriptor is not valid schema v3: ${error instanceof Error ? error.message : String(error)}`); + } + return true; +} + +function deploymentScriptDialect(path) { + if (path.endsWith(".sh")) return "bash"; + if (path.endsWith(".ps1")) return "powershell"; + throw new Error(`${path}: unknown deployment script dialect`); +} + + +function powerShellHereStringOpener(line, state) { + let quote = null; + for (let index = 0; index < line.length; index += 1) { + if (state.blockComment) { + const close = line.indexOf("#>", index); + if (close < 0) return null; + state.blockComment = false; + index = close + 1; + continue; + } + const character = line[index]; + if (quote === null && character === "`") { + index += 1; + continue; + } + if (quote === "'") { + if (character === "'" && line[index + 1] === "'") index += 1; + else if (character === "'") quote = null; + continue; + } + if (quote === '"') { + if (character === "`") index += 1; + else if (character === '"') quote = null; + continue; + } + if (character === "#") return null; + if (character === "<" && line[index + 1] === "#") { + state.blockComment = true; + index += 1; + continue; + } + if (character === "@" && (line[index + 1] === "'" || line[index + 1] === '"') && /^[ \t]*$/u.test(line.slice(index + 2))) return line[index + 1]; + if (character === "'" || character === '"') quote = character; + } + return null; +} + +function extractPowerShellDocuments(source, label) { + const records = physicalLines(source); + const lines = records.map((record) => record.text); + const extracted = []; + const state = { blockComment: false }; + for (let index = 0; index < lines.length; index += 1) { + const quote = powerShellHereStringOpener(lines[index], state); + if (quote === null) continue; + const delimiter = `${quote}@`; + const opener = index; + const body = []; + const start = index + 2; + let closed = false; + for (index += 1; index < lines.length; index += 1) { + if (lines[index].trimEnd() === delimiter) { + closed = true; + break; + } + body.push(lines[index]); + } + extracted.push({ + source: `${body.join("\n")}\n`, + label: `${label}:${start} PowerShell here-string${closed ? "" : " (unclosed)"}`, + expandable: quote === '"', + path: label, + rawBlock: records.slice(opener, Math.min(index + 1, records.length)).map((record) => record.raw).join(""), + }); + } + return extracted; +} + +export function extractScriptDocuments(source, label = "deployment script") { + const dialect = deploymentScriptDialect(label); + if (dialect === "bash") return extractBashDocuments(source, label); + return extractPowerShellDocuments(source, label); +} + +async function safeFile(root, path) { + if (typeof path !== "string" || path.length === 0 || isAbsolute(path) || path.includes("\\")) { + throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`); + } + const segments = path.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new Error(`unsafe verifier path: ${JSON.stringify(path)}`); + } + const absolute = resolve(root, ...segments); + const fromRoot = relative(root, absolute); + if (fromRoot.startsWith(`..${sep}`) || fromRoot === ".." || isAbsolute(fromRoot)) { + throw new Error(`verifier path escapes root: ${JSON.stringify(path)}`); + } + const entry = await lstat(absolute); + if (!entry.isFile() || entry.isSymbolicLink()) { + throw new Error(`verifier input is not a regular file: ${path}`); + } + const canonical = await realpath(absolute); + const canonicalRelative = relative(root, canonical); + if (canonicalRelative.startsWith(`..${sep}`) || canonicalRelative === ".." || isAbsolute(canonicalRelative)) { + throw new Error(`verifier input resolves outside root: ${path}`); + } + return absolute; +} + +export async function verifyEntries({ root, entries }) { + const canonicalRoot = await realpath(root); + const seen = new Set(); + const pythonPolicies = []; + for (const entry of entries) { + if (!entry || !allowedKinds.has(entry.kind) || typeof entry.path !== "string") { + throw new Error("workspace verifier manifest contains an invalid entry"); + } + const identity = `${entry.kind}\0${entry.path}`; + if (seen.has(identity)) throw new Error(`workspace verifier manifest duplicates: ${entry.path}`); + seen.add(identity); + const absolute = await safeFile(canonicalRoot, entry.path); + const bytes = await readFile(absolute); + let source; + try { + source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error(`${entry.path}: input is not valid UTF-8`); + } + if (source.includes("\0")) throw new Error(`${entry.path}: NUL byte is forbidden`); + if (entry.kind === "policy_text") { + validatePolicySource(source, entry.path); + if (/\.pyw?$/iu.test(entry.path) && !isPolicyImplementationException(entry.path, "revision-state")) { + pythonPolicies.push({ label: entry.path, source }); + } + continue; + } + if (entry.kind === "workspace_descriptor") { + validateWorkspaceSource(source, entry.path, { requireWorkspace: true }); + continue; + } + for (const candidate of extractScriptDocuments(source, entry.path)) { + validateWorkspaceSource(candidate.source, candidate.label, { + requireWorkspace: false, + expandable: candidate.expandable, + path: entry.path, + rawBlock: candidate.rawBlock, + }); + } + } + validatePythonRevisionStates(pythonPolicies); +} + +export function decodeManifest(bytes) { + const fields = bytes.toString("utf8").split("\0"); + if (fields.at(-1) !== "") throw new Error("workspace verifier manifest is not NUL-terminated"); + fields.pop(); + if (fields.length % 2 !== 0) throw new Error("workspace verifier manifest has an incomplete record"); + const entries = []; + for (let index = 0; index < fields.length; index += 2) { + entries.push({ kind: fields[index], path: fields[index + 1] }); + } + return entries; +} + +function cliArguments(argv) { + let root; + let manifest; + for (let index = 0; index < argv.length; index += 1) { + const option = argv[index]; + const value = argv[index + 1]; + if ((option === "--root" || option === "--manifest") && value !== undefined) { + if (option === "--root" && root === undefined) root = value; + else if (option === "--manifest" && manifest === undefined) manifest = value; + else throw new Error(`duplicate or invalid option: ${option}`); + index += 1; + } else { + throw new Error(`unknown or incomplete option: ${option}`); + } + } + if (root === undefined || manifest === undefined) { + throw new Error("usage: verify-workspace-descriptor-files.mjs --root ROOT --manifest NUL_FILE"); + } + return { root, manifest }; +} + +async function main(argv) { + const { root, manifest } = cliArguments(argv); + const manifestEntry = await lstat(manifest); + if (!manifestEntry.isFile() || manifestEntry.isSymbolicLink()) { + throw new Error("workspace verifier manifest is not a regular file"); + } + const entries = decodeManifest(await readFile(manifest)); + await verifyEntries({ root, entries }); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main(process.argv.slice(2)).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/backend/scripts/verify-workspace-descriptor-files.test.mjs b/backend/scripts/verify-workspace-descriptor-files.test.mjs new file mode 100644 index 00000000..97e4fe85 --- /dev/null +++ b/backend/scripts/verify-workspace-descriptor-files.test.mjs @@ -0,0 +1,992 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { extractScriptDocuments, verifyEntries } from "./verify-workspace-descriptor-files.mjs"; + +const repositoryRoot = fileURLToPath(new URL("../..", import.meta.url)); +const canonicalDescriptor = await readFile(join(repositoryRoot, "deploy/workspaces/example.yaml"), "utf8"); + +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), "thoth-workspace-yaml-verifier-")); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} + +async function put(root, path, content) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), content); +} + +function entry(kind, path) { + return { kind, path }; +} + +function bashN(root, path) { + execFileSync("/bin/bash", ["-n", join(root, path)], { stdio: "pipe" }); +} + +function replaceWorkspaceKeys(source, workspaceKey, schemaLine) { + return source + .replace(/^workspace:$/m, workspaceKey) + .replace(/^ schema_version: 3$/m, schemaLine); +} + +test("production parser accepts semantic v3 with quoted Unicode/tagged keys and spacing", async (t) => { + const root = await fixture(t); + const unicode = replaceWorkspaceKeys( + canonicalDescriptor, + '"\\u0077orkspace" :', + ' "\\u0073chema_version" : 3', + ); + const tagged = replaceWorkspaceKeys( + canonicalDescriptor, + "!!str workspace :", + " !!str schema_version : 3", + ); + await put(root, "deploy/workspaces/unicode.yaml", unicode); + await put(root, "deploy/workspaces/tagged.yaml", tagged); + await verifyEntries({ + root, + entries: [ + entry("workspace_descriptor", "deploy/workspaces/unicode.yaml"), + entry("workspace_descriptor", "deploy/workspaces/tagged.yaml"), + ], + }); +}); + +test("production parser rejects fancy keys with every non-v3 or ambiguous value", async (t) => { + const invalid = [ + ["unicode-v2", '"\\u0077orkspace" :', ' "\\u0073chema_version" : 2'], + ["tagged-leading-zero", "!!str workspace :", " !!str schema_version : 02"], + ["hexadecimal", "workspace :", " schema_version : 0x2"], + ["multiline", "workspace :", " schema_version : >\n 3"], + ["duplicate", "workspace :", " schema_version : 3\n schema_version: 3"], + ["inline", "workspace: { schema_version: 3 }", " schema_version: 3"], + ]; + for (const [name, workspaceKey, schemaLine] of invalid) { + await t.test(name, async () => { + const root = await mkdtemp(join(tmpdir(), `thoth-workspace-yaml-${name}-`)); + try { + const source = replaceWorkspaceKeys(canonicalDescriptor, workspaceKey, schemaLine); + const path = `deploy/workspaces/${name}.yaml`; + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }), + /workspace descriptor/i, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + } +}); + +test("Bash embedded workspace mappings are rejected while tracked-fixture-only bundles pass", async (t) => { + const root = await fixture(t); + const validScript = [ + "#!/usr/bin/env bash", + "cat <<'WORKSPACE_YAML'", + canonicalDescriptor.trimEnd(), + "WORKSPACE_YAML", + "cat <<'BUNDLE_YAML'", + "bundle:", + " name: deploy", + "schema_version: 1", + "job:", + " state: operational", + "BUNDLE_YAML", + "", + ].join("\n"); + await put(root, "scripts/operator-smoke.sh", validScript); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator-smoke.sh")] }), + /embedded workspace descriptor/i, + ); + + const bundleScript = validScript.replace(canonicalDescriptor.trimEnd(), "job:\n name: deploy"); + await put(root, "scripts/operator-smoke.sh", bundleScript); + await verifyEntries({ + root, + entries: [entry("deployment_script", "scripts/operator-smoke.sh")], + }); +}); + +test("PowerShell embedded workspace mappings are rejected while bundle-only strings pass", async (t) => { + const root = await fixture(t); + const source = [ + "$workspace = @'", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 0x2").trimEnd(), + "'@", + '$bundle = @"', + "bundle:", + " schema_version: 1", + '"@', + "", + ].join("\n"); + await put(root, "scripts/operator.ps1", source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/operator.ps1")] }), + /workspace descriptor/i, + ); +}); + +test("workspace descriptor family entries require a top-level workspace", async (t) => { + const root = await fixture(t); + await put(root, "scripts/fixtures/workspace-registry-future.yaml", "bundle:\n schema_version: 3\n"); + await assert.rejects( + verifyEntries({ + root, + entries: [entry("workspace_descriptor", "scripts/fixtures/workspace-registry-future.yaml")], + }), + /top-level workspace/i, + ); +}); + + +test("script scalar workspace remains a bundle even with descriptor-like siblings", async (t) => { + const root = await fixture(t); + const path = "scripts/job-smoke.sh"; + const job = [ + "#!/usr/bin/env bash", + "cat <<'JOB-YAML'", + "job: refresh", + "workspace: analytics", + "schema_version: 2", + "state: operational", + "JOB-YAML", + "", + ].join("\n"); + await put(root, path, job); + bashN(root, path); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); + + const bundles = [ + job.replace("job: refresh", "dwh:\n engine: postgres"), + job.replace("job: refresh", "evidence:\n source: bundle"), + ]; + for (const bundle of bundles) { + await put(root, path, bundle); + bashN(root, path); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); + } +}); + +test("standalone descriptor files require workspace to be a mapping", async (t) => { + const root = await fixture(t); + const path = "scripts/fixtures/workspace-registry-scalar.yaml"; + await put(root, path, "workspace: analytics\nschema_version: 3\n"); + await assert.rejects( + verifyEntries({ root, entries: [entry("workspace_descriptor", path)] }), + /workspace.*mapping/i, + ); +}); + +test("Bash extractor supports hyphen, digit, escaped delimiters, and tab stripping", async (t) => { + const root = await fixture(t); + const cases = [ + { + name: "hyphen-v2", + opener: "cat <<'WORKSPACE-YAML'", + delimiter: "WORKSPACE-YAML", + descriptor: canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2"), + rejected: true, + }, + { + name: "digit-v3", + opener: "cat <<2YAML", + delimiter: "2YAML", + descriptor: canonicalDescriptor, + rejected: true, + }, + { + name: "escaped-v2", + opener: "cat < `\t${line}`).join("\n"), + rejected: true, + }, + ]; + for (const item of cases) { + await t.test(item.name, async () => { + const path = `scripts/${item.name}-smoke.sh`; + const source = ["#!/usr/bin/env bash", item.opener, item.descriptor.trimEnd(), item.delimiter, ""].join("\n"); + await put(root, path, source); + bashN(root, path); + const verification = verifyEntries({ root, entries: [entry("deployment_script", path)] }); + if (item.rejected) await assert.rejects(verification, /workspace descriptor/i); + else await verification; + }); + } +}); + +test("unsupported Bash heredoc opener fails closed while a bundle heredoc stays allowed", async (t) => { + const root = await fixture(t); + const unsupportedPath = "scripts/unsupported-smoke.sh"; + const unsupported = [ + "#!/usr/bin/env bash", + "cat <<$DELIMITER", + canonicalDescriptor.trimEnd(), + "$DELIMITER", + "", + ].join("\n"); + await put(root, unsupportedPath, unsupported); + bashN(root, unsupportedPath); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", unsupportedPath)] }), + /unsupported Bash heredoc opener/i, + ); + + const bundlePath = "scripts/bundle-smoke.sh"; + const bundle = [ + "#!/usr/bin/env bash", + "cat <<'BUNDLE-YAML'", + "job: refresh", + "workspace: analytics", + "schema_version: 1", + "state: operational", + "BUNDLE-YAML", + "", + ].join("\n"); + await put(root, bundlePath, bundle); + bashN(root, bundlePath); + await verifyEntries({ root, entries: [entry("deployment_script", bundlePath)] }); +}); + + +test("non-stripping heredoc close requires an exact physical delimiter line", async (t) => { + const root = await fixture(t); + const path = "scripts/trailing-close-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat <<'---'", + "--- ", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "---", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("delimiter-like body lines remain content until a real exact close", async (t) => { + const root = await fixture(t); + const path = "scripts/delimiter-content-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat <<'END'", + "END ", + " END", + "job: refresh", + "workspace: analytics", + "schema_version: 1", + "END", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + const [candidate] = extractScriptDocuments(source, path); + assert.match(candidate.source, /^END \n END\n/u); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + + +test("double-quoted non-special backslash is preserved in the delimiter", async (t) => { + const root = await fixture(t); + const path = "scripts/double-quoted-nonspecial-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + 'cat <<"\\---"', + "---", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "\\---", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("double-quoted delimiter quote removal matches Bash special escapes", async (t) => { + const root = await fixture(t); + const cases = [ + ["dollar", 'cat <<"DOL\\$LAR"', "DOL$LAR"], + ["backtick", 'cat <<"TIC\\`K"', "TIC`K"], + ["quote", 'cat <<"QUO\\\"TE"', 'QUO"TE'], + ["backslash", 'cat <<"SLA\\\\SH"', "SLA\\SH"], + ["newline", 'cat <<"LINE\\\nBREAK"', "LINEBREAK"], + ["nonspecial", 'cat <<"NON\\-SPECIAL"', "NON\\-SPECIAL"], + ]; + for (const [name, opener, close] of cases) { + const path = `scripts/double-quoted-${name}-smoke.sh`; + const source = ["#!/usr/bin/env bash", opener, "job: refresh", close, ""].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.equal(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), "job: refresh\n"); + assert.equal(extractScriptDocuments(source, path)[0].source, "job: refresh\n"); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); + } +}); + + +test("split heredoc operator continuation cannot bypass v2 validation", async (t) => { + const root = await fixture(t); + const path = "scripts/split-operator-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat <\\", + "<'YAML'", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.match(execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), /schema_version: 2/u); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("multiple opener continuations are joined before heredoc discovery", async (t) => { + const root = await fixture(t); + const path = "scripts/multiple-continuation-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "cat \\", + "<\\", + "<'YAML'", + "job: refresh", + "workspace: analytics", + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.equal( + execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), + "job: refresh\nworkspace: analytics\n", + ); + const [candidate] = extractScriptDocuments(source, path); + assert.equal(candidate.label, `${path}:5 Bash heredoc`); + assert.equal(candidate.source, "job: refresh\nworkspace: analytics\n"); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + +test("backslash-newline inside single quotes is not removed", async (t) => { + const root = await fixture(t); + const path = "scripts/single-quoted-noncontinuation-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + "printf '%s' 'literal\\", + "continued'", + "cat <<'YAML'", + "job: refresh", + "workspace: analytics", + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + assert.equal( + execFileSync("/bin/bash", [join(root, path)], { encoding: "utf8" }), + "literal\\\ncontinuedjob: refresh\nworkspace: analytics\n", + ); + const [candidate] = extractScriptDocuments(source, path); + assert.equal(candidate.label, `${path}:5 Bash heredoc`); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + + +test("PowerShell comment backslash cannot hide a following v2 here-string", async (t) => { + const root = await fixture(t); + const path = "scripts/powershell-comment-smoke.ps1"; + const source = [ + "# harmless PowerShell comment \\", + "$workspace = @'", + canonicalDescriptor.replace(" schema_version: 3", " schema_version: 2").trimEnd(), + "'@", + "", + ].join("\n"); + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /workspace descriptor/i, + ); +}); + +test("PowerShell dialect accepts normal v3 and non-workspace bundle here-strings", async (t) => { + const root = await fixture(t); + const path = "scripts/powershell-valid-smoke.ps1"; + const source = [ + "$workspace = @'", + canonicalDescriptor.trimEnd(), + "'@", + "$bundle = @'", + "evidence:", + " source: bundle", + "schema_version: 2", + "'@", + "", + ].join("\n"); + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /embedded workspace descriptor/i, + ); + + const bundleOnly = [ + "$bundle = @'", + "evidence:", + " source: bundle", + "schema_version: 2", + "'@", + "", + ].join("\n"); + await put(root, path, bundleOnly); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + +test("unknown deployment script dialect fails closed", async (t) => { + const root = await fixture(t); + const path = "scripts/operator-smoke.cmd"; + await put(root, path, "echo harmless\n"); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /unknown deployment script dialect/i, + ); +}); + + +test("PowerShell cast and concatenation openers cannot hide embedded descriptors", async (t) => { + const root = await fixture(t); + for (const [name, opener] of [["cast", "[string]@'"], ["concat", "+@'"]]) { + const path = `scripts/powershell-${name}-smoke.ps1`; + const source = [opener, canonicalDescriptor.trimEnd(), "'@", ""].join("\n"); + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /embedded workspace descriptor/i, + ); + } +}); + +test("expandable YAML interpolation that can hide a workspace descriptor fails closed", async (t) => { + const root = await fixture(t); + const cases = [ + ["braced-key", "${key}:\n schema_version: 3"], + ["plain-key", "$key:\n schema_version: 3"], + ["quoted-key", '"$key" :\n schema_version: 3'], + ["subexpression-key", "$($key):\n schema_version: 3"], + ["version", "workspace:\n schema_version: $version"], + ]; + for (const [name, body] of cases) { + const path = `scripts/powershell-interpolation-${name}.ps1`; + await put(root, path, [`$yaml = @\"`, body, `\"@`, ""].join("\n")); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /interpolation|embedded workspace descriptor/i, + ); + } +}); + +test("Bash heredoc discovery ignores quoted, comment, here-string, and arithmetic tokens", async (t) => { + const root = await fixture(t); + const path = "scripts/bash-lexer-smoke.sh"; + const source = [ + "#!/usr/bin/env bash", + `printf '%s\\n' \"cat <<'QUOTED'\"`, + `printf '%s\\n' 'cat <<\"SINGLE\"'`, + "# cat <<'COMMENT'", + "value=$((1 << 2))", + `cat <<< \"not a heredoc\"`, + "cat <<'YAML'", + "job: refresh", + "YAML", + "", + ].join("\n"); + await put(root, path, source); + bashN(root, path); + const extracted = extractScriptDocuments(source, path); + assert.equal(extracted.length, 1); + assert.equal(extracted[0].source, "job: refresh\n"); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + +test("UTF-8 decoding is fatal but literal replacement characters are valid text", async (t) => { + const root = await fixture(t); + const validPath = "deploy/workspaces/replacement.yaml"; + await put(root, validPath, `${canonicalDescriptor}# literal replacement: �\n`); + await verifyEntries({ root, entries: [entry("workspace_descriptor", validPath)] }); + + const invalidPath = "deploy/workspaces/malformed.yaml"; + await mkdir(dirname(join(root, invalidPath)), { recursive: true }); + await writeFile(join(root, invalidPath), Buffer.concat([Buffer.from(canonicalDescriptor), Buffer.from([0xff])])); + await assert.rejects( + verifyEntries({ root, entries: [entry("workspace_descriptor", invalidPath)] }), + /valid UTF-8/i, + ); +}); + + +test("unmarked expandable Bash YAML cannot generate descriptor keys or values at runtime", async (t) => { + const root = await fixture(t); + const cases = [ + ["quoted", '"$key" :'], + ["command", "$(printf workspace):"], + ["braced", "${key}:"], + ["plain", "$key:"], + ]; + for (const [name, generatedKey] of cases) { + const path = `scripts/bash-dynamic-${name}.sh`; + const source = [ + "#!/usr/bin/env bash", + "key=workspace", + "cat < { + const root = await fixture(t); + for (const [path, source] of [ + ["scripts/fake-marker.sh", [ + "#!/usr/bin/env bash", + "# schema-v3-only: expandable-nonworkspace", + "cat < { + const reviewedPaths = [ + "scripts/preprocess-smoke.sh", + "scripts/test-server-pi-state-topology.sh", + "scripts/test-vector-backup-restore-safety.sh", + "scripts/test-windows-clone-contract.ps1", + "scripts/unified-deployment-smoke.sh", + "scripts/vector-backup.sh", + "scripts/vector-restore.sh", + ]; + await verifyEntries({ + root: repositoryRoot, + entries: reviewedPaths.map((path) => entry("deployment_script", path)), + }); + + const root = await fixture(t); + const original = await readFile(join(repositoryRoot, "scripts/preprocess-smoke.sh"), "utf8"); + await put(root, "scripts/copied-preprocess.sh", original); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/copied-preprocess.sh")] }), + /exact-content reviewed allowlist/, + ); + await put(root, "scripts/preprocess-smoke.sh", original.replace('$tmp/smoke.yaml', '$tmp/other.yaml')); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", "scripts/preprocess-smoke.sh")] }), + /exact-content reviewed allowlist/, + ); +}); + +test("PowerShell tokenizer ignores opener text in comments and ordinary strings", async (t) => { + const root = await fixture(t); + const path = "scripts/powershell-lexical-context.ps1"; + const source = [ + "# example @'", + '\"example @\'\"', + "'example @\"'", + "<# block @'", + "still @\" #>", + "$cast = [string]@'", + "job: cast", + "'@", + "$concat = $cast +@'", + "job: concat", + "'@", + "", + ].join("\n"); + await put(root, path, source); + const extracted = extractScriptDocuments(source, path); + assert.equal(extracted.length, 2); + assert.deepEqual(extracted.map((item) => item.source), ["job: cast\n", "job: concat\n"]); + await verifyEntries({ root, entries: [entry("deployment_script", path)] }); +}); + + +test("policy text rejects NUL and prescribed symbol substrings but permits lower-camel legacy identifiers", async (t) => { + const root = await fixture(t); + await put(root, "backend/src/nul.ts", Buffer.from("safe\0WorkspaceV2")); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", "backend/src/nul.ts")] }), /NUL byte/); + + for (const [name, text] of [ + ["compat", "type X = WorkspaceV2Compat;"], + ["mixed-prescribed", "type X = wOrKsPaCeV2;"], + ["lower-deprecated", "type X = deprecatedV2Descriptor;"], + ["upper-function", "WRITEMIGRATEDWORKSPACE(value);"], + ["adapter", "type X = LegacyWorkspaceAdapter;"], + ["lower", "type X = legacyworkspace;"], + ["mixed", "type X = LeGaCyWoRkSpAcE;"], + ]) { + const path = `backend/src/${name}.ts`; + await put(root, path, text); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /forbidden/); + } + await put(root, "backend/src/allowed.ts", "const legacyWorkspacePath = value;"); + await verifyEntries({ root, entries: [entry("policy_text", "backend/src/allowed.ts")] }); +}); + +test("revision-state structural scan permits only the exact historical decoder occurrence", async (t) => { + const root = await fixture(t); + const registry = "backend/src/workspaces/registry.ts"; + await put(root, registry, 'if (revision.state !== "operational") return;\n'); + await verifyEntries({ root, entries: [entry("policy_text", registry)] }); + + const variants = [ + 'if (revision.state !== "operational") return;\nif (revision["state"] === value) return;\n', + 'if (workspaceRevision\n .state === value) return;\n', + "if (selectedWorkspace [ 'state' ] === value) return;\n", + ]; + for (let index = 0; index < variants.length; index += 1) { + const path = index === 0 ? registry : `frontend/src/revision-${index}.ts`; + await put(root, path, variants[index]); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); + } +}); + + +test("complete descriptors supplied only through Bash or PowerShell variables require exact review", async (t) => { + const root = await fixture(t); + const cases = [ + ["scripts/variable-descriptor.sh", ["#!/usr/bin/env bash", "cat < { + const root = await fixture(t); + const cases = [ + ["scripts/positional.sh", "cat < { + const root = await fixture(t); + for (const [name, prefix] of [ + ["escaped-hash", "Write-Output `# harmless"], + ["escaped-quote", 'Write-Output `" harmless'], + ]) { + const path = `scripts/${name}.ps1`; + const source = [prefix, "$yaml = @'", "workspace:", " schema_version: 2", "'@", ""].join("\n"); + await put(root, path, source); + assert.equal(extractScriptDocuments(source, path).length, 1); + await assert.rejects( + verifyEntries({ root, entries: [entry("deployment_script", path)] }), + /embedded workspace descriptor/, + ); + } +}); + +test("TypeScript AST rejects comment-separated and destructured revision state", async (t) => { + const root = await fixture(t); + for (const [index, source] of [ + "const value = revision /*legacy*/ . state;", + "const { state } = revision;", + "const { state: oldState } = selectedWorkspace;", + ].entries()) { + const path = `frontend/src/ast-revision-${index}.ts`; + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); + } + const registry = "backend/src/workspaces/registry.ts"; + await put(root, registry, 'if (revision.state !== "operational") return;\nconst { state } = revision;\n'); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/); + await put(root, "backend/src/unrelated.ts", "const { state } = lease; const jobState = job.state;"); + await verifyEntries({ root, entries: [entry("policy_text", "backend/src/unrelated.ts")] }); +}); + + +test("AST recognizes semantic state keys in every revision destructuring form", async (t) => { + const root = await fixture(t); + const cases = [ + ["backend/src/computed.mts", 'const { ["state"]: oldState } = revision;'], + ["frontend/src/renamed.cts", 'const { "state": oldState = fallback } = workspaceRevision;'], + ["backend/scripts/template.TS", 'const { [`state`]: oldState } = selectedWorkspace;'], + ["scripts/parameter.txt", 'function read({ state: oldState = fallback } = revision) {}'], + ["scripts/assignment.sh", '({ state } = workspaceRevision);'], + ["scripts/computed-assignment.data", '({ ["state"]: oldState = fallback } = selectedWorkspace);'], + ]; + for (const [path, source] of cases) { + await put(root, path, source); + await assert.rejects( + verifyEntries({ root, entries: [entry("policy_text", path)] }), + /revision-state/, + path, + ); + } + + const registry = "backend/src/workspaces/registry.ts"; + await put(root, registry, [ + 'if (revision.state !== "operational") return;', + 'function read({ ["state"]: oldState } = revision) {}', + "", + ].join("\n")); + await assert.rejects( + verifyEntries({ root, entries: [entry("policy_text", registry)] }), + /revision-state/, + ); +}); + +test("tolerant all-suffix AST scan ignores strings/comments and unrelated state", async (t) => { + const root = await fixture(t); + const path = "scripts/arbitrary.weird"; + await put(root, path, [ + '// const { state } = revision;', + '"revision.state";', + "'({ [\\\"state\\\"]: oldState } = selectedWorkspace)';", + "const { state } = lease;", + "const jobState = job.state;", + "record.state = 'ready';", + "", + ].join("\n")); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); +}); + + +test("computed revision destructuring keys fold parentheses assertions templates and string concatenation", async (t) => { + const root = await fixture(t); + const cases = [ + ["backend/src/paren.ts", 'const { [("state")]: oldState } = revision;'], + ["backend/src/concat.ts", 'const { ["st" + "ate"]: oldState } = workspaceRevision;'], + ["frontend/src/template.ts", 'const { [`st${"ate"}`]: oldState } = selectedWorkspace;'], + ["scripts/assertion.data", 'const { [("st" as string) + (`ate` satisfies string)]: oldState } = revision;'], + ["scripts/assignment.txt", '({ ["st" + "ate"]: oldState } = selectedWorkspace);'], + ]; + for (const [path, source] of cases) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + + const registry = "backend/src/workspaces/registry.ts"; + for (const injected of [ + 'const { [("state")]: oldState } = revision;', + '({ ["st" + "ate"]: oldState } = revision);', + ]) { + await put(root, registry, `if (revision.state !== "operational") return;\n${injected}\n`); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", registry)] }), /revision-state/); + } +}); + +test("polyglot masking and JSX syntax prevent comment and string false positives", async (t) => { + const root = await fixture(t); + const passing = [ + ["backend/scripts/comment.py", '# revision.state\nvalue = "revision.state"\ntext = """selectedWorkspace.state"""\n'], + ["scripts/comment.ps1", '# revision.state\n<# workspaceRevision.state #>\n$value = "revision.state"\n'], + ["scripts/comment.sh", '# revision.state\nprintf \'%s\\n\' "selectedWorkspace.state"\n'], + ["frontend/src/content.tsx", 'export const view =
revision.state
;'], + ["frontend/src/attribute.tsx", 'export const view =
;'], + ["frontend/src/expression.tsx", 'export const view =
{"revision.state"}
;'], + ["scripts/arbitrary.data", 'title: "revision.state"\n# const { state } = revision\nlease:\n state: ready\n'], + ]; + for (const [path, source] of passing) { + await put(root, path, source); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } + + for (const [path, source] of [ + ["scripts/code.txt", "const { state } = revision;"], + ["scripts/code.data", '({ ["st" + "ate"]: oldState } = workspaceRevision);'], + ]) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); + } +}); + + +test("rest bindings and dynamic computed keys are not semantic state-property access", async (t) => { + const root = await fixture(t); + const cases = [ + ["backend/src/rest.ts", "const { ...state } = revision;"], + ["frontend/src/renamed.ts", "const { other: state } = workspaceRevision;"], + ["scripts/dynamic.txt", "const { [state]: value } = selectedWorkspace;"], + ["scripts/dynamic-assignment.data", "({ [state]: value } = revision);"], + ["scripts/spread-assignment.data", "({ ...state } = workspaceRevision);"], + ]; + for (const [path, source] of cases) { + await put(root, path, source); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } +}); + +test("polyglot code remains structural across shell Python PowerShell YAML TSX and JSX", async (t) => { + const root = await fixture(t); + const failing = [ + ["scripts/code.sh", "value=revision.state\n"], + ["scripts/code.ps1", "$value = workspaceRevision.state\n"], + ["backend/scripts/code.py", "value = selectedWorkspace.state\n"], + ["scripts/code.yaml", "value: revision.state\n"], + ["frontend/src/code.tsx", "export const view =
{revision.state}
;"], + ["frontend/src/code.jsx", "export const view =
{workspaceRevision.state}
;"], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } +}); + + +test("PowerShell executable subexpressions expose dollar-prefixed revision access", async (t) => { + const root = await fixture(t); + const failing = [ + ["scripts/ps-property.ps1", 'Write-Output "revision: $($revision.state)"\n'], + ["scripts/ps-element.ps1", 'Write-Output "$($workspaceRevision[\'state\'])"\n'], + ["scripts/ps-workspace.ps1", '$value = $workspaceRevision.state\n'], + ["scripts/ps-nested.ps1", 'Write-Output "$($($revision.state))"\n'], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + const passing = [ + '# $revision.state\nWrite-Output "revision.state"\n', + "Write-Output '$selectedWorkspace[\"state\"]'\n", + ]; + for (let index = 0; index < passing.length; index += 1) { + const path = `scripts/ps-literal-${index}.ps1`; + await put(root, path, passing[index]); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } +}); + +test("Python f-string fields expose revision access while literal text remains masked", async (t) => { + const root = await fixture(t); + const failing = [ + ["backend/scripts/f-property.py", 'value = f"{revision.state}"\n'], + ["backend/scripts/fr-element.py", 'value = fr"{workspaceRevision[\'state\']}"\n'], + ["backend/scripts/rf-element.py", 'value = rf"prefix {selectedWorkspace[\"state\"]}"\n'], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + const passing = [ + 'value = f"revision.state"\n', + 'value = f"{{revision.state}}"\n', + 'value = "revision.state"\n', + 'value = r"workspaceRevision.state"\n', + 'value = """selectedWorkspace.state"""\n', + 'value = r"""revision.state"""\n', + ]; + for (let index = 0; index < passing.length; index += 1) { + const path = `backend/scripts/python-literal-${index}.py`; + await put(root, path, passing[index]); + await verifyEntries({ root, entries: [entry("policy_text", path)] }); + } +}); + + +test("Bash masking preserves parameter trimming and executable command consumers", async (t) => { + const root = await fixture(t); + const failing = [ + ["scripts/trim.sh", "trimmed=${value#prefix}; old=revision.state\n"], + ["scripts/base.sh", "base=${path##*/}; old=workspaceRevision.state\n"], + ["scripts/backtick.sh", "old=`echo revision.state`\n"], + ["scripts/quoted-backtick.sh", 'echo "old: `echo revision.state`"\n'], + ["scripts/jq.sh", "jq '.revision.state' snapshot.json\n"], + ["scripts/substitution.sh", 'echo "$(echo revision.state)"\n'], + ]; + for (const [path, source] of failing) { + await put(root, path, source); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/, path); + } + await put(root, "scripts/echo.sh", 'echo "revision.state"\n# workspaceRevision.state\n'); + await verifyEntries({ root, entries: [entry("policy_text", "scripts/echo.sh")] }); + await put(root, "scripts/literal.yaml", '# revision.state\nvalue: "selectedWorkspace.state"\n'); + await verifyEntries({ root, entries: [entry("policy_text", "scripts/literal.yaml")] }); +}); + +test("YAML keeps URL slashes as data rather than a false line comment", async (t) => { + const root = await fixture(t); + const path = "scripts/url.yaml"; + await put(root, path, "url: https://host/x; old: selectedWorkspace.state\n"); + await assert.rejects(verifyEntries({ root, entries: [entry("policy_text", path)] }), /revision-state/); +}); diff --git a/backend/src/app.ts b/backend/src/app.ts index 190da83a..f7fdfe5f 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -1,19 +1,42 @@ -import Fastify, { type FastifyInstance } from "fastify"; +import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify"; import cors from "@fastify/cors"; +import cookie from "@fastify/cookie"; +import rateLimit from "@fastify/rate-limit"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import type { AppConfig } from "./config.js"; import { ThtRunner } from "./tht/tht-runner.js"; import { PiProcessManager } from "./pi/pi-process-manager.js"; import { SseHub } from "./sse/sse-hub.js"; -import { authPreHandler } from "./auth/auth.js"; -import { getPrincipal } from "./auth/auth.js"; +import { authenticateSession, captureAuthConfigSnapshot, configuredOrigin } from "./auth/auth.js"; import type { PrincipalContext } from "./auth/principal.js"; +import type { LoadedAuthConfig } from "./auth/types.js"; +import { createCurrentLocalUserRegistryResolver, type LocalUserRegistry } from "./auth/local-registry.js"; +import { AuthSessionOperationalError, createFileAuthSessionStore, type AuthSessionStore, type AuthSessionValidity } from "./auth/session-store.js"; +import type { WindowsAuthStorageBridge } from "./auth/windows-auth-storage.js"; +import { registerAuthRoutes } from "./auth/routes.js"; +import { createOidcProtocol, type OidcProtocol, type OidcProtocolOptions } from "./auth/oidc-client.js"; +import { createConfiguredAuthDiagnoser } from "./auth/diagnostic-command.js"; +import type { AuthDiagnoser } from "./auth/diagnostics.js"; +import { isUsableAuthenticationSecret } from "./auth/secret-policy.js"; +import { secretValue } from "./config/secret-bundle.js"; import { sessionRoutes } from "./routes/sessions.js"; import { sqlRoutes } from "./routes/sql.js"; import { metaRoutes, type ListModelsFn } from "./routes/meta.js"; import { settingsRoutes, effectiveSettings } from "./routes/settings.js"; import { createPiModelLister } from "./pi/list-models.js"; -import { loadSettings, saveSettings, type Settings } from "./settings/settings-store.js"; +import { createPiManagement, type PiManagementService } from "./pi/management.js"; +import { loadSettings, type Settings } from "./settings/settings-store.js"; import { ReadinessManager } from "./runtime/readiness-manager.js"; +import { MaintenanceBarrier } from "./runtime/maintenance-gate.js"; +import { WorkspaceRegistry } from "./workspaces/registry.js"; +import { createProductionWorkspaceDiagnoser } from "./workspaces/diagnostics.js"; +import { workspaceRoutes, type WorkspaceDiagnoser } from "./routes/workspaces.js"; +import { piManagementRoutes } from "./routes/pi-management.js"; +import { supportsSessionRuntime } from "./workspaces/bindings.js"; +import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js"; +import type { WorkspaceDescriptor } from "./workspaces/schema.js"; +import { WorkspaceSecretStore } from "./workspaces/secret-store.js"; export interface BuildAppDeps { thtRunner?: ThtRunner; @@ -23,28 +46,101 @@ export interface BuildAppDeps { getSettings?: (principal?: PrincipalContext) => Settings | Promise; readiness?: ReadinessManager; hub?: SseHub; + workspaceRegistry?: WorkspaceRegistry; + workspaceDiagnoser?: WorkspaceDiagnoser; + workspaceSecretStore?: WorkspaceSecretStore; + workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean; + maintenanceBarrier?: MaintenanceBarrier; + piManagement?: PiManagementService; + localUserRegistry?: LocalUserRegistry; + authSessionStore?: AuthSessionStore; + /** Explicit test-only transport seam; production always invokes the hidden tht bridge. */ + authStorageBridgeForTest?: WindowsAuthStorageBridge; + oidcProtocol?: OidcProtocol; + authDiagnoser?: AuthDiagnoser; + /** Explicit test seam; production uses the provider-neutral OIDC constructor. */ + oidcProtocolFactory?: (options: OidcProtocolOptions) => OidcProtocol; +} + +export interface AppWithAuthSessionStore extends FastifyInstance { + thothiiAuthSessionStore?: AuthSessionStore; } export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstance { const app = Fastify({ logger: { level: "warn" }, disableRequestLogging: true }); - - // Allow any origin in dev/e2e; tighten in production via config if needed. - app.register(cors, { - origin: true, - credentials: true, - methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"], + app.decorateRequest("authConfigSnapshot", undefined); + app.decorateRequest("authConfigSnapshotCaptured", false); + app.decorateRequest("authConfigSnapshotUnavailable", false); + const isolatedTestRoot = process.env.VITEST === "true" + ? join(tmpdir(), `thothii-workspace-secrets-vitest-${process.pid}`) + : undefined; + const workspaceSecretStore = deps?.workspaceSecretStore ?? new WorkspaceSecretStore({ + root: isolatedTestRoot ?? config.workspaceSecretStoreRoot, + runtimeRoot: isolatedTestRoot === undefined + ? config.workspaceSecretRuntimeRoot + : join(isolatedTestRoot, "runtime"), + installationId: config.workspaceRegistry.installationId, }); + const cookieAuth = config.authMode === "local" || config.authMode === "oidc"; + app.register(cors, { + // The delegator runs at CORS's onRequest hook. It owns the one request-scoped config load + // which subsequent auth hooks and routes consume, including preflights that end here. + delegator: (request, callback) => { + const snapshot = captureAuthConfigSnapshot(request, config.authentication); + const origin = configuredOrigin(snapshot); + const snapshotUsesCookies = snapshot?.value.mode === "local" || snapshot?.value.mode === "oidc"; + callback(null, { + origin: snapshotUsesCookies && origin ? corsOrigin(request, origin) : cookieAuth ? false : true, + credentials: snapshotUsesCookies, + methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], + }); + }, + }); + // Cookie parsing and the rate-limit plugin must precede every auth/application route. + app.register(cookie); + app.register(rateLimit, { global: false }); + const tht = deps?.thtRunner ?? new ThtRunner({ thtBin: config.thtBin, harnessDir: config.harnessDir, configPath: process.env.THT_CONFIG ?? "config/tht.yaml", dataRoot: config.dataRoot, + runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"), + secretRoots: config.workspaceRegistry.secretRoots, secretsFile: config.secretsFile, secretFiles: config.secretFiles, + workspaceSecretStore, + semanticRuntime: { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }, }); const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined); const hub = deps?.hub ?? new SseHub(); + const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry); + const workspaceDiagnoser = deps?.workspaceDiagnoser + ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }); + const workspaceRuntimeSupport = deps?.workspaceRuntimeSupport ?? ((workspace: WorkspaceDescriptor) => { + const lease = resolveRuntimeBindingsWithWorkspaceSecrets( + workspace, + process.env, + config.workspaceRegistry.secretRoots, + workspaceSecretStore, + ); + try { + return supportsSessionRuntime(lease.bindings); + } finally { + lease.release(); + } + }); const readiness = deps?.readiness ?? new ReadinessManager( tht as ThtRunner, Math.round(config.ollamaEnsureTimeoutMs / 1000), @@ -62,45 +158,199 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc }; const getSettings = async (principal: PrincipalContext): Promise => { if (deps?.getSettings) return await deps.getSettings(principal); - const runner = runnerFor(principal); - // The real runner persists preferences through the harness repository. The file fallback - // only keeps older isolated route tests and externally injected runners compatible. - if (typeof runner.preferencesGet === "function") { - const stored = await runner.preferencesGet() as Settings; - if (Object.keys(stored).length === 0) { - const seeded = effectiveSettings(config, loadSettings(config)); - await runner.preferencesSet(seeded); - return seeded; + const stored = loadSettings(config); + const effective = effectiveSettings(config, stored); + // In the registry system the legacy `harness/workspaces/*.yaml` default is obsolete: when no + // installation workspace is pinned, default to the first active registry workspace. + if (!stored.workspace) { + try { + const revisions = await workspaceRegistry.list(); + if (revisions.length > 0) effective.workspace = revisions[0].id; + } catch { + // Registry not bootstrapped yet; keep the legacy fallback. } - return effectiveSettings(config, stored); } - return effectiveSettings(config, loadSettings(config)); - }; - const saveUserSettings = async (principal: PrincipalContext, settings: Settings): Promise => { - const runner = runnerFor(principal); - if (typeof runner.preferencesSet === "function") { - await runner.preferencesSet(settings); - return; - } - saveSettings(config, settings); + return effective; }; + const piManagement = deps?.piManagement ?? createPiManagement(config, { listModels }); - const authenticate = authPreHandler(config.authMode); - app.addHook("preHandler", async (req, reply) => { - // Process readiness is intentionally unauthenticated for local container/proxy probes. - if (req.url === "/health" || req.url === "/health/dwh") return; - return authenticate(req, reply); + const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile); + const localRegistryResolver = deps?.localUserRegistry === undefined + ? createCurrentLocalUserRegistryResolver() + : undefined; + const resolveLocalUserRegistry = (loaded: LoadedAuthConfig) => { + return deps?.localUserRegistry ?? localRegistryResolver?.resolve(loaded); + }; + const localUserForSnapshot = async (loaded: LoadedAuthConfig, subject: string) => { + try { + if (loaded.value.mode !== "local") return { revision: loaded.revision, user: undefined }; + const registry = resolveLocalUserRegistry(loaded); + if (!registry) throw new AuthSessionOperationalError(); + const user = await registry.findBySubject(subject); + return { + revision: loaded.revision, + user: user === undefined ? undefined : { + enabled: user.enabled, + authRevision: user.authRevision, + roles: user.roles, + }, + }; + } catch (error) { + if (error instanceof AuthSessionOperationalError) throw error; + throw new AuthSessionOperationalError(); + } + }; + const sessionValidityForSnapshot = (loaded: LoadedAuthConfig): AuthSessionValidity => ({ + currentAuthConfigRevision: () => loaded.revision, + currentLocalUser: (subject) => localUserForSnapshot(loaded, subject), }); + const resolveOidcProtocol = (loaded: LoadedAuthConfig): OidcProtocol | undefined => { + if (deps?.oidcProtocol) return deps.oidcProtocol; + if (loaded.value.mode !== "oidc") return undefined; + try { + const clientSecret = secretValue(config, loaded.value.oidc.clientSecretRef); + if (!isUsableAuthenticationSecret("THT_OIDC_CLIENT_SECRET", clientSecret)) return undefined; + return (deps?.oidcProtocolFactory ?? createOidcProtocol)({ + issuer: loaded.value.oidc.issuer, + clientId: loaded.value.oidc.clientId, + clientSecret, + callbackUrl: new URL("/api/auth/oidc/callback", loaded.value.publicUrl).href, + scopes: loaded.value.oidc.scopes, + groupsClaim: loaded.value.oidc.groupsClaim, + }); + } catch { + return undefined; + } + }; + const authDiagnoser = deps?.authDiagnoser ?? createConfiguredAuthDiagnoser(config, { + localUserRegistry: resolveLocalUserRegistry, + oidcProtocol: resolveOidcProtocol, + }); + const authSessionStore = deps?.authSessionStore ?? (config.authMode === "local" || config.authMode === "oidc" + ? createFileAuthSessionStore(config.authStateRoot, { + currentAuthConfigRevision: () => { + try { + return config.authentication?.current().revision ?? ""; + } catch { + throw new AuthSessionOperationalError(); + } + }, + currentLocalUser: async (subject) => { + try { + const loaded = config.authentication?.current(); + if (!loaded) return { revision: "", user: undefined }; + return await localUserForSnapshot(loaded, subject); + } catch (error) { + if (error instanceof AuthSessionOperationalError) throw error; + throw new AuthSessionOperationalError(); + } + }, + }, deps?.authStorageBridgeForTest === undefined + ? undefined + : process.platform === "win32" + ? { windowsStorageBridge: deps.authStorageBridgeForTest } + : { posixStorageBridge: deps.authStorageBridgeForTest }) + : undefined); + (app as AppWithAuthSessionStore).thothiiAuthSessionStore = authSessionStore; + const authenticate = authenticateSession({ + mode: config.authMode, + publicExposure: config.publicExposure, + authentication: config.authentication, + sessionStore: authSessionStore, + sessionValidityForSnapshot, + }); + app.addHook("preHandler", (req, reply, done) => { + if (isMaintenanceControl(req.url)) { + if (!isLoopback(req.ip)) { + reply.code(403).send({ error: "loopback maintenance control required" }); + } + } + done(); + }); + app.addHook("preHandler", authenticate); app.get("/health", async () => ({ status: "ok" })); - app.get("/health/dwh", async () => tht.dbPing()); - app.get("/me", async (req) => getPrincipal(req)); - sessionRoutes(app, { - mgr, tht: tht as ThtRunner, hub, getSettings, readiness, listModels, - dwhPrecheck: config.dwhPrecheck, + app.get("/health/dwh", async () => { + // In the registry system there is no single legacy DWH config: ping the first active + // workspace's rendered runtime config. If the registry is not bootstrapped yet, do not + // block the app — per-workspace diagnostics and the session precheck own reachability. + try { + const revisions = await workspaceRegistry.list(); + if (revisions.length > 0) { + return await tht.dbPing(revisions[0].snapshotPath); + } + } catch { + // fall through + } + return { ok: true, detail: "workspace diagnostics own DWH reachability" }; }); - sqlRoutes(app, { tht: tht as ThtRunner, getSettings }); + registerAuthRoutes(app, { + authMode: config.authMode, + authentication: config.authentication, + sessionStore: authSessionStore, + localUserRegistry: deps?.localUserRegistry, + resolveLocalUserRegistry, + resolveOidcProtocol, + }); + sessionRoutes(app, { + mgr, tht: tht as ThtRunner, hub, getSettings, readiness, listModels, workspaceRegistry, + dwhPrecheck: config.dwhPrecheck, + legacyWorkspaceMode: config.legacyWorkspaceMode, + workspaceRuntimeSupport, + maintenanceBarrier, + }); + app.post("/internal/maintenance/activate", async (req, reply) => { + try { + await maintenanceBarrier.activate(); + return maintenanceBarrier.status(); + } catch { + return reply.code(500).send({ + ...maintenanceBarrier.status(), + code: "maintenance_durability_failed", + error: "maintenance activation durability was not acknowledged", + }); + } + }); + app.post("/internal/maintenance/deactivate", async (req, reply) => { + try { + maintenanceBarrier.deactivate(); + return maintenanceBarrier.status(); + } catch { + return reply.code(500).send({ + ...maintenanceBarrier.status(), + code: "maintenance_durability_failed", + error: "maintenance deactivation durability was not acknowledged", + }); + } + }); + app.get("/internal/maintenance/status", async (req, reply) => { + return maintenanceBarrier.status(); + }); + sqlRoutes(app, { tht: tht as ThtRunner, getSettings, workspaceRegistry }); metaRoutes(app, { harnessDir: config.harnessDir, listModels }); - settingsRoutes(app, { cfg: config, listModels, getSettings, saveSettings: saveUserSettings }); + workspaceRoutes(app, { + registry: workspaceRegistry, + config: config.workspaceRegistry, + diagnose: workspaceDiagnoser, + authDiagnoser, + secretStore: workspaceSecretStore, + }); + settingsRoutes(app, { cfg: config, listModels, getSettings }); + piManagementRoutes(app, { service: piManagement }); return app; } + +function corsOrigin(request: FastifyRequest, expectedOrigin: string): string | false { + const supplied = request.headers.origin; + if (typeof supplied !== "string") return false; + try { + return new URL(supplied).origin === expectedOrigin ? expectedOrigin : false; + } catch { + return false; + } +} + +function isLoopback(ip: string): boolean { return ip === "127.0.0.1" || ip === "::1" || ip === "::ffff:127.0.0.1"; } +function isMaintenanceControl(url: string): boolean { + return /^\/internal\/maintenance\/(?:activate|deactivate|status)(?:\?|$)/.test(url); +} diff --git a/backend/src/auth/auth.ts b/backend/src/auth/auth.ts index 8379e8ba..e73008ac 100644 --- a/backend/src/auth/auth.ts +++ b/backend/src/auth/auth.ts @@ -1,17 +1,64 @@ -import type { FastifyRequest, FastifyReply } from "fastify"; +import type { FastifyRequest, FastifyReply, preHandlerHookHandler } from "fastify"; import { localPrincipal, type PrincipalContext, upstreamPrincipal } from "./principal.js"; +import { rolesToPermissions } from "./config.js"; +import type { AuthenticationConfigProvider, AuthMode, AuthSessionRecord, LoadedAuthConfig } from "./types.js"; +import { AuthSessionOperationalError, type AuthSessionStore, type AuthSessionValidity } from "./session-store.js"; +import { deriveCsrfToken, csrfTokensEqual } from "./csrf.js"; +import { requireSameOriginOrNonBrowser } from "./authorization.js"; declare module "fastify" { - interface FastifyRequest { principal?: PrincipalContext } + interface FastifyRequest { + principal?: PrincipalContext; + authSession?: AuthSessionRecord; + /** Internal only: never serialize or write this opaque cookie token to logs. */ + authSessionToken?: string; + authPublicOrigin?: string; + /** One immutable configuration load for the whole request, including CORS. */ + authConfigSnapshot?: LoadedAuthConfig; + authConfigSnapshotCaptured?: boolean; + authConfigSnapshotUnavailable?: boolean; + } } -export function authPreHandler(mode: "none" | "mock" | "upstream") { +const SESSION_COOKIE = "thothii_session"; +const SESSION_TOKEN = /^[A-Za-z0-9_-]{43}$/; +const STATE_CHANGING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); + +export interface AuthDependencies { + mode: AuthMode; + publicExposure?: boolean; + authentication?: AuthenticationConfigProvider; + sessionStore?: AuthSessionStore; + sessionValidityForSnapshot?: (snapshot: LoadedAuthConfig) => AuthSessionValidity; +} + +/** Capture the authentication configuration once; CORS calls this before every other hook. */ +export function captureAuthConfigSnapshot( + request: FastifyRequest, + authentication: AuthenticationConfigProvider | undefined, +): LoadedAuthConfig | undefined { + if (request.authConfigSnapshotCaptured) return request.authConfigSnapshot; + request.authConfigSnapshotCaptured = true; + try { + request.authConfigSnapshot = authentication?.current(); + } catch { + request.authConfigSnapshotUnavailable = true; + } + return request.authConfigSnapshot; +} + +export function authPreHandler(mode: "none" | "mock" | "upstream", publicExposure = false) { return async (req: FastifyRequest, reply: FastifyReply) => { if (mode === "none") { - req.principal = localPrincipal(); + req.principal = localPrincipal(publicExposure); } else if (mode === "mock") { const subject = typeof req.headers["x-mock-user"] === "string" ? req.headers["x-mock-user"].trim() : "mock"; - req.principal = { issuer: "mock", subject: subject || "mock", displayName: subject || "mock", isAdmin: false }; + const elevated = req.headers["x-thoth-is-admin"] === "1" || req.headers["x-thoth-is-admin"] === "true"; + const roles = elevated ? ["admin"] as const : ["user"] as const; + req.principal = { + issuer: "mock", subject: subject || "mock", displayName: subject || "mock", roles, + permissions: rolesToPermissions(roles), isAdmin: elevated, + }; } else { const principal = upstreamPrincipal(req.headers); if (!principal) { @@ -22,6 +69,157 @@ export function authPreHandler(mode: "none" | "mock" | "upstream") { }; } +/** + * The one application boundary for principal resolution. Auth protocol endpoints are the only + * public exceptions; all other routes get either a resolved principal or a sanitized denial. + */ +export function authenticateSession(deps: AuthDependencies): preHandlerHookHandler { + const legacy = deps.mode === "none" || deps.mode === "mock" || deps.mode === "upstream" + ? authPreHandler(deps.mode, deps.publicExposure) + : undefined; + + const handle = async (request: FastifyRequest, reply: FastifyReply): Promise => { + const snapshot = captureAuthConfigSnapshot(request, deps.authentication); + if (isPublicRoute(request)) return; + + if (legacy) { + await legacy(request, reply); + if (reply.sent || !STATE_CHANGING_METHODS.has(request.method)) return; + return requireSameOriginOrNonBrowser(request, reply); + } + + const origin = configuredOrigin(snapshot); + if (!snapshot || !origin || !deps.sessionStore) { + return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" }); + } + const token = readSessionCookie(request); + if (token === undefined || token === false) return authenticationRequired(reply); + + let session: AuthSessionRecord | undefined; + try { + session = await deps.sessionStore.resolve(token, undefined, deps.sessionValidityForSnapshot?.(snapshot)); + if (session && session.authConfigRevision !== snapshot.revision) { + try { await deps.sessionStore.revoke(token); } catch { /* the mismatch remains denied */ } + return authenticationRequired(reply); + } + if (session) await deps.sessionStore.touch(token); + } catch (error) { + if (error instanceof AuthSessionOperationalError) { + return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" }); + } + return authenticationRequired(reply); + } + if (!session) return authenticationRequired(reply); + + request.authSession = session; + request.authSessionToken = token; + request.authPublicOrigin = origin; + request.principal = { + issuer: session.issuer, + subject: session.subject, + ...(session.displayName === undefined ? {} : { displayName: session.displayName }), + roles: session.roles, + permissions: session.permissions, + isAdmin: session.roles.includes("admin"), + }; + if (STATE_CHANGING_METHODS.has(request.method)) { + requireCsrf(request, reply); + return; + } + }; + return (request, reply, done) => { + void handle(request, reply).then( + () => done(), + () => { + if (!reply.sent) reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" }); + done(); + }, + ); + }; +} + +export function requireCsrf(request: FastifyRequest, reply: FastifyReply): true | FastifyReply { + const expectedOrigin = request.authPublicOrigin; + const token = request.authSessionToken; + if (!expectedOrigin || !token) return authenticationRequired(reply); + if (!matchesOrigin(request, expectedOrigin)) return csrfFailed(reply); + + const header = singleHeader(request.headers["x-thothii-csrf"]); + const supplied = header === false || header === undefined || !SESSION_TOKEN.test(header) ? undefined : header; + let expected = ""; + try { + expected = deriveCsrfToken(token); + } catch { + return authenticationRequired(reply); + } + if (!csrfTokensEqual(expected, supplied)) return csrfFailed(reply); + return true; +} + +/** Require an exact configured public origin and browser Fetch Metadata when supplied. */ +export function requireExactOrigin( + request: FastifyRequest, + reply: FastifyReply, + expectedOrigin: string, +): true | FastifyReply { + return matchesOrigin(request, expectedOrigin) ? true : csrfFailed(reply); +} + +export function sessionCookieName(): string { return SESSION_COOKIE; } + +function authenticationRequired(reply: FastifyReply): FastifyReply { + return reply.code(401).send({ code: "authentication_required", error: "Authentication is required" }); +} + +function csrfFailed(reply: FastifyReply): FastifyReply { + return reply.code(403).send({ code: "csrf_failed", error: "Request origin validation failed" }); +} + +export function configuredOrigin(snapshot: LoadedAuthConfig | undefined): string | undefined { + try { + const publicUrl = snapshot?.value.publicUrl; + return publicUrl ? new URL(publicUrl).origin : undefined; + } catch { + return undefined; + } +} + +function readSessionCookie(request: FastifyRequest): string | false | undefined { + const raw = request.headers.cookie; + if (raw === undefined) return undefined; + if (Array.isArray(raw) || typeof raw !== "string" || raw.length > 4096) return false; + const values = raw.split(";").filter((part) => /^\s*thothii_session(?:=|\s*$)/.test(part)); + if (values.length !== 1) return values.length === 0 ? undefined : false; + const match = /^\s*thothii_session=([A-Za-z0-9_-]{43})\s*$/.exec(values[0]); + return match?.[1] ?? false; +} + +function singleHeader(value: string | string[] | undefined): string | false | undefined { + if (value === undefined) return undefined; + if (Array.isArray(value) || typeof value !== "string" || value.includes(",")) return false; + return value; +} + +function matchesOrigin(request: FastifyRequest, expectedOrigin: string): boolean { + const origin = singleHeader(request.headers.origin); + try { + if (origin === undefined || origin === false || new URL(origin).origin !== expectedOrigin) return false; + } catch { + return false; + } + const fetchSite = singleHeader(request.headers["sec-fetch-site"]); + return fetchSite === undefined || fetchSite === "same-origin"; +} + +function isPublicRoute(request: FastifyRequest): boolean { + const rawUrl = request.raw.url ?? request.url; + const query = rawUrl.indexOf("?"); + const pathname = query === -1 ? rawUrl : rawUrl.slice(0, query); + return (request.method === "GET" && (pathname === "/health" || pathname === "/auth/config" + || pathname === "/auth/oidc/login" || pathname === "/auth/oidc/callback")) + || (request.method === "POST" && pathname === "/auth/local/login"); +} + export function getPrincipal(req: FastifyRequest): PrincipalContext { if (!req.principal) throw new Error("principal missing after authentication"); return req.principal; diff --git a/backend/src/auth/authentik-group-catalog.ts b/backend/src/auth/authentik-group-catalog.ts new file mode 100644 index 00000000..6d9f5180 --- /dev/null +++ b/backend/src/auth/authentik-group-catalog.ts @@ -0,0 +1,236 @@ +import type { AuthDiagnostic, GroupCatalog } from "./group-catalog.js"; +import { isUsableAuthenticationSecret } from "./secret-policy.js"; +import { parseConfiguredTransportUrl } from "./url-policy.js"; + +const MAX_RESPONSE_BYTES = 1024 * 1024; +const REQUEST_TIMEOUT_MS = 5_000; + +export interface AuthentikGroupCatalogOptions { + baseUrl: string; + apiToken: string; + fetch?: typeof globalThis.fetch; +} + +function diagnostic( + code: AuthDiagnostic["code"], + message: string, + field?: string, +): AuthDiagnostic { + return { level: "error", code, message, ...(field === undefined ? {} : { field }) }; +} + +function catalogUnreachable(): AuthDiagnostic { + return diagnostic("oidc_group_catalog_unreachable", "The configured group catalog is unavailable."); +} + +function catalogUnauthorized(): AuthDiagnostic { + return diagnostic("oidc_group_catalog_unauthorized", "The configured group catalog credentials were rejected."); +} + +function missing(name: string): AuthDiagnostic { + return diagnostic("oidc_mapped_group_missing", "A configured authorization group does not exist.", name); +} + +function ambiguous(name: string): AuthDiagnostic { + return diagnostic("oidc_mapped_group_ambiguous", "A configured authorization group is ambiguous.", name); +} + +function stableCompare(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("The operation was aborted", "AbortError"); +} + +function cancelResponse(response: Response): void { + try { + const cancelled = response.body?.cancel(); + if (cancelled) void cancelled.catch(() => undefined); + } catch { /* cancellation is advisory and never changes the diagnostic */ } +} + +function cancelReader(reader: ReadableStreamDefaultReader): void { + try { + const cancelled = reader.cancel(); + void cancelled.catch(() => undefined); + } catch { /* cancellation is advisory and never changes the diagnostic */ } +} + +function awaitWithAbort( + operation: Promise, + signal: AbortSignal, + onLateResolution?: (value: T) => void, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const abort = () => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", abort); + reject(abortReason(signal)); + }; + if (signal.aborted) { + abort(); + return; + } + signal.addEventListener("abort", abort, { once: true }); + operation.then( + (value) => { + if (settled) { + try { onLateResolution?.(value); } catch { /* best-effort cleanup only */ } + return; + } + settled = true; + signal.removeEventListener("abort", abort); + resolve(value); + }, + (error: unknown) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", abort); + reject(error); + }, + ); + }); +} + +function validContentLength(response: Response): boolean { + const value = response.headers.get("content-length"); + if (value === null) return true; + if (!/^\d+$/.test(value)) return false; + const length = Number(value); + return Number.isSafeInteger(length) && length <= MAX_RESPONSE_BYTES; +} + +async function readBounded(response: Response, signal: AbortSignal): Promise { + if (!validContentLength(response)) { + cancelResponse(response); + return undefined; + } + const reader = response.body?.getReader(); + if (!reader) return new Uint8Array(); + const chunks: Uint8Array[] = []; + let size = 0; + let complete = false; + try { + while (true) { + const { done, value } = await awaitWithAbort(reader.read(), signal); + if (done) break; + if (value.byteLength > MAX_RESPONSE_BYTES - size) return undefined; + chunks.push(value); + size += value.byteLength; + } + complete = true; + const body = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return body; + } finally { + if (!complete) cancelReader(reader); + try { reader.releaseLock(); } catch { /* reader may already be unusable */ } + } +} + +type GroupResult = "present" | "missing" | "ambiguous" | "unauthorized" | "unreachable"; + +function exactResult(name: string, parsed: unknown): GroupResult { + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return "unreachable"; + const record = parsed as { results?: unknown; pagination?: unknown }; + if (!Array.isArray(record.results) || record.results.length > 2 + || !record.pagination || typeof record.pagination !== "object" + || Array.isArray(record.pagination)) return "unreachable"; + if (!Object.prototype.hasOwnProperty.call(record.pagination, "next")) return "unreachable"; + const next = (record.pagination as { next: unknown }).next; + if (next !== null) { + if (typeof next !== "string" || next.length === 0 || next.length > 2048 || /\p{Cc}/u.test(next)) return "unreachable"; + try { + const continuation = new URL(next); + if (continuation.protocol !== "https:" || continuation.username || continuation.password || continuation.hash) return "unreachable"; + } catch { + return "unreachable"; + } + return "ambiguous"; + } + const resultNames: string[] = []; + for (const result of record.results) { + if (!result || typeof result !== "object" || Array.isArray(result) + || typeof (result as { name?: unknown }).name !== "string") return "unreachable"; + resultNames.push((result as { name: string }).name); + } + const exactMatches = resultNames.filter((candidate) => candidate === name).length; + if (exactMatches === 0) return "missing"; + return exactMatches === 1 ? "present" : "ambiguous"; +} + +export function createAuthentikGroupCatalog(options: AuthentikGroupCatalogOptions): GroupCatalog { + const origin = parseConfiguredTransportUrl(options.baseUrl, { allowLoopbackHttp: false, originOnly: true }); + const fetchImplementation = options.fetch ?? globalThis.fetch; + const valid = origin !== undefined + && isUsableAuthenticationSecret("THT_AUTHENTIK_API_TOKEN", options.apiToken) + && typeof fetchImplementation === "function"; + + async function verify(name: string, signal: AbortSignal): Promise { + if (!origin || !valid || signal.aborted) return "unreachable"; + const target = new URL("/api/v3/core/groups/", origin); + target.searchParams.set("name", name); + target.searchParams.set("include_users", "false"); + target.searchParams.set("page_size", "2"); + const timeout = new AbortController(); + const timer = setTimeout(() => timeout.abort(), REQUEST_TIMEOUT_MS); + timer.unref(); + const requestSignal = AbortSignal.any([signal, timeout.signal]); + try { + const response = await awaitWithAbort( + Promise.resolve().then(() => fetchImplementation(target, { + headers: { accept: "application/json", authorization: `Bearer ${options.apiToken}` }, + redirect: "error", + signal: requestSignal, + })), + requestSignal, + cancelResponse, + ); + if (response.redirected || response.type === "opaqueredirect" || response.status >= 300 && response.status < 400) { + cancelResponse(response); + return "unreachable"; + } + if (response.status === 401 || response.status === 403) { + cancelResponse(response); + return "unauthorized"; + } + if (!response.ok) { + cancelResponse(response); + return "unreachable"; + } + const body = await readBounded(response, requestSignal); + if (body === undefined) return "unreachable"; + try { + return exactResult(name, JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(body))); + } catch { + return "unreachable"; + } + } catch { + return "unreachable"; + } finally { + clearTimeout(timer); + } + } + + return { + async verifyConfiguredGroups(names, signal) { + const diagnostics: AuthDiagnostic[] = []; + for (const name of [...new Set(names)].sort(stableCompare)) { + const outcome = await verify(name, signal); + if (outcome === "present") continue; + if (outcome === "missing") diagnostics.push(missing(name)); + else if (outcome === "ambiguous") diagnostics.push(ambiguous(name)); + else if (outcome === "unauthorized") return [catalogUnauthorized()]; + else return [catalogUnreachable()]; + } + return diagnostics; + }, + }; +} diff --git a/backend/src/auth/authorization.ts b/backend/src/auth/authorization.ts new file mode 100644 index 00000000..0fbf152e --- /dev/null +++ b/backend/src/auth/authorization.ts @@ -0,0 +1,54 @@ +import type { FastifyReply, FastifyRequest } from "fastify"; +import type { Permission } from "./types.js"; +import { getPrincipal } from "./auth.js"; +import type { PrincipalContext } from "./principal.js"; + +export function hasPermission(principal: PrincipalContext, permission: Permission): boolean { + return principal.permissions.includes(permission); +} + +export function isPrincipalContext( + value: PrincipalContext | FastifyReply, +): value is PrincipalContext { + return "issuer" in value; +} + +export function requirePermission( + request: FastifyRequest, + reply: FastifyReply, + permission: Permission, +): PrincipalContext | FastifyReply { + const principal = getPrincipal(request); + if (hasPermission(principal, permission)) return principal; + return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" }); +} + +/** + * A resolved cookie session is populated only by the central auth boundary, after its + * request-snapshot Origin and CSRF checks. Route-specific legacy guards must not reinterpret + * the internal transport host/protocol for that already-authorized browser request. + */ +export function hasCookieBackedAuthSession(request: FastifyRequest): boolean { + return request.authSession !== undefined; +} + +/** Permit non-browser clients and browsers whose declared origin matches the request host. */ +export function requireSameOriginOrNonBrowser( + request: FastifyRequest, + reply: FastifyReply, +): FastifyReply | undefined { + if (hasCookieBackedAuthSession(request)) return undefined; + const origin = request.headers.origin; + if (origin === undefined) return undefined; + if (typeof origin !== "string" || typeof request.headers.host !== "string") { + return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" }); + } + try { + const supplied = new URL(origin); + const expected = new URL(`${request.protocol}://${request.headers.host}`); + if (supplied.origin === expected.origin) return undefined; + } catch { + // Invalid browser origins are forbidden below. + } + return reply.code(403).send({ code: "auth_forbidden", error: "This operation is not permitted" }); +} diff --git a/backend/src/auth/config.ts b/backend/src/auth/config.ts new file mode 100644 index 00000000..f3c97aa6 --- /dev/null +++ b/backend/src/auth/config.ts @@ -0,0 +1,320 @@ +import { createHash } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, + realpathSync, +} from "node:fs"; +import type { Stats } from "node:fs"; +import { dirname, isAbsolute, normalize } from "node:path"; +import { parseDocument } from "yaml"; +import { z } from "zod"; +import type { + AuthenticationConfig, + AuthenticationConfigProvider, + AuthMode, + LoadedAuthConfig, + Permission, + Role, +} from "./types.js"; +import { parseConfiguredTransportUrl } from "./url-policy.js"; +import { createWindowsAuthStorageBridge, type WindowsAuthStorageBridge } from "./windows-auth-storage.js"; + +export type { + AuthenticationConfig, + AuthenticationConfigProvider, + AuthMode, + LoadedAuthConfig, + Permission, + Role, +} from "./types.js"; + +const MAX_AUTH_CONFIG_BYTES = 1024 * 1024; +// Keep live catalog work within the same deterministic bound as the mandatory direct groups claim. +const MAX_MAPPED_GROUPS = 128; +const ROLES = ["user", "admin"] as const; +export const PERMISSION_CATALOG: readonly Permission[] = [ + "session.use", "session.read_all", "session.manage_all", "settings.manage", + "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", +]; + +const invalid = (): Error => new Error("authentication configuration is invalid"); +const nonEmptyText = z.string().min(1).max(512).refine( + (value) => value.trim() === value && !/[\u0000-\u001f\u007f]/.test(value), +); +const positiveSeconds = z.number().int().min(1).max(365 * 24 * 60 * 60); +const sessionSchema = z.strictObject({ + regularTtlSeconds: positiveSeconds.default(43_200), + regularIdleSeconds: positiveSeconds.default(7_200), + rememberTtlSeconds: positiveSeconds.default(2_592_000), + rememberIdleSeconds: positiveSeconds.default(604_800), + oidcTtlSeconds: positiveSeconds.default(28_800), +}); +const roleSchema = z.enum(ROLES); +const groupNameSchema = nonEmptyText.max(256); +const groupRolesSchema = z.record(groupNameSchema, z.array(roleSchema).min(1)) + .refine((value) => Object.keys(value).length <= MAX_MAPPED_GROUPS); + +const localSchema = z.strictObject({ + version: z.literal(1), mode: z.literal("local"), publicUrl: nonEmptyText, session: sessionSchema.optional(), + local: z.strictObject({ usersFile: nonEmptyText.max(255) }), +}); +const oidcSchema = z.strictObject({ + version: z.literal(1), mode: z.literal("oidc"), publicUrl: nonEmptyText, session: sessionSchema.optional(), + oidc: z.strictObject({ + issuer: nonEmptyText, clientId: nonEmptyText, clientSecretRef: z.literal("THT_OIDC_CLIENT_SECRET"), + scopes: z.array(nonEmptyText).min(1).max(16), groupsClaim: z.literal("groups"), + }), + groupCatalog: z.strictObject({ + driver: z.literal("authentik"), baseUrl: nonEmptyText, apiTokenRef: z.literal("THT_AUTHENTIK_API_TOKEN"), + }), + authorization: z.strictObject({ groupRoles: groupRolesSchema }), +}); + +interface FileIdentity { + dev: number; + ino: number; + uid: number; + size: number; + mtimeMs: number; + ctimeMs: number; + mode: number; + nlink: number; +} + +interface DirectoryIdentity { + dev: number; + ino: number; + uid: number; + mode: number; + ctimeMs: number; +} + +interface StorageIdentity { + file: FileIdentity; + directory: DirectoryIdentity; +} + +export interface AuthenticationConfigLoadOptions { + /** Test seam; production creates the existing bounded internal tht auth-storage bridge. */ + windowsStorageBridge?: Pick; +} + +function validateCanonicalPath(path: string): void { + if (typeof path !== "string" || path.length === 0 || path.trim() !== path + || path.includes("\0") || !isAbsolute(path) || normalize(path) !== path + || realpathSync(path) !== path || realpathSync(dirname(path)) !== dirname(path)) throw invalid(); +} + +function runtimeOwner(): number { + if (process.platform === "win32" || typeof process.geteuid !== "function") throw invalid(); + const owner = process.geteuid(); + if (!Number.isSafeInteger(owner) || owner < 0) throw invalid(); + return owner; +} + +function fileMetadata(info: Stats): FileIdentity { + const mode = info.mode & 0o7777; + if (!info.isFile() || info.uid !== runtimeOwner() || info.nlink !== 1 || mode !== 0o600 + || info.size < 0 || info.size > MAX_AUTH_CONFIG_BYTES) throw invalid(); + return { + dev: info.dev, ino: info.ino, uid: info.uid, size: info.size, + mtimeMs: info.mtimeMs, ctimeMs: info.ctimeMs, mode, nlink: info.nlink, + }; +} + +function directoryMetadata(info: Stats): DirectoryIdentity { + const mode = info.mode & 0o7777; + if (!info.isDirectory() || info.uid !== runtimeOwner() || mode !== 0o700) throw invalid(); + return { dev: info.dev, ino: info.ino, uid: info.uid, mode, ctimeMs: info.ctimeMs }; +} + +function sameFileIdentity(left: FileIdentity, right: FileIdentity): boolean { + return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid + && left.size === right.size && left.mtimeMs === right.mtimeMs && left.ctimeMs === right.ctimeMs + && left.mode === right.mode && left.nlink === right.nlink; +} + +function sameDirectoryIdentity(left: DirectoryIdentity, right: DirectoryIdentity): boolean { + return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid + && left.mode === right.mode && left.ctimeMs === right.ctimeMs; +} + +function sameIdentity(left: StorageIdentity, right: StorageIdentity): boolean { + return sameFileIdentity(left.file, right.file) && sameDirectoryIdentity(left.directory, right.directory); +} + +function storageIdentity(path: string): StorageIdentity { + try { + validateCanonicalPath(path); + return { + file: fileMetadata(lstatSync(path) as Stats), + directory: directoryMetadata(lstatSync(dirname(path)) as Stats), + }; + } catch { + throw invalid(); + } +} + +function openDirectoryDescriptor(path: string): number { + return openSync(path, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0) + | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0)); +} + +function readBoundedConfig(path: string): { source: string; identity: StorageIdentity } { + let directoryDescriptor: number | undefined; + let fd: number | undefined; + try { + const before = storageIdentity(path); + directoryDescriptor = openDirectoryDescriptor(dirname(path)); + const openedDirectory = directoryMetadata(fstatSync(directoryDescriptor) as Stats); + if (!sameDirectoryIdentity(before.directory, openedDirectory)) throw invalid(); + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const opened = fileMetadata(fstatSync(fd) as Stats); + if (!sameFileIdentity(before.file, opened)) throw invalid(); + const buffer = Buffer.allocUnsafe(MAX_AUTH_CONFIG_BYTES + 1); + let offset = 0; + while (offset < buffer.length) { + const bytesRead = readSync(fd, buffer, offset, buffer.length - offset, null); + if (bytesRead === 0) break; + offset += bytesRead; + } + if (offset > MAX_AUTH_CONFIG_BYTES) throw invalid(); + const afterFile = fileMetadata(fstatSync(fd) as Stats); + const afterPath = storageIdentity(path); + const afterOpenedDirectory = directoryMetadata(fstatSync(directoryDescriptor) as Stats); + if (!sameFileIdentity(opened, afterFile) || !sameFileIdentity(afterFile, afterPath.file) + || !sameDirectoryIdentity(before.directory, afterPath.directory) + || !sameDirectoryIdentity(openedDirectory, afterOpenedDirectory)) throw invalid(); + validateCanonicalPath(path); + return { + source: new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, offset)), + identity: afterPath, + }; + } catch { + throw invalid(); + } finally { + if (fd !== undefined) try { closeSync(fd); } catch { /* sanitized by design */ } + if (directoryDescriptor !== undefined) try { closeSync(directoryDescriptor); } catch { /* sanitized by design */ } + } +} + +function validOrigin(value: string, httpLoopbackAllowed: boolean): boolean { + return parseConfiguredTransportUrl(value, { allowLoopbackHttp: httpLoopbackAllowed, originOnly: true }) !== undefined; +} + +function validIssuer(value: string): boolean { + return parseConfiguredTransportUrl(value, { allowLoopbackHttp: false }) !== undefined; +} + +function validUsersFile(value: string): boolean { + return /^[A-Za-z0-9][A-Za-z0-9._-]*\.yaml$/.test(value); +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value as Record) + .sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0) + .map(([key, nested]) => [key, canonicalize(nested)])); + } + return value; +} + +function canonicalRevision(value: AuthenticationConfig): string { + return createHash("sha256").update(JSON.stringify(canonicalize(value))).digest("hex"); +} + +function parseAuthenticationConfig(source: string): AuthenticationConfig { + try { + const document = parseDocument(source, { uniqueKeys: true }); + if (document.errors.length > 0 || document.warnings.length > 0) throw invalid(); + const parsed = document.toJSON(); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw invalid(); + const config = parsed as Record; + const schema = config.mode === "local" ? localSchema : config.mode === "oidc" ? oidcSchema : undefined; + if (!schema) throw invalid(); + const validated = schema.parse(config); + const session = sessionSchema.parse(validated.session ?? {}); + if (!validOrigin(validated.publicUrl, true)) throw invalid(); + if (validated.mode === "local") { + if (!validUsersFile(validated.local.usersFile)) throw invalid(); + return { ...validated, session }; + } + if (!validIssuer(validated.oidc.issuer) || !validOrigin(validated.groupCatalog.baseUrl, false)) throw invalid(); + if (!validated.oidc.scopes.includes("openid")) throw invalid(); + const mappings = Object.entries(validated.authorization.groupRoles); + if (mappings.length === 0 || mappings.filter(([, roles]) => roles.includes("admin")).length !== 1) throw invalid(); + return { ...validated, session }; + } catch { throw invalid(); } +} + +function loadAuthenticationConfigWithIdentity(path: string): { loaded: LoadedAuthConfig; identity: StorageIdentity } { + const read = readBoundedConfig(path); + const value = parseAuthenticationConfig(read.source); + return { loaded: { value, revision: canonicalRevision(value), sourcePath: path }, identity: read.identity }; +} + +function loadWindowsAuthenticationConfig( + path: string, + bridge: Pick, +): LoadedAuthConfig { + try { + const contents = bridge.readAuthConfig(path); + if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > MAX_AUTH_CONFIG_BYTES) throw invalid(); + const source = new TextDecoder("utf-8", { fatal: true }).decode(contents); + const value = parseAuthenticationConfig(source); + return { value, revision: canonicalRevision(value), sourcePath: path }; + } catch { + throw invalid(); + } +} + +export function loadAuthenticationConfig(path: string, options: AuthenticationConfigLoadOptions = {}): LoadedAuthConfig { + if (process.platform === "win32") { + return loadWindowsAuthenticationConfig(path, options.windowsStorageBridge ?? createWindowsAuthStorageBridge()); + } + return loadAuthenticationConfigWithIdentity(path).loaded; +} + +export function createAuthenticationConfigProvider( + path: string, + options: AuthenticationConfigLoadOptions = {}, +): AuthenticationConfigProvider { + if (process.platform === "win32") { + const bridge = options.windowsStorageBridge ?? createWindowsAuthStorageBridge(); + return { current: () => loadWindowsAuthenticationConfig(path, bridge) }; + } + let cached: { identity: StorageIdentity; loaded: LoadedAuthConfig } | undefined; + return { current(): LoadedAuthConfig { + const before = storageIdentity(path); + if (cached && sameIdentity(cached.identity, before)) return cached.loaded; + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + const { loaded, identity } = loadAuthenticationConfigWithIdentity(path); + if (sameIdentity(identity, storageIdentity(path))) { + cached = { identity, loaded }; + return loaded; + } + } catch { /* retry one concurrent atomic replacement, then fail closed */ } + } + throw invalid(); + } }; +} + +export function rolesToPermissions(roles: readonly Role[]): readonly Permission[] { + const requested = new Set(); + for (const role of roles) { + if (!ROLES.includes(role)) throw invalid(); + requested.add(role); + } + if (requested.has("admin")) return PERMISSION_CATALOG; + return requested.has("user") ? ["session.use"] : []; +} + +export function isPermission(value: string): value is Permission { + return PERMISSION_CATALOG.includes(value as Permission); +} diff --git a/backend/src/auth/csrf.ts b/backend/src/auth/csrf.ts new file mode 100644 index 00000000..bcb4c062 --- /dev/null +++ b/backend/src/auth/csrf.ts @@ -0,0 +1,13 @@ +import { timingSafeEqual } from "node:crypto"; +import { deriveCsrfToken as deriveStoredCsrfToken } from "./session-store.js"; + +export { deriveStoredCsrfToken as deriveCsrfToken }; + +/** Compare a client-supplied CSRF value without exposing a useful length timing oracle. */ +export function csrfTokensEqual(expectedToken: string, suppliedToken: string | undefined): boolean { + const expected = Buffer.from(expectedToken, "utf8"); + const supplied = Buffer.from(suppliedToken ?? "", "utf8"); + const padded = Buffer.alloc(expected.length); + supplied.copy(padded, 0, 0, expected.length); + return timingSafeEqual(expected, padded) && supplied.length === expected.length; +} diff --git a/backend/src/auth/diagnostic-command.ts b/backend/src/auth/diagnostic-command.ts new file mode 100644 index 00000000..47f9f4ee --- /dev/null +++ b/backend/src/auth/diagnostic-command.ts @@ -0,0 +1,338 @@ +import { fileURLToPath } from "node:url"; +import { resolve } from "node:path"; +import { + closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, +} from "node:fs"; +import { loadConfig, type AppConfig } from "../config.js"; +import { loadSecretBundle, secretValue } from "../config/secret-bundle.js"; +import { createAuthentikGroupCatalog } from "./authentik-group-catalog.js"; +import { createCurrentLocalUserRegistryResolver, type LocalUserRegistry } from "./local-registry.js"; +import { createOidcProtocol, OidcDeviceFlowUnavailableError, type OidcProtocol } from "./oidc-client.js"; +import { createAuthDiagnoser, type AuthDiagnoser, type AuthDiagnostic, type AuthDiagnostics } from "./diagnostics.js"; +import { decodeAuthDiagnostics, type GroupCatalog } from "./group-catalog.js"; +import type { LoadedAuthConfig } from "./types.js"; + +const AUTH_SECRET_REFERENCES = ["THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN"] as const; +const MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES = 64 * 1024; +const MAX_DIAGNOSTIC_SECRET_VALUES = 4096; +const MAX_DIAGNOSTIC_SECRET_DEPTH = 32; + +function unavailableSecretCorpus(): Error { + return new Error("diagnostic secret corpus is unavailable"); +} + +function readMountedSecretSource(file: string): string { + let fd: number | undefined; + try { + if (!file || file.trim() !== file || file.includes("\0")) throw unavailableSecretCorpus(); + const before = lstatSync(file); + if (!before.isFile() || before.isSymbolicLink() || before.size > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES) { + throw unavailableSecretCorpus(); + } + fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const opened = fstatSync(fd); + if (!opened.isFile() || opened.size > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES + || before.dev !== opened.dev || before.ino !== opened.ino) { + throw unavailableSecretCorpus(); + } + const value = readFileSync(fd, "utf8"); + if (Buffer.byteLength(value, "utf8") > MAX_DIAGNOSTIC_SECRET_SOURCE_BYTES) { + throw unavailableSecretCorpus(); + } + return value; + } catch { + throw unavailableSecretCorpus(); + } finally { + if (fd !== undefined) try { closeSync(fd); } catch { /* fixed failure surface above */ } + } +} + +function parsedSecretValues(raw: string, requireJson: boolean): readonly string[] { + const trimmed = raw.trim(); + if (!trimmed) return []; + const values = new Set([raw.replace(/[\r\n]+$/u, "")]); + const looksJson = trimmed.startsWith("{") || trimmed.startsWith("["); + if (!looksJson) { + if (requireJson) throw unavailableSecretCorpus(); + return [...values]; + } + let document: unknown; + try { document = JSON.parse(trimmed); } catch { throw unavailableSecretCorpus(); } + if (requireJson && (!document || typeof document !== "object" || Array.isArray(document))) { + throw unavailableSecretCorpus(); + } + const pending: Array<{ value: unknown; depth: number }> = [{ value: document, depth: 0 }]; + let scalarCount = 0; + while (pending.length > 0) { + const current = pending.pop()!; + if (current.depth > MAX_DIAGNOSTIC_SECRET_DEPTH) throw unavailableSecretCorpus(); + if (Array.isArray(current.value)) { + for (const item of current.value) pending.push({ value: item, depth: current.depth + 1 }); + } else if (current.value && typeof current.value === "object") { + for (const item of Object.values(current.value as Record)) { + pending.push({ value: item, depth: current.depth + 1 }); + } + } else { + scalarCount += 1; + if (scalarCount > 1024) throw unavailableSecretCorpus(); + if (typeof current.value === "string" && current.value.length > 0) values.add(current.value); + } + } + return [...values]; +} + +export function configuredSecretValues(config: AppConfig): readonly string[] { + try { + const values = new Set(); + if (config.secretsFile) { + for (const value of loadSecretBundle(config.secretsFile).values()) values.add(value); + } + const legacyFiles = new Set(Object.values(config.secretFiles).filter( + (file): file is string => file !== undefined, + )); + for (const file of legacyFiles) { + for (const value of parsedSecretValues(readMountedSecretSource(file), false)) values.add(value); + } + if (config.piAuthFile) { + for (const value of parsedSecretValues(readMountedSecretSource(config.piAuthFile), true)) values.add(value); + } + if (values.size > MAX_DIAGNOSTIC_SECRET_VALUES) throw unavailableSecretCorpus(); + return [...values]; + } catch { + throw unavailableSecretCorpus(); + } +} + +export interface ConfiguredAuthDiagnoserOptions { + localUserRegistry?: (loaded: LoadedAuthConfig) => LocalUserRegistry | undefined; + oidcProtocol?: (loaded: LoadedAuthConfig) => OidcProtocol | undefined; + groupCatalog?: (loaded: LoadedAuthConfig) => GroupCatalog | undefined; + sessionRootValidator?: (root: string) => void | Promise; +} + +/** Builds the one shared auth diagnostic implementation used by app routes and the one-shot CLI. */ +export function createConfiguredAuthDiagnoser( + config: AppConfig, + options: ConfiguredAuthDiagnoserOptions = {}, +): AuthDiagnoser { + const localResolver = options.localUserRegistry === undefined + ? createCurrentLocalUserRegistryResolver() + : undefined; + const secretValues = (): ReadonlyMap => { + const values = new Map(); + for (const reference of AUTH_SECRET_REFERENCES) { + try { + const value = secretValue(config, reference); + if (value !== undefined) values.set(reference, value); + } catch { + // The shared diagnoser emits the fixed missing-secret diagnostic below. + } + } + return values; + }; + const loaded = (): LoadedAuthConfig | undefined => { + try { return config.authentication?.current(); } catch { return undefined; } + }; + return { + async inspect(request): Promise { + const current = loaded(); + const protocol = current?.value.mode === "oidc" + ? options.oidcProtocol?.(current) ?? (() => { + try { + const clientSecret = secretValues().get("THT_OIDC_CLIENT_SECRET"); + if (!clientSecret) return undefined; + return createOidcProtocol({ + issuer: current.value.oidc.issuer, + clientId: current.value.oidc.clientId, + clientSecret, + callbackUrl: new URL("/api/auth/oidc/callback", current.value.publicUrl).href, + scopes: current.value.oidc.scopes, + groupsClaim: current.value.oidc.groupsClaim, + }); + } catch { return undefined; } + })() + : undefined; + const groupCatalog = current?.value.mode === "oidc" ? options.groupCatalog?.(current) ?? (() => { + try { + const token = secretValues().get("THT_AUTHENTIK_API_TOKEN"); + return token === undefined ? undefined : createAuthentikGroupCatalog({ + baseUrl: current.value.groupCatalog.baseUrl, + apiToken: token, + }); + } catch { return undefined; } + })() : undefined; + const report = await createAuthDiagnoser({ + authMode: config.authMode, + authStateRoot: config.authStateRoot, + ...(options.sessionRootValidator === undefined ? {} : { sessionRootValidator: options.sessionRootValidator }), + authentication: config.authentication, + secrets: secretValues(), + localUserRegistry: current?.value.mode === "local" + ? options.localUserRegistry?.(current) ?? localResolver?.resolve(current) + : undefined, + oidcProtocol: protocol, + groupCatalog, + }).inspect(request); + if (!request.interactive || !report.ready) return report; + if (current?.value.mode !== "oidc" || !protocol?.verifyDeviceFlow || !request.presentDeviceCode) { + return { + ready: false, + mode: report.mode, + checks: [{ + level: "error", + code: "oidc_device_flow_unavailable", + message: "Interactive authentication diagnostics require OIDC device authorization.", + }], + }; + } + try { + const identity = await protocol.verifyDeviceFlow( + request.signal ?? AbortSignal.timeout(10 * 60_000), request.presentDeviceCode, + ); + // Exact names only: unrelated provider groups are neither emitted nor retained. + const mappedRoles = new Set(); + for (const [configuredGroup, roles] of Object.entries(current.value.authorization.groupRoles)) { + if (!identity.groups.includes(configuredGroup)) continue; + for (const role of roles) mappedRoles.add(role); + } + if (mappedRoles.size === 0) { + return { + ready: false, + mode: "oidc", + checks: [{ + level: "error", + code: "oidc_groups_claim_invalid", + message: "The OIDC device-flow identity could not be validated.", + }], + }; + } + return report; + } catch (error) { + return { + ready: false, + mode: "oidc", + checks: [{ + level: "error", + code: error instanceof OidcDeviceFlowUnavailableError + ? "oidc_device_flow_unavailable" + : "oidc_groups_claim_invalid", + message: error instanceof OidcDeviceFlowUnavailableError + ? "OIDC device authorization is unavailable." + : "The OIDC device-flow identity could not be validated.", + }], + }; + } + }, + }; +} + +export interface DiagnosticCommandDependencies { + diagnoser: AuthDiagnoser; + secretValues?: readonly string[]; + stdout: (line: string) => void; + stderr: (line: string) => void; +} + +function genericFailure(): AuthDiagnostics { + return { + ready: false, + mode: "none", + checks: [{ level: "error", code: "auth_config_invalid", message: "Authentication configuration is unavailable." }], + }; +} + +function redact(value: string, secrets: readonly string[]): string { + let result = value; + for (const secret of [...secrets].filter(Boolean).sort((left, right) => right.length - left.length)) { + result = result.replaceAll(secret, "[REDACTED]"); + } + return result; +} + +function redactedReport(report: AuthDiagnostics, secrets: readonly string[]): AuthDiagnostics { + return { + ...report, + checks: report.checks.map((check): AuthDiagnostic => ({ + ...check, + message: redact(check.message, secrets), + ...(check.field === undefined ? {} : { field: redact(check.field, secrets) }), + })), + }; +} + +function parseArguments(args: readonly string[]): { json: true; interactive: boolean } | undefined { + let json = false; + let interactive = false; + for (const arg of args) { + if (arg === "--json" && !json) json = true; + else if (arg === "--interactive" && !interactive) interactive = true; + else return undefined; + } + return json ? { json: true, interactive } : undefined; +} + +/** A bounded machine command: stdout receives exactly one final report and no progress text. */ +export async function runDiagnosticCommand( + args: readonly string[], + dependencies: DiagnosticCommandDependencies, +): Promise { + const options = parseArguments(args); + if (!options) { + dependencies.stderr("usage: diagnostic-command.js --json [--interactive]"); + return 2; + } + let report: AuthDiagnostics; + const secrets = dependencies.secretValues ?? []; + try { + report = await dependencies.diagnoser.inspect({ + live: true, + ...(options.interactive ? { + interactive: true, + presentDeviceCode: (uri: string, code: string) => dependencies.stderr( + redact(`Open ${uri} and enter code ${code}`, secrets), + ), + } : {}), + }); + } catch { + report = genericFailure(); + } + const decoded = decodeAuthDiagnostics(report) ?? genericFailure(); + const safe = decodeAuthDiagnostics(redactedReport(decoded, secrets)) ?? genericFailure(); + dependencies.stdout(`${JSON.stringify(safe)}\n`); + return safe.ready ? 0 : 1; +} + +async function main(): Promise { + const exitCode = await runConfiguredDiagnosticCommand( + process.argv.slice(2), process.env, + (line) => process.stdout.write(line), + (line) => process.stderr.write(`${line}\n`), + ); + process.exitCode = exitCode; +} + +export async function runConfiguredDiagnosticCommand( + args: readonly string[], + env: Record, + stdout: (line: string) => void, + stderr: (line: string) => void, +): Promise { + let diagnoser: AuthDiagnoser = { inspect: async () => genericFailure() }; + let secretValues: readonly string[] | undefined; + try { + const config = loadConfig(env); + // Complete this preflight before constructing a diagnoser that may forward a device prompt. + secretValues = configuredSecretValues(config); + diagnoser = createConfiguredAuthDiagnoser(config); + } catch { /* turn startup or corpus faults into the closed report below */ } + return runDiagnosticCommand(args, { + diagnoser, + ...(secretValues === undefined ? {} : { secretValues }), + stdout, + stderr, + }); +} + +if (process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + void main(); +} diff --git a/backend/src/auth/diagnostics.ts b/backend/src/auth/diagnostics.ts new file mode 100644 index 00000000..32858529 --- /dev/null +++ b/backend/src/auth/diagnostics.ts @@ -0,0 +1,213 @@ +import type { AuthenticationConfigProvider, AuthMode } from "./types.js"; +import type { LocalUserRegistry } from "./local-registry.js"; +import { OidcIssuerMismatchError, OidcJwksUnavailableError, type OidcProtocol } from "./oidc-client.js"; +import { + createPosixAuthStorageBridge, + createWindowsAuthStorageBridge, + type WindowsAuthStorageBridge, +} from "./windows-auth-storage.js"; +import { isUsableAuthenticationSecret, type AuthenticationSecretReference } from "./secret-policy.js"; +import type { AuthDiagnostic, AuthDiagnosticCode, AuthDiagnostics, GroupCatalog } from "./group-catalog.js"; + +export type { AuthDiagnostic, AuthDiagnosticCode, AuthDiagnostics } from "./group-catalog.js"; + +const LIVE_DIAGNOSTIC_TIMEOUT_MS = 30_000; + +export interface AuthDiagnoser { + inspect(options: { + live: boolean; + interactive?: boolean; + signal?: AbortSignal; + /** Device-code presentation is transient operator output, never persisted diagnostic state. */ + presentDeviceCode?: (uri: string, code: string) => void; + }): Promise; +} + +export interface AuthDiagnoserDependencies { + authMode: AuthMode; + authStateRoot: string; + /** Platform integrations may inject an equivalent side-effect-free owner/ACL validator. */ + sessionRootValidator?: (root: string) => void | Promise; + windowsStorageBridge?: Pick; + posixStorageBridge?: Pick; + authentication?: AuthenticationConfigProvider; + secrets?: ReadonlyMap; + localUserRegistry?: LocalUserRegistry; + oidcProtocol?: OidcProtocol; + groupCatalog?: GroupCatalog; +} + +function check(code: AuthDiagnosticCode, message: string, field?: string): AuthDiagnostic { + return { level: "error", code, message, ...(field === undefined ? {} : { field }) }; +} + +function ordered(checks: readonly AuthDiagnostic[]): readonly AuthDiagnostic[] { + const unique = new Map(); + for (const item of checks) unique.set(`${item.code}\u0000${item.field ?? ""}`, item); + return [...unique.values()].sort((left, right) => { + const leftKey = `${left.code}\u0000${left.field ?? ""}`; + const rightKey = `${right.code}\u0000${right.field ?? ""}`; + return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0; + }); +} + +function stableCompare(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("The operation was aborted", "AbortError"); +} + +function awaitWithAbort(operation: Promise, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const abort = () => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", abort); + reject(abortReason(signal)); + }; + if (signal.aborted) abort(); + else signal.addEventListener("abort", abort, { once: true }); + operation.then( + (value) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", abort); + resolve(value); + }, + (error: unknown) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", abort); + reject(error); + }, + ); + }); +} + +function startBeforeAbort(signal: AbortSignal, operation: () => Promise): Promise { + return Promise.resolve().then(() => { + if (signal.aborted) throw abortReason(signal); + return operation(); + }); +} + +async function localRegistryIsUsable(deps: AuthDiagnoserDependencies): Promise { + try { + if (!deps.localUserRegistry) { + return check("local_user_registry_invalid", "The local user registry is unavailable."); + } + if (!await deps.localUserRegistry.hasEnabledAdmin()) { + return check("local_admin_missing", "No enabled local administrator is configured."); + } + return undefined; + } catch { + return check("local_user_registry_invalid", "The local user registry is invalid."); + } +} + +export function createAuthDiagnoser(deps: AuthDiagnoserDependencies): AuthDiagnoser { + const validateSessionRoot = deps.sessionRootValidator ?? (process.platform === "win32" + ? (root: string) => (deps.windowsStorageBridge ?? createWindowsAuthStorageBridge()).validateRoot(root) + : (root: string) => (deps.posixStorageBridge ?? createPosixAuthStorageBridge()).validateRoot(root)); + return { + async inspect(options): Promise { + const checks: AuthDiagnostic[] = []; + const signal = options.signal ?? new AbortController().signal; + try { + await validateSessionRoot(deps.authStateRoot); + } catch { + checks.push(check("auth_session_store_invalid", "The authentication session store is invalid.")); + } + + if (deps.authMode === "none" || deps.authMode === "mock") { + const result = ordered(checks); + return result.length === 0 + ? { ready: true, mode: deps.authMode, checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] } + : { ready: false, mode: deps.authMode, checks: result }; + } + if (deps.authMode === "upstream") { + checks.push(check("auth_config_incomplete", "The deprecated upstream authentication mode is not certifiable.")); + return { ready: false, mode: deps.authMode, checks: ordered(checks) }; + } + + let loaded; + try { + if (!deps.authentication) throw new Error("missing authentication configuration"); + loaded = deps.authentication.current(); + } catch { + checks.push(check(deps.authentication ? "auth_config_invalid" : "auth_config_incomplete", "Authentication configuration is unavailable.")); + return { ready: false, mode: deps.authMode, checks: ordered(checks) }; + } + if (loaded.value.mode !== deps.authMode) { + checks.push(check("auth_config_invalid", "Authentication mode does not match its configuration.")); + return { ready: false, mode: deps.authMode, checks: ordered(checks) }; + } + + if (loaded.value.mode === "local") { + const local = await localRegistryIsUsable(deps); + if (local) checks.push(local); + const result = ordered(checks); + return result.length === 0 + ? { ready: true, mode: "local", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] } + : { ready: false, mode: "local", checks: result }; + } + + const requiredSecrets: readonly AuthenticationSecretReference[] = ["THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN"]; + if (requiredSecrets.some((name) => !isUsableAuthenticationSecret(name, deps.secrets?.get(name)))) { + checks.push(check("oidc_secret_missing", "A required OIDC or group catalog secret is unavailable.")); + } + if (!options.live || checks.length > 0) { + const result = ordered(checks); + return result.length === 0 + ? { ready: true, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] } + : { ready: false, mode: "oidc", checks: result }; + } + const mappedGroupNames = Object.keys(loaded.value.authorization.groupRoles).sort(stableCompare); + + const deadline = new AbortController(); + const deadlineTimer = setTimeout(() => deadline.abort(), LIVE_DIAGNOSTIC_TIMEOUT_MS); + deadlineTimer.unref(); + const liveSignal = AbortSignal.any([signal, deadline.signal]); + try { + if (!deps.oidcProtocol) { + checks.push(check("oidc_discovery_unreachable", "The OIDC provider is unavailable.")); + } else { + try { + await awaitWithAbort(startBeforeAbort(liveSignal, () => deps.oidcProtocol!.diagnose(liveSignal)), liveSignal); + } catch (error) { + checks.push(check( + error instanceof OidcIssuerMismatchError + ? "oidc_issuer_mismatch" + : error instanceof OidcJwksUnavailableError + ? "oidc_jwks_unreachable" + : "oidc_discovery_unreachable", + "The OIDC provider could not be validated.", + )); + } + } + if (!liveSignal.aborted) { + if (!deps.groupCatalog) { + checks.push(check("oidc_group_catalog_unreachable", "The configured group catalog cannot be certified.")); + } else { + try { + checks.push(...await awaitWithAbort(startBeforeAbort(liveSignal, () => deps.groupCatalog!.verifyConfiguredGroups( + mappedGroupNames, liveSignal, + )), liveSignal)); + } catch { + checks.push(check("oidc_group_catalog_unreachable", "The configured group catalog is unavailable.")); + } + } + } + } finally { + clearTimeout(deadlineTimer); + } + const result = ordered(checks); + return result.length === 0 + ? { ready: true, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] } + : { ready: false, mode: "oidc", checks: result }; + }, + }; +} diff --git a/backend/src/auth/group-catalog.ts b/backend/src/auth/group-catalog.ts new file mode 100644 index 00000000..eb06f7ca --- /dev/null +++ b/backend/src/auth/group-catalog.ts @@ -0,0 +1,96 @@ +/** The fixed machine contract shared by the Authentik catalog and auth diagnostics. */ +export type AuthDiagnosticCode = + | "auth_ready" + | "auth_config_incomplete" + | "auth_config_invalid" + | "auth_session_store_invalid" + | "local_user_registry_invalid" + | "local_admin_missing" + | "oidc_secret_missing" + | "oidc_discovery_unreachable" + | "oidc_issuer_mismatch" + | "oidc_jwks_unreachable" + | "oidc_group_catalog_unreachable" + | "oidc_group_catalog_unauthorized" + | "oidc_mapped_group_missing" + | "oidc_mapped_group_ambiguous" + | "oidc_groups_claim_invalid" + | "oidc_device_flow_unavailable"; + +export interface AuthDiagnostic { + level: "error" | "info"; + code: AuthDiagnosticCode; + message: string; + field?: string; +} + +export interface AuthDiagnostics { + ready: boolean; + mode: "local" | "oidc" | "upstream" | "none" | "mock"; + checks: readonly AuthDiagnostic[]; +} + +const diagnosticCodes = new Set([ + "auth_ready", "auth_config_incomplete", "auth_config_invalid", "auth_session_store_invalid", + "local_user_registry_invalid", "local_admin_missing", "oidc_secret_missing", + "oidc_discovery_unreachable", "oidc_issuer_mismatch", "oidc_jwks_unreachable", + "oidc_group_catalog_unreachable", "oidc_group_catalog_unauthorized", "oidc_mapped_group_missing", + "oidc_mapped_group_ambiguous", "oidc_groups_claim_invalid", "oidc_device_flow_unavailable", +]); +const diagnosticModes = new Set(["local", "oidc", "upstream", "none", "mock"]); +const fieldCodes = new Set(["oidc_mapped_group_missing", "oidc_mapped_group_ambiguous"]); + +function exactObject(value: unknown, keys: readonly string[]): Record | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const source = value as Record; + const actual = Object.keys(source); + return actual.length === keys.length && actual.every((key) => keys.includes(key)) ? source : undefined; +} + +function safeText(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 512 + && value.trim() === value && !/\p{Cc}/u.test(value); +} + +/** Strict decoder for the machine contract shared with tht and the frontend. */ +export function decodeAuthDiagnostics(value: unknown): AuthDiagnostics | undefined { + const source = exactObject(value, ["ready", "mode", "checks"]); + if (!source || typeof source.ready !== "boolean" || typeof source.mode !== "string" + || !diagnosticModes.has(source.mode as AuthDiagnostics["mode"]) + || !Array.isArray(source.checks) || source.checks.length === 0 || source.checks.length > 129) return undefined; + const seen = new Set(); + const checks: AuthDiagnostic[] = []; + for (const value of source.checks) { + const raw = value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; + const check = raw && exactObject(raw, raw.field === undefined + ? ["level", "code", "message"] + : ["level", "code", "message", "field"]); + if (!check || (check.level !== "error" && check.level !== "info") + || typeof check.code !== "string" || !diagnosticCodes.has(check.code as AuthDiagnosticCode) + || !safeText(check.message) || (check.field !== undefined && !safeText(check.field))) return undefined; + const code = check.code as AuthDiagnosticCode; + if (check.field !== undefined && !fieldCodes.has(code)) return undefined; + const key = `${code}\u0000${check.field ?? ""}`; + if (seen.has(key)) return undefined; + seen.add(key); + checks.push({ + level: check.level, + code, + message: check.message, + ...(check.field === undefined ? {} : { field: check.field }), + }); + } + if (source.ready) { + if (checks.length !== 1 || checks[0].level !== "info" || checks[0].code !== "auth_ready" + || checks[0].field !== undefined) return undefined; + } else if (!checks.some(({ level }) => level === "error") + || checks.some(({ code }) => code === "auth_ready")) return undefined; + return { ready: source.ready, mode: source.mode as AuthDiagnostics["mode"], checks }; +} + +/** A provider-specific proof that only the configured authorization groups exist. */ +export interface GroupCatalog { + verifyConfiguredGroups(names: readonly string[], signal: AbortSignal): Promise; +} diff --git a/backend/src/auth/local-registry.ts b/backend/src/auth/local-registry.ts new file mode 100644 index 00000000..7d70d99d --- /dev/null +++ b/backend/src/auth/local-registry.ts @@ -0,0 +1,305 @@ +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readSync, + realpathSync, +} from "node:fs"; +import type { Stats } from "node:fs"; +import { dirname, isAbsolute, join, normalize } from "node:path"; +import { parseDocument } from "yaml"; +import { z } from "zod"; +import { isValidPasswordHash, verifyPassword, verifyWithDummy } from "./password.js"; +import type { LoadedAuthConfig, Role } from "./types.js"; +import { createWindowsAuthStorageBridge, type WindowsAuthStorageBridge } from "./windows-auth-storage.js"; + +const MAX_USERS_YAML_BYTES = 1 << 20; +const USERNAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._@-]{2,63}$/; +const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const ROLES = ["user", "admin"] as const; +const invalid = (): Error => new Error("local_user_registry_invalid"); + +function runtimeOwner(): number { + if (process.platform === "win32" || typeof process.geteuid !== "function") throw invalid(); + const owner = process.geteuid(); + if (!Number.isSafeInteger(owner) || owner < 0) throw invalid(); + return owner; +} + +export interface LocalUserRecord { + id: string; + username: string; + normalizedUsername: string; + displayName?: string; + passwordHash: string; + roles: readonly Role[]; + enabled: boolean; + authRevision: number; +} + +export interface LocalUserRegistry { + /** Safe production diagnostic probe; never returns user records or hashes. */ + hasEnabledAdmin(): Promise; + findByUsername(username: string): Promise; + findBySubject(id: string): Promise; + verify(user: LocalUserRecord | undefined, password: string): Promise; +} + +/** Keeps only the registry named by the current coherent authentication-config snapshot. */ +export interface CurrentLocalUserRegistryResolver { + resolve(loaded: LoadedAuthConfig): LocalUserRegistry | undefined; +} + +/** Native Windows obtains protected registry bytes only from the hidden tht bridge. */ +export interface LocalUserRegistryOptions { + windowsStorageBridge?: Pick; +} + +interface FileIdentity { + dev: number; + ino: number; + uid: number; + size: number; + mtimeMs: number; +} + +interface DirectoryIdentity { + dev: number; + ino: number; + uid: number; + mode: number; +} + +interface RegistryIdentity { + file: FileIdentity; + directory: DirectoryIdentity; +} + +const roleSchema = z.enum(ROLES); +const userSchema = z.strictObject({ + id: z.string().regex(UUID_V4_PATTERN), + username: z.string().regex(USERNAME_PATTERN), + displayName: z.string().optional().refine((value) => value === undefined || !/\p{Cc}/u.test(value)), + passwordHash: z.string().refine(isValidPasswordHash), + roles: z.array(roleSchema).min(1).superRefine((roles, context) => { + if (new Set(roles).size !== roles.length) context.addIssue({ code: "custom", message: "duplicate role" }); + }), + enabled: z.boolean(), + authRevision: z.number().int().positive().safe(), +}); +const registrySchema = z.strictObject({ version: z.literal(1), users: z.array(userSchema).min(1) }); + +function normalizeUsername(username: string): string { + return username.replace(/[A-Z]/g, (character) => character.toLowerCase()); +} + +function sameFileIdentity(left: FileIdentity, right: FileIdentity): boolean { + return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid + && left.size === right.size && left.mtimeMs === right.mtimeMs; +} + +function sameDirectoryIdentity(left: DirectoryIdentity, right: DirectoryIdentity): boolean { + return left.dev === right.dev && left.ino === right.ino && left.uid === right.uid && left.mode === right.mode; +} + +function sameIdentity(left: RegistryIdentity, right: RegistryIdentity): boolean { + return sameFileIdentity(left.file, right.file) && sameDirectoryIdentity(left.directory, right.directory); +} + +function validateCanonicalPath(path: string): void { + if (typeof path !== "string" || path.length === 0 || path.includes("\0") || !isAbsolute(path) || normalize(path) !== path) throw invalid(); + const parent = dirname(path); + if (realpathSync(parent) !== parent) throw invalid(); +} + +function fileMetadata(info: Stats, owner: number): FileIdentity { + if (!info.isFile() || info.uid !== owner || info.nlink !== 1 || (info.mode & 0o7777) !== 0o600) throw invalid(); + if (info.size < 0 || info.size > MAX_USERS_YAML_BYTES) throw invalid(); + return { dev: info.dev, ino: info.ino, uid: info.uid, size: info.size, mtimeMs: info.mtimeMs }; +} + +function directoryMetadata(info: Stats, owner: number): DirectoryIdentity { + if (!info.isDirectory() || info.uid !== owner || (info.mode & 0o7777) !== 0o700) throw invalid(); + return { dev: info.dev, ino: info.ino, uid: info.uid, mode: info.mode & 0o7777 }; +} + +function directoryIdentity(path: string, owner: number): DirectoryIdentity { + const parent = dirname(path); + if (realpathSync(parent) !== parent) throw invalid(); + return directoryMetadata(lstatSync(parent) as Stats, owner); +} + +function registryIdentity(path: string, owner: number): RegistryIdentity { + validateCanonicalPath(path); + const info = lstatSync(path); + return { file: fileMetadata(info as Stats, owner), directory: directoryIdentity(path, owner) }; +} + +function openDirectoryDescriptor(path: string): number | undefined { + if (process.platform === "win32") return undefined; + const flags = constants.O_RDONLY + | (constants.O_DIRECTORY ?? 0) + | (constants.O_NOFOLLOW ?? 0) + | (constants.O_NONBLOCK ?? 0); + return openSync(path, flags); +} + +function readBounded(path: string, owner: number): { source: string; identity: RegistryIdentity } { + validateCanonicalPath(path); + const beforeDirectory = directoryIdentity(path, owner); + const beforePath = lstatSync(path); + const before = fileMetadata(beforePath as Stats, owner); + let directoryDescriptor: number | undefined; + let descriptor: number | undefined; + try { + directoryDescriptor = openDirectoryDescriptor(dirname(path)); + const openedDirectory = directoryDescriptor === undefined + ? beforeDirectory + : directoryMetadata(fstatSync(directoryDescriptor) as Stats, owner); + if (!sameDirectoryIdentity(beforeDirectory, openedDirectory)) throw invalid(); + descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const opened = fileMetadata(fstatSync(descriptor) as Stats, owner); + if (!sameFileIdentity(before, opened)) throw invalid(); + const buffer = Buffer.allocUnsafe(MAX_USERS_YAML_BYTES + 1); + let offset = 0; + while (offset < buffer.length) { + const bytesRead = readSync(descriptor, buffer, offset, buffer.length - offset, null); + if (bytesRead === 0) break; + offset += bytesRead; + } + if (offset > MAX_USERS_YAML_BYTES) throw invalid(); + const after = fileMetadata(fstatSync(descriptor) as Stats, owner); + const afterPath = fileMetadata(lstatSync(path) as Stats, owner); + const afterDirectory = directoryMetadata(lstatSync(dirname(path)) as Stats, owner); + const afterOpenedDirectory = directoryDescriptor === undefined + ? afterDirectory + : directoryMetadata(fstatSync(directoryDescriptor) as Stats, owner); + if (!sameFileIdentity(opened, after) || !sameFileIdentity(after, afterPath) + || !sameDirectoryIdentity(beforeDirectory, afterDirectory) + || !sameDirectoryIdentity(openedDirectory, afterOpenedDirectory)) throw invalid(); + const source = new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, offset)); + return { source, identity: { file: after, directory: afterDirectory } }; + } catch { + throw invalid(); + } finally { + if (descriptor !== undefined) { + try { closeSync(descriptor); } catch { /* sanitized by design */ } + } + if (directoryDescriptor !== undefined) { + try { closeSync(directoryDescriptor); } catch { /* sanitized by design */ } + } + } +} + +function parseRegistry(source: string): LocalUserRecord[] { + try { + const document = parseDocument(source, { uniqueKeys: true }); + if (document.errors.length > 0 || document.warnings.length > 0) throw invalid(); + const parsed = registrySchema.parse(document.toJSON()); + const ids = new Set(); + const usernames = new Set(); + const records = parsed.users.map((user) => { + const normalizedUsername = normalizeUsername(user.username); + if (ids.has(user.id) || usernames.has(normalizedUsername)) throw invalid(); + ids.add(user.id); + usernames.add(normalizedUsername); + return Object.freeze({ + id: user.id, + username: user.username, + normalizedUsername, + ...(user.displayName === undefined ? {} : { displayName: user.displayName }), + passwordHash: user.passwordHash, + roles: Object.freeze([...user.roles]) as readonly Role[], + enabled: user.enabled, + authRevision: user.authRevision, + }); + }); + return records; + } catch { + throw invalid(); + } +} + +function load(path: string, owner: number): { records: LocalUserRecord[]; identity: RegistryIdentity } { + const read = readBounded(path, owner); + return { records: parseRegistry(read.source), identity: read.identity }; +} + +export function createLocalUserRegistry(usersPath: string, options: LocalUserRegistryOptions = {}): LocalUserRegistry { + let cached: { records: LocalUserRecord[]; identity: RegistryIdentity } | undefined; + const windowsStorage = process.platform === "win32" + ? options.windowsStorageBridge ?? createWindowsAuthStorageBridge() + : undefined; + + function currentPosix(): LocalUserRecord[] { + try { + const owner = runtimeOwner(); + const before = registryIdentity(usersPath, owner); + if (cached && sameIdentity(cached.identity, before)) return cached.records; + for (let attempt = 0; attempt < 2; attempt += 1) { + const loaded = load(usersPath, owner); + if (sameIdentity(loaded.identity, registryIdentity(usersPath, owner))) { + cached = loaded; + return loaded.records; + } + } + } catch { + throw invalid(); + } + throw invalid(); + } + + async function current(): Promise { + if (process.platform !== "win32") return currentPosix(); + try { + if (!windowsStorage) throw invalid(); + const contents = await windowsStorage.readLocalUsers(usersPath); + if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > MAX_USERS_YAML_BYTES) throw invalid(); + return parseRegistry(new TextDecoder("utf-8", { fatal: true }).decode(contents)); + } catch { + throw invalid(); + } + } + + async function operationalRecords(): Promise { + const records = await current(); + if (!records.some((user) => user.enabled && user.roles.includes("admin"))) throw invalid(); + return records; + } + + return { + async hasEnabledAdmin(): Promise { + return (await current()).some((user) => user.enabled && user.roles.includes("admin")); + }, + async findByUsername(username: string): Promise { + const normalized = normalizeUsername(username); + return (await operationalRecords()).find((user) => user.normalizedUsername === normalized); + }, + async findBySubject(id: string): Promise { + return (await operationalRecords()).find((user) => user.id === id); + }, + async verify(user: LocalUserRecord | undefined, password: string): Promise { + if (!user || !user.enabled) { + await verifyWithDummy(password); + return false; + } + return await verifyPassword(password, user.passwordHash); + }, + }; +} + +export function createCurrentLocalUserRegistryResolver(options: LocalUserRegistryOptions = {}): CurrentLocalUserRegistryResolver { + let current: { usersPath: string; registry: LocalUserRegistry } | undefined; + return { + resolve(loaded: LoadedAuthConfig): LocalUserRegistry | undefined { + if (loaded.value.mode !== "local") return undefined; + const usersPath = join(dirname(loaded.sourcePath), loaded.value.local.usersFile); + if (current?.usersPath === usersPath) return current.registry; + const registry = createLocalUserRegistry(usersPath, options); + current = { usersPath, registry }; + return registry; + }, + }; +} diff --git a/backend/src/auth/oidc-client.ts b/backend/src/auth/oidc-client.ts new file mode 100644 index 00000000..70c8a833 --- /dev/null +++ b/backend/src/auth/oidc-client.ts @@ -0,0 +1,658 @@ +import { + authorizationCodeGrant, + buildAuthorizationUrl, + calculatePKCECodeChallenge, + customFetch, + discovery, + initiateDeviceAuthorization, + pollDeviceAuthorizationGrant, + type Configuration, + type CustomFetch, +} from "openid-client"; +import { constants, createPublicKey, verify as verifySignature } from "node:crypto"; +import { parseConfiguredTransportUrl } from "./url-policy.js"; +import { isUsableAuthenticationSecret } from "./secret-policy.js"; + +export interface OidcIdentity { + issuer: string; + subject: string; + displayName?: string; + groups: readonly string[]; + tokenExpiresAt: Date; +} + +export interface OidcProtocol { + authorizationUrl(input: { state: string; nonce: string; codeVerifier: string }): Promise; + callback(input: { currentUrl: URL; state: string; nonce: string; codeVerifier: string }): Promise; + diagnose(signal: AbortSignal): Promise; + verifyDeviceFlow?(signal: AbortSignal, present: (uri: string, code: string) => void): Promise; +} + +export class OidcProtocolError extends Error { + constructor(message = "oidc_protocol_invalid") { + super(message); + this.name = "OidcProtocolError"; + } +} + +/** A safe operational distinction for callers and diagnostics; provider details never cross this boundary. */ +export class OidcProviderUnavailableError extends OidcProtocolError { + constructor() { + super("oidc_provider_unavailable"); + this.name = "OidcProviderUnavailableError"; + } +} + +/** The discovery document resolved, but its signed-token key set could not be certified. */ +export class OidcJwksUnavailableError extends OidcProtocolError { + constructor() { + super("oidc_jwks_unreachable"); + this.name = "OidcJwksUnavailableError"; + } +} + +/** Discovery completed with metadata for a different issuer than the configured trust anchor. */ +export class OidcIssuerMismatchError extends OidcProtocolError { + constructor() { + super("oidc_issuer_mismatch"); + this.name = "OidcIssuerMismatchError"; + } +} + +/** Device authorization is optional OIDC metadata and must never fall back to a browser flow. */ +export class OidcDeviceFlowUnavailableError extends OidcProtocolError { + constructor() { + super("oidc_device_flow_unavailable"); + this.name = "OidcDeviceFlowUnavailableError"; + } +} + +export interface OidcProtocolOptions { + issuer: string; + clientId: string; + clientSecret: string; + callbackUrl: string; + scopes: readonly string[]; + groupsClaim: string; + fetch?: typeof globalThis.fetch; + httpTimeoutMs?: number; + jwksTimeoutMs?: number; +} + +const MAX_GROUPS = 128; +const MAX_GROUP_LENGTH = 256; +const MAX_ID_TOKEN_LENGTH = 16 * 1024; +const MAX_OIDC_RESPONSE_BYTES = 1024 * 1024; +const DEFAULT_HTTP_TIMEOUT_MS = 5_000; +const MAX_HTTP_TIMEOUT_MS = 30_000; +const DEFAULT_JWKS_TIMEOUT_MS = 5_000; +const MAX_JWKS_TIMEOUT_MS = 30_000; +const MAX_DEVICE_FLOW_TIMEOUT_MS = 10 * 60_000; +const text = (value: unknown, maximum = 2048): value is string => + typeof value === "string" && value.length > 0 && value.length <= maximum && !/\p{Cc}/u.test(value); + +function discoveryStringList(value: unknown): value is readonly string[] { + return Array.isArray(value) && value.length > 0 && value.length <= 128 + && value.every((item) => text(item, 128)); +} + +function schemaValidDiscoveryMetadata(value: unknown): value is Record & { issuer: string } { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const metadata = value as Record; + const issuer = metadata.issuer; + const authorizationEndpoint = metadata.authorization_endpoint; + const tokenEndpoint = metadata.token_endpoint; + const jwksUri = metadata.jwks_uri; + if (!text(issuer, 2048) || !text(authorizationEndpoint, 2048) + || !text(tokenEndpoint, 2048) || !text(jwksUri, 2048) + || !discoveryStringList(metadata.response_types_supported) + || !discoveryStringList(metadata.subject_types_supported) + || !discoveryStringList(metadata.id_token_signing_alg_values_supported)) return false; + try { + configuredHttpsUrl(issuer); + httpsEndpoint(authorizationEndpoint); + httpsEndpoint(tokenEndpoint); + httpsEndpoint(jwksUri); + return true; + } catch { + return false; + } +} + +function configuredHttpsUrl(value: string): URL { + const url = parseConfiguredTransportUrl(value, { allowLoopbackHttp: false }); + if (!url) throw new OidcProtocolError(); + return url; +} + +function configuredCallbackUrl(value: string): URL { + const url = parseConfiguredTransportUrl(value, { allowLoopbackHttp: true }); + if (!url) throw new OidcProtocolError(); + return url; +} + +function httpsEndpoint(value: unknown): URL { + if (!text(value, 2048)) throw new OidcProtocolError(); + let url: URL; + try { + url = new URL(value); + } catch { + throw new OidcProtocolError(); + } + if (url.protocol !== "https:" || url.username || url.password || url.hash) throw new OidcProtocolError(); + return url; +} + +function groupsFromClaims(claims: Record, name: string): string[] { + const indirect = claims._claim_names; + if ((indirect && typeof indirect === "object" && !Array.isArray(indirect) + && Object.prototype.hasOwnProperty.call(indirect, name)) + || claims.hasgroups === true) throw new OidcProtocolError(); + const raw = claims[name]; + if (!Array.isArray(raw) || raw.length === 0 || raw.length > MAX_GROUPS) throw new OidcProtocolError(); + const groups: string[] = []; + const unique = new Set(); + for (const group of raw) { + if (!text(group, MAX_GROUP_LENGTH) || group.trim().length === 0 || unique.has(group)) throw new OidcProtocolError(); + unique.add(group); + groups.push(group); + } + return groups; +} + +function identityFromClaims(claims: Record, options: OidcProtocolOptions): OidcIdentity { + if (claims.iss !== options.issuer || !text(claims.sub, 512)) throw new OidcProtocolError(); + const audience = claims.aud; + if (!(audience === options.clientId || (Array.isArray(audience) && audience.includes(options.clientId)))) { + throw new OidcProtocolError(); + } + if (typeof claims.exp !== "number" || !Number.isSafeInteger(claims.exp) || claims.exp * 1000 <= Date.now()) { + throw new OidcProtocolError(); + } + const tokenExpiresAt = new Date(claims.exp * 1000); + if (Number.isNaN(tokenExpiresAt.getTime())) throw new OidcProtocolError(); + return { + issuer: options.issuer, + subject: claims.sub, + ...(text(claims.name, 256) ? { displayName: claims.name } : {}), + groups: groupsFromClaims(claims, options.groupsClaim), + tokenExpiresAt, + }; +} + +function jsonPart(part: string): Record { + if (!/^[A-Za-z0-9_-]+$/.test(part) || part.length > MAX_ID_TOKEN_LENGTH) throw new OidcProtocolError(); + try { + const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.from(part, "base64url"))); + if (!value || typeof value !== "object" || Array.isArray(value)) throw new OidcProtocolError(); + return value as Record; + } catch { + throw new OidcProtocolError(); + } +} + +function signatureAlgorithm(algorithm: string): { + digest: string | null; + pss?: boolean; + saltLength?: number; + ecdsaPartLength?: number; +} { + switch (algorithm) { + case "RS256": return { digest: "RSA-SHA256" }; + case "RS384": return { digest: "RSA-SHA384" }; + case "RS512": return { digest: "RSA-SHA512" }; + case "PS256": return { digest: "sha256", pss: true, saltLength: 32 }; + case "PS384": return { digest: "sha384", pss: true, saltLength: 48 }; + case "PS512": return { digest: "sha512", pss: true, saltLength: 64 }; + case "ES256": return { digest: "sha256", ecdsaPartLength: 32 }; + case "ES384": return { digest: "sha384", ecdsaPartLength: 48 }; + case "ES512": return { digest: "sha512", ecdsaPartLength: 66 }; + case "EdDSA": return { digest: null }; + default: throw new OidcProtocolError(); + } +} + +function derLength(length: number): Buffer { + if (length < 128) return Buffer.from([length]); + if (length < 256) return Buffer.from([0x81, length]); + throw new OidcProtocolError(); +} + +function derInteger(raw: Buffer): Buffer { + let start = 0; + while (start < raw.length - 1 && raw[start] === 0) start += 1; + let value = raw.subarray(start); + if ((value[0] & 0x80) !== 0) value = Buffer.concat([Buffer.from([0]), value]); + return Buffer.concat([Buffer.from([0x02]), derLength(value.length), value]); +} + +function joseEcdsaSignatureToDer(signature: Buffer, partLength: number): Buffer { + if (signature.length !== partLength * 2) throw new OidcProtocolError(); + const sequence = Buffer.concat([ + derInteger(signature.subarray(0, partLength)), + derInteger(signature.subarray(partLength)), + ]); + return Buffer.concat([Buffer.from([0x30]), derLength(sequence.length), sequence]); +} + +class OidcTransportError extends Error { + constructor(readonly availability: boolean) { + super(availability ? "oidc_provider_unavailable" : "oidc_provider_response_invalid"); + this.name = "OidcTransportError"; + } +} + +interface BoundedOidcTransport { + customFetch: CustomFetch; + request( + input: RequestInfo | URL, + init?: RequestInit, + options?: { timeoutMs?: number; requireSuccess?: boolean }, + ): Promise; +} + +function cancelReaderBestEffort(reader: ReadableStreamDefaultReader): void { + try { + void Promise.resolve(reader.cancel()).catch(() => undefined); + } catch { + // Cancellation is advisory; the deadline and rejection remain authoritative. + } +} + +function cancelResponseBestEffort(response: Response): void { + if (!response.body) return; + try { + void Promise.resolve(response.body.cancel()).catch(() => undefined); + } catch { + // A late response is never allowed to turn an already-bounded request into an unhandled rejection. + } +} + +function abortedReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("The operation was aborted", "AbortError"); +} + +async function awaitWithAbort( + operation: Promise, + signal: AbortSignal, + onLateResolution?: (value: T) => void, +): Promise { + return await new Promise((resolve, reject) => { + let settled = signal.aborted; + const cleanup = () => signal.removeEventListener("abort", aborted); + const aborted = () => { + if (settled) return; + settled = true; + cleanup(); + reject(abortedReason(signal)); + }; + if (signal.aborted) reject(abortedReason(signal)); + else signal.addEventListener("abort", aborted, { once: true }); + operation.then( + (value) => { + if (settled) { + try { onLateResolution?.(value); } catch { /* best-effort late cleanup only */ } + return; + } + settled = true; + cleanup(); + resolve(value); + }, + (error: unknown) => { + if (settled) return; + settled = true; + cleanup(); + reject(error); + }, + ); + }); +} + +function providerRequestUrl(input: RequestInfo | URL): URL { + const value = input instanceof URL ? input.href : input instanceof Request ? input.url : input; + return httpsEndpoint(value); +} + +function declaredResponseLength(response: Response): number | undefined { + const declared = response.headers.get("content-length"); + if (declared === null) return undefined; + if (!/^\d+$/.test(declared)) throw new OidcTransportError(false); + const length = Number(declared); + if (!Number.isSafeInteger(length) || length > MAX_OIDC_RESPONSE_BYTES) throw new OidcTransportError(false); + return length; +} + +function safeBufferedResponse(response: Response, body: Buffer): Response { + const noBodyStatus = response.status === 204 || response.status === 205 || response.status === 304; + // Node accepts Buffer as a fetch body; the DOM declaration in this project does not model it. + return new Response(noBodyStatus ? null : body as unknown as BodyInit, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function boundedResponse( + response: Response, + signal: AbortSignal, + requireSuccess: boolean, +): Promise { + const reader = response.body?.getReader(); + const chunks: Buffer[] = []; + let total = 0; + let completed = false; + try { + if (response.redirected || response.type === "opaqueredirect" || response.status >= 300 && response.status < 400) { + throw new OidcTransportError(false); + } + if (requireSuccess && !response.ok) throw new OidcTransportError(false); + declaredResponseLength(response); + if (!reader) { + completed = true; + return safeBufferedResponse(response, Buffer.alloc(0)); + } + while (true) { + const { done, value } = await awaitWithAbort(reader.read(), signal); + if (done) break; + if (value.byteLength > MAX_OIDC_RESPONSE_BYTES - total) throw new OidcTransportError(false); + total += value.byteLength; + chunks.push(Buffer.from(value)); + } + completed = true; + return safeBufferedResponse(response, Buffer.concat(chunks, total)); + } finally { + try { + if (!completed && reader) cancelReaderBestEffort(reader); + } finally { + try { reader?.releaseLock(); } catch { /* cancellation already made the response unusable */ } + } + } +} + +function createBoundedOidcTransport( + fetchImplementation: typeof globalThis.fetch, + defaultTimeoutMs: number, +): BoundedOidcTransport { + const request: BoundedOidcTransport["request"] = async (input, init, requestOptions) => { + let target: URL; + try { + target = providerRequestUrl(input); + } catch { + throw new OidcTransportError(false); + } + const timeoutMs = requestOptions?.timeoutMs ?? defaultTimeoutMs; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + timeout.unref(); + const signal = init?.signal ? AbortSignal.any([init.signal, controller.signal]) : controller.signal; + try { + const requestInit: RequestInit = { ...init, redirect: "manual", signal }; + const response = await awaitWithAbort( + Promise.resolve().then(() => fetchImplementation(target, requestInit)), + signal, + cancelResponseBestEffort, + ); + return await boundedResponse(response, signal, requestOptions?.requireSuccess === true); + } catch (error) { + if (error instanceof OidcTransportError) throw error; + if (signal.aborted) throw new OidcTransportError(true); + throw new OidcTransportError(true); + } finally { + clearTimeout(timeout); + } + }; + return { + request, + // openid-client's FetchBody is Fetch-compatible, but comes from a distinct declaration graph. + customFetch: (url, options) => request(url, options as unknown as RequestInit), + }; +} + +function availabilityFailure(error: unknown): boolean { + let current = error; + const seen = new Set(); + for (let depth = 0; depth < 8; depth += 1) { + if (current instanceof OidcTransportError) return current.availability; + if (!current || typeof current !== "object" || seen.has(current)) return false; + seen.add(current); + current = (current as { cause?: unknown }).cause; + } + return false; +} + +function protocolFailure(error: unknown): OidcProtocolError { + let current = error; + const seen = new Set(); + for (let depth = 0; depth < 8; depth += 1) { + if (current instanceof OidcProtocolError) return current; + if (!current || typeof current !== "object" || seen.has(current)) break; + seen.add(current); + current = (current as { cause?: unknown }).cause; + } + return availabilityFailure(error) ? new OidcProviderUnavailableError() : new OidcProtocolError(); +} + +async function verifyIdTokenSignature( + idToken: unknown, + config: Configuration, + transport: BoundedOidcTransport, + jwksTimeoutMs: number, +): Promise { + if (!text(idToken, MAX_ID_TOKEN_LENGTH)) throw new OidcProtocolError(); + const [protectedPart, payloadPart, signaturePart, extra] = idToken.split("."); + if (!protectedPart || !payloadPart || !signaturePart || extra) throw new OidcProtocolError(); + const header = jsonPart(protectedPart); + if (!text(header.alg, 16) || !text(header.kid, 256)) throw new OidcProtocolError(); + const metadata = config.serverMetadata(); + if (!Array.isArray(metadata.id_token_signing_alg_values_supported) + || !metadata.id_token_signing_alg_values_supported.includes(header.alg) + || !text(metadata.jwks_uri, 2048)) throw new OidcProtocolError(); + const jwksUrl = httpsEndpoint(metadata.jwks_uri); + try { + const response = await transport.request( + jwksUrl, + { headers: { accept: "application/json" }, redirect: "manual" }, + { timeoutMs: jwksTimeoutMs, requireSuccess: true }, + ); + const body = new TextDecoder("utf-8", { fatal: true }).decode(await response.arrayBuffer()); + const parsed = JSON.parse(body) as { keys?: unknown }; + if (!Array.isArray(parsed.keys) || parsed.keys.length === 0 || parsed.keys.length > 16) throw new OidcProtocolError(); + const matching = parsed.keys.filter((key): key is Record => + Boolean(key) && typeof key === "object" && !Array.isArray(key) && key.kid === header.kid); + if (matching.length !== 1) throw new OidcProtocolError(); + const key = matching[0]; + if (key.use !== undefined && key.use !== "sig") throw new OidcProtocolError(); + if (key.alg !== undefined && key.alg !== header.alg) throw new OidcProtocolError(); + const algorithm = signatureAlgorithm(header.alg); + if (!/^[A-Za-z0-9_-]+$/.test(signaturePart)) throw new OidcProtocolError(); + const signature = Buffer.from(signaturePart, "base64url"); + if (signature.length === 0) throw new OidcProtocolError(); + const publicKey = createPublicKey({ key: key as never, format: "jwk" }); + const normalizedSignature = algorithm.ecdsaPartLength + ? joseEcdsaSignatureToDer(signature, algorithm.ecdsaPartLength) + : signature; + const verified = algorithm.pss + ? verifySignature(algorithm.digest, Buffer.from(`${protectedPart}.${payloadPart}`), { + key: publicKey, padding: constants.RSA_PKCS1_PSS_PADDING, saltLength: algorithm.saltLength, + }, normalizedSignature) + : verifySignature(algorithm.digest, Buffer.from(`${protectedPart}.${payloadPart}`), publicKey, normalizedSignature); + if (!verified) throw new OidcProtocolError(); + } catch (error) { + throw protocolFailure(error); + } +} + +async function verifyJwksAvailability( + config: Configuration, + transport: BoundedOidcTransport, + jwksTimeoutMs: number, + signal: AbortSignal, +): Promise { + const metadata = config.serverMetadata(); + if (!text(metadata.jwks_uri, 2048)) throw new OidcProtocolError(); + const response = await transport.request( + httpsEndpoint(metadata.jwks_uri), + { headers: { accept: "application/json" }, redirect: "manual", signal }, + { timeoutMs: jwksTimeoutMs, requireSuccess: true }, + ); + const parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(await response.arrayBuffer())); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) + || !Array.isArray((parsed as { keys?: unknown }).keys)) throw new OidcProtocolError(); +} + +export function createOidcProtocol(options: OidcProtocolOptions): OidcProtocol { + const issuerUrl = configuredHttpsUrl(options.issuer); + const callbackUrl = configuredCallbackUrl(options.callbackUrl); + if (!text(options.clientId, 512) || !isUsableAuthenticationSecret("THT_OIDC_CLIENT_SECRET", options.clientSecret) + || !text(options.groupsClaim, 128) || options.scopes.length === 0 || options.scopes.length > 16 + || options.scopes.some((scope) => !text(scope, 128)) + || (options.httpTimeoutMs !== undefined && (!Number.isSafeInteger(options.httpTimeoutMs) + || options.httpTimeoutMs < 1 || options.httpTimeoutMs > MAX_HTTP_TIMEOUT_MS)) + || (options.jwksTimeoutMs !== undefined && (!Number.isSafeInteger(options.jwksTimeoutMs) + || options.jwksTimeoutMs < 1 || options.jwksTimeoutMs > MAX_JWKS_TIMEOUT_MS))) throw new OidcProtocolError(); + const httpTimeoutMs = options.httpTimeoutMs ?? DEFAULT_HTTP_TIMEOUT_MS; + const jwksTimeoutMs = options.jwksTimeoutMs ?? DEFAULT_JWKS_TIMEOUT_MS; + const transport = createBoundedOidcTransport(options.fetch ?? globalThis.fetch, httpTimeoutMs); + + let discovered: Promise | undefined; + const configuration = async (): Promise => { + if (!discovered) { + discovered = (async () => { + let certifiedIssuerMismatch = false; + const issuerCheckingFetch: CustomFetch = async (input, init) => { + const response = await transport.customFetch(input, init); + if (!response.ok) return response; + try { + const metadata: unknown = await response.clone().json(); + if (schemaValidDiscoveryMetadata(metadata) && metadata.issuer !== options.issuer) { + certifiedIssuerMismatch = true; + const headers = new Headers(response.headers); + headers.delete("content-length"); + return new Response(JSON.stringify({ ...metadata, issuer: options.issuer }), { + status: response.status, + statusText: response.statusText, + headers, + }); + } + } catch { + // The OIDC library owns malformed discovery-document classification. + } + return response; + }; + try { + const config = await discovery( + issuerUrl, + options.clientId, + { client_secret: options.clientSecret, redirect_uris: [callbackUrl.href], response_types: ["code"] }, + undefined, + { [customFetch]: issuerCheckingFetch, timeout: httpTimeoutMs / 1000 }, + ); + const metadata = config.serverMetadata(); + if (metadata.issuer !== options.issuer) throw new OidcProtocolError(); + httpsEndpoint(metadata.authorization_endpoint); + httpsEndpoint(metadata.token_endpoint); + httpsEndpoint(metadata.jwks_uri); + if (certifiedIssuerMismatch) throw new OidcIssuerMismatchError(); + return config; + } catch (error) { + throw protocolFailure(error); + } + })(); + } + return await discovered; + }; + + return { + async authorizationUrl(input) { + try { + const challenge = await calculatePKCECodeChallenge(input.codeVerifier); + return buildAuthorizationUrl(await configuration(), { + response_type: "code", + redirect_uri: callbackUrl.href, + scope: options.scopes.join(" "), + state: input.state, + nonce: input.nonce, + code_challenge: challenge, + code_challenge_method: "S256", + }); + } catch (error) { + throw protocolFailure(error); + } + }, + async callback(input) { + if (input.currentUrl.origin !== callbackUrl.origin || input.currentUrl.pathname !== callbackUrl.pathname) { + throw new OidcProtocolError(); + } + try { + const config = await configuration(); + const tokens = await authorizationCodeGrant(config, input.currentUrl, { + expectedState: input.state, + expectedNonce: input.nonce, + pkceCodeVerifier: input.codeVerifier, + idTokenExpected: true, + }); + await verifyIdTokenSignature(tokens.id_token, config, transport, jwksTimeoutMs); + const claims = tokens.claims(); + if (!claims || Array.isArray(claims)) throw new OidcProtocolError(); + return identityFromClaims(claims as Record, options); + } catch (error) { + throw protocolFailure(error); + } + }, + async diagnose(signal) { + signal.throwIfAborted(); + const config = await configuration(); + signal.throwIfAborted(); + try { + await verifyJwksAvailability(config, transport, jwksTimeoutMs, signal); + } catch { + throw new OidcJwksUnavailableError(); + } + signal.throwIfAborted(); + }, + async verifyDeviceFlow(signal, present) { + const maximumDeadline = AbortSignal.timeout(MAX_DEVICE_FLOW_TIMEOUT_MS); + const operationSignal = AbortSignal.any([signal, maximumDeadline]); + let config: Configuration; + try { + operationSignal.throwIfAborted(); + config = await configuration(); + operationSignal.throwIfAborted(); + const endpoint = config.serverMetadata().device_authorization_endpoint; + httpsEndpoint(endpoint); + } catch (error) { + if (error instanceof OidcIssuerMismatchError || error instanceof OidcProviderUnavailableError) throw error; + throw new OidcDeviceFlowUnavailableError(); + } + try { + operationSignal.throwIfAborted(); + const device = await awaitWithAbort( + initiateDeviceAuthorization(config, { scope: options.scopes.join(" ") }), + operationSignal, + ); + if (!text(device.verification_uri, 2048) || !text(device.user_code, 256)) { + throw new OidcDeviceFlowUnavailableError(); + } + const providerLifetimeMs = device.expires_in * 1000; + if (!Number.isSafeInteger(providerLifetimeMs) || providerLifetimeMs <= 0) { + throw new OidcDeviceFlowUnavailableError(); + } + const deviceSignal = AbortSignal.any([ + signal, + maximumDeadline, + AbortSignal.timeout(Math.min(providerLifetimeMs, MAX_DEVICE_FLOW_TIMEOUT_MS)), + ]); + const verificationUri = httpsEndpoint(device.verification_uri); + present(verificationUri.href, device.user_code); + const tokens = await pollDeviceAuthorizationGrant(config, device, undefined, { signal: deviceSignal }); + await verifyIdTokenSignature(tokens.id_token, config, transport, jwksTimeoutMs); + const claims = tokens.claims(); + if (!claims || Array.isArray(claims)) throw new OidcProtocolError(); + return identityFromClaims(claims as Record, options); + } catch (error) { + if (error instanceof OidcDeviceFlowUnavailableError) throw error; + throw protocolFailure(error); + } + }, + }; +} diff --git a/backend/src/auth/password.ts b/backend/src/auth/password.ts new file mode 100644 index 00000000..2074982c --- /dev/null +++ b/backend/src/auth/password.ts @@ -0,0 +1,157 @@ +import { argon2, timingSafeEqual } from "node:crypto"; + +const MAXIMUM_PHC_BYTES = 256; +const ARGON2_MEMORY_KIB = 65_536; +const ARGON2_PASSES = 3; +const ARGON2_PARALLELISM = 1; +const ARGON2_SALT_BYTES = 16; +const ARGON2_KEY_BYTES = 32; +const MINIMUM_PASSWORD_BYTES = 12; +const MAXIMUM_PASSWORD_BYTES = 1024; + +interface Argon2Parameters { + memory: number; + passes: number; + parallelism: number; + salt: Buffer; + digest: Buffer; +} + +/** Internal-only signal for unavailable native Argon2 work; never expose its cause. */ +export class LocalPasswordVerificationError extends Error { + constructor() { + super("local_password_verification_failed"); + } +} + +function parseDecimal(value: string, maximum: number): number | undefined { + if (!/^\d+$/.test(value) || (value.length > 1 && value[0] === "0") || value.length > 10) return undefined; + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed <= maximum ? parsed : undefined; +} + +function decodeRawBase64(value: string, minimum: number, maximum: number): Buffer | undefined { + if (!/^[A-Za-z0-9+/]+$/.test(value)) return undefined; + const decoded = Buffer.from(value, "base64"); + if (decoded.length < minimum || decoded.length > maximum) { + decoded.fill(0); + return undefined; + } + if (decoded.toString("base64").replace(/=+$/, "") !== value) { + decoded.fill(0); + return undefined; + } + return decoded; +} + +function parsePHC(encoded: string): Argon2Parameters | undefined { + if (typeof encoded !== "string" || encoded.length === 0 || Buffer.byteLength(encoded, "utf8") > MAXIMUM_PHC_BYTES) return undefined; + const parts = encoded.split("$"); + if (parts.length !== 6 || parts[0] !== "" || parts[1] !== "argon2id" || parts[2] !== "v=19") return undefined; + + const parameterParts = parts[3].split(","); + if (parameterParts.length !== 3 || !parameterParts[0].startsWith("m=") || !parameterParts[1].startsWith("t=") || !parameterParts[2].startsWith("p=")) return undefined; + const memory = parseDecimal(parameterParts[0].slice(2), ARGON2_MEMORY_KIB); + const passes = parseDecimal(parameterParts[1].slice(2), ARGON2_PASSES); + const parallelism = parseDecimal(parameterParts[2].slice(2), ARGON2_PARALLELISM); + if (memory !== ARGON2_MEMORY_KIB || passes !== ARGON2_PASSES || parallelism !== ARGON2_PARALLELISM) return undefined; + + const salt = decodeRawBase64(parts[4], ARGON2_SALT_BYTES, ARGON2_SALT_BYTES); + const digest = decodeRawBase64(parts[5], ARGON2_KEY_BYTES, ARGON2_KEY_BYTES); + if (!salt || !digest) { + salt?.fill(0); + digest?.fill(0); + return undefined; + } + return { memory, passes, parallelism, salt, digest }; +} + +function hasValidPasswordBytes(password: string): boolean { + if (typeof password !== "string") return false; + const typedPassword = password as string & { isWellFormed?: () => boolean }; + if (typeof typedPassword.isWellFormed === "function") { + if (!typedPassword.isWellFormed()) return false; + } else if (/[\uD800-\uDFFF]/.test(password)) { + return false; + } + const byteLength = Buffer.byteLength(password, "utf8"); + return byteLength >= MINIMUM_PASSWORD_BYTES && byteLength <= MAXIMUM_PASSWORD_BYTES; +} + +function passwordBytes(password: string): Buffer | undefined { + return hasValidPasswordBytes(password) ? Buffer.from(password, "utf8") : undefined; +} + +function clearParameters(parameters: Argon2Parameters): void { + parameters.salt.fill(0); + parameters.digest.fill(0); +} + +function deriveArgon2(message: Buffer, parameters: Argon2Parameters): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const complete = (error: Error | null, derived?: Buffer): void => { + if (settled) { + derived?.fill(0); + return; + } + settled = true; + if (error || !derived) { + derived?.fill(0); + reject(error ?? new Error("argon2_failed")); + return; + } + resolve(derived); + }; + try { + argon2("argon2id", { + message, + nonce: parameters.salt, + memory: parameters.memory, + passes: parameters.passes, + parallelism: parameters.parallelism, + tagLength: parameters.digest.length, + }, complete); + } catch (error) { + if (!settled) { + settled = true; + reject(error); + } + } + }); +} + +export function isValidPasswordHash(encoded: string): boolean { + const parameters = parsePHC(encoded); + if (!parameters) return false; + clearParameters(parameters); + return true; +} + +export async function verifyPassword(password: string, encoded: string): Promise { + const parameters = parsePHC(encoded); + const message = passwordBytes(password); + if (!message || !parameters) { + message?.fill(0); + if (parameters) clearParameters(parameters); + return false; + } + let derived: Buffer | undefined; + try { + derived = await deriveArgon2(message, parameters); + return derived.length === parameters.digest.length && timingSafeEqual(derived, parameters.digest); + } catch { + throw new LocalPasswordVerificationError(); + } finally { + message.fill(0); + derived?.fill(0); + clearParameters(parameters); + } +} + +const DUMMY_PASSWORD = "thothii-process-local-dummy-password"; +const DUMMY_HASH = "$argon2id$v=19$m=65536,t=3,p=1$ABEiM0RVZneImaq7zN3u/w$/YuK14HV5biXcVIYqaJs07meu0nRgo+d62KnM02MSoU"; + +export async function verifyWithDummy(password: string): Promise { + await verifyPassword(hasValidPasswordBytes(password) ? password : DUMMY_PASSWORD, DUMMY_HASH); +} diff --git a/backend/src/auth/principal.ts b/backend/src/auth/principal.ts index a58aff1f..95302a56 100644 --- a/backend/src/auth/principal.ts +++ b/backend/src/auth/principal.ts @@ -2,16 +2,21 @@ import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { randomUUID } from "node:crypto"; +import { isPermission, rolesToPermissions } from "./config.js"; +import type { Permission, Role } from "./types.js"; export interface PrincipalContext { issuer: string; subject: string; displayName?: string; + roles: readonly Role[]; + permissions: readonly Permission[]; isAdmin: boolean; } const principalEnvKeys = [ "THT_PRINCIPAL_ISSUER", "THT_PRINCIPAL_SUBJECT", "THT_PRINCIPAL_DISPLAY_NAME", "THT_PRINCIPAL_IS_ADMIN", + "THT_PRINCIPAL_PERMISSIONS", ] as const; export function clearPrincipalEnvironment(env: NodeJS.ProcessEnv): void { @@ -42,6 +47,19 @@ function optional(value: unknown): string | undefined { return required(value); } +function principal( + issuer: string, subject: string, roles: readonly Role[], displayName?: string, +): PrincipalContext { + return { + issuer, + subject, + ...(displayName ? { displayName } : {}), + roles, + permissions: rolesToPermissions(roles), + isAdmin: roles.includes("admin"), + }; +} + export function upstreamPrincipal(headers: Record): PrincipalContext | undefined { const issuer = required(headers["x-thoth-principal-issuer"]); const subject = required(headers["x-thoth-principal-subject"]); @@ -49,10 +67,15 @@ export function upstreamPrincipal(headers: Record): PrincipalCo const adminHeader = headers["x-thoth-is-admin"]; if (!issuer || !subject || (headers["x-thoth-principal-display-name"] !== undefined && !displayName)) return undefined; if (adminHeader !== "0" && adminHeader !== "1" && adminHeader !== "true" && adminHeader !== "false") return undefined; - return { issuer, subject, displayName, isAdmin: adminHeader === "1" || adminHeader === "true" }; + return principal( + issuer, + subject, + adminHeader === "1" || adminHeader === "true" ? ["user", "admin"] : ["user"], + displayName, + ); } -export function localPrincipal(): PrincipalContext { +export function localPrincipal(publicExposure = false): PrincipalContext { const home = expandLocalHome(process.env.THT_HOME ?? join(homedir(), ".thothii")); const identityPath = join(home, "identity.json"); mkdirSync(home, { recursive: true, mode: 0o700 }); @@ -61,29 +84,34 @@ export function localPrincipal(): PrincipalContext { const stored = JSON.parse(readFileSync(identityPath, "utf8")); if (stored?.issuer === "local" && typeof stored.subject === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(stored.subject)) { harden(identityPath, 0o600); - return { issuer: "local", subject: stored.subject, isAdmin: false }; + return principal("local", stored.subject, publicExposure ? ["user"] : ["admin"]); } throw new Error("invalid local identity"); } catch (error: any) { if (error?.code !== "ENOENT") throw error; - const principal = { issuer: "local", subject: randomUUID() }; + const created = { issuer: "local", subject: randomUUID() }; try { - writeFileSync(identityPath, JSON.stringify(principal) + "\n", { mode: 0o600, flag: "wx" }); + writeFileSync(identityPath, JSON.stringify(created) + "\n", { mode: 0o600, flag: "wx" }); harden(identityPath, 0o600); - return { ...principal, isAdmin: false }; + return principalContext("local", created.subject, publicExposure); } catch (writeError: any) { // Another local request won the identity creation race; always converge on its UUID. - if (writeError?.code === "EEXIST") return localPrincipal(); + if (writeError?.code === "EEXIST") return localPrincipal(publicExposure); throw writeError; } } } +function principalContext(issuer: string, subject: string, publicExposure: boolean): PrincipalContext { + return principal(issuer, subject, publicExposure ? ["user"] : ["admin"]); +} + export function principalEnvironment(principal: PrincipalContext): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { THT_PRINCIPAL_ISSUER: principal.issuer, THT_PRINCIPAL_SUBJECT: principal.subject, THT_PRINCIPAL_IS_ADMIN: principal.isAdmin ? "true" : "false", + THT_PRINCIPAL_PERMISSIONS: principal.permissions.filter(isPermission).join(","), }; if (principal.displayName) env.THT_PRINCIPAL_DISPLAY_NAME = principal.displayName; return env; diff --git a/backend/src/auth/routes.ts b/backend/src/auth/routes.ts new file mode 100644 index 00000000..71e7ad97 --- /dev/null +++ b/backend/src/auth/routes.ts @@ -0,0 +1,659 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import type { + AuthenticationConfigProvider, + LoadedAuthConfig, + OidcAuthenticationConfig, + OidcStateRecord, + OidcTransactionTransport, + Role, +} from "./types.js"; +import type { LocalUserRecord, LocalUserRegistry } from "./local-registry.js"; +import { OidcStateCapacityError, type AuthSessionStore } from "./session-store.js"; +import { rolesToPermissions } from "./config.js"; +import { captureAuthConfigSnapshot, getPrincipal, requireExactOrigin, sessionCookieName } from "./auth.js"; +import { requirePermission, isPrincipalContext } from "./authorization.js"; +import { deriveCsrfToken } from "./csrf.js"; +import { verifyWithDummy } from "./password.js"; +import type { OidcProtocol } from "./oidc-client.js"; +import { parseConfiguredTransportUrl } from "./url-policy.js"; + +const TEN_MINUTES_MS = 10 * 60 * 1000; +const REMEMBER_COOKIE_SECONDS = 2_592_000; +const MAX_USERNAME_LENGTH = 64; +const MAX_PASSWORD_LENGTH = 1024; +const MAX_LIMIT_ENTRIES = 10_000; +const MAX_OIDC_INITIATIONS_PER_ADDRESS = 20; +const MAX_OIDC_CALLBACK_QUERY_LENGTH = 4096; +const OIDC_CALLBACK_PATH = "/api/auth/oidc/callback"; +const OIDC_TRANSACTION_COOKIE_SECONDS = TEN_MINUTES_MS / 1000; +const OIDC_VALUE_PATTERN = /^[A-Za-z0-9_-]{43}$/; + +interface OidcTransactionCookieProfile { + transport: OidcTransactionTransport; + name: string; + secure: boolean; +} + +const OIDC_TRANSACTION_COOKIE_PROFILES: Readonly> = { + https: { transport: "https", name: "__Host-thothii_oidc_tx", secure: true }, + loopback_http: { transport: "loopback_http", name: "thothii_oidc_tx", secure: false }, +}; + +export interface AuthRouteDependencies { + authMode: "local" | "oidc" | "upstream" | "none" | "mock"; + authentication?: AuthenticationConfigProvider; + sessionStore?: AuthSessionStore; + /** Test-only compatibility seam; production resolves from each loaded config snapshot. */ + localUserRegistry?: LocalUserRegistry; + resolveLocalUserRegistry?: (loaded: LoadedAuthConfig) => LocalUserRegistry | undefined; + oidcProtocol?: OidcProtocol; + resolveOidcProtocol?: (loaded: LoadedAuthConfig) => OidcProtocol | undefined; +} + +interface LoginPayload { + username: string; + password: string; + remember: boolean; +} + +function countActiveAttempts( + bucket: ReadonlyMap, + key: string, + now: number, +): number { + const attempts = bucket.get(key); + if (!attempts) return 0; + const earliest = now - TEN_MINUTES_MS; + let count = 0; + for (const timestamp of attempts) { + if (timestamp > earliest) count += 1; + } + return count; +} + +function pruneExpiredAttempts(bucket: Map, now: number): void { + const earliest = now - TEN_MINUTES_MS; + for (const [key, attempts] of bucket) { + const active = attempts.filter((timestamp) => timestamp > earliest); + if (active.length === 0) bucket.delete(key); + else if (active.length !== attempts.length) bucket.set(key, active); + } +} + +function canRecordAttempt( + bucket: ReadonlyMap, + key: string, + limit: number, + maximumEntries: number, +): boolean { + return (bucket.get(key)?.length ?? 0) < limit + && (bucket.has(key) || bucket.size < maximumEntries); +} + +function appendAttempt(bucket: Map, key: string, now: number): void { + bucket.set(key, [...(bucket.get(key) ?? []), now]); +} + +export class LoginFailureLimiter { + private readonly usernames = new Map(); + private readonly addresses = new Map(); + private readonly maximumEntries: number; + + constructor(options: { maximumEntries?: number } = {}) { + this.maximumEntries = options.maximumEntries ?? MAX_LIMIT_ENTRIES; + } + + isLimited(username: string, address: string, now = Date.now()): boolean { + return countActiveAttempts(this.usernames, username, now) >= 10 + || countActiveAttempts(this.addresses, address, now) >= 20; + } + + recordFailure(username: string, address: string, now = Date.now()): boolean { + pruneExpiredAttempts(this.usernames, now); + pruneExpiredAttempts(this.addresses, now); + if (!canRecordAttempt(this.usernames, username, 10, this.maximumEntries) + || !canRecordAttempt(this.addresses, address, 20, this.maximumEntries)) return false; + appendAttempt(this.usernames, username, now); + appendAttempt(this.addresses, address, now); + return true; + } +} + +class OidcInitiationLimiter { + private readonly addresses = new Map(); + + consume(address: string, now = Date.now()): boolean { + pruneExpiredAttempts(this.addresses, now); + if (!canRecordAttempt( + this.addresses, + address, + MAX_OIDC_INITIATIONS_PER_ADDRESS, + MAX_LIMIT_ENTRIES, + )) return false; + appendAttempt(this.addresses, address, now); + return true; + } +} + +class VerificationGate { + private active = 0; + + async run(operation: () => Promise): Promise { + if (this.active >= 2) return undefined; + this.active += 1; + try { + return await operation(); + } finally { + this.active -= 1; + } + } +} + +async function unavailableAfterDummy( + gate: VerificationGate, + password: string, + reply: FastifyReply, +): Promise { + try { + const completed = await gate.run(async () => { + await verifyWithDummy(password); + return true; + }); + if (completed === undefined) return loginLimited(reply); + } catch { + // Preserve the sanitized operational outcome below. + } + return unavailable(reply); +} + +export function registerAuthRoutes(app: FastifyInstance, deps: AuthRouteDependencies): void { + const limiter = new LoginFailureLimiter(); + const oidcInitiationLimiter = new OidcInitiationLimiter(); + const verificationGate = new VerificationGate(); + + app.get("/auth/config", async (request, reply) => { + const snapshot = captureAuthConfigSnapshot(request, deps.authentication); + if (!snapshot) return unavailable(reply); + const mode = snapshot.value.mode; + return reply.send({ mode, localLogin: mode === "local", oidcLogin: mode === "oidc" }); + }); + + app.post("/auth/local/login", async (request, reply) => { + const snapshot = captureAuthConfigSnapshot(request, deps.authentication); + const configured = currentLocalConfig(snapshot, deps); + if (configured.kind === "unavailable") { + return unavailableAfterDummy(verificationGate, argon2SafePassword(loginPayload(request).password), reply); + } + if (configured.kind !== "local" || !deps.sessionStore) return unavailable(reply); + const originCheck = requireExactOrigin(request, reply, configured.origin); + if (originCheck !== true) return originCheck; + + const payload = loginPayload(request); + const safePassword = argon2SafePassword(payload.password); + const normalizedUsername = payload.username.replace(/[A-Z]/g, (character) => character.toLowerCase()); + const sourceAddress = boundedAddress(request.ip); + if (limiter.isLimited(normalizedUsername, sourceAddress)) return loginLimited(reply); + + let user: LocalUserRecord | undefined; + try { + if (payload.username.length > 0) user = await configured.registry.findByUsername(payload.username); + } catch { + return unavailableAfterDummy(verificationGate, safePassword, reply); + } + let verified: boolean | undefined; + try { + verified = await verificationGate.run(async () => + configured.registry.verify(user, safePassword)); + } catch { + return unavailable(reply); + } + if (verified === undefined) return loginLimited(reply); + if (!verified || !user || !user.enabled) { + if (!limiter.recordFailure(normalizedUsername, sourceAddress)) return loginLimited(reply); + return invalidCredentials(reply); + } + + try { + const created = await deps.sessionStore.create({ + principal: { + issuer: "local", + subject: user.id, + displayName: user.displayName ?? user.username, + roles: user.roles, + permissions: rolesToPermissions(user.roles), + isAdmin: user.roles.includes("admin"), + }, + method: "local", + remembered: payload.remember, + userAuthRevision: user.authRevision, + authConfigRevision: configured.revision, + idleTtlMs: (payload.remember ? configured.session.rememberIdleSeconds : configured.session.regularIdleSeconds) * 1000, + absoluteTtlMs: (payload.remember ? configured.session.rememberTtlSeconds : configured.session.regularTtlSeconds) * 1000, + }); + reply.setCookie(sessionCookieName(), created.token, cookieOptions(snapshot, payload.remember)); + return reply.send({}); + } catch { + return unavailable(reply); + } + }); + + app.get("/auth/oidc/login", async (request, reply) => { + if (!oidcInitiationLimiter.consume(boundedAddress(request.ip))) return loginLimited(reply); + const loaded = captureAuthConfigSnapshot(request, deps.authentication); + const configured = currentOidcConfig(loaded, deps); + const transactionProfile = configured === undefined + ? undefined + : oidcTransactionCookieProfile(configured.loaded); + if (!configured || !transactionProfile || !deps.sessionStore) { + clearOidcTransactionCookies(reply); + return unavailable(reply); + } + const nonce = randomOidcValue(); + const codeVerifier = randomOidcValue(); + const browserTransaction = randomOidcValue(); + try { + const created = await deps.sessionStore.createOidcState({ + nonce, + codeVerifier, + returnTo: "/", + authConfigRevision: configured.loaded.revision, + issuer: configured.config.oidc.issuer, + browserTransactionDigest: oidcTransactionDigest(browserTransaction).toString("hex"), + browserTransactionTransport: transactionProfile.transport, + }); + try { + const location = await configured.protocol.authorizationUrl({ state: created.state, nonce, codeVerifier }); + clearOidcTransactionCookie(reply, otherOidcTransactionCookieProfile(transactionProfile)); + reply.setCookie(transactionProfile.name, browserTransaction, oidcTransactionCookieOptions(transactionProfile)); + return reply.redirect(location.href); + } catch { + await deps.sessionStore.consumeOidcState(created.state).catch(() => undefined); + clearOidcTransactionCookies(reply, transactionProfile); + return unavailable(reply); + } + } catch (error) { + clearOidcTransactionCookies(reply, transactionProfile); + if (error instanceof OidcStateCapacityError) return loginLimited(reply); + return unavailable(reply); + } + }); + + app.get("/auth/oidc/callback", async (request, reply) => { + const loaded = captureAuthConfigSnapshot(request, deps.authentication); + const callback = oidcCallbackUrl(request, loaded?.value.publicUrl); + if (!deps.sessionStore || !callback.state) { + clearOidcTransactionCookies(reply); + return oidcCallbackFailed(reply); + } + let state: OidcStateRecord | undefined; + try { + state = await deps.sessionStore.consumeOidcState(callback.state); + } catch { + clearOidcTransactionCookies(reply); + return oidcCallbackFailed(reply); + } + const stateTransactionProfile = oidcTransactionCookieProfileForTransport(state?.browserTransactionTransport); + clearOidcTransactionCookies(reply, stateTransactionProfile); + const configured = currentOidcConfig(loaded, deps); + const configuredTransactionProfile = configured === undefined + ? undefined + : oidcTransactionCookieProfile(configured.loaded); + const transaction = readOidcTransactionCookie(request, stateTransactionProfile); + const transactionMatches = oidcTransactionMatches(transaction, state?.browserTransactionDigest ?? ""); + if (!callback.currentUrl || !configured || !configuredTransactionProfile || !state || !stateTransactionProfile + || state.returnTo !== "/" || !transactionMatches + || state.authConfigRevision !== configured.loaded.revision + || state.issuer !== configured.config.oidc.issuer + || stateTransactionProfile.transport !== configuredTransactionProfile.transport) { + return oidcCallbackFailed(reply); + } + try { + const identity = await configured.protocol.callback({ + currentUrl: callback.currentUrl, + state: callback.state, + nonce: state.nonce, + codeVerifier: state.codeVerifier, + }); + if (identity.issuer !== configured.config.oidc.issuer || !Array.isArray(identity.groups) + || identity.groups.length === 0) return oidcCallbackFailed(reply); + const roles = oidcRoles(identity.groups, configured.config); + const now = new Date(); + const absoluteTtlMs = Math.min( + configured.config.session.oidcTtlSeconds * 1000, + identity.tokenExpiresAt.getTime() - now.getTime(), + ); + if (!Number.isSafeInteger(absoluteTtlMs) || absoluteTtlMs <= 0) return oidcCallbackFailed(reply); + const created = await deps.sessionStore.create({ + principal: { + issuer: identity.issuer, + subject: identity.subject, + ...(identity.displayName === undefined ? {} : { displayName: identity.displayName }), + roles, + permissions: rolesToPermissions(roles), + isAdmin: roles.includes("admin"), + }, + method: "oidc", + remembered: false, + authConfigRevision: configured.loaded.revision, + idleTtlMs: configured.config.session.regularIdleSeconds * 1000, + absoluteTtlMs, + }, now); + reply.setCookie(sessionCookieName(), created.token, cookieOptions(configured.loaded, false)); + return reply.redirect(state.returnTo); + } catch { + return oidcCallbackFailed(reply); + } + }); + + app.post("/auth/logout", async (request, reply) => { + const token = request.authSessionToken; + if (!token || !deps.sessionStore) return unavailable(reply); + try { + await deps.sessionStore.revoke(token); + reply.clearCookie(sessionCookieName(), cookieOptions(request.authConfigSnapshot, false)); + return reply.code(204).send(); + } catch { + return unavailable(reply); + } + }); + + app.get("/me", async (request, reply) => { + const principal = requirePermission(request, reply, "session.use"); + if (!isPrincipalContext(principal)) return principal; + const session = request.authSession; + const token = request.authSessionToken; + if (!session || !token) { + return { + issuer: principal.issuer, + subject: principal.subject, + ...(principal.displayName === undefined ? {} : { displayName: principal.displayName }), + roles: principal.roles, + permissions: principal.permissions, + isAdmin: principal.isAdmin, + csrfToken: null, + session: null, + }; + } + try { + return { + issuer: principal.issuer, + subject: principal.subject, + ...(principal.displayName === undefined ? {} : { displayName: principal.displayName }), + roles: principal.roles, + permissions: principal.permissions, + isAdmin: principal.isAdmin, + csrfToken: deriveCsrfToken(token), + session: { + method: session.method, + remembered: session.remembered, + idleExpiresAt: session.idleExpiresAt, + absoluteExpiresAt: session.absoluteExpiresAt, + }, + }; + } catch { + return unavailable(reply); + } + }); +} + +function currentLocalConfig(loaded: LoadedAuthConfig | undefined, deps: AuthRouteDependencies): + | { + revision: string; + origin: string; + session: { regularTtlSeconds: number; regularIdleSeconds: number; rememberTtlSeconds: number; rememberIdleSeconds: number }; + registry: LocalUserRegistry; + kind: "local"; +} + | { kind: "not_local" } + | { kind: "unavailable" } { + try { + if (!loaded) return { kind: "unavailable" }; + if (loaded.value.mode !== "local") return { kind: "not_local" }; + const registry = deps.resolveLocalUserRegistry?.(loaded) ?? deps.localUserRegistry; + if (!registry) return { kind: "unavailable" }; + const url = new URL(loaded.value.publicUrl); + return { + kind: "local", + revision: loaded.revision, + origin: url.origin, + session: loaded.value.session, + registry, + }; + } catch { + return { kind: "unavailable" }; + } +} + +function cookieOptions(snapshot: LoadedAuthConfig | undefined, remembered: boolean) { + let secure = false; + try { + secure = snapshot !== undefined && new URL(snapshot.value.publicUrl).protocol === "https:"; + } catch { + // Invalid auth configurations are rejected before they can reach this route. + } + return { + httpOnly: true, + sameSite: "lax" as const, + path: "/", + secure, + ...(remembered ? { maxAge: REMEMBER_COOKIE_SECONDS } : {}), + }; +} + +function currentOidcConfig(loaded: LoadedAuthConfig | undefined, deps: AuthRouteDependencies): + | { loaded: LoadedAuthConfig; config: OidcAuthenticationConfig; protocol: OidcProtocol } + | undefined { + if (!loaded || loaded.value.mode !== "oidc") return undefined; + const protocol = deps.resolveOidcProtocol?.(loaded) ?? deps.oidcProtocol; + return protocol ? { loaded, config: loaded.value, protocol } : undefined; +} + +function randomOidcValue(): string { + return randomBytes(32).toString("base64url"); +} + +function oidcTransactionDigest(value: string): Buffer { + return createHash("sha256").update(value, "utf8").digest(); +} + +function oidcTransactionMatches(value: string | undefined, expectedDigest: string): boolean { + const canonical = typeof value === "string" && OIDC_VALUE_PATTERN.test(value); + const expectedCanonical = /^[a-f0-9]{64}$/.test(expectedDigest); + const supplied = oidcTransactionDigest(canonical ? value : ""); + const expected = expectedCanonical ? Buffer.from(expectedDigest, "hex") : Buffer.alloc(32); + const matches = timingSafeEqual(supplied, expected); + return canonical && expectedCanonical && matches; +} + +function oidcTransactionCookieProfile(loaded: LoadedAuthConfig): OidcTransactionCookieProfile | undefined { + try { + if (loaded.value.mode !== "oidc") return undefined; + const publicUrl = parseConfiguredTransportUrl(loaded.value.publicUrl, { + allowLoopbackHttp: true, + originOnly: true, + }); + if (!publicUrl) return undefined; + if (publicUrl.protocol === "https:") return OIDC_TRANSACTION_COOKIE_PROFILES.https; + if (publicUrl.protocol === "http:") return OIDC_TRANSACTION_COOKIE_PROFILES.loopback_http; + return undefined; + } catch { + return undefined; + } +} + +function oidcTransactionCookieProfileForTransport( + transport: OidcTransactionTransport | undefined, +): OidcTransactionCookieProfile | undefined { + return transport === "https" || transport === "loopback_http" + ? OIDC_TRANSACTION_COOKIE_PROFILES[transport] + : undefined; +} + +function otherOidcTransactionCookieProfile( + profile: OidcTransactionCookieProfile, +): OidcTransactionCookieProfile { + return profile.transport === "https" + ? OIDC_TRANSACTION_COOKIE_PROFILES.loopback_http + : OIDC_TRANSACTION_COOKIE_PROFILES.https; +} + +function oidcTransactionCookieOptions(profile: OidcTransactionCookieProfile) { + return { + httpOnly: true, + sameSite: "lax" as const, + path: "/", + secure: profile.secure, + maxAge: OIDC_TRANSACTION_COOKIE_SECONDS, + }; +} + +function clearOidcTransactionCookie(reply: FastifyReply, profile: OidcTransactionCookieProfile): void { + reply.clearCookie(profile.name, { + httpOnly: true, + sameSite: "lax", + path: "/", + secure: profile.secure, + }); +} + +function clearOidcTransactionCookies(reply: FastifyReply, preferred?: OidcTransactionCookieProfile): void { + if (preferred) { + clearOidcTransactionCookie(reply, preferred); + clearOidcTransactionCookie(reply, otherOidcTransactionCookieProfile(preferred)); + return; + } + clearOidcTransactionCookie(reply, OIDC_TRANSACTION_COOKIE_PROFILES.https); + clearOidcTransactionCookie(reply, OIDC_TRANSACTION_COOKIE_PROFILES.loopback_http); +} + +/** + * Browser parsers choose one duplicate cookie value differently. Parse just our two fixed names + * from the raw header and reject duplicates or a cross-transport sibling before hashing. + */ +function readOidcTransactionCookie( + request: FastifyRequest, + expected: OidcTransactionCookieProfile | undefined, +): string | undefined { + const raw = request.headers.cookie; + if (!expected || typeof raw !== "string" || raw.length > 4096 || Array.isArray(raw)) return undefined; + let transactionCookies = 0; + let expectedCookies = 0; + let expectedValue: string | undefined; + for (const part of raw.split(";")) { + const match = /^\s*(__Host-thothii_oidc_tx|thothii_oidc_tx)(?:=([^;]*))?\s*$/.exec(part); + if (!match) continue; + transactionCookies += 1; + if (match[1] !== expected.name) continue; + expectedCookies += 1; + expectedValue = match[2]; + } + return transactionCookies === 1 && expectedCookies === 1 ? expectedValue : undefined; +} + +function oidcCallbackUrl( + request: FastifyRequest, + publicUrl: string | undefined, +): { currentUrl?: URL; state?: string } { + if (request.url.length > MAX_OIDC_CALLBACK_QUERY_LENGTH) { + return { state: oversizedOidcCallbackState(request.url) }; + } + let supplied: URL; + try { + supplied = new URL(request.url, "http://callback.invalid"); + } catch { + return {}; + } + if (supplied.pathname !== "/auth/oidc/callback") return {}; + const allowed = new Set(["code", "state", "error", "error_description", "error_uri", "iss"]); + const copied = new URLSearchParams(); + let state: string | undefined; + let valid = true; + for (const [key, value] of supplied.searchParams) { + if (key === "state" && state === undefined && OIDC_VALUE_PATTERN.test(value)) state = value; + if (!allowed.has(key) || value.length > 2048 || /\p{Cc}/u.test(value) || copied.has(key)) { + valid = false; + continue; + } + copied.set(key, value); + } + if (!state || !valid || copied.get("state") !== state || publicUrl === undefined) return { state }; + let target: URL; + try { + target = new URL(OIDC_CALLBACK_PATH, publicUrl); + } catch { + return { state }; + } + target.search = copied.toString(); + return { currentUrl: target, state }; +} + +function oversizedOidcCallbackState(rawUrl: string): string | undefined { + const prefix = "/auth/oidc/callback?"; + if (!rawUrl.startsWith(prefix)) return undefined; + const boundedQuery = rawUrl.slice(prefix.length, MAX_OIDC_CALLBACK_QUERY_LENGTH); + let offset = 0; + while (offset < boundedQuery.length) { + const separator = boundedQuery.indexOf("&", offset); + const end = separator === -1 ? boundedQuery.length : separator; + const parameter = boundedQuery.slice(offset, end); + if (parameter.startsWith("state=")) { + const value = parameter.slice("state=".length); + if (OIDC_VALUE_PATTERN.test(value)) return value; + } + if (separator === -1) break; + offset = separator + 1; + } + return undefined; +} + +function oidcCallbackFailed(reply: FastifyReply) { + return reply.code(401).send({ code: "oidc_callback_failed", error: "OIDC sign-in could not be completed" }); +} + +function oidcRoles(groups: readonly string[], config: OidcAuthenticationConfig): Role[] { + const roles = new Set(); + for (const group of groups) { + for (const role of config.authorization.groupRoles[group] ?? []) roles.add(role); + } + return [...roles]; +} + +function loginPayload(request: FastifyRequest): LoginPayload { + const body = request.body; + if (!body || typeof body !== "object" || Array.isArray(body)) return { username: "", password: "", remember: false }; + const input = body as Record; + return { + username: typeof input.username === "string" && input.username.length <= MAX_USERNAME_LENGTH ? input.username : "", + password: typeof input.password === "string" && input.password.length <= MAX_PASSWORD_LENGTH ? input.password : "", + remember: input.remember === true, + }; +} + +function boundedAddress(address: string): string { + return typeof address === "string" && address.length > 0 && address.length <= 128 ? address : "unknown"; +} + +function argon2SafePassword(value: string): string { + const typed = value as string & { isWellFormed?: () => boolean }; + const wellFormed = typeof typed.isWellFormed === "function" + ? typed.isWellFormed() + : !/[\uD800-\uDFFF]/.test(value); + const bytes = Buffer.byteLength(value, "utf8"); + if (wellFormed && bytes >= 12 && bytes <= MAX_PASSWORD_LENGTH) return value; + // A per-attempt random value preserves the Argon2 work without turning an invalid input into + // a reusable password that could happen to match a user's configured secret. + return randomBytes(32).toString("base64url"); +} + +function invalidCredentials(reply: FastifyReply): FastifyReply { + return reply.code(401).send({ code: "invalid_credentials", error: "Invalid username or password" }); +} + +function loginLimited(reply: FastifyReply): FastifyReply { + return reply.code(429).send({ code: "login_rate_limited", error: "Too many login attempts" }); +} + +function unavailable(reply: FastifyReply): FastifyReply { + return reply.code(503).send({ code: "auth_unavailable", error: "Authentication is unavailable" }); +} diff --git a/backend/src/auth/secret-policy.ts b/backend/src/auth/secret-policy.ts new file mode 100644 index 00000000..e7d400d4 --- /dev/null +++ b/backend/src/auth/secret-policy.ts @@ -0,0 +1,19 @@ +export const AUTHENTICATION_SECRET_LIMITS = Object.freeze({ + THT_OIDC_CLIENT_SECRET: 4096, + THT_AUTHENTIK_API_TOKEN: 16 * 1024, +} as const); + +export type AuthenticationSecretReference = keyof typeof AUTHENTICATION_SECRET_LIMITS; + +export function isAuthenticationSecretReference(value: string): value is AuthenticationSecretReference { + return Object.prototype.hasOwnProperty.call(AUTHENTICATION_SECRET_LIMITS, value); +} + +/** One policy shared by bundle loading, runtime adapters, and static diagnostics. */ +export function isUsableAuthenticationSecret( + name: AuthenticationSecretReference, + value: unknown, +): value is string { + return typeof value === "string" && value.length > 0 + && value.length <= AUTHENTICATION_SECRET_LIMITS[name] && !/\p{Cc}/u.test(value); +} diff --git a/backend/src/auth/session-store.ts b/backend/src/auth/session-store.ts new file mode 100644 index 00000000..6e5fa08a --- /dev/null +++ b/backend/src/auth/session-store.ts @@ -0,0 +1,754 @@ +import { createHash, hkdfSync, randomBytes } from "node:crypto"; +import { z } from "zod"; +import type { PrincipalContext } from "./principal.js"; +import type { + AuthSessionRecord, + OidcStateRecord, + OidcTransactionTransport, + Permission, + Role, +} from "./types.js"; +import { + createPosixAuthStorageBridge, + createWindowsAuthStorageBridge, + type WindowsAuthStorageBridge, +} from "./windows-auth-storage.js"; + +const TOKEN_BYTES = 32; +const TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/; +const DIGEST_FILENAME_PATTERN = /^[a-f0-9]{64}\.json$/; +const CLAIM_FILENAME_PATTERN = /^[a-f0-9]{64}\.claim$/; +const OIDC_SLOT_FILENAME_PATTERN = /^slot-(\d{2})\.json$/; +const MAX_SESSION_RECORD_BYTES = 16 * 1024; +const MAX_OIDC_STATE_RECORD_BYTES = 8 * 1024; +const MAX_OIDC_SLOT_RECORD_BYTES = 512; +const MAX_TTL_MS = 365 * 24 * 60 * 60 * 1000; +const OIDC_STATE_TTL_MS = 10 * 60 * 1000; +const OIDC_STATE_CAPACITY = 64; +const MAX_OIDC_STORAGE_ENTRIES = OIDC_STATE_CAPACITY * 3; +const MAX_SESSION_PRUNE_ENTRIES = 512; +const TOUCH_INTERVAL_MS = 5 * 60 * 1000; +const CSRF_CONTEXT = Buffer.from("thothii-csrf-v1", "utf8"); +const EMPTY_HKDF_SALT = Buffer.alloc(0); +const ROLES = ["user", "admin"] as const; +const PERMISSIONS = [ + "session.use", "session.read_all", "session.manage_all", "settings.manage", + "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", +] as const satisfies readonly Permission[]; + +const invalid = (): Error => new Error("auth_session_store_invalid"); + +export interface SessionCreateInput { + principal: PrincipalContext; + method: "local" | "oidc" | "upstream"; + remembered: boolean; + userAuthRevision?: number; + authConfigRevision: string; + idleTtlMs: number; + absoluteTtlMs: number; +} + +export interface CreatedAuthSession { + token: string; + csrfToken: string; + record: AuthSessionRecord; +} + +export interface OidcStateCreateInput { + nonce: string; + codeVerifier: string; + returnTo: "/"; + authConfigRevision: string; + issuer: string; + browserTransactionDigest: string; + browserTransactionTransport: OidcTransactionTransport; +} + +export interface CreatedOidcState { + state: string; + record: OidcStateRecord; +} + +export interface LocalSessionUser { + enabled: boolean; + authRevision: number; + roles: readonly Role[]; +} + +export interface CurrentLocalSessionUser { + revision: string; + user: LocalSessionUser | undefined; +} + +/** Operational validity-source failures must not masquerade as revoked credentials. */ +export class AuthSessionOperationalError extends Error { + constructor() { + super("auth_session_operational_error"); + } +} + +export class OidcStateCapacityError extends Error { + constructor() { + super("auth_oidc_state_capacity"); + } +} + +/** + * The route layer supplies the current installation revision and local-registry lookup. + * Supplying this hook makes every resolve an authorization-generation check. + */ +export interface AuthSessionValidity { + currentAuthConfigRevision(): string | Promise; + findLocalUser?(subject: string): LocalSessionUser | undefined | Promise; + currentLocalUser?(subject: string): CurrentLocalSessionUser | Promise; +} + +export interface AuthSessionStore { + create(input: SessionCreateInput, now?: Date): Promise; + resolve(token: string, now?: Date, validity?: AuthSessionValidity): Promise; + touch(token: string, now?: Date): Promise; + revoke(token: string): Promise; + prune(now?: Date): Promise; + createOidcState(input: OidcStateCreateInput, now?: Date): Promise; + consumeOidcState(state: string, now?: Date): Promise; +} + +/** Narrow test seams for the native tht-backed storage adaptors. */ +export interface FileAuthSessionStoreOptions { + windowsStorageBridge?: WindowsAuthStorageBridge; + /** Test seam; production uses the bounded hidden tht bridge for every POSIX record operation. */ + posixStorageBridge?: WindowsAuthStorageBridge; + /** Test-only capacity seam; production always uses the fixed 64-state bound. */ + oidcStateCapacity?: number; +} + +interface SessionDirectoryPage { + entries: string[]; + more: boolean; +} + +const text = z.string().min(1).max(512).refine((value) => !/[\u0000-\u001f\u007f]/.test(value)); +const timestamp = z.string().length(24).refine((value) => { + const parsed = Date.parse(value); + return Number.isFinite(parsed) && new Date(parsed).toISOString() === value; +}); +const role = z.enum(ROLES); +const permission = z.enum(PERMISSIONS); +const distinct = (items: readonly T[]): boolean => new Set(items).size === items.length; +const sessionRecordSchema = z.strictObject({ + version: z.literal(1), + issuer: text, + subject: text, + displayName: text.optional(), + method: z.enum(["local", "oidc", "upstream"]), + roles: z.array(role).max(ROLES.length).refine(distinct), + permissions: z.array(permission).max(PERMISSIONS.length).refine(distinct), + userAuthRevision: z.number().int().positive().safe().optional(), + authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/), + remembered: z.boolean(), + createdAt: timestamp, + lastSeenAt: timestamp, + idleExpiresAt: timestamp, + absoluteExpiresAt: timestamp, +}).superRefine((record, context) => { + const createdAt = Date.parse(record.createdAt); + const lastSeenAt = Date.parse(record.lastSeenAt); + const idleExpiresAt = Date.parse(record.idleExpiresAt); + const absoluteExpiresAt = Date.parse(record.absoluteExpiresAt); + if (lastSeenAt < createdAt || idleExpiresAt < lastSeenAt || idleExpiresAt > absoluteExpiresAt + || absoluteExpiresAt < createdAt || absoluteExpiresAt - createdAt > MAX_TTL_MS) { + context.addIssue({ code: "custom", message: "invalid session lifetime" }); + } + if (record.method === "local" && record.userAuthRevision === undefined) { + context.addIssue({ code: "custom", message: "local revision is required" }); + } + if (record.method !== "local" && record.userAuthRevision !== undefined) { + context.addIssue({ code: "custom", message: "non-local revision is forbidden" }); + } +}); +const oidcStateRecordSchema = z.strictObject({ + version: z.literal(1), + nonce: z.string().min(16).max(512).regex(/^[A-Za-z0-9_-]+$/), + codeVerifier: z.string().min(43).max(128).regex(/^[A-Za-z0-9._~-]+$/), + returnTo: z.literal("/"), + authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/), + issuer: z.string().min(1).max(2048).refine((value) => !/\p{Cc}/u.test(value)), + browserTransactionDigest: z.string().regex(/^[a-f0-9]{64}$/), + // Existing ten-minute records from before this field was introduced can be consumed and + // rejected by the route. New records always receive the required input field below. + browserTransactionTransport: z.enum(["https", "loopback_http"]).optional(), + capacitySlot: z.number().int().min(0).max(OIDC_STATE_CAPACITY - 1).optional(), + createdAt: timestamp, + expiresAt: timestamp, +}).superRefine((record, context) => { + const lifetime = Date.parse(record.expiresAt) - Date.parse(record.createdAt); + if (lifetime <= 0 || lifetime > OIDC_STATE_TTL_MS) { + context.addIssue({ code: "custom", message: "invalid OIDC state lifetime" }); + } +}); +const oidcSlotRecordSchema = z.strictObject({ + version: z.literal(1), + stateFilename: z.string().regex(DIGEST_FILENAME_PATTERN), + expiresAt: timestamp, +}); +const sessionInputSchema = z.strictObject({ + principal: z.strictObject({ + issuer: text, + subject: text, + displayName: text.optional(), + roles: z.array(role).max(ROLES.length).refine(distinct), + permissions: z.array(permission).max(PERMISSIONS.length).refine(distinct), + isAdmin: z.boolean(), + }), + method: z.enum(["local", "oidc", "upstream"]), + remembered: z.boolean(), + userAuthRevision: z.number().int().positive().safe().optional(), + authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/), + idleTtlMs: z.number().int().min(1).max(MAX_TTL_MS), + absoluteTtlMs: z.number().int().min(1).max(MAX_TTL_MS), +}).superRefine((input, context) => { + if (input.principal.isAdmin !== input.principal.roles.includes("admin")) { + context.addIssue({ code: "custom", message: "principal roles disagree" }); + } + if (input.method === "local" && input.userAuthRevision === undefined) { + context.addIssue({ code: "custom", message: "local revision is required" }); + } + if (input.method !== "local" && input.userAuthRevision !== undefined) { + context.addIssue({ code: "custom", message: "non-local revision is forbidden" }); + } +}); +const oidcStateInputSchema = z.strictObject({ + nonce: z.string().min(16).max(512).regex(/^[A-Za-z0-9_-]+$/), + codeVerifier: z.string().min(43).max(128).regex(/^[A-Za-z0-9._~-]+$/), + returnTo: z.literal("/"), + authConfigRevision: z.string().regex(/^[a-f0-9]{64}$/), + issuer: z.string().min(1).max(2048).refine((value) => !/\p{Cc}/u.test(value)), + browserTransactionDigest: z.string().regex(/^[a-f0-9]{64}$/), + browserTransactionTransport: z.enum(["https", "loopback_http"]), +}); + +function canonicalRawValue(value: string): boolean { + if (typeof value !== "string" || !TOKEN_PATTERN.test(value)) return false; + try { + const bytes = Buffer.from(value, "base64url"); + return bytes.length === TOKEN_BYTES && bytes.toString("base64url") === value; + } catch { + return false; + } +} + +function digestFilename(rawValue: string): string { + return `${createHash("sha256").update(rawValue).digest("hex")}.json`; +} + +function claimFilename(filename: string): string { + if (!DIGEST_FILENAME_PATTERN.test(filename)) throw invalid(); + return filename.slice(0, -".json".length) + ".claim"; +} + +function oidcSlotFilename(index: number): string { + if (!Number.isInteger(index) || index < 0 || index >= OIDC_STATE_CAPACITY) throw invalid(); + return `slot-${String(index).padStart(2, "0")}.json`; +} + +function oidcSlotIndex(filename: string): number | undefined { + const match = OIDC_SLOT_FILENAME_PATTERN.exec(filename); + if (!match) return undefined; + const index = Number(match[1]); + return Number.isInteger(index) && index >= 0 && index < OIDC_STATE_CAPACITY ? index : undefined; +} + +function parseSessionRecord(source: string): AuthSessionRecord { + try { + return sessionRecordSchema.parse(JSON.parse(source)) as AuthSessionRecord; + } catch { + throw invalid(); + } +} + +function parseOidcStateRecord(source: string): OidcStateRecord { + try { + return oidcStateRecordSchema.parse(JSON.parse(source)) as OidcStateRecord; + } catch { + throw invalid(); + } +} + +type OidcSlotRecord = z.infer; + +function parseOidcSlotRecord(source: string): OidcSlotRecord { + try { + return oidcSlotRecordSchema.parse(JSON.parse(source)); + } catch { + throw invalid(); + } +} + +function parseWindowsRecord(contents: Buffer, maximumBytes: number, parse: (source: string) => T): T { + if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > maximumBytes) throw invalid(); + try { + return parse(new TextDecoder("utf-8", { fatal: true }).decode(contents)); + } catch { + throw invalid(); + } +} + +interface StoredOidcSlot { + filename: string; + index: number; + record: OidcSlotRecord; +} + +function serialize(record: AuthSessionRecord | OidcStateRecord | OidcSlotRecord, maximumBytes: number): Buffer { + const contents = Buffer.from(`${JSON.stringify(record)}\n`, "utf8"); + if (contents.length > maximumBytes) throw invalid(); + return contents; +} + +function dateMilliseconds(now: Date): number { + if (!(now instanceof Date) || !Number.isFinite(now.getTime())) throw invalid(); + return now.getTime(); +} + +function isoAt(milliseconds: number): string { + if (!Number.isSafeInteger(milliseconds) || !Number.isFinite(milliseconds)) throw invalid(); + try { + return new Date(milliseconds).toISOString(); + } catch { + throw invalid(); + } +} + +function sessionExpired(record: AuthSessionRecord, nowMs: number): boolean { + return nowMs >= Date.parse(record.idleExpiresAt) || nowMs >= Date.parse(record.absoluteExpiresAt); +} + +function oidcStateExpired(record: OidcStateRecord, nowMs: number): boolean { + return nowMs >= Date.parse(record.expiresAt); +} + +function equalRoleSets(left: readonly Role[], right: readonly Role[]): boolean { + if (!distinct(left) || !distinct(right) || left.length !== right.length) return false; + if (!left.every((value) => ROLES.includes(value)) || !right.every((value) => ROLES.includes(value))) return false; + return [...left].sort().every((value, index) => value === [...right].sort()[index]); +} + +function validLocalUser(user: LocalSessionUser | undefined, record: AuthSessionRecord): boolean { + return user !== undefined && user.enabled === true && Number.isSafeInteger(user.authRevision) + && user.authRevision > 0 && user.authRevision === record.userAuthRevision + && equalRoleSets(user.roles, record.roles); +} + +async function recordIsCurrent(record: AuthSessionRecord, validity: AuthSessionValidity | undefined): Promise { + // A root-only store remains useful for creation/diagnostics, but is intentionally incapable + // of authenticating a principal. Task 8 must supply config and local-registry dependencies. + if (!validity) return false; + if (record.method === "local" && validity.currentLocalUser) { + const current = await validity.currentLocalUser(record.subject); + return typeof current.revision === "string" && current.revision === record.authConfigRevision + && validLocalUser(current.user, record); + } + const revision = await validity.currentAuthConfigRevision(); + if (typeof revision !== "string" || revision !== record.authConfigRevision) return false; + if (record.method !== "local") return true; + return validity.findLocalUser === undefined ? false : validLocalUser(await validity.findLocalUser(record.subject), record); +} + +const locks = new Map>(); + +async function withLock(key: string, operation: () => Promise): Promise { + const previous = locks.get(key) ?? Promise.resolve(); + let release: (() => void) | undefined; + const current = new Promise((resolve) => { release = resolve; }); + locks.set(key, current); + await previous; + try { + return await operation(); + } finally { + release?.(); + if (locks.get(key) === current) locks.delete(key); + } +} + +function lockKey(root: string, directory: "sessions" | "oidc", filename: string): string { + return `${root}\0${directory}\0${filename}`; +} + +/** Derive a one-way, domain-separated 256-bit CSRF value without persisting it. */ +export function deriveCsrfToken(sessionToken: string): string { + if (!canonicalRawValue(sessionToken)) throw invalid(); + try { + const sessionBytes = Buffer.from(sessionToken, "base64url"); + return Buffer.from(hkdfSync("sha256", sessionBytes, EMPTY_HKDF_SALT, CSRF_CONTEXT, TOKEN_BYTES)) + .toString("base64url"); + } catch { + throw invalid(); + } +} + +export function createFileAuthSessionStore( + root: string, + validity?: AuthSessionValidity, + options: FileAuthSessionStoreOptions = {}, +): AuthSessionStore { + const oidcStateCapacity = options.oidcStateCapacity ?? OIDC_STATE_CAPACITY; + if (!Number.isInteger(oidcStateCapacity) || oidcStateCapacity < 1 || oidcStateCapacity > OIDC_STATE_CAPACITY) { + throw invalid(); + } + const rawStorage = process.platform === "win32" + ? options.windowsStorageBridge ?? createWindowsAuthStorageBridge() + : options.posixStorageBridge ?? createPosixAuthStorageBridge(); + // A malformed or failed helper must be indistinguishable from any other storage failure to + // callers. This also keeps narrow test seams from accidentally exposing transport details. + const storage: WindowsAuthStorageBridge = { + validateRoot: async (value) => { try { await rawStorage.validateRoot(value); } catch { throw invalid(); } }, + ensureLayout: async (value) => { try { await rawStorage.ensureLayout(value); } catch { throw invalid(); } }, + readAuthConfig: (value) => { try { return rawStorage.readAuthConfig(value); } catch { throw invalid(); } }, + readLocalUsers: async (value) => { try { return await rawStorage.readLocalUsers(value); } catch { throw invalid(); } }, + create: async (...args) => { try { return await rawStorage.create(...args); } catch { throw invalid(); } }, + read: async (...args) => { try { return await rawStorage.read(...args); } catch { throw invalid(); } }, + replace: async (...args) => { try { await rawStorage.replace(...args); } catch { throw invalid(); } }, + remove: async (...args) => { try { return await rawStorage.remove(...args); } catch { throw invalid(); } }, + list: async (...args) => { try { return await rawStorage.list(...args); } catch { throw invalid(); } }, + listPage: async (...args) => { try { return await rawStorage.listPage(...args); } catch { throw invalid(); } }, + claimConsume: async (...args) => { try { return await rawStorage.claimConsume(...args); } catch { throw invalid(); } }, + readClaim: async (...args) => { try { return await rawStorage.readClaim(...args); } catch { throw invalid(); } }, + removeClaim: async (...args) => { try { return await rawStorage.removeClaim(...args); } catch { throw invalid(); } }, + }; + let sessionPruneCursor: string | undefined; + + function requiredStorage(): WindowsAuthStorageBridge { + if (!storage) throw invalid(); + return storage; + } + + async function ordinarySessionPage(after: string | undefined): Promise { + const page = await requiredStorage().listPage(root, "sessions", after, MAX_SESSION_PRUNE_ENTRIES); + if (!page || !Array.isArray(page.entries) || typeof page.more !== "boolean") throw invalid(); + return { entries: page.entries.map((entry) => entry.name), more: page.more }; + } + + function nextSessionPruneCursor(page: SessionDirectoryPage, after: string | undefined): string | undefined { + if (!Array.isArray(page.entries) || typeof page.more !== "boolean" + || page.entries.length > MAX_SESSION_PRUNE_ENTRIES) throw invalid(); + const seen = new Set(); + let previous = after; + for (const filename of page.entries) { + if (!DIGEST_FILENAME_PATTERN.test(filename) || seen.has(filename) + || (previous !== undefined && filename <= previous)) throw invalid(); + seen.add(filename); + previous = filename; + } + if (!page.more) return undefined; + if (page.entries.length !== MAX_SESSION_PRUNE_ENTRIES || previous === undefined || previous === after) throw invalid(); + return previous; + } + + async function pruneOrdinarySessions(nowMs: number): Promise { + const after = sessionPruneCursor; + const page = await ordinarySessionPage(after); + const next = nextSessionPruneCursor(page, after); + let removed = 0; + const bridge = requiredStorage(); + for (const filename of page.entries) { + const contents = await bridge.read(root, "sessions", filename); + if (!contents) continue; + const record = parseWindowsRecord(contents, MAX_SESSION_RECORD_BYTES, parseSessionRecord); + if (sessionExpired(record, nowMs) && await bridge.remove(root, "sessions", filename)) removed += 1; + } + sessionPruneCursor = next; + return removed; + } + + async function oidcStorageEntries(): Promise { + const entries = (await requiredStorage().list(root, "oidc", MAX_OIDC_STORAGE_ENTRIES)).map((entry) => entry.name); + if (entries.length > MAX_OIDC_STORAGE_ENTRIES) throw invalid(); + if (entries.some((entry) => !DIGEST_FILENAME_PATTERN.test(entry) + && !CLAIM_FILENAME_PATTERN.test(entry) && oidcSlotIndex(entry) === undefined)) throw invalid(); + return entries; + } + + async function storedOidcSlots(suppliedEntries?: string[]): Promise { + const entries = suppliedEntries ?? await oidcStorageEntries(); + const slots: StoredOidcSlot[] = []; + const stateFilenames = new Set(); + for (const filename of entries) { + const index = oidcSlotIndex(filename); + if (index === undefined) continue; + const contents = await requiredStorage().read(root, "oidc", filename); + if (!contents) continue; + const record = parseWindowsRecord(contents, MAX_OIDC_SLOT_RECORD_BYTES, parseOidcSlotRecord); + if (stateFilenames.has(record.stateFilename)) throw invalid(); + stateFilenames.add(record.stateFilename); + slots.push({ filename, index, record }); + } + return slots; + } + + async function removeOidcSlot(slot: StoredOidcSlot): Promise { + const current = await requiredStorage().read(root, "oidc", slot.filename); + if (!current) return; + const record = parseWindowsRecord(current, MAX_OIDC_SLOT_RECORD_BYTES, parseOidcSlotRecord); + if (record.stateFilename !== slot.record.stateFilename || record.expiresAt !== slot.record.expiresAt + || !await requiredStorage().remove(root, "oidc", slot.filename)) throw invalid(); + } + + async function releaseOidcSlot(index: number | undefined, stateFilename: string): Promise { + if (index === undefined) return; + const filename = oidcSlotFilename(index); + const slot = (await storedOidcSlots([filename]))[0]; + if (!slot || slot.record.stateFilename !== stateFilename) throw invalid(); + await removeOidcSlot(slot); + } + + async function reserveOidcSlot(stateFilename: string, expiresAt: string): Promise { + const entries = await oidcStorageEntries(); + const slots = await storedOidcSlots(entries); + const representedStates = new Set(slots.map((slot) => slot.record.stateFilename)); + const legacyStates = new Set(); + for (const entry of entries) { + const filename = CLAIM_FILENAME_PATTERN.test(entry) + ? `${entry.slice(0, -".claim".length)}.json` + : entry; + if (DIGEST_FILENAME_PATTERN.test(filename) && !representedStates.has(filename)) legacyStates.add(filename); + } + const availableSlotCount = oidcStateCapacity - legacyStates.size; + if (availableSlotCount <= 0) throw new OidcStateCapacityError(); + const occupied = new Set(slots.map((slot) => slot.index)); + const record: OidcSlotRecord = { version: 1, stateFilename, expiresAt }; + const contents = serialize(record, MAX_OIDC_SLOT_RECORD_BYTES); + for (let index = 0; index < availableSlotCount; index += 1) { + if (occupied.has(index)) continue; + const filename = oidcSlotFilename(index); + const created = await requiredStorage().create(root, "oidc", filename, contents); + if (created) return index; + } + throw new OidcStateCapacityError(); + } + + async function createSession(input: SessionCreateInput, now = new Date()): Promise { + const nowMs = dateMilliseconds(now); + let validated: z.infer; + try { + validated = sessionInputSchema.parse(input); + } catch { + throw invalid(); + } + const absoluteExpiresMs = nowMs + validated.absoluteTtlMs; + const idleExpiresMs = Math.min(nowMs + validated.idleTtlMs, absoluteExpiresMs); + if (!Number.isSafeInteger(absoluteExpiresMs) || !Number.isSafeInteger(idleExpiresMs)) throw invalid(); + const record: AuthSessionRecord = { + version: 1, + issuer: validated.principal.issuer, + subject: validated.principal.subject, + ...(validated.principal.displayName === undefined ? {} : { displayName: validated.principal.displayName }), + method: validated.method, + roles: [...validated.principal.roles], + permissions: [...validated.principal.permissions], + ...(validated.userAuthRevision === undefined ? {} : { userAuthRevision: validated.userAuthRevision }), + authConfigRevision: validated.authConfigRevision, + remembered: validated.remembered, + createdAt: isoAt(nowMs), + lastSeenAt: isoAt(nowMs), + idleExpiresAt: isoAt(idleExpiresMs), + absoluteExpiresAt: isoAt(absoluteExpiresMs), + }; + const contents = serialize(record, MAX_SESSION_RECORD_BYTES); + const bridge = requiredStorage(); + for (let attempt = 0; attempt < 8; attempt += 1) { + const token = randomBytes(TOKEN_BYTES).toString("base64url"); + const filename = digestFilename(token); + if (await bridge.create(root, "sessions", filename, contents)) { + return { token, csrfToken: deriveCsrfToken(token), record }; + } + } + throw invalid(); + } + + async function resolveSession( + token: string, + now = new Date(), + requestValidity = validity, + ): Promise { + if (!canonicalRawValue(token)) return undefined; + const nowMs = dateMilliseconds(now); + const filename = digestFilename(token); + return withLock(lockKey(root, "sessions", filename), async () => { + const bridge = requiredStorage(); + const contents = await bridge.read(root, "sessions", filename); + if (!contents) return undefined; + const record = parseWindowsRecord(contents, MAX_SESSION_RECORD_BYTES, parseSessionRecord); + if (sessionExpired(record, nowMs)) { + await bridge.remove(root, "sessions", filename); + return undefined; + } + try { + if (await recordIsCurrent(record, requestValidity)) return record; + } catch (error) { + if (error instanceof AuthSessionOperationalError) throw error; + await bridge.remove(root, "sessions", filename); + throw invalid(); + } + await bridge.remove(root, "sessions", filename); + return undefined; + }); + } + + async function touchSession(token: string, now = new Date()): Promise { + if (!canonicalRawValue(token)) return; + const nowMs = dateMilliseconds(now); + const filename = digestFilename(token); + await withLock(lockKey(root, "sessions", filename), async () => { + const bridge = requiredStorage(); + const contents = await bridge.read(root, "sessions", filename); + if (!contents) return; + const record = parseWindowsRecord(contents, MAX_SESSION_RECORD_BYTES, parseSessionRecord); + if (sessionExpired(record, nowMs)) { + await bridge.remove(root, "sessions", filename); + return undefined; + } + const lastSeenMs = Date.parse(record.lastSeenAt); + if (nowMs <= lastSeenMs || nowMs - lastSeenMs < TOUCH_INTERVAL_MS) return; + const idleWindowMs = Date.parse(record.idleExpiresAt) - lastSeenMs; + if (idleWindowMs <= 0 || idleWindowMs > MAX_TTL_MS) throw invalid(); + const touched: AuthSessionRecord = { + ...record, + lastSeenAt: isoAt(nowMs), + idleExpiresAt: isoAt(Math.min(nowMs + idleWindowMs, Date.parse(record.absoluteExpiresAt))), + }; + await bridge.replace(root, "sessions", filename, serialize(touched, MAX_SESSION_RECORD_BYTES)); + }); + } + + async function revokeSession(token: string): Promise { + if (!canonicalRawValue(token)) return; + const filename = digestFilename(token); + await withLock(lockKey(root, "sessions", filename), async () => { + await requiredStorage().remove(root, "sessions", filename); + }); + } + + async function pruneOidcStates(nowMs: number): Promise { + const bridge = requiredStorage(); + const oidcEntries = await bridge.list(root, "oidc", MAX_OIDC_STORAGE_ENTRIES); + if (oidcEntries.length > MAX_OIDC_STORAGE_ENTRIES) throw invalid(); + const stateNames = new Set(oidcEntries + .filter((entry) => DIGEST_FILENAME_PATTERN.test(entry.name)) + .map((entry) => entry.name)); + const claimEntries = new Map(oidcEntries + .filter((entry) => CLAIM_FILENAME_PATTERN.test(entry.name)) + .map((entry) => [entry.name, entry])); + const slotEntries = oidcEntries.filter((entry) => oidcSlotIndex(entry.name) !== undefined); + if (stateNames.size + claimEntries.size + slotEntries.length !== oidcEntries.length) throw invalid(); + let removed = 0; + for (const filename of stateNames) { + const claim = claimFilename(filename); + const contents = claimEntries.has(claim) + ? await bridge.readClaim(root, filename) + : await bridge.read(root, "oidc", filename); + if (!contents) continue; + const record = parseWindowsRecord(contents, MAX_OIDC_STATE_RECORD_BYTES, parseOidcStateRecord); + if (oidcStateExpired(record, nowMs)) { + const didRemove = claimEntries.has(claim) + ? await bridge.removeClaim(root, filename) + : await bridge.remove(root, "oidc", filename); + if (didRemove) removed += 1; + } + } + for (const [claim, entry] of claimEntries) { + const filename = `${claim.slice(0, -".claim".length)}.json`; + if (stateNames.has(filename)) continue; + if (nowMs >= entry.modifiedUnixMs + OIDC_STATE_TTL_MS + && await bridge.remove(root, "oidc", claim)) removed += 1; + } + for (const entry of slotEntries) { + const contents = await bridge.read(root, "oidc", entry.name); + if (!contents) continue; + const slot = parseWindowsRecord(contents, MAX_OIDC_SLOT_RECORD_BYTES, parseOidcSlotRecord); + if (nowMs < Date.parse(slot.expiresAt)) continue; + const claim = claimFilename(slot.stateFilename); + const stateContents = claimEntries.has(claim) + ? await bridge.readClaim(root, slot.stateFilename) + : await bridge.read(root, "oidc", slot.stateFilename); + if (stateContents) { + const state = parseWindowsRecord(stateContents, MAX_OIDC_STATE_RECORD_BYTES, parseOidcStateRecord); + if (!oidcStateExpired(state, nowMs)) throw invalid(); + const didRemove = claimEntries.has(claim) + ? await bridge.removeClaim(root, slot.stateFilename) + : await bridge.remove(root, "oidc", slot.stateFilename); + if (didRemove) removed += 1; + } + if (!await bridge.remove(root, "oidc", entry.name)) throw invalid(); + } + return removed; + } + + async function createOidcState(input: OidcStateCreateInput, now = new Date()): Promise { + const nowMs = dateMilliseconds(now); + let validated: z.infer; + try { + validated = oidcStateInputSchema.parse(input); + } catch { + throw invalid(); + } + const expiresMs = nowMs + OIDC_STATE_TTL_MS; + if (!Number.isSafeInteger(expiresMs)) throw invalid(); + return withLock(lockKey(root, "oidc", "capacity"), async () => { + await pruneOidcStates(nowMs); + for (let attempt = 0; attempt < 8; attempt += 1) { + const state = randomBytes(TOKEN_BYTES).toString("base64url"); + const filename = digestFilename(state); + const capacitySlot = await reserveOidcSlot(filename, isoAt(expiresMs)); + const record: OidcStateRecord = { + version: 1, + nonce: validated.nonce, + codeVerifier: validated.codeVerifier, + returnTo: validated.returnTo, + authConfigRevision: validated.authConfigRevision, + issuer: validated.issuer, + browserTransactionDigest: validated.browserTransactionDigest, + browserTransactionTransport: validated.browserTransactionTransport, + capacitySlot, + createdAt: isoAt(nowMs), + expiresAt: isoAt(expiresMs), + }; + const contents = serialize(record, MAX_OIDC_STATE_RECORD_BYTES); + const created = await requiredStorage().create(root, "oidc", filename, contents); + if (created) return { state, record }; + await releaseOidcSlot(capacitySlot, filename); + } + throw invalid(); + }); + } + + async function consumeOidcState(state: string, now = new Date()): Promise { + if (!canonicalRawValue(state)) return undefined; + const nowMs = dateMilliseconds(now); + const filename = digestFilename(state); + return withLock(lockKey(root, "oidc", filename), async () => { + const contents = await requiredStorage().claimConsume(root, filename); + if (!contents) return undefined; + const record = parseWindowsRecord(contents, MAX_OIDC_STATE_RECORD_BYTES, parseOidcStateRecord); + await releaseOidcSlot(record.capacitySlot, filename); + return oidcStateExpired(record, nowMs) ? undefined : record; + }); + } + + async function prune(now = new Date()): Promise { + const nowMs = dateMilliseconds(now); + // Cursor advancement is process-local, so concurrent timer/manual invocations must not + // observe the same page and strand a later page forever. + return await withLock(lockKey(root, "sessions", "maintenance"), async () => + (await pruneOrdinarySessions(nowMs)) + (await pruneOidcStates(nowMs))); + } + + return { + create: createSession, + resolve: resolveSession, + touch: touchSession, + revoke: revokeSession, + prune, + createOidcState, + consumeOidcState, + }; +} diff --git a/backend/src/auth/types.ts b/backend/src/auth/types.ts new file mode 100644 index 00000000..88292315 --- /dev/null +++ b/backend/src/auth/types.ts @@ -0,0 +1,93 @@ +export type AuthMode = "local" | "oidc" | "upstream" | "none" | "mock"; + +export type Role = "user" | "admin"; + +export type Permission = + | "session.use" | "session.read_all" | "session.manage_all" + | "settings.manage" | "workspace.manage" | "workspace.secrets.manage" + | "pi.manage" | "auth.diagnostics.read"; + +export interface AuthenticationSessionConfig { + regularTtlSeconds: number; + regularIdleSeconds: number; + rememberTtlSeconds: number; + rememberIdleSeconds: number; + oidcTtlSeconds: number; +} + +export interface LocalAuthenticationConfig { + version: 1; + mode: "local"; + publicUrl: string; + session: AuthenticationSessionConfig; + local: { usersFile: string }; +} + +export interface OidcAuthenticationConfig { + version: 1; + mode: "oidc"; + publicUrl: string; + session: AuthenticationSessionConfig; + oidc: { + issuer: string; + clientId: string; + clientSecretRef: "THT_OIDC_CLIENT_SECRET"; + scopes: readonly string[]; + groupsClaim: "groups"; + }; + groupCatalog: { + driver: "authentik"; + baseUrl: string; + apiTokenRef: "THT_AUTHENTIK_API_TOKEN"; + }; + authorization: { groupRoles: Readonly> }; +} + +export type AuthenticationConfig = LocalAuthenticationConfig | OidcAuthenticationConfig; + +export interface LoadedAuthConfig { + value: AuthenticationConfig; + revision: string; + sourcePath: string; +} + +export interface AuthenticationConfigProvider { + current(): LoadedAuthConfig; +} + +/** The only browser transport modes accepted for an OIDC transaction cookie. */ +export type OidcTransactionTransport = "https" | "loopback_http"; + +/** Durable, server-side representation of an opaque browser session. */ +export interface AuthSessionRecord { + version: 1; + issuer: string; + subject: string; + displayName?: string; + method: "local" | "oidc" | "upstream"; + roles: readonly Role[]; + permissions: readonly Permission[]; + userAuthRevision?: number; + authConfigRevision: string; + remembered: boolean; + createdAt: string; + lastSeenAt: string; + idleExpiresAt: string; + absoluteExpiresAt: string; +} + +/** Server-side OIDC callback material keyed by a separately generated opaque state value. */ +export interface OidcStateRecord { + version: 1; + nonce: string; + codeVerifier: string; + returnTo: "/"; + authConfigRevision: string; + issuer: string; + browserTransactionDigest: string; + /** Optional only to safely consume and reject a short-lived pre-transport legacy state. */ + browserTransactionTransport?: OidcTransactionTransport; + capacitySlot?: number; + createdAt: string; + expiresAt: string; +} diff --git a/backend/src/auth/url-policy.ts b/backend/src/auth/url-policy.ts new file mode 100644 index 00000000..7143e996 --- /dev/null +++ b/backend/src/auth/url-policy.ts @@ -0,0 +1,42 @@ +export interface ConfiguredTransportUrlOptions { + allowLoopbackHttp: boolean; + originOnly?: boolean; +} + +function canonicalLoopbackAuthority(value: string): boolean { + const match = /^http:\/\/([^/?#]+)(?:[/?#]|$)/.exec(value); + if (!match) return false; + const authority = match[1]; + let port: string | undefined; + if (authority.startsWith("[")) { + const ipv6 = /^(\[::1\])(?::([^:]+))?$/.exec(authority); + if (!ipv6) return false; + port = ipv6[2]; + } else { + const ipv4 = /^([^:]+)(?::([^:]+))?$/.exec(authority); + if (!ipv4) return false; + const octets = ipv4[1].split("."); + if (octets.length !== 4 || octets.some((octet) => !/^(?:0|[1-9]\d{0,2})$/.test(octet) + || Number(octet) > 255) || Number(octets[0]) !== 127) return false; + port = ipv4[2]; + } + return port === undefined || (/^(?:0|[1-9]\d{0,4})$/.test(port) && Number(port) <= 65_535); +} + +export function parseConfiguredTransportUrl( + value: string, + options: ConfiguredTransportUrlOptions, +): URL | undefined { + let url: URL; + try { + url = new URL(value); + } catch { + return undefined; + } + if (url.username || url.password || url.search || url.hash || (options.originOnly && url.pathname !== "/")) { + return undefined; + } + if (url.protocol === "https:") return url; + if (options.allowLoopbackHttp && url.protocol === "http:" && canonicalLoopbackAuthority(value)) return url; + return undefined; +} diff --git a/backend/src/auth/windows-auth-storage.ts b/backend/src/auth/windows-auth-storage.ts new file mode 100644 index 00000000..0e03ef8e --- /dev/null +++ b/backend/src/auth/windows-auth-storage.ts @@ -0,0 +1,639 @@ +import { spawn, spawnSync } from "node:child_process"; +import { posix, win32 } from "node:path"; +import type { Readable, Writable } from "node:stream"; +import { z } from "zod"; + +const PROTOCOL_VERSION = 1; +const MAX_PROTOCOL_BYTES = 64 * 1024; +const MAX_RESPONSE_BYTES = 64 * 1024; +const MAX_AUTH_CONFIG_BYTES = 1024 * 1024; +const MAX_AUTH_CONFIG_BASE64_BYTES = 4 * Math.ceil(MAX_AUTH_CONFIG_BYTES / 3); +const MAX_AUTH_CONFIG_RESPONSE_BYTES = MAX_AUTH_CONFIG_BASE64_BYTES + 1024; +const MAX_SESSION_BYTES = 16 * 1024; +const MAX_OIDC_BYTES = 8 * 1024; +const DEFAULT_MAX_ENTRIES = 256; +const MAX_ENTRIES = 512; +const TIMEOUT_MS = 5_000; +const TERMINATION_GRACE_MS = 100; +const FINAL_SETTLEMENT_MS = 750; +const DIGEST_FILENAME = /^[a-f0-9]{64}\.json$/; +const CLAIM_FILENAME = /^[a-f0-9]{64}\.claim$/; +const OIDC_SLOT_FILENAME = /^slot-(?:[0-5][0-9]|6[0-3])\.json$/; +const AUTH_CONFIG_FILENAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,249}\.yaml$/; + +const invalid = (): Error => new Error("auth_session_store_invalid"); + +export type WindowsAuthStorageDirectory = "sessions" | "oidc"; + +export interface WindowsAuthStorageEntry { + name: string; + modifiedUnixMs: number; +} + +export interface WindowsAuthStoragePage { + entries: WindowsAuthStorageEntry[]; + more: boolean; +} + +/** Internal adapter boundary for the file-session store's native Windows path. */ +export interface WindowsAuthStorageBridge { + validateRoot(root: string): Promise; + ensureLayout(root: string): Promise; + readAuthConfig(path: string): Buffer; + readLocalUsers(path: string): Promise; + create(root: string, directory: WindowsAuthStorageDirectory, filename: string, contents: Buffer): Promise; + read(root: string, directory: WindowsAuthStorageDirectory, filename: string): Promise; + replace(root: string, directory: WindowsAuthStorageDirectory, filename: string, contents: Buffer): Promise; + remove(root: string, directory: WindowsAuthStorageDirectory, filename: string): Promise; + list( + root: string, + directory: WindowsAuthStorageDirectory, + maximumEntries?: number, + ): Promise; + listPage( + root: string, + directory: "sessions", + afterName: string | undefined, + maximumEntries: number, + ): Promise; + claimConsume(root: string, filename: string): Promise; + readClaim(root: string, filename: string): Promise; + removeClaim(root: string, filename: string): Promise; +} + +export interface WindowsAuthStorageInvocation { + executable: string; + args: readonly string[]; + input: Buffer; + timeoutMs: number; + maximumOutputBytes: number; +} + +export interface WindowsAuthStorageInvocationResult { + code: number; + stdout: Buffer; + stderr: Buffer; +} + +interface WindowsAuthStorageChild { + readonly stdin: Writable | null; + readonly stdout: Readable | null; + readonly stderr: Readable | null; + kill(signal?: NodeJS.Signals | number): boolean; + unref?(): void; + on(event: "error", listener: (error: Error) => void): this; + once(event: "error", listener: (error: Error) => void): this; + once(event: "close", listener: (code: number | null, signal: NodeJS.Signals | null) => void): this; + removeListener?(event: "error" | "close", listener: (...args: any[]) => void): this; +} + +type WindowsAuthStorageSpawn = ( + executable: string, + args: readonly string[], + options: { shell: false; windowsHide: true; stdio: ["pipe", "pipe", "pipe"]; env: NodeJS.ProcessEnv }, +) => WindowsAuthStorageChild; + +export interface WindowsAuthStorageBridgeOptions { + /** Test-only transport seam. Production always uses the no-shell child-process invocation. */ + invoke?: (invocation: WindowsAuthStorageInvocation) => Promise; + /** Test-only synchronous seam used by the synchronous authentication-config provider. */ + invokeSync?: (invocation: WindowsAuthStorageInvocation) => WindowsAuthStorageInvocationResult; + /** Optional configured tht path. Defaults to THT_BIN, then the safe bare command `tht`. */ + thtExecutable?: string; + /** Test-only child-launch seam; production uses the fixed no-shell Node child-process launcher. */ + spawnChild?: WindowsAuthStorageSpawn; + /** Test-only input scheduling seam for real child-process lifecycle tests. */ + beforeInputForTest?: () => Promise; + /** Test-only bounded lifecycle timings. Production always uses the fixed deadlines below. */ + deadlinesForTest?: { + timeoutMs?: number; + terminationGraceMs?: number; + finalSettlementMs?: number; + }; +} + +type AuthStoragePathStyle = "posix" | "windows"; + +const responseSchema = z.strictObject({ + version: z.literal(PROTOCOL_VERSION), + ok: z.literal(true), + created: z.boolean().optional(), + replaced: z.boolean().optional(), + removed: z.boolean().optional(), + found: z.boolean().optional(), + contentBase64: z.string().max(MAX_AUTH_CONFIG_BASE64_BYTES).optional(), + entries: z.array(z.strictObject({ + name: z.string().max(128), + modifiedUnixMs: z.number().int().safe().nonnegative(), + })).max(MAX_ENTRIES).optional(), + more: z.boolean().optional(), + validated: z.boolean().optional(), + prepared: z.boolean().optional(), +}); + +type BridgeResponse = z.infer; + +interface BridgeRequest { + version: typeof PROTOCOL_VERSION; + operation: "validate-root" | "ensure-layout" | "read-auth-config" | "read-local-users" | "create" | "read" | "replace" | "remove" | "list" | "claim-consume" | "read-claim" | "remove-claim"; + root: string; + directory?: WindowsAuthStorageDirectory; + filename?: string; + contentBase64?: string; + maximumEntries?: number; + afterName?: string; + continuation?: true; +} + +function directoryMaximum(directory: WindowsAuthStorageDirectory): number { + return directory === "sessions" ? MAX_SESSION_BYTES : MAX_OIDC_BYTES; +} + +function canonicalBase64(value: string, maximum: number): Buffer { + if (typeof value !== "string" || value.length > Math.ceil(maximum / 3) * 4) throw invalid(); + try { + const decoded = Buffer.from(value, "base64"); + if (decoded.length === 0 || decoded.length > maximum || decoded.toString("base64") !== value) throw invalid(); + return decoded; + } catch { + throw invalid(); + } +} + +function validateRoot(root: string, pathStyle: AuthStoragePathStyle): void { + const paths = pathStyle === "windows" ? win32 : posix; + if (typeof root !== "string" || root.length === 0 || /[\u0000-\u001f\u007f]/.test(root) + || !paths.isAbsolute(root) || paths.normalize(root) !== root) throw invalid(); +} + +function validateFilename(filename: string, allowClaim = false, allowOidcSlot = false): void { + if (typeof filename !== "string" || (!DIGEST_FILENAME.test(filename) + && !(allowClaim && CLAIM_FILENAME.test(filename)) + && !(allowOidcSlot && OIDC_SLOT_FILENAME.test(filename)))) throw invalid(); +} + +function safeThtExecutable(value: string | undefined, pathStyle: AuthStoragePathStyle): string { + const executable = value ?? process.env.THT_AUTH_STORAGE_BIN ?? process.env.THT_BIN ?? "tht"; + if (typeof executable !== "string" || executable.length === 0 || /[\u0000-\u001f\u007f]/.test(executable)) throw invalid(); + if (executable === "tht" || (pathStyle === "windows" && executable === "tht.exe")) return executable; + const paths = pathStyle === "windows" ? win32 : posix; + if (paths.isAbsolute(executable) && paths.normalize(executable) === executable + && (pathStyle === "posix" || /\.exe$/i.test(executable))) return executable; + throw invalid(); +} + +function parseResponse(result: WindowsAuthStorageInvocationResult, maximumOutputBytes: number): BridgeResponse { + if (!Number.isInteger(result.code) || result.code !== 0 || !Buffer.isBuffer(result.stdout) + || !Buffer.isBuffer(result.stderr) || result.stderr.length > MAX_RESPONSE_BYTES + || result.stdout.length === 0 || result.stdout.length > maximumOutputBytes) { + throw invalid(); + } + try { + const source = new TextDecoder("utf-8", { fatal: true }).decode(result.stdout); + return responseSchema.parse(JSON.parse(source)); + } catch { + throw invalid(); + } +} + +function encodedRequest(request: BridgeRequest, pathStyle: AuthStoragePathStyle): Buffer { + validateRoot(request.root, pathStyle); + if (request.operation === "validate-root" || request.operation === "ensure-layout") { + if (request.directory !== undefined || request.filename !== undefined || request.contentBase64 !== undefined + || request.maximumEntries !== undefined || request.afterName !== undefined || request.continuation !== undefined) throw invalid(); + } else if (request.operation === "read-auth-config" || request.operation === "read-local-users") { + if (request.directory !== undefined || request.contentBase64 !== undefined || request.maximumEntries !== undefined + || request.afterName !== undefined || request.continuation !== undefined + || request.filename === undefined || !AUTH_CONFIG_FILENAME.test(request.filename)) throw invalid(); + } else if (request.operation === "list") { + if (request.directory === undefined) throw invalid(); + if (request.filename !== undefined || request.contentBase64 !== undefined) throw invalid(); + if (request.maximumEntries !== undefined && (!Number.isInteger(request.maximumEntries) + || request.maximumEntries < 1 || request.maximumEntries > MAX_ENTRIES)) throw invalid(); + if (request.continuation === true) { + if (request.directory !== "sessions" || (request.afterName !== undefined && !DIGEST_FILENAME.test(request.afterName))) { + throw invalid(); + } + } else if (request.afterName !== undefined || request.continuation !== undefined) { + throw invalid(); + } + } else { + if (request.directory === undefined) throw invalid(); + if (request.maximumEntries !== undefined || request.afterName !== undefined || request.continuation !== undefined) throw invalid(); + if (request.filename === undefined) throw invalid(); + const allowClaim = request.operation === "remove" && request.directory === "oidc"; + const allowOidcSlot = request.directory === "oidc" + && (request.operation === "create" || request.operation === "read" || request.operation === "remove"); + validateFilename(request.filename, allowClaim, allowOidcSlot); + if (request.contentBase64 !== undefined && request.operation !== "create" && request.operation !== "replace") throw invalid(); + } + if ((request.operation === "claim-consume" || request.operation === "read-claim" || request.operation === "remove-claim") + && request.directory !== "oidc") throw invalid(); + if (request.contentBase64 !== undefined) { + if (request.directory === undefined) throw invalid(); + canonicalBase64(request.contentBase64, directoryMaximum(request.directory)); + } + const encoded = Buffer.from(JSON.stringify(request), "utf8"); + if (encoded.length === 0 || encoded.length > MAX_PROTOCOL_BYTES) throw invalid(); + return encoded; +} + +function environmentForBridge(): NodeJS.ProcessEnv { + const path = process.env.PATH; + const systemRoot = process.env.SystemRoot ?? process.env.SYSTEMROOT; + return { + ...(path === undefined ? {} : { PATH: path }), + ...(systemRoot === undefined ? {} : { SystemRoot: systemRoot }), + }; +} + +const spawnTht: WindowsAuthStorageSpawn = (executable, args, options) => spawn(executable, [...args], options); + +function invokeThtSync(invocation: WindowsAuthStorageInvocation): WindowsAuthStorageInvocationResult { + try { + const result = spawnSync(invocation.executable, [...invocation.args], { + shell: false, + windowsHide: true, + env: environmentForBridge(), + input: invocation.input, + timeout: invocation.timeoutMs, + maxBuffer: invocation.maximumOutputBytes, + encoding: "buffer", + }); + if (result.error || result.signal !== null || typeof result.status !== "number" + || !Buffer.isBuffer(result.stdout) || !Buffer.isBuffer(result.stderr)) throw invalid(); + return { code: result.status, stdout: result.stdout, stderr: result.stderr }; + } catch { + throw invalid(); + } +} + +async function invokeTht( + invocation: WindowsAuthStorageInvocation, + spawnChild: WindowsAuthStorageSpawn = spawnTht, + beforeInputForTest?: () => Promise, + terminationGraceMs = TERMINATION_GRACE_MS, + finalSettlementMs = FINAL_SETTLEMENT_MS, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + let aborted = false; + let timeout: NodeJS.Timeout | undefined; + let terminationTimer: NodeJS.Timeout | undefined; + let finalSettlementTimer: NodeJS.Timeout | undefined; + let lateErrorReleaseTimer: NodeJS.Timeout | undefined; + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let stdoutBytes = 0; + let stderrBytes = 0; + let child: WindowsAuthStorageChild | undefined; + let stdin: Writable | undefined; + let stdoutStream: Readable | undefined; + let stderrStream: Readable | undefined; + const swallowChildError = (): void => undefined; + const swallowStreamError = (): void => undefined; + const releaseQuarantine = (): void => { + if (lateErrorReleaseTimer !== undefined) clearTimeout(lateErrorReleaseTimer); + lateErrorReleaseTimer = undefined; + removeChildListener("error", swallowChildError); + removeChildListener("close", releaseQuarantine); + removeStreamListener(stdin, "error", swallowStreamError); + removeStreamListener(stdoutStream, "error", swallowStreamError); + removeStreamListener(stderrStream, "error", swallowStreamError); + }; + const quarantineLateErrors = (): void => { + // A final-deadline settlement can precede a broken ChildProcess object's terminal events. + // Keep only no-capture listeners for a bounded grace period so a late EventEmitter error + // cannot become uncaught, including after an already-observed close event. + child?.on("error", swallowChildError); + child?.once("close", releaseQuarantine); + stdin?.on("error", swallowStreamError); + stdoutStream?.on("error", swallowStreamError); + stderrStream?.on("error", swallowStreamError); + lateErrorReleaseTimer = setTimeout(releaseQuarantine, finalSettlementMs); + lateErrorReleaseTimer.unref?.(); + }; + const removeChildListener = (event: "error" | "close", listener: (...args: any[]) => void): void => { + try { child?.removeListener?.(event, listener); } catch { /* the helper is already terminal */ } + }; + const removeStreamListener = (stream: Writable | Readable | undefined, event: "data" | "error", listener: (...args: any[]) => void): void => { + try { stream?.removeListener(event, listener); } catch { /* the helper is already terminal */ } + }; + const stopStream = (stream: Writable | Readable | null | undefined): void => { + try { stream?.destroy(); } catch { /* abort is already fail-closed */ } + }; + const onChildError = (): void => abort(); + const onStdinError = (): void => abort(); + const onStdoutError = (): void => abort(); + const onStderrError = (): void => abort(); + const onStdoutData = (chunk: Buffer): void => { + if (aborted || settled) return; + stdoutBytes += chunk.length; + if (stdoutBytes > invocation.maximumOutputBytes) { + abort(); + return; + } + stdout.push(Buffer.from(chunk)); + }; + const onStderrData = (chunk: Buffer): void => { + if (aborted || settled) return; + stderrBytes += chunk.length; + if (stderrBytes > MAX_RESPONSE_BYTES) { + abort(); + return; + } + stderr.push(Buffer.from(chunk)); + }; + const onClose = (code: number | null, signal: NodeJS.Signals | null): void => { + if (settled) return; + if (aborted || code === null || !Number.isInteger(code) || signal !== null) { + settle(() => reject(invalid()), true); + return; + } + settle(() => resolve({ + code, + stdout: Buffer.concat(stdout), + stderr: Buffer.concat(stderr), + })); + }; + const cleanup = (quarantine = false): void => { + if (timeout !== undefined) clearTimeout(timeout); + if (terminationTimer !== undefined) clearTimeout(terminationTimer); + if (finalSettlementTimer !== undefined) clearTimeout(finalSettlementTimer); + removeChildListener("error", onChildError); + removeChildListener("close", onClose as (...args: any[]) => void); + removeStreamListener(stdin, "error", onStdinError); + removeStreamListener(stdoutStream, "data", onStdoutData); + removeStreamListener(stdoutStream, "error", onStdoutError); + removeStreamListener(stderrStream, "data", onStderrData); + removeStreamListener(stderrStream, "error", onStderrError); + if (quarantine) quarantineLateErrors(); + }; + const settle = (callback: () => void, quarantine = false): void => { + if (settled) return; + settled = true; + cleanup(quarantine); + callback(); + }; + const abort = (): void => { + if (aborted || settled) return; + aborted = true; + if (timeout !== undefined) clearTimeout(timeout); + if (child !== undefined) { + stopStream(stdin); + stopStream(stdoutStream); + stopStream(stderrStream); + try { child.kill("SIGTERM"); } catch { /* final settlement still owns completion */ } + try { child.unref?.(); } catch { /* the bounded timers still own completion */ } + } + terminationTimer = setTimeout(() => { + if (settled || child === undefined) return; + try { child.kill("SIGKILL"); } catch { /* final settlement still owns completion */ } + }, terminationGraceMs); + finalSettlementTimer = setTimeout(() => { + settle(() => reject(invalid()), true); + }, finalSettlementMs); + }; + try { + child = spawnChild(invocation.executable, invocation.args, { + shell: false, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"], + env: environmentForBridge(), + }); + } catch { + settle(() => reject(invalid())); + return; + } + child.once("close", onClose); + child.on("error", onChildError); + if (!child.stdin || !child.stdout || !child.stderr) { + abort(); + return; + } + stdin = child.stdin; + stdoutStream = child.stdout; + stderrStream = child.stderr; + timeout = setTimeout(() => { + abort(); + }, invocation.timeoutMs); + stdoutStream.on("data", onStdoutData); + stdoutStream.once("error", onStdoutError); + stderrStream.on("data", onStderrData); + stderrStream.once("error", onStderrError); + stdin.once("error", onStdinError); + const writeInput = (): void => { + if (aborted || settled) return; + try { + stdin.end(invocation.input); + } catch { + abort(); + } + }; + if (beforeInputForTest === undefined) { + writeInput(); + } else { + void Promise.resolve().then(beforeInputForTest).then(writeInput, abort); + } + }); +} + +function contentFrom(response: BridgeResponse, maximum: number): Buffer | undefined { + if (response.found !== true) { + if (response.contentBase64 !== undefined) throw invalid(); + return undefined; + } + if (response.contentBase64 === undefined) throw invalid(); + return canonicalBase64(response.contentBase64, maximum); +} + +function listedEntries( + response: BridgeResponse, + directory: WindowsAuthStorageDirectory, + maximumEntries: number, +): WindowsAuthStorageEntry[] { + if (response.entries === undefined || response.entries.length > maximumEntries) throw invalid(); + const names = new Set(); + for (const entry of response.entries) { + if (!DIGEST_FILENAME.test(entry.name) && !(directory === "oidc" + && (CLAIM_FILENAME.test(entry.name) || OIDC_SLOT_FILENAME.test(entry.name)))) throw invalid(); + if (names.has(entry.name)) throw invalid(); + names.add(entry.name); + } + return response.entries.map((entry) => ({ name: entry.name, modifiedUnixMs: entry.modifiedUnixMs })); +} + +function createAuthStorageBridge( + pathStyle: AuthStoragePathStyle, + options: WindowsAuthStorageBridgeOptions = {}, +): WindowsAuthStorageBridge { + const executable = safeThtExecutable(options.thtExecutable, pathStyle); + const testDeadlines = options.deadlinesForTest; + const timeoutMs = testDeadlines?.timeoutMs ?? TIMEOUT_MS; + const terminationGraceMs = testDeadlines?.terminationGraceMs ?? TERMINATION_GRACE_MS; + const finalSettlementMs = testDeadlines?.finalSettlementMs ?? FINAL_SETTLEMENT_MS; + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > TIMEOUT_MS + || !Number.isSafeInteger(terminationGraceMs) || terminationGraceMs < 1 || terminationGraceMs > TIMEOUT_MS + || !Number.isSafeInteger(finalSettlementMs) || finalSettlementMs <= terminationGraceMs || finalSettlementMs > TIMEOUT_MS) { + throw invalid(); + } + const invoke = options.invoke ?? ((invocation: WindowsAuthStorageInvocation) => invokeTht( + invocation, + options.spawnChild, + options.beforeInputForTest, + terminationGraceMs, + finalSettlementMs, + )); + const invokeSync = options.invokeSync ?? invokeThtSync; + const request = async (value: BridgeRequest): Promise => { + try { + const maximumOutputBytes = value.operation === "read-local-users" + ? MAX_AUTH_CONFIG_RESPONSE_BYTES + : MAX_RESPONSE_BYTES; + const response = await invoke({ + executable, + args: ["_auth-storage"], + input: encodedRequest(value, pathStyle), + timeoutMs, + maximumOutputBytes, + }); + return parseResponse(response, maximumOutputBytes); + } catch { + throw invalid(); + } + }; + const syncRequest = (value: BridgeRequest): BridgeResponse => { + try { + const response = invokeSync({ + executable, + args: ["_auth-storage"], + input: encodedRequest(value, pathStyle), + timeoutMs, + maximumOutputBytes: MAX_AUTH_CONFIG_RESPONSE_BYTES, + }); + return parseResponse(response, MAX_AUTH_CONFIG_RESPONSE_BYTES); + } catch { + throw invalid(); + } + }; + const recordRequest = (operation: "create" | "read" | "replace" | "remove" | "claim-consume" | "read-claim" | "remove-claim", root: string, directory: WindowsAuthStorageDirectory, filename: string, contents?: Buffer): BridgeRequest => ({ + version: PROTOCOL_VERSION, + operation, + root, + directory, + filename, + ...(contents === undefined ? {} : { contentBase64: contents.toString("base64") }), + }); + + return { + async validateRoot(root) { + const response = await request({ version: PROTOCOL_VERSION, operation: "validate-root", root }); + if (response.validated !== true + || Object.keys(response).some((key) => !["version", "ok", "validated"].includes(key))) throw invalid(); + }, + async ensureLayout(root) { + const response = await request({ version: PROTOCOL_VERSION, operation: "ensure-layout", root }); + if (response.prepared !== true + || Object.keys(response).some((key) => !["version", "ok", "prepared"].includes(key))) throw invalid(); + }, + readAuthConfig(path) { + const paths = pathStyle === "windows" ? win32 : posix; + if (typeof path !== "string" || path.length === 0 || /[\u0000-\u001f\u007f]/.test(path) + || !paths.isAbsolute(path) || paths.normalize(path) !== path) throw invalid(); + const root = paths.dirname(path); + const filename = paths.basename(path); + if (!AUTH_CONFIG_FILENAME.test(filename) || paths.join(root, filename) !== path) throw invalid(); + const response = syncRequest({ version: PROTOCOL_VERSION, operation: "read-auth-config", root, filename }); + if (Object.keys(response).some((key) => !["version", "ok", "found", "contentBase64"].includes(key))) throw invalid(); + const contents = contentFrom(response, MAX_AUTH_CONFIG_BYTES); + if (contents === undefined) throw invalid(); + return contents; + }, + async readLocalUsers(path) { + const paths = pathStyle === "windows" ? win32 : posix; + if (typeof path !== "string" || path.length === 0 || /[\u0000-\u001f\u007f]/.test(path) + || !paths.isAbsolute(path) || paths.normalize(path) !== path) throw invalid(); + const root = paths.dirname(path); + const filename = paths.basename(path); + if (!AUTH_CONFIG_FILENAME.test(filename) || paths.join(root, filename) !== path) throw invalid(); + const response = await request({ version: PROTOCOL_VERSION, operation: "read-local-users", root, filename }); + if (Object.keys(response).some((key) => !["version", "ok", "found", "contentBase64"].includes(key))) throw invalid(); + const contents = contentFrom(response, MAX_AUTH_CONFIG_BYTES); + if (contents === undefined) throw invalid(); + return contents; + }, + async create(root, directory, filename, contents) { + if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > directoryMaximum(directory)) throw invalid(); + const response = await request(recordRequest("create", root, directory, filename, contents)); + if (response.created === undefined) throw invalid(); + return response.created; + }, + async read(root, directory, filename) { + return contentFrom(await request(recordRequest("read", root, directory, filename)), directoryMaximum(directory)); + }, + async replace(root, directory, filename, contents) { + if (!Buffer.isBuffer(contents) || contents.length === 0 || contents.length > directoryMaximum(directory)) throw invalid(); + const response = await request(recordRequest("replace", root, directory, filename, contents)); + if (response.replaced !== true) throw invalid(); + }, + async remove(root, directory, filename) { + const response = await request(recordRequest("remove", root, directory, filename)); + return response.removed === true; + }, + async list(root, directory, maximumEntries = DEFAULT_MAX_ENTRIES) { + if (!Number.isInteger(maximumEntries) || maximumEntries < 1 || maximumEntries > MAX_ENTRIES) throw invalid(); + const response = await request({ + version: PROTOCOL_VERSION, + operation: "list", + root, + directory, + maximumEntries, + }); + if (response.more !== undefined) throw invalid(); + return listedEntries(response, directory, maximumEntries); + }, + async listPage(root, directory, afterName, maximumEntries) { + if (directory !== "sessions" || !Number.isInteger(maximumEntries) + || maximumEntries < 1 || maximumEntries > MAX_ENTRIES + || (afterName !== undefined && !DIGEST_FILENAME.test(afterName))) throw invalid(); + const response = await request({ + version: PROTOCOL_VERSION, + operation: "list", + root, + directory, + maximumEntries, + continuation: true, + ...(afterName === undefined ? {} : { afterName }), + }); + if (response.more === undefined) throw invalid(); + const entries = listedEntries(response, directory, maximumEntries); + let previous = afterName; + for (const entry of entries) { + if (previous !== undefined && entry.name <= previous) throw invalid(); + previous = entry.name; + } + if (response.more && entries.length !== maximumEntries) throw invalid(); + if (response.more && (previous === undefined || previous === afterName)) throw invalid(); + return { entries, more: response.more }; + }, + async claimConsume(root, filename) { + return contentFrom(await request(recordRequest("claim-consume", root, "oidc", filename)), MAX_OIDC_BYTES); + }, + async readClaim(root, filename) { + return contentFrom(await request(recordRequest("read-claim", root, "oidc", filename)), MAX_OIDC_BYTES); + }, + async removeClaim(root, filename) { + const response = await request(recordRequest("remove-claim", root, "oidc", filename)); + return response.removed === true; + }, + }; +} + +export function createWindowsAuthStorageBridge(options: WindowsAuthStorageBridgeOptions = {}): WindowsAuthStorageBridge { + return createAuthStorageBridge("windows", options); +} + +/** POSIX uses the same single hidden tht protocol and bounds, with native canonical path rules. */ +export function createPosixAuthStorageBridge(options: WindowsAuthStorageBridgeOptions = {}): WindowsAuthStorageBridge { + return createAuthStorageBridge("posix", options); +} diff --git a/backend/src/bridge/session-bridge.ts b/backend/src/bridge/session-bridge.ts index 1e65926a..f1dbf2c2 100644 --- a/backend/src/bridge/session-bridge.ts +++ b/backend/src/bridge/session-bridge.ts @@ -1,5 +1,19 @@ import type { RpcClient } from "../rpc/rpc-client.js"; +const GENERIC_MODEL_FAILURE = + "Model request failed. Check provider connectivity, then Resume the session."; +const SUBSCRIPTION_MODEL_FAILURE = + "The selected model is unavailable for the current subscription. Choose another model and start a new session."; + +function safeModelFailure(error: unknown): string { + const detail = typeof error === "string" ? error : ""; + const isSubscriptionFailure = + /\b429\b/.test(detail) && + /(subscription plan|code["':\s]+1311|does not yet include access)/i.test(detail); + + return isSubscriptionFailure ? SUBSCRIPTION_MODEL_FAILURE : GENERIC_MODEL_FAILURE; +} + export type ToolActivity = { kind: "tool"; toolCallId: string; @@ -53,7 +67,7 @@ export class SessionBridge { this.fan({ type: "info", level: "error", - text: "Model request failed. Check provider connectivity, then Resume the session.", + text: safeModelFailure(m.message.errorMessage), }); } } else if (m.type === "extension_ui_request" && m.method === "notify") { diff --git a/backend/src/config.ts b/backend/src/config.ts index b5331436..c3c00a10 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -1,8 +1,19 @@ import path from "node:path"; +import { statSync } from "node:fs"; +import { + createAuthenticationConfigProvider, + type AuthenticationConfigProvider, + type AuthMode, +} from "./auth/config.js"; +import type { WorkspaceRegistryConfig } from "./workspaces/types.js"; export interface AppConfig { host: string; port: number; harnessDir: string; thtBin: string; piBin: string; - authMode: "none" | "mock" | "upstream"; + authMode: AuthMode; + authConfigFile: string; + authStateRoot: string; + authentication?: AuthenticationConfigProvider; + publicExposure: boolean; sessionStorage: { mode: "local" | "postgres"; host?: string; port?: number; database?: string; runtimeUser?: string; @@ -11,9 +22,12 @@ export interface AppConfig { defaults: { provider?: string; model?: string; thinking?: string }; maxPiProcesses: number; settingsFile: string; + maintenanceFile: string; dataRoot?: string; ollamaEnsureTimeoutMs: number; + piManagementTimeoutMs: number; secretsFile?: string; + piAuthFile?: string; secretFiles: Readonly>; modelApiKeyFile?: string; /** @@ -22,22 +36,186 @@ export interface AppConfig { * pay the probe; the local dev launcher (run-stack.sh) opts in via THT_DWH_PRECHECK. */ dwhPrecheck: boolean; + /** Explicit compatibility mode for old loopback clients that send `workspace` in POST /sessions. */ + legacyWorkspaceMode: boolean; + workspaceDiagnosticTimeoutMs: number; + workspaceRegistry: WorkspaceRegistryConfig; + workspaceSecretStoreRoot: string; + workspaceSecretRuntimeRoot: string; + internalQdrantUrl: string; + internalEmbeddingUrl: string; + internalEmbeddingModel: string; + internalEmbeddingDimensions: number; } -export function loadConfig(env: Record): AppConfig { - const authMode = env.AUTH_MODE ?? "none"; - if (!(["none", "mock", "upstream"] as const).includes(authMode as AppConfig["authMode"])) { - throw new Error(`unsupported AUTH_MODE=${authMode}; use none, mock, or upstream`); + +export const MAX_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS = 10_000; + +function requiredRegistryValue(value: string, label: string): string { + if (value.length === 0 || value.trim() !== value || value.includes("\0")) { + throw new Error(`workspace registry ${label} configuration is invalid`); } - if (env.THOTH_PUBLIC_EXPOSURE === "true" && authMode !== "upstream") { - throw new Error("public exposure requires AUTH_MODE=upstream behind a trusted proxy"); + return value; +} + +function absoluteRegistryPath(value: string, label: string): string { + const pathValue = requiredRegistryValue(value, label); + if (!path.isAbsolute(pathValue)) { + throw new Error(`workspace registry ${label} must be absolute`); + } + return pathValue; +} + +function absoluteAuthPath(value: string, label: string): string { + if (value.length === 0 || value.trim() !== value || value.includes("\0") || !path.isAbsolute(value)) { + throw new Error(`authentication ${label} configuration is invalid`); + } + return value; +} + +function authConfigFileExists(file: string): boolean { + try { + if (!statSync(file).isFile()) throw new Error("authentication configuration is invalid"); + return true; + } catch (error: any) { + if (error?.code === "ENOENT") return false; + throw new Error("authentication configuration is invalid"); + } +} + +function refSafeGitBranch(value: string): string { + const branch = requiredRegistryValue(value, "branch"); + if ( + branch === "@" + || branch === "HEAD" + || branch.startsWith("-") + || branch.startsWith("/") + || branch.endsWith("/") + || branch.endsWith(".") + || branch.includes("..") + || branch.includes("@{") + || branch.includes("//") + || branch.split("/").some((component) => component.startsWith(".") || component.endsWith(".lock")) + || /[\p{Cc} ~^:?*\[\\]/u.test(branch) + ) { + throw new Error("workspace registry branch configuration is invalid"); + } + return branch; +} + +function safeInstallationId(value: string): string { + const installationId = requiredRegistryValue(value, "installation ID"); + if (/\p{Cc}/u.test(installationId)) { + throw new Error("workspace registry installation ID configuration is invalid"); + } + return installationId; +} + +function positiveImportLimit(value: string | undefined, fallback: number): number { + const limit = Number(value ?? fallback); + if (!Number.isSafeInteger(limit) || limit <= 0) { + throw new Error("workspace limit configuration is invalid"); + } + return limit; +} + +function diagnosticTimeout(value: string | undefined): number { + const timeout = Number(value ?? 5_000); + if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > MAX_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS) { + throw new Error("workspace diagnostic timeout configuration is invalid"); + } + return timeout; +} + +function piManagementTimeout(value: string | undefined): number { + const timeout = Number(value ?? 8_000); + if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 30_000) { + throw new Error("Pi management timeout configuration is invalid"); + } + return timeout; +} + +function loopbackHost(host: string): boolean { + return host === "::1" + || host === "127.0.0.1" + || /^127(?:\.\d{1,3}){3}$/.test(host); +} + +function internalServiceUrl( + value: string | undefined, + fallback: string, + label: string, + allowedHosts: readonly string[], +): string { + const raw = value ?? fallback; + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + throw new Error(`${label} configuration is invalid`); + } + if ( + parsed.protocol !== "http:" + || parsed.username.length > 0 + || parsed.password.length > 0 + || parsed.pathname !== "/" + || parsed.search.length > 0 + || parsed.hash.length > 0 + || (!allowedHosts.includes(parsed.hostname) && !loopbackHost(parsed.hostname)) + ) { + throw new Error(`${label} configuration is invalid`); + } + return parsed.toString().replace(/\/$/, ""); +} + +function positiveDimension(value: string | undefined, fallback: number): number { + const parsed = Number(value ?? fallback); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error("internal embedding dimensions configuration is invalid"); + } + return parsed; +} + +export function loadConfig(env: Record): AppConfig { + const authConfigFile = absoluteAuthPath(env.THT_AUTH_CONFIG_FILE ?? "/run/thothii-auth/auth.yaml", "file"); + const authStateRoot = absoluteAuthPath(env.THT_AUTH_STATE_ROOT ?? "/data/auth", "state root"); + const hasAuthenticationConfig = authConfigFileExists(authConfigFile); + if (hasAuthenticationConfig && env.AUTH_MODE !== undefined) { + throw new Error("authentication configuration and AUTH_MODE cannot both be set"); + } + const authentication = hasAuthenticationConfig ? createAuthenticationConfigProvider(authConfigFile) : undefined; + let authMode: AuthMode; + if (authentication) { + authMode = authentication.current().value.mode; + } else { + const requestedMode = env.AUTH_MODE ?? "none"; + if (!(["none", "mock", "upstream"] as const).includes(requestedMode as "none" | "mock" | "upstream")) { + throw new Error(`unsupported AUTH_MODE=${requestedMode}; use none, mock, or upstream`); + } + const nodeEnvironment = env.NODE_ENV ?? process.env.NODE_ENV; + if ((requestedMode === "none" || requestedMode === "mock") + && nodeEnvironment !== "development" && nodeEnvironment !== "test") { + throw new Error("production requires auth.yaml or AUTH_MODE=upstream"); + } + authMode = requestedMode as "none" | "mock" | "upstream"; + } + const publicExposure = env.THOTH_PUBLIC_EXPOSURE === "true"; + if (publicExposure && authMode !== "oidc" && authMode !== "upstream") { + throw new Error("public exposure requires AUTH_MODE=upstream or configured OIDC behind a trusted proxy"); } const sessionStorageMode = env.THT_SESSION_STORAGE ?? "local"; if (sessionStorageMode !== "local" && sessionStorageMode !== "postgres") { throw new Error("session storage configuration is invalid"); } - if (sessionStorageMode === "local" && env.THOTH_PUBLIC_EXPOSURE === "true") { + if (sessionStorageMode === "local" && publicExposure) { throw new Error("local session storage requires loopback-only deployment"); } + const legacyWorkspaceMode = env.THT_LEGACY_WORKSPACE_MODE; + if (legacyWorkspaceMode !== undefined && legacyWorkspaceMode !== "local") { + throw new Error("legacy workspace mode configuration is invalid"); + } + if (legacyWorkspaceMode === "local" && sessionStorageMode !== "local") { + throw new Error("legacy workspace mode requires local session storage"); + } const sessionStorage: AppConfig["sessionStorage"] = { mode: sessionStorageMode }; if (sessionStorageMode === "postgres") { const host = env.THT_SESSION_DB_HOST; @@ -48,15 +226,15 @@ export function loadConfig(env: Record): AppConfig { const sslrootcert = env.THT_SESSION_DB_SSLROOTCERT; const port = Number(env.THT_SESSION_DB_PORT ?? 5432); if ( - authMode !== "upstream" + (authMode !== "upstream" && authMode !== "oidc") || !host || !database || !runtimeUser || !runtimePasswordFile || !path.isAbsolute(runtimePasswordFile) || (sslmode !== "verify-ca" && sslmode !== "verify-full") || !sslrootcert || !path.isAbsolute(sslrootcert) || !Number.isInteger(port) || port < 1 || port > 65535 ) { - if (authMode !== "upstream") { - throw new Error("server session storage requires AUTH_MODE=upstream"); + if (authMode !== "upstream" && authMode !== "oidc") { + throw new Error("server session storage requires AUTH_MODE=upstream or configured OIDC"); } throw new Error("server session storage configuration is invalid"); } @@ -82,6 +260,11 @@ export function loadConfig(env: Record): AppConfig { secretsFile.trim() !== secretsFile || secretsFile.length === 0 || secretsFile.includes("\0") || !path.isAbsolute(secretsFile) )) throw new Error("secret bundle configuration is invalid"); + const piAuthFile = env.THT_PI_AUTH_FILE; + if (piAuthFile !== undefined && ( + piAuthFile.trim() !== piAuthFile || piAuthFile.length === 0 || piAuthFile.includes("\0") + || !path.isAbsolute(piAuthFile) + )) throw new Error("Pi authentication source configuration is invalid"); const secretFiles: Record = {}; for (const name of [ "THT_MODEL_API_KEY_SECRET_FILE", "THT_DWH_API_KEY_SECRET_FILE", "THT_VEC_API_KEY_SECRET_FILE", @@ -89,22 +272,87 @@ export function loadConfig(env: Record): AppConfig { "THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE", "THT_VECTOR_READER_PASSWORD_SECRET_FILE", "THT_VECTOR_WRITER_PASSWORD_SECRET_FILE", ]) secretFiles[name] = env[name]; + const registryRoot = absoluteRegistryPath( + env.THT_WORKSPACE_REGISTRY_ROOT ?? "/data/workspace-registry", + "root", + ); + const registryBranch = refSafeGitBranch(env.THT_WORKSPACE_GIT_BRANCH ?? "main"); + const installationId = safeInstallationId( + env.THT_WORKSPACE_INSTALLATION_ID ?? "local", + ); + const remoteUrl = env.THT_WORKSPACE_GIT_REMOTE === undefined + ? undefined + : requiredRegistryValue(env.THT_WORKSPACE_GIT_REMOTE, "remote"); + const secretRoots = (env.THT_WORKSPACE_SECRET_ROOTS ?? "") + .split(",") + .filter((root) => root.length > 0) + .map((root) => absoluteRegistryPath(root, "secret root")); + const workspaceRegistry: WorkspaceRegistryConfig = { + root: registryRoot, + remoteUrl, + branch: registryBranch, + installationId, + secretRoots, + dataRoot: env.THT_DATA_ROOT, + maxEvidenceEntries: positiveImportLimit(env.THT_WORKSPACE_MAX_EVIDENCE_ENTRIES, 4096), + maxEvidenceBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_EVIDENCE_BYTES, 64 * 1024 * 1024), + maxEvidenceFileBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_EVIDENCE_FILE_BYTES, 8 * 1024 * 1024), + maxEvidencePathBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_EVIDENCE_PATH_BYTES, 4096), + maxEvidenceManifestBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_EVIDENCE_MANIFEST_BYTES, 1024 * 1024), + }; + const workspaceSecretStoreRoot = absoluteRegistryPath( + env.THT_WORKSPACE_SECRET_STORE_ROOT ?? path.join(path.dirname(registryRoot), "workspace-secrets"), + "secret store root", + ); + const workspaceSecretRuntimeRoot = absoluteRegistryPath( + env.THT_WORKSPACE_SECRET_RUNTIME_ROOT ?? "/tmp/thothii-workspace-secrets", + "secret runtime root", + ); + const settingsFile = env.SETTINGS_FILE ?? "data/settings.json"; + const internalQdrantUrl = internalServiceUrl( + env.THT_INTERNAL_QDRANT_URL, + "http://qdrant:6333", + "internal Qdrant URL", + ["qdrant", "localhost"], + ); + const internalEmbeddingUrl = internalServiceUrl( + env.THT_INTERNAL_EMBEDDING_URL, + "http://embedding:11434", + "internal embedding URL", + ["embedding", "localhost"], + ); return { host: env.HOST ?? "127.0.0.1", port: Number(env.PORT ?? 8787), harnessDir: env.THT_HARNESS_DIR ?? "../harness", thtBin: env.THT_BIN ?? "tht", piBin: env.PI_BIN ?? "pi", - authMode: authMode as AppConfig["authMode"], + authMode, + authConfigFile, + authStateRoot, + authentication, + publicExposure, sessionStorage, defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING }, maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4), - settingsFile: env.SETTINGS_FILE ?? "data/settings.json", + settingsFile, + maintenanceFile: env.THT_MAINTENANCE_FILE ?? path.join(path.dirname(settingsFile), "maintenance.json"), dataRoot: env.THT_DATA_ROOT, ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000), + piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS), secretsFile, + piAuthFile, secretFiles, modelApiKeyFile, dwhPrecheck: env.THT_DWH_PRECHECK === "true" || env.THT_DWH_PRECHECK === "1", + legacyWorkspaceMode: legacyWorkspaceMode === "local", + workspaceDiagnosticTimeoutMs: diagnosticTimeout(env.THT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS), + workspaceRegistry, + workspaceSecretStoreRoot, + workspaceSecretRuntimeRoot, + internalQdrantUrl, + internalEmbeddingUrl, + internalEmbeddingModel: env.THT_INTERNAL_EMBEDDING_MODEL ?? "qwen3-embedding:0.6b", + internalEmbeddingDimensions: positiveDimension(env.THT_INTERNAL_EMBEDDING_DIMENSIONS, 1024), }; } diff --git a/backend/src/config/secret-bundle.ts b/backend/src/config/secret-bundle.ts index b46746a3..5dfe95b8 100644 --- a/backend/src/config/secret-bundle.ts +++ b/backend/src/config/secret-bundle.ts @@ -2,12 +2,18 @@ import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, type Stats, } from "node:fs"; +import { + AUTHENTICATION_SECRET_LIMITS, + isAuthenticationSecretReference, + isUsableAuthenticationSecret, +} from "../auth/secret-policy.js"; /** Keys accepted by the deployment bundle. Keep this list intentionally explicit. */ export const SECRET_BUNDLE_KEYS = Object.freeze([ "THT_MODEL_API_KEY", "THT_DWH_API_KEY", "THT_VEC_API_KEY", "THT_VEC_WRITE_API_KEY", "THT_CA", "THT_SSL_CA", "THT_VECTOR_BOOTSTRAP_PASSWORD", "THT_VECTOR_MIGRATOR_PASSWORD", "THT_VECTOR_READER_PASSWORD", "THT_VECTOR_WRITER_PASSWORD", "PI_PROVIDER_API_KEY", + "THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN", ] as const); const ALLOWED = new Set(SECRET_BUNDLE_KEYS); @@ -25,7 +31,10 @@ const LEGACY_FILES: Readonly> = { }; const MAX_BUNDLE_BYTES = 64 * 1024; -const MAX_LINE_BYTES = 16 * 1024; +const MAX_LINE_BYTES = Math.max( + 16 * 1024, + ...Object.entries(AUTHENTICATION_SECRET_LIMITS).map(([name, maximum]) => name.length + 1 + maximum), +); export interface SecretBundleConfig { secretsFile?: string; @@ -97,7 +106,8 @@ function parseBundle(text: string): ReadonlyMap { const match = /^([A-Z][A-Z0-9_]*)=(.*)$/.exec(line); if (!match) throw unavailable(); const [, key, value] = match; - if (!ALLOWED.has(key) || values.has(key) || value.length === 0 || /[\r\n]/.test(value)) { + if (!ALLOWED.has(key) || values.has(key) || value.length === 0 || /[\r\n]/.test(value) + || isAuthenticationSecretReference(key) && !isUsableAuthenticationSecret(key, value)) { throw unavailable(); } values.set(key, value); diff --git a/backend/src/operator-command.ts b/backend/src/operator-command.ts new file mode 100644 index 00000000..db820c45 --- /dev/null +++ b/backend/src/operator-command.ts @@ -0,0 +1,135 @@ +/** + * Installation-scoped host operations that must not impersonate an HTTP administrator. + * This command runs only through `docker compose exec core`; it never accepts credentials, + * headers, paths, or arbitrary code from the caller. + */ +import { pathToFileURL } from "node:url"; +import { join } from "node:path"; +import { loadConfig, type AppConfig } from "./config.js"; +import { rolesToPermissions } from "./auth/config.js"; +import type { PrincipalContext } from "./auth/principal.js"; +import { createPiModelLister } from "./pi/list-models.js"; +import { createPiManagement } from "./pi/management.js"; +import { effectiveSettings } from "./routes/settings.js"; +import { MaintenanceBarrier } from "./runtime/maintenance-gate.js"; +import { loadSettings } from "./settings/settings-store.js"; +import { ThtRunner, type SessionRow } from "./tht/tht-runner.js"; +import { WorkspaceRegistry } from "./workspaces/registry.js"; +import { WorkspaceSecretStore } from "./workspaces/secret-store.js"; + +type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status" + | "session-inventory" | "workflow-doctor" | "workspace-integrity" + | "pi-options" | "pi-test" | "effective-settings"; + +const lifecyclePrincipal: PrincipalContext = { + issuer: "tht-operator-command", + subject: "installation-lifecycle", + displayName: "Installation lifecycle", + roles: ["admin"], + permissions: rolesToPermissions(["admin"]), + isAdmin: true, +}; + +function operatorRunner(config: AppConfig): ThtRunner { + const workspaceSecretStore = new WorkspaceSecretStore({ + root: config.workspaceSecretStoreRoot, + runtimeRoot: config.workspaceSecretRuntimeRoot, + installationId: config.workspaceRegistry.installationId, + }); + return new ThtRunner({ + thtBin: config.thtBin, + harnessDir: config.harnessDir, + configPath: process.env.THT_CONFIG ?? "config/tht.yaml", + dataRoot: config.dataRoot, + runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"), + secretRoots: config.workspaceRegistry.secretRoots, + secretsFile: config.secretsFile, + secretFiles: config.secretFiles, + workspaceSecretStore, + semanticRuntime: { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }, + }).withPrincipal(lifecyclePrincipal); +} + +async function sessionInventory(config: AppConfig): Promise>> { + const registry = new WorkspaceRegistry(config.workspaceRegistry); + const revisions = await registry.listRetainedSnapshots(); + const runner = operatorRunner(config); + const sessions = new Map(); + for (const revision of revisions) { + for (const session of await runner.sessionList(revision.snapshotPath)) sessions.set(session.id, session); + } + return [...sessions.values()].map(({ status, archived }) => ({ status, archived: archived === true })); +} + +async function workflowDiagnostics(config: AppConfig): Promise<{ ready: true; workspaces: number }> { + const registry = new WorkspaceRegistry(config.workspaceRegistry); + const revisions = await registry.listRetainedSnapshots(); + if (revisions.length === 0) throw new Error("workflow diagnostics unavailable"); + const runner = operatorRunner(config); + for (const revision of revisions) { + const result = await runner.run(["doctor", "--json"], revision.snapshotPath); + let payload: unknown; + try { + payload = JSON.parse(result.stdout); + } catch { + throw new Error("workflow diagnostics failed"); + } + if ( + result.code !== 0 || !payload || typeof payload !== "object" + || (payload as { ok?: unknown }).ok !== true + ) throw new Error("workflow diagnostics failed"); + } + return { ready: true, workspaces: revisions.length }; +} + +async function workspaceIntegrity(config: AppConfig): Promise<{ + ready: true; + state: "uninitialized" | "active"; + workspaces: number; + fingerprint: string; +}> { + const integrity = await new WorkspaceRegistry(config.workspaceRegistry).verifyStoredState(); + return { ready: true, ...integrity }; +} + +export async function runOperatorAction( + action: OperatorAction, + config: AppConfig, +): Promise { + if (action.startsWith("maintenance-")) { + const barrier = new MaintenanceBarrier(config.maintenanceFile); + if (action === "maintenance-activate") await barrier.activate(); + if (action === "maintenance-deactivate") barrier.deactivate(); + return barrier.status(); + } + if (action === "session-inventory") return await sessionInventory(config); + if (action === "workflow-doctor") return await workflowDiagnostics(config); + if (action === "workspace-integrity") return await workspaceIntegrity(config); + if (action === "effective-settings") return effectiveSettings(config, loadSettings(config)); + const service = createPiManagement(config, { listModels: createPiModelLister(config) }); + if (action === "pi-options") return await service.options(); + if (action === "pi-test") return await service.test(); + throw new Error("unsupported operator action"); +} + +async function main(): Promise { + const action = process.argv[2] as OperatorAction | undefined; + if (!action || ![ + "maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory", + "workflow-doctor", "workspace-integrity", "pi-options", "pi-test", "effective-settings", + ].includes(action)) throw new Error("invalid operator action"); + const result = await runOperatorAction(action, loadConfig(process.env)); + process.stdout.write(`${JSON.stringify(result)}\n`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + void main().catch(() => { + process.stderr.write("operator command failed\n"); + process.exitCode = 2; + }); +} diff --git a/backend/src/pi/list-models.ts b/backend/src/pi/list-models.ts index 7d68d16b..6616221f 100644 --- a/backend/src/pi/list-models.ts +++ b/backend/src/pi/list-models.ts @@ -6,6 +6,10 @@ import { loadPiEnabledModels, type PiEnabledModelsResult, } from "./enabled-models.js"; +import { + readConfiguredPiAgentFile, + validateDeclarativePiConfig, +} from "./managed-config.js"; export interface PiModel { provider: string; @@ -23,6 +27,7 @@ interface Opts { ttlMs?: number; nowMs?: () => number; loadEnabledModels?: () => PiEnabledModelsResult; + readModelsStore?: () => string | undefined; warn?: (message: string) => void; } @@ -39,6 +44,11 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): () => Prom let cache: { at: number; models: PiModel[] } | null = null; return async function listModels(): Promise { + const managedModels = opts.readModelsStore + ? opts.readModelsStore() + : readConfiguredPiAgentFile("models.json", true); + if (managedModels !== undefined) validateDeclarativePiConfig(managedModels); + if (cache && now() - cache.at < ttlMs) return cache.models; const enabled = (opts.loadEnabledModels diff --git a/backend/src/pi/managed-config.ts b/backend/src/pi/managed-config.ts new file mode 100644 index 00000000..42c12603 --- /dev/null +++ b/backend/src/pi/managed-config.ts @@ -0,0 +1,161 @@ +import { + chmodSync, closeSync, constants, fstatSync, lstatSync, mkdtempSync, openSync, + readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync, type Dirent, +} from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const MAX_AGENT_CONFIG_BYTES = 1024 * 1024; + +export const PI_MANAGED_CONFIG_ERROR_CODE = "PI_MANAGED_CONFIG_INVALID"; +export const PI_MANAGED_CONFIG_ERROR_MESSAGE = "Pi provider/model configuration is invalid"; + +export class PiManagedConfigError extends Error { + readonly code = PI_MANAGED_CONFIG_ERROR_CODE; + + constructor() { + super(PI_MANAGED_CONFIG_ERROR_MESSAGE); + } +} + +export function isPiManagedConfigError(error: unknown): boolean { + return Boolean( + error && typeof error === "object" + && (error as { code?: unknown }).code === PI_MANAGED_CONFIG_ERROR_CODE, + ); +} + +export function parsePiConfigJson(raw: string): unknown { + try { + return JSON.parse(raw); + } catch { + throw new PiManagedConfigError(); + } +} + +/** Reject every Pi shell-backed configuration value, including unknown future nested fields. */ +export function assertDeclarativePiConfig(value: unknown): void { + const pending: unknown[] = [value]; + while (pending.length > 0) { + const current = pending.pop(); + if (typeof current === "string") { + if (current.startsWith("!")) throw new PiManagedConfigError(); + continue; + } + if (Array.isArray(current)) { + for (const nested of current) pending.push(nested); + continue; + } + if (current && typeof current === "object") { + for (const nested of Object.values(current as Record)) pending.push(nested); + } + } +} + +export function validateDeclarativePiConfig(raw: string): void { + assertDeclarativePiConfig(parsePiConfigJson(raw)); +} + +function configuredPiAgentDir(): string { + return resolve(process.env.PI_CODING_AGENT_DIR ?? join(homedir(), ".pi", "agent")); +} + +function readPiAgentFile( + configuredAgentDir: string, + name: "auth.json" | "models.json", + optional: boolean, +): string | undefined { + const path = join(configuredAgentDir, name); + let fd: number | undefined; + try { + const before = lstatSync(path); + if (!before.isFile() || before.isSymbolicLink() || before.size > MAX_AGENT_CONFIG_BYTES) { + throw new PiManagedConfigError(); + } + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const opened = fstatSync(fd); + if (!opened.isFile() || opened.size > MAX_AGENT_CONFIG_BYTES + || before.dev !== opened.dev || before.ino !== opened.ino) { + throw new PiManagedConfigError(); + } + return readFileSync(fd, "utf8"); + } catch (error) { + if (optional && (error as NodeJS.ErrnoException)?.code === "ENOENT") return undefined; + throw new PiManagedConfigError(); + } finally { + if (fd !== undefined) { + try { closeSync(fd); } catch { /* preserve the stable validation outcome */ } + } + } +} + +export function readConfiguredPiAgentFile(name: "auth.json"): string; +export function readConfiguredPiAgentFile(name: "auth.json", optional: true): string | undefined; +export function readConfiguredPiAgentFile(name: "models.json", optional: true): string | undefined; +export function readConfiguredPiAgentFile( + name: "auth.json" | "models.json", + optional = false, +): string | undefined { + return readPiAgentFile(configuredPiAgentDir(), name, optional); +} + +export interface PiRuntimeAgentSnapshot { + agentDir: string; + sessionDir: string; + cleanup: () => void; +} + +/** + * Bind a session Pi process to the exact managed auth/model bytes validated at spawn time. + * Other agent resources remain live through symlinks, while session storage stays persistent. + */ +export function createPiRuntimeAgentSnapshot(): PiRuntimeAgentSnapshot { + const sourceAgentDir = configuredPiAgentDir(); + const auth = readPiAgentFile(sourceAgentDir, "auth.json", true); + const models = readPiAgentFile(sourceAgentDir, "models.json", true); + if (auth !== undefined) validateDeclarativePiConfig(auth); + if (models !== undefined) validateDeclarativePiConfig(models); + + let snapshotDir: string | undefined; + try { + snapshotDir = mkdtempSync(join(tmpdir(), "thoth-pi-runtime-agent-")); + chmodSync(snapshotDir, 0o700); + let entries: Dirent[]; + try { + entries = readdirSync(sourceAgentDir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") throw error; + entries = []; + } + for (const entry of entries) { + if (entry.name === "auth.json" || entry.name === "models.json") continue; + symlinkSync( + join(sourceAgentDir, entry.name), + join(snapshotDir, entry.name), + entry.isDirectory() ? (process.platform === "win32" ? "junction" : "dir") : "file", + ); + } + if (auth !== undefined) { + writeFileSync(join(snapshotDir, "auth.json"), auth, { flag: "wx", mode: 0o600 }); + } + if (models !== undefined) { + writeFileSync(join(snapshotDir, "models.json"), models, { flag: "wx", mode: 0o600 }); + } + } catch { + if (snapshotDir !== undefined) { + try { rmSync(snapshotDir, { recursive: true, force: true }); } catch { /* sanitized */ } + } + throw new PiManagedConfigError(); + } + + let cleaned = false; + return { + agentDir: snapshotDir, + sessionDir: process.env.PI_CODING_AGENT_SESSION_DIR || join(sourceAgentDir, "sessions"), + cleanup: () => { + if (cleaned) return; + cleaned = true; + try { rmSync(snapshotDir, { recursive: true, force: true }); } catch { /* sanitized */ } + }, + }; +} diff --git a/backend/src/pi/management.ts b/backend/src/pi/management.ts new file mode 100644 index 00000000..114db34f --- /dev/null +++ b/backend/src/pi/management.ts @@ -0,0 +1,341 @@ +import { execFile as nodeExecFile } from "node:child_process"; +import { promisify } from "node:util"; +import type { AppConfig } from "../config.js"; +import { secretValue } from "../config/secret-bundle.js"; +import { + loadSettings, + saveSettings, + type Settings, +} from "../settings/settings-store.js"; +import type { PiModel } from "./list-models.js"; +import { + PI_MANAGED_CONFIG_ERROR_MESSAGE, + isPiManagedConfigError, +} from "./managed-config.js"; +import { createPiProviderSmoke, type PiProviderSmoke } from "./provider-smoke.js"; +import { loadPiAuthProviders } from "./auth-providers.js"; +import { + piProviderCredentialStatus, + type PiCredentialStatus, +} from "./provider-credentials.js"; + +const execFile = promisify(nodeExecFile); +const REASONING_CHOICES = ["low", "medium", "high"] as const; +const VERSION_PATTERN = /^(?:pi(?:\s+version)?\s+)?v?(\d+(?:\.\d+){1,3}(?:[-+][0-9A-Za-z.-]+)?)$/; +const MAX_LOG_LINES = 200; +const MAX_LOG_LINE_LENGTH = 4_096; +const MAX_EXEC_OUTPUT_BYTES = 64 * 1024; + +export type PiReasoning = typeof REASONING_CHOICES[number]; + +export interface PiInstallationConfig { + provider?: string; + model?: string; + reasoning?: PiReasoning; +} + +export interface PiStatus { + version?: string; + ready: boolean; + credentials: PiCredentialStatus; + config: PiInstallationConfig; + checkedAt: string; + message?: string; +} + +export interface PiOptions { + providers: string[]; + models: Array<{ provider: string; id: string }>; + reasoning: PiReasoning[]; + checkedAt: string; +} + +export interface PiTestResult { + ready: boolean; + checkedAt: string; + message?: string; +} + +export interface PiLogs { + lines: string[]; + checkedAt: string; +} + +export interface PiExecFileOptions { + timeout: number; + maxBuffer: number; +} + +export type PiExecFile = ( + command: string, + args: string[], + options: PiExecFileOptions, +) => Promise<{ stdout: string; stderr: string }>; + +export interface PiManagementService { + status(): Promise; + options(): Promise; + configure(value: PiInstallationConfig): Promise; + test(): Promise; + logs(): Promise; +} + +export class PiManagementError extends Error { + constructor( + public readonly code: "pi_management_invalid_config" | "pi_management_unavailable" | "pi_management_write_failed", + message: string, + ) { + super(message); + } +} + +interface PiManagementDeps { + execute?: PiExecFile; + listModels: () => Promise; + smokeProvider?: PiProviderSmoke; + readSettings?: () => Settings; + saveSettings?: (settings: Settings) => Settings; + readLogs?: () => string | Promise; + credentialStatus?: (provider: string | undefined) => PiCredentialStatus; + now?: () => Date; +} + +export function createPiManagement(config: AppConfig, deps: PiManagementDeps): PiManagementService { + const now = deps.now ?? (() => new Date()); + const diagnostics: string[] = []; + const addDiagnostic = (message: string): void => { + diagnostics.push(`${now().toISOString()} ${redact(message)}`); + if (diagnostics.length > MAX_LOG_LINES) diagnostics.splice(0, diagnostics.length - MAX_LOG_LINES); + }; + const execute = deps.execute ?? defaultExecFile; + const readSettings = deps.readSettings ?? (() => loadSettings(config)); + const persistSettings = deps.saveSettings ?? ((settings) => saveSettings(config, settings)); + const readLogs = deps.readLogs ?? (() => diagnostics.join("\n")); + const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config); + const credentialStatus = deps.credentialStatus ?? ((provider: string | undefined) => { + try { + return piProviderCredentialStatus({ + provider, + authProviders: loadPiAuthProviders(), + resolveCredentialValue: () => secretValue(config, "THT_MODEL_API_KEY"), + credentialFile: config.modelApiKeyFile, + }); + } catch { + return "missing"; + } + }); + + const closedOptions = async (): Promise> => { + let listed: PiModel[]; + try { + listed = await deps.listModels(); + } catch (error) { + if (isPiManagedConfigError(error)) { + throw new PiManagementError("pi_management_unavailable", PI_MANAGED_CONFIG_ERROR_MESSAGE); + } + throw new PiManagementError("pi_management_unavailable", "Pi model choices are unavailable"); + } + const models: Array<{ provider: string; id: string }> = []; + const providers: string[] = []; + const seenModels = new Set(); + const seenProviders = new Set(); + for (const model of listed) { + if (!isChoice(model?.provider) || !isChoice(model?.id)) continue; + const key = `${model.provider}\u0000${model.id}`; + if (seenModels.has(key)) continue; + seenModels.add(key); + models.push({ provider: model.provider, id: model.id }); + if (!seenProviders.has(model.provider)) { + seenProviders.add(model.provider); + providers.push(model.provider); + } + } + return { providers, models, reasoning: [...REASONING_CHOICES] }; + }; + + const version = async (timeoutMs = config.piManagementTimeoutMs): Promise => { + let output: { stdout: string; stderr: string }; + try { + // The Pi executable and every argument are installation-owned constants. Do not add a shell. + output = await execute(config.piBin, ["--version"], { + timeout: timeoutMs, + maxBuffer: MAX_EXEC_OUTPUT_BYTES, + }); + } catch (error) { + if (isTimeout(error)) { + throw new PiManagementError("pi_management_unavailable", "Pi smoke check timed out"); + } + throw new PiManagementError("pi_management_unavailable", "Pi runtime is unavailable"); + } + const matched = VERSION_PATTERN.exec(output.stdout.trim()); + if (!matched) throw new PiManagementError("pi_management_unavailable", "Pi runtime returned an invalid version"); + return matched[1]; + }; + + const installationConfig = (): PiInstallationConfig => { + const settings = readSettings(); + const provider = config.defaults.provider ?? settings.provider; + const model = config.defaults.model ?? settings.model; + const reasoning = config.defaults.thinking ?? settings.thinking; + return { + ...(isChoice(provider) ? { provider } : {}), + ...(isChoice(model) ? { model } : {}), + ...(isReasoning(reasoning) ? { reasoning } : {}), + }; + }; + + return { + async status(): Promise { + const checkedAt = now().toISOString(); + const current = installationConfig(); + const credentials = credentialStatus(current.provider); + try { + const currentVersion = await version(); + addDiagnostic("Pi version probe succeeded"); + return { version: currentVersion, ready: true, credentials, config: current, checkedAt }; + } catch (error) { + const message = stableMessage(error, "Pi runtime is unavailable"); + addDiagnostic(message); + return { ready: false, credentials, config: current, checkedAt, message }; + } + }, + + async options(): Promise { + const choices = await closedOptions(); + return { ...choices, checkedAt: now().toISOString() }; + }, + + async configure(value: PiInstallationConfig): Promise { + if (!isInstallationConfig(value)) { + throw new PiManagementError("pi_management_invalid_config", "Pi installation configuration is invalid"); + } + const choices = await closedOptions(); + if (!choices.models.some((model) => model.provider === value.provider && model.id === value.model)) { + throw new PiManagementError("pi_management_invalid_config", "Pi provider and model must be selected from available choices"); + } + try { + persistSettings({ ...readSettings(), provider: value.provider, model: value.model, thinking: value.reasoning }); + } catch { + throw new PiManagementError("pi_management_write_failed", "Pi installation configuration could not be saved"); + } + addDiagnostic("Pi installation defaults updated"); + return { ...value, updatedAt: now().toISOString() }; + }, + + async test(): Promise { + const checkedAt = now().toISOString(); + const deadline = Date.now() + config.piManagementTimeoutMs; + let timer: NodeJS.Timeout | undefined; + try { + const check = async (): Promise => { + await version(remainingBudget(deadline)); + const current = installationConfig(); + if (!current.provider || !current.model || !current.reasoning) { + throw new PiManagementError( + "pi_management_unavailable", + "Pi installation configuration is incomplete", + ); + } + await smokeProvider({ + provider: current.provider, + model: current.model, + reasoning: current.reasoning, + timeoutMs: remainingBudget(deadline), + }); + }; + await Promise.race([ + check(), + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new PiManagementError("pi_management_unavailable", "Pi smoke check timed out")), + config.piManagementTimeoutMs, + ); + }), + ]); + addDiagnostic("Pi smoke check succeeded"); + return { ready: true, checkedAt }; + } catch (error) { + return smokeFailure(stableMessage(error, "Pi provider smoke check failed"), checkedAt, addDiagnostic); + } finally { + if (timer) clearTimeout(timer); + } + }, + + async logs(): Promise { + let source = ""; + try { + source = await readLogs(); + } catch { + source = "Pi diagnostics are unavailable"; + } + const lines = source + .split(/\r?\n/u) + .filter((line) => line.length > 0) + .slice(-MAX_LOG_LINES) + .map((line) => redact(line.slice(0, MAX_LOG_LINE_LENGTH))); + return { lines, checkedAt: now().toISOString() }; + }, + }; +} + +async function defaultExecFile(command: string, args: string[], options: PiExecFileOptions) { + const result = await execFile(command, args, { + timeout: options.timeout, + maxBuffer: options.maxBuffer, + windowsHide: true, + }); + return { stdout: String(result.stdout), stderr: String(result.stderr) }; +} + +function isChoice(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 128 && value.trim() === value + && /^[A-Za-z0-9][A-Za-z0-9._/-]*$/u.test(value); +} + +function isReasoning(value: unknown): value is PiReasoning { + return typeof value === "string" && (REASONING_CHOICES as readonly string[]).includes(value); +} + +function isInstallationConfig(value: unknown): value is Required { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const candidate = value as Record; + if (Object.keys(candidate).length !== 3 || Object.keys(candidate).some((key) => !["provider", "model", "reasoning"].includes(key))) { + return false; + } + return isChoice(candidate.provider) && isChoice(candidate.model) && isReasoning(candidate.reasoning); +} + +function isTimeout(error: unknown): boolean { + return Boolean( + error && typeof error === "object" && ( + (error as { code?: unknown }).code === "ETIMEDOUT" + || (error as { killed?: unknown }).killed === true + ), + ); +} + +function stableMessage(error: unknown, fallback: string): string { + if (isPiManagedConfigError(error)) return PI_MANAGED_CONFIG_ERROR_MESSAGE; + return error instanceof PiManagementError ? error.message : fallback; +} + +function smokeFailure( + message: string, + checkedAt: string, + addDiagnostic: (message: string) => void, +): PiTestResult { + addDiagnostic(message); + return { ready: false, message, checkedAt }; +} + +export function redact(value: string): string { + return value + .replace(/(\bauthorization\b\s*:\s*Bearer\s+)[^\s,;]+/giu, "$1[REDACTED]") + .replace(/((?:["']?)[A-Za-z0-9_-]*(?:api[_-]?key|token|password|secret|authorization)[A-Za-z0-9_-]*(?:["']?)\s*(?:=|:)\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;}]+)/giu, "$1[REDACTED]") + .replace(/(\bBearer\s+)[^\s,;]+/giu, "$1[REDACTED]") + .replace(/(\w+:\/\/[^:/\s]+:)[^@/\s]+@/gu, "$1[REDACTED]@"); +} + +function remainingBudget(deadline: number): number { + return Math.max(1, deadline - Date.now()); +} diff --git a/backend/src/pi/pi-process-manager.ts b/backend/src/pi/pi-process-manager.ts index a12750e4..226f967f 100644 --- a/backend/src/pi/pi-process-manager.ts +++ b/backend/src/pi/pi-process-manager.ts @@ -2,17 +2,19 @@ import { spawn as nodeSpawn, type ChildProcessWithoutNullStreams } from "node:ch import type { AppConfig } from "../config.js"; import { RpcClient } from "../rpc/rpc-client.js"; import { SessionBridge } from "../bridge/session-bridge.js"; -import type { ThtRunner } from "../tht/tht-runner.js"; +import type { RuntimeConfigLease, ThtRunner } from "../tht/tht-runner.js"; import { buildPiChildEnv, canonicalPiProvider } from "./provider-credentials.js"; import { loadPiAuthProviders } from "./auth-providers.js"; import { secretValue } from "../config/secret-bundle.js"; import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js"; +import { createPiRuntimeAgentSnapshot } from "./managed-config.js"; export interface SessionRuntime { rpc: RpcClient; bridge: SessionBridge; child: ChildProcessWithoutNullStreams; ownerKey?: string; + releaseRuntimeConfig?: () => void; } export interface RuntimeOptions { @@ -23,6 +25,7 @@ export interface RuntimeOptions { question?: string; mode?: "new" | "resume"; principal?: PrincipalContext; + runtimeConfig?: RuntimeConfigLease; } /** Injectable child-process boundary; callbacks may ignore arguments in simpler tests. */ @@ -34,67 +37,94 @@ type SpawnFn = ( export class PiProcessManager { private runtimes = new Map(); + private agentSnapshotCleanups = new WeakMap void>(); private spawnFn: ( sessionId: string, author: string, provider: string | undefined, principal?: PrincipalContext, + runtimeConfigPath?: string, ) => ChildProcessWithoutNullStreams; - private loadAuthProviders: () => ReadonlySet; + private loadAuthProviders: (agentDir: string) => ReadonlySet; constructor( private cfg: AppConfig, - opts?: { spawnFn?: SpawnFn; authProviders?: () => ReadonlySet }, + opts?: { spawnFn?: SpawnFn; authProviders?: (agentDir: string) => ReadonlySet }, ) { - this.loadAuthProviders = opts?.authProviders ?? (() => loadPiAuthProviders()); + this.loadAuthProviders = opts?.authProviders + ?? ((agentDir) => loadPiAuthProviders({ agentDir })); if (opts?.spawnFn) { - this.spawnFn = (sessionId, author, provider, principal) => - this.spawnPi(opts.spawnFn!, sessionId, author, provider, principal); + this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) => + this.spawnPi(opts.spawnFn!, sessionId, author, provider, principal, runtimeConfigPath); } else { - this.spawnFn = (sessionId, author, provider, principal) => - this.spawnPi(nodeSpawn, sessionId, author, provider, principal); + this.spawnFn = (sessionId, author, provider, principal, runtimeConfigPath) => + this.spawnPi(nodeSpawn, sessionId, author, provider, principal, runtimeConfigPath); } } + private cleanupAgentSnapshot(child: ChildProcessWithoutNullStreams): void { + const cleanup = this.agentSnapshotCleanups.get(child); + if (!cleanup) return; + this.agentSnapshotCleanups.delete(child); + cleanup(); + } + private spawnPi( - spawnFn: SpawnFn, sessionId: string, author: string, provider: string | undefined, principal?: PrincipalContext, + spawnFn: SpawnFn, sessionId: string, author: string, provider: string | undefined, + principal?: PrincipalContext, runtimeConfigPath?: string, ): ChildProcessWithoutNullStreams { - const env = buildPiChildEnv({ - provider, - authProviders: this.loadAuthProviders(), - credentialValue: secretValue(this.cfg, "THT_MODEL_API_KEY"), - credentialFile: this.cfg.modelApiKeyFile, - additions: { THT_SESSION: sessionId, THT_AUTHOR: author }, - }); - clearPrincipalEnvironment(env); - if (principal) Object.assign(env, principalEnvironment(principal)); - // The Thoth gate executes the deterministic `tht` CLI as a Pi tool. Give only - // this managed session process the adapter values already loaded by the core - // entrypoint; the generic provider helper continues to scrub them by default. - for (const name of [ - "THT_DWH_API_KEY", "THT_VEC_API_KEY", "THT_VEC_WRITE_API_KEY", - ] as const) { - const value = secretValue(this.cfg, name) ?? process.env[name]; - if (value !== undefined) env[name] = value; - } - const ca = secretValue(this.cfg, "THT_SSL_CA") - ?? secretValue(this.cfg, "THT_CA") - ?? process.env.THT_SSL_CA - ?? process.env.THT_CA; - if (ca !== undefined) { - env.THT_CA = ca; - env.THT_SSL_CA = ca; - } - delete env.THT_DATA_ROOT; - if (this.cfg.dataRoot !== undefined) env.THT_DATA_ROOT = this.cfg.dataRoot; - // pi 0.73 removed `--approve`: rpc mode is headless and its argv is intentionally minimal. - const child = spawnFn(this.cfg.piBin, ["--mode", "rpc"], { - cwd: this.cfg.harnessDir, - env, - }); + // This is the final shared boundary for createFor(), spawnFor(), and resume(). Validate + // before auth-provider inspection, then make Pi consume the exact copied bytes rather than + // reopening mutable mounted auth/models files after this check. + const agent = createPiRuntimeAgentSnapshot(); + let child: ChildProcessWithoutNullStreams | undefined; try { + const env = buildPiChildEnv({ + provider, + authProviders: this.loadAuthProviders(agent.agentDir), + credentialValue: secretValue(this.cfg, "THT_MODEL_API_KEY"), + credentialFile: this.cfg.modelApiKeyFile, + additions: { THT_SESSION: sessionId, THT_AUTHOR: author }, + }); + env.PI_CODING_AGENT_DIR = agent.agentDir; + env.PI_CODING_AGENT_SESSION_DIR = agent.sessionDir; + clearPrincipalEnvironment(env); + if (principal) Object.assign(env, principalEnvironment(principal)); + // The Thoth gate executes the deterministic `tht` CLI as a Pi tool. Give only + // this managed session process the adapter values already loaded by the core + // entrypoint; the generic provider helper continues to scrub them by default. + for (const name of [ + "THT_DWH_API_KEY", "THT_VEC_API_KEY", "THT_VEC_WRITE_API_KEY", + ] as const) { + const value = secretValue(this.cfg, name) ?? process.env[name]; + if (value !== undefined) env[name] = value; + } + const ca = secretValue(this.cfg, "THT_SSL_CA") + ?? secretValue(this.cfg, "THT_CA") + ?? process.env.THT_SSL_CA + ?? process.env.THT_CA; + if (ca !== undefined) { + env.THT_CA = ca; + env.THT_SSL_CA = ca; + } + delete env.THT_DATA_ROOT; + if (this.cfg.dataRoot !== undefined) env.THT_DATA_ROOT = this.cfg.dataRoot; + if (runtimeConfigPath !== undefined) env.THT_CONFIG = runtimeConfigPath; + // pi 0.73 removed `--approve`: rpc mode is headless and its argv is intentionally minimal. + child = spawnFn(this.cfg.piBin, ["--mode", "rpc"], { + cwd: this.cfg.harnessDir, + env, + }); + this.agentSnapshotCleanups.set(child, agent.cleanup); + child.once("exit", () => this.cleanupAgentSnapshot(child!)); + child.once("close", () => this.cleanupAgentSnapshot(child!)); // Log stderr for debugging (was silently drained) child.stderr.on("data", (d: Buffer) => console.error(`[pi:${sessionId}] stderr:`, d.toString().trim())); return child; } catch (error) { - try { child.kill(); } catch { /* preserve the initialization error */ } + if (child) { + try { child.kill(); } catch { /* preserve the initialization error */ } + this.cleanupAgentSnapshot(child); + } else { + agent.cleanup(); + } throw error; } } @@ -118,15 +148,31 @@ export class PiProcessManager { // SIGTERM to the in-flight Pi process and lose its pending gate. const existing = this.runtimes.get(sessionId); if (existing) { + o.runtimeConfig?.release(); throw new Error(`session runtime already active: ${sessionId}`); } if (o.principal) this.teardownForPrincipal(o.principal); if (this.runtimes.size >= this.cfg.maxPiProcesses) { + o.runtimeConfig?.release(); throw new Error("max Pi processes reached"); } const author = o.author ?? "dev@local"; const provider = canonicalPiProvider(o.provider ?? this.cfg.defaults.provider); - const child = this.spawnFn(sessionId, author, provider, o.principal); + let child: ChildProcessWithoutNullStreams; + try { + child = this.spawnFn(sessionId, author, provider, o.principal, o.runtimeConfig?.path); + } catch (error) { + o.runtimeConfig?.release(); + throw error; + } + let runtimeConfigReleased = false; + const releaseRuntimeConfig = () => { + if (runtimeConfigReleased) return; + runtimeConfigReleased = true; + o.runtimeConfig?.release(); + }; + child.once("exit", releaseRuntimeConfig); + child.once("close", releaseRuntimeConfig); let rt: SessionRuntime | undefined; try { const rpc = new RpcClient(child); @@ -136,6 +182,7 @@ export class PiProcessManager { bridge, child, ownerKey: o.principal ? `${o.principal.issuer}\0${o.principal.subject}` : undefined, + ...(o.runtimeConfig ? { releaseRuntimeConfig } : {}), }; rt = runtime; bridge.beginTurn(); @@ -170,7 +217,9 @@ export class PiProcessManager { return runtime; } catch (error) { if (rt && this.runtimes.get(sessionId) === rt) this.runtimes.delete(sessionId); + releaseRuntimeConfig(); try { child.kill(); } catch { /* preserve the initialization error */ } + this.cleanupAgentSnapshot(child); throw error; } } @@ -237,7 +286,9 @@ export class PiProcessManager { // Delete before signalling the child so its asynchronous exit cannot be mistaken for a // crash, and so a replacement installed by a later lifecycle operation is never targeted. this.runtimes.delete(id); + expected.releaseRuntimeConfig?.(); expected.child.kill(); + this.cleanupAgentSnapshot(expected.child); return true; } } diff --git a/backend/src/pi/provider-credentials.ts b/backend/src/pi/provider-credentials.ts index dfac758e..a01d6669 100644 --- a/backend/src/pi/provider-credentials.ts +++ b/backend/src/pi/provider-credentials.ts @@ -53,6 +53,8 @@ export function canonicalPiProvider(provider: string | undefined): string | unde return value; } +export type PiCredentialStatus = "present" | "missing"; + export interface CredentialFsOps { lstat(path: string): Stats; open(path: string, flags: number): number; @@ -172,3 +174,29 @@ export function buildPiChildEnv(opts: { } return env; } + +/** Report only whether the selected hosted provider has a usable credential source. */ +export function piProviderCredentialStatus(opts: { + provider?: string; + credentialFile?: string; + resolveCredentialValue?: () => string | undefined; + authProviders?: ReadonlySet; + fsOps?: CredentialFsOps; +}): PiCredentialStatus { + const provider = canonicalPiProvider(opts.provider); + if (!provider || LOCAL_PROVIDERS.has(provider)) return "missing"; + if (opts.authProviders?.has(provider)) return "present"; + try { + buildPiChildEnv({ + ambient: {}, + provider, + credentialFile: opts.credentialFile, + credentialValue: opts.resolveCredentialValue?.(), + authProviders: opts.authProviders, + fsOps: opts.fsOps, + }); + return "present"; + } catch { + return "missing"; + } +} diff --git a/backend/src/pi/provider-smoke.ts b/backend/src/pi/provider-smoke.ts new file mode 100644 index 00000000..976b0e3c --- /dev/null +++ b/backend/src/pi/provider-smoke.ts @@ -0,0 +1,274 @@ +import { spawn as nodeSpawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AppConfig } from "../config.js"; +import { secretValue } from "../config/secret-bundle.js"; +import { clearPrincipalEnvironment } from "../auth/principal.js"; +import { RpcClient } from "../rpc/rpc-client.js"; +import { loadPiAuthProviders } from "./auth-providers.js"; +import { buildPiChildEnv, canonicalPiProvider } from "./provider-credentials.js"; +import type { PiReasoning } from "./management.js"; +import { + PiManagedConfigError, + isPiManagedConfigError, + parsePiConfigJson, + readConfiguredPiAgentFile, + validateDeclarativePiConfig, +} from "./managed-config.js"; + +const SMOKE_PROMPT = "Provider health check. Reply with exactly OK."; +const SMOKE_ARGS = [ + "--mode", "rpc", + "--no-session", + "--no-tools", + "--no-extensions", + "--no-skills", + "--no-prompt-templates", + "--no-themes", + "--no-context-files", + "--no-approve", +] as const; + +export interface PiProviderSmokeRequest { + provider: string; + model: string; + reasoning: PiReasoning; + timeoutMs: number; +} + +export type PiProviderSmoke = (request: PiProviderSmokeRequest) => Promise; + +interface ProviderSmokeOptions { + spawnFn?: ( + command: string, + args: string[], + options: { cwd: string; env: NodeJS.ProcessEnv }, + ) => ChildProcessWithoutNullStreams; + authProviders?: () => ReadonlySet; + readAuthStore?: () => string; + readModelsStore?: () => string | undefined; +} + +export function createPiProviderSmoke( + config: AppConfig, + options: ProviderSmokeOptions = {}, +): PiProviderSmoke { + const spawnFn = options.spawnFn ?? nodeSpawn; + const authProviders = options.authProviders ?? (() => loadPiAuthProviders()); + + return async ({ provider, model, reasoning, timeoutMs }): Promise => { + let child: ChildProcessWithoutNullStreams | undefined; + let isolatedRoot: string | undefined; + let timer: NodeJS.Timeout | undefined; + try { + const canonicalProvider = canonicalPiProvider(provider); + if (!canonicalProvider || timeoutMs <= 0) throw providerFailure(); + const configuredAuthProviders = authProviders(); + const env = buildPiChildEnv({ + provider: canonicalProvider, + authProviders: configuredAuthProviders, + credentialValue: secretValue(config, "THT_MODEL_API_KEY"), + credentialFile: config.modelApiKeyFile, + }); + clearPrincipalEnvironment(env); + delete env.THT_DATA_ROOT; + delete env.THT_SESSION; + delete env.THT_AUTHOR; + delete env.THT_CONFIG; + delete env.PI_CODING_AGENT_SESSION_DIR; + + isolatedRoot = mkdtempSync(join(tmpdir(), "thothii-pi-smoke-")); + const isolatedCwd = join(isolatedRoot, "work"); + const isolatedAgentDir = join(isolatedRoot, "agent"); + mkdirSync(isolatedCwd, { mode: 0o700 }); + mkdirSync(isolatedAgentDir, { mode: 0o700 }); + if (configuredAuthProviders.has(canonicalProvider)) { + const authStore = selectedProviderAuthStore( + options.readAuthStore?.() ?? readConfiguredPiAgentFile("auth.json"), + canonicalProvider, + ); + writeDeclarativeAgentConfig(join(isolatedAgentDir, "auth.json"), authStore); + } + const configuredModels = options.readModelsStore + ? options.readModelsStore() + : readConfiguredPiAgentFile("models.json", true); + if (configuredModels !== undefined) { + const modelsStore = selectedProviderModelsStore( + configuredModels, + canonicalProvider, + model, + ); + if (modelsStore !== undefined) { + writeDeclarativeAgentConfig(join(isolatedAgentDir, "models.json"), modelsStore); + } + } + env.PI_CODING_AGENT_DIR = isolatedAgentDir; + + child = spawnFn(config.piBin, [...SMOKE_ARGS], { cwd: isolatedCwd, env }); + child.stderr.resume(); + const rpc = new RpcClient(child); + const capabilityGuard = failOnUnexpectedCapabilities(rpc); + const turn = async (): Promise => { + requireSuccessfulResponse(await rpc.request({ + type: "set_model", provider: canonicalProvider, modelId: model, + } as object & { type: string })); + requireSuccessfulResponse(await rpc.request({ + type: "set_thinking_level", level: reasoning, + } as object & { type: string })); + await waitForProviderTurn(rpc, child!); + }; + await Promise.race([ + turn(), + capabilityGuard, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(providerTimeout()), timeoutMs); + }), + ]); + } catch (error) { + if (isProviderTimeout(error)) throw providerTimeout(); + if (isPiManagedConfigError(error)) throw new PiManagedConfigError(); + throw providerFailure(); + } finally { + if (timer) clearTimeout(timer); + if (child) { + try { child.kill(); } catch { /* preserve the sanitized smoke outcome */ } + } + if (isolatedRoot) { + try { rmSync(isolatedRoot, { recursive: true, force: true, maxRetries: 2 }); } catch { + /* preserve the sanitized smoke outcome; the OS temp directory remains isolated */ + } + } + } + }; +} + +function failOnUnexpectedCapabilities(rpc: RpcClient): Promise { + return new Promise((_resolve, reject) => { + rpc.on("event", (event) => { + if (isUnexpectedCapabilityEvent(event)) reject(providerFailure()); + }); + }); +} + +function isUnexpectedCapabilityEvent(event: any): boolean { + const type = typeof event?.type === "string" ? event.type : ""; + if (type.startsWith("tool_") || type.startsWith("toolcall_") || type.startsWith("extension_")) { + return true; + } + const updateType = event?.assistantMessageEvent?.type; + if (typeof updateType === "string" && updateType.startsWith("toolcall_")) return true; + if (Array.isArray(event?.toolResults) && event.toolResults.length > 0) return true; + if (messageUsesTool(event?.message)) return true; + return Array.isArray(event?.messages) && event.messages.some(messageUsesTool); +} + +function messageUsesTool(message: any): boolean { + return message?.role === "toolResult" || message?.stopReason === "toolUse" + || (Array.isArray(message?.content) + && message.content.some((content: any) => content?.type === "toolCall")); +} + +function writeDeclarativeAgentConfig(path: string, raw: string): void { + validateDeclarativePiConfig(raw); + writeFileSync(path, raw, { mode: 0o600, flag: "wx" }); +} + +function selectedProviderAuthStore(raw: string, provider: string): string { + const parsed = parsePiConfigJson(raw); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new PiManagedConfigError(); + } + const entry = Object.entries(parsed as Record) + .find(([key]) => key.trim().toLowerCase() === provider); + if (!entry) throw new PiManagedConfigError(); + return JSON.stringify({ [entry[0]]: entry[1] }); +} + +const PROVIDER_CONFIG_FIELDS = [ + "name", "baseUrl", "apiKey", "api", "headers", "compat", "authHeader", +] as const; + +function selectedProviderModelsStore(raw: string, provider: string, model: string): string | undefined { + const parsed = parsePiConfigJson(raw); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new PiManagedConfigError(); + } + const providers = (parsed as { providers?: unknown }).providers; + if (!providers || typeof providers !== "object" || Array.isArray(providers)) { + throw new PiManagedConfigError(); + } + const entry = Object.entries(providers as Record) + .find(([key]) => key.trim().toLowerCase() === provider); + if (!entry) return undefined; + const providerConfig = entry[1]; + if (!providerConfig || typeof providerConfig !== "object" || Array.isArray(providerConfig)) { + throw new PiManagedConfigError(); + } + const source = providerConfig as Record; + const selected: Record = {}; + for (const field of PROVIDER_CONFIG_FIELDS) { + if (Object.hasOwn(source, field)) selected[field] = source[field]; + } + if (Object.hasOwn(source, "models")) { + if (!Array.isArray(source.models)) throw new PiManagedConfigError(); + let selectedModel: unknown; + for (const candidate of source.models) { + if (candidate && typeof candidate === "object" && !Array.isArray(candidate) + && (candidate as { id?: unknown }).id === model) { + selectedModel = candidate; + } + } + if (selectedModel !== undefined) selected.models = [selectedModel]; + } + if (Object.hasOwn(source, "modelOverrides")) { + const overrides = source.modelOverrides; + if (!overrides || typeof overrides !== "object" || Array.isArray(overrides)) { + throw new PiManagedConfigError(); + } + if (Object.hasOwn(overrides, model)) { + selected.modelOverrides = { [model]: (overrides as Record)[model] }; + } + } + return JSON.stringify({ providers: { [entry[0]]: selected } }); +} + +function waitForProviderTurn(rpc: RpcClient, child: ChildProcessWithoutNullStreams): Promise { + return new Promise((resolve, reject) => { + let failed = false; + rpc.on("event", (event) => { + if (event?.type === "message_end" && event.message?.role === "assistant" + && event.message.stopReason === "error") { + failed = true; + reject(providerFailure()); + return; + } + if (event?.type === "agent_end") { + const messages = Array.isArray(event.messages) ? event.messages : []; + const eventFailed = messages.some((message: any) => ( + message?.role === "assistant" && message?.stopReason === "error" + )); + if (failed || eventFailed) reject(providerFailure()); + else resolve(); + } + }); + child.once("exit", () => reject(providerFailure())); + rpc.send({ type: "prompt", message: SMOKE_PROMPT }); + }); +} + +function requireSuccessfulResponse(response: any): void { + if (!response || response.success !== true) throw providerFailure(); +} + +function providerFailure(): Error { + return new Error("Pi provider smoke check failed"); +} + +function providerTimeout(): Error { + return Object.assign(new Error("Pi smoke check timed out"), { code: "ETIMEDOUT" }); +} + +function isProviderTimeout(error: unknown): boolean { + return Boolean(error && typeof error === "object" && (error as { code?: unknown }).code === "ETIMEDOUT"); +} diff --git a/backend/src/routes/meta.ts b/backend/src/routes/meta.ts index db8149c5..018536a3 100644 --- a/backend/src/routes/meta.ts +++ b/backend/src/routes/meta.ts @@ -2,6 +2,7 @@ import { readdirSync } from "node:fs"; import { join } from "node:path"; import type { FastifyInstance } from "fastify"; import type { PiModel } from "../pi/list-models.js"; +import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; export type ListModelsFn = () => Promise; @@ -26,11 +27,8 @@ export function metaRoutes( app: FastifyInstance, deps: { harnessDir: string; listModels?: ListModelsFn }, ): void { - app.get("/workspaces", async () => { - return listWorkspaces(deps.harnessDir); - }); - - app.get("/models", async () => { + app.get("/models", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply; const fn = deps.listModels ?? (async () => []); try { return { models: await fn() }; diff --git a/backend/src/routes/pi-management.ts b/backend/src/routes/pi-management.ts new file mode 100644 index 00000000..8a3838e6 --- /dev/null +++ b/backend/src/routes/pi-management.ts @@ -0,0 +1,50 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { + isPrincipalContext, + requirePermission, + requireSameOriginOrNonBrowser, +} from "../auth/authorization.js"; +import { PiManagementError, type PiManagementService } from "../pi/management.js"; + +export function piManagementRoutes( + app: FastifyInstance, + deps: { service: PiManagementService }, +): void { + app.get("/pi-management/status", async (request, reply) => run(request, reply, deps, () => deps.service.status())); + app.get("/pi-management/options", async (request, reply) => run(request, reply, deps, () => deps.service.options())); + app.put("/pi-management/config", async (request, reply) => run( + request, + reply, + deps, + () => deps.service.configure((request.body ?? {}) as Record), + )); + app.post("/pi-management/test", async (request, reply) => run(request, reply, deps, () => deps.service.test())); + app.get("/pi-management/logs", async (request, reply) => run(request, reply, deps, () => deps.service.logs())); +} + +async function run( + request: FastifyRequest, + reply: FastifyReply, + deps: { service: PiManagementService }, + action: () => Promise, +): Promise { + const principal = requirePermission(request, reply, "pi.manage"); + if (!isPrincipalContext(principal)) return principal; + if (principal.issuer === "local" && isWrite(request.method)) { + const csrfDenied = requireSameOriginOrNonBrowser(request, reply); + if (csrfDenied) return csrfDenied; + } + try { + return await action(); + } catch (error) { + if (error instanceof PiManagementError) { + const statusCode = error.code === "pi_management_invalid_config" ? 400 : 503; + return reply.code(statusCode).send({ code: error.code, error: error.message }); + } + return reply.code(503).send({ code: "pi_management_unavailable", error: "Pi management is unavailable" }); + } +} + +function isWrite(method: string): boolean { + return method === "POST" || method === "PUT" || method === "PATCH" || method === "DELETE"; +} diff --git a/backend/src/routes/sessions.ts b/backend/src/routes/sessions.ts index f7834422..c7d5ed84 100644 --- a/backend/src/routes/sessions.ts +++ b/backend/src/routes/sessions.ts @@ -7,6 +7,10 @@ import { getPrincipal } from "../auth/auth.js"; import type { PrincipalContext } from "../auth/principal.js"; import type { ReadinessManager } from "../runtime/readiness-manager.js"; import type { ListModelsFn } from "./meta.js"; +import type { WorkspaceRegistry } from "../workspaces/registry.js"; +import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js"; +import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js"; +import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js"; const BOOTSTRAP_FAILURE_MESSAGE = "Session startup failed. Check configuration and connectivity, then Resume the session."; @@ -18,6 +22,8 @@ const DWH_UNREACHABLE_MESSAGE = "Cannot start a session: the database is unreachable. Check the VPN connection and try again."; const MODEL_UNAVAILABLE_MESSAGE = "Selected model is unavailable. Check Pi authentication and model settings, then try again."; +const WORKSPACE_REVISION_UNAVAILABLE_MESSAGE = + "Session workspace configuration is unavailable. Check configuration and try again."; export function sessionRoutes( app: FastifyInstance, @@ -26,8 +32,14 @@ export function sessionRoutes( getSettings: (principal: PrincipalContext) => Promise; readiness: ReadinessManager; listModels: ListModelsFn; + workspaceRegistry: WorkspaceRegistry; /** Local-only guard: probe DWH reachability before creating a session (run-stack.sh). */ dwhPrecheck?: boolean; + /** Explicit loopback-only compatibility path for old clients that send `workspace`. */ + legacyWorkspaceMode?: boolean; + /** Fail-closed installation/runtime transport capability check. */ + workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean; + maintenanceBarrier: MaintenanceBarrier; }, ) { const lifecycleTails = new Map>(); @@ -65,26 +77,137 @@ export function sessionRoutes( const runner = d.tht as any; return typeof runner.withPrincipal === "function" ? runner.withPrincipal(principal) : runner; }; + const ownershipPrincipal = (principal: PrincipalContext, permission: "session.read_all" | "session.manage_all") => ({ + ...principal, + // The harness transition remains isAdmin-based, but only the relevant all-session + // permission can enable its RLS bypass. + isAdmin: hasPermission(principal, permission), + }); + + const optionsWithRuntimeConfig = (runner: any, workspaceConfigPath: string | undefined, options: any) => ( + workspaceConfigPath && typeof runner.acquireWorkspaceRuntime === "function" + ? { ...options, runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath) } + : options + ); + + const maintenanceReply = (reply: any) => reply.code(503).send({ + code: "maintenance", + error: "Session admission is temporarily paused for maintenance. Try again shortly.", + }); + const admissionLeases = new WeakMap void>(); + app.addHook("preHandler", async (req, reply) => { + if (req.method !== "POST" || !(req.url === "/sessions" || /^\/sessions\/[^/]+\/resume(?:\?|$)/.test(req.url))) return; + const release = d.maintenanceBarrier.acquire(); + if (!release) return maintenanceReply(reply); + admissionLeases.set(req, release); + }); + app.addHook("preHandler", async (req, reply) => { + const pathname = req.url.split("?", 1)[0]; + if (pathname === "/runtime/prewarm" || pathname === "/sessions" || pathname.startsWith("/sessions/")) { + const principal = requirePermission(req, reply, "session.use"); + if (!isPrincipalContext(principal)) return principal; + } + }); + app.addHook("onResponse", async (req) => { admissionLeases.get(req)?.(); }); + + /** Include retained historical descriptors so removed workspaces remain resumable. */ + const sessionRevisions = async () => { + const registry = d.workspaceRegistry as Partial; + if (typeof registry.listRetainedSnapshots === "function") { + return await registry.listRetainedSnapshots(); + } + return await d.workspaceRegistry.list(); + }; const isNotFound = (error: unknown) => /not found|non trovata|inesistente|404/i.test(error instanceof Error ? error.message : String(error)); - /** RLS makes a foreign session indistinguishable from a missing one. */ - const authorize = async (principal: PrincipalContext, id: string, workspace?: string): Promise => { + type LocatedSession = { + manifest: any; + workspaceConfigPath: string; + workspace?: WorkspaceDescriptor; + }; + + const workspaceRevisionUnavailable = () => Object.assign( + new Error("workspace revision unavailable"), { code: "workspace_revision_unavailable" }, + ); + + const unavailableWorkspaceReply = (reply: any) => reply.code(409).send({ + error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE, + code: "workspace_revision_unavailable", + }); + + /** + * Find a session by asking every active registry snapshot, never by using the installation + * default. `tht` applies RLS for the supplied principal, so a foreign ID remains a 404. + */ + const locateSession = async ( + principal: PrincipalContext, id: string, permission: "session.read_all" | "session.manage_all" = "session.read_all", + ): Promise => { + const runner = runnerFor(ownershipPrincipal(principal, permission)); + // Dependency-injected runners in legacy route tests may model only the mutation under test. + if (typeof runner.sessionShow !== "function") return { manifest: {}, workspaceConfigPath: "" }; + const legacySession = async (): Promise => { + try { + const manifest = await runner.sessionShow(id); + return manifest && !manifest.workspace_id && !manifest.workspace_revision + ? { manifest, workspaceConfigPath: "" } + : undefined; + } catch (error) { + if (isNotFound(error)) return undefined; + throw error; + } + }; + let revisions: Awaited>; try { - const runner = runnerFor(principal); - // Dependency-injected runners in legacy route tests may model only the mutation under - // test. Production ThtRunner always exposes sessionShow; keep that test seam harmless. - if (typeof runner.sessionShow !== "function") return {}; - const manifest = await runner.sessionShow(id, workspace); - return manifest ?? undefined; - } catch (error) { - if (isNotFound(error)) return undefined; - throw error; + revisions = await sessionRevisions(); + } catch (registryError) { + // Sessions created before revision pinning still live under the installation's legacy + // default config. Keep that compatibility path available when a fresh installation has + // no registry snapshot yet; a pinned session remains fail-closed below. + const legacy = await legacySession(); + if (legacy) return legacy; + throw registryError; + } + for (const revision of revisions) { + try { + const manifest = await runner.sessionShow(id, revision.snapshotPath); + if (manifest) return { manifest, workspaceConfigPath: revision.snapshotPath }; + } catch (error) { + if (isNotFound(error)) continue; + throw error; + } + } + return await legacySession(); + }; + + /** Read the durable pinned descriptor only after the owner-visible manifest is located. */ + const resolveSessionWorkspace = async (located: LocatedSession): Promise => { + const saved = located.manifest as { workspace_id?: string; workspace_revision?: string }; + if (!saved.workspace_id || !saved.workspace_revision) return located; + try { + const pinned = await d.workspaceRegistry.readPinned(saved.workspace_id, saved.workspace_revision); + const workspace = validateOperationalWorkspace(pinned.workspace); + return { + ...located, + workspace, + workspaceConfigPath: pinned.workspaceConfigPath ?? (pinned as any).revision?.snapshotPath, + }; + } catch { + throw workspaceRevisionUnavailable(); } }; + /** RLS makes a foreign session indistinguishable from a missing one. */ + const authorize = async ( + principal: PrincipalContext, id: string, permission: "session.read_all" | "session.manage_all" = "session.read_all", + ): Promise => await locateSession(principal, id, permission); + const storageFailure = (reply: any) => reply.code(503).send({ error: "session storage is unavailable" }); + const lifecycleFailure = (reply: any, error: unknown) => + (error as { code?: string } | undefined)?.code === "workspace_revision_unavailable" + ? unavailableWorkspaceReply(reply) + : storageFailure(reply); const releaseIfFinalized = async ( id: string, rt: ReturnType, @@ -195,97 +318,201 @@ export function sessionRoutes( }); app.post("/sessions", async (req, reply) => { - const b = req.body as { question: string; name?: string }; + const b = req.body as { + question: string; name?: string; workspace?: string; workspaceId?: string; + provider?: string; model?: string; thinking?: string; + }; const principal = getPrincipal(req); let s: Settings; try { s = await d.getSettings(principal); } catch { return storageFailure(reply); } const runner = runnerFor(principal); - // A persisted session is resumable without keeping Pi alive. New work replaces every - // runtime owned by this principal, while runtimes belonging to other users remain intact. - // Optional chaining preserves the deliberately narrow manager stubs used by route tests. - for (const id of d.mgr.teardownForPrincipal?.(principal) ?? []) boundRuntimes.delete(id); - const ensure = await d.readiness.ensure(s.workspace ?? "", principal); - if (!ensure.ok) return reply.code(503).send({ error: READINESS_FAILURE_MESSAGE }); - // Local-only: verify the DWH is reachable BEFORE creating the session, so a dropped - // VPN surfaces as an up-front alert instead of a session that spawns Pi and then dies - // in bootstrap retrieval. `code` lets the client show a specific message. - if (d.dwhPrecheck) { - const ping = await runner.dbPing(s.workspace); - if (!ping.ok) { - console.error(`[dwh-precheck] refusing new session — DWH unreachable: ${ping.detail}`); - return reply.code(503).send({ error: DWH_UNREACHABLE_MESSAGE, code: "dwh_unreachable" }); - } - } - if (s.provider && s.model) { - let available: Awaited>; - try { - available = await d.listModels(); - } catch { - return reply.code(503).send({ - error: MODEL_UNAVAILABLE_MESSAGE, - code: "model_unavailable", - }); - } - const selectedAvailable = available.some( - (candidate) => candidate.provider === s.provider && candidate.id === s.model, - ); - if (!selectedAvailable) { - return reply.code(503).send({ - error: MODEL_UNAVAILABLE_MESSAGE, - code: "model_unavailable", - }); - } - } - // Settings (global) supply workspace/provider/model/thinking. The new-question - // form sends only the question text. `workspace` selects the tht `-c `. - let id: string; - try { - ({ id } = await runner.sessionNew({ - question: b.question, name: b.name, workspace: s.workspace, - provider: s.provider, model: s.model, thinking: s.thinking, - })); - } catch { return storageFailure(reply); } - const options = { - provider: s.provider, - model: s.model, - thinking: s.thinking, - author: principal.displayName ?? principal.subject, - principal, - question: b.question, - }; - let rt: ReturnType | undefined; - try { - rt = d.mgr.createFor(id, options); - bindRuntime(id, rt, runner, s.workspace); - } catch (error) { - if (rt) d.mgr.teardownIfCurrent(id, rt); - console.error( - `[pi:${id}] runtime construction failed:`, - error instanceof Error ? error.message : "unknown error", - ); - await runner.failSession(id, s.workspace).catch((persistenceError: unknown) => { - console.error(`[session:${id}] failSession persistence failed:`, persistenceError); + // `workspace` was the legacy request field before browser-local registry preferences. + // It is available only through the explicit loopback-only compatibility mode; every normal + // new session must resolve and pin an immutable registry revision. + const legacyWorkspaceRequest = d.legacyWorkspaceMode + && typeof b.workspace === "string" && b.workspace.length > 0; + const requestedWorkspaceId = b.workspaceId ?? (legacyWorkspaceRequest ? undefined : s.workspace); + if (!requestedWorkspaceId && !legacyWorkspaceRequest) { + return reply.code(409).send({ + error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE, + code: "workspace_revision_unavailable", }); - return reply.code(503).send({ error: BOOTSTRAP_FAILURE_MESSAGE }); } - info(id, "Session created"); - bootstrap( - id, rt, runner, s.workspace, d.mgr.configure(rt, options), - runner.searchPack(b.question, id, s.workspace), - () => d.mgr.start(id, rt, options), - ); - return { id }; + let revisionLease: Awaited> | undefined; + let manifestPersisted = false; + try { + let workspaceConfigPath: string | undefined; + let workspaceId: string | undefined; + let workspaceRevision: string | undefined; + let workspaceDescriptor: WorkspaceDescriptor | undefined; + let allowedModels: readonly string[] | undefined; + if (requestedWorkspaceId) { + try { + const registry = d.workspaceRegistry as Partial; + const resolved = typeof registry.acquireSessionRevision === "function" + ? await registry.acquireSessionRevision.call(d.workspaceRegistry, requestedWorkspaceId) + : await d.workspaceRegistry.read(requestedWorkspaceId); + if ("markPersisted" in resolved && "abort" in resolved) { + revisionLease = resolved as Awaited>; + } + if (!d.workspaceRuntimeSupport(resolved.workspace)) { + return reply.code(409).send({ + error: "This workspace transport is not available to runtime sessions.", + code: "workspace_not_activatable", + }); + } + workspaceConfigPath = resolved.revision.snapshotPath; + workspaceId = resolved.revision.id; + workspaceRevision = resolved.revision.commit; + workspaceDescriptor = resolved.workspace; + allowedModels = resolved.workspace.llm_policy.allowed; + } catch { + return reply.code(409).send({ + error: WORKSPACE_REVISION_UNAVAILABLE_MESSAGE, + code: "workspace_revision_unavailable", + }); + } + } + const provider = b.provider ?? s.provider; + const model = b.model ?? s.model; + const thinking = b.thinking ?? s.thinking; + if (allowedModels && provider && model && !allowedModels.includes(`${provider}/${model}`)) { + return reply.code(400).send({ error: "Selected model is not allowed by this workspace." }); + } + // A persisted session is resumable without keeping Pi alive. New work replaces every + // runtime owned by this principal, while runtimes belonging to other users remain intact. + // Optional chaining preserves the deliberately narrow manager stubs used by route tests. + for (const id of d.mgr.teardownForPrincipal?.(principal) ?? []) boundRuntimes.delete(id); + const ensure = await d.readiness.ensure( + workspaceConfigPath ?? "", principal, workspaceDescriptor, + ); + if (!ensure.ok) return reply.code(503).send({ + error: READINESS_FAILURE_MESSAGE, + ...(ensure.code ? { code: ensure.code } : {}), + }); + // Local-only: verify the DWH is reachable BEFORE creating the session, so a dropped + // VPN surfaces as an up-front alert instead of a session that spawns Pi and then dies + // in bootstrap retrieval. `code` lets the client show a specific message. + if (d.dwhPrecheck) { + const ping = await runner.dbPing(workspaceConfigPath); + if (!ping.ok) { + console.error(`[dwh-precheck] refusing new session — DWH unreachable: ${ping.detail}`); + return reply.code(503).send({ error: DWH_UNREACHABLE_MESSAGE, code: "dwh_unreachable" }); + } + } + if (provider && model) { + let available: Awaited>; + try { + available = await d.listModels(); + } catch { + return reply.code(503).send({ + error: MODEL_UNAVAILABLE_MESSAGE, + code: "model_unavailable", + }); + } + const selectedAvailable = available.some( + (candidate) => candidate.provider === provider && candidate.id === model, + ); + if (!selectedAvailable) { + return reply.code(503).send({ + error: MODEL_UNAVAILABLE_MESSAGE, + code: "model_unavailable", + }); + } + } + // Browser choices are copied to the persisted manifest together with the immutable + // registry snapshot. The legacy fallback stays available for sessions created before + // the browser-local preference migration. + let id: string; + try { + ({ id } = await runner.sessionNew({ + question: b.question, name: b.name, workspaceConfigPath, + workspaceId, workspaceRevision, provider, model, thinking, + })); + manifestPersisted = true; + if (revisionLease) { + await revisionLease.markPersisted().catch((error: unknown) => { + console.error( + `[session:${id}] revision lease hand-off failed:`, + error instanceof Error ? error.message : "unknown error", + ); + }); + } + } catch { return storageFailure(reply); } + const options = { + provider, model, thinking, + author: principal.displayName ?? principal.subject, + principal, + question: b.question, + }; + let runtimeOptions = options; + let rt: ReturnType | undefined; + try { + runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options); + rt = d.mgr.createFor(id, runtimeOptions); + bindRuntime(id, rt, runner, workspaceConfigPath); + } catch (error) { + if (rt) d.mgr.teardownIfCurrent(id, rt); + console.error( + `[pi:${id}] runtime construction failed:`, + error instanceof Error ? error.message : "unknown error", + ); + await runner.failSession(id, workspaceConfigPath).catch((persistenceError: unknown) => { + console.error(`[session:${id}] failSession persistence failed:`, persistenceError); + }); + return reply.code(503).send({ error: BOOTSTRAP_FAILURE_MESSAGE }); + } + info(id, "Session created"); + bootstrap( + id, rt, runner, workspaceConfigPath, d.mgr.configure(rt, runtimeOptions), + runner.searchPack(b.question, id, workspaceConfigPath), + () => d.mgr.start(id, rt, runtimeOptions), + ); + return { id }; + } finally { + if (revisionLease && !manifestPersisted) { + await revisionLease.abort().catch((error: unknown) => { + console.error( + "[session] revision lease cleanup failed:", + error instanceof Error ? error.message : "unknown error", + ); + }); + } + } }); app.get("/sessions", async (req, reply) => { const principal = getPrincipal(req); const scope = (req.query as { scope?: string }).scope ?? "mine"; if (scope !== "mine" && scope !== "all") return reply.code(400).send({ error: "scope must be mine or all" }); - if (scope === "all" && !principal.isAdmin) return reply.code(403).send({ error: "admin scope required" }); + if (scope === "all") { + const allPrincipal = requirePermission(req, reply, "session.read_all"); + if (!isPrincipalContext(allPrincipal)) return allPrincipal; + } try { - const settings = await d.getSettings(principal); // Admin RLS is deliberately disabled for a normal 'mine' listing. - const scopedPrincipal = scope === "mine" ? { ...principal, isAdmin: false } : principal; - const list: SessionRow[] = await runnerFor(scopedPrincipal).sessionList(settings.workspace); + const scopedPrincipal = scope === "mine" + ? { ...principal, isAdmin: false } + : ownershipPrincipal(principal, "session.read_all"); + const runner = runnerFor(scopedPrincipal); + const revisions = await sessionRevisions(); + const lists = await Promise.all(revisions + .map((revision) => runner.sessionList(revision.snapshotPath) as Promise)); + const sessions = new Map(); + for (const row of lists.flat()) { + if (!sessions.has(row.id)) sessions.set(row.id, row); + } + const list = [...sessions.values()]; + // Only an administrator-visible complete list (or the single local principal) is safe + // input for retention. A remote per-user view can never discard another principal's pin. + const reconcileSnapshotRetention = (d.workspaceRegistry as Partial).reconcileSnapshotRetention; + const hasCompleteRetentionView = (scope === "all" || principal.issuer === "local") + && hasPermission(principal, "session.read_all"); + if (hasCompleteRetentionView && typeof reconcileSnapshotRetention === "function") { + const retained = [...new Set(list + .filter((row) => row.status !== "finalized" && !row.archived && typeof row.workspace_revision === "string") + .map((row) => row.workspace_revision!))]; + await reconcileSnapshotRetention.call(d.workspaceRegistry, retained); + } // Annotate each row with whether a live Pi runtime is currently bound. The client // opens an `active` session straight into its live view (reconnecting to its pending // gate), while a cold session keeps its explicit Resume affordance — so a mere click @@ -296,18 +523,20 @@ export function sessionRoutes( app.get("/sessions/:id", async (req, reply) => { const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - const manifest = await authorize(principal, (req.params as any).id, settings.workspace); - return manifest ?? reply.code(404).send({ error: "session not found" }); - } catch { return storageFailure(reply); } + const session = await authorize(principal, (req.params as any).id); + if (!session) return reply.code(404).send({ error: "session not found" }); + const manifest = session.manifest; + return (!manifest.workspace_id || !manifest.workspace_revision) + ? { ...manifest, warning: "Legacy session: this session is not pinned to a workspace revision." } + : manifest; + } catch (error) { return lifecycleFailure(reply, error); } }); app.post("/sessions/:id/response", async (req, reply) => { const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - } catch { return storageFailure(reply); } + if (!await authorize(principal, id, "session.manage_all")) return reply.code(404).send({ error: "session not found" }); + } catch (error) { return lifecycleFailure(reply, error); } const rt = d.mgr.get(id); if (!rt) return reply.code(404).send({ error: "sessione non attiva" }); if (!rt.bridge.respond((req.body as any).ui_response)) { @@ -319,9 +548,8 @@ export function sessionRoutes( const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - } catch { return storageFailure(reply); } + if (!await authorize(principal, id, "session.manage_all")) return reply.code(404).send({ error: "session not found" }); + } catch (error) { return lifecycleFailure(reply, error); } const rt = d.mgr.get(id); if (!rt) return reply.code(404).send({ error: "sessione non attiva" }); rt.bridge.steer((req.body as any).text); @@ -332,18 +560,31 @@ export function sessionRoutes( const principal = getPrincipal(req); return withSessionLifecycle(id, async () => { let settings: Settings; - let manifest: any; + let located: LocatedSession | undefined; try { - settings = await d.getSettings(principal); - manifest = await authorize(principal, id, settings.workspace); + located = await locateSession(principal, id, "session.manage_all"); } catch { return storageFailure(reply); } - if (!manifest) return reply.code(404).send({ error: "session not found" }); + if (!located) return reply.code(404).send({ error: "session not found" }); + const manifest = located.manifest; const runner = runnerFor(principal); - // Read-only contract FIRST: a finalized/archived session must refuse resume even - // when a lingering runtime still looks active — the manifest is the truth. + // Read-only contract FIRST: finalized or archived sessions never attempt compatibility + // resolution, even when their historical snapshot was subsequently pruned. if (manifest?.status === "finalized" || manifest?.archived) { return reply.code(409).send({ error: "sessione in sola lettura (finalizzata o archiviata)" }); } + const saved = manifest as { + provider?: string; model?: string; thinking?: string; + workspace_id?: string; workspace_revision?: string; + }; + let workspaceConfigPath: string; + let workspaceDescriptor: WorkspaceDescriptor | undefined; + try { + const resolved = await resolveSessionWorkspace(located); + workspaceConfigPath = resolved.workspaceConfigPath; + workspaceDescriptor = resolved.workspace; + } + catch { return unavailableWorkspaceReply(reply); } + try { settings = await d.getSettings(principal); } catch { return storageFailure(reply); } // This check belongs inside the per-session lock: a preceding cold Resume may have // installed a running runtime while this request was waiting. const existing = d.mgr.get(id); @@ -353,9 +594,13 @@ export function sessionRoutes( return reply.code(200).send({ id, alreadyActive: true }); } } - const ensure = await d.readiness.ensure(settings.workspace ?? "", principal); - if (!ensure.ok) return reply.code(503).send({ error: READINESS_FAILURE_MESSAGE }); - const saved = manifest as { provider?: string; model?: string; thinking?: string } | null; + const ensure = await d.readiness.ensure( + workspaceConfigPath ?? "", principal, workspaceDescriptor, + ); + if (!ensure.ok) return reply.code(503).send({ + error: READINESS_FAILURE_MESSAGE, + ...(ensure.code ? { code: ensure.code } : {}), + }); const options = { provider: saved?.provider, model: saved?.model, @@ -364,11 +609,12 @@ export function sessionRoutes( principal, mode: "resume" as const, }; + let runtimeOptions = options; // Reopening is validation, not the transport commit point. Keep the old hub intact if // persistence cannot be reopened. try { - await runner.reopenSession(id, settings.workspace); + await runner.reopenSession(id, workspaceConfigPath); } catch { return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE }); } @@ -393,8 +639,9 @@ export function sessionRoutes( if (boundRuntimes.get(id) === current) boundRuntimes.delete(id); d.mgr.teardownIfCurrent(id, current); } - rt = d.mgr.createFor(id, options); - bindRuntime(id, rt, runner, settings.workspace); + runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options); + rt = d.mgr.createFor(id, runtimeOptions); + bindRuntime(id, rt, runner, workspaceConfigPath); } catch { // A created-but-unbound runtime is not usable. The old hub remains attached because // clear() has not happened yet. @@ -409,7 +656,11 @@ export function sessionRoutes( // immediately before the first event produced by the new Resume. d.hub.clear(id); info(id, "Resuming session"); - bootstrap(id, rt, runner, settings.workspace, d.mgr.configure(rt, options), null, () => d.mgr.start(id, rt, options)); + bootstrap( + id, rt, runner, workspaceConfigPath, + d.mgr.configure(rt, runtimeOptions), null, + () => d.mgr.start(id, rt, runtimeOptions), + ); return reply.code(200).send({ id, alreadyActive: false }); }); }); @@ -417,20 +668,20 @@ export function sessionRoutes( const id = (req.params as { id: string }).id; const principal = getPrincipal(req); return withSessionLifecycle(id, async () => { - let settings: Settings; + let session: LocatedSession | undefined; try { - settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - } catch { return storageFailure(reply); } + session = await authorize(principal, id, "session.manage_all"); + if (!session) return reply.code(404).send({ error: "session not found" }); + } catch (error) { return lifecycleFailure(reply, error); } // Invalidate the live generation before persistence can yield. Otherwise its deferred // bootstrap may start Pi while Close is already in progress. const current = d.mgr.get(id); boundRuntimes.delete(id); if (current) d.mgr.teardownIfCurrent(id, current); try { - await runnerFor(principal).closeSession(id, settings.workspace); - } catch { - return storageFailure(reply); + await runnerFor(principal).closeSession(id, session.workspaceConfigPath); + } catch (error) { + return lifecycleFailure(reply, error); } finally { // clear, NOT forget: a closed session can be reopened, and the per-session seq // monotonicity is what keeps a browser's old cursor detectable. The buffer is @@ -444,20 +695,32 @@ export function sessionRoutes( const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - } catch { return storageFailure(reply); } + if (!await authorize(principal, id)) return reply.code(404).send({ error: "session not found" }); + } catch (error) { return lifecycleFailure(reply, error); } const rt = d.mgr.get(id); - // Add CORS headers manually: reply.raw.writeHead bypasses Fastify's onSend hooks - // (where @fastify/cors injects headers), so we must set them explicitly here. - const origin = (req.headers.origin as string | undefined) ?? "*"; + // reply.raw.writeHead bypasses Fastify's CORS hook. Only a cookie-authenticated request + // from the exact configured public origin receives credentialed SSE CORS headers. + const origin = typeof req.headers.origin === "string" ? req.headers.origin : undefined; + let isConfiguredOrigin = false; + try { + isConfiguredOrigin = req.authPublicOrigin !== undefined + && origin !== undefined + && new URL(origin).origin === req.authPublicOrigin; + } catch { + isConfiguredOrigin = false; + } + const corsHeaders = isConfiguredOrigin + ? { + "Access-Control-Allow-Origin": req.authPublicOrigin, + "Access-Control-Allow-Credentials": "true", + } + : {}; reply.raw.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", "X-Accel-Buffering": "no", Connection: "keep-alive", - "Access-Control-Allow-Origin": origin, - "Access-Control-Allow-Credentials": "true", + ...corsHeaders, }); // Send the handshake immediately. Without this, Node waits for the first event body and // proxies/clients cannot establish an idle SSE subscription or inspect its headers. @@ -481,58 +744,58 @@ export function sessionRoutes( const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - await runnerFor(principal).setName(id, (req.body as any).name, settings.workspace); - } catch { return storageFailure(reply); } + const session = await authorize(principal, id, "session.manage_all"); + if (!session) return reply.code(404).send({ error: "session not found" }); + await runnerFor(principal).setName(id, (req.body as any).name, session.workspaceConfigPath); + } catch (error) { return lifecycleFailure(reply, error); } return reply.code(204).send(); }); app.post("/sessions/:id/group", async (req, reply) => { const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - await runnerFor(principal).setGroup(id, (req.body as any).group, settings.workspace); - } catch { return storageFailure(reply); } + const session = await authorize(principal, id, "session.manage_all"); + if (!session) return reply.code(404).send({ error: "session not found" }); + await runnerFor(principal).setGroup(id, (req.body as any).group, session.workspaceConfigPath); + } catch (error) { return lifecycleFailure(reply, error); } return reply.code(204).send(); }); app.post("/sessions/:id/archive", async (req, reply) => { const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - await runnerFor(principal).archive(id, settings.workspace); - } catch { return storageFailure(reply); } + const session = await authorize(principal, id, "session.manage_all"); + if (!session) return reply.code(404).send({ error: "session not found" }); + await runnerFor(principal).archive(id, session.workspaceConfigPath); + } catch (error) { return lifecycleFailure(reply, error); } return reply.code(204).send(); }); app.post("/sessions/:id/unarchive", async (req, reply) => { const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - await runnerFor(principal).unarchive(id, settings.workspace); - } catch { return storageFailure(reply); } + const session = await authorize(principal, id, "session.manage_all"); + if (!session) return reply.code(404).send({ error: "session not found" }); + await runnerFor(principal).unarchive(id, session.workspaceConfigPath); + } catch (error) { return lifecycleFailure(reply, error); } return reply.code(204).send(); }); app.delete("/sessions/:id", async (req, reply) => { const id = (req.params as any).id; const principal = getPrincipal(req); return withSessionLifecycle(id, async () => { - let settings: Settings; + let session: LocatedSession | undefined; try { - settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - } catch { return storageFailure(reply); } + session = await authorize(principal, id, "session.manage_all"); + if (!session) return reply.code(404).send({ error: "session not found" }); + } catch (error) { return lifecycleFailure(reply, error); } const current = d.mgr.get(id); boundRuntimes.delete(id); if (current) d.mgr.teardownIfCurrent(id, current); try { - await runnerFor(principal).deleteSession(id, settings.workspace); - } catch { - return storageFailure(reply); + await runnerFor(principal).deleteSession(id, session.workspaceConfigPath); + } catch (error) { + return lifecycleFailure(reply, error); } d.hub.forget(id); return reply.code(204).send(); @@ -542,9 +805,9 @@ export function sessionRoutes( const id = (req.params as any).id; const principal = getPrincipal(req); try { - const settings = await d.getSettings(principal); - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); - return await runnerFor(principal).documents(id, settings.workspace); - } catch { return storageFailure(reply); } + const session = await authorize(principal, id); + if (!session) return reply.code(404).send({ error: "session not found" }); + return await runnerFor(principal).documents(id, session.workspaceConfigPath); + } catch (error) { return lifecycleFailure(reply, error); } }); } diff --git a/backend/src/routes/settings.ts b/backend/src/routes/settings.ts index d6a542ca..71afd333 100644 --- a/backend/src/routes/settings.ts +++ b/backend/src/routes/settings.ts @@ -1,18 +1,18 @@ import type { FastifyInstance } from "fastify"; import type { AppConfig } from "../config.js"; -import { loadSettings, saveSettings, type Settings } from "../settings/settings-store.js"; +import type { Settings } from "../settings/settings-store.js"; import { listWorkspaces, type ListModelsFn } from "./meta.js"; -import { getPrincipal } from "../auth/auth.js"; import type { PrincipalContext } from "../auth/principal.js"; +import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; /** Merge stored settings over env/first-workspace defaults. */ export function effectiveSettings(cfg: AppConfig, stored: Settings): Settings { const workspaces = listWorkspaces(cfg.harnessDir); return { - workspace: stored.workspace ?? (workspaces[0]?.name), - provider: stored.provider ?? cfg.defaults.provider, - model: stored.model ?? cfg.defaults.model, - thinking: stored.thinking ?? cfg.defaults.thinking, + workspace: stored.workspace ?? workspaces[0]?.name, + provider: cfg.defaults.provider ?? stored.provider, + model: cfg.defaults.model ?? stored.model, + thinking: cfg.defaults.thinking ?? stored.thinking, }; } @@ -21,18 +21,21 @@ export function settingsRoutes( deps: { cfg: AppConfig; listModels: ListModelsFn; getSettings: (principal: PrincipalContext) => Promise; - saveSettings: (principal: PrincipalContext, settings: Settings) => Promise; }, ): void { app.get("/settings", async (req, reply) => { + const principal = requirePermission(req, reply, "session.use"); + if (!isPrincipalContext(principal)) return principal; try { - return await deps.getSettings(getPrincipal(req)); + return await deps.getSettings(principal); } catch { return reply.code(503).send({ error: "settings storage is unavailable" }); } }); app.put("/settings", async (req, reply) => { + const principal = requirePermission(req, reply, "settings.manage"); + if (!isPrincipalContext(principal)) return principal; const b = (req.body ?? {}) as Settings; if (b.model) { let available: { provider: string; id: string }[] = []; @@ -50,15 +53,10 @@ export function settingsRoutes( }); } } - const next: Settings = { - workspace: b.workspace, - provider: b.provider, - model: b.model, - thinking: b.thinking, - }; try { - await deps.saveSettings(getPrincipal(req), next); - return effectiveSettings(deps.cfg, next); + // Retain this endpoint as a validating compatibility surface for older clients, but do + // not write anonymous users' choices to shared server storage. + return await deps.getSettings(principal); } catch { return reply.code(503).send({ error: "settings storage is unavailable" }); } diff --git a/backend/src/routes/sql.ts b/backend/src/routes/sql.ts index f47304cb..c547fdeb 100644 --- a/backend/src/routes/sql.ts +++ b/backend/src/routes/sql.ts @@ -1,24 +1,55 @@ import type { FastifyInstance } from "fastify"; import type { ThtRunner } from "../tht/tht-runner.js"; -import { getPrincipal } from "../auth/auth.js"; import type { PrincipalContext } from "../auth/principal.js"; import type { Settings } from "../settings/settings-store.js"; +import type { WorkspaceRegistry } from "../workspaces/registry.js"; +import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js"; export function sqlRoutes(app: FastifyInstance, deps: { tht: ThtRunner; getSettings: (principal: PrincipalContext) => Promise; + workspaceRegistry: WorkspaceRegistry; }): void { const runnerFor = (principal: PrincipalContext): any => { const runner = deps.tht as any; return typeof runner.withPrincipal === "function" ? runner.withPrincipal(principal) : runner; }; - const authorize = async (principal: PrincipalContext, id: string, workspace?: string) => { - try { - const runner = runnerFor(principal); - if (typeof runner.sessionShow !== "function") return {}; - return await runner.sessionShow(id, workspace); + const readPrincipal = (principal: PrincipalContext): PrincipalContext => ({ + ...principal, + // The harness still consumes this compatibility bit; it must never exceed session.read_all. + isAdmin: hasPermission(principal, "session.read_all"), + }); + const isNotFound = (error: unknown) => /not found|non trovata|inesistente|404/i.test( + error instanceof Error ? error.message : String(error), + ); + const locate = async (principal: PrincipalContext, id: string, legacyWorkspace?: string) => { + const runner = runnerFor(readPrincipal(principal)); + if (typeof runner.sessionShow !== "function") return { manifest: {}, workspace: legacyWorkspace }; + const registry = deps.workspaceRegistry as Partial; + const revisions = typeof registry.listRetainedSnapshots === "function" + ? await registry.listRetainedSnapshots.call(deps.workspaceRegistry) + : await deps.workspaceRegistry.list(); + for (const revision of revisions) { + try { + const manifest = await runner.sessionShow(id, revision.snapshotPath); + if (!manifest) continue; + const saved = manifest as { workspace_id?: string; workspace_revision?: string }; + if (saved.workspace_id && saved.workspace_revision) { + const pinned = await deps.workspaceRegistry.readPinned(saved.workspace_id, saved.workspace_revision); + return { + manifest, + workspace: pinned.workspaceConfigPath ?? (pinned as any).revision?.snapshotPath, + }; + } + return { manifest, workspace: revision.snapshotPath }; + } catch (error) { + if (!isNotFound(error)) throw error; + } } - catch (error) { - if (/not found|non trovata|inesistente|404/i.test(error instanceof Error ? error.message : String(error))) return undefined; + try { + const manifest = await runner.sessionShow(id, legacyWorkspace); + return manifest ? { manifest, workspace: legacyWorkspace } : undefined; + } catch (error) { + if (isNotFound(error)) return undefined; throw error; } }; @@ -28,15 +59,18 @@ export function sqlRoutes(app: FastifyInstance, deps: { let principal: PrincipalContext; let workspace: string | undefined; try { - principal = getPrincipal(req); + const authorized = requirePermission(req, reply, "session.use"); + if (!isPrincipalContext(authorized)) return authorized; + principal = authorized; const settings = await deps.getSettings(principal); - workspace = settings.workspace; - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); + const located = await locate(principal, id, settings.workspace); + if (!located) return reply.code(404).send({ error: "session not found" }); + workspace = located.workspace; } catch { return reply.code(503).send({ error: "session storage is unavailable" }); } try { - return await runnerFor(principal).sqlPreview(id, { limit, offset }, workspace); + return await runnerFor(readPrincipal(principal)).sqlPreview(id, { limit, offset }, workspace); } catch (error: any) { return reply.code(500).send({ error: error.message ?? String(error) }); } @@ -47,15 +81,18 @@ export function sqlRoutes(app: FastifyInstance, deps: { let principal: PrincipalContext; let workspace: string | undefined; try { - principal = getPrincipal(req); + const authorized = requirePermission(req, reply, "session.use"); + if (!isPrincipalContext(authorized)) return authorized; + principal = authorized; const settings = await deps.getSettings(principal); - workspace = settings.workspace; - if (!await authorize(principal, id, settings.workspace)) return reply.code(404).send({ error: "session not found" }); + const located = await locate(principal, id, settings.workspace); + if (!located) return reply.code(404).send({ error: "session not found" }); + workspace = located.workspace; } catch { return reply.code(503).send({ error: "session storage is unavailable" }); } try { - return await runnerFor(principal).sqlExport(id, workspace); + return await runnerFor(readPrincipal(principal)).sqlExport(id, workspace); } catch (error: any) { return reply.code(500).send({ error: error.message ?? String(error) }); } diff --git a/backend/src/routes/workspaces.ts b/backend/src/routes/workspaces.ts new file mode 100644 index 00000000..a2b4a563 --- /dev/null +++ b/backend/src/routes/workspaces.ts @@ -0,0 +1,259 @@ +import type { FastifyInstance, FastifyReply } from "fastify"; +import { z } from "zod"; +import type { WorkspaceRegistryConfig } from "../workspaces/types.js"; +import { WorkspaceRegistryError } from "../workspaces/git-repository.js"; +import type { WorkspaceRegistry } from "../workspaces/registry.js"; +import { buildInstallationContract } from "../workspaces/contracts.js"; +import { + discoverWorkspaceSecretRequirements, + resolveRuntimeBindingsWithWorkspaceSecrets, +} from "../workspaces/secret-requirements.js"; +import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; +import { + validateOperationalWorkspace, + validateWorkspaceDescriptor, + type CanonicalWorkspace, + type WorkspaceDescriptor, +} from "../workspaces/schema.js"; +import type { RuntimeBindings } from "../workspaces/runtime-renderer.js"; +import type { ConnectorDiagnostics } from "../workspaces/diagnostics.js"; +import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; +import type { AuthDiagnoser } from "../auth/diagnostics.js"; +import { decodeAuthDiagnostics, type AuthDiagnostics } from "../auth/group-catalog.js"; + +export type WorkspaceDiagnoser = ( + workspace: WorkspaceDescriptor, + bindings: RuntimeBindings, + options: { writeProbe: boolean }, +) => Promise; + +interface WorkspaceRoutesDeps { + registry: WorkspaceRegistry; + config: WorkspaceRegistryConfig; + diagnose: WorkspaceDiagnoser; + authDiagnoser: AuthDiagnoser; + secretStore: WorkspaceSecretStore; +} + +const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/); +const workspacePayload = z.object({ workspace: z.unknown() }).strict(); +const secretRequirementId = z.string().regex(/^[a-z0-9][a-z0-9._-]{1,127}$/); +const secretValuesPayload = z.object({ + values: z.record(secretRequirementId, z.string()).refine( + (values) => Object.keys(values).length > 0 && Object.keys(values).length <= 16, + ), +}).strict(); + +const SAFE_MESSAGES = { + workspace_invalid: "Workspace request is invalid.", + binding_missing: "Installation binding is missing or invalid.", + workspace_not_activatable: "Workspace cannot be activated on this installation.", + workspace_stale: "Workspace repository state is stale.", + git_unavailable: "Workspace Git service is unavailable.", + git_auth_failed: "Workspace Git authentication failed.", + git_non_fast_forward: "Workspace Git branch has changed.", + connector_unavailable: "Workspace connector is unavailable.", + semantic_index_incompatible: "Semantic index is incompatible with this workspace.", +} as const; + +function authenticationReport(value: unknown): AuthDiagnostics { + const report = decodeAuthDiagnostics(value); + if (!report) throw new Error("invalid authentication diagnostic report"); + return report; +} + +function workspaceErrorCode(error: unknown): keyof typeof SAFE_MESSAGES { + return error instanceof WorkspaceRegistryError ? error.code : "workspace_invalid"; +} + +function workspaceErrorStatus(code: keyof typeof SAFE_MESSAGES): number { + if (code === "workspace_stale" || code === "git_non_fast_forward") return 409; + if (code === "git_unavailable" || code === "git_auth_failed") return 503; + return 400; +} + +function errorReply(reply: FastifyReply, error: unknown) { + const code = workspaceErrorCode(error); + return reply.code(workspaceErrorStatus(code)).send({ code, message: SAFE_MESSAGES[code] }); +} + +export function workspaceRoutes(app: FastifyInstance, deps: WorkspaceRoutesDeps): void { + const runtimeConfiguration = async (id: string) => { + const { workspace, revision } = await deps.registry.read(id); + const operational = validateOperationalWorkspace(workspace); + const requirements = discoverWorkspaceSecretRequirements(operational, process.env) + .map((requirement) => ({ + id: requirement.id, + connector: requirement.connector, + label: requirement.label, + description: requirement.description, + input: requirement.input, + required: requirement.required, + configured: deps.secretStore.has(id, requirement.id), + })); + return { + workspaceId: id, + revision, + configurationState: requirements.some(({ required, configured }) => required && !configured) + ? "configuration_required" as const + : "ready" as const, + requirements, + }; + }; + + app.get("/workspace-registry/status", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "workspace.manage"))) return reply; + try { + return await deps.registry.bootstrap(); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspace-registry/pull", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "workspace.manage"))) return reply; + try { + return await deps.registry.pull(); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.get("/workspaces", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply; + try { + const records = await deps.registry.listCatalog(); + return await Promise.all(records.map(async (record) => { + const configuration = await runtimeConfiguration(record.id); + return { + id: record.id, + // Retain the metadata endpoint's selector field while adding catalog metadata. + name: record.id, + file: `${record.id}/workspace.yaml`, + displayName: record.name, + description: record.description, + configurationState: configuration.configurationState, + ...(record.revision ? { revision: record.revision } : {}), + }; + })); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.get("/workspaces/:id", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply; + try { + const { id } = z.object({ id: workspaceId }).parse(request.params); + return await deps.registry.read(id); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspaces/validate", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "workspace.manage"))) return reply; + try { + const { workspace } = workspacePayload.parse(request.body); + const canonical = validateWorkspaceDescriptor(workspace); + const authentication = authenticationReport(await deps.authDiagnoser.inspect({ live: false })); + return { + workspace: canonical, + contract: buildInstallationContract(canonical), + activatable: authentication.ready, + diagnostics: [], + authentication, + }; + } catch (error) { + return errorReply(reply, error); + } + }); + + app.get("/workspaces/:id/runtime-configuration", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "session.use"))) return reply; + try { + const { id } = z.object({ id: workspaceId }).parse(request.params); + return await runtimeConfiguration(id); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.put("/workspaces/:id/secrets", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "workspace.secrets.manage"))) return reply; + try { + const { id } = z.object({ id: workspaceId }).parse(request.params); + const { values } = secretValuesPayload.parse(request.body); + const { workspace } = await deps.registry.read(id); + const declared = new Set( + discoverWorkspaceSecretRequirements(validateOperationalWorkspace(workspace), process.env) + .map(({ id: requirementId }) => requirementId), + ); + if (Object.keys(values).some((requirementId) => !declared.has(requirementId))) { + throw new Error("undeclared workspace secret"); + } + deps.secretStore.putMany(id, values); + return await runtimeConfiguration(id); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.delete("/workspaces/:id/secrets/:requirementId", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "workspace.secrets.manage"))) return reply; + try { + const { id, requirementId } = z.object({ + id: workspaceId, + requirementId: secretRequirementId, + }).parse(request.params); + const { workspace } = await deps.registry.read(id); + const declared = discoverWorkspaceSecretRequirements( + validateOperationalWorkspace(workspace), process.env, + ).some(({ id: candidate }) => candidate === requirementId); + if (!declared) throw new Error("undeclared workspace secret"); + deps.secretStore.forget(id, requirementId); + return await runtimeConfiguration(id); + } catch (error) { + return errorReply(reply, error); + } + }); + + app.post("/workspaces/:id/test", async (request, reply) => { + if (!isPrincipalContext(requirePermission(request, reply, "workspace.manage"))) return reply; + try { + const { id } = z.object({ id: workspaceId }).parse(request.params); + const { workspace } = await deps.registry.read(id); + let operational: CanonicalWorkspace; + try { + operational = validateOperationalWorkspace(workspace); + } catch { + throw new WorkspaceRegistryError( + "workspace_not_activatable", "Workspace requires explicit migration", + ); + } + const lease = resolveRuntimeBindingsWithWorkspaceSecrets( + operational, + process.env, + deps.config.secretRoots, + deps.secretStore, + ); + try { + const [workspaceDiagnostics, inspectedAuthentication] = await Promise.all([ + deps.diagnose(operational, lease.bindings, { writeProbe: false }), + deps.authDiagnoser.inspect({ live: true }), + ]); + const authentication = authenticationReport(inspectedAuthentication); + return { + ...workspaceDiagnostics, + activatable: workspaceDiagnostics.activatable && authentication.ready, + authentication, + }; + } finally { + lease.release(); + } + } catch (error) { + return errorReply(reply, error); + } + }); + +} diff --git a/backend/src/runtime/maintenance-gate.ts b/backend/src/runtime/maintenance-gate.ts new file mode 100644 index 00000000..569c79da --- /dev/null +++ b/backend/src/runtime/maintenance-gate.ts @@ -0,0 +1,134 @@ +import { + closeSync, + existsSync, + fsyncSync, + mkdirSync, + openSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname } from "node:path"; + +export interface MaintenanceDurability { + writeFile?(descriptor: number, contents: string): void; + syncFile?(descriptor: number): void; + syncDirectory(directory: string): void; +} + +/** A durable admission barrier. A lease spans the complete create/resume decision. */ +export class MaintenanceBarrier { + private active: boolean; + private admissions = 0; + private waiters: (() => void)[] = []; + private recoveryRequired = false; + + constructor( + private readonly markerFile?: string, + private readonly durability: MaintenanceDurability = defaultDurability, + ) { + this.active = markerFile === undefined ? false : existsSync(markerFile); + } + + acquire(): (() => void) | undefined { + this.reconcileActive(); + if (this.active) return undefined; + this.admissions += 1; + let released = false; + return () => { + if (released) return; + released = true; + this.admissions -= 1; + if (this.admissions === 0) this.waiters.splice(0).forEach((resolve) => resolve()); + }; + } + + async activate(): Promise { + let persistError: unknown; + if (this.markerFile === undefined) { + this.active = true; + this.recoveryRequired = false; + } else { + try { + this.persistMarker(); + this.recoveryRequired = false; + } catch (error) { + persistError = error; + this.recoveryRequired = true; + } finally { + this.reconcileActive(); + } + } + if (!this.active) throw persistError; + if (this.admissions > 0) { + await new Promise((resolve) => this.waiters.push(resolve)); + } + if (persistError !== undefined) throw persistError; + } + + deactivate(): void { + if (this.markerFile === undefined) { + this.active = false; + this.recoveryRequired = false; + return; + } + try { + this.removeMarker(); + this.recoveryRequired = false; + } catch (error) { + try { this.persistMarker(); } catch { /* marker existence is reconciled below */ } + this.recoveryRequired = true; + throw error; + } finally { + this.reconcileActive(); + } + } + status(): { active: boolean; admissions: number; recoveryRequired?: true } { + this.reconcileActive(); + const status = { active: this.active, admissions: this.admissions }; + return this.recoveryRequired ? { ...status, recoveryRequired: true } : status; + } + + private reconcileActive(): void { + if (this.markerFile !== undefined) this.active = existsSync(this.markerFile); + } + + private persistMarker(): void { + if (!this.markerFile) return; + const directory = dirname(this.markerFile); + mkdirSync(directory, { recursive: true }); + const temporary = `${this.markerFile}.tmp-${process.pid}-${Date.now()}`; + const fd = openSync(temporary, "wx", 0o600); + let closed = false; + try { + const contents = '{"version":1,"active":true}\n'; + if (this.durability.writeFile) this.durability.writeFile(fd, contents); + else writeFileSync(fd, contents, "utf8"); + if (this.durability.syncFile) this.durability.syncFile(fd); + else fsyncSync(fd); + closeSync(fd); + closed = true; + renameSync(temporary, this.markerFile); + this.durability.syncDirectory(directory); + } catch (error) { + if (!closed) try { closeSync(fd); } catch { /* preserve the original failure */ } + try { unlinkSync(temporary); } catch { /* already renamed or best-effort cleanup */ } + throw error; + } + } + + private removeMarker(): void { + if (!this.markerFile) return; + if (existsSync(this.markerFile)) unlinkSync(this.markerFile); + else if (!this.recoveryRequired) return; + this.durability.syncDirectory(dirname(this.markerFile)); + } +} + +const defaultDurability: MaintenanceDurability = { + syncDirectory(directory: string): void { + if (process.platform === "win32") return; + const fd = openSync(directory, "r"); + try { fsyncSync(fd); } finally { closeSync(fd); } + }, +}; diff --git a/backend/src/runtime/readiness-manager.ts b/backend/src/runtime/readiness-manager.ts index 1c49494e..0257c3d3 100644 --- a/backend/src/runtime/readiness-manager.ts +++ b/backend/src/runtime/readiness-manager.ts @@ -1,9 +1,16 @@ -import type { OllamaEnsureResult, ThtRunner } from "../tht/tht-runner.js"; +import type { + OllamaEnsureResult, + SemanticReadinessCode, + ThtRunner, +} from "../tht/tht-runner.js"; import type { PrincipalContext } from "../auth/principal.js"; +import type { WorkspaceDescriptor } from "../workspaces/schema.js"; + +export type ReadinessResult = OllamaEnsureResult & { code?: SemanticReadinessCode }; interface ReadyEntry { expiresAt: number; - result: OllamaEnsureResult; + result: ReadinessResult; } /** @@ -11,7 +18,7 @@ interface ReadyEntry { * Failures are deliberately not cached so a submit can retry after a transient outage. */ export class ReadinessManager { - private inFlight = new Map>(); + private inFlight = new Map>(); private ready = new Map(); constructor( @@ -21,7 +28,11 @@ export class ReadinessManager { private now: () => number = Date.now, ) {} - ensure(workspace = "", principal?: PrincipalContext): Promise { + ensure( + workspace = "", + principal?: PrincipalContext, + descriptor?: WorkspaceDescriptor, + ): Promise { const key = `${principal?.issuer ?? ""}\0${principal?.subject ?? ""}\0${workspace}`; const cached = this.ready.get(key); if (cached && cached.expiresAt > this.now()) return Promise.resolve(cached.result); @@ -32,7 +43,20 @@ export class ReadinessManager { const runner = principal && typeof (this.tht as any).withPrincipal === "function" ? this.tht.withPrincipal(principal) : this.tht; - const pending = runner.ollamaEnsure(workspace, this.timeoutSec) + const pending = (async (): Promise => { + try { + if (descriptor) { + const qdrant = await runner.qdrantEnsure(descriptor, this.timeoutSec, "self_heal"); + if (!qdrant.ok) return qdrant; + } + const ollama = await runner.ollamaEnsure(workspace, this.timeoutSec); + return ollama.ok + ? ollama + : { ...ollama, code: "workspace_not_activatable" }; + } catch { + return { ok: false, code: "workspace_not_activatable" }; + } + })() .then((result) => { if (result.ok) { this.ready.set(key, { result, expiresAt: this.now() + this.ttlMs }); diff --git a/backend/src/server.ts b/backend/src/server.ts index 6cef6b0d..726aedb3 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -1,6 +1,24 @@ -import { buildApp } from "./app.js"; +import { buildApp, type AppWithAuthSessionStore } from "./app.js"; import { loadConfig } from "./config.js"; -const config = loadConfig(process.env); -const app = buildApp(config); -app.listen({ port: config.port, host: config.host }) - .then((addr) => console.log(`backend listening on ${addr}`)); +import { formatStartupFailure } from "./startup-error.js"; + +async function start(): Promise { + const config = loadConfig(process.env); + const app = buildApp(config) as AppWithAuthSessionStore; + const sessions = app.thothiiAuthSessionStore; + if (sessions) { + await sessions.prune(); + const interval = setInterval(() => { + void sessions.prune().catch(() => app.log.warn({ component: "auth-session-prune" }, "auth session pruning failed")); + }, 15 * 60 * 1000); + interval.unref(); + app.addHook("onClose", async () => clearInterval(interval)); + } + const address = await app.listen({ port: config.port, host: config.host }); + console.log(`backend listening on ${address}`); +} + +void start().catch((error: unknown) => { + console.error(formatStartupFailure(error)); + process.exitCode = 1; +}); diff --git a/backend/src/settings/settings-cli.ts b/backend/src/settings/settings-cli.ts new file mode 100644 index 00000000..5bfbc817 --- /dev/null +++ b/backend/src/settings/settings-cli.ts @@ -0,0 +1,48 @@ +/* Core-side, non-interactive installation-default writer used only through compose exec. + * It accepts no credentials and writes the same SETTINGS_FILE consumed by session creation. */ +import { readFileSync } from "node:fs"; +import { loadConfig } from "../config.js"; +import { + captureSettingsSnapshot, + loadSettings, + restoreSettingsSnapshot, + saveSettings, + type Settings, + type SettingsSnapshot, +} from "./settings-store.js"; + +const choice = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$/; + +function value(args: string[], flag: string): string { + const at = args.indexOf(flag); + if (at < 0 || at + 1 >= args.length || args.filter((part) => part === flag).length !== 1) { + throw new Error(`missing ${flag}`); + } + return args[at + 1]; +} + +try { + const args = process.argv.slice(2); + const cfg = loadConfig(process.env); + if (args.length === 1 && args[0] === "--snapshot") { + process.stdout.write(`${JSON.stringify(captureSettingsSnapshot(cfg))}\n`); + } else if (args.length === 1 && args[0] === "--restore") { + const parsed = JSON.parse(readFileSync(0, "utf8")) as Partial; + if (Object.keys(parsed).some((key) => key !== "exists" && key !== "rawBase64")) { + throw new Error("invalid settings snapshot"); + } + restoreSettingsSnapshot(cfg, parsed as SettingsSnapshot); + } else { + if (args.length !== 6) throw new Error("only provider, model, and thinking may be configured"); + const provider = value(args, "--provider"); + const model = value(args, "--model"); + const thinking = value(args, "--thinking"); + if (!choice.test(provider) || !choice.test(model)) throw new Error("invalid provider or model"); + if (!["low", "medium", "high"].includes(thinking)) throw new Error("invalid thinking level"); + const next: Settings = { ...loadSettings(cfg), provider, model, thinking }; + saveSettings(cfg, next); + } +} catch (error) { + process.stderr.write(`settings-cli: ${error instanceof Error ? error.message : "invalid configuration"}\n`); + process.exitCode = 2; +} diff --git a/backend/src/settings/settings-store.ts b/backend/src/settings/settings-store.ts index 7783a40f..d42792f5 100644 --- a/backend/src/settings/settings-store.ts +++ b/backend/src/settings/settings-store.ts @@ -1,4 +1,13 @@ -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { + closeSync, + fsyncSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { dirname } from "node:path"; import type { AppConfig } from "../config.js"; @@ -9,6 +18,20 @@ export interface Settings { thinking?: string; } +export interface SettingsDurability { + syncDirectory(directory: string): void; +} + +export interface SettingsSnapshot { + exists: boolean; + rawBase64: string; +} + +/** + * Settings files are installation defaults only. Personal workspace/model/thinking choices + * belong to the browser and must never be written back here by request handlers. + */ + /** Read settings from cfg.settingsFile. Returns {} if missing or invalid. */ export function loadSettings(cfg: AppConfig): Settings { try { @@ -21,9 +44,105 @@ export function loadSettings(cfg: AppConfig): Settings { } } +/** Capture exact file existence and bytes so host-side configuration can compensate losslessly. */ +export function captureSettingsSnapshot(cfg: AppConfig): SettingsSnapshot { + try { + return { exists: true, rawBase64: readFileSync(cfg.settingsFile).toString("base64") }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { exists: false, rawBase64: "" }; + } + throw error; + } +} + +/** Restore a previously captured settings file exactly, including the clean absent state. */ +export function restoreSettingsSnapshot( + cfg: AppConfig, + snapshot: SettingsSnapshot, + durability: SettingsDurability = defaultDurability, +): void { + if (typeof snapshot.exists !== "boolean" || typeof snapshot.rawBase64 !== "string") { + throw new Error("invalid settings snapshot"); + } + const raw = Buffer.from(snapshot.rawBase64, "base64"); + if (raw.toString("base64") !== snapshot.rawBase64 || (!snapshot.exists && raw.length !== 0)) { + throw new Error("invalid settings snapshot"); + } + const directory = dirname(cfg.settingsFile); + mkdirSync(directory, { recursive: true }); + if (snapshot.exists) { + replaceSettingsFile(cfg.settingsFile, raw); + } else { + try { + unlinkSync(cfg.settingsFile); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + durability.syncDirectory(directory); +} + /** Persist settings (pretty JSON). Creates the parent directory if needed. */ -export function saveSettings(cfg: AppConfig, s: Settings): Settings { +export function saveSettings( + cfg: AppConfig, + s: Settings, + durability: SettingsDurability = defaultDurability, +): Settings { mkdirSync(dirname(cfg.settingsFile), { recursive: true }); - writeFileSync(cfg.settingsFile, JSON.stringify(s, null, 2) + "\n", "utf8"); + const directory = dirname(cfg.settingsFile); + let previous: Buffer | undefined; + try { + previous = readFileSync(cfg.settingsFile); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + + replaceSettingsFile(cfg.settingsFile, Buffer.from(JSON.stringify(s, null, 2) + "\n", "utf8")); + try { + durability.syncDirectory(directory); + } catch (durabilityError) { + try { + if (previous === undefined) unlinkSync(cfg.settingsFile); + else replaceSettingsFile(cfg.settingsFile, previous); + durability.syncDirectory(directory); + } catch { + throw new Error("settings durability failed and previous settings could not be restored", { + cause: durabilityError, + }); + } + throw durabilityError; + } return s; } + +let temporarySequence = 0; + +function replaceSettingsFile(path: string, contents: Buffer): void { + const temporary = `${path}.tmp-${process.pid}-${Date.now()}-${temporarySequence++}`; + const fd = openSync(temporary, "wx", 0o600); + try { + writeFileSync(fd, contents); + fsyncSync(fd); + } catch (error) { + try { closeSync(fd); } catch { /* preserve the write error */ } + try { unlinkSync(temporary); } catch { /* best effort */ } + throw error; + } + closeSync(fd); + try { + renameSync(temporary, path); + } catch (error) { + try { unlinkSync(temporary); } catch { /* best effort */ } + throw error; + } +} + +const defaultDurability: SettingsDurability = { + syncDirectory(directory: string): void { + // The core image runs Linux. Windows durability is owned by the host-side Go executable. + if (process.platform === "win32") return; + const dirFd = openSync(directory, "r"); + try { fsyncSync(dirFd); } finally { closeSync(dirFd); } + }, +}; diff --git a/backend/src/startup-error.ts b/backend/src/startup-error.ts new file mode 100644 index 00000000..feb49ed8 --- /dev/null +++ b/backend/src/startup-error.ts @@ -0,0 +1,18 @@ +const STARTUP_CAUSES = new Set([ + "auth_config_invalid", + "auth_session_store_invalid", + "workspace_registry_invalid", +]); + +const EXACT_CAUSES = new Map([ + ["authentication configuration is invalid", "auth_config_invalid"], + ["authentication session state is invalid", "auth_session_store_invalid"], + ["workspace registry configuration is invalid", "workspace_registry_invalid"], +]); + +/** Return one bounded machine cause; never include the original error text or stack. */ +export function formatStartupFailure(error: unknown): string { + const message = error instanceof Error ? error.message : ""; + const cause = STARTUP_CAUSES.has(message) ? message : EXACT_CAUSES.get(message) ?? "startup_unknown"; + return `backend startup failed: ${cause}`; +} diff --git a/backend/src/tht/tht-runner.ts b/backend/src/tht/tht-runner.ts index 26dabe3d..8b681a9f 100644 --- a/backend/src/tht/tht-runner.ts +++ b/backend/src/tht/tht-runner.ts @@ -1,14 +1,47 @@ import { spawn } from "node:child_process"; -import { existsSync } from "node:fs"; -import { join } from "node:path"; +import { createHash, randomUUID } from "node:crypto"; +import { + closeSync, constants as fsConstants, existsSync, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, + openSync, readFileSync, readSync, realpathSync, statSync, unlinkSync, writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { parseAllDocuments } from "yaml"; import { clearPrincipalEnvironment, principalEnvironment, type PrincipalContext } from "../auth/principal.js"; import { secretValue, type SecretBundleConfig } from "../config/secret-bundle.js"; +import { renderWorkspaceRuntimeFromSnapshotPath } from "../workspaces/runtime-config-lease.js"; +import { + DEFAULT_SEMANTIC_RUNTIME, + type RuntimeInstallationOverlay, + type RuntimePaths, + type SemanticRuntimeConfig, +} from "../workspaces/runtime-renderer.js"; +import { + parseWorkspaceYaml, + validateOperationalWorkspace, + type WorkspaceDescriptor, +} from "../workspaces/schema.js"; +import { reconcileCollection } from "../workspaces/qdrant-collection.js"; +import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; export interface ThtConfig extends SecretBundleConfig { thtBin: string; harnessDir: string; configPath: string; dataRoot?: string; + runtimeSnapshotRoot?: string; + secretRoots?: readonly string[]; + semanticRuntime: SemanticRuntimeConfig; + qdrantRequest?: typeof fetch; + /** "self_heal" for session admission (create missing collections/indexes), default "require_existing". */ + qdrantCollectionMode?: "self_heal" | "require_existing"; + workspaceSecretStore?: WorkspaceSecretStore; +} + +export interface RuntimeConfigLease { + path: string; + workspaceId: string; + workspaceRevision: string; + release(): void; } export interface SessionRow { @@ -19,6 +52,9 @@ export interface SessionRow { created_at: string; updated_at: string | null; author: string | null; + workspace_id?: string | null; + workspace_revision?: string | null; + archived?: boolean; } export interface SessionDocument { @@ -38,7 +74,36 @@ export interface OllamaEnsureResult { model_name?: string; } +export type SemanticReadinessCode = "workspace_not_activatable" | "semantic_index_incompatible"; + +export interface QdrantEnsureResult { + ok: boolean; + code?: SemanticReadinessCode; +} + +const REQUIRED_QDRANT_PAYLOAD_INDEXES = [ + "content_hash", + "document_id", + "kind", + "record_key", + "record_kind", + "vector_generation", + "workspace_id", + "workspace_revision", +] as const; + +interface RuntimeSnapshot { + path: string; + dev: number; + ino: number; + size: number; + mode: number; + digest: string; +} + export class ThtRunner { + private readonly runtimeSnapshots = new Map(); + constructor(private cfg: ThtConfig, private principal?: PrincipalContext) {} /** Bind one trusted request principal to every child spawned by this runner. */ @@ -49,8 +114,17 @@ export class ThtRunner { * back to the default config would point every operation at the wrong workspace * (wrong DB, wrong sessions dir) — fail loud instead. */ - private configArg(workspace?: string): string[] { - if (workspace) { + private configArg(workspaceConfigPath?: string): string[] { + if (workspaceConfigPath) { + if (isAbsolute(workspaceConfigPath)) { + if (this.runtimeSnapshots.has(workspaceConfigPath)) this.assertTrustedRuntimeSnapshot(workspaceConfigPath); + else this.assertWorkspaceSnapshot(workspaceConfigPath); + return ["-c", workspaceConfigPath]; + } + if (workspaceConfigPath.includes("/")) { + throw new Error("workspace snapshot config path must be absolute"); + } + const workspace = workspaceConfigPath; if (!existsSync(join(this.cfg.harnessDir, "workspaces", `${workspace}.yaml`))) { throw new Error(`workspace non trovato: workspaces/${workspace}.yaml (harness: ${this.cfg.harnessDir})`); } @@ -59,6 +133,228 @@ export class ThtRunner { return ["-c", this.cfg.configPath]; } + private assertWorkspaceSnapshot(path: string): { workspaceId: string; workspaceRevision: string } { + if (!this.cfg.runtimeSnapshotRoot) throw new Error("workspace snapshot root is not configured"); + const snapshotsRoot = dirname(this.cfg.runtimeSnapshotRoot); + const pathRelative = relative(snapshotsRoot, path); + const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(pathRelative); + if ( + pathRelative.startsWith("..") || isAbsolute(pathRelative) + || !match + ) throw new Error("config path is not a trusted runtime snapshot"); + const entry = lstatSync(path); + if (!entry.isFile() || entry.isSymbolicLink()) { + throw new Error("config path is not a trusted runtime snapshot"); + } + return { workspaceRevision: match[1], workspaceId: match[2] }; + } + + private readCanonicalWorkspaceSnapshot(path: string): { + workspace: ReturnType; + workspaceId: string; + workspaceRevision: string; + revisionContentRoot: string; + } { + const identity = this.assertWorkspaceSnapshot(path); + const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const before = fstatSync(fd); + if (!before.isFile()) throw new Error("workspace snapshot is not a file"); + const source = readFileSync(fd, "utf8"); + const after = fstatSync(fd); + if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) { + throw new Error("workspace snapshot changed while reading"); + } + const workspace = validateOperationalWorkspace(parseWorkspaceYaml(source)); + if (workspace.workspace.id !== identity.workspaceId) { + throw new Error("workspace snapshot identity does not match its path"); + } + return { workspace, ...identity, revisionContentRoot: dirname(path) }; + } finally { + closeSync(fd); + } + } + + private runtimePaths(workspaceId: string): RuntimePaths { + if (!this.cfg.dataRoot || !isAbsolute(this.cfg.dataRoot)) { + throw new Error("registry workspace runtime requires an absolute data root"); + } + // The portable stack persists one `sessions` store at /sessions. Keep every + // workspace's mutable harness roots below that mounted boundary. + const root = join(this.cfg.dataRoot, "sessions", workspaceId); + return { + sessions: join(root, "sessions"), + artifacts: join(root, "artifacts"), + indexes: join(root, "indexes"), + memory: join(root, "memory"), + }; + } + + private installationOverlay(): RuntimeInstallationOverlay { + const path = isAbsolute(this.cfg.configPath) + ? this.cfg.configPath + : resolve(this.cfg.harnessDir, this.cfg.configPath); + if (!existsSync(path)) return {}; + const documents = parseAllDocuments(readFileSync(path, "utf8"), { uniqueKeys: true }); + if (documents.length !== 1) throw new Error("installation config must contain one YAML document"); + const document = documents[0]; + if (document.errors.length > 0 || document.warnings.length > 0) { + throw new Error("installation config contains invalid YAML"); + } + const parsed = document.toJSON(); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("installation config must be a YAML mapping"); + } + const source = parsed as Record; + return { + ...(source.session_storage === undefined ? {} : { session_storage: source.session_storage }), + ...(source.profile === undefined ? {} : { profile: source.profile }), + }; + } + + /** Render one immutable canonical registry revision into a backend-owned harness config. */ + acquireWorkspaceRuntime(workspaceConfigPath: string): RuntimeConfigLease { + const rendered = renderWorkspaceRuntimeFromSnapshotPath({ + snapshotPath: workspaceConfigPath, + harnessDir: this.cfg.harnessDir, + configPath: this.cfg.configPath, + dataRoot: this.cfg.dataRoot ?? (() => { + throw new Error("registry workspace runtime requires an absolute data root"); + })(), + secretRoots: this.cfg.secretRoots ?? [], + semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME, + workspaceSecretStore: this.cfg.workspaceSecretStore, + }); + let path: string; + try { + path = this.createRuntimeSnapshot(rendered.renderedConfig); + } catch (error) { + rendered.releaseSecrets(); + throw error; + } + let released = false; + return { + path, + workspaceId: rendered.workspaceId, + workspaceRevision: rendered.workspaceRevision, + release: () => { + if (released) return; + released = true; + this.cleanupRuntimeSnapshot(path); + rendered.releaseSecrets(); + }, + }; + } + + private runtimeSnapshotDirectory(): string { + if (!this.cfg.runtimeSnapshotRoot) throw new Error("runtime snapshot root is not configured"); + if (!isAbsolute(this.cfg.runtimeSnapshotRoot)) throw new Error("runtime snapshot root must be absolute"); + mkdirSync(this.cfg.runtimeSnapshotRoot, { recursive: true, mode: 0o700 }); + const directory = lstatSync(this.cfg.runtimeSnapshotRoot); + if (!directory.isDirectory() || directory.isSymbolicLink() || (directory.mode & 0o077) !== 0) { + throw new Error("runtime snapshot root is not trusted"); + } + return realpathSync(this.cfg.runtimeSnapshotRoot); + } + + private static isRestrictiveMode(mode: number): boolean { + const permissions = mode & 0o777; + return (permissions & 0o400) !== 0 && (permissions & ~0o600) === 0; + } + + private assertTrustedRuntimeSnapshot(path: string): RuntimeSnapshot { + const snapshot = this.runtimeSnapshots.get(path); + if (!snapshot) throw new Error("config path is not a trusted runtime snapshot"); + try { + const entry = lstatSync(path); + const stat = statSync(path); + if ( + !entry.isFile() || entry.isSymbolicLink() + || stat.dev !== snapshot.dev || stat.ino !== snapshot.ino || stat.size !== snapshot.size + || (stat.mode & 0o777) !== snapshot.mode || !ThtRunner.isRestrictiveMode(stat.mode) + || createHash("sha256").update(readFileSync(path)).digest("hex") !== snapshot.digest + ) throw new Error("changed runtime snapshot"); + return snapshot; + } catch { + throw new Error("config path is not a trusted runtime snapshot"); + } + } + + /** Create an opaque, backend-owned temporary config that is safe to hand to `tht`. */ + createRuntimeSnapshot(config: string): string { + const directory = this.runtimeSnapshotDirectory(); + const path = join(directory, `runtime-${randomUUID()}.yaml`); + const fd = openSync( + path, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, + 0o600, + ); + try { + writeFileSync(fd, config, "utf8"); + fsyncSync(fd); + fchmodSync(fd, 0o400); + const stat = fstatSync(fd); + this.runtimeSnapshots.set(path, { + path, + dev: stat.dev, + ino: stat.ino, + size: stat.size, + mode: stat.mode & 0o777, + digest: createHash("sha256").update(config, "utf8").digest("hex"), + }); + return path; + } catch (error) { + try { unlinkSync(path); } catch { /* creation did not produce a removable file */ } + throw error; + } finally { + closeSync(fd); + } + } + + cleanupRuntimeSnapshot(path: string): void { + const snapshot = this.runtimeSnapshots.get(path); + if (!snapshot) return; + this.runtimeSnapshots.delete(path); + try { unlinkSync(snapshot.path); } catch { /* a changed path is never removed recursively */ } + } + + private openTrustedRuntimeSnapshot(path: string): number { + const snapshot = this.assertTrustedRuntimeSnapshot(path); + const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const stat = fstatSync(fd); + if ( + !stat.isFile() || stat.dev !== snapshot.dev || stat.ino !== snapshot.ino || stat.size !== snapshot.size + || (stat.mode & 0o777) !== snapshot.mode || !ThtRunner.isRestrictiveMode(stat.mode) + ) throw new Error("changed runtime snapshot"); + const contents = Buffer.alloc(snapshot.size); + let offset = 0; + while (offset < contents.length) { + const bytes = readSync(fd, contents, offset, contents.length - offset, offset); + if (bytes === 0) throw new Error("truncated runtime snapshot"); + offset += bytes; + } + if (createHash("sha256").update(contents).digest("hex") !== snapshot.digest) { + throw new Error("changed runtime snapshot"); + } + return fd; + } catch { + closeSync(fd); + throw new Error("config path is not a trusted runtime snapshot"); + } + } + + async runWithRuntimeSnapshot( + args: string[], config: string, timeoutMs: number = ThtRunner.DEFAULT_TIMEOUT_MS, + ): Promise<{ code: number; stdout: string; stderr: string }> { + const snapshot = this.createRuntimeSnapshot(config); + try { + return await this.run(args, snapshot, timeoutMs); + } finally { + this.cleanupRuntimeSnapshot(snapshot); + } + } + /** * Build the full argv for a `tht` invocation. `--config`/`-c` is a PER-COMMAND * option in the `tht` CLI (there is NO global `-c`), so it MUST be appended @@ -75,8 +371,20 @@ export class ThtRunner { static readonly DWH_TIMEOUT_MS = 120_000; run( - args: string[], workspace?: string, timeoutMs: number = ThtRunner.DEFAULT_TIMEOUT_MS, + args: string[], workspaceConfigPath?: string, timeoutMs: number = ThtRunner.DEFAULT_TIMEOUT_MS, ): Promise<{ code: number; stdout: string; stderr: string }> { + if ( + workspaceConfigPath && isAbsolute(workspaceConfigPath) + && !this.runtimeSnapshots.has(workspaceConfigPath) + ) { + let runtime: RuntimeConfigLease; + try { + runtime = this.acquireWorkspaceRuntime(workspaceConfigPath); + } catch (error) { + return Promise.reject(error); + } + return this.run(args, runtime.path, timeoutMs).finally(runtime.release); + } return new Promise((resolve) => { const env: NodeJS.ProcessEnv = { ...process.env }; delete env.THT_DATA_ROOT; @@ -94,10 +402,26 @@ export class ThtRunner { env.THT_CA = ca; env.THT_SSL_CA = ca; } - const ch = spawn(this.cfg.thtBin, this.buildArgv(args, workspace), { - cwd: this.cfg.harnessDir, - env, - }); + let snapshotFd: number | undefined; + let ch; + try { + snapshotFd = workspaceConfigPath && this.runtimeSnapshots.has(workspaceConfigPath) + ? this.openTrustedRuntimeSnapshot(workspaceConfigPath) + : undefined; + ch = spawn( + this.cfg.thtBin, + snapshotFd === undefined + ? this.buildArgv(args, workspaceConfigPath) + : [...args, "-c", "/dev/fd/3"], + { + cwd: this.cfg.harnessDir, + env, + ...(snapshotFd === undefined ? {} : { stdio: ["ignore", "pipe", "pipe", snapshotFd] }), + }, + ); + } finally { + if (snapshotFd !== undefined) closeSync(snapshotFd); + } let stdout = ""; let stderr = ""; let settled = false; @@ -114,8 +438,8 @@ export class ThtRunner { finish({ code: 124, stdout, stderr: stderr || `timed out after ${timeoutMs}ms` }); }, timeoutMs); } - ch.stdout.on("data", (d: Buffer) => (stdout += d)); - ch.stderr.on("data", (d: Buffer) => (stderr += d)); + ch.stdout?.on("data", (d: Buffer) => (stdout += d)); + ch.stderr?.on("data", (d: Buffer) => (stderr += d)); ch.on("error", (error) => finish({ code: 1, stdout, stderr: stderr || error.message })); ch.on("close", (code) => finish({ code: code ?? 0, stdout, stderr })); }); @@ -138,7 +462,11 @@ export class ThtRunner { model?: string; thinking?: string; name?: string; + /** Legacy named-workspace compatibility; pinned sessions use workspaceConfigPath. */ workspace?: string; + workspaceConfigPath?: string; + workspaceId?: string; + workspaceRevision?: string; }) { const a = ["session", "new", o.question]; for (const [f, v] of [ @@ -146,11 +474,13 @@ export class ThtRunner { ["--model", o.model], ["--thinking", o.thinking], ["--name", o.name], + ["--workspace-id", o.workspaceId], + ["--workspace-revision", o.workspaceRevision], ] as const) { if (v) a.push(f, v); } a.push("--json"); - return this.json<{ id: string }>(a, o.workspace); + return this.json<{ id: string }>(a, o.workspaceConfigPath ?? o.workspace); } /** Build and persist the deterministic F1 retrieval pack for a new session. */ @@ -241,4 +571,37 @@ export class ThtRunner { error: parsed?.error ?? (stderr.trim() || `tht ollama ensure exit ${code}`), }; } + + async qdrantEnsure( + workspace: WorkspaceDescriptor, + timeoutSec: number, + mode: "self_heal" | "require_existing" = "require_existing", + ): Promise { + let descriptor; + try { + descriptor = validateOperationalWorkspace(workspace); + } catch { + return { ok: false, code: "workspace_not_activatable" }; + } + const collection = descriptor.semantic_index.vector_store; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutSec) * 1000); + try { + const checked = await reconcileCollection({ + baseUrl: this.cfg.semanticRuntime.internalQdrantUrl, + collection: collection.collection, + dimensions: collection.dimensions, + distance: collection.distance, + mode, + request: this.cfg.qdrantRequest ?? fetch, + signal: controller.signal, + }); + return checked; + } catch { + return { ok: false, code: "workspace_not_activatable" }; + } finally { + clearTimeout(timer); + controller.abort(); + } + } } diff --git a/backend/src/workspace-maintenance.ts b/backend/src/workspace-maintenance.ts new file mode 100644 index 00000000..0ec8bbce --- /dev/null +++ b/backend/src/workspace-maintenance.ts @@ -0,0 +1,331 @@ +import { spawn } from "node:child_process"; +import { closeSync, constants as fsConstants, openSync } from "node:fs"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { parse } from "yaml"; +import { loadConfig } from "./config.js"; +import { ThtRunner } from "./tht/tht-runner.js"; +import { WorkspaceRegistry } from "./workspaces/registry.js"; +import { publishDeterministicRuntimeConfigLease, renderActiveWorkspaceRuntime } from "./workspaces/runtime-config-lease.js"; +import { WorkspaceSecretStore } from "./workspaces/secret-store.js"; +import { WorkspacePreprocessingService, type WorkspaceOperationResult } from "./workspaces/preprocessing-service.js"; +import type { SessionInventoryRow } from "./workspaces/preprocessing-state.js"; + +export interface WorkspaceMaintenanceIo { + stdin: string; + stdout: string[]; + stderr: string[]; + writeStdout(value: string): void; + writeStderr(value: string): void; +} + +type Command = "inspect" | "preprocess-dwh" | "schema-suggest-fks" | "schema-check" | "schema-accept" | "index-schema" | "preprocess-evidence" | "preprocess-run" | "vector-inspect" | "vector-rebuild"; + +function failureResult( + operation: string, + workspaceId = "", + code: WorkspaceOperationResult["code"] = "workspace_not_activatable", +): WorkspaceOperationResult { + return { + schemaVersion: 1, + status: "failed", + code, + workspaceId, + workspaceRevision: "", + descriptorBlob: "", + operation, + completedStages: [], + }; +} + +const STATE_ERROR_CODES: Record = { + preprocessing_resume_mismatch: "preprocessing_resume_mismatch", + preprocessing_conflict: "preprocessing_conflict", + effective_config_mismatch: "effective_config_mismatch", +}; + +function boundedJson(result: WorkspaceOperationResult): string { + const encoded = JSON.stringify(result); + if (Buffer.byteLength(encoded, "utf8") > 1024 * 1024) { + return JSON.stringify(failureResult(result.operation || "unknown", result.workspaceId)); + } + return encoded; +} + +function sanitizeStderr(_error: unknown): string { + // Never return raw exception text: it may embed endpoints, tokens, or SQL. + return "workspace maintenance failed\n"; +} + +function parseRequest(command: string, stdin: string): Record { + const parsed = JSON.parse(stdin) as Record; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || parsed.schemaVersion !== 1) { + throw new Error("invalid request"); + } + const allowedByCommand: Record = { + inspect: ["schemaVersion", "workspaceId"], + "preprocess-dwh": ["schemaVersion", "workspaceId", "resumeRunId"], + "schema-suggest-fks": ["schemaVersion", "workspaceId", "fromSql", "assume", "resumeRunId"], + "schema-check": ["schemaVersion", "workspaceId", "annotationsYaml", "reviewedCandidatesDigest"], + "schema-accept": ["schemaVersion", "workspaceId", "runId", "yes"], + "index-schema": ["schemaVersion", "workspaceId", "resumeRunId"], + "preprocess-evidence": ["schemaVersion", "workspaceId", "dryRun", "resumeRunId"], + "preprocess-run": ["schemaVersion", "workspaceId", "resumeRunId"], + "vector-inspect": ["schemaVersion", "workspaceId"], + "vector-rebuild": ["schemaVersion", "workspaceId", "collection", "confirm", "destroy"], + }; + const allowed = allowedByCommand[command]; + if (!allowed) throw new Error("unknown command"); + if (typeof parsed.workspaceId !== "string") throw new Error("invalid workspace id"); + for (const key of Object.keys(parsed)) if (!allowed.includes(key)) throw new Error("unexpected request field"); + return parsed; +} + +function exitCodeFor(result: WorkspaceOperationResult): number { + if (["succeeded", "unchanged", "dry_run"].includes(result.status)) return 0; + if (result.status === "blocked") return 3; + return 1; +} + +async function dispatch(command: Command, service: WorkspacePreprocessingService, request: Record): Promise { + switch (command) { + case "inspect": + return await service.inspect({ workspaceId: request.workspaceId as string }); + case "preprocess-dwh": + return await service.preprocessDwh({ + workspaceId: request.workspaceId as string, + resumeRunId: request.resumeRunId as string | undefined, + }); + case "schema-suggest-fks": + return await service.suggestFks({ + workspaceId: request.workspaceId as string, + fromSql: request.fromSql as any, + assume: request.assume as any, + resumeRunId: request.resumeRunId as string | undefined, + }); + case "schema-check": + return await service.checkSchema({ + workspaceId: request.workspaceId as string, + annotationsYaml: request.annotationsYaml as string | undefined, + reviewedCandidatesDigest: request.reviewedCandidatesDigest as string | undefined, + }); + case "schema-accept": + return await service.acceptSchema({ + workspaceId: request.workspaceId as string, + runId: request.runId as string, + yes: request.yes === true, + }); + case "index-schema": + return await service.indexSchema({ + workspaceId: request.workspaceId as string, + resumeRunId: request.resumeRunId as string | undefined, + }); + case "preprocess-evidence": + return await service.preprocessEvidence({ + workspaceId: request.workspaceId as string, + dryRun: request.dryRun as boolean | undefined, + resumeRunId: request.resumeRunId as string | undefined, + }); + case "preprocess-run": + return await service.run({ + workspaceId: request.workspaceId as string, + resumeRunId: request.resumeRunId as string | undefined, + }); + case "vector-inspect": + return await service.vectorInspect({ workspaceId: request.workspaceId as string }); + case "vector-rebuild": + return await service.vectorRebuild({ + workspaceId: request.workspaceId as string, + collection: request.collection as string | undefined, + confirm: request.confirm as string | undefined, + destroy: request.destroy === true, + }); + } +} + +export async function runWorkspaceMaintenanceCli( + argv: readonly string[], + service: WorkspacePreprocessingService, + io: WorkspaceMaintenanceIo, +): Promise { + const command = argv[2]; + if (!command) { + const result = failureResult("unknown"); + io.writeStdout(boundedJson(result)); + return 2; + } + try { + const request = parseRequest(command, io.stdin); + const result = await dispatch(command as Command, service, request); + io.writeStdout(boundedJson(result)); + return exitCodeFor(result); + } catch (error) { + const failureCode = error instanceof Error + && "code" in error + && typeof (error as { code?: unknown }).code === "string" + && (error as { code: string }).code in STATE_ERROR_CODES + ? STATE_ERROR_CODES[(error as { code: string }).code] + : "workspace_not_activatable"; + const result = failureResult(command, (() => { + try { return JSON.parse(io.stdin).workspaceId ?? ""; } catch { return ""; } + })(), failureCode); + io.writeStdout(boundedJson(result)); + io.writeStderr(sanitizeStderr(error)); + const message = String((error as Error).message ?? ""); + const requestError = error instanceof SyntaxError + || message === "invalid request" + || message === "unknown command" + || message === "unexpected request field" + || message === "invalid workspace id"; + return command in { + inspect: true, "preprocess-dwh": true, "schema-suggest-fks": true, "schema-check": true, + "schema-accept": true, + "index-schema": true, "preprocess-evidence": true, "preprocess-run": true, + } ? (requestError ? 2 : 1) : 2; + } +} + +async function readSessionInventory(dataRoot: string, workspaceId: string): Promise { + const directory = join(dataRoot, "sessions", workspaceId, "sessions"); + try { + const entries = await readdir(directory, { withFileTypes: true }); + const rows: SessionInventoryRow[] = []; + for (const entry of entries) { + if (!entry.isDirectory() || entry.isSymbolicLink()) continue; + try { + const source = await readFile(join(directory, entry.name, "session_manifest.yaml"), "utf8"); + const manifest = parse(source) as Record; + rows.push({ + id: entry.name, + status: typeof manifest.status === "string" ? manifest.status : "open", + archived: manifest.archived === true, + workspaceRevision: typeof manifest.workspace_revision === "string" ? manifest.workspace_revision : null, + }); + } catch { + // fail closed at mutation time by ignoring unreadable manifests from the resumable scan + } + } + return rows; + } catch { + return []; + } +} + +function createProductionService(): WorkspacePreprocessingService { + const config = loadConfig(process.env); + const registry = new WorkspaceRegistry(config.workspaceRegistry); + const workspaceSecretStore = new WorkspaceSecretStore({ + root: config.workspaceSecretStoreRoot, + runtimeRoot: config.workspaceSecretRuntimeRoot, + installationId: config.workspaceRegistry.installationId, + }); + const runner = new ThtRunner({ + thtBin: config.thtBin, + harnessDir: config.harnessDir, + configPath: process.env.THT_CONFIG ?? "config/tht.yaml", + dataRoot: config.dataRoot, + runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"), + secretRoots: config.workspaceRegistry.secretRoots, + secretsFile: config.secretsFile, + secretFiles: config.secretFiles, + workspaceSecretStore, + semanticRuntime: { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }, + }); + return new WorkspacePreprocessingService({ + dataRoot: config.dataRoot ?? "/data", + httpPrivateHostAllowlist: (process.env.THT_EVIDENCE_PRIVATE_HOST_ALLOWLIST ?? "") + .split(",").map((value) => value.trim()).filter((value) => value.length > 0), + acquireActiveRuntime: async (workspaceId) => { + const active = await renderActiveWorkspaceRuntime({ + workspaceId, + registry, + registryConfig: config.workspaceRegistry, + harnessDir: config.harnessDir, + configPath: process.env.THT_CONFIG ?? "config/tht.yaml", + dataRoot: config.dataRoot ?? "/data", + secretRoots: config.workspaceRegistry.secretRoots, + semanticRuntime: { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }, + }); + const configLease = await publishDeterministicRuntimeConfigLease({ + workspaceId, + registry, + registryConfig: config.workspaceRegistry, + harnessDir: config.harnessDir, + configPath: process.env.THT_CONFIG ?? "config/tht.yaml", + dataRoot: config.dataRoot ?? "/data", + secretRoots: config.workspaceRegistry.secretRoots, + semanticRuntime: { + internalQdrantUrl: config.internalQdrantUrl, + internalEmbeddingUrl: config.internalEmbeddingUrl, + internalEmbeddingModel: config.internalEmbeddingModel, + internalEmbeddingDimensions: config.internalEmbeddingDimensions, + }, + workspaceSecretStore, + }); + return { + workspace: active.workspace, + workspaceId: active.workspaceId, + workspaceRevision: active.workspaceRevision, + descriptorBlob: active.descriptorBlob, + catalogBlob: active.catalogBlob, + configLease, + }; + }, + runChild: async ({ argv, configPath }) => { + const configFd = openSync(configPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + return await new Promise((resolve) => { + const child = spawn(config.thtBin, argv, { + cwd: config.harnessDir, + env: { ...process.env, ...(config.dataRoot ? { THT_DATA_ROOT: config.dataRoot } : {}) }, + stdio: ["ignore", "pipe", "pipe", configFd], + }); + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk: Buffer) => { stdout += chunk.toString("utf8"); }); + child.stderr?.on("data", (chunk: Buffer) => { stderr += chunk.toString("utf8"); }); + child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr: stderr.slice(0, 64 * 1024) })); + child.on("error", (error) => resolve({ exitCode: 1, stdout, stderr: String(error.message).slice(0, 4096) })); + }); + } finally { + closeSync(configFd); + } + }, + listSessions: async (workspaceId) => await readSessionInventory(config.dataRoot ?? "/data", workspaceId), + semanticPreflight: async (workspace) => { + const result = await runner.qdrantEnsure(workspace, 30); + return result.ok ? { ok: true as const } : { ok: false as const, code: result.code ?? "workspace_not_activatable" }; + }, + }); +} + +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + const stdout: string[] = []; + const stderr: string[] = []; + const io: WorkspaceMaintenanceIo = { + stdin: await new Promise((resolve) => { + let input = ""; + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (chunk) => { input += chunk; }); + process.stdin.on("end", () => resolve(input)); + }), + stdout, + stderr, + writeStdout: (value) => { stdout.push(value); }, + writeStderr: (value) => { stderr.push(value); }, + }; + const exitCode = await runWorkspaceMaintenanceCli(process.argv, createProductionService(), io); + process.stdout.write(stdout.join("")); + if (stderr.length > 0) process.stderr.write(stderr.join("").slice(0, 64 * 1024)); + process.exit(exitCode); +} diff --git a/backend/src/workspaces/annotations-sync.ts b/backend/src/workspaces/annotations-sync.ts new file mode 100644 index 00000000..9eb8e136 --- /dev/null +++ b/backend/src/workspaces/annotations-sync.ts @@ -0,0 +1,209 @@ +import { createHash, randomBytes } from "node:crypto"; +import { + closeSync, + constants as fsConstants, + fchmodSync, + fstatSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; +import { parseAnnotationsYaml } from "./annotations.js"; + +export interface AnnotationsSyncInput { + dataRoot: string; + workspaceId: string; + commit: string; + blobId: string; + contents: Buffer; +} + +export interface AnnotationsSyncResult { + path: string; + manifestPath: string; + contentDigest: string; +} + +interface AnnotationsOwnershipManifest { + workspace: string; + commit: string; + blobId: string; + contentDigest: string; + destination: string; +} + +export function annotationsSyncRoot(dataRoot: string, workspaceId: string, commit: string): string { + return join(dataRoot, "sessions", workspaceId, "revisions", commit, "artifacts"); +} + +function sha256(value: Buffer | string): string { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +function ensureDirectory(path: string): void { + mkdirSync(path, { recursive: true, mode: 0o700 }); + const entry = lstatSync(path); + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error("annotations sync directory is unavailable"); + } +} + +function syncDirectory(directory: string): void { + if (process.platform === "win32") return; + const fd = openSync(directory, "r"); + try { fsyncSync(fd); } finally { closeSync(fd); } +} + +function writeAtomicFile(path: string, contents: string | Buffer, mode: number): void { + ensureDirectory(dirname(path)); + const staging = `${path}.tmp-${process.pid}-${Date.now()}-${randomBytes(6).toString("hex")}`; + const fd = openSync( + staging, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, + 0o600, + ); + let closed = false; + try { + writeFileSync(fd, contents); + fsyncSync(fd); + fchmodSync(fd, mode); + closeSync(fd); + closed = true; + renameSync(staging, path); + syncDirectory(dirname(path)); + } catch (error) { + if (!closed) try { closeSync(fd); } catch { /* preserve original failure */ } + try { unlinkSync(staging); } catch { /* best effort */ } + throw error; + } +} + +function readTrustedFile(path: string): Buffer { + const entry = lstatSync(path); + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("annotations sync file is invalid"); + const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1) throw new Error("annotations sync file is invalid"); + const contents = readFileSync(fd); + const after = fstatSync(fd); + if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.nlink !== after.nlink) { + throw new Error("annotations sync file changed while reading"); + } + return contents; + } finally { + closeSync(fd); + } +} + +function parseManifest(source: string): AnnotationsOwnershipManifest { + const parsed = JSON.parse(source) as Record; + if ( + typeof parsed.workspace !== "string" + || typeof parsed.commit !== "string" + || typeof parsed.blobId !== "string" + || typeof parsed.contentDigest !== "string" + || typeof parsed.destination !== "string" + ) { + throw new Error("annotations ownership manifest is invalid"); + } + return parsed as unknown as AnnotationsOwnershipManifest; +} + +/** + * Atomically synchronize a curated annotation blob to its immutable revision-qualified runtime + * root and write an adjacent ownership manifest. Idempotent: an existing destination is re-verified + * against the exact blob/digest and fails closed on any mismatch. Never follows symlinks. + */ +export function syncAnnotations(input: AnnotationsSyncInput): AnnotationsSyncResult { + if (!isAbsolute(input.dataRoot)) throw new Error("annotations sync data root must be absolute"); + if (!/^[a-z][a-z0-9-]{2,62}$/.test(input.workspaceId) || input.workspaceId === "workspace-docs") { + throw new Error("annotations sync workspace id is invalid"); + } + if (!/^[0-9a-f]{40}$/.test(input.commit)) throw new Error("annotations sync commit is invalid"); + if (!/^[0-9a-f]{40}$/.test(input.blobId)) throw new Error("annotations sync blob id is invalid"); + parseAnnotationsYaml(input.contents.toString("utf8")); + + const root = annotationsSyncRoot(input.dataRoot, input.workspaceId, input.commit); + const directory = join(root, "mschema"); + const path = join(directory, "annotations.yaml"); + const manifestPath = join(directory, "annotations.ownership.json"); + const contentDigest = sha256(input.contents); + const manifest: AnnotationsOwnershipManifest = { + workspace: input.workspaceId, + commit: input.commit, + blobId: input.blobId, + contentDigest, + destination: path, + }; + + let existingContents: Buffer | undefined; + try { + existingContents = readTrustedFile(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + existingContents = undefined; + } + + if (existingContents !== undefined) { + if (sha256(existingContents) !== contentDigest) { + throw new Error("annotations sync destination does not match the pinned revision"); + } + const existingManifest = parseManifest(readTrustedFile(manifestPath).toString("utf8")); + if ( + existingManifest.workspace !== manifest.workspace + || existingManifest.commit !== manifest.commit + || existingManifest.blobId !== manifest.blobId + || existingManifest.contentDigest !== manifest.contentDigest + || existingManifest.destination !== manifest.destination + ) { + throw new Error("annotations ownership manifest does not match the pinned revision"); + } + return { path, manifestPath, contentDigest }; + } + + writeAtomicFile(path, input.contents, 0o400); + writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}\n`, 0o600); + return { path, manifestPath, contentDigest }; +} + +export interface SyncedAnnotations { + blobId: string; + contents: Buffer; + contentDigest: string; + manifestPath: string; +} + +/** Read the synced annotations and verify their ownership manifest; undefined when not yet synced. */ +export function readAnnotationsSync( + dataRoot: string, + workspaceId: string, + commit: string, +): SyncedAnnotations | undefined { + const directory = join(annotationsSyncRoot(dataRoot, workspaceId, commit), "mschema"); + const path = join(directory, "annotations.yaml"); + const manifestPath = join(directory, "annotations.ownership.json"); + let contents: Buffer; + try { + contents = readTrustedFile(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + const manifest = parseManifest(readTrustedFile(manifestPath).toString("utf8")); + const contentDigest = sha256(contents); + if ( + manifest.workspace !== workspaceId + || manifest.commit !== commit + || manifest.contentDigest !== contentDigest + ) { + throw new Error("annotations ownership manifest does not match the pinned revision"); + } + return { blobId: manifest.blobId, contents, contentDigest, manifestPath }; +} diff --git a/backend/src/workspaces/annotations.ts b/backend/src/workspaces/annotations.ts new file mode 100644 index 00000000..60d2376d --- /dev/null +++ b/backend/src/workspaces/annotations.ts @@ -0,0 +1,28 @@ +import { parseAllDocuments } from "yaml"; +import { WorkspaceRegistryError } from "./git-repository.js"; + +/** + * Coarse structural validation for a curated annotation blob at activation time. The harness + * Pydantic parser remains the authority for per-table shapes; this check rejects only YAML that + * cannot possibly be a canonical `Annotations` document (single mapping, optional `tables` mapping). + */ +export function parseAnnotationsYaml(source: string): void { + try { + const documents = parseAllDocuments(source, { uniqueKeys: true }); + if (documents.length !== 1) throw new Error("malformed annotations"); + const document = documents[0]; + if (document.errors.length > 0 || document.warnings.length > 0) throw new Error("malformed annotations"); + const parsed = document.toJSON(); + if (parsed === null || parsed === undefined) return; // empty canonical set + if (typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("malformed annotations"); + const record = parsed as Record; + if ( + record.tables !== undefined + && (typeof record.tables !== "object" || record.tables === null || Array.isArray(record.tables)) + ) { + throw new Error("malformed annotations"); + } + } catch { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations are malformed"); + } +} diff --git a/backend/src/workspaces/bindings.ts b/backend/src/workspaces/bindings.ts new file mode 100644 index 00000000..9d74f7e0 --- /dev/null +++ b/backend/src/workspaces/bindings.ts @@ -0,0 +1,176 @@ +import { constants, realpathSync, statSync, accessSync } from "node:fs"; +import { isAbsolute, relative } from "node:path"; +import { buildInstallationContract, type InstallationSuffix } from "./contracts.js"; +import { + DWH_TRANSPORTS, + validateWorkspaceDescriptor, + type DwhTransport, + type WorkspaceDescriptor, +} from "./schema.js"; + +export interface ResolvedEvidenceBinding { + values: Record; + missing: string[]; +} + +export interface RuntimeBindings { + dwh: ResolvedBinding; + evidence: ResolvedEvidenceBinding; +} + +export interface ResolvedBinding { + transport: DwhTransport; + values: Record; + missing: string[]; +} + +const REQUIRED_SUFFIXES: Record = { + postgres_direct: ["HOST", "PORT", "USER", "PASSWORD_FILE"], + rest_api: ["BASE_URL", "API_KEY_FILE"], + ssh_tunnel: [ + "USER", "PASSWORD_FILE", "SSH_HOST", "SSH_PORT", "SSH_USER", + "SSH_PRIVATE_KEY_FILE", "SSH_KNOWN_HOSTS_FILE", "SSH_TARGET_HOST", "SSH_TARGET_PORT", + ], +}; + +function isTransport(value: string | undefined): value is DwhTransport { + return value !== undefined && (DWH_TRANSPORTS as readonly string[]).includes(value); +} + +function isInside(path: string, root: string): boolean { + const pathRelative = relative(root, path); + return pathRelative !== "" && !pathRelative.startsWith("..") && !isAbsolute(pathRelative); +} + +function safeSecretFilePath(path: string, secretRoots: readonly string[]): string | undefined { + if (!isAbsolute(path)) return undefined; + + try { + const resolvedPath = realpathSync(path); + const resolvedRoots = secretRoots.map((root) => realpathSync(root)); + if (!resolvedRoots.some((root) => isInside(resolvedPath, root))) return undefined; + if (!statSync(resolvedPath).isFile()) return undefined; + accessSync(resolvedPath, constants.R_OK); + return resolvedPath; + } catch { + return undefined; + } +} + +function requireSupportedDescriptor(workspace: unknown): void { + if (typeof workspace !== "object" || workspace === null) { + throw new Error("Workspace bindings support only workspace schema version 3"); + } + const metadata = Reflect.get(workspace, "workspace"); + if (typeof metadata !== "object" || metadata === null + || Reflect.get(metadata, "schema_version") !== 3) { + throw new Error("Workspace bindings support only workspace schema version 3"); + } +} + +function requiredSuffixes( + workspace: WorkspaceDescriptor, + transport: DwhTransport, +): readonly InstallationSuffix[] { + const required = REQUIRED_SUFFIXES[transport]; + return transport === "rest_api" && workspace.diagnostics?.dwh_rest?.auth === "none" + ? required.filter((suffix) => suffix !== "API_KEY_FILE") + : required; +} + +/** + * Resolve only installation-local values. Secret files remain file paths: their contents are + * deliberately left for the harness secret-file loader, so bindings cannot leak credentials. + */ +export function resolveBinding( + workspace: WorkspaceDescriptor, + role: "DWH", + env: NodeJS.ProcessEnv, + secretRoots: readonly string[], +): ResolvedBinding { + requireSupportedDescriptor(workspace); + const descriptor = validateWorkspaceDescriptor(workspace); + const contract = buildInstallationContract(descriptor); + const variables = contract.variables.filter((variable) => variable.role === role); + const transportVariable = variables.find((variable) => variable.suffix === "TRANSPORT"); + const supported = descriptor.dwh.supported_transports; + const selectedValue = transportVariable ? env[transportVariable.name] : undefined; + const selectedTransport = isTransport(selectedValue) ? selectedValue : supported[0]; + const missing: string[] = []; + + if (transportVariable && (!isTransport(selectedValue) || !supported.includes(selectedTransport))) { + missing.push(transportVariable.name); + } + + const required = new Set(requiredSuffixes(descriptor, selectedTransport)); + const values: Record = {}; + for (const variable of variables) { + if (variable.suffix === "TRANSPORT") continue; + if (variable.transports && !variable.transports.includes(selectedTransport)) continue; + + const value = env[variable.name]; + const present = value !== undefined && value.trim() !== ""; + const safePath = variable.secret && present ? safeSecretFilePath(value, secretRoots) : undefined; + const safe = !variable.secret || safePath !== undefined; + if ((required.has(variable.suffix) && !present) || (present && !safe)) { + missing.push(variable.name); + } + if (present && safe) values[variable.name] = variable.secret ? safePath! : value; + } + + return { transport: selectedTransport, values, missing }; +} + +/** Resolve descriptor-selected Evidence credentials without reading any secret file contents. */ +export function resolveEvidenceBinding( + workspace: WorkspaceDescriptor, + env: NodeJS.ProcessEnv, + secretRoots: readonly string[], +): ResolvedEvidenceBinding { + requireSupportedDescriptor(workspace); + const descriptor = validateWorkspaceDescriptor(workspace); + const variables = buildInstallationContract(descriptor).variables + .filter((variable) => variable.role === "EVIDENCE"); + if (variables.length === 0) return { values: {}, missing: [] }; + + const source = descriptor.evidence?.source; + const required = new Set( + source?.type === "http" + ? ["SIGNED_URLS_FILE"] + : source?.type === "s3" + ? ["ACCESS_KEY_FILE", "SECRET_KEY_FILE"] + : [], + ); + const values: Record = {}; + const missing: string[] = []; + for (const variable of variables) { + const value = env[variable.name]; + const present = value !== undefined && value.trim() !== ""; + const safePath = present ? safeSecretFilePath(value, secretRoots) : undefined; + if ((required.has(variable.suffix) && !present) || (present && safePath === undefined)) { + missing.push(variable.name); + } + if (safePath !== undefined) values[variable.name] = safePath; + } + return { values, missing }; +} + +/** Resolve the complete schema-v3 runtime binding set. */ +export function resolveRuntimeBindings( + workspace: WorkspaceDescriptor, + env: NodeJS.ProcessEnv, + secretRoots: readonly string[], +): RuntimeBindings { + requireSupportedDescriptor(workspace); + const descriptor = validateWorkspaceDescriptor(workspace); + + return { + dwh: resolveBinding(descriptor, "DWH", env, secretRoots), + evidence: resolveEvidenceBinding(descriptor, env, secretRoots), + }; +} + +/** SSH bindings remain diagnostic-only until the session runtime owns a long-lived tunnel. */ +export function supportsSessionRuntime(bindings: RuntimeBindings): boolean { + return bindings.dwh.transport !== "ssh_tunnel" && bindings.evidence.missing.length === 0; +} diff --git a/backend/src/workspaces/catalog.ts b/backend/src/workspaces/catalog.ts new file mode 100644 index 00000000..95b09cac --- /dev/null +++ b/backend/src/workspaces/catalog.ts @@ -0,0 +1,75 @@ +import { parseAllDocuments } from "yaml"; +import { z } from "zod"; +import type { WorkspaceDescriptor } from "./schema.js"; + +export const CATALOG_PATH = "thoth-workspaces.yaml"; + +export interface WorkspaceCatalogEntry { + id: string; + name: string; + description?: string; +} + +export interface WorkspaceCatalog { + schema_version: 1; + workspaces: WorkspaceCatalogEntry[]; +} + +const workspaceId = z.string().trim().regex(/^[a-z][a-z0-9-]{2,62}$/, { + message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$", +}).refine((value) => value !== "workspace-docs", { + message: "workspace id is reserved", +}); + +const catalogEntry = z.object({ + id: workspaceId, + name: z.string().trim().min(1), + description: z.string().trim().min(1).optional(), +}).strict(); + +const catalogSchema = z.object({ + schema_version: z.literal(1), + workspaces: z.array(catalogEntry), +}).strict().superRefine((catalog, context) => { + const seen = new Set(); + catalog.workspaces.forEach((entry, index) => { + if (seen.has(entry.id)) { + context.addIssue({ + code: "custom", + path: ["workspaces", index, "id"], + message: "workspace id is duplicated in the catalog", + }); + } + seen.add(entry.id); + }); +}); + +function safeCatalogError(): Error { + return new Error("Workspace catalog is invalid"); +} + +export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog { + try { + const documents = parseAllDocuments(source, { uniqueKeys: true }); + if (documents.length !== 1) throw safeCatalogError(); + const document = documents[0]; + if (document.errors.length > 0 || document.warnings.length > 0) throw safeCatalogError(); + return catalogSchema.parse(document.toJSON()) as WorkspaceCatalog; + } catch (error) { + if (error instanceof Error && error.message === "Workspace catalog is invalid") throw error; + throw safeCatalogError(); + } +} + +export function assertCatalogMatchesDescriptor( + entry: WorkspaceCatalogEntry, + workspace: WorkspaceDescriptor, +): void { + if ( + entry.id !== workspace.workspace.id + || entry.name !== workspace.workspace.name + || entry.description !== workspace.workspace.description + ) { + throw new Error("Workspace catalog metadata does not match descriptor"); + } +} diff --git a/backend/src/workspaces/contracts.ts b/backend/src/workspaces/contracts.ts new file mode 100644 index 00000000..e13baf17 --- /dev/null +++ b/backend/src/workspaces/contracts.ts @@ -0,0 +1,282 @@ +import { validateWorkspaceDescriptor } from "./schema.js"; +import type { DwhTransport, WorkspaceDescriptor } from "./schema.js"; + +export type InstallationRole = "DWH" | "EVIDENCE"; +export type InstallationSuffix = + | "TRANSPORT" + | "HOST" + | "PORT" + | "BASE_URL" + | "USER" + | "PASSWORD_FILE" + | "API_KEY_FILE" + | "TLS_CA_FILE" + | "SSH_HOST" + | "SSH_PORT" + | "SSH_USER" + | "SSH_PRIVATE_KEY_FILE" + | "SSH_KNOWN_HOSTS_FILE" + | "SSH_TARGET_HOST" + | "SSH_TARGET_PORT" + | "SIGNED_URLS_FILE" + | "ACCESS_KEY_FILE" + | "SECRET_KEY_FILE" + | "SESSION_TOKEN_FILE"; + +type ConnectorTransport = DwhTransport; + +export interface InstallationVariable { + name: string; + role: InstallationRole; + suffix: InstallationSuffix; + secret: boolean; + transports?: readonly ConnectorTransport[]; +} + +export interface InstallationContract { + workspaceId: string; + namespace: string; + variables: InstallationVariable[]; +} + +const DIRECT_SUFFIXES: readonly InstallationSuffix[] = [ + "HOST", + "PORT", + "USER", + "PASSWORD_FILE", + "TLS_CA_FILE", +]; + +const REST_SUFFIXES: readonly InstallationSuffix[] = [ + "BASE_URL", + "API_KEY_FILE", + "TLS_CA_FILE", +]; + +const SSH_SUFFIXES: readonly InstallationSuffix[] = [ + "USER", + "PASSWORD_FILE", + "TLS_CA_FILE", + "SSH_HOST", + "SSH_PORT", + "SSH_USER", + "SSH_PRIVATE_KEY_FILE", + "SSH_KNOWN_HOSTS_FILE", + "SSH_TARGET_HOST", + "SSH_TARGET_PORT", +]; + +function namespaceFor(workspace: WorkspaceDescriptor): string { + return workspace.workspace.id.replaceAll("-", "_").toUpperCase(); +} + +function createVariable( + namespace: string, + role: InstallationRole, + suffix: InstallationSuffix, + transports?: readonly ConnectorTransport[], +): InstallationVariable { + return { + name: `THT_WS_${namespace}_${role}_${suffix}`, + role, + suffix, + secret: suffix.endsWith("_FILE"), + ...(transports ? { transports } : {}), + }; +} + +function connectorVariables( + namespace: string, + transports: readonly DwhTransport[], +): InstallationVariable[] { + const suffixTransports = new Map(); + const add = (suffixes: readonly InstallationSuffix[], transport: DwhTransport) => { + for (const suffix of suffixes) { + const applicable = suffixTransports.get(suffix) ?? []; + applicable.push(transport); + suffixTransports.set(suffix, applicable); + } + }; + + for (const transport of transports) { + if (transport === "postgres_direct") { + add(DIRECT_SUFFIXES, transport); + } else if (transport === "rest_api") { + add(REST_SUFFIXES, transport); + } else { + add(SSH_SUFFIXES, transport); + } + } + + return [ + createVariable(namespace, "DWH", "TRANSPORT", transports), + ...[...suffixTransports.entries()].map(([suffix, applicable]) => ( + createVariable(namespace, "DWH", suffix, applicable) + )), + ]; +} + +function evidenceVariables( + namespace: string, + workspace: WorkspaceDescriptor, +): InstallationVariable[] { + if (!("evidence" in workspace) || workspace.evidence === undefined) return []; + const source = workspace.evidence.source; + if (source.type === "http" && source.authentication === "signed_urls_file") { + return [createVariable(namespace, "EVIDENCE", "SIGNED_URLS_FILE")]; + } + if (source.type === "s3" && source.credentials === "static_files") { + return [ + createVariable(namespace, "EVIDENCE", "ACCESS_KEY_FILE"), + createVariable(namespace, "EVIDENCE", "SECRET_KEY_FILE"), + createVariable(namespace, "EVIDENCE", "SESSION_TOKEN_FILE"), + ]; + } + return []; +} + +function requireSupportedDescriptor(workspace: unknown): void { + if (typeof workspace !== "object" || workspace === null) { + throw new Error("Installation contract supports only workspace schema version 3"); + } + const metadata = Reflect.get(workspace, "workspace"); + if (typeof metadata !== "object" || metadata === null + || Reflect.get(metadata, "schema_version") !== 3) { + throw new Error("Installation contract supports only workspace schema version 3"); + } +} + +export function buildInstallationContract(workspace: WorkspaceDescriptor): InstallationContract { + requireSupportedDescriptor(workspace); + const descriptor = validateWorkspaceDescriptor(workspace); + const namespace = namespaceFor(descriptor); + + return { + workspaceId: descriptor.workspace.id, + namespace, + variables: [ + ...connectorVariables(namespace, descriptor.dwh.supported_transports), + ...evidenceVariables(namespace, descriptor), + ], + }; +} + +function localizedIntroduction(workspace: WorkspaceDescriptor): string { + return workspace.workspace.language === "it" + ? `Configurazione dell'installazione per ${workspace.workspace.name}. Imposta solo i binding supportati da questa installazione.` + : `Installation setup for ${workspace.workspace.name}. Configure only the bindings supported by this installation.`; +} + +function evidenceDocumentation( + workspace: WorkspaceDescriptor, + variables: readonly InstallationVariable[], +): string[] { + if (!("evidence" in workspace) || workspace.evidence === undefined) return []; + const { source, policy } = workspace.evidence; + const common = [ + "## Evidence source", + "", + `- Type: \`${source.type}\``, + ]; + let details: string[]; + if (source.type === "filesystem") { + details = [ + `- URI: \`${source.uri}\``, + `- Patterns: ${source.patterns.map((pattern) => `\`${pattern}\``).join(", ")}`, + `- Maximum source bytes: \`${source.max_bytes}\``, + "- Ownership: the descriptor and its Evidence tree are owned by the same Git revision.", + "- Materialization: P6 materializes that revision-pinned tree and verifies real containment, including symlink safety.", + "- Export boundary: the browser/API ZIP does not include Evidence file bytes.", + ]; + } else if (source.type === "http") { + details = [ + "- URIs:", + ...source.uris.map((uri) => ` - \`${uri}\``), + `- Authentication: \`${source.authentication}\`. ${source.authentication === "none" + ? "No credential file is required." + : "Provide the signed URL file through the installation file variable listed below."}`, + `- Connect timeout (ms): \`${source.connect_timeout_ms}\``, + `- Read timeout (ms): \`${source.read_timeout_ms}\``, + `- Maximum source bytes: \`${source.max_bytes}\``, + `- Maximum redirects: \`${source.max_redirects}\``, + `- Private hosts allowed: \`${source.allow_private_hosts}\``, + `- Maximum cache bytes: \`${source.max_cache_bytes}\``, + ]; + } else { + details = [ + `- URI: \`${source.uri}\``, + ...(source.endpoint_url === undefined ? [] : [`- Endpoint URL: \`${source.endpoint_url}\``]), + ...(source.region === undefined ? [] : [`- Region: \`${source.region}\``]), + `- Credentials: \`${source.credentials}\`. ${source.credentials === "ambient" + ? "Use ambient credentials; no Evidence credential file is required." + : "Provide credentials through the installation file variables listed below."}`, + `- Trusted endpoint: \`${source.trusted_endpoint}\``, + `- Private endpoint allowed: \`${source.allow_private_endpoint}\``, + `- Insecure endpoint allowed: \`${source.allow_insecure_endpoint}\``, + `- Maximum source bytes: \`${source.max_bytes}\``, + `- Maximum objects: \`${source.max_objects}\``, + `- Maximum pages: \`${source.max_pages}\``, + `- Page size: \`${source.page_size}\``, + ]; + } + const evidenceVariables = variables.filter(({ role }) => role === "EVIDENCE"); + const requiredVariables = evidenceVariables.filter(({ suffix }) => suffix !== "SESSION_TOKEN_FILE"); + const optionalVariables = evidenceVariables.filter(({ suffix }) => suffix === "SESSION_TOKEN_FILE"); + return [ + ...common, + ...details, + `- Maximum chunk characters: \`${policy.max_chunk_chars}\``, + `- Retained published generations: \`${policy.retain_published_generations}\``, + ...(requiredVariables.length === 0 ? [] : [ + "- Required installation file variables:", + ...requiredVariables.map(({ name }) => ` - \`${name}\``), + ]), + ...(optionalVariables.length === 0 ? [] : [ + "- Optional installation file variables:", + ...optionalVariables.map(({ name }) => ` - \`${name}\``), + ]), + "", + ]; +} + +export function renderWorkspaceDocs(workspace: WorkspaceDescriptor): { envExample: string; markdown: string } { + const descriptor = validateWorkspaceDescriptor(workspace); + const contract = buildInstallationContract(descriptor); + const variablesByRole = new Map(); + for (const variable of contract.variables) { + const variables = variablesByRole.get(variable.role) ?? []; + variables.push(variable); + variablesByRole.set(variable.role, variables); + } + + const envExample = [ + `# Generated installation bindings for ${descriptor.workspace.id}`, + "# Provide secret file paths only; never paste secret values here.", + ...contract.variables.map((variable) => `${variable.name}=`), + "", + ].join("\n"); + + const markdown = [ + "# Installation requirements", + "", + `**Workspace:** ${descriptor.workspace.name}`, + "", + localizedIntroduction(descriptor), + "", + "Use the following UI fields as installation bindings. Secret fields always contain file paths, never secret values.", + "", + ...(["DWH", "EVIDENCE"] as const) + .filter((role) => variablesByRole.has(role)) + .flatMap((role) => [ + `## ${role === "DWH" ? "Data warehouse" : "Evidence"}`, + "", + ...(variablesByRole.get(role) ?? []).map((variable) => ( + `- \`${variable.name}\`${variable.transports ? ` (for: ${variable.transports.join(", ")})` : ""}` + )), + "", + ]), + ...evidenceDocumentation(descriptor, contract.variables), + ].join("\n"); + + return { envExample, markdown }; +} diff --git a/backend/src/workspaces/diagnostics.ts b/backend/src/workspaces/diagnostics.ts new file mode 100644 index 00000000..48e9b253 --- /dev/null +++ b/backend/src/workspaces/diagnostics.ts @@ -0,0 +1,560 @@ +import { readFile } from "node:fs/promises"; +import { Client } from "pg"; +import { MAX_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS } from "../config.js"; +import { buildInstallationContract } from "./contracts.js"; +import type { RuntimeBindings } from "./runtime-renderer.js"; +import { + resolveDiagnosticUrl, + validateWorkspaceDescriptor, + type RestDiagnosticRequest, + type WorkspaceDescriptor, +} from "./schema.js"; +import type { WorkspaceErrorCode } from "./types.js"; +import type { SemanticRuntimeConfig } from "./runtime-renderer.js"; +import type { AuthDiagnostics } from "../auth/diagnostics.js"; + +export interface Diagnostic { + level: "error" | "warning" | "info"; + code: WorkspaceErrorCode | "binding_ok"; + field?: string; + variable?: string; + message: string; +} + +/** Connector-only result produced before the route aggregates authentication. */ +export interface ConnectorDiagnostics { + activatable: boolean; + diagnostics: Diagnostic[]; +} + +/** The HTTP workspace diagnostic contract always includes the shared authentication report. */ +export interface WorkspaceDiagnostics extends ConnectorDiagnostics { + authentication: AuthDiagnostics; +} + +interface DiagnosticResource { + database?: string; + schema?: string; +} + +type RestConnectorDiagnostic = RestDiagnosticRequest & { + response?: Record; +}; + +export interface ConnectorDiagnosticRequest { + role: "dwh"; + transport: "postgres_direct" | "rest_api"; + host?: string; + port?: number; + baseUrl?: string; + user?: string; + credentialFile?: string; + tlsCaFile?: string; + tlsServername?: string; + resource: DiagnosticResource; + timeoutMs: number; + signal: AbortSignal; + diagnostic?: RestConnectorDiagnostic; +} + +export interface ConnectorDiagnosticResult { + resolved: boolean; + tlsVerified: boolean; + authenticated: boolean; + resource: DiagnosticResource; +} + +export interface QdrantDiagnosticRequest { + baseUrl: string; + collection: string; + timeoutMs: number; + signal: AbortSignal; +} + +export interface QdrantDiagnosticResult { + collection: string; + dimensions?: number; + distance?: string; +} + +export interface EmbeddingDiagnosticRequest { + baseUrl: string; + model: string; + timeoutMs: number; + signal: AbortSignal; +} + +export interface EmbeddingDiagnosticResult { + available: boolean; + dimensions?: number; +} + +export interface DirectProtocolFactory { + probe(request: ConnectorDiagnosticRequest): Promise; +} + +export interface DatabaseDiagnosticClient { + query(sql: string, values: readonly unknown[]): Promise<{ rows: Array> }>; + end(): Promise; +} + +export interface DatabaseDiagnosticClientFactory { + connect(request: { + host: string; + port: number; + database: string; + user: string; + credentialFile: string; + tlsCaFile?: string; + tlsServername?: string; + signal: AbortSignal; + }): Promise; +} + +export interface ConcreteDiagnosticAdapterDependencies { + directProtocol?: DirectProtocolFactory; + databaseClient?: DatabaseDiagnosticClientFactory; +} + +/** Adapters retain only diagnostic metadata and never return credential contents or bodies. */ +export interface DiagnosticAdapters { + probeConnector(request: ConnectorDiagnosticRequest): Promise; + inspectQdrant(request: QdrantDiagnosticRequest): Promise; + probeEmbedding(request: EmbeddingDiagnosticRequest): Promise; +} + +export const DEFAULT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS = 5_000; + +async function secretPresent(file: string): Promise { + return (await readFile(file, "utf8")).trim().length > 0; +} + +async function restHeaders( + diagnostic: RestDiagnosticRequest, + credentialFile: string | undefined, +): Promise> { + if (diagnostic.auth === "none") return {}; + if (!credentialFile || !(await secretPresent(credentialFile))) throw new Error("REST probe failed"); + const secret = (await readFile(credentialFile, "utf8")).trim(); + return diagnostic.auth === "bearer" ? { authorization: `Bearer ${secret}` } : { "x-api-key": secret }; +} + +/** + * Concrete production adapters deliberately retain only probe metadata. Protocol failures and + * response bodies are discarded at this boundary; callers receive fixed diagnostics instead. + */ +export function createConcreteDiagnosticAdapters( + dependencies: ConcreteDiagnosticAdapterDependencies = {}, +): DiagnosticAdapters { + const databaseClient = dependencies.databaseClient ?? { + async connect(request: { + host: string; + port: number; + database: string; + user: string; + credentialFile: string; + tlsCaFile?: string; + tlsServername?: string; + signal: AbortSignal; + }) { + const client = new Client({ + host: request.host, + port: request.port, + database: request.database, + user: request.user, + password: (await readFile(request.credentialFile, "utf8")).trim(), + ssl: { + ...(request.tlsCaFile ? { ca: await readFile(request.tlsCaFile, "utf8") } : {}), + ...(request.tlsServername ? { servername: request.tlsServername } : {}), + rejectUnauthorized: true, + }, + connectionTimeoutMillis: 5_000, + }); + const abort = () => { void client.end(); }; + request.signal.addEventListener("abort", abort, { once: true }); + try { + await client.connect(); + return { + query: async (sql: string, values: readonly unknown[]) => await client.query(sql, [...values]), + end: async () => { + request.signal.removeEventListener("abort", abort); + await client.end(); + }, + }; + } catch (error) { + request.signal.removeEventListener("abort", abort); + await client.end().catch(() => undefined); + throw error; + } + }, + }; + const directProtocol = dependencies.directProtocol ?? { + async probe(request: ConnectorDiagnosticRequest): Promise { + if (!request.host || !request.port || !request.user || !request.credentialFile + || !(await secretPresent(request.credentialFile))) { + throw new Error("direct probe failed"); + } + const database = request.resource.database; + const schema = request.resource.schema; + if (!database || !schema) throw new Error("direct probe failed"); + const client = await databaseClient.connect({ + host: request.host, + port: request.port, + database, + user: request.user, + credentialFile: request.credentialFile, + tlsCaFile: request.tlsCaFile, + tlsServername: request.tlsServername, + signal: request.signal, + }); + try { + const result = await client.query( + "SELECT current_database() AS database, current_schema() AS schema", + [], + ); + const row = result.rows[0]; + if (row?.database !== database || row.schema !== schema) throw new Error("direct probe failed"); + return { + resolved: true, + tlsVerified: true, + authenticated: true, + resource: request.resource, + }; + } finally { + await client.end().catch(() => undefined); + } + }, + }; + + return { + async probeConnector(request) { + if (request.transport === "rest_api") { + if (!request.baseUrl || !request.diagnostic || request.tlsCaFile) { + throw new Error("REST probe failed"); + } + const endpoint = resolveDiagnosticUrl(request.baseUrl, request.diagnostic.path); + const response = await fetch(endpoint.toString(), { + method: request.diagnostic.method, + headers: await restHeaders(request.diagnostic, request.credentialFile), + signal: request.signal, + redirect: "error", + }); + if (!response.ok) throw new Error("REST probe failed"); + if ("response" in request.diagnostic) { + const payload = await response.json().catch(() => undefined) as Record | undefined; + const declared = request.diagnostic.response; + // Validate a declared field only when the probe response actually carries it, so a + // health-style ping (2xx + JSON without database/schema identity) still proves a + // reachable, authenticated connector. Declared fields that are present must match. + const databaseMatches = declared?.database === undefined + || payload?.[declared.database] === undefined + || payload[declared.database] === request.resource.database; + const schemaMatches = declared?.schema === undefined + || payload?.[declared.schema] === undefined + || payload[declared.schema] === request.resource.schema; + if (!payload || !databaseMatches || !schemaMatches) { + throw new Error("REST probe failed"); + } + } + return { + resolved: true, + tlsVerified: new URL(request.baseUrl).protocol === "https:", + authenticated: true, + resource: request.resource, + }; + } + if (request.transport !== "postgres_direct") throw new Error("direct probe failed"); + return await directProtocol.probe(request); + }, + async inspectQdrant(request) { + const response = await fetch( + new URL(`/collections/${request.collection}`, `${request.baseUrl}/`).toString(), + { method: "GET", signal: request.signal, redirect: "error" }, + ); + const payload = await response.json().catch(() => undefined) as { + result?: { config?: { params?: { vectors?: { size?: unknown; distance?: unknown } } } }; + } | undefined; + const size = payload?.result?.config?.params?.vectors?.size; + const distance = payload?.result?.config?.params?.vectors?.distance; + if (!response.ok || !Number.isInteger(size) || typeof distance !== "string" + || distance.length === 0) { + throw new Error("Qdrant metadata probe failed"); + } + return { + collection: request.collection, + dimensions: size as number, + distance: distance.toLowerCase(), + }; + }, + async probeEmbedding(request) { + const response = await fetch(new URL("/api/embed", `${request.baseUrl}/`).toString(), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: request.model, input: "diagnostic" }), + signal: request.signal, + redirect: "error", + }); + const payload = await response.json().catch(() => undefined) as { + embeddings?: unknown[]; + } | undefined; + const vector = Array.isArray(payload?.embeddings) ? payload.embeddings[0] : undefined; + if (!response.ok || !Array.isArray(vector)) throw new Error("embedding probe failed"); + return { available: true, dimensions: vector.length }; + }, + }; +} + +function configuredTimeout(value: number | undefined): number { + return Math.min( + Math.max(1, value ?? DEFAULT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS), + MAX_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS, + ); +} + +function boundedTimeout(value: number | undefined, ceiling: number): number { + return Math.min(Math.max(1, value ?? ceiling), ceiling, MAX_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS); +} + +async function withTimeout(timeoutMs: number, operation: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + let timer: NodeJS.Timeout | undefined; + try { + return await new Promise((resolve, reject) => { + timer = setTimeout(() => { + controller.abort(); + reject(new Error("diagnostic timed out")); + }, timeoutMs); + void operation(controller.signal).then(resolve, reject); + }); + } finally { + if (timer !== undefined) clearTimeout(timer); + controller.abort(); + } +} + +function sameResource(expected: DiagnosticResource, actual: DiagnosticResource): boolean { + return Object.entries(expected).every(([key, value]) => actual[key as keyof DiagnosticResource] === value); +} + +function hasRequiredConnectorChecks( + result: ConnectorDiagnosticResult, + resource: DiagnosticResource, +): boolean { + return result.resolved + && result.tlsVerified + && result.authenticated + && sameResource(resource, result.resource); +} + +function diagnosticError(code: WorkspaceErrorCode, field?: string): Diagnostic { + return { + level: "error", + code, + ...(field ? { field } : {}), + message: code === "binding_missing" + ? "Installation binding is missing or invalid." + : code === "semantic_index_incompatible" + ? "Semantic index metadata is incompatible with this workspace." + : code === "workspace_not_activatable" + ? "This transport can be tested, but it is not available to runtime sessions." + : "Connector diagnostic failed.", + }; +} + +function bindingName(workspace: WorkspaceDescriptor, suffix: string): string { + const entry = buildInstallationContract(workspace).variables.find((variable) => ( + variable.role === "DWH" && variable.suffix === suffix + )); + if (!entry) throw new Error(`workspace contract is missing DWH_${suffix}`); + return entry.name; +} + +function numericBinding(binding: Record, name: string): number | undefined { + const value = Number(binding[name]); + return Number.isInteger(value) && value > 0 && value <= 65_535 ? value : undefined; +} + +function requireSupportedDescriptor(workspace: unknown): void { + if (typeof workspace !== "object" || workspace === null) { + throw new Error("Workspace diagnoser supports only workspace schema version 3"); + } + const metadata = Reflect.get(workspace, "workspace"); + if (typeof metadata !== "object" || metadata === null + || Reflect.get(metadata, "schema_version") !== 3) { + throw new Error("Workspace diagnoser supports only workspace schema version 3"); + } +} + +async function diagnoseValidatedWorkspace( + descriptor: WorkspaceDescriptor, + bindings: RuntimeBindings, + adapters: DiagnosticAdapters, + timeoutMs: number, + semanticRuntime: SemanticRuntimeConfig, +): Promise { + const evidenceField = descriptor.evidence?.source.type === "http" + ? "evidence.source.authentication" + : descriptor.evidence?.source.type === "s3" + ? "evidence.source.credentials" + : undefined; + const evidenceDiagnostics = [...bindings.evidence.missing].sort().map((variable): Diagnostic => ({ + ...diagnosticError("binding_missing", evidenceField), + variable, + })); + const diagnostics = [ + ...[...bindings.dwh.missing].sort().map((field) => diagnosticError("binding_missing", field)), + ...evidenceDiagnostics, + ]; + if (diagnostics.length > 0) return { activatable: false, diagnostics }; + + const dwhTimeout = boundedTimeout(descriptor.dwh.timeout_ms, timeoutMs); + let activatable = true; + const dwhValues = bindings.dwh.values; + const dwhField = (suffix: string) => bindingName(descriptor, suffix); + const dwhResource = { database: descriptor.dwh.database, schema: descriptor.dwh.schema }; + let dwhRequest: ConnectorDiagnosticRequest | undefined; + if (bindings.dwh.transport === "rest_api") { + const diagnostic = descriptor.diagnostics?.dwh_rest; + const baseUrl = dwhValues[dwhField("BASE_URL")]; + if (diagnostic && baseUrl) { + const credentialFile = diagnostic.auth === "none" ? undefined : dwhValues[dwhField("API_KEY_FILE")]; + if (diagnostic.auth === "none" || credentialFile !== undefined) { + dwhRequest = { + role: "dwh", + transport: "rest_api", + baseUrl, + credentialFile, + tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")], + resource: dwhResource, + timeoutMs: dwhTimeout, + signal: new AbortController().signal, + diagnostic, + }; + } + } + } else if (bindings.dwh.transport === "postgres_direct") { + const host = dwhValues[dwhField("HOST")]; + const port = numericBinding(dwhValues, dwhField("PORT")); + const user = dwhValues[dwhField("USER")]; + const credentialFile = dwhValues[dwhField("PASSWORD_FILE")]; + if (host && port && user && credentialFile) { + dwhRequest = { + role: "dwh", + transport: "postgres_direct", + host, + port, + user, + credentialFile, + tlsCaFile: dwhValues[dwhField("TLS_CA_FILE")], + resource: dwhResource, + timeoutMs: dwhTimeout, + signal: new AbortController().signal, + }; + } + } + + if (!dwhRequest) { + diagnostics.push(diagnosticError("workspace_not_activatable")); + return { activatable: false, diagnostics }; + } + + try { + const dwhResult = await withTimeout(dwhTimeout, (signal) => adapters.probeConnector({ + ...dwhRequest, + signal, + timeoutMs: dwhTimeout, + })); + if (!hasRequiredConnectorChecks(dwhResult, dwhRequest.resource)) { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + } catch { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + + try { + const vector = await withTimeout(timeoutMs, (signal) => adapters.inspectQdrant({ + baseUrl: semanticRuntime.internalQdrantUrl, + collection: descriptor.semantic_index.vector_store.collection, + timeoutMs, + signal, + })); + const expected = descriptor.semantic_index.vector_store; + if (vector.collection !== expected.collection + || vector.dimensions !== expected.dimensions + || vector.distance !== expected.distance) { + diagnostics.push(diagnosticError("semantic_index_incompatible")); + activatable = false; + } + } catch { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + + try { + const embedding = await withTimeout(timeoutMs, (signal) => adapters.probeEmbedding({ + baseUrl: semanticRuntime.internalEmbeddingUrl, + model: semanticRuntime.internalEmbeddingModel, + timeoutMs, + signal, + })); + if (semanticRuntime.internalEmbeddingModel !== descriptor.semantic_index.embedding.model + || semanticRuntime.internalEmbeddingDimensions !== descriptor.semantic_index.embedding.dimensions + || !embedding.available + || embedding.dimensions !== descriptor.semantic_index.embedding.dimensions) { + diagnostics.push(diagnosticError("semantic_index_incompatible")); + activatable = false; + } + } catch { + diagnostics.push(diagnosticError("connector_unavailable")); + activatable = false; + } + + return { + activatable, + diagnostics: diagnostics.length > 0 + ? diagnostics + : [{ + level: "info", + code: "binding_ok", + message: "Installation bindings and diagnostics succeeded.", + }], + }; +} + +const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, +}; + +export function createWorkspaceDiagnoser( + adapters: DiagnosticAdapters, + options: { timeoutMs?: number; semanticRuntime?: SemanticRuntimeConfig } = {}, +) { + const timeoutMs = configuredTimeout(options.timeoutMs); + const semanticRuntime = options.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME; + return async function diagnose( + workspace: WorkspaceDescriptor, + bindings: RuntimeBindings, + _options: { writeProbe: boolean }, + ): Promise { + requireSupportedDescriptor(workspace); + const descriptor = validateWorkspaceDescriptor(workspace); + return await diagnoseValidatedWorkspace(descriptor, bindings, adapters, timeoutMs, semanticRuntime); + }; +} + +export function createProductionWorkspaceDiagnoser( + timeoutMs: number, + adapters: DiagnosticAdapters = createConcreteDiagnosticAdapters(), + semanticRuntime: SemanticRuntimeConfig = DEFAULT_SEMANTIC_RUNTIME, +) { + return createWorkspaceDiagnoser(adapters, { timeoutMs, semanticRuntime }); +} + +export const diagnoseWorkspace = createProductionWorkspaceDiagnoser( + DEFAULT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS, +); diff --git a/backend/src/workspaces/effective-config.ts b/backend/src/workspaces/effective-config.ts new file mode 100644 index 00000000..29e06bf1 --- /dev/null +++ b/backend/src/workspaces/effective-config.ts @@ -0,0 +1,219 @@ +import { createHash } from "node:crypto"; +import { normalize } from "node:path"; + +export interface CanonicalEffectiveConfig { + schemaVersion: 1; + dwh: CanonicalDwhConfig; + vector: CanonicalVectorConfig; + embedding: CanonicalEmbeddingConfig; + roots: CanonicalRootsConfig; +} + +export interface CanonicalDwhConfig { + engine: "postgres"; + database: string; + schema: string; + transport: "postgres_direct" | "rest_api" | "ssh_tunnel"; + host?: string; + port?: number; + baseUrl?: string; + user?: string; +} + +export interface CanonicalVectorConfig { + collection: string; + dimensions: number; + distance: string; +} + +export interface CanonicalEmbeddingConfig { + model: string; + dimensions: number; +} + +export interface CanonicalRootsConfig { + artifacts: string; + indexes: string; +} + +function asRecord(value: unknown): Record | undefined { + if (typeof value === "object" && value !== null && !Array.isArray(value)) { + return value as Record; + } + return undefined; +} + +function requireString(value: Record, key: string): string { + const candidate = value[key]; + if (typeof candidate !== "string" || candidate.length === 0) { + throw new TypeError(`effective config is missing ${key}`); + } + return candidate; +} + +function optionalString(value: Record, key: string): string | undefined { + const candidate = value[key]; + if (candidate === undefined || candidate === null) return undefined; + if (typeof candidate !== "string") return undefined; + return candidate; +} + +function optionalNumber(value: Record, key: string): number | undefined { + const candidate = value[key]; + if (candidate === undefined || candidate === null) return undefined; + if (typeof candidate !== "number" || Number.isNaN(candidate)) return undefined; + return candidate; +} + +function requireNumber(value: Record, key: string): number { + const candidate = value[key]; + if (typeof candidate !== "number" || Number.isNaN(candidate)) { + throw new TypeError(`effective config is missing numeric ${key}`); + } + return candidate; +} + +function normalizeRoot(value: string): string { + return normalize(value); +} + +function dwhTransport(rendered: Record): CanonicalDwhConfig["transport"] { + const dwh = asRecord(rendered.dwh); + const type = dwh ? requireString(dwh, "type") : undefined; + if (type === "postgres_direct") return "postgres_direct"; + if (type === "thoth_rest") return "rest_api"; + if (type === "ssh_tunnel") return "ssh_tunnel"; + throw new TypeError(`effective config has unsupported dwh transport ${type}`); +} + +function buildDwhConfig(rendered: Record): CanonicalDwhConfig { + const transport = dwhTransport(rendered); + const databaseRecord = asRecord(rendered.database) ?? asRecord(asRecord(asRecord(rendered.dwh)?.connection)?.database); + if (!databaseRecord) { + throw new TypeError("effective config is missing database identity"); + } + const engine = "postgres"; + const database = requireString(databaseRecord, "database"); + const schema = requireString(databaseRecord, "schema"); + const dwh: Record = { engine, database, schema, transport }; + + if (transport === "postgres_direct") { + const host = optionalString(databaseRecord, "host"); + const port = optionalNumber(databaseRecord, "port"); + const user = optionalString(databaseRecord, "user"); + if (host !== undefined) dwh.host = host; + if (port !== undefined) dwh.port = port; + if (user !== undefined) dwh.user = user; + } else if (transport === "rest_api") { + const rest = asRecord(rendered.rest) ?? asRecord(asRecord(asRecord(rendered.dwh)?.endpoint)); + const baseUrl = rest ? optionalString(rest, "base_url") : undefined; + if (baseUrl !== undefined) dwh.baseUrl = baseUrl; + } + + return dwh as unknown as CanonicalDwhConfig; +} + +function buildVectorConfig(rendered: Record): CanonicalVectorConfig { + const resources = asRecord(rendered.resources); + const vector = resources ? asRecord(resources.vector) : undefined; + if (!vector) { + throw new TypeError("effective config is missing vector resources"); + } + const collection = requireString(vector, "collection"); + const semanticIndex = asRecord(rendered.semantic_index); + const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined; + const dimensions = vectorStore + ? requireNumber(vectorStore, "dimensions") + : requireNumber(vector, "dimensions"); + const distance = vectorStore + ? requireString(vectorStore, "distance") + : (optionalString(vector, "distance") ?? "cosine"); + return { collection, dimensions, distance }; +} + +function buildEmbeddingConfig(rendered: Record): CanonicalEmbeddingConfig { + const resources = asRecord(rendered.resources); + const embeddings = resources ? asRecord(resources.embeddings) : undefined; + if (!embeddings) { + throw new TypeError("effective config is missing embedding resources"); + } + return { + model: requireString(embeddings, "model"), + dimensions: requireNumber(embeddings, "dimensions"), + }; +} + +function buildRootsConfig(rendered: Record): CanonicalRootsConfig { + const roots = asRecord(rendered.roots) ?? asRecord(rendered.paths); + if (!roots) { + throw new TypeError("effective config is missing roots"); + } + return { + artifacts: normalizeRoot(requireString(roots, "artifacts")), + indexes: normalizeRoot(requireString(roots, "indexes")), + }; +} + +/** + * Build the versioned, non-secret effective DWH/preprocessing configuration from a + * rendered runtime configuration object. The result contains only the fields that + * affect DWH generation identity; credentials, runtime identity, session storage, + * evidence, and service endpoints are excluded. + */ +export function buildCanonicalEffectiveConfig(renderedConfig: unknown): CanonicalEffectiveConfig { + const rendered = asRecord(renderedConfig); + if (!rendered) { + throw new TypeError("effective config requires a rendered configuration object"); + } + return { + schemaVersion: 1, + dwh: buildDwhConfig(rendered), + vector: buildVectorConfig(rendered), + embedding: buildEmbeddingConfig(rendered), + roots: buildRootsConfig(rendered), + }; +} + +function sha256(value: string | Buffer): string { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +/** + * Serialize the canonical effective config to a deterministic JSON string with the + * fixed key order defined by the shared contract. No whitespace is included. + */ +export function canonicalEffectiveConfigJson(config: CanonicalEffectiveConfig): string { + const ordered: Record = { schemaVersion: config.schemaVersion }; + ordered.dwh = { ...config.dwh }; + ordered.vector = { ...config.vector }; + ordered.embedding = { ...config.embedding }; + ordered.roots = { ...config.roots }; + return JSON.stringify(ordered); +} + +/** + * Return the stable logical config-source identity for a workspace revision. + * This is `workspace://@v1:`. + */ +export function effectiveConfigIdentity(workspaceId: string, renderedConfig: unknown): string { + const canonical = buildCanonicalEffectiveConfig(renderedConfig); + const digest = createHash("sha256").update(canonicalEffectiveConfigJson(canonical)).digest("hex"); + return `workspace://${workspaceId}@v1:${digest}`; +} + +/** + * Return the config fingerprint: `sha256:` + the SHA-256 of the canonical effective + * config JSON bytes. + */ +export function configFingerprint(renderedConfig: unknown): string { + const canonical = buildCanonicalEffectiveConfig(renderedConfig); + return sha256(canonicalEffectiveConfigJson(canonical)); +} + +/** + * Return the input fingerprint: `sha256:` + the SHA-256 of the logical config-source + * identity string. + */ +export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string { + return sha256(effectiveConfigIdentity(workspaceId, renderedConfig)); +} diff --git a/backend/src/workspaces/evidence-materialization.ts b/backend/src/workspaces/evidence-materialization.ts new file mode 100644 index 00000000..c9eea887 --- /dev/null +++ b/backend/src/workspaces/evidence-materialization.ts @@ -0,0 +1,155 @@ +import { createHash } from "node:crypto"; +import { + closeSync, + constants as fsConstants, + fchmodSync, + fsyncSync, + mkdirSync, + openSync, + writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; +import { GitWorkspaceRepository } from "./git-repository.js"; + +export interface EvidenceMaterializationLimits { + maxEntries: number; + maxTotalBytes: number; + maxFileBytes: number; + maxPathBytes: number; + maxManifestBytes: number; +} + +export const DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS: EvidenceMaterializationLimits = { + maxEntries: 4096, + maxTotalBytes: 64 * 1024 * 1024, + maxFileBytes: 8 * 1024 * 1024, + maxPathBytes: 4096, + maxManifestBytes: 1024 * 1024, +}; + +export interface EvidenceManifestFile { + mode: "100644" | "100755"; + oid: string; + digest: string; + bytes: number; +} + +export interface EvidenceManifest { + schemaVersion: 1; + workspace: string; + commit: string; + tree: string; + entryCount: number; + totalBytes: number; + files: Record; +} + +export interface MaterializedEvidence { + root: string; + manifestPath: string; + manifest: EvidenceManifest; + /** 64-hex sha256 of the manifest bytes, for the snapshot manifest integrity chain. */ + manifestDigest: string; +} + +function sha256Hex(value: Buffer | string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function sha256Prefixed(value: Buffer | string): string { + return `sha256:${sha256Hex(value)}`; +} + +function writeExclusiveNoFollow(path: string, contents: Buffer, mode: number): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const fd = openSync( + path, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, + 0o600, + ); + let closed = false; + try { + writeFileSync(fd, contents); + fsyncSync(fd); + fchmodSync(fd, mode); + closeSync(fd); + closed = true; + } catch (error) { + if (!closed) try { closeSync(fd); } catch { /* preserve original failure */ } + throw error; + } +} + +export interface MaterializeEvidenceTreeOptions { + repository: GitWorkspaceRepository; + revision: string; + id: string; + /** The workspace directory (e.g. `/`) that will receive `evidence/` and the manifest. */ + targetDirectory: string; + limits?: Partial; +} + +/** + * Materialize a canonical `/evidence` tree from an exact commit into an owned staging + * directory with a bounded manifest. Never follows symlinks; a bound violation or unsafe object + * aborts before any atomic publication. The caller is responsible for the final atomic rename. + */ +export async function materializeEvidenceTree(options: MaterializeEvidenceTreeOptions): Promise { + const limits: EvidenceMaterializationLimits = { ...DEFAULT_EVIDENCE_MATERIALIZATION_LIMITS, ...options.limits }; + if (!/^[0-9a-f]{40}$/.test(options.revision)) throw new Error("evidence revision is invalid"); + if (!/^[a-z][a-z0-9-]{2,62}$/.test(options.id)) throw new Error("evidence workspace id is invalid"); + if (!isAbsolute(options.targetDirectory)) throw new Error("evidence target directory must be absolute"); + + const objects = await options.repository.evidenceTreeObjects(options.revision, options.id); + if (objects.length > limits.maxEntries) throw new Error("evidence entry count exceeds the bound"); + const tree = await options.repository.evidenceTreeId(options.revision, options.id); + + // Disk-space preflight: sum the real object sizes before writing anything. + const sizes = new Map(); + let totalBytes = 0; + for (const entry of objects) { + if (Buffer.byteLength(entry.posixPath, "utf8") > limits.maxPathBytes) { + throw new Error("evidence path exceeds the bound"); + } + const size = await options.repository.gitObjectSize(entry.oid); + if (size > limits.maxFileBytes) throw new Error("evidence file exceeds the bound"); + sizes.set(entry.oid, size); + totalBytes += size; + if (totalBytes > limits.maxTotalBytes) throw new Error("evidence total bytes exceed the bound"); + } + + const root = join(options.targetDirectory, "evidence"); + mkdirSync(root, { recursive: true, mode: 0o700 }); + const files: Record = {}; + for (const entry of objects) { + const contents = await options.repository.evidenceBlobBytes(entry.oid, limits.maxFileBytes); + if (contents.length !== sizes.get(entry.oid)) { + throw new Error("evidence object changed while materializing"); + } + const target = join(root, ...entry.posixPath.split("/")); + writeExclusiveNoFollow(target, contents, entry.mode === "100755" ? 0o755 : 0o644); + files[entry.posixPath] = { + mode: entry.mode, + oid: entry.oid, + digest: sha256Prefixed(contents), + bytes: contents.length, + }; + } + + const manifest: EvidenceManifest = { + schemaVersion: 1, + workspace: options.id, + commit: options.revision, + tree, + entryCount: objects.length, + totalBytes, + files, + }; + const manifestJson = `${JSON.stringify(manifest)}\n`; + if (Buffer.byteLength(manifestJson, "utf8") > limits.maxManifestBytes) { + throw new Error("evidence manifest exceeds the bound"); + } + const manifestPath = join(options.targetDirectory, "evidence.manifest.json"); + writeExclusiveNoFollow(manifestPath, Buffer.from(manifestJson, "utf8"), 0o600); + return { root, manifestPath, manifest, manifestDigest: sha256Hex(manifestJson) }; +} diff --git a/backend/src/workspaces/git-repository.ts b/backend/src/workspaces/git-repository.ts new file mode 100644 index 00000000..a01203c1 --- /dev/null +++ b/backend/src/workspaces/git-repository.ts @@ -0,0 +1,621 @@ +import { execFile, spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { lstatSync, mkdirSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { isAbsolute, join } from "node:path"; +import { promisify, TextDecoder } from "node:util"; +import type { WorkspaceErrorCode, WorkspaceRegistryConfig } from "./types.js"; + +const execFileAsync = promisify(execFile); + +export interface GitStatus { + branch: string; + repository?: WorkspaceRepositoryIdentity; + head?: string; + ahead: number; + behind: number; + degraded: boolean; + lastError?: WorkspaceErrorCode; +} + +export interface WorkspaceRepositoryIdentity { + host: string; + repository: string; + transport: "https" | "ssh" | "local"; +} + +export interface EvidenceTreeObject { + mode: "100644" | "100755"; + oid: string; + posixPath: string; +} + +export class WorkspaceRegistryError extends Error { + constructor(readonly code: WorkspaceErrorCode, message: string) { + super(message); + this.name = "WorkspaceRegistryError"; + } +} + +function invalidRemote(): never { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Git remote is invalid"); +} + +function safeRepositoryPath(raw: string): string { + let decoded: string; + try { + decoded = decodeURIComponent(raw).replace(/^\/+/, "").replace(/\/+$/, "").replace(/\.git$/, ""); + } catch { + return invalidRemote(); + } + if ( + decoded.length === 0 + || decoded.includes("\\") + || decoded.split("/").some((part) => part === "" || part === "." || part === "..") + || /[\p{Cc}\s?#]/u.test(decoded) + ) return invalidRemote(); + return decoded; +} + +/** Convert a configured remote to the only repository identity safe for API/UI responses. */ +export function normalizeRepositoryIdentity(remote: string): WorkspaceRepositoryIdentity { + if (remote.length === 0 || remote.trim() !== remote || remote.includes("\0")) return invalidRemote(); + if (isAbsolute(remote) || remote.startsWith("file://")) { + return { host: "local", repository: "configured-repository", transport: "local" }; + } + const scp = /^git@([^:/\s]+):(.+)$/.exec(remote); + if (scp) { + return { host: scp[1].toLowerCase(), repository: safeRepositoryPath(scp[2]), transport: "ssh" }; + } + let parsed: URL; + try { + parsed = new URL(remote); + } catch { + return invalidRemote(); + } + if (parsed.search || parsed.hash || !parsed.hostname || parsed.port) return invalidRemote(); + if (parsed.protocol === "https:") { + if (parsed.username || parsed.password) return invalidRemote(); + return { + host: parsed.hostname.toLowerCase(), + repository: safeRepositoryPath(parsed.pathname), + transport: "https", + }; + } + if (parsed.protocol === "ssh:") { + if (parsed.password || (parsed.username !== "" && parsed.username !== "git")) return invalidRemote(); + return { + host: parsed.hostname.toLowerCase(), + repository: safeRepositoryPath(parsed.pathname), + transport: "ssh", + }; + } + return invalidRemote(); +} + +function isMissing(path: string): boolean { + try { + lstatSync(path); + return false; + } catch { + return true; + } +} + +function assertDirectory(path: string): void { + const entry = lstatSync(path); + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new WorkspaceRegistryError("git_unavailable", "Workspace registry path is unavailable"); + } +} + +function isValidUtf8(buffer: Buffer): boolean { + try { + new TextDecoder("utf-8", { fatal: true }).decode(buffer); + return true; + } catch { + return false; + } +} + +function gitErrorCode(error: unknown): WorkspaceErrorCode { + const detail = [ + error instanceof Error ? error.message : "", + typeof error === "object" && error !== null && "stderr" in error + ? String((error as { stderr?: unknown }).stderr ?? "") + : "", + ].join("\n").toLowerCase(); + if (/authentication failed|could not read username|permission denied|publickey/.test(detail)) { + return "git_auth_failed"; + } + if (/non-fast-forward|not possible to fast-forward|fast-forward/.test(detail)) { + return "git_non_fast_forward"; + } + return "git_unavailable"; +} + +/** + * A persistent checkout that executes Git only through fixed argument vectors. Git's stdout and + * stderr are intentionally never exposed: they can contain remote URLs or credential hints. + */ +export class GitWorkspaceRepository { + readonly root: string; + readonly repoPath: string; + readonly snapshotsPath: string; + readonly statePath: string; + readonly locksPath: string; + private readonly hooksPath: string; + private readonly identity?: WorkspaceRepositoryIdentity; + + constructor(private readonly config: WorkspaceRegistryConfig) { + if (!isAbsolute(config.root)) { + throw new WorkspaceRegistryError("git_unavailable", "Workspace registry root is unavailable"); + } + this.root = config.root; + this.repoPath = join(this.root, "repo"); + this.snapshotsPath = join(this.root, "snapshots"); + this.statePath = join(this.root, "state"); + this.locksPath = join(this.root, "locks"); + this.hooksPath = join(this.locksPath, "empty-hooks"); + this.identity = config.remoteUrl === undefined + ? undefined + : normalizeRepositoryIdentity(config.remoteUrl); + } + + async ensureLayout(): Promise { + try { + for (const path of [this.root, this.snapshotsPath, this.statePath, this.locksPath, this.hooksPath]) { + await mkdir(path, { recursive: true, mode: 0o700 }); + assertDirectory(path); + } + } catch (error) { + if (error instanceof WorkspaceRegistryError) throw error; + throw new WorkspaceRegistryError("git_unavailable", "Workspace registry storage is unavailable"); + } + } + + async bootstrap(): Promise { + await this.ensureLayout(); + if (isMissing(this.repoPath)) { + if (!this.config.remoteUrl) { + throw new WorkspaceRegistryError("git_unavailable", "Workspace registry remote is unavailable"); + } + await this.clone(); + } else { + assertDirectory(this.repoPath); + await this.refresh(); + } + return await this.status(); + } + + async pull(): Promise { + await this.ensureLayout(); + if (isMissing(this.repoPath)) return await this.bootstrap(); + assertDirectory(this.repoPath); + await this.refresh(); + return await this.status(); + } + + async status(): Promise { + const head = (await this.git(["rev-parse", "HEAD"])).trim(); + const tracking = await this.gitOptional(["rev-list", "--left-right", "--count", "HEAD...@{upstream}"]); + const [ahead = "0", behind = "0"] = tracking ? tracking.trim().split(/\s+/) : []; + return { + branch: this.config.branch, + ...(this.identity ? { repository: this.identity } : {}), + head, + ahead: Number(ahead), + behind: Number(behind), + degraded: false, + }; + } + + async workspaceDirectories(): Promise { + const output = await this.git(["ls-tree", "-d", "--name-only", "HEAD"]); + const directories = output.trim() === "" ? [] : output.trim().split("\n"); + for (const id of directories) { + if (id === "workspace-docs") continue; + if (!/^[a-z][a-z0-9-]{2,62}$/.test(id)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository contains an invalid path"); + } + } + return directories.filter((id) => id !== "workspace-docs").sort(); + } + + async workspacePaths(): Promise { + const paths: string[] = []; + for (const id of await this.workspaceDirectories()) { + const path = `${id}/workspace.yaml`; + const type = await this.gitOptional(["cat-file", "-t", `HEAD:${path}`]); + if (type === undefined) continue; + if (type.trim() !== "blob") { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace descriptor is invalid"); + } + paths.push(path); + } + return paths.sort(); + } + + async readCatalog(revision = "HEAD"): Promise { + return await this.git(["show", `${revision}:thoth-workspaces.yaml`], {}, "Workspace catalog is invalid"); + } + + async catalogBlob(revision = "HEAD"): Promise { + return (await this.git(["rev-parse", `${revision}:thoth-workspaces.yaml`], {}, + "Workspace catalog is invalid")).trim(); + } + + async readWorkspace(path: string, revision = "HEAD"): Promise { + if (!/^(?!workspace-docs\/)[a-z][a-z0-9-]{2,62}\/workspace\.yaml$/.test(path)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository path is invalid"); + } + return await this.git(["show", `${revision}:${path}`]); + } + + async blob(path: string, revision = "HEAD"): Promise { + if (!/^(?!workspace-docs\/)[a-z][a-z0-9-]{2,62}\/workspace\.yaml$/.test(path)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository path is invalid"); + } + return (await this.git(["rev-parse", `${revision}:${path}`])).trim(); + } + + /** Read-only object type at an exact revision, or undefined when absent. */ + async gitObjectType(revision: string, path: string): Promise { + if (!/^[0-9a-f]{40}$/.test(revision)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace revision is invalid"); + } + if (!/^[a-z][a-z0-9-]{2,62}\/workspace\.yaml$/.test(path)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository path is invalid"); + } + return await this.gitOptional(["cat-file", "-t", `${revision}:${path}`]); + } + + /** Read a generated-doc blob at an exact revision, or undefined when absent. */ + async readObjectOrAbsent(revision: string, path: string): Promise { + if (!/^[0-9a-f]{40}$/.test(revision)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace revision is invalid"); + } + if (!/^workspace-docs\/[a-z][a-z0-9-]{2,62}\/(?:contract\.env\.example|README\.md)$/.test(path)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository path is invalid"); + } + const output = await this.gitOptional(["show", `${revision}:${path}`]); + return output === undefined ? undefined : output; + } + + /** List committed generated-doc paths at an exact revision. */ + async workspaceDocsPaths(revision: string): Promise { + if (!/^[0-9a-f]{40}$/.test(revision)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace revision is invalid"); + } + const output = await this.git(["ls-tree", "-r", "--name-only", revision, "--", "workspace-docs"]); + if (output.trim() === "") return []; + const paths = output.trim().split("\n"); + for (const path of paths) { + if (!/^workspace-docs\/[a-z][a-z0-9-]{2,62}\/(?:contract\.env\.example|README\.md)$/.test(path)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace repository contains an invalid docs path"); + } + } + return paths; + } + + /** Assert that a canonical Evidence root is a Git tree at an exact commit. */ + async assertTreeAtRevision(revision: string, repoRelativePath: string): Promise { + if (!/^[0-9a-f]{40}$/.test(revision) + || !/^[a-z][a-z0-9-]{2,62}\/evidence$/.test(repoRelativePath)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence revision is invalid"); + } + const type = (await this.git( + ["cat-file", "-t", `${revision}:${repoRelativePath}`], + {}, + "Workspace Evidence root is invalid", + )).trim(); + if (type !== "tree") { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence root is invalid"); + } + } + + /** Read the curated FK annotations object at an exact commit, or undefined when absent. */ + async annotationsObject(revision: string, id: string): Promise<{ blobId: string; contents: Buffer } | undefined> { + if (!/^[0-9a-f]{40}$/.test(revision)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations revision is invalid"); + } + if (!/^[a-z][a-z0-9-]{2,62}$/.test(id) || id === "workspace-docs") { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations path is invalid"); + } + const path = `${id}/schema/annotations.yaml`; + // ls-tree -z reports the exact object at the path (or its children when the path is a tree). + const listing = await this.git(["ls-tree", "-z", "--full-tree", revision, "--", path]); + const entries = listing.split("\0").filter((entry) => entry.length > 0); + if (entries.length === 0) return undefined; + const exact = entries.find((entry) => entry.slice(entry.lastIndexOf("\t") + 1) === path); + if (exact === undefined) { + // The path resolves to a tree (its children are listed) or another non-blob object. + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid"); + } + const match = /^([0-9]{6})\s+(blob|tree|commit)\s+([0-9a-f]{40})\t/.exec(exact); + // Only regular Git blobs are accepted: symlinks (120000) and gitlinks (160000) are refused. + if (match === null || match[2] !== "blob" || (match[1] !== "100644" && match[1] !== "100755")) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is invalid"); + } + const blobId = match[3]; + const contents = await this.gitBlobBytes(blobId, 16 * 1024 * 1024, "annotations"); + if (!isValidUtf8(contents)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace annotations object is not valid UTF-8"); + } + return { blobId, contents }; + } + + /** + * Recursively enumerate a canonical `/evidence` tree at an exact commit as regular Git blobs. + * Symlinks (120000), gitlinks (160000), non-regular modes, non-blob types, traversal/absolute/ + * duplicate/cross-namespace paths, and NUL/newline-bearing names are refused. + */ + async evidenceTreeObjects(revision: string, id: string): Promise { + if (!/^[0-9a-f]{40}$/.test(revision)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence revision is invalid"); + } + if (!/^[a-z][a-z0-9-]{2,62}$/.test(id)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence path is invalid"); + } + const prefix = `${id}/evidence`; + const listing = await this.git(["ls-tree", "-r", "-z", "--full-tree", revision, "--", prefix]); + const entries = listing.split("\0").filter((entry) => entry.length > 0); + const seen = new Set(); + const objects: EvidenceTreeObject[] = []; + for (const entry of entries) { + const tab = entry.lastIndexOf("\t"); + if (tab < 0) throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object is invalid"); + const name = entry.slice(tab + 1); + const meta = entry.slice(0, tab); + const match = /^([0-9]{6}) (blob|commit|tree) ([0-9a-f]{40})$/.exec(meta); + if (match === null || match[2] !== "blob" || (match[1] !== "100644" && match[1] !== "100755")) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object is invalid"); + } + if (name === prefix) { + // The Evidence root resolves to a single regular blob (or symlink/gitlink already refused above). + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence root is invalid"); + } + if (!name.startsWith(`${prefix}/`)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object escapes its namespace"); + } + const rel = name.slice(prefix.length + 1); + if (rel.length === 0 || rel.includes("\0") || rel.includes("\n") || rel.includes("\r")) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence path is invalid"); + } + const segments = rel.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence path is invalid"); + } + if (seen.has(rel)) throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence path is duplicated"); + seen.add(rel); + objects.push({ mode: match[1] as "100644" | "100755", oid: match[3], posixPath: rel }); + } + return objects; + } + + /** Read one Evidence blob with a per-object byte bound. */ + evidenceBlobBytes(objectId: string, maxBytes: number): Promise { + return this.gitBlobBytes(objectId, maxBytes, "Evidence"); + } + + /** Return the 40-hex tree id of a canonical Evidence root at an exact commit. */ + async evidenceTreeId(revision: string, id: string): Promise { + if (!/^[0-9a-f]{40}$/.test(revision) || !/^[a-z][a-z0-9-]{2,62}$/.test(id)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence revision is invalid"); + } + const objectId = (await this.git(["rev-parse", `${revision}:${id}/evidence`])).trim(); + const type = (await this.git(["cat-file", "-t", objectId])).trim(); + if (!/^[0-9a-f]{40}$/.test(objectId) || type !== "tree") { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence root is invalid"); + } + return objectId; + } + + /** Return the byte size of one Git object without reading its contents. */ + async gitObjectSize(objectId: string): Promise { + if (!/^[0-9a-f]{40}$/.test(objectId)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object is invalid"); + } + const raw = (await this.git(["cat-file", "-s", objectId])).trim(); + const size = Number(raw); + if (!Number.isSafeInteger(size) || size < 0) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace Evidence object size is invalid"); + } + return size; + } + + private async gitBlobBytes(objectId: string, maxBytes: number, label: string): Promise { + if (!/^[0-9a-f]{40}$/.test(objectId)) { + throw new WorkspaceRegistryError("workspace_invalid", `Workspace ${label} object is invalid`); + } + try { + const { stdout } = await execFileAsync( + "git", + ["-c", `core.hooksPath=${this.hooksPath}`, "cat-file", "blob", objectId], + { + cwd: this.repoPath, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, + encoding: "buffer", + maxBuffer: maxBytes + 1024 * 1024, + }, + ); + if (stdout.length > maxBytes) { + throw new WorkspaceRegistryError("workspace_invalid", `Workspace ${label} object is too large`); + } + return stdout; + } catch (error) { + if (error instanceof WorkspaceRegistryError) throw error; + const detail = error instanceof Error ? error.message : ""; + if (/maxBuffer|stdout maxBuffer/i.test(detail)) { + throw new WorkspaceRegistryError("workspace_invalid", `Workspace ${label} object is too large`); + } + throw this.sanitizeGitError(error); + } + } + + private async clone(): Promise { + try { + await execFileAsync("git", [ + "-c", `core.hooksPath=${this.hooksPath}`, + "clone", "--branch", this.config.branch, "--single-branch", "--", this.config.remoteUrl!, this.repoPath, + ], { cwd: this.root, env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); + assertDirectory(this.repoPath); + } catch (error) { + throw this.sanitizeGitError(error); + } + } + + private async refresh(): Promise { + if ((await this.git(["status", "--porcelain"])).trim() !== "") { + throw new WorkspaceRegistryError("workspace_stale", "Workspace checkout has local changes"); + } + if (this.config.remoteUrl) { + await this.git(["remote", "set-url", "origin", "--", this.config.remoteUrl]); + } + await this.git(["fetch", "--no-tags", "origin", this.config.branch]); + const remoteHead = (await this.git(["rev-parse", "FETCH_HEAD"])).trim(); + const localHead = (await this.git(["rev-parse", "HEAD"])).trim(); + if (localHead !== remoteHead) { + const commonAncestor = (await this.git(["merge-base", "HEAD", "FETCH_HEAD"])).trim(); + if (commonAncestor !== localHead) { + throw new WorkspaceRegistryError("git_non_fast_forward", "Workspace checkout diverged from remote"); + } + await this.git(["merge", "--ff-only", "FETCH_HEAD"]); + } + if ((await this.git(["rev-parse", "HEAD"])).trim() !== remoteHead) { + throw new WorkspaceRegistryError("git_non_fast_forward", "Workspace checkout does not match remote"); + } + } + + private async git( + args: string[], + env: NodeJS.ProcessEnv = {}, + invalidObjectMessage?: string, + ): Promise { + try { + const { stdout } = await execFileAsync( + "git", + ["-c", `core.hooksPath=${this.hooksPath}`, ...args], + { cwd: this.repoPath, env: { ...process.env, GIT_TERMINAL_PROMPT: "0", ...env } }, + ); + return stdout; + } catch (error) { + const stderr = typeof error === "object" && error !== null && "stderr" in error + && typeof error.stderr === "string" ? error.stderr : ""; + if (invalidObjectMessage + && /^fatal: path '[^']+' does not exist in '[0-9a-f]{40}'\s*$/u.test(stderr)) { + throw new WorkspaceRegistryError("workspace_invalid", invalidObjectMessage); + } + throw this.sanitizeGitError(error); + } + } + + private async gitOptional(args: string[]): Promise { + try { + return await this.git(args); + } catch (error) { + if (error instanceof WorkspaceRegistryError && error.code === "git_unavailable") return undefined; + throw error; + } + } + + private sanitizeGitError(error: unknown): WorkspaceRegistryError { + return new WorkspaceRegistryError(gitErrorCode(error), "Workspace Git operation failed"); + } +} + +export class WorkspaceRepositoryLock { + private queue = Promise.resolve(); + + constructor(private readonly locksPath: string) {} + + async run(operation: () => Promise): Promise { + const previous = this.queue; + let releaseQueue!: () => void; + this.queue = new Promise((resolve) => { releaseQueue = resolve; }); + await previous; + + let holder: ChildProcessWithoutNullStreams | undefined; + const lockPath = join(this.locksPath, "repository.lock"); + try { + try { + mkdirSync(this.locksPath, { recursive: true, mode: 0o700 }); + assertDirectory(this.locksPath); + } catch (error) { + throw new WorkspaceRegistryError("git_unavailable", "Workspace registry lock is unavailable"); + } + try { + holder = await this.acquire(lockPath); + } catch (error) { + throw this.lockError(error); + } + return await operation(); + } finally { + try { + if (holder !== undefined) await this.release(holder); + } finally { + releaseQueue(); + } + } + } + + private async acquire(lockPath: string): Promise { + try { + const entry = lstatSync(lockPath); + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("invalid lock path"); + } catch (error) { + if (!(typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT")) { + throw error; + } + } + const holder = spawn("python3", ["-c", WorkspaceRepositoryLock.HOLDER_PROGRAM, lockPath], { + stdio: ["pipe", "pipe", "pipe"], + }); + await new Promise((resolve, reject) => { + let output = ""; + const fail = (error: WorkspaceRegistryError) => { + holder.stdout.removeAllListeners("data"); + reject(error); + }; + holder.once("error", () => fail(new WorkspaceRegistryError("git_unavailable", "Workspace registry lock is unavailable"))); + holder.once("exit", (code) => { + fail(new WorkspaceRegistryError( + code === 73 ? "workspace_stale" : "git_unavailable", + code === 73 ? "Workspace registry is busy" : "Workspace registry lock is unavailable", + )); + }); + holder.stdout.on("data", (chunk: Buffer) => { + output += chunk.toString("utf8"); + if (output === "locked\n") { + holder.stdout.removeAllListeners("data"); + resolve(); + } + }); + }); + return holder; + } + + private async release(holder: ChildProcessWithoutNullStreams): Promise { + if (!holder.stdin.destroyed) holder.stdin.end(); + await new Promise((resolve) => holder.once("exit", () => resolve())); + } + + private lockError(error: unknown): WorkspaceRegistryError { + if (error instanceof WorkspaceRegistryError) return error; + if (typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST") { + return new WorkspaceRegistryError("workspace_stale", "Workspace registry is busy"); + } + return new WorkspaceRegistryError("git_unavailable", "Workspace registry lock is unavailable"); + } + + private static readonly HOLDER_PROGRAM = [ + "import fcntl, os, sys", + "fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT | getattr(os, 'O_NOFOLLOW', 0), 0o600)", + "try:", + " fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)", + "except BlockingIOError:", + " sys.exit(73)", + "sys.stdout.write('locked\\n')", + "sys.stdout.flush()", + "sys.stdin.buffer.read()", + ].join("\n"); +} diff --git a/backend/src/workspaces/preprocessing-service.ts b/backend/src/workspaces/preprocessing-service.ts new file mode 100644 index 00000000..c8180cab --- /dev/null +++ b/backend/src/workspaces/preprocessing-service.ts @@ -0,0 +1,613 @@ +import { createHash, randomBytes } from "node:crypto"; +import { readdirSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { isIP } from "node:net"; +import { join } from "node:path"; +import type { WorkspaceDescriptor } from "./schema.js"; +import { + PreprocessingStateStore, + type FkReviewRecord, + type PreprocessingJobState, + type SessionInventoryRow, +} from "./preprocessing-state.js"; +import type { DeterministicRuntimeConfigLease } from "./runtime-config-lease.js"; +import { readAnnotationsSync } from "./annotations-sync.js"; +import { reconcileCollection } from "./qdrant-collection.js"; + +export interface WorkspaceOperationResult { + schemaVersion: 1; + status: "succeeded" | "unchanged" | "dry_run" | "blocked" | "failed"; + code: + | "ok" | "workspace_not_found" | "workspace_not_activatable" + | "binding_missing" | "preprocessing_conflict" + | "preprocessing_resume_mismatch" | "manual_review_required" + | "evidence_materialization_required" | "effective_config_mismatch" + | "semantic_index_incompatible" | "annotation_invalid" + | "egress_policy_refused"; + workspaceId: string; + workspaceRevision: string; + descriptorBlob: string; + operation: string; + runId?: string; + childRuns?: Record; + completedStages: string[]; + counts?: Record; + artifactIdentities?: Array<{ kind: string; digest: string }>; + effectiveConfigIdentity?: string; + configFingerprint?: string; + inputFingerprint?: string; + /** Suggested FK annotations YAML for the operator to write to --output (schema suggest-fks). */ + suggestedFksYaml?: string; + warnings?: string[]; +} + +export interface ChildProcessRequest { + argv: string[]; + configPath: string; +} + +interface ActiveRuntime { + workspace: WorkspaceDescriptor; + workspaceId: string; + workspaceRevision: string; + descriptorBlob: string; + catalogBlob: string; + configLease: DeterministicRuntimeConfigLease; +} + +interface ChildProcessResult { + exitCode: number; + stdout: string; + stderr: string; +} + +export interface WorkspacePreprocessingServiceDeps { + dataRoot: string; + acquireActiveRuntime(workspaceId: string): Promise; + runChild(request: ChildProcessRequest): Promise; + listSessions(workspaceId: string): Promise; + semanticPreflight(workspace: WorkspaceDescriptor): Promise< + { ok: true } | { ok: false; code: "workspace_not_activatable" | "semantic_index_incompatible" } + >; + httpPrivateHostAllowlist?: readonly string[]; +} + +interface RunScope { + runtime: ActiveRuntime; + state: PreprocessingStateStore; + job: PreprocessingJobState; +} + +function digest(value: string | Buffer): string { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +function baseResult( + runtime: ActiveRuntime, + operation: string, + status: WorkspaceOperationResult["status"], + code: WorkspaceOperationResult["code"], + extra: Omit, "schemaVersion" | "status" | "code" | "workspaceId" | "workspaceRevision" | "descriptorBlob" | "operation"> = {}, +): WorkspaceOperationResult { + return { + schemaVersion: 1, + status, + code, + workspaceId: runtime.workspaceId, + workspaceRevision: runtime.workspaceRevision, + descriptorBlob: runtime.descriptorBlob, + operation, + completedStages: [], + effectiveConfigIdentity: runtime.configLease.effectiveConfigIdentity, + configFingerprint: runtime.configLease.configFingerprint, + inputFingerprint: runtime.configLease.inputFingerprint, + ...extra, + }; +} + +function isPrivateHost(hostname: string): boolean { + if (hostname === "localhost" || hostname === "metadata.google.internal") return true; + const address = isIP(hostname); + if (address === 4) { + if (/^127\./.test(hostname) || /^10\./.test(hostname) || /^192\.168\./.test(hostname)) return true; + if (/^169\.254\./.test(hostname) || /^0\./.test(hostname)) return true; + const match = /^172\.(\d+)\./.exec(hostname); + return Boolean(match && Number(match[1]) >= 16 && Number(match[1]) <= 31); + } + if (address === 6) { + const normalized = hostname.toLowerCase(); + return normalized === "::1" || normalized.startsWith("fe80:") || normalized.startsWith("fd") || normalized.startsWith("fc"); + } + return hostname.endsWith(".internal"); +} + +function noEvidenceWarning(workspace: WorkspaceDescriptor): string[] { + return workspace.evidence === undefined ? ["workspace has no Evidence source"] : []; +} + +export class WorkspacePreprocessingService { + constructor(private readonly deps: WorkspacePreprocessingServiceDeps) {} + + + async vectorInspect(options: { workspaceId: string }): Promise { + const runtime = await this.deps.acquireActiveRuntime(options.workspaceId); + const collection = runtime.workspace.semantic_index.vector_store.collection; + const res = await fetch(`${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`, { method: "GET" }); + if (!res.ok) return baseResult(runtime, "vector inspect", "failed", "semantic_index_incompatible", { warnings: ["collection unavailable"] }); + const body = await res.json() as any; + const info = body?.result; + const vectors = info?.config?.params?.vectors; + return baseResult(runtime, "vector inspect", "succeeded", "ok", { + counts: { dimensions: vectors?.size ?? 0 }, + warnings: [`collection=${collection} distance=${vectors?.distance ?? "unknown"}`], + }); + } + + async vectorRebuild(options: { workspaceId: string; collection?: string; confirm?: string; destroy?: boolean }): Promise { + const runtime = await this.deps.acquireActiveRuntime(options.workspaceId); + const collection = runtime.workspace.semantic_index.vector_store.collection; + if (options.collection !== collection || options.confirm !== collection || options.destroy !== true) { + return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["rebuild requires exact confirmation and --destroy"] }); + } + const q = `${runtime.configLease.semanticQdrantUrl}/collections/${encodeURIComponent(collection)}`; + const del = await fetch(q, { method: "DELETE" }); + if (!del.ok && del.status !== 404) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection delete failed"] }); + // Recreate the complete contract (dimensions + distance + the 8 required keyword indexes). + const recreated = await reconcileCollection({ + baseUrl: runtime.configLease.semanticQdrantUrl, + collection, + dimensions: runtime.workspace.semantic_index.vector_store.dimensions, + distance: runtime.workspace.semantic_index.vector_store.distance, + mode: "self_heal", + }); + if (!recreated.ok) return baseResult(runtime, "vector rebuild", "failed", "semantic_index_incompatible", { warnings: ["collection recreate failed"] }); + return baseResult(runtime, "vector rebuild", "succeeded", "ok", { warnings: [`recreated collection=${collection}`] }); + } + async inspect(options: { workspaceId: string }): Promise { + try { + const runtime = await this.deps.acquireActiveRuntime(options.workspaceId); + return baseResult(runtime, "inspect", "succeeded", "ok", { + artifactIdentities: [ + { kind: "descriptor", digest: runtime.descriptorBlob }, + { kind: "catalog", digest: runtime.catalogBlob }, + { kind: "runtime_config", digest: runtime.configLease.configDigest }, + ], + }); + } catch { + return { + schemaVersion: 1, + status: "failed", + code: "workspace_not_activatable", + workspaceId: options.workspaceId, + workspaceRevision: "", + descriptorBlob: "", + operation: "inspect", + completedStages: [], + }; + } + } + + async preprocessDwh(options: { workspaceId: string; resumeRunId?: string }): Promise { + const scope = await this.startRun(options.workspaceId, "preprocess dwh", options.resumeRunId); + if (scope.job.completedStages.includes("dwh")) { + return baseResult(scope.runtime, "preprocess dwh", "unchanged", "ok", { + runId: scope.job.runId, + childRuns: scope.job.childRuns, + completedStages: [...scope.job.completedStages], + }); + } + const payload = await this.runJsonStage(scope.runtime, [ + "preprocess", "dwh", "--steps", "introspect,lsh", + ...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []), + "--json", "-c", "/dev/fd/3", + ]); + const childRun = this.requireRunId(payload.run_id); + scope.job.childRuns.dwh = childRun; + if (!scope.job.completedStages.includes("dwh")) scope.job.completedStages.push("dwh"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + return baseResult(scope.runtime, "preprocess dwh", "succeeded", "ok", { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + }); + } + + async suggestFks(options: { + workspaceId: string; + fromSql?: ReadonlyArray<{ name: string; sql: string }>; + assume?: readonly string[]; + resumeRunId?: string; + }): Promise { + const scope = await this.startRun(options.workspaceId, "schema suggest-fks", options.resumeRunId); + return await this.runSuggestStage(scope, options.fromSql ?? [], options.assume ?? []); + } + + async checkSchema(options: { + workspaceId: string; + annotationsYaml?: string; + reviewedCandidatesDigest?: string; + }): Promise { + const runtime = await this.deps.acquireActiveRuntime(options.workspaceId); + const state = this.state(runtime.workspaceId); + if ((options.annotationsYaml === undefined) !== (options.reviewedCandidatesDigest === undefined)) { + return baseResult(runtime, "schema check", "failed", "annotation_invalid"); + } + if (options.reviewedCandidatesDigest === undefined) { + const payload = await this.runJsonStage(runtime, ["schema", "check", "--json", "-c", "/dev/fd/3"]); + return baseResult(runtime, "schema check", Number(payload.orphan_count ?? 0) === 0 ? "succeeded" : "failed", Number(payload.orphan_count ?? 0) === 0 ? "ok" : "annotation_invalid"); + } + const reviewedCandidatesDigest = options.reviewedCandidatesDigest; + const runId = this.findRunIdByCandidateDigest(state, reviewedCandidatesDigest); + if (!runId) return baseResult(runtime, "schema check", "failed", "annotation_invalid"); + const request = await this.withStagedInputs(runtime.workspaceId, [ + { flag: "--annotations", name: "annotations.yaml", contents: options.annotationsYaml! }, + ], async (argv) => await this.runJsonStage(runtime, [ + "schema", "check", ...argv, + "--reviewed-candidates", reviewedCandidatesDigest, + "--json", "-c", "/dev/fd/3", + ])); + if (request.reviewed_candidates_digest !== reviewedCandidatesDigest || typeof request.annotations_digest !== "string") { + return baseResult(runtime, "schema check", "failed", "annotation_invalid", { runId }); + } + // P5 supersedes the host-file FK review: schema check is read-only validation and never + // records a review. Only `schema accept` records a human review for the curated Git blob. + return baseResult(runtime, "schema check", "succeeded", "ok", { runId }); + } + + async acceptSchema(options: { workspaceId: string; runId: string; yes?: boolean }): Promise { + const runtime = await this.deps.acquireActiveRuntime(options.workspaceId); + const state = this.state(runtime.workspaceId); + if (options.yes !== true) { + return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { + runId: options.runId, + warnings: ["accept requires --yes"], + }); + } + if (!/^[0-9a-f]{32}$/.test(options.runId)) { + return baseResult(runtime, "schema accept", "failed", "annotation_invalid"); + } + const candidate = state.readFkCandidates(options.runId); + if (candidate === undefined) { + return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { + runId: options.runId, + warnings: ["candidate run is unavailable"], + }); + } + const synced = readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision); + if (synced === undefined || synced.contents.toString("utf8").trim() === "") { + return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { + runId: options.runId, + warnings: ["curated annotations are not synchronized"], + }); + } + // The harness parser validates the curated blob against the physical schema; the recorded + // candidate digest must round-trip and the blob digest must match the synced destination. + const payload = await this.runJsonStage(runtime, [ + "schema", "check", "--reviewed-candidates", candidate.digest, "--json", "-c", "/dev/fd/3", + ]); + if (payload.annotations_digest !== synced.contentDigest + || payload.reviewed_candidates_digest !== candidate.digest + || Number(payload.orphan_count ?? 0) !== 0) { + return baseResult(runtime, "schema accept", "failed", "annotation_invalid", { runId: options.runId }); + } + const review = state.writeFkReview(options.runId, { + reviewedCandidatesDigest: candidate.digest, + annotationsDigest: synced.contentDigest, + workspaceRevision: runtime.workspaceRevision, + blobId: synced.blobId, + }); + const job = state.readJob(options.runId); + job.reviewDigest = review.digest; + if (!job.completedStages.includes("fk_review")) job.completedStages.push("fk_review"); + state.writeJob(job); + return baseResult(runtime, "schema accept", "succeeded", "ok", { + runId: options.runId, + completedStages: [...job.completedStages], + artifactIdentities: [ + { kind: "fk_review", digest: review.digest }, + { kind: "annotations", digest: synced.contentDigest }, + ], + }); + } + + async indexSchema(options: { workspaceId: string; resumeRunId?: string }): Promise { + const scope = await this.startRun(options.workspaceId, "index-schema", options.resumeRunId); + const semantic = await this.deps.semanticPreflight(scope.runtime.workspace); + if (!semantic.ok) return baseResult(scope.runtime, "index-schema", "failed", semantic.code, { runId: scope.job.runId }); + if (scope.job.completedStages.includes("schema_index")) { + return baseResult(scope.runtime, "index-schema", "unchanged", "ok", { + runId: scope.job.runId, + completedStages: [...scope.job.completedStages], + }); + } + const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]); + const counts = this.numberRecord(payload.counts); + scope.job.completedStages.push("schema_index"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + return baseResult(scope.runtime, "index-schema", "succeeded", "ok", { + runId: scope.job.runId, + completedStages: [...scope.job.completedStages], + counts, + }); + } + + async preprocessEvidence(options: { workspaceId: string; dryRun?: boolean; resumeRunId?: string }): Promise { + const scope = await this.startRun(options.workspaceId, "preprocess evidence", options.resumeRunId); + if (scope.runtime.workspace.evidence === undefined) { + return baseResult(scope.runtime, "preprocess evidence", "unchanged", "ok", { + warnings: noEvidenceWarning(scope.runtime.workspace), + }); + } + const policy = this.evidencePolicy(scope.runtime.workspace); + if (policy !== undefined) return baseResult(scope.runtime, "preprocess evidence", policy.status, policy.code, { warnings: policy.warnings }); + const semantic = await this.deps.semanticPreflight(scope.runtime.workspace); + if (!semantic.ok) return baseResult(scope.runtime, "preprocess evidence", "failed", semantic.code, { runId: scope.job.runId }); + if (scope.job.completedStages.includes("evidence") && !options.dryRun) { + return baseResult(scope.runtime, "preprocess evidence", "unchanged", "ok", { + runId: scope.job.runId, + completedStages: [...scope.job.completedStages], + }); + } + const payload = await this.runJsonStage(scope.runtime, [ + "preprocess", "evidence", + ...(options.dryRun ? ["--dry-run"] : []), + ...(scope.job.childRuns.evidence ? ["--resume", scope.job.childRuns.evidence] : []), + "--json", "-c", "/dev/fd/3", + ]); + if (typeof payload.run_id === "string") scope.job.childRuns.evidence = this.requireRunId(payload.run_id); + if (!options.dryRun && !scope.job.completedStages.includes("evidence")) scope.job.completedStages.push("evidence"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + return baseResult(scope.runtime, "preprocess evidence", options.dryRun ? "dry_run" : "succeeded", "ok", { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + counts: this.numberRecord(payload.counts), + }); + } + + async run(options: { workspaceId: string; resumeRunId?: string }): Promise { + const scope = await this.startRun(options.workspaceId, "preprocess run", options.resumeRunId); + if (!scope.job.completedStages.includes("dwh")) { + const payload = await this.runJsonStage(scope.runtime, [ + "preprocess", "dwh", "--steps", "introspect,lsh", + ...(scope.job.childRuns.dwh ? ["--resume", scope.job.childRuns.dwh] : []), + "--json", "-c", "/dev/fd/3", + ]); + scope.job.childRuns.dwh = this.requireRunId(payload.run_id); + scope.job.completedStages.push("dwh"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + } + if (!scope.job.completedStages.includes("fk_suggest")) { + const suggest = await this.runSuggestStage(scope, [], []); + if (suggest.code === "manual_review_required") return suggest; + } + const candidate = this.state(scope.runtime.workspaceId).readFkCandidates(scope.job.runId); + if (candidate && !scope.job.completedStages.includes("fk_review")) { + // P5: continuation requires a review accepted for this candidate whose accepted blob digest + // equals the current revision's synced annotations. A revision change (or a missing curated + // blob) therefore records a new review checkpoint instead of silently reusing the old one. + const accepted = this.findAcceptedReviewForDigest(scope.runtime.workspaceId, candidate.digest); + const currentDigest = this.currentAnnotationsDigest(scope.runtime); + if (accepted === undefined || currentDigest === undefined || accepted.annotationsDigest !== currentDigest) { + return baseResult(scope.runtime, "preprocess run", "blocked", "manual_review_required", { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + artifactIdentities: [{ kind: "fk_candidates", digest: candidate.digest }], + }); + } + scope.job.reviewDigest = accepted.reviewedCandidatesDigest; + scope.job.completedStages.push("fk_review"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + } + const semantic = await this.deps.semanticPreflight(scope.runtime.workspace); + if (!semantic.ok) return baseResult(scope.runtime, "preprocess run", "failed", semantic.code, { runId: scope.job.runId }); + if (!scope.job.completedStages.includes("schema_index")) { + const payload = await this.runJsonStage(scope.runtime, ["vector", "index-schema", "--json", "-c", "/dev/fd/3"]); + scope.job.completedStages.push("schema_index"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + const warnings = noEvidenceWarning(scope.runtime.workspace); + if (scope.runtime.workspace.evidence === undefined) { + return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + counts: this.numberRecord(payload.counts), + warnings, + }); + } + } + if (scope.runtime.workspace.evidence === undefined) { + return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + warnings: noEvidenceWarning(scope.runtime.workspace), + }); + } + const policy = this.evidencePolicy(scope.runtime.workspace); + if (policy !== undefined) { + return baseResult(scope.runtime, "preprocess run", policy.status, policy.code, { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + warnings: policy.warnings, + }); + } + if (!scope.job.completedStages.includes("evidence")) { + const payload = await this.runJsonStage(scope.runtime, [ + "preprocess", "evidence", + ...(scope.job.childRuns.evidence ? ["--resume", scope.job.childRuns.evidence] : []), + "--json", "-c", "/dev/fd/3", + ]); + if (typeof payload.run_id === "string") scope.job.childRuns.evidence = this.requireRunId(payload.run_id); + scope.job.completedStages.push("evidence"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + return baseResult(scope.runtime, "preprocess run", "succeeded", "ok", { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + counts: this.numberRecord(payload.counts), + }); + } + return baseResult(scope.runtime, "preprocess run", "unchanged", "ok", { + runId: scope.job.runId, + childRuns: { ...scope.job.childRuns }, + completedStages: [...scope.job.completedStages], + }); + } + + private async startRun(workspaceId: string, operation: string, resumeRunId?: string): Promise { + const runtime = await this.deps.acquireActiveRuntime(workspaceId); + const state = this.state(runtime.workspaceId); + await state.assertSessionInventoryCompatible(runtime.workspaceRevision, await this.deps.listSessions(runtime.workspaceId)); + const job = await state.beginJob({ + operation, + runId: resumeRunId, + workspaceRevision: runtime.workspaceRevision, + descriptorBlob: runtime.descriptorBlob, + catalogBlob: runtime.catalogBlob, + configDigest: runtime.configLease.configDigest, + bindingDigest: runtime.configLease.bindingDigest, + }); + return { runtime, state, job }; + } + + private state(workspaceId: string): PreprocessingStateStore { + return new PreprocessingStateStore({ dataRoot: this.deps.dataRoot, workspaceId }); + } + + private async runSuggestStage( + scope: RunScope, + fromSql: ReadonlyArray<{ name: string; sql: string }>, + assume: readonly string[], + ): Promise { + const payload = await this.withStagedInputs(scope.runtime.workspaceId, fromSql.map((entry) => ({ + flag: "--from-sql", + name: entry.name, + contents: entry.sql, + })), async (stagedArgv) => await this.runJsonStage(scope.runtime, [ + "schema", "suggest-fks", ...stagedArgv, + ...assume.flatMap((value) => ["--assume", value]), + "--json", "-c", "/dev/fd/3", + ])); + const candidateCount = Number(payload.candidate_count ?? 0); + const candidateYaml = typeof payload.candidate_yaml === "string" ? payload.candidate_yaml : ""; + let artifactIdentities: Array<{ kind: string; digest: string }> | undefined; + if (candidateCount > 0) { + const persisted = this.state(scope.runtime.workspaceId).writeFkCandidates(scope.job.runId, candidateYaml); + scope.job.candidateDigest = persisted.digest; + artifactIdentities = [{ kind: "fk_candidates", digest: persisted.digest }]; + } + if (!scope.job.completedStages.includes("fk_suggest")) scope.job.completedStages.push("fk_suggest"); + this.state(scope.runtime.workspaceId).writeJob(scope.job); + const resultExtra = { + runId: scope.job.runId, + completedStages: [...scope.job.completedStages], + ...(artifactIdentities ? { artifactIdentities } : {}), + ...(candidateYaml.length > 0 ? { suggestedFksYaml: candidateYaml } : {}), + }; + if (candidateCount > 0) { + return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "blocked", "manual_review_required", { + ...resultExtra, + childRuns: { ...scope.job.childRuns }, + }); + } + return baseResult(scope.runtime, scope.job.operation === "preprocess run" ? "preprocess run" : "schema suggest-fks", "succeeded", "ok", resultExtra); + } + + private async runJsonStage(runtime: ActiveRuntime, argv: string[]): Promise> { + const result = await this.deps.runChild({ argv, configPath: runtime.configLease.path }); + if (result.exitCode !== 0) throw new Error("workspace child failed"); + return JSON.parse(result.stdout) as Record; + } + + private requireRunId(value: unknown): string { + if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) throw new Error("child run id is invalid"); + return value; + } + + private numberRecord(value: unknown): Record | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + return Object.fromEntries(Object.entries(value as Record).map(([key, nested]) => [key, Number(nested)])); + } + + private async withStagedInputs( + workspaceId: string, + inputs: ReadonlyArray<{ flag: string; name: string; contents: string }>, + fn: (argv: string[]) => Promise, + ): Promise { + if (inputs.length === 0) return await fn([]); + const root = join(this.deps.dataRoot, "sessions", workspaceId, "preprocessing", `.stage-${randomBytes(6).toString("hex")}`); + mkdirSync(root, { recursive: true, mode: 0o700 }); + const argv: string[] = []; + const paths: string[] = []; + try { + for (const input of inputs) { + const path = join(root, input.name); + writeFileSync(path, input.contents, { encoding: "utf8", flag: "wx", mode: 0o600 }); + paths.push(path); + argv.push(input.flag, path); + } + return await fn(argv); + } finally { + rmSync(root, { recursive: true, force: true }); + } + } + + private currentAnnotationsDigest(runtime: ActiveRuntime): string | undefined { + return readAnnotationsSync(this.deps.dataRoot, runtime.workspaceId, runtime.workspaceRevision)?.contentDigest; + } + + private findAcceptedReviewForDigest( + workspaceId: string, + digestValue: string, + ): FkReviewRecord | undefined { + const state = this.state(workspaceId); + for (const entry of readdirSync(state.fkReviewsDirectory(), { withFileTypes: true })) { + if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.json$/.test(entry.name)) continue; + const runId = entry.name.slice(0, -".json".length); + const review = state.readFkReview(runId); + if (review && review.reviewedCandidatesDigest === digestValue) return review; + } + return undefined; + } + + private findRunIdByCandidateDigest(state: PreprocessingStateStore, digestValue: string): string | undefined { + for (const entry of readdirSync(state.fkCandidatesDirectory(), { withFileTypes: true })) { + if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f]{32}\.yaml$/.test(entry.name)) continue; + const runId = entry.name.slice(0, -".yaml".length); + if (state.readFkCandidates(runId)?.digest === digestValue) return runId; + } + return undefined; + } + + private evidencePolicy(workspace: WorkspaceDescriptor): { + status: WorkspaceOperationResult["status"]; + code: WorkspaceOperationResult["code"]; + warnings?: string[]; + } | undefined { + const evidence = workspace.evidence; + if (!evidence) return undefined; + // P6: filesystem Evidence is materialized from the pinned commit at activation, so the + // engine may proceed directly against the immutable revision content root. + if (evidence.source.type === "filesystem") return undefined; + if (evidence.source.type === "http") { + for (const value of evidence.source.uris) { + const host = new URL(value).hostname; + if (isPrivateHost(host) && !(evidence.source.allow_private_hosts && this.deps.httpPrivateHostAllowlist?.includes(host))) { + return { status: "failed", code: "egress_policy_refused" }; + } + } + return undefined; + } + if ( + evidence.source.endpoint_url !== undefined + || evidence.source.credentials === "ambient" + || evidence.source.allow_private_endpoint + || evidence.source.allow_insecure_endpoint + ) { + return { status: "failed", code: "egress_policy_refused" }; + } + return undefined; + } +} diff --git a/backend/src/workspaces/preprocessing-state.ts b/backend/src/workspaces/preprocessing-state.ts new file mode 100644 index 00000000..f950c4ef --- /dev/null +++ b/backend/src/workspaces/preprocessing-state.ts @@ -0,0 +1,392 @@ +import { createHash, randomBytes } from "node:crypto"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { + closeSync, + constants as fsConstants, + fchmodSync, + fstatSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; + +export type PreprocessingConflictCode = "preprocessing_conflict" | "preprocessing_resume_mismatch"; + +export class PreprocessingStateError extends Error { + constructor(readonly code: PreprocessingConflictCode, message: string) { + super(message); + this.name = "PreprocessingStateError"; + } +} + +export interface SessionInventoryRow { + id: string; + status: string; + archived?: boolean; + workspaceRevision?: string | null; +} + +export interface WriterLockLease { + holderPid: number; + release(): Promise; +} + +export interface BeginPreprocessingJobOptions { + operation: string; + workspaceRevision: string; + descriptorBlob: string; + catalogBlob: string; + configDigest: string; + bindingDigest: string; + runId?: string; +} + +export interface PreprocessingJobState { + schemaVersion: 1; + runId: string; + operation: string; + workspaceId: string; + workspaceRevision: string; + descriptorBlob: string; + catalogBlob: string; + configDigest: string; + bindingDigest: string; + completedStages: string[]; + childRuns: Record; + status: "active" | "succeeded" | "blocked" | "failed"; + candidateDigest?: string; + reviewDigest?: string; +} + +export interface FkReviewRecord { + reviewedCandidatesDigest: string; + annotationsDigest: string; + workspaceRevision: string; + /** Curated Git blob id accepted at review time (P5); absent for legacy host-file reviews. */ + blobId?: string; +} + +function sha256(value: string | Buffer): string { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +function syncDirectory(directory: string): void { + if (process.platform === "win32") return; + const fd = openSync(directory, "r"); + try { fsyncSync(fd); } finally { closeSync(fd); } +} + +function ensureDirectory(path: string): string { + mkdirSync(path, { recursive: true, mode: 0o700 }); + const entry = lstatSync(path); + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error("preprocessing state directory is unavailable"); + } + return path; +} + +function validateRunId(runId: string): string { + if (!/^[0-9a-f]{32}$/.test(runId)) throw new Error("preprocessing run id is invalid"); + return runId; +} + +function writeAtomicFile(path: string, contents: string, mode: number): void { + ensureDirectory(join(path, "..")); + const directory = path.slice(0, path.lastIndexOf("/")); + ensureDirectory(directory); + const staging = `${path}.tmp-${process.pid}-${Date.now()}-${randomBytes(6).toString("hex")}`; + const fd = openSync( + staging, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, + 0o600, + ); + let closed = false; + try { + writeFileSync(fd, contents, "utf8"); + fsyncSync(fd); + fchmodSync(fd, mode); + closeSync(fd); + closed = true; + renameSync(staging, path); + syncDirectory(directory); + } catch (error) { + if (!closed) try { closeSync(fd); } catch { /* preserve original failure */ } + try { unlinkSync(staging); } catch { /* best effort */ } + throw error; + } +} + +function readTrustedFile(path: string): string { + const entry = lstatSync(path); + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("preprocessing state file is invalid"); + const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1) throw new Error("preprocessing state file is invalid"); + const contents = readFileSync(fd, "utf8"); + const after = fstatSync(fd); + if ( + before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.nlink !== after.nlink + ) throw new Error("preprocessing state file changed while reading"); + return contents; + } finally { + closeSync(fd); + } +} + +function decodeJob(value: unknown): PreprocessingJobState { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("preprocessing job state is invalid"); + } + const record = value as Record; + if ( + record.schemaVersion !== 1 + || typeof record.runId !== "string" + || typeof record.operation !== "string" + || typeof record.workspaceId !== "string" + || typeof record.workspaceRevision !== "string" + || typeof record.descriptorBlob !== "string" + || typeof record.catalogBlob !== "string" + || typeof record.configDigest !== "string" + || typeof record.bindingDigest !== "string" + || !Array.isArray(record.completedStages) + || typeof record.childRuns !== "object" || record.childRuns === null || Array.isArray(record.childRuns) + || !["active", "succeeded", "blocked", "failed"].includes(String(record.status)) + ) { + throw new Error("preprocessing job state is invalid"); + } + return record as unknown as PreprocessingJobState; +} + +function decodeReview(value: unknown): FkReviewRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("preprocessing review state is invalid"); + } + const record = value as Record; + if ( + typeof record.reviewedCandidatesDigest !== "string" + || typeof record.annotationsDigest !== "string" + || typeof record.workspaceRevision !== "string" + || (record.blobId !== undefined && typeof record.blobId !== "string") + ) throw new Error("preprocessing review state is invalid"); + return record as unknown as FkReviewRecord; +} + +export class PreprocessingStateStore { + private readonly root: string; + + constructor(private readonly options: { dataRoot: string; workspaceId: string }) { + if (!/^[a-z][a-z0-9-]{2,62}$/.test(options.workspaceId)) { + throw new Error("workspace id is invalid"); + } + this.root = join(options.dataRoot, "sessions", options.workspaceId, "preprocessing"); + } + + writerLockPath(): string { return join(this.root, "writer.lock"); } + runtimeConfigDirectory(): string { return join(this.root, "runtime-config"); } + runtimeConfigManifestDirectory(): string { return join(this.root, "runtime-config-manifests"); } + jobsDirectory(): string { return join(this.root, "jobs"); } + fkCandidatesDirectory(): string { return join(this.root, "fk-candidates"); } + fkReviewsDirectory(): string { return join(this.root, "fk-reviews"); } + jobPath(runId: string): string { return join(this.jobsDirectory(), `${validateRunId(runId)}.json`); } + fkCandidatesPath(runId: string): string { return join(this.fkCandidatesDirectory(), `${validateRunId(runId)}.yaml`); } + fkReviewPath(runId: string): string { return join(this.fkReviewsDirectory(), `${validateRunId(runId)}.json`); } + + private ensureLayout(): void { + ensureDirectory(this.root); + for (const directory of [ + this.runtimeConfigDirectory(), + this.runtimeConfigManifestDirectory(), + this.jobsDirectory(), + this.fkCandidatesDirectory(), + this.fkReviewsDirectory(), + ]) ensureDirectory(directory); + } + + async acquireWriterLock(): Promise { + this.ensureLayout(); + const lockPath = this.writerLockPath(); + try { + const entry = lstatSync(lockPath); + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("invalid writer lock path"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw new PreprocessingStateError("preprocessing_conflict", "Workspace preprocessing lock is unavailable"); + } + } + const holder = spawn("python3", ["-c", PreprocessingStateStore.HOLDER_PROGRAM, lockPath], { + stdio: ["pipe", "pipe", "pipe"], + }); + await new Promise((resolve, reject) => { + let output = ""; + const fail = (error: Error) => { + holder.stdout.removeAllListeners("data"); + reject(error); + }; + holder.once("error", () => fail(new PreprocessingStateError( + "preprocessing_conflict", + "Workspace preprocessing lock is unavailable", + ))); + holder.once("exit", (code) => { + fail(new PreprocessingStateError( + "preprocessing_conflict", + code === 73 ? "Workspace preprocessing is busy" : "Workspace preprocessing lock is unavailable", + )); + }); + holder.stdout.on("data", (chunk: Buffer) => { + output += chunk.toString("utf8"); + if (output === "locked\n") { + holder.stdout.removeAllListeners("data"); + resolve(); + } + }); + }); + let released = false; + return { + holderPid: holder.pid ?? 0, + release: async () => { + if (released) return; + released = true; + if (!holder.stdin.destroyed) holder.stdin.end(); + await new Promise((resolve) => holder.once("exit", () => resolve())); + }, + }; + } + + async beginJob(options: BeginPreprocessingJobOptions): Promise { + this.ensureLayout(); + if (!/^[0-9a-f]{40}$/.test(options.workspaceRevision)) { + throw new Error("workspace revision is invalid"); + } + const runId = options.runId ?? randomBytes(16).toString("hex"); + const path = this.jobPath(runId); + try { + const existing = this.readJob(runId); + if ( + existing.operation !== options.operation + || existing.workspaceRevision !== options.workspaceRevision + || existing.descriptorBlob !== options.descriptorBlob + || existing.catalogBlob !== options.catalogBlob + || existing.configDigest !== options.configDigest + || existing.bindingDigest !== options.bindingDigest + ) { + throw new PreprocessingStateError( + "preprocessing_resume_mismatch", + "Workspace preprocessing resume no longer matches the pinned revision", + ); + } + return existing; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + if (error instanceof PreprocessingStateError) throw error; + throw error; + } + if (options.runId) { + throw new PreprocessingStateError( + "preprocessing_resume_mismatch", + "Workspace preprocessing run is unavailable", + ); + } + } + const job: PreprocessingJobState = { + schemaVersion: 1, + runId, + operation: options.operation, + workspaceId: this.options.workspaceId, + workspaceRevision: options.workspaceRevision, + descriptorBlob: options.descriptorBlob, + catalogBlob: options.catalogBlob, + configDigest: options.configDigest, + bindingDigest: options.bindingDigest, + completedStages: [], + childRuns: {}, + status: "active", + }; + writeAtomicFile(path, `${JSON.stringify(job)} +`, 0o600); + return job; + } + + readJob(runId: string): PreprocessingJobState { + return decodeJob(JSON.parse(readTrustedFile(this.jobPath(runId)))); + } + + writeJob(job: PreprocessingJobState): PreprocessingJobState { + this.ensureLayout(); + writeAtomicFile(this.jobPath(job.runId), `${JSON.stringify(job)} +`, 0o600); + return job; + } + + writeFkCandidates(runId: string, yaml: string): { path: string; digest: string } { + this.ensureLayout(); + const path = this.fkCandidatesPath(runId); + writeAtomicFile(path, yaml, 0o600); + return { path, digest: sha256(yaml) }; + } + + readFkCandidates(runId: string): { path: string; yaml: string; digest: string } | undefined { + const path = this.fkCandidatesPath(runId); + try { + const yaml = readTrustedFile(path); + return { path, yaml, digest: sha256(yaml) }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + } + + writeFkReview(runId: string, review: FkReviewRecord): { path: string; digest: string } { + this.ensureLayout(); + const path = this.fkReviewPath(runId); + const json = `${JSON.stringify(review)} +`; + writeAtomicFile(path, json, 0o600); + return { path, digest: sha256(json) }; + } + + readFkReview(runId: string): FkReviewRecord | undefined { + try { + return decodeReview(JSON.parse(readTrustedFile(this.fkReviewPath(runId)))); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + } + + async assertSessionInventoryCompatible( + workspaceRevision: string, + sessions: readonly SessionInventoryRow[], + ): Promise { + const conflicting = sessions.find((session) => ( + session.workspaceRevision + && session.workspaceRevision !== workspaceRevision + && session.status !== "finalized" + && !session.archived + )); + if (conflicting) { + throw new PreprocessingStateError( + "preprocessing_conflict", + "A resumable session is pinned to a different workspace revision", + ); + } + } + + private static readonly HOLDER_PROGRAM = [ + "import fcntl, os, sys", + "fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT | getattr(os, 'O_NOFOLLOW', 0), 0o600)", + "try:", + " fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)", + "except BlockingIOError:", + " sys.exit(73)", + "sys.stdout.write('locked\\n')", + "sys.stdout.flush()", + "sys.stdin.buffer.read()", + ].join("\n"); +} diff --git a/backend/src/workspaces/qdrant-collection.ts b/backend/src/workspaces/qdrant-collection.ts new file mode 100644 index 00000000..a934b423 --- /dev/null +++ b/backend/src/workspaces/qdrant-collection.ts @@ -0,0 +1,105 @@ +export const QDRANT_REQUIRED_INDEXES = Object.freeze([ + "content_hash", "document_id", "kind", "record_key", + "record_kind", "vector_generation", "workspace_id", "workspace_revision", +]); + +export type CollectionMode = "self_heal" | "require_existing"; + +export interface CollectionCheck { + ok: boolean; + code?: "semantic_index_incompatible" | "workspace_not_activatable"; + state?: "ready" | "created" | "repaired"; +} + +export interface ReconcileCollectionOptions { + baseUrl: string; + collection: string; + dimensions: number; + distance: string; + mode: CollectionMode; + request?: typeof fetch; + signal?: AbortSignal; +} + +function qdrantDistance(distance: string): string { + return distance.length === 0 ? distance : distance.charAt(0).toUpperCase() + distance.slice(1); +} + +function qdrantUrl(baseUrl: string, path: string): string { + return new URL(path, baseUrl).toString(); +} + +async function collectionInfo(opts: ReconcileCollectionOptions, request: typeof fetch): Promise { + const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}`), { method: "GET", signal: opts.signal }); + if (res.status === 404) return undefined; + if (!res.ok) throw new Error("qdrant collection check failed"); + return (await res.json() as any)?.result; +} + +function vectorCompatibility(info: any, opts: ReconcileCollectionOptions): boolean { + const vectors = info?.config?.params?.vectors; + return Boolean(vectors && vectors.size === opts.dimensions && typeof vectors.distance === "string" + && vectors.distance.toLowerCase() === opts.distance); +} + +async function missingIndexes(opts: ReconcileCollectionOptions, info: any): Promise { + const payloadSchema = info?.payload_schema; + if (!payloadSchema || typeof payloadSchema !== "object") return [...QDRANT_REQUIRED_INDEXES]; + return QDRANT_REQUIRED_INDEXES.filter((field) => payloadSchema[field]?.data_type !== "keyword"); +} + +async function createCollection(opts: ReconcileCollectionOptions, request: typeof fetch): Promise { + const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}`), { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ vectors: { size: opts.dimensions, distance: qdrantDistance(opts.distance) } }), + signal: opts.signal, + }); + if (!res.ok && res.status !== 409) throw new Error("qdrant collection creation failed"); +} + +async function createIndex(opts: ReconcileCollectionOptions, field: string, request: typeof fetch): Promise { + const res = await request(qdrantUrl(opts.baseUrl, `/collections/${encodeURIComponent(opts.collection)}/index`), { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ field_name: field, field_schema: "keyword" }), + signal: opts.signal, + }); + if (!res.ok && res.status !== 409) throw new Error("qdrant index creation failed"); +} + +/** Reconcile a Qdrant collection: self-heal creates missing collections/indexes; require_existing + * only validates and refuses incompatible contracts (never mutates). */ +export async function reconcileCollection(opts: ReconcileCollectionOptions): Promise { + const request = opts.request ?? fetch; + let info = await collectionInfo(opts, request); + if (info === undefined) { + if (opts.mode !== "self_heal") return { ok: false, code: "semantic_index_incompatible" }; + await createCollection(opts, request); + // Tolerate an already-compatible concurrent creator: re-read the final state. + info = await collectionInfo(opts, request); + if (info === undefined) return { ok: false, code: "workspace_not_activatable" }; + } + if (!vectorCompatibility(info, opts)) { + return { ok: false, code: "semantic_index_incompatible" }; + } + const missing = await missingIndexes(opts, info); + if (missing.length > 0) { + if (opts.mode !== "self_heal") return { ok: false, code: "semantic_index_incompatible" }; + for (const field of missing) await createIndex(opts, field, request); + // Qdrant payload indexes become visible asynchronously: poll until the + // contract is complete or a bounded deadline passes (fail closed). + const deadline = Date.now() + 15000; + let current: any = info; + while (Date.now() < deadline) { + current = await collectionInfo(opts, request); + if (!vectorCompatibility(current, opts)) break; + if ((await missingIndexes(opts, current)).length === 0) { + return { ok: true, state: "repaired" }; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + return { ok: false, code: "semantic_index_incompatible" }; + } + return { ok: true, state: "ready" }; +} diff --git a/backend/src/workspaces/registry.ts b/backend/src/workspaces/registry.ts new file mode 100644 index 00000000..e69b4e87 --- /dev/null +++ b/backend/src/workspaces/registry.ts @@ -0,0 +1,914 @@ +import { createHash, randomUUID } from "node:crypto"; +import { lstatSync, readdirSync, readFileSync } from "node:fs"; +import { mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { isAbsolute, join } from "node:path"; +import { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js"; +import { parseAnnotationsYaml } from "./annotations.js"; +import { syncAnnotations } from "./annotations-sync.js"; +import { materializeEvidenceTree } from "./evidence-materialization.js"; +import { assertCatalogMatchesDescriptor, parseWorkspaceCatalogYaml, type WorkspaceCatalog, type WorkspaceCatalogEntry } from "./catalog.js"; +import { + GitWorkspaceRepository, + WorkspaceRegistryError, + WorkspaceRepositoryLock, + normalizeRepositoryIdentity, + type GitStatus, +} from "./git-repository.js"; +import { + parseWorkspaceYaml, + serializeWorkspaceYaml, + validateOperationalWorkspace, + type WorkspaceDescriptor, +} from "./schema.js"; +import type { WorkspaceErrorCode, WorkspaceRegistryConfig } from "./types.js"; + +export type { GitStatus } from "./git-repository.js"; + +export interface WorkspaceRevision { + id: string; + commit: string; + blob: string; + snapshotPath: string; +} + +export interface StoredWorkspaceIntegrity { + state: "uninitialized" | "active"; + workspaces: number; + fingerprint: string; +} + +export interface SessionRevisionLease { + workspace: WorkspaceDescriptor; + revision: WorkspaceRevision; + /** Mark the manifest durable; retention removes the lease only after observing that manifest. */ + markPersisted(): Promise; + /** Remove a lease for a session that failed before its manifest was durable. */ + abort(): Promise; +} + +interface ActiveState { + head: string; + revisions: WorkspaceRevision[]; + catalog?: WorkspaceCatalog; +} + +interface SnapshotManifest extends ActiveState { + files: Record; +} + +interface RevisionLeaseRecord { + version: 1; + token: string; + workspaceId: string; + commit: string; + state: "creating" | "persisted"; +} + +function workspacePath(id: string): string { + if (!/^[a-z][a-z0-9-]{2,62}$/.test(id)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace ID is invalid"); + } + return `${id}/workspace.yaml`; +} + +function safeCommit(commit: string): string { + if (!/^[0-9a-f]{40}$/.test(commit)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace revision is invalid"); + } + return commit; +} + +function safeBlob(blob: string): string { + if (!/^[0-9a-f]{40}$/.test(blob)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace snapshot blob is invalid"); + } + return blob; +} + +function digest(contents: string | Buffer): string { + return createHash("sha256").update(contents).digest("hex"); +} + +function workspaceError(error: unknown): WorkspaceRegistryError { + if (error instanceof WorkspaceRegistryError) return error; + return new WorkspaceRegistryError("workspace_invalid", "Workspace repository content is invalid"); +} + +/** Immutable canonical workspace snapshots backed by the configured Git checkout. */ +export class WorkspaceRegistry { + private readonly repository: GitWorkspaceRepository; + private readonly lock: WorkspaceRepositoryLock; + + constructor(private readonly config: WorkspaceRegistryConfig) { + this.repository = new GitWorkspaceRepository(config); + this.lock = new WorkspaceRepositoryLock(this.repository.locksPath); + } + + snapshotPath(commit: string, id: string): string { + return join(this.repository.snapshotsPath, safeCommit(commit), `${id}.yaml`); + } + + async bootstrap(): Promise { + await this.repository.ensureLayout(); + return await this.lock.run(async () => { + try { + const status = await this.repository.bootstrap(); + await this.activate(status.head!); + return status; + } catch (error) { + return await this.gitFallback(error); + } + }); + } + + async pull(): Promise { + await this.repository.ensureLayout(); + return await this.lock.run(async () => { + try { + const status = await this.repository.pull(); + await this.activate(status.head!); + return status; + } catch (error) { + return await this.gitFallback(error); + } + }); + } + + async listCatalog(): Promise> { + const active = await this.tryActiveState(); + if (!active) { + await this.bootstrap(); + return await this.listCatalog(); + } + const catalog = active.catalog ?? { schema_version: 1 as const, workspaces: [] }; + return catalog.workspaces.map((entry) => ({ + ...entry, + configurationState: "ready" as const, + revision: active.revisions.find((revision) => revision.id === entry.id)!, + })); + } + + async list(): Promise { + const active = await this.tryActiveState(); + if (active) return active.revisions; + // A clean installation has no active snapshot until the first registry operation. Keep + // this lazy so health/startup remain available when Git is temporarily unreachable, while + // still refusing corrupted existing state (tryActiveState throws instead of returning none). + await this.bootstrap(); + return (await this.activeState()).revisions; + } + + /** + * List every intact retained snapshot, current snapshots first. Session discovery and + * retention use this rather than only the active revision so removing a workspace from + * Git cannot strand a resumable session that still pins one of its older descriptors. + */ + async listRetainedSnapshots(): Promise { + await this.repository.ensureLayout(); + return await this.lock.run(async () => { + try { + const active = await this.activeState(); + const revisions = [...active.revisions]; + const entries = await readdir(this.repository.snapshotsPath, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory() || entry.isSymbolicLink() || !/^[0-9a-f]{40}$/.test(entry.name)) continue; + if (entry.name === active.head) continue; + const state = await this.snapshotState(entry.name); + revisions.push(...state.revisions); + } + return revisions; + } catch (error) { + throw workspaceError(error); + } + }); + } + + /** + * Validate only persisted local registry state. Restore uses this path while the installation + * is stopped: it must neither contact Git nor turn a never-used registry into initialized state. + * The only never-initialized shape is an existing empty root. An initialized root is exactly + * repo/, snapshots/, state/, and locks/: locks contains repository.lock plus an empty + * empty-hooks/, state contains active.json plus an optional empty revision-leases/, and + * snapshots contains exact immutable commit snapshots plus an optional empty runtime/. + * Descendants may contain only ordinary directories and regular files; links and special files + * are rejected by the stable whole-tree fingerprint before any shape is accepted. + */ + async verifyStoredState(): Promise { + try { + const before = await this.storedStateFingerprint(); + const rootEntries = await readdir(this.repository.root, { withFileTypes: true }); + if (rootEntries.length === 0) { + const after = await this.storedStateFingerprint(); + if (after !== before) throw new Error("workspace registry changed during inspection"); + return { state: "uninitialized", workspaces: 0, fingerprint: `sha256:${before}` }; + } + this.assertExactDirectoryEntries(rootEntries, { + locks: "directory", repo: "directory", snapshots: "directory", state: "directory", + }); + + this.assertExactDirectoryEntries( + await readdir(this.repository.locksPath, { withFileTypes: true }), + { "empty-hooks": "directory", "repository.lock": "file" }, + ); + this.assertExactDirectoryEntries( + await readdir(join(this.repository.locksPath, "empty-hooks"), { withFileTypes: true }), + {}, + ); + + const stateEntries = await readdir(this.repository.statePath, { withFileTypes: true }); + const stateShape: Record = { "active.json": "file" }; + if (stateEntries.some((entry) => entry.name === "revision-leases")) { + stateShape["revision-leases"] = "directory"; + } + this.assertExactDirectoryEntries(stateEntries, stateShape); + if (stateShape["revision-leases"] !== undefined) { + this.assertExactDirectoryEntries( + await readdir(join(this.repository.statePath, "revision-leases"), { withFileTypes: true }), + {}, + ); + } + + const active = this.decodeActiveState(JSON.parse(await readFile( + join(this.repository.statePath, "active.json"), "utf8", + ))); + const snapshotEntries = await readdir(this.repository.snapshotsPath, { withFileTypes: true }); + if (snapshotEntries.length === 0) throw new Error("workspace snapshots are unavailable"); + let activeSnapshotFound = false; + for (const entry of snapshotEntries) { + if (entry.name === "runtime") { + if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("workspace runtime path is invalid"); + this.assertExactDirectoryEntries( + await readdir(join(this.repository.snapshotsPath, "runtime"), { withFileTypes: true }), + {}, + ); + continue; + } + if (!/^[0-9a-f]{40}$/.test(entry.name) || !entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error("workspace snapshot path is invalid"); + } + const state = entry.name === active.head ? active : await this.readStoredSnapshotState(entry.name); + await this.assertSnapshotIntegrity(state, false, true); + if (entry.name === active.head) activeSnapshotFound = true; + } + if (!activeSnapshotFound) throw new Error("active workspace snapshot is unavailable"); + + const after = await this.storedStateFingerprint(); + if (after !== before) throw new Error("workspace registry changed during inspection"); + return { state: "active", workspaces: active.revisions.length, fingerprint: `sha256:${before}` }; + } catch (error) { + throw workspaceError(error); + } + } + + private assertExactDirectoryEntries( + entries: Array<{ name: string; isFile(): boolean; isDirectory(): boolean; isSymbolicLink(): boolean }>, + expected: Record, + ): void { + if (entries.length !== Object.keys(expected).length) throw new Error("workspace registry shape is invalid"); + for (const entry of entries) { + const kind = expected[entry.name]; + if (kind === undefined || entry.isSymbolicLink() + || (kind === "file" && !entry.isFile()) + || (kind === "directory" && !entry.isDirectory())) { + throw new Error("workspace registry shape is invalid"); + } + } + } + + private async storedStateFingerprint(): Promise { + const records: string[] = []; + let entries = 0; + let totalBytes = 0; + const visit = async (path: string, relative: string): Promise => { + const before = lstatSync(path); + if (before.isSymbolicLink()) throw new Error("workspace registry link is invalid"); + const metadata = [ + before.dev, before.ino, before.mode, before.uid, before.gid, + before.size, before.mtimeMs, before.ctimeMs, + ].join(":"); + entries += 1; + if (entries > 65_536) throw new Error("workspace registry contains too many entries"); + if (before.isDirectory()) { + records.push(`directory:${relative}:${metadata}`); + const children = await readdir(path); + children.sort(); + for (const name of children) { + await visit(join(path, name), relative === "." ? name : `${relative}/${name}`); + } + } else if (before.isFile()) { + if (before.size > 256 * 1024 * 1024) throw new Error("workspace registry file is too large"); + totalBytes += before.size; + if (totalBytes > 2 * 1024 * 1024 * 1024) throw new Error("workspace registry is too large"); + const contents = await readFile(path); + records.push(`file:${relative}:${metadata}:${contents.length}:${digest(contents)}`); + } else { + throw new Error("workspace registry entry is invalid"); + } + const after = lstatSync(path); + if (before.dev !== after.dev || before.ino !== after.ino || before.mode !== after.mode + || before.uid !== after.uid || before.gid !== after.gid || before.size !== after.size + || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) { + throw new Error("workspace registry changed during inspection"); + } + }; + await visit(this.repository.root, "."); + return digest(records.join("\n")); + } + + private async readStoredSnapshotState(head: string): Promise { + return this.decodeSnapshotManifest(await this.readSnapshotManifest(safeCommit(head))); + } + + async read(id: string): Promise<{ workspace: WorkspaceDescriptor; revision: WorkspaceRevision }> { + const state = await this.activeState(); + const revision = state.revisions.find((candidate) => candidate.id === id); + if (!revision) throw new WorkspaceRegistryError("workspace_invalid", "Workspace is unavailable"); + try { + const source = await readFile(revision.snapshotPath, "utf8"); + return { workspace: parseWorkspaceYaml(source), revision }; + } catch (error) { + throw workspaceError(error); + } + } + + /** + * Resolve the active revision and create its cross-process retention lease under the same + * repository lock. The lease bridges the interval before `session_manifest.yaml` is durable. + */ + async acquireSessionRevision(id: string): Promise { + await this.repository.ensureLayout(); + return await this.lock.run(async () => { + const state = await this.activeState(); + const revision = state.revisions.find((candidate) => candidate.id === id); + if (!revision) throw new WorkspaceRegistryError("workspace_invalid", "Workspace is unavailable"); + let workspace: WorkspaceDescriptor; + try { + workspace = validateOperationalWorkspace( + parseWorkspaceYaml(await readFile(revision.snapshotPath, "utf8")), + ); + } catch (error) { + throw workspaceError(error); + } + + const token = randomUUID(); + const record: RevisionLeaseRecord = { + version: 1, + token, + workspaceId: id, + commit: revision.commit, + state: "creating", + }; + const path = await this.writeRevisionLease(record, true); + let localState: RevisionLeaseRecord["state"] | "aborted" = "creating"; + + return { + workspace, + revision, + markPersisted: async () => { + if (localState === "persisted") return; + if (localState === "aborted") throw new WorkspaceRegistryError( + "workspace_invalid", "Workspace revision lease is unavailable", + ); + await this.lock.run(async () => { + await this.replaceRevisionLease(path, { ...record, state: "persisted" }); + }); + localState = "persisted"; + }, + abort: async () => { + if (localState !== "creating") return; + await this.lock.run(async () => { await rm(path, { force: true }); }); + localState = "aborted"; + }, + }; + }); + } + + /** Read a retained immutable snapshot for a session pinned to a historical commit. */ + async readPinned(id: string, commit: string): Promise<{ workspace: WorkspaceDescriptor; workspaceConfigPath: string }> { + const snapshotPath = this.snapshotPath(safeCommit(commit), id); + try { + const source = await readFile(snapshotPath, "utf8"); + return { + workspace: validateOperationalWorkspace(parseWorkspaceYaml(source)), + workspaceConfigPath: snapshotPath, + }; + } catch (error) { + throw workspaceError(error); + } + } + + /** + * Garbage-collect obsolete immutable snapshots without breaking cold Resume. + * Callers must supply revisions collected from an administrator-visible complete session list; + * a partial, per-user list could otherwise remove another user's resumable workspace pin. + */ + async reconcileSnapshotRetention(referencedCommits: readonly string[]): Promise { + const manifestReferences = new Set(referencedCommits.map(safeCommit)); + const retained = new Set(manifestReferences); + await this.repository.ensureLayout(); + await this.lock.run(async () => { + const leases = await this.revisionLeases(); + for (const { record } of leases) retained.add(record.commit); + retained.add((await this.activeState()).head); + const entries = await readdir(this.repository.snapshotsPath, { withFileTypes: true }); + for (const entry of entries) { + // Leave staging and unexpected entries untouched: this cleanup only owns finalized, + // commit-addressed snapshot directories. + if (!entry.isDirectory() || entry.isSymbolicLink() || !/^[0-9a-f]{40}$/.test(entry.name)) continue; + if (retained.has(entry.name)) continue; + const path = join(this.repository.snapshotsPath, entry.name); + const current = lstatSync(path); + if (!current.isDirectory() || current.isSymbolicLink()) continue; + await rm(path, { recursive: true, force: true }); + } + // A persisted lease is handed off only when this exact authoritative scan has observed a + // manifest pin for its commit. A stale scan therefore keeps the lease and cannot prune it. + for (const { path, record } of leases) { + if (record.state === "persisted" && manifestReferences.has(record.commit)) { + await rm(path, { force: true }); + } + } + }); + } + + private revisionLeaseDirectory(): string { + return join(this.repository.statePath, "revision-leases"); + } + + private async writeRevisionLease(record: RevisionLeaseRecord, exclusive: boolean): Promise { + const directory = this.revisionLeaseDirectory(); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const path = join(directory, `${record.token}.json`); + await writeFile(path, JSON.stringify(record), { + encoding: "utf8", + mode: 0o600, + flush: true, + ...(exclusive ? { flag: "wx" } : {}), + }); + return path; + } + + private async replaceRevisionLease(path: string, record: RevisionLeaseRecord): Promise { + const staging = `${path}.staging-${randomUUID()}`; + try { + await writeFile(staging, JSON.stringify(record), { + encoding: "utf8", mode: 0o600, flag: "wx", flush: true, + }); + await rename(staging, path); + } catch (error) { + await rm(staging, { force: true }); + throw error; + } + } + + private async revisionLeases(): Promise> { + const directory = this.revisionLeaseDirectory(); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const entries = await readdir(directory, { withFileTypes: true }); + const leases: Array<{ path: string; record: RevisionLeaseRecord }> = []; + for (const entry of entries) { + if (!entry.isFile() || entry.isSymbolicLink() || !/^[0-9a-f-]{36}\.json$/.test(entry.name)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace revision lease is invalid"); + } + const path = join(directory, entry.name); + let record: RevisionLeaseRecord; + try { + record = JSON.parse(await readFile(path, "utf8")) as RevisionLeaseRecord; + } catch { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace revision lease is invalid"); + } + if ( + record.version !== 1 + || `${record.token}.json` !== entry.name + || !/^[0-9a-f-]{36}$/.test(record.token) + || !/^[a-z][a-z0-9-]{2,62}$/.test(record.workspaceId) + || !/^[0-9a-f]{40}$/.test(record.commit) + || (record.state !== "creating" && record.state !== "persisted") + ) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace revision lease is invalid"); + } + leases.push({ path, record }); + } + return leases; + } + + private async assertEvidenceContext(workspace: WorkspaceDescriptor, revision: string): Promise { + if (workspace.evidence?.source.type !== "filesystem") return; + // P6 owns recursive containment. Here we deliberately validate only the declared root object. + await this.repository.assertTreeAtRevision(revision, workspace.evidence.source.uri); + } + + private async assertSnapshotEvidenceContexts(state: ActiveState): Promise { + for (const revision of state.revisions) { + const workspace = parseWorkspaceYaml(await readFile(revision.snapshotPath, "utf8")); + await this.assertEvidenceContext(workspace, revision.commit); + } + } + + private async activate(commit: string): Promise { + const safeHead = safeCommit(commit); + const catalog = parseWorkspaceCatalogYaml(await this.repository.readCatalog(safeHead)); + const catalogById = new Map(catalog.workspaces.map((entry) => [entry.id, entry])); + for (const id of await this.repository.workspaceDirectories()) { + if (!catalogById.has(id)) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace directory is not listed in the catalog"); + } + } + const files = await this.repository.workspacePaths(); + const descriptorIds = new Set(files.map((path) => path.slice(0, -"/workspace.yaml".length))); + for (const id of catalogById.keys()) { + if (!descriptorIds.has(id)) { + throw new WorkspaceRegistryError( + "workspace_invalid", + "Every catalog workspace must have a published descriptor", + ); + } + } + + const snapshots: Array<{ + id: string; + source: string; + workspace: WorkspaceDescriptor; + blob: string; + }> = []; + const collectionOwners = new Map(); + try { + for (const path of files) { + const id = path.slice(0, -"/workspace.yaml".length); + const entry = catalogById.get(id); + if (!entry) throw new WorkspaceRegistryError("workspace_invalid", "Workspace descriptor is not listed in the catalog"); + const source = await this.repository.readWorkspace(path, safeHead); + const workspace = parseWorkspaceYaml(source); + if (workspace.workspace.id !== id) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace ID does not match its repository path"); + } + assertCatalogMatchesDescriptor(entry, workspace); + await this.assertEvidenceContext(workspace, safeHead); + const annotations = await this.repository.annotationsObject(safeHead, id); + if (annotations !== undefined) { + parseAnnotationsYaml(annotations.contents.toString("utf8")); + if (this.config.dataRoot !== undefined) { + syncAnnotations({ + dataRoot: this.config.dataRoot, + workspaceId: id, + commit: safeHead, + blobId: annotations.blobId, + contents: annotations.contents, + }); + } + } + const collection = workspace.semantic_index.vector_store.collection; + const owner = collectionOwners.get(collection); + if (owner !== undefined) { + throw new Error(`duplicate qdrant collection ownership: ${collection} (${owner}, ${id})`); + } + collectionOwners.set(collection, id); + buildInstallationContract(workspace); + renderWorkspaceDocs(workspace); + snapshots.push({ + id, + source: serializeWorkspaceYaml(workspace), + workspace, + blob: await this.repository.blob(path, safeHead), + }); + } + } catch (error) { + throw workspaceError(error); + } + + const snapshotDirectory = join(this.repository.snapshotsPath, safeHead); + const revisions = snapshots.map((snapshot) => ({ + id: snapshot.id, + commit: safeHead, + blob: snapshot.blob, + snapshotPath: this.snapshotPath(safeHead, snapshot.id), + })); + if (this.pathExists(snapshotDirectory)) { + await this.assertSnapshotIntegrity({ head: safeHead, revisions, catalog }); + } else { + const staging = join(this.repository.snapshotsPath, `.staging-${randomUUID()}`); + await mkdir(staging, { mode: 0o700 }); + try { + const files: Record = {}; + for (const snapshot of snapshots) { + const yamlName = `${snapshot.id}.yaml`; + const envName = `${snapshot.id}.env.example`; + const docsName = `${snapshot.id}.md`; + await writeFile(join(staging, yamlName), snapshot.source, { encoding: "utf8", mode: 0o400 }); + files[yamlName] = digest(snapshot.source); + const docs = renderWorkspaceDocs(snapshot.workspace); + await writeFile(join(staging, envName), docs.envExample, { encoding: "utf8", mode: 0o400 }); + await writeFile(join(staging, docsName), docs.markdown, { encoding: "utf8", mode: 0o400 }); + files[envName] = digest(docs.envExample); + files[docsName] = digest(docs.markdown); + if (snapshot.workspace.evidence?.source.type === "filesystem") { + const materialized = await materializeEvidenceTree({ + repository: this.repository, + revision: safeHead, + id: snapshot.id, + targetDirectory: join(staging, snapshot.id), + limits: this.evidenceMaterializationLimits(), + }); + files[`${snapshot.id}/evidence.manifest.json`] = materialized.manifestDigest; + } + } + await writeFile(join(staging, "snapshot.json"), JSON.stringify({ head: safeHead, revisions, catalog, files }), { + encoding: "utf8", mode: 0o400, + }); + await rename(staging, snapshotDirectory); + } catch (error) { + await rm(staging, { recursive: true, force: true }); + throw error; + } + } + + await this.writeActiveState({ head: safeHead, revisions, catalog }); + } + + private async gitFallback(error: unknown): Promise { + const safeError = workspaceError(error); + if (safeError.code !== "git_unavailable" && safeError.code !== "git_auth_failed") throw safeError; + const active = await this.tryActiveState(); + if (!active) throw safeError; + return { + branch: this.config.branch, + ...(this.config.remoteUrl + ? { repository: normalizeRepositoryIdentity(this.config.remoteUrl) } + : {}), + head: active.head, + ahead: 0, + behind: 0, + degraded: true, + lastError: safeError.code, + }; + } + + private async activeState(): Promise { + const active = await this.tryActiveState(); + if (!active) throw new WorkspaceRegistryError("workspace_invalid", "No active workspace snapshot is available"); + return active; + } + + private async tryActiveState(): Promise { + const file = join(this.repository.statePath, "active.json"); + try { + const state = this.decodeActiveState(JSON.parse(await readFile(file, "utf8"))); + await this.assertSnapshotIntegrity(state); + return state; + } catch (error) { + if (this.pathIsMissing(file)) return undefined; + if (error instanceof WorkspaceRegistryError) throw error; + throw new WorkspaceRegistryError("workspace_invalid", "Workspace active snapshot is invalid"); + } + } + + private async writeActiveState(state: ActiveState): Promise { + const target = join(this.repository.statePath, "active.json"); + const staging = join(this.repository.statePath, `.active-${randomUUID()}.json`); + await writeFile(staging, JSON.stringify(state), { encoding: "utf8", mode: 0o600 }); + await rename(staging, target); + } + + private decodeActiveState(value: unknown): ActiveState { + const record = this.optionalKeyObject(value, ["head", "revisions"], ["catalog"]); + const state = this.decodeStateRevisions(record.head, record.revisions); + return record.catalog === undefined ? state : { ...state, catalog: this.decodeCatalog(record.catalog) }; + } + + private decodeSnapshotManifest(value: unknown): SnapshotManifest { + const manifest = this.optionalKeyObject(value, ["head", "revisions", "files"], ["catalog"]); + const state = this.decodeStateRevisions(manifest.head, manifest.revisions); + const catalog = manifest.catalog === undefined ? undefined : this.decodeCatalog(manifest.catalog); + if (!manifest.files || typeof manifest.files !== "object" || Array.isArray(manifest.files)) { + throw new Error("bad manifest files"); + } + const entries = Object.entries(manifest.files as Record); + if (entries.some(([, contentsDigest]) => typeof contentsDigest !== "string")) { + throw new Error("bad manifest files"); + } + return { ...state, ...(catalog ? { catalog } : {}), files: Object.fromEntries(entries) as Record }; + } + + private decodeStateRevisions(headValue: unknown, revisionsValue: unknown): ActiveState { + if (typeof headValue !== "string" || !Array.isArray(revisionsValue)) throw new Error("bad state"); + const head = safeCommit(headValue); + const ids = new Set(); + const revisions = revisionsValue.map((value) => { + const revision = this.decodeRevision(value, head); + if (ids.has(revision.id)) throw new Error("duplicate revision"); + ids.add(revision.id); + return revision; + }); + return { head, revisions }; + } + + private decodeRevision(value: unknown, head: string): WorkspaceRevision { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("bad revision"); + const revision = value as Record; + const keys = Object.keys(revision); + const required = ["id", "commit", "blob", "snapshotPath"]; + const hasHistoricalState = Object.prototype.hasOwnProperty.call(revision, "state"); + if ( + keys.length !== required.length + (hasHistoricalState ? 1 : 0) + || !required.every((key) => Object.prototype.hasOwnProperty.call(revision, key)) + || (hasHistoricalState && revision.state !== "operational") + ) { + throw new Error("bad revision"); + } + if ( + typeof revision.id !== "string" + || typeof revision.commit !== "string" + || typeof revision.blob !== "string" + || typeof revision.snapshotPath !== "string" + ) { + throw new Error("bad revision"); + } + const id = revision.id; + const commit = safeCommit(revision.commit); + const blob = safeBlob(revision.blob); + const snapshotPath = revision.snapshotPath; + workspacePath(id); + if ( + commit !== head + || !isAbsolute(snapshotPath) + || snapshotPath !== this.snapshotPath(commit, id) + ) { + throw new Error("bad revision"); + } + // Always reconstruct a fresh public revision. The sole accepted historical state field is + // compatibility input and must never cross the registry boundary. + return { id, commit, blob, snapshotPath }; + } + + private decodeCatalog(value: unknown): WorkspaceCatalog { + if (typeof value !== "object" || value === null) throw new Error("bad catalog"); + return parseWorkspaceCatalogYaml(JSON.stringify(value)); + } + + private optionalKeyObject( + value: unknown, + requiredKeys: readonly string[], + optionalKeys: readonly string[], + ): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("bad state"); + const record = value as Record; + const allowed = new Set([...requiredKeys, ...optionalKeys]); + const keys = Object.keys(record); + if ( + keys.length !== requiredKeys.length + optionalKeys.length + || !requiredKeys.every((key) => Object.prototype.hasOwnProperty.call(record, key)) + || !keys.every((key) => allowed.has(key)) + ) { + throw new Error("bad state"); + } + return record; + } + + private strictObject(value: unknown, expectedKeys: readonly string[]): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("bad state"); + const record = value as Record; + const keys = Object.keys(record); + if ( + keys.length !== expectedKeys.length + || !expectedKeys.every((key) => Object.prototype.hasOwnProperty.call(record, key)) + ) { + throw new Error("bad state"); + } + return record; + } + + private async readSnapshotManifest(head: string): Promise { + const path = join(this.repository.snapshotsPath, head, "snapshot.json"); + return JSON.parse(await readFile(path, "utf8")); + } + + private async snapshotState(head: string): Promise { + const state = this.decodeSnapshotManifest(await this.readSnapshotManifest(safeCommit(head))); + await this.assertSnapshotIntegrity(state); + return state; + } + + private async assertSnapshotIntegrity( + state: ActiveState, + verifyGitEvidence = true, + exactStoredShape = false, + ): Promise { + const directory = join(this.repository.snapshotsPath, state.head); + try { + const manifest = this.decodeSnapshotManifest(await this.readSnapshotManifest(state.head)); + if (manifest.head !== state.head || !this.sameRevisions(manifest.revisions, state.revisions) + || JSON.stringify(manifest.catalog ?? null) !== JSON.stringify(state.catalog ?? null)) { + throw new Error("manifest state does not match active state"); + } + await this.assertManifestFiles(directory, manifest.files, this.expectedSnapshotFiles(state, directory)); + if (exactStoredShape) this.assertStoredSnapshotShape(directory, state); + if (verifyGitEvidence) await this.assertSnapshotEvidenceContexts(state); + } catch (error) { + if (error instanceof WorkspaceRegistryError) throw error; + throw new WorkspaceRegistryError("workspace_invalid", "Workspace snapshot integrity check failed"); + } + } + + private assertStoredSnapshotShape(directory: string, state: ActiveState): void { + const expected: Record = { "snapshot.json": "file" }; + for (const revision of state.revisions) { + expected[`${revision.id}.yaml`] = "file"; + expected[`${revision.id}.env.example`] = "file"; + expected[`${revision.id}.md`] = "file"; + const workspace = parseWorkspaceYaml(readFileSync(join(directory, `${revision.id}.yaml`), "utf8")); + if (workspace.evidence?.source.type === "filesystem") expected[revision.id] = "directory"; + } + this.assertExactDirectoryEntries(readdirSync(directory, { withFileTypes: true }), expected); + for (const revision of state.revisions) { + if (expected[revision.id] !== "directory") continue; + this.assertExactDirectoryEntries( + readdirSync(join(directory, revision.id), { withFileTypes: true }), + { evidence: "directory", "evidence.manifest.json": "file" }, + ); + } + } + + private expectedSnapshotFiles(state: ActiveState, directory: string): string[] { + return state.revisions.flatMap((revision) => { + const names = [`${revision.id}.yaml`, `${revision.id}.env.example`, `${revision.id}.md`]; + const workspace = parseWorkspaceYaml(readFileSync(join(directory, `${revision.id}.yaml`), "utf8")); + if (workspace.evidence?.source.type === "filesystem") { + names.push(`${revision.id}/evidence.manifest.json`); + } + return names; + }); + } + + private evidenceMaterializationLimits(): Partial<{ + maxEntries: number; + maxTotalBytes: number; + maxFileBytes: number; + maxPathBytes: number; + maxManifestBytes: number; + }> { + return { + ...(this.config.maxEvidenceEntries === undefined ? {} : { maxEntries: this.config.maxEvidenceEntries }), + ...(this.config.maxEvidenceBytes === undefined ? {} : { maxTotalBytes: this.config.maxEvidenceBytes }), + ...(this.config.maxEvidenceFileBytes === undefined ? {} : { maxFileBytes: this.config.maxEvidenceFileBytes }), + ...(this.config.maxEvidencePathBytes === undefined ? {} : { maxPathBytes: this.config.maxEvidencePathBytes }), + ...(this.config.maxEvidenceManifestBytes === undefined ? {} : { maxManifestBytes: this.config.maxEvidenceManifestBytes }), + }; + } + + private async assertManifestFiles( + directory: string, + files: Record, + expected: string[], + ): Promise { + if (!files || typeof files !== "object" || Object.keys(files).length !== expected.length || !expected.every((name) => ( + /^[0-9a-f]{64}$/.test(files[name] ?? "") + ))) throw new Error("manifest files are invalid"); + for (const name of expected) { + const path = join(directory, name); + const entry = lstatSync(path); + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("snapshot file is invalid"); + const contents = await readFile(path); + if (digest(contents) !== files[name]) throw new Error("snapshot file does not match manifest"); + if (name.endsWith(".yaml")) { + const workspace = parseWorkspaceYaml(contents.toString("utf8")); + if (workspace.workspace.id !== name.slice(0, -".yaml".length)) { + throw new Error("snapshot workspace is invalid"); + } + } + } + } + + private sameRevisions(left: WorkspaceRevision[], right: WorkspaceRevision[]): boolean { + return left.length === right.length && left.every((revision, index) => { + const candidate = right[index]; + return candidate !== undefined + && candidate.id === revision.id && candidate.commit === revision.commit + && candidate.blob === revision.blob && candidate.snapshotPath === revision.snapshotPath; + }); + } + + private pathExists(path: string): boolean { + try { + const entry = lstatSync(path); + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace snapshot path is invalid"); + } + return true; + } catch (error) { + if (error instanceof WorkspaceRegistryError) throw error; + return false; + } + } + + private pathIsMissing(path: string): boolean { + try { + lstatSync(path); + return false; + } catch { + return true; + } + } +} diff --git a/backend/src/workspaces/runtime-config-lease.ts b/backend/src/workspaces/runtime-config-lease.ts new file mode 100644 index 00000000..912fb872 --- /dev/null +++ b/backend/src/workspaces/runtime-config-lease.ts @@ -0,0 +1,611 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + closeSync, + constants as fsConstants, + fchmodSync, + fstatSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readSync, + renameSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { readFile as readFileAsync } from "node:fs/promises"; +import { parse, parseAllDocuments, stringify } from "yaml"; +import { + buildCanonicalEffectiveConfig, + canonicalEffectiveConfigJson, + configFingerprint, + effectiveConfigIdentity, + inputFingerprint, + type CanonicalEffectiveConfig, +} from "./effective-config.js"; +import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js"; +import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./secret-requirements.js"; +import type { WorkspaceSecretStore } from "./secret-store.js"; +import { GitWorkspaceRepository } from "./git-repository.js"; +import { WorkspaceRegistry } from "./registry.js"; +import { + renderRuntimeConfig, + type RuntimeIdentity, + type RuntimeInstallationOverlay, + type RuntimePaths, + type RuntimeRenderContext, + type SemanticRuntimeConfig, +} from "./runtime-renderer.js"; +import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js"; +import type { WorkspaceRegistryConfig } from "./types.js"; + +export interface RuntimeConfigLease { + path: string; + workspaceId: string; + workspaceRevision: string; + release(): void; +} + +export interface RenderedWorkspaceRuntime { + workspace: WorkspaceDescriptor; + workspaceId: string; + workspaceRevision: string; + revisionContentRoot: string; + runtimePaths: RuntimePaths; + installationOverlay: RuntimeInstallationOverlay; + bindings: RuntimeBindings; + bindingDigest: string; + renderedConfig: string; + semanticQdrantUrl: string; + releaseSecrets(): void; +} + +export interface ActiveRenderedWorkspaceRuntime extends RenderedWorkspaceRuntime { + snapshotPath: string; + descriptorBlob: string; + catalogBlob: string; +} + +export interface DeterministicRuntimeConfigLease extends RuntimeConfigLease { + manifestPath: string; + descriptorBlob: string; + catalogBlob: string; + configDigest: string; + bindingDigest: string; + semanticQdrantUrl: string; + effectiveConfig: CanonicalEffectiveConfig; + effectiveConfigIdentity: string; + configFingerprint: string; + inputFingerprint: string; +} + +export class RuntimeConfigLeaseError extends Error { + constructor(readonly code: "effective_config_mismatch", message: string) { + super(message); + this.name = "RuntimeConfigLeaseError"; + } +} + +interface SnapshotIdentity extends RuntimeIdentity { + snapshotPath: string; +} + +interface PublishedRuntimeConfigManifest { + schemaVersion: 1; + workspaceId: string; + workspaceRevision: string; + descriptorBlob: string; + catalogBlob: string; + configDigest: string; + bindingDigest: string; + effectiveConfigIdentity: string; + configFingerprint: string; + inputFingerprint: string; + path: string; + file: { + dev: number; + ino: number; + size: number; + mode: number; + nlink: number; + }; +} + +function sha256(value: string | Buffer): string { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +function stableBindingDigest(bindings: RuntimeBindings): string { + const encodeRecord = (value: Record) => Object.entries(value).sort(([left], [right]) => ( + left.localeCompare(right) + )); + return sha256(JSON.stringify({ + dwh: { + transport: bindings.dwh.transport, + values: encodeRecord(bindings.dwh.values), + missing: [...bindings.dwh.missing].sort(), + }, + evidence: { + values: encodeRecord(bindings.evidence.values), + missing: [...bindings.evidence.missing].sort(), + }, + })); +} + +function syncDirectory(directory: string): void { + if (process.platform === "win32") return; + const fd = openSync(directory, "r"); + try { fsyncSync(fd); } finally { closeSync(fd); } +} + +function ensureTrustedDirectory(directory: string): string { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + const entry = lstatSync(directory); + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error("runtime configuration directory is unavailable"); + } + return directory; +} + +function writeAtomicFile(path: string, contents: string, mode: number): void { + ensureTrustedDirectory(dirname(path)); + const staging = `${path}.tmp-${process.pid}-${Date.now()}-${randomUUID()}`; + const fd = openSync( + staging, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, + 0o600, + ); + let closed = false; + try { + writeFileSync(fd, contents, "utf8"); + fsyncSync(fd); + fchmodSync(fd, mode); + closeSync(fd); + closed = true; + renameSync(staging, path); + syncDirectory(dirname(path)); + } catch (error) { + if (!closed) try { closeSync(fd); } catch { /* preserve original error */ } + try { unlinkSync(staging); } catch { /* best effort */ } + throw error; + } +} + +function readTrustedFile(path: string): { contents: string; stat: ReturnType } { + const entry = lstatSync(path); + if (!entry.isFile() || entry.isSymbolicLink()) { + throw new Error("runtime configuration file is untrusted"); + } + const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1) { + throw new Error("runtime configuration file is untrusted"); + } + const contents = readFileSync(fd, "utf8"); + const after = fstatSync(fd); + if ( + before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.nlink !== after.nlink + ) { + throw new Error("runtime configuration file changed while reading"); + } + return { contents, stat: statSync(path) }; + } finally { + closeSync(fd); + } +} + +function readStrictJson(path: string): unknown { + return JSON.parse(readTrustedFile(path).contents); +} + +function trustedSnapshotIdentity(snapshotPath: string): SnapshotIdentity { + if (!isAbsolute(snapshotPath)) throw new Error("config path is not a trusted runtime snapshot"); + const commitDirectory = dirname(snapshotPath); + const snapshotsRoot = dirname(commitDirectory); + const pathRelative = relative(snapshotsRoot, snapshotPath); + const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(pathRelative); + if (pathRelative.startsWith("..") || isAbsolute(pathRelative) || !match) { + throw new Error("config path is not a trusted runtime snapshot"); + } + const entry = lstatSync(snapshotPath); + if (!entry.isFile() || entry.isSymbolicLink()) { + throw new Error("config path is not a trusted runtime snapshot"); + } + return { snapshotPath, workspaceRevision: match[1], workspaceId: match[2] }; +} + +function readSnapshotWorkspace(snapshotPath: string): { + workspace: WorkspaceDescriptor; + identity: SnapshotIdentity; + revisionContentRoot: string; +} { + const identity = trustedSnapshotIdentity(snapshotPath); + const fd = openSync(snapshotPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const before = fstatSync(fd); + if (!before.isFile() || before.nlink !== 1) throw new Error("workspace snapshot is not a file"); + const source = readFileSync(fd, "utf8"); + const after = fstatSync(fd); + if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) { + throw new Error("workspace snapshot changed while reading"); + } + const workspace = validateOperationalWorkspace(parseWorkspaceYaml(source)); + if (workspace.workspace.id !== identity.workspaceId) { + throw new Error("workspace snapshot identity does not match its path"); + } + return { workspace, identity, revisionContentRoot: dirname(snapshotPath) }; + } finally { + closeSync(fd); + } +} + +function runtimePaths(dataRoot: string, workspaceId: string, workspaceRevision?: string): RuntimePaths { + if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root"); + const root = join(dataRoot, "sessions", workspaceId); + return { + sessions: join(root, "sessions"), + artifacts: join(root, "artifacts"), + indexes: join(root, "indexes"), + memory: join(root, "memory"), + ...(workspaceRevision === undefined + ? {} + : { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }), + }; +} + +function installationOverlay(harnessDir: string, configPath: string): RuntimeInstallationOverlay { + const path = isAbsolute(configPath) ? configPath : resolve(harnessDir, configPath); + try { + const documents = parseAllDocuments(readFileSync(path, "utf8"), { uniqueKeys: true }); + if (documents.length !== 1) throw new Error("installation config must contain one YAML document"); + const document = documents[0]; + if (document.errors.length > 0 || document.warnings.length > 0) { + throw new Error("installation config contains invalid YAML"); + } + const parsed = document.toJSON(); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("installation config must be a YAML mapping"); + } + const source = parsed as Record; + return { + ...(source.session_storage === undefined ? {} : { session_storage: source.session_storage }), + ...(source.profile === undefined ? {} : { profile: source.profile }), + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return {}; + throw error; + } +} + +function applyCollectionLifecycle(config: string, lifecycle: "require_existing"): string { + const parsed = parse(config) as Record; + if (!parsed.resources || typeof parsed.resources !== "object") { + throw new Error("runtime configuration is missing resources"); + } + parsed.resources = { ...parsed.resources }; + parsed.resources.vector = { ...(parsed.resources.vector ?? {}), collection_lifecycle: lifecycle }; + return stringify(parsed, { lineWidth: 0, sortMapEntries: false }); +} + +function renderWorkspaceRuntimeFromWorkspace(options: { + workspace: WorkspaceDescriptor; + workspaceId: string; + workspaceRevision: string; + revisionContentRoot: string; + harnessDir: string; + configPath: string; + dataRoot: string; + secretRoots: readonly string[]; + semanticRuntime: SemanticRuntimeConfig; + workspaceSecretStore?: WorkspaceSecretStore; +}): RenderedWorkspaceRuntime { + const secretLease = options.workspaceSecretStore === undefined + ? undefined + : resolveRuntimeBindingsWithWorkspaceSecrets( + options.workspace, + process.env, + options.secretRoots, + options.workspaceSecretStore, + ); + const bindings = secretLease?.bindings + ?? resolveRuntimeBindings(options.workspace, process.env, options.secretRoots); + const overlay = installationOverlay(options.harnessDir, options.configPath); + const context: RuntimeRenderContext = { + workspaceId: options.workspaceId, + workspaceRevision: options.workspaceRevision, + revisionContentRoot: options.revisionContentRoot, + }; + try { + return { + workspace: options.workspace, + workspaceId: options.workspaceId, + workspaceRevision: options.workspaceRevision, + revisionContentRoot: options.revisionContentRoot, + runtimePaths: runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision), + installationOverlay: overlay, + bindings, + bindingDigest: stableBindingDigest(bindings), + semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl, + releaseSecrets: () => secretLease?.release(), + renderedConfig: renderRuntimeConfig( + options.workspace, + bindings, + runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision), + context, + overlay, + options.semanticRuntime, + ), + }; + } catch (error) { + secretLease?.release(); + throw error; + } +} + +export function renderWorkspaceRuntimeFromSnapshotPath(options: { + snapshotPath: string; + harnessDir: string; + configPath: string; + dataRoot: string; + secretRoots: readonly string[]; + semanticRuntime: SemanticRuntimeConfig; + workspaceSecretStore?: WorkspaceSecretStore; +}): RenderedWorkspaceRuntime { + const snapshot = readSnapshotWorkspace(options.snapshotPath); + return renderWorkspaceRuntimeFromWorkspace({ + workspace: snapshot.workspace, + workspaceId: snapshot.identity.workspaceId, + workspaceRevision: snapshot.identity.workspaceRevision, + revisionContentRoot: snapshot.revisionContentRoot, + harnessDir: options.harnessDir, + configPath: options.configPath, + dataRoot: options.dataRoot, + secretRoots: options.secretRoots, + semanticRuntime: options.semanticRuntime, + workspaceSecretStore: options.workspaceSecretStore, + }); +} + +export async function renderActiveWorkspaceRuntime(options: { + workspaceId: string; + registry: WorkspaceRegistry; + registryConfig: WorkspaceRegistryConfig; + harnessDir: string; + configPath: string; + dataRoot: string; + secretRoots: readonly string[]; + semanticRuntime: SemanticRuntimeConfig; + workspaceSecretStore?: WorkspaceSecretStore; +}): Promise { + // The persisted active state may reference host-side snapshot paths (written by another + // process or installation). Read the active state directly and resolve the immutable snapshot + // beneath this process's own configured registry root, so the path is correct inside the + // maintenance container and on the host. This deliberately bypasses WorkspaceRegistry.read, + // whose integrity check would fail on host-side paths inside the container. + const activePath = join(options.registryConfig.root, "state", "active.json"); + const active = JSON.parse(await readFileAsync(activePath, "utf8")) as { + head: string; + revisions?: Array<{ id: string; commit: string; blob: string }>; + }; + const revision = (active.revisions ?? []).find((entry) => entry.id === options.workspaceId); + if (!revision) throw new Error("workspace is not active"); + const repository = new GitWorkspaceRepository(options.registryConfig); + await repository.ensureLayout(); + const snapshotPath = options.registry.snapshotPath(revision.commit, revision.id); + const descriptorSource = await readFileAsync(snapshotPath, "utf8"); + const workspace = parseWorkspaceYaml(descriptorSource); + const catalogSource = await repository.readCatalog(revision.commit); + const rendered = renderWorkspaceRuntimeFromWorkspace({ + workspace, + workspaceId: revision.id, + workspaceRevision: revision.commit, + revisionContentRoot: dirname(snapshotPath), + harnessDir: options.harnessDir, + configPath: options.configPath, + dataRoot: options.dataRoot, + secretRoots: options.secretRoots, + semanticRuntime: options.semanticRuntime, + workspaceSecretStore: options.workspaceSecretStore, + }); + return { + ...rendered, + snapshotPath, + descriptorBlob: `sha256:${createHash("sha256").update(descriptorSource).digest("hex")}`, + catalogBlob: `sha256:${createHash("sha256").update(catalogSource).digest("hex")}`, + }; +} + +function decodePublishedRuntimeConfigManifest(value: unknown): PublishedRuntimeConfigManifest { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration manifest is invalid"); + } + const manifest = value as Record; + const file = manifest.file as Record | undefined; + if ( + manifest.schemaVersion !== 1 + || typeof manifest.workspaceId !== "string" + || typeof manifest.workspaceRevision !== "string" + || typeof manifest.descriptorBlob !== "string" + || typeof manifest.catalogBlob !== "string" + || typeof manifest.configDigest !== "string" + || typeof manifest.bindingDigest !== "string" + || typeof manifest.effectiveConfigIdentity !== "string" + || typeof manifest.configFingerprint !== "string" + || typeof manifest.inputFingerprint !== "string" + || typeof manifest.path !== "string" + || !file + || typeof file.dev !== "number" + || typeof file.ino !== "number" + || typeof file.size !== "number" + || typeof file.mode !== "number" + || typeof file.nlink !== "number" + ) { + throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration manifest is invalid"); + } + return manifest as unknown as PublishedRuntimeConfigManifest; +} + +export async function publishDeterministicRuntimeConfigLease(options: { + workspaceId: string; + registry: WorkspaceRegistry; + registryConfig: WorkspaceRegistryConfig; + harnessDir: string; + configPath: string; + dataRoot: string; + secretRoots: readonly string[]; + semanticRuntime: SemanticRuntimeConfig; + workspaceSecretStore?: WorkspaceSecretStore; +}): Promise { + const rendered = await renderActiveWorkspaceRuntime(options); + const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing"); + const renderedConfigObject = parse(rendered.renderedConfig) as Record; + const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject); + const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject); + const configFingerprintValue = configFingerprint(renderedConfigObject); + const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject); + const identitySuffix = inputFingerprintValue.slice(7, 23); + + const preprocessingRoot = ensureTrustedDirectory(join( + options.dataRoot, + "sessions", + rendered.workspaceId, + "preprocessing", + )); + const configDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config")); + const manifestDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config-manifests")); + const path = join(configDirectory, `${rendered.workspaceRevision}-${identitySuffix}.yaml`); + const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}-${identitySuffix}.json`); + const configDigest = sha256(publishedConfig); + + const verifyPublished = (): PublishedRuntimeConfigManifest | undefined => { + let manifest: PublishedRuntimeConfigManifest | undefined; + try { + manifest = decodePublishedRuntimeConfigManifest(readStrictJson(manifestPath)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + let configStat: ReturnType | undefined; + let contents: string | undefined; + try { + const file = readTrustedFile(path); + contents = file.contents; + configStat = file.stat; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (contents === undefined) return manifest; + if ((Number(configStat!.mode) & 0o777) !== 0o400 || configStat!.nlink !== 1) { + throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed"); + } + if (sha256(contents) !== configDigest) { + throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed"); + } + if (!manifest) return undefined; + if ( + manifest.workspaceId !== rendered.workspaceId + || manifest.workspaceRevision !== rendered.workspaceRevision + || manifest.descriptorBlob !== rendered.descriptorBlob + || manifest.catalogBlob !== rendered.catalogBlob + || manifest.configDigest !== configDigest + || manifest.bindingDigest !== rendered.bindingDigest + || manifest.effectiveConfigIdentity !== effectiveConfigIdentityValue + || manifest.configFingerprint !== configFingerprintValue + || manifest.inputFingerprint !== inputFingerprintValue + || manifest.path !== path + || manifest.file.dev !== configStat!.dev + || manifest.file.ino !== configStat!.ino + || manifest.file.size !== configStat!.size + || manifest.file.mode !== (Number(configStat!.mode) & 0o777) + || manifest.file.nlink !== configStat!.nlink + ) { + throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed"); + } + return manifest; + }; + + const existing = verifyPublished(); + if (existing) { + return { + path, + manifestPath, + workspaceId: rendered.workspaceId, + workspaceRevision: rendered.workspaceRevision, + descriptorBlob: rendered.descriptorBlob, + catalogBlob: rendered.catalogBlob, + configDigest, + bindingDigest: rendered.bindingDigest, + semanticQdrantUrl: rendered.semanticQdrantUrl, + effectiveConfig, + effectiveConfigIdentity: effectiveConfigIdentityValue, + configFingerprint: configFingerprintValue, + inputFingerprint: inputFingerprintValue, + release: () => rendered.releaseSecrets(), + }; + } + + try { + readTrustedFile(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + writeAtomicFile(path, publishedConfig, 0o400); + } else { + throw error; + } + } + const published = readTrustedFile(path); + const publishedStat = published.stat!; + if (sha256(published.contents) !== configDigest) { + throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed"); + } + const manifest: PublishedRuntimeConfigManifest = { + schemaVersion: 1, + workspaceId: rendered.workspaceId, + workspaceRevision: rendered.workspaceRevision, + descriptorBlob: rendered.descriptorBlob, + catalogBlob: rendered.catalogBlob, + configDigest, + bindingDigest: rendered.bindingDigest, + effectiveConfigIdentity: effectiveConfigIdentityValue, + configFingerprint: configFingerprintValue, + inputFingerprint: inputFingerprintValue, + path, + file: { + dev: Number(publishedStat.dev), + ino: Number(publishedStat.ino), + size: Number(publishedStat.size), + mode: Number(publishedStat.mode) & 0o777, + nlink: Number(publishedStat.nlink), + }, + }; + try { + readStrictJson(manifestPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + writeAtomicFile(manifestPath, `${JSON.stringify(manifest)} +`, 0o600); + } else { + throw error; + } + } + verifyPublished(); + return { + path, + manifestPath, + workspaceId: rendered.workspaceId, + workspaceRevision: rendered.workspaceRevision, + descriptorBlob: rendered.descriptorBlob, + catalogBlob: rendered.catalogBlob, + configDigest, + bindingDigest: rendered.bindingDigest, + semanticQdrantUrl: rendered.semanticQdrantUrl, + effectiveConfig, + effectiveConfigIdentity: effectiveConfigIdentityValue, + configFingerprint: configFingerprintValue, + inputFingerprint: inputFingerprintValue, + release: () => rendered.releaseSecrets(), + }; +} diff --git a/backend/src/workspaces/runtime-renderer.ts b/backend/src/workspaces/runtime-renderer.ts new file mode 100644 index 00000000..23db59a1 --- /dev/null +++ b/backend/src/workspaces/runtime-renderer.ts @@ -0,0 +1,294 @@ +import { basename, join } from "node:path"; +import { stringify } from "yaml"; +import { buildInstallationContract } from "./contracts.js"; +import { validateWorkspaceDescriptor, type WorkspaceDescriptor } from "./schema.js"; +import type { ResolvedBinding, ResolvedEvidenceBinding, RuntimeBindings } from "./bindings.js"; +export type { RuntimeBindings } from "./bindings.js"; + +export interface RuntimePaths { + sessions: string; + artifacts: string; + indexes: string; + memory: string; + /** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */ + annotations_root?: string; +} + +export interface RuntimeIdentity { + workspaceId: string; + workspaceRevision: string; +} + +/** Immutable, explicit inputs needed to bind descriptor-relative content to one revision. */ +export interface RuntimeRenderContext extends RuntimeIdentity { + revisionContentRoot: string; +} + +export interface RuntimeInstallationOverlay { + session_storage?: unknown; + profile?: unknown; +} + +export interface SemanticRuntimeConfig { + internalQdrantUrl: string; + internalEmbeddingUrl: string; + internalEmbeddingModel: string; + internalEmbeddingDimensions: number; +} + +export const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, +}; + +function bindingValue(binding: ResolvedBinding, name: string): string | undefined { + return binding.values[name]; +} + +function requireBinding(binding: ResolvedBinding, name: string): string { + const value = bindingValue(binding, name); + if (value === undefined) throw new Error(`runtime binding is missing ${name}`); + return value; +} + +function directConnection( + binding: ResolvedBinding, + names: { host: string; port: string; user: string; passwordFile: string; tlsCaFile: string }, + identity: { database: string; schema: string }, +): Record { + const connection: Record = { + host: requireBinding(binding, names.host), + port: Number(requireBinding(binding, names.port)), + database: identity.database, + schema: identity.schema, + user: requireBinding(binding, names.user), + password_file: requireBinding(binding, names.passwordFile), + }; + const tlsCaFile = bindingValue(binding, names.tlsCaFile); + if (tlsCaFile !== undefined) connection.ssl_ca_file = tlsCaFile; + return connection; +} + +function restEndpoint( + binding: ResolvedBinding, + names: { baseUrl: string; apiKeyFile: string; tlsCaFile: string }, + requiresCredential: boolean, +): Record { + const endpoint: Record = { + base_url: requireBinding(binding, names.baseUrl), + }; + if (requiresCredential) endpoint.api_key_file = requireBinding(binding, names.apiKeyFile); + const tlsCaFile = bindingValue(binding, names.tlsCaFile); + if (tlsCaFile !== undefined) endpoint.ssl_ca_file = tlsCaFile; + return endpoint; +} + +function exactSeconds(timeoutMs: number): number { + return timeoutMs / 1_000; +} + +function requireRuntimeRenderContext( + identity: RuntimeIdentity | RuntimeRenderContext | undefined, +): RuntimeRenderContext { + if (!identity || !("revisionContentRoot" in identity)) { + throw new Error("runtime Evidence requires an immutable revision content root"); + } + return identity; +} + +function evidenceBindingValue(binding: ResolvedEvidenceBinding, name: string): string | undefined { + return binding.values[name]; +} + +function requireEvidenceBinding(binding: ResolvedEvidenceBinding, name: string): string { + const value = evidenceBindingValue(binding, name); + if (value === undefined) throw new Error(`runtime binding is missing ${name}`); + return value; +} + +function renderEvidence( + workspace: WorkspaceDescriptor, + binding: ResolvedEvidenceBinding, + context: RuntimeRenderContext, + bindingName: (suffix: string) => string, +): { evidence: Record; vector: Record } | undefined { + if (workspace.evidence === undefined) return undefined; + if (binding.missing.length > 0) { + throw new Error("runtime configuration requires complete Evidence bindings"); + } + if (basename(context.revisionContentRoot) !== context.workspaceRevision) { + throw new Error("runtime revision content root does not match workspace revision"); + } + + const source = workspace.evidence.source; + let renderedSource: Record; + if (source.type === "filesystem") { + renderedSource = { + type: "filesystem", + root: join(context.revisionContentRoot, source.uri), + patterns: source.patterns, + max_bytes: source.max_bytes, + }; + } else if (source.type === "http") { + renderedSource = { + type: "http", + ...(source.authentication === "none" + ? { urls: source.uris } + : { + provenance_urls: source.uris, + signed_urls_file: requireEvidenceBinding(binding, bindingName("SIGNED_URLS_FILE")), + }), + connect_timeout: exactSeconds(source.connect_timeout_ms), + read_timeout: exactSeconds(source.read_timeout_ms), + max_bytes: source.max_bytes, + max_redirects: source.max_redirects, + allow_private_hosts: source.allow_private_hosts, + max_cache_bytes: source.max_cache_bytes, + }; + } else { + const uri = new URL(source.uri); + const sessionTokenFile = source.credentials === "static_files" + ? evidenceBindingValue(binding, bindingName("SESSION_TOKEN_FILE")) + : undefined; + renderedSource = { + type: "s3", + bucket: uri.hostname, + prefix: uri.pathname.replace(/^\//, ""), + ...(source.endpoint_url === undefined ? {} : { endpoint_url: source.endpoint_url }), + ...(source.region === undefined ? {} : { region: source.region }), + ...(source.credentials === "ambient" ? {} : { + access_key_file: requireEvidenceBinding(binding, bindingName("ACCESS_KEY_FILE")), + secret_key_file: requireEvidenceBinding(binding, bindingName("SECRET_KEY_FILE")), + ...(sessionTokenFile === undefined ? {} : { session_token_file: sessionTokenFile }), + }), + trusted_endpoint: source.trusted_endpoint, + allow_private_endpoint: source.allow_private_endpoint, + allow_insecure_endpoint: source.allow_insecure_endpoint, + max_bytes: source.max_bytes, + max_objects: source.max_objects, + max_pages: source.max_pages, + page_size: source.page_size, + }; + } + + return { + evidence: { sources: [renderedSource] }, + vector: { + max_chunk_chars: workspace.evidence.policy.max_chunk_chars, + retain_published_generations: workspace.evidence.policy.retain_published_generations, + }, + }; +} + + +function placeholderConnection(identity: { database: string; schema: string }): Record { + return { + host: "localhost", + port: 5432, + database: identity.database, + schema: identity.schema, + user: "rest", + password: "", + transport: "rest", + }; +} + +function requireSupportedDescriptor(workspace: unknown): void { + if (typeof workspace !== "object" || workspace === null) { + throw new Error("Runtime renderer supports only workspace schema version 3"); + } + const metadata = Reflect.get(workspace, "workspace"); + if (typeof metadata !== "object" || metadata === null + || Reflect.get(metadata, "schema_version") !== 3) { + throw new Error("Runtime renderer supports only workspace schema version 3"); + } +} + +/** Render the schema-v3 compatibility fields consumed by the current Python harness. */ +export function renderRuntimeConfig( + workspace: WorkspaceDescriptor, + bindings: RuntimeBindings, + paths: RuntimePaths, + identity?: RuntimeIdentity | RuntimeRenderContext, + installation: RuntimeInstallationOverlay = {}, + semanticRuntime: SemanticRuntimeConfig = DEFAULT_SEMANTIC_RUNTIME, +): string { + requireSupportedDescriptor(workspace); + const descriptor = validateWorkspaceDescriptor(workspace); + const contract = buildInstallationContract(descriptor); + const name = (role: "DWH" | "EVIDENCE", suffix: string) => { + const variable = contract.variables.find((entry) => entry.role === role && entry.suffix === suffix); + if (!variable) throw new Error(`workspace contract is missing ${role}_${suffix}`); + return variable.name; + }; + const renderedEvidence = descriptor.evidence === undefined + ? undefined + : renderEvidence( + descriptor, + bindings.evidence, + requireRuntimeRenderContext(identity), + (suffix) => name("EVIDENCE", suffix), + ); + if (bindings.dwh.missing.length > 0) { + throw new Error("runtime configuration requires complete bindings"); + } + + const dwhIdentity = { database: descriptor.dwh.database, schema: descriptor.dwh.schema }; + const database = bindings.dwh.transport === "postgres_direct" + ? { ...directConnection(bindings.dwh, { + host: name("DWH", "HOST"), + port: name("DWH", "PORT"), + user: name("DWH", "USER"), + passwordFile: name("DWH", "PASSWORD_FILE"), + tlsCaFile: name("DWH", "TLS_CA_FILE"), + }, dwhIdentity), transport: "direct" } + : placeholderConnection(dwhIdentity); + const rendered: Record = { + ...(identity ? { + runtime_identity: { + workspace_id: identity.workspaceId, + workspace_revision: identity.workspaceRevision, + source_identity: `workspace://${identity.workspaceId}`, + }, + } : {}), + ...(installation.session_storage === undefined + ? {} : { session_storage: installation.session_storage }), + ...(installation.profile === undefined ? {} : { profile: installation.profile }), + language: descriptor.workspace.language, + database, + semantic_index: descriptor.semantic_index, + resources: { + vector: { + engine: "qdrant", + base_url: semanticRuntime.internalQdrantUrl, + collection: descriptor.semantic_index.vector_store.collection, + }, + embeddings: { + provider: "ollama_internal", + base_url: semanticRuntime.internalEmbeddingUrl, + model: semanticRuntime.internalEmbeddingModel, + dimensions: semanticRuntime.internalEmbeddingDimensions, + }, + }, + roots: paths, + paths, + ...(renderedEvidence ?? {}), + }; + if (bindings.dwh.transport === "postgres_direct") { + rendered.dwh = { type: "postgres_direct", connection: database }; + } else if (bindings.dwh.transport === "rest_api") { + const rest = restEndpoint(bindings.dwh, { + baseUrl: name("DWH", "BASE_URL"), + apiKeyFile: name("DWH", "API_KEY_FILE"), + tlsCaFile: name("DWH", "TLS_CA_FILE"), + }, descriptor.diagnostics?.dwh_rest?.auth !== "none"); + rendered.rest = rest; + rendered.database = placeholderConnection(dwhIdentity); + rendered.dwh = { type: "thoth_rest", database: dwhIdentity, endpoint: rest }; + } else { + throw new Error("ssh_tunnel runtime configuration requires a diagnostic tunnel"); + } + return stringify(rendered, { lineWidth: 0, sortMapEntries: false }); +} diff --git a/backend/src/workspaces/schema.ts b/backend/src/workspaces/schema.ts new file mode 100644 index 00000000..666fae3e --- /dev/null +++ b/backend/src/workspaces/schema.ts @@ -0,0 +1,439 @@ +import { parseAllDocuments, stringify } from "yaml"; +import { z } from "zod"; + +export const DWH_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"] as const; +export type DwhTransport = (typeof DWH_TRANSPORTS)[number]; + +export const VECTOR_TRANSPORTS = ["pgvector_direct", "rest_api", "ssh_tunnel"] as const; +export type VectorTransport = (typeof VECTOR_TRANSPORTS)[number]; + +export const REST_DIAGNOSTIC_METHODS = ["GET", "POST"] as const; +export type RestDiagnosticMethod = (typeof REST_DIAGNOSTIC_METHODS)[number]; + +export const DIAGNOSTIC_AUTH_MODES = ["none", "bearer", "x-api-key"] as const; +export type DiagnosticAuthMode = (typeof DIAGNOSTIC_AUTH_MODES)[number]; + +export interface RestDiagnosticRequest { + method: RestDiagnosticMethod; + path: string; + auth: DiagnosticAuthMode; +} + +export interface CanonicalDiagnostics { + dwh_rest?: RestDiagnosticRequest & { response: { database: string; schema: string } }; + vector_rest?: { + metadata: RestDiagnosticRequest & { + response: { collection: string; dimensions: string; distance: string }; + }; + reversible_probe?: RestDiagnosticRequest & { + method: "POST"; + auth: Exclude; + response: { operation: string }; + }; + }; + embedding?: RestDiagnosticRequest & { response: { model: string; dimensions: string } }; +} + +interface WorkspaceMetadata { + schema_version: 3; + id: string; + name: string; + description?: string; + language: "en" | "it"; +} + +interface WorkspaceDwh { + engine: "postgres"; + database: string; + schema: string; + port?: number; + timeout_ms?: number; + supported_transports: DwhTransport[]; +} + +interface WorkspaceBase { + workspace: WorkspaceMetadata; + dwh: WorkspaceDwh; + semantic_index: { + vector_store: TVectorStore; + embedding: { + provider: "ollama_internal"; + model: "qwen3-embedding:0.6b"; + dimensions: 1024; + }; + }; + llm_policy: { + default?: `${string}/${string}`; + allowed: `${string}/${string}`[]; + }; + diagnostics?: Pick; +} + +interface QdrantVectorStore { + engine: "qdrant"; + collection: string; + dimensions: 1024; + distance: "cosine"; +} + +export interface EvidencePolicy { + max_chunk_chars: number; + retain_published_generations: number; +} + +export type EvidenceSource = + | { + type: "filesystem"; + uri: string; + patterns: string[]; + max_bytes: number; + } + | { + type: "http"; + uris: string[]; + authentication: "none" | "signed_urls_file"; + connect_timeout_ms: number; + read_timeout_ms: number; + max_bytes: number; + max_redirects: number; + allow_private_hosts: boolean; + max_cache_bytes: number; + } + | { + type: "s3"; + uri: string; + endpoint_url?: string; + region?: string; + credentials: "ambient" | "static_files"; + trusted_endpoint: boolean; + allow_private_endpoint: boolean; + allow_insecure_endpoint: boolean; + max_bytes: number; + max_objects: number; + max_pages: number; + page_size: number; + }; + +export interface WorkspaceEvidence { + source: EvidenceSource; + policy: EvidencePolicy; +} + +export interface WorkspaceV3 extends WorkspaceBase { + evidence?: WorkspaceEvidence; +} + +export type CanonicalWorkspace = WorkspaceV3; +export type WorkspaceDescriptor = WorkspaceV3; + +const workspaceId = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/, { + message: "workspace id must match ^[a-z][a-z0-9-]{2,62}$", +}); +const identifier = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, { + message: "database identifiers must start with a letter or underscore", +}); +const port = z.number().int().min(1).max(65_535); +const timeoutMs = z.number().int().positive(); +const modelReference = z.string().regex(/^[^/\s]+\/[^/\s]+$/, { + message: "model must use provider/model syntax", +}); + +function isOriginRelativeDiagnosticPath(value: string): boolean { + return /^\/(?!\/)[^\\\u0000-\u001F\u007F?#]*$/.test(value) && !/%5c/i.test(value); +} + +const diagnosticPath = z.string().refine(isOriginRelativeDiagnosticPath, { + message: "diagnostic paths must be origin-relative and cannot contain backslashes, control characters, queries, or fragments", +}); +const responseField = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/, { + message: "diagnostic response fields must be identifiers", +}); +const restDiagnosticRequest = z.object({ + method: z.enum(REST_DIAGNOSTIC_METHODS), + path: diagnosticPath, + auth: z.enum(DIAGNOSTIC_AUTH_MODES), +}).strict(); +const dwhRestDiagnostic = restDiagnosticRequest.extend({ + response: z.object({ database: responseField, schema: responseField }).strict(), +}).strict(); + +const dwhSchema = z.object({ + engine: z.literal("postgres"), + database: identifier, + schema: identifier, + port: port.optional(), + timeout_ms: timeoutMs.optional(), + supported_transports: z.array(z.enum(DWH_TRANSPORTS)).min(1), +}).strict(); +const internalEmbeddingSchema = z.object({ + provider: z.literal("ollama_internal"), + model: z.literal("qwen3-embedding:0.6b"), + dimensions: z.literal(1024), +}).strict(); +const qdrantVectorStoreSchema = z.object({ + engine: z.literal("qdrant"), + collection: workspaceId, + dimensions: z.literal(1024), + distance: z.literal("cosine"), +}).strict(); +const llmPolicySchema = z.object({ + default: modelReference.optional(), + allowed: z.array(modelReference).min(1), +}).strict(); + +const positiveSafeInteger = z.number().int().safe().positive(); +const nonnegativeSafeInteger = z.number().int().safe().nonnegative(); + +function isSafeEvidencePattern(value: string): boolean { + const parts = value.split("/"); + return value.length > 0 + && !value.startsWith("/") + && !value.includes("\\") + && !/[\u0000-\u001f\u007f]/u.test(value) + && parts.every((part) => part !== "" && part !== "." && part !== ".."); +} + +function parsePublicHttpUri(value: string): URL | undefined { + if (value.trim() !== value || /[\u0000-\u001f\u007f\\]/u.test(value)) return undefined; + try { + const parsed = new URL(value); + if ( + !["http:", "https:"].includes(parsed.protocol) + || parsed.hostname.length === 0 + || parsed.username !== "" + || parsed.password !== "" + || parsed.search !== "" + || parsed.hash !== "" + ) return undefined; + return parsed; + } catch { + return undefined; + } +} + +function canonicalPublicHttpUri(value: string): string | undefined { + return parsePublicHttpUri(value)?.href; +} + +function isSafeS3Uri(value: string): boolean { + if (value.trim() !== value || /[\u0000-\u001f\u007f\\]/u.test(value)) return false; + try { + const parsed = new URL(value); + const bucket = parsed.hostname; + const validBucket = /^(?=.{3,63}$)(?!-)(?!.*\.\.)(?!.*\.-)(?!.*-\.)[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/.test(bucket) + && !/^\d{1,3}(?:\.\d{1,3}){3}$/.test(bucket); + return parsed.protocol === "s3:" + && validBucket + && parsed.port === "" + && parsed.username === "" + && parsed.password === "" + && parsed.search === "" + && parsed.hash === "" + && parsed.href === value; + } catch { + return false; + } +} + +function isSafeS3Endpoint(value: string): boolean { + const parsed = parsePublicHttpUri(value); + return parsed !== undefined && (parsed.pathname === "/" || parsed.pathname === ""); +} + +const evidencePattern = z.string().refine(isSafeEvidencePattern, { + message: "evidence patterns must be normalized relative globs", +}); +const filesystemEvidenceSourceSchema = z.object({ + type: z.literal("filesystem"), + uri: z.string(), + patterns: z.array(evidencePattern).min(1).default(["**/*.md"]), + max_bytes: positiveSafeInteger.default(10 * 1024 * 1024), +}).strict().superRefine((source, context) => { + if (new Set(source.patterns).size !== source.patterns.length) { + context.addIssue({ code: "custom", path: ["patterns"], message: "evidence patterns must not repeat" }); + } +}); +const httpEvidenceUri = z.string().refine((value) => parsePublicHttpUri(value) !== undefined, { + message: "HTTP evidence URIs must be public http(s) identities without credentials, query, or fragment", +}); +const httpEvidenceSourceSchema = z.object({ + type: z.literal("http"), + uris: z.array(httpEvidenceUri).min(1), + authentication: z.enum(["none", "signed_urls_file"]).default("none"), + connect_timeout_ms: positiveSafeInteger.default(5_000), + read_timeout_ms: positiveSafeInteger.default(30_000), + max_bytes: positiveSafeInteger.default(10 * 1024 * 1024), + max_redirects: nonnegativeSafeInteger.default(5), + allow_private_hosts: z.boolean().default(false), + max_cache_bytes: positiveSafeInteger.default(64 * 1024 * 1024), +}).strict().superRefine((source, context) => { + const canonical = source.uris.map(canonicalPublicHttpUri); + if (new Set(canonical).size !== canonical.length) { + context.addIssue({ code: "custom", path: ["uris"], message: "HTTP evidence URIs must not repeat" }); + } +}); +const s3EvidenceSourceSchema = z.object({ + type: z.literal("s3"), + uri: z.string().refine(isSafeS3Uri, { + message: "S3 evidence URI must use s3:// without credentials, query, or fragment", + }), + endpoint_url: z.string().refine(isSafeS3Endpoint, { + message: "S3 endpoint must be an origin-only http(s) URL without credentials", + }).optional(), + region: z.string().trim().min(1).optional(), + credentials: z.enum(["ambient", "static_files"]).default("ambient"), + trusted_endpoint: z.boolean().default(false), + allow_private_endpoint: z.boolean().default(false), + allow_insecure_endpoint: z.boolean().default(false), + max_bytes: positiveSafeInteger.default(10 * 1024 * 1024), + max_objects: positiveSafeInteger.default(10_000), + max_pages: positiveSafeInteger.default(100), + page_size: positiveSafeInteger.max(1_000).default(1_000), +}).strict().superRefine((source, context) => { + if (source.endpoint_url === undefined) { + if (source.trusted_endpoint || source.allow_private_endpoint || source.allow_insecure_endpoint) { + context.addIssue({ + code: "custom", path: ["endpoint_url"], + message: "S3 endpoint policy requires endpoint_url", + }); + } + return; + } + if (!source.trusted_endpoint) { + context.addIssue({ + code: "custom", path: ["trusted_endpoint"], + message: "custom S3 endpoints must be explicitly trusted", + }); + } + const endpoint = parsePublicHttpUri(source.endpoint_url); + if (endpoint?.protocol === "http:" && !source.allow_insecure_endpoint) { + context.addIssue({ + code: "custom", path: ["allow_insecure_endpoint"], + message: "HTTP S3 endpoints require an explicit insecure opt-in", + }); + } +}); +const evidenceSourceSchema = z.discriminatedUnion("type", [ + filesystemEvidenceSourceSchema, + httpEvidenceSourceSchema, + s3EvidenceSourceSchema, +]); +const evidencePolicySchema = z.object({ + max_chunk_chars: positiveSafeInteger.default(4_000), + retain_published_generations: positiveSafeInteger.default(3), +}).strict(); +const workspaceEvidenceSchema = z.object({ + source: evidenceSourceSchema, + policy: evidencePolicySchema.default({ + max_chunk_chars: 4_000, + retain_published_generations: 3, + }), +}).strict(); + +function unique(values: readonly T[], context: z.RefinementCtx, path: PropertyKey[]) { + if (new Set(values).size !== values.length) { + context.addIssue({ code: "custom", path, message: "supported transports must not repeat" }); + } +} + +function workspaceInvariants(workspace: any, context: z.RefinementCtx): void { + unique(workspace.dwh.supported_transports, context, ["dwh", "supported_transports"]); + unique(workspace.llm_policy.allowed, context, ["llm_policy", "allowed"]); + + if (workspace.evidence?.source.type === "filesystem") { + const expected = `${workspace.workspace.id}/evidence`; + if (workspace.evidence.source.uri !== expected) { + context.addIssue({ + code: "custom", + path: ["evidence", "source", "uri"], + message: "filesystem evidence URI must be the canonical workspace Evidence root", + }); + } + } + + if (workspace.semantic_index.vector_store.dimensions !== workspace.semantic_index.embedding.dimensions) { + context.addIssue({ + code: "custom", + path: ["semantic_index", "embedding", "dimensions"], + message: "embedding dimensions must match vector store dimensions", + }); + } + if (workspace.llm_policy.default && !workspace.llm_policy.allowed.includes(workspace.llm_policy.default)) { + context.addIssue({ + code: "custom", + path: ["llm_policy", "default"], + message: "LLM default must be included in the allowlist", + }); + } + if (workspace.diagnostics?.dwh_rest && !workspace.dwh.supported_transports.includes("rest_api")) { + context.addIssue({ + code: "custom", + path: ["diagnostics", "dwh_rest"], + message: "diagnostics.dwh_rest requires dwh rest_api transport support", + }); + } +} + +const WorkspaceV3Schema = z.object({ + dwh: dwhSchema, + llm_policy: llmPolicySchema, + evidence: workspaceEvidenceSchema.optional(), + diagnostics: z.object({ + dwh_rest: dwhRestDiagnostic.optional(), + }).strict().optional(), + workspace: z.object({ + schema_version: z.literal(3), id: workspaceId, name: z.string().trim().min(1), + description: z.string().trim().min(1).optional(), language: z.enum(["en", "it"]), + }).strict(), + semantic_index: z.object({ + vector_store: qdrantVectorStoreSchema, + embedding: internalEmbeddingSchema, + }).strict(), +}).strict().superRefine(workspaceInvariants); +const WorkspaceDescriptorSchema = WorkspaceV3Schema; + +export function parseWorkspaceYaml(source: string): WorkspaceDescriptor { + const documents = parseAllDocuments(source, { uniqueKeys: true }); + if (documents.length !== 1) throw new Error("Workspace YAML must contain exactly one document"); + const document = documents[0]; + if (document.errors.length > 0 || document.warnings.length > 0) { + throw new Error(`Invalid workspace YAML: ${[...document.errors, ...document.warnings] + .map((error) => error.message).join("; ")}`); + } + return validateWorkspaceDescriptor(document.toJSON()); +} + +export function validateWorkspaceDescriptor(workspace: unknown): WorkspaceDescriptor { + return WorkspaceDescriptorSchema.parse(workspace) as WorkspaceDescriptor; +} + +export function isCanonicalWorkspace(workspace: unknown): workspace is CanonicalWorkspace { + return WorkspaceDescriptorSchema.safeParse(workspace).success; +} + +export function isOperationalWorkspace(workspace: unknown): workspace is WorkspaceV3 { + return WorkspaceDescriptorSchema.safeParse(workspace).success; +} + +export function validateOperationalWorkspace(workspace: unknown): WorkspaceV3 { + return validateWorkspaceDescriptor(workspace); +} + +/** Builds a request URL only after rejecting values that can leave the declared service origin. */ +export function resolveDiagnosticUrl(baseUrl: string, path: string): URL { + if (!isOriginRelativeDiagnosticPath(path)) throw new Error("Diagnostic path must remain on the configured origin"); + const base = new URL(baseUrl); + // Resolve the origin-relative path beneath the configured base path (e.g. `/dwh/`), not the + // origin root: a leading slash must append to the base path instead of resetting it. + const basePath = base.pathname.endsWith("/") ? base.pathname : `${base.pathname}/`; + const resolved = new URL(`${basePath}${path.replace(/^\/+/, "")}`, base.origin); + if (resolved.origin !== base.origin) throw new Error("Diagnostic URL must remain on the configured origin"); + return resolved; +} + +export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string { + const canonical = validateOperationalWorkspace(workspace); + return stringify(canonical, { lineWidth: 0, sortMapEntries: true }); +} + +export { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js"; diff --git a/backend/src/workspaces/secret-requirements.ts b/backend/src/workspaces/secret-requirements.ts new file mode 100644 index 00000000..b0c15836 --- /dev/null +++ b/backend/src/workspaces/secret-requirements.ts @@ -0,0 +1,199 @@ +import { dirname } from "node:path"; + +import { + buildInstallationContract, + type InstallationRole, + type InstallationSuffix, + type InstallationVariable, +} from "./contracts.js"; +import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js"; +import { + DWH_TRANSPORTS, + validateWorkspaceDescriptor, + type DwhTransport, + type WorkspaceDescriptor, +} from "./schema.js"; +import { + WorkspaceSecretStore, + type WorkspaceSecretMaterialization, +} from "./secret-store.js"; + +export type WorkspaceSecretInput = "password" | "textarea"; + +export interface WorkspaceSecretRequirement { + id: string; + variable: string; + connector: "dwh" | "evidence"; + label: string; + description: string; + input: WorkspaceSecretInput; + required: boolean; +} + +export interface WorkspaceRuntimeBindingLease { + bindings: RuntimeBindings; + release(): void; +} + +interface RequirementDefinition { + id: string; + label: string; + description: string; + input: WorkspaceSecretInput; +} + +const DEFINITIONS: Readonly>> = { + "DWH.PASSWORD_FILE": { + id: "dwh.password", + label: "Data warehouse password", + description: "Password used by the selected data warehouse connection.", + input: "password", + }, + "DWH.API_KEY_FILE": { + id: "dwh.api_key", + label: "Data warehouse API key", + description: "API key sent to the configured data warehouse REST endpoint.", + input: "password", + }, + "DWH.SSH_PRIVATE_KEY_FILE": { + id: "dwh.ssh_private_key", + label: "SSH private key", + description: "Private key used to open the configured SSH tunnel to the data warehouse.", + input: "textarea", + }, + "EVIDENCE.SIGNED_URLS_FILE": { + id: "evidence.signed_urls", + label: "Evidence signed URLs", + description: "Signed URLs that authorize ThothII to retrieve the workspace Evidence sources.", + input: "textarea", + }, + "EVIDENCE.ACCESS_KEY_FILE": { + id: "evidence.access_key", + label: "Evidence access key", + description: "Access-key identifier used for the configured S3 Evidence source.", + input: "password", + }, + "EVIDENCE.SECRET_KEY_FILE": { + id: "evidence.secret_key", + label: "Evidence secret key", + description: "Secret access key used for the configured S3 Evidence source.", + input: "password", + }, + "EVIDENCE.SESSION_TOKEN_FILE": { + id: "evidence.session_token", + label: "Evidence session token", + description: "Optional temporary session token used with the S3 Evidence credentials.", + input: "password", + }, +}; + +const REQUIRED_DWH_SECRETS: Readonly> = { + postgres_direct: ["PASSWORD_FILE"], + rest_api: ["API_KEY_FILE"], + ssh_tunnel: ["PASSWORD_FILE", "SSH_PRIVATE_KEY_FILE"], +}; + +function isTransport(value: string | undefined): value is DwhTransport { + return value !== undefined && (DWH_TRANSPORTS as readonly string[]).includes(value); +} + +function selectedTransport( + descriptor: WorkspaceDescriptor, + variables: readonly InstallationVariable[], + env: NodeJS.ProcessEnv, +): DwhTransport { + const transportVariable = variables.find(({ role, suffix }) => role === "DWH" && suffix === "TRANSPORT"); + const value = transportVariable === undefined ? undefined : env[transportVariable.name]; + return isTransport(value) && descriptor.dwh.supported_transports.includes(value) + ? value + : descriptor.dwh.supported_transports[0]; +} + +function requirementFor( + variable: InstallationVariable, + required: boolean, +): WorkspaceSecretRequirement | undefined { + const definition = DEFINITIONS[`${variable.role}.${variable.suffix}`]; + if (definition === undefined) return undefined; + return { + ...definition, + variable: variable.name, + connector: variable.role === "DWH" ? "dwh" : "evidence", + required, + }; +} + +/** + * Discover the credential fields for the connector and authentication mechanisms selected by + * this installation. Paths, hostnames and trust files remain installation configuration rather + * than user-entered secrets. + */ +export function discoverWorkspaceSecretRequirements( + workspace: WorkspaceDescriptor, + env: NodeJS.ProcessEnv, +): WorkspaceSecretRequirement[] { + const descriptor = validateWorkspaceDescriptor(workspace); + const variables = buildInstallationContract(descriptor).variables; + const transport = selectedTransport(descriptor, variables, env); + const restHasNoAuthentication = transport === "rest_api" && descriptor.diagnostics?.dwh_rest?.auth === "none"; + const requiredDwh = new Set(restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport]); + + const requirements: WorkspaceSecretRequirement[] = []; + for (const variable of variables) { + if (variable.role === "DWH") { + if (variable.transports !== undefined && !variable.transports.includes(transport)) continue; + const requirement = requirementFor(variable, requiredDwh.has(variable.suffix)); + if (requirement !== undefined && requirement.required) requirements.push(requirement); + continue; + } + const requirement = requirementFor(variable, variable.suffix !== "SESSION_TOKEN_FILE"); + if (requirement !== undefined) requirements.push(requirement); + } + return requirements; +} + +/** + * Materialize only the selected workspace credentials, translate them to the existing file-based + * harness contract, and bind cleanup to the returned lease. + */ +export function resolveRuntimeBindingsWithWorkspaceSecrets( + workspace: WorkspaceDescriptor, + env: NodeJS.ProcessEnv, + secretRoots: readonly string[], + store: WorkspaceSecretStore, +): WorkspaceRuntimeBindingLease { + const descriptor = validateWorkspaceDescriptor(workspace); + const requirements = discoverWorkspaceSecretRequirements(descriptor, env); + let materialization: WorkspaceSecretMaterialization | undefined; + try { + materialization = store.materialize( + descriptor.workspace.id, + requirements.map(({ id }) => id), + ); + const effectiveEnvironment: NodeJS.ProcessEnv = { ...env }; + const materializedRoots = new Set(); + for (const requirement of requirements) { + const path = materialization.files.get(requirement.id); + if (path === undefined) continue; + effectiveEnvironment[requirement.variable] = path; + materializedRoots.add(dirname(path)); + } + const bindings = resolveRuntimeBindings( + descriptor, + effectiveEnvironment, + [...secretRoots, ...materializedRoots], + ); + let released = false; + return { + bindings, + release: () => { + if (released) return; + released = true; + materialization?.release(); + }, + }; + } catch (error) { + materialization?.release(); + throw error; + } +} diff --git a/backend/src/workspaces/secret-store.ts b/backend/src/workspaces/secret-store.ts new file mode 100644 index 00000000..3431ea07 --- /dev/null +++ b/backend/src/workspaces/secret-store.ts @@ -0,0 +1,351 @@ +import { + chmodSync, + closeSync, + constants, + fchmodSync, + fsyncSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; +import { basename, join } from "node:path"; + +const STORE_ERROR = "Workspace secret store is unavailable."; +const DEFAULT_MAX_SECRET_BYTES = 64 * 1024; +const ID_PATTERN = /^[a-z0-9](?:[a-z0-9._-]{0,126}[a-z0-9])?$/; + +interface EncryptedEntry { + workspaceId: string; + requirementId: string; + iv: string; + tag: string; + ciphertext: string; +} + +interface VaultDocument { + version: 1; + generation: number; + entries: Record; +} + +export interface WorkspaceSecretMaterialization { + files: ReadonlyMap; + release(): void; +} + +export interface WorkspaceSecretStoreOptions { + root: string; + runtimeRoot?: string; + installationId: string; + maxSecretBytes?: number; +} + +function assertIdentifier(value: string, label: string): void { + if (!ID_PATTERN.test(value)) throw new Error(`Invalid ${label}.`); +} + +function entryKey(workspaceId: string, requirementId: string): string { + return Buffer.from(`${workspaceId}\0${requirementId}`, "utf8").toString("base64url"); +} + +function directorySync(path: string): void { + mkdirSync(path, { recursive: true, mode: 0o700 }); + chmodSync(path, 0o700); +} + +function syncDirectory(path: string): void { + const fd = openSync(path, constants.O_RDONLY); + try { + fsyncSync(fd); + } finally { + closeSync(fd); + } +} + +function atomicPrivateWrite(path: string, contents: string | Buffer): void { + const parent = join(path, ".."); + const temporary = join(parent, `.${basename(path)}.${process.pid}.${randomBytes(6).toString("hex")}`); + const fd = openSync(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + try { + fchmodSync(fd, 0o600); + writeFileSync(fd, contents); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(temporary, path); + chmodSync(path, 0o600); + syncDirectory(parent); +} + +function emptyVault(): VaultDocument { + return { version: 1, generation: 0, entries: {} }; +} + +export class WorkspaceSecretStore { + private readonly root: string; + private readonly runtimeRoot: string; + private readonly installationId: string; + private readonly maxSecretBytes: number; + private readonly keyPath: string; + private readonly vaultPath: string; + private readonly materializationReferences = new Map(); + + constructor(options: WorkspaceSecretStoreOptions) { + if (!options.installationId.trim()) throw new Error("Installation identifier is required."); + this.root = options.root; + this.runtimeRoot = options.runtimeRoot ?? join(options.root, "runtime"); + this.installationId = options.installationId; + this.maxSecretBytes = options.maxSecretBytes ?? DEFAULT_MAX_SECRET_BYTES; + this.keyPath = join(this.root, "master.key"); + this.vaultPath = join(this.root, "vault.json"); + directorySync(this.root); + directorySync(this.runtimeRoot); + this.ensureInitialized(); + } + + configured(workspaceId: string): string[] { + assertIdentifier(workspaceId, "workspace identifier"); + const vault = this.readVault(); + return Object.values(vault.entries) + .filter((entry) => entry.workspaceId === workspaceId) + .map((entry) => entry.requirementId) + .sort(); + } + + has(workspaceId: string, requirementId: string): boolean { + this.assertIds(workspaceId, requirementId); + return this.readVault().entries[entryKey(workspaceId, requirementId)] !== undefined; + } + + generation(workspaceId: string): number { + assertIdentifier(workspaceId, "workspace identifier"); + const vault = this.readVault(); + return Object.values(vault.entries).some((entry) => entry.workspaceId === workspaceId) + ? vault.generation + : 0; + } + + put(workspaceId: string, requirementId: string, value: string): void { + this.putMany(workspaceId, { [requirementId]: value }); + } + + putMany(workspaceId: string, values: Readonly>): void { + assertIdentifier(workspaceId, "workspace identifier"); + const items = Object.entries(values); + if (items.length === 0) throw new Error("At least one workspace secret is required."); + for (const [requirementId, value] of items) { + assertIdentifier(requirementId, "secret requirement identifier"); + const size = Buffer.byteLength(value, "utf8"); + if (size === 0) throw new Error("Workspace secrets cannot be empty."); + if (size > this.maxSecretBytes) throw new Error("Workspace secret exceeds the size limit."); + } + + const vault = this.readVault(); + const key = this.readKey(); + for (const [requirementId, value] of items) { + const iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", key, iv); + cipher.setAAD(this.additionalData(workspaceId, requirementId)); + const ciphertext = Buffer.concat([cipher.update(value, "utf8"), cipher.final()]); + vault.entries[entryKey(workspaceId, requirementId)] = { + workspaceId, + requirementId, + iv: iv.toString("base64"), + tag: cipher.getAuthTag().toString("base64"), + ciphertext: ciphertext.toString("base64"), + }; + } + vault.generation += 1; + this.writeVault(vault); + } + + forget(workspaceId: string, requirementId: string): void { + this.assertIds(workspaceId, requirementId); + const vault = this.readVault(); + const key = entryKey(workspaceId, requirementId); + if (vault.entries[key] === undefined) return; + delete vault.entries[key]; + vault.generation += 1; + this.writeVault(vault); + } + + materialize( + workspaceId: string, + requirementIds: readonly string[], + ): WorkspaceSecretMaterialization { + assertIdentifier(workspaceId, "workspace identifier"); + for (const requirementId of requirementIds) { + assertIdentifier(requirementId, "secret requirement identifier"); + } + + let directory: string | undefined; + let retained = false; + try { + const vault = this.readVault(); + const key = this.readKey(); + const workspaceEntries = requirementIds + .map((requirementId) => vault.entries[entryKey(workspaceId, requirementId)]) + .filter((entry): entry is EncryptedEntry => entry !== undefined) + .sort((left, right) => left.requirementId.localeCompare(right.requirementId)); + const materializationId = createHash("sha256") + .update(this.installationId) + .update("\0") + .update(workspaceId) + .update("\0") + .update(JSON.stringify(workspaceEntries)) + .digest("hex") + .slice(0, 24); + directory = join( + this.runtimeRoot, + `workspace-${createHash("sha256").update(workspaceId).digest("hex").slice(0, 16)}-${materializationId}`, + ); + const references = this.materializationReferences.get(directory) ?? 0; + if (references === 0) { + try { + mkdirSync(directory, { recursive: false, mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + rmSync(directory, { recursive: true, force: true }); + mkdirSync(directory, { recursive: false, mode: 0o700 }); + } + chmodSync(directory, 0o700); + } + const files = new Map(); + for (const requirementId of [...new Set(requirementIds)]) { + const entry = vault.entries[entryKey(workspaceId, requirementId)]; + if (entry === undefined) continue; + if (entry.workspaceId !== workspaceId || entry.requirementId !== requirementId) { + throw new Error(STORE_ERROR); + } + const decipher = createDecipheriv( + "aes-256-gcm", + key, + Buffer.from(entry.iv, "base64"), + ); + decipher.setAAD(this.additionalData(workspaceId, requirementId)); + decipher.setAuthTag(Buffer.from(entry.tag, "base64")); + const plaintext = Buffer.concat([ + decipher.update(Buffer.from(entry.ciphertext, "base64")), + decipher.final(), + ]); + const path = join(directory, createHash("sha256").update(requirementId).digest("hex")); + if (references === 0) { + const fd = openSync(path, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o400); + try { + fchmodSync(fd, 0o400); + writeFileSync(fd, plaintext); + fsyncSync(fd); + } finally { + plaintext.fill(0); + closeSync(fd); + } + } else { + plaintext.fill(0); + } + files.set(requirementId, path); + } + this.materializationReferences.set(directory, references + 1); + retained = true; + let released = false; + const leasedDirectory = directory; + return { + files, + release: () => { + if (released) return; + released = true; + const remaining = (this.materializationReferences.get(leasedDirectory) ?? 1) - 1; + if (remaining > 0) { + this.materializationReferences.set(leasedDirectory, remaining); + } else { + this.materializationReferences.delete(leasedDirectory); + rmSync(leasedDirectory, { recursive: true, force: true }); + } + }, + }; + } catch { + if (directory !== undefined && !retained && !this.materializationReferences.has(directory)) { + rmSync(directory, { recursive: true, force: true }); + } + throw new Error(STORE_ERROR); + } + } + + private assertIds(workspaceId: string, requirementId: string): void { + assertIdentifier(workspaceId, "workspace identifier"); + assertIdentifier(requirementId, "secret requirement identifier"); + } + + private additionalData(workspaceId: string, requirementId: string): Buffer { + return Buffer.from(`${this.installationId}\0${workspaceId}\0${requirementId}`, "utf8"); + } + + private ensureInitialized(): void { + try { + readFileSync(this.keyPath); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ENOENT") throw new Error(STORE_ERROR); + try { + atomicPrivateWrite(this.keyPath, randomBytes(32)); + } catch (writeError) { + if ((writeError as NodeJS.ErrnoException).code !== "EEXIST") throw new Error(STORE_ERROR); + } + } + try { + readFileSync(this.vaultPath); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ENOENT") throw new Error(STORE_ERROR); + atomicPrivateWrite(this.vaultPath, `${JSON.stringify(emptyVault())}\n`); + } + this.readKey(); + this.readVault(); + } + + private readKey(): Buffer { + try { + const key = readFileSync(this.keyPath); + if (key.length !== 32) throw new Error(STORE_ERROR); + chmodSync(this.keyPath, 0o600); + return key; + } catch { + throw new Error(STORE_ERROR); + } + } + + private readVault(): VaultDocument { + try { + const parsed = JSON.parse(readFileSync(this.vaultPath, "utf8")) as Partial; + if (parsed.version !== 1 || !Number.isSafeInteger(parsed.generation) || + parsed.generation! < 0 || typeof parsed.entries !== "object" || parsed.entries === null) { + throw new Error(STORE_ERROR); + } + for (const [key, entry] of Object.entries(parsed.entries)) { + if (entry === null || typeof entry !== "object" || + typeof entry.workspaceId !== "string" || typeof entry.requirementId !== "string" || + typeof entry.iv !== "string" || typeof entry.tag !== "string" || + typeof entry.ciphertext !== "string" || + key !== entryKey(entry.workspaceId, entry.requirementId)) { + throw new Error(STORE_ERROR); + } + } + chmodSync(this.vaultPath, 0o600); + return parsed as VaultDocument; + } catch { + throw new Error(STORE_ERROR); + } + } + + private writeVault(vault: VaultDocument): void { + try { + atomicPrivateWrite(this.vaultPath, `${JSON.stringify(vault)}\n`); + } catch { + throw new Error(STORE_ERROR); + } + } +} diff --git a/backend/src/workspaces/types.ts b/backend/src/workspaces/types.ts new file mode 100644 index 00000000..dd40c1fd --- /dev/null +++ b/backend/src/workspaces/types.ts @@ -0,0 +1,22 @@ +export interface WorkspaceRegistryConfig { + root: string; + remoteUrl?: string; + branch: string; + installationId: string; + secretRoots: readonly string[]; + /** Absolute runtime data root; when set, activation also syncs curated annotations per revision. */ + dataRoot?: string; + /** P6 Evidence materialization bounds; defaults are applied by the materializer. */ + maxEvidenceEntries?: number; + maxEvidenceBytes?: number; + maxEvidenceFileBytes?: number; + maxEvidencePathBytes?: number; + maxEvidenceManifestBytes?: number; +} + +export type WorkspaceErrorCode = + | "workspace_invalid" | "binding_missing" | "workspace_not_activatable" + | "workspace_stale" | "git_unavailable" | "git_auth_failed" | "git_non_fast_forward" + | "connector_unavailable" | "semantic_index_incompatible"; + +export type { WorkspaceV3 } from "./schema.js"; diff --git a/backend/test/annotations-sync.test.ts b/backend/test/annotations-sync.test.ts new file mode 100644 index 00000000..e93a4652 --- /dev/null +++ b/backend/test/annotations-sync.test.ts @@ -0,0 +1,116 @@ +import { chmodSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { syncAnnotations } from "../src/workspaces/annotations-sync.js"; + +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +function input(overrides: Partial<{ + dataRoot: string; workspaceId: string; commit: string; blobId: string; contents: Buffer; +}> = {}) { + return { + dataRoot: overrides.dataRoot ?? "", + workspaceId: overrides.workspaceId ?? "research", + commit: overrides.commit ?? "a".repeat(40), + blobId: overrides.blobId ?? "b".repeat(40), + contents: overrides.contents ?? Buffer.from("tables: {}\n"), + }; +} + +test("writes the revision-qualified annotations file and ownership manifest", () => { + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-annotations-sync-")); + temporaryRoots.push(dataRoot); + const target = input({ dataRoot }); + + const result = syncAnnotations(target); + + expect(readFileSync(result.path, "utf8")).toBe("tables: {}\n"); + const manifest = JSON.parse(readFileSync(result.manifestPath, "utf8")); + expect(manifest).toMatchObject({ + workspace: "research", + commit: "a".repeat(40), + blobId: "b".repeat(40), + contentDigest: result.contentDigest, + destination: result.path, + }); + expect(result.path).toContain("/revisions/" + "a".repeat(40) + "/artifacts/mschema/annotations.yaml"); + expect(lstatSync(result.path).mode & 0o777).toBe(0o400); +}); + +test("is idempotent for the exact same blob and manifest", () => { + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-annotations-sync-")); + temporaryRoots.push(dataRoot); + const target = input({ dataRoot }); + + expect(syncAnnotations(target)).toEqual(syncAnnotations(target)); +}); + +test("fails closed when the destination was tampered", () => { + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-annotations-sync-")); + temporaryRoots.push(dataRoot); + const target = input({ dataRoot }); + syncAnnotations(target); + + const dest = join(dataRoot, "sessions", "research", "revisions", "a".repeat(40), "artifacts", "mschema", "annotations.yaml"); + chmodSync(dest, 0o600); + writeFileSync(dest, "tampered\n"); + + expect(() => syncAnnotations(target)).toThrow(); +}); + +test("fails closed when the ownership manifest does not match", () => { + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-annotations-sync-")); + temporaryRoots.push(dataRoot); + const target = input({ dataRoot }); + syncAnnotations(target); + + const manifestPath = join(dataRoot, "sessions", "research", "revisions", "a".repeat(40), "artifacts", "mschema", "annotations.ownership.json"); + writeFileSync(manifestPath, JSON.stringify({ workspace: "other", commit: "c".repeat(40), blobId: "d".repeat(40), contentDigest: "sha256:x", destination: "/other" })); + + expect(() => syncAnnotations(target)).toThrow(); +}); + +test("writes different revisions to different directories", () => { + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-annotations-sync-")); + temporaryRoots.push(dataRoot); + + const first = syncAnnotations(input({ dataRoot, commit: "a".repeat(40) })); + const second = syncAnnotations(input({ dataRoot, commit: "b".repeat(40) })); + + expect(first.path).not.toBe(second.path); + expect(readFileSync(second.path, "utf8")).toBe("tables: {}\n"); +}); + +test("rejects a symlink destination and malformed inputs before writing", () => { + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-annotations-sync-")); + temporaryRoots.push(dataRoot); + const root = join(dataRoot, "sessions", "research", "revisions", "a".repeat(40), "artifacts"); + mkdirSync(root, { recursive: true }); + symlinkSync(join(root, "mschema-target"), join(root, "mschema")); + mkdirSync(join(root, "mschema-target")); + + expect(() => syncAnnotations(input({ dataRoot }))).toThrow(); + expect(() => syncAnnotations(input({ dataRoot: "relative" }))).toThrow(); + expect(() => syncAnnotations(input({ workspaceId: "../x" }))).toThrow(); + expect(() => syncAnnotations(input({ commit: "HEAD" }))).toThrow(); + expect(() => syncAnnotations(input({ blobId: "not-hex" }))).toThrow(); + expect(() => syncAnnotations(input({ contents: Buffer.from("tables: [bad]\n") }))).toThrow(); +}); + +test("does not follow a symlinked destination when verifying", () => { + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-annotations-sync-")); + temporaryRoots.push(dataRoot); + const target = input({ dataRoot }); + syncAnnotations(target); + + const dest = join(dataRoot, "sessions", "research", "revisions", "a".repeat(40), "artifacts", "mschema", "annotations.yaml"); + rmSync(dest, { force: true }); + symlinkSync("/etc/hosts", dest); + + expect(() => syncAnnotations(target)).toThrow(); +}); diff --git a/backend/test/app-auth-mode.test.ts b/backend/test/app-auth-mode.test.ts new file mode 100644 index 00000000..7b4515dd --- /dev/null +++ b/backend/test/app-auth-mode.test.ts @@ -0,0 +1,90 @@ +import { expect, test, vi } from "vitest"; +import { chmodSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; + +test("configured OIDC advertises login but fails closed without its runtime client secret", async () => { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-app-oidc-mode-")); + chmodSync(directory, 0o700); + const file = join(directory, "auth.yaml"); + writeFileSync(file, stringify({ + version: 1, mode: "oidc", publicUrl: "https://thothii.example.org", + oidc: { + issuer: "https://authentik.example.org/application/o/thothii/", clientId: "thothii", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", scopes: ["openid"], groupsClaim: "groups", + }, + groupCatalog: { driver: "authentik", baseUrl: "https://authentik.example.org", apiTokenRef: "THT_AUTHENTIK_API_TOKEN" }, + authorization: { groupRoles: { "TOT Users": ["user"], "TOT Admin": ["admin"] } }, + }), { encoding: "utf8", mode: 0o600 }); + chmodSync(file, 0o600); + try { + const app = buildApp(loadConfig({ THT_AUTH_CONFIG_FILE: file, THT_AUTH_STATE_ROOT: join(directory, "auth-state") })); + try { + expect((await app.inject({ method: "GET", url: "/auth/config" })).json()) + .toEqual({ mode: "oidc", localLogin: false, oidcLogin: true }); + const placeholder = await app.inject({ method: "GET", url: "/auth/oidc/login" }); + expect(placeholder.statusCode).toBe(503); + expect(placeholder.json()).toEqual({ code: "auth_unavailable", error: "Authentication is unavailable" }); + } finally { + await app.close(); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("configured OIDC initializes login from the literal secret bundle without an environment duplicate", async () => { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-app-oidc-bundle-")); + chmodSync(directory, 0o700); + const file = join(directory, "auth.yaml"); + const bundle = join(directory, "thothii.secrets"); + const clientSecret = "bundle-only-oidc-client-secret"; + writeFileSync(file, stringify({ + version: 1, mode: "oidc", publicUrl: "https://thothii.example.org", + oidc: { + issuer: "https://authentik.example.org/application/o/thothii/", clientId: "thothii", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", scopes: ["openid"], groupsClaim: "groups", + }, + groupCatalog: { driver: "authentik", baseUrl: "https://authentik.example.org", apiTokenRef: "THT_AUTHENTIK_API_TOKEN" }, + authorization: { groupRoles: { "TOT Users": ["user"], "TOT Admin": ["admin"] } }, + }), { encoding: "utf8", mode: 0o600 }); + writeFileSync(bundle, `THT_OIDC_CLIENT_SECRET=${clientSecret}\nTHT_AUTHENTIK_API_TOKEN=bundle-only-authentik-token\n`, { + encoding: "utf8", mode: 0o600, + }); + chmodSync(file, 0o600); + chmodSync(bundle, 0o600); + const original = process.env.THT_OIDC_CLIENT_SECRET; + delete process.env.THT_OIDC_CLIENT_SECRET; + const oidcProtocolFactory = vi.fn((input: { clientSecret: string }) => ({ + authorizationUrl: async ({ state }: { state: string }) => new URL(`https://authentik.example.org/authorize?state=${state}`), + callback: async () => { throw new Error("callback is outside this login-start regression"); }, + diagnose: async () => undefined, + })); + const authSessionStore = { + createOidcState: async () => ({ + state: "s".repeat(43), + record: { version: 1 }, + }), + }; + try { + const app = buildApp(loadConfig({ + NODE_ENV: "test", THT_AUTH_CONFIG_FILE: file, THT_AUTH_STATE_ROOT: join(directory, "auth-state"), + THT_SECRETS_FILE: bundle, + }), { oidcProtocolFactory, authSessionStore } as never); + try { + const response = await app.inject({ method: "GET", url: "/auth/oidc/login" }); + expect(response.statusCode).toBe(302); + expect(oidcProtocolFactory).toHaveBeenCalledWith(expect.objectContaining({ clientSecret })); + expect(process.env.THT_OIDC_CLIENT_SECRET).toBeUndefined(); + } finally { + await app.close(); + } + } finally { + if (original === undefined) delete process.env.THT_OIDC_CLIENT_SECRET; + else process.env.THT_OIDC_CLIENT_SECRET = original; + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/backend/test/auth-config.test.ts b/backend/test/auth-config.test.ts new file mode 100644 index 00000000..153755db --- /dev/null +++ b/backend/test/auth-config.test.ts @@ -0,0 +1,364 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { + chmodSync, + linkSync, + mkdirSync, + mkdtempSync, + realpathSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { stringify } from "yaml"; +import { + createAuthenticationConfigProvider, + loadAuthenticationConfig, + rolesToPermissions, +} from "../src/auth/config.js"; + +const readHook = vi.hoisted(() => ({ callback: undefined as undefined | (() => void) })); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readSync: (...args: any[]) => { + const result = (actual.readSync as any)(...args); + const callback = readHook.callback; + readHook.callback = undefined; + callback?.(); + return result; + }, + }; +}); + +const directories: string[] = []; + +afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +function writeFixture(value: unknown): string { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-config-")); + chmodSync(directory, 0o700); + directories.push(directory); + const file = join(directory, "auth.yaml"); + writeFileSync(file, stringify(value), { encoding: "utf8", mode: 0o600 }); + chmodSync(file, 0o600); + return file; +} + +function localConfig(overrides: Record = {}): Record { + return { + version: 1, + mode: "local", + publicUrl: "http://127.0.0.1:8080", + local: { usersFile: "users.yaml" }, + ...overrides, + }; +} + +function oidcConfig(overrides: Record = {}): Record { + return { + version: 1, + mode: "oidc", + publicUrl: "https://thothii.example.org", + oidc: { + issuer: "https://authentik.example.org/application/o/thothii/", + clientId: "thothii", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", + scopes: ["openid", "profile", "email"], + groupsClaim: "groups", + }, + groupCatalog: { + driver: "authentik", + baseUrl: "https://authentik.example.org", + apiTokenRef: "THT_AUTHENTIK_API_TOKEN", + }, + authorization: { + groupRoles: { + "TOT Users": ["user"], + "TOT Admin": ["admin"], + }, + }, + ...overrides, + }; +} + +test("loads local configuration with the specified default lifetimes", () => { + const loaded = loadAuthenticationConfig(writeFixture(localConfig())); + + expect(loaded.value).toMatchObject({ + mode: "local", + publicUrl: "http://127.0.0.1:8080", + session: { + regularTtlSeconds: 43_200, + regularIdleSeconds: 7_200, + rememberTtlSeconds: 2_592_000, + rememberIdleSeconds: 604_800, + oidcTtlSeconds: 28_800, + }, + local: { usersFile: "users.yaml" }, + }); + expect(loaded.revision).toMatch(/^[a-f0-9]{64}$/); +}); + +test("loads the fixed OIDC secret references and preserves exact group names", () => { + const loaded = loadAuthenticationConfig(writeFixture(oidcConfig())); + + expect(loaded.value).toMatchObject({ + mode: "oidc", + oidc: { clientSecretRef: "THT_OIDC_CLIENT_SECRET", groupsClaim: "groups" }, + groupCatalog: { driver: "authentik", apiTokenRef: "THT_AUTHENTIK_API_TOKEN" }, + authorization: { + groupRoles: { "TOT Users": ["user"], "TOT Admin": ["admin"] }, + }, + }); + expect(loaded.value.authorization.groupRoles["tot users"]).toBeUndefined(); +}); + +test.each([ + ["unknown root key", localConfig({ unexpected: true })], + ["relative users file", localConfig({ local: { usersFile: "../users.yaml" } })], + ["OIDC without groups claim", oidcConfig({ oidc: { + issuer: "https://authentik.example.org/application/o/thothii/", clientId: "thothii", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", scopes: ["openid"], + } })], + ["OIDC without admin mapping", oidcConfig({ authorization: { groupRoles: {} } })], + ["HTTP non-loopback public URL", localConfig({ publicUrl: "http://thoth.example" })], +])("rejects %s", (_label, value) => { + expect(() => loadAuthenticationConfig(writeFixture(value))).toThrow("authentication configuration is invalid"); +}); + +test.each([ + "http://127.0.0.1:8787", + "http://127.255.255.254:8787", + "http://[::1]:8787", +])("accepts the literal loopback HTTP OIDC exception %s", (publicUrl) => { + expect(loadAuthenticationConfig(writeFixture(oidcConfig({ publicUrl }))).value.mode).toBe("oidc"); +}); + +test.each([ + "http://localhost:8787", + "http://loopback.example.test:8787", + "http://user@127.0.0.1:8787", + "http://127.1:8787", + "http://127.0.0.01:8787", + "http://0177.0.0.1:8787", + "http://0x7f000001:8787", + "http://2130706433:8787", + "http://[::ffff:127.0.0.1]:8787", + "http://128.0.0.1:8787", +])("rejects non-canonical or non-loopback HTTP public URL %s", (publicUrl) => { + expect(() => loadAuthenticationConfig(writeFixture(oidcConfig({ publicUrl })))) + .toThrow("authentication configuration is invalid"); +}); + +test("rejects unknown roles and requires exactly one admin group", () => { + expect(() => loadAuthenticationConfig(writeFixture(oidcConfig({ + authorization: { groupRoles: { "TOT Users": ["user", "operator"], "TOT Admin": ["admin"] } }, + })))).toThrow("authentication configuration is invalid"); + expect(() => loadAuthenticationConfig(writeFixture(oidcConfig({ + authorization: { groupRoles: { "TOT Users": ["user"], "TOT Admin": ["admin"], "Other Admin": ["admin"] } }, + })))).toThrow("authentication configuration is invalid"); +}); + +test("caps configured group mappings at the OIDC direct-groups bound", () => { + const mappings = Object.fromEntries(Array.from({ length: 128 }, (_unused, index) => [ + `Mapped Group ${String(index).padStart(3, "0")}`, + index === 0 ? ["admin"] : ["user"], + ])); + expect(loadAuthenticationConfig(writeFixture(oidcConfig({ authorization: { groupRoles: mappings } }))).value.mode) + .toBe("oidc"); + mappings["Mapped Group overflow"] = ["user"]; + expect(() => loadAuthenticationConfig(writeFixture(oidcConfig({ authorization: { groupRoles: mappings } })))) + .toThrow("authentication configuration is invalid"); +}); + +test("roles collapse duplicates and admin contains all administrative permissions", () => { + expect(rolesToPermissions(["admin", "admin", "user"])).toEqual([ + "session.use", + "session.read_all", + "session.manage_all", + "settings.manage", + "workspace.manage", + "workspace.secrets.manage", + "pi.manage", + "auth.diagnostics.read", + ]); + expect(() => rolesToPermissions(["unknown"] as never)).toThrow("authentication configuration is invalid"); +}); + +test("rejects duplicate YAML keys and does not leak invalid reference values", () => { + const duplicate = writeFixture(`version: 1\nmode: local\nmode: oidc\npublicUrl: http://127.0.0.1:8787\nlocal:\n usersFile: users.yaml\n`); + expect(() => loadAuthenticationConfig(duplicate)).toThrow("authentication configuration is invalid"); + + const referenceCanary = "never-load-or-emit-this-secret"; + const invalid = writeFixture(oidcConfig({ oidc: { + issuer: "https://authentik.example.org/application/o/thothii/", clientId: "thothii", + clientSecretRef: referenceCanary, scopes: ["openid"], groupsClaim: "groups", + } })); + try { + loadAuthenticationConfig(invalid); + } catch (error) { + expect(String(error)).not.toContain(referenceCanary); + } +}); + +test("canonical group map order produces one stable revision", () => { + const first = writeFixture(oidcConfig()); + const reordered = writeFixture(oidcConfig({ + authorization: { groupRoles: { "TOT Admin": ["admin"], "TOT Users": ["user"] } }, + })); + expect(loadAuthenticationConfig(first).revision).toBe(loadAuthenticationConfig(reordered).revision); +}); + +test("canonical revisions use code-unit ordering for non-ASCII group names", () => { + const loaded = loadAuthenticationConfig(writeFixture(oidcConfig({ + authorization: { groupRoles: { "Ångström users": ["user"], "Zebra admins": ["admin"] } }, + }))); + const canonicalize = (value: unknown): unknown => Array.isArray(value) + ? value.map(canonicalize) + : value && typeof value === "object" + ? Object.fromEntries(Object.entries(value as Record) + .sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0) + .map(([key, nested]) => [key, canonicalize(nested)])) + : value; + const expected = createHash("sha256").update(JSON.stringify(canonicalize(loaded.value))).digest("hex"); + + expect(loaded.revision).toBe(expected); +}); + +test("provider reloads after an atomic configuration replacement", () => { + const file = writeFixture(localConfig()); + const provider = createAuthenticationConfigProvider(file); + const original = provider.current(); + const replacement = `${file}.replacement`; + writeFileSync(replacement, stringify(localConfig({ publicUrl: "http://127.0.0.1:9999" })), { encoding: "utf8", mode: 0o600 }); + chmodSync(replacement, 0o600); + renameSync(replacement, file); + + const reloaded = provider.current(); + expect(reloaded.revision).not.toBe(original.revision); + expect(reloaded.value.publicUrl).toBe("http://127.0.0.1:9999"); +}); + +test("loads and reloads Windows auth.yaml through the production storage bridge boundary", () => { + const originalPlatform = process.platform; + const windowsPath = "C:\\ProgramData\\ThothII\\auth\\auth.yaml"; + let source = stringify(localConfig()); + const readAuthConfig = vi.fn(() => Buffer.from(source)); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const options = { windowsStorageBridge: { readAuthConfig } as never }; + expect(loadAuthenticationConfig(windowsPath, options).value.publicUrl).toBe("http://127.0.0.1:8080"); + const provider = createAuthenticationConfigProvider(windowsPath, options); + const original = provider.current(); + source = stringify(localConfig({ publicUrl: "http://127.0.0.1:9999" })); + const reloaded = provider.current(); + + expect(reloaded.value.publicUrl).toBe("http://127.0.0.1:9999"); + expect(reloaded.revision).not.toBe(original.revision); + expect(readAuthConfig).toHaveBeenCalledTimes(3); + expect(readAuthConfig).toHaveBeenCalledWith(windowsPath); + } finally { + Object.defineProperty(process, "platform", { configurable: true, value: originalPlatform }); + } +}); + +test("provider retries when replacement occurs between its read and cache identity check", () => { + const file = writeFixture(localConfig()); + const replacement = `${file}.replacement`; + writeFileSync(replacement, stringify(localConfig({ publicUrl: "http://127.0.0.1:9999" })), { encoding: "utf8", mode: 0o600 }); + chmodSync(replacement, 0o600); + const provider = createAuthenticationConfigProvider(file); + readHook.callback = () => renameSync(replacement, file); + + expect(provider.current().value.publicUrl).toBe("http://127.0.0.1:9999"); +}); + +test.each(["symlink", "hard link", "mode wider than 0600", "non-private parent"])( + "rejects auth.yaml with unsafe %s storage", + (kind) => { + const file = writeFixture(localConfig()); + if (kind === "symlink") { + const target = `${file}.target`; + renameSync(file, target); + symlinkSync(target, file); + } else if (kind === "hard link") linkSync(file, `${file}.link`); + else if (kind === "mode wider than 0600") chmodSync(file, 0o640); + else chmodSync(dirname(file), 0o750); + + expect(() => loadAuthenticationConfig(file)).toThrow("authentication configuration is invalid"); + }, +); + +test("rejects auth.yaml beneath a symlinked parent without exposing its path", () => { + const outer = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-config-parent-")); + chmodSync(outer, 0o700); + directories.push(outer); + const realDirectory = join(outer, "real-auth"); + const linkedDirectory = join(outer, "linked-auth"); + mkdirSync(realDirectory, { mode: 0o700 }); + chmodSync(realDirectory, 0o700); + const realFile = join(realDirectory, "auth.yaml"); + writeFileSync(realFile, stringify(localConfig()), { encoding: "utf8", mode: 0o600 }); + chmodSync(realFile, 0o600); + symlinkSync(realDirectory, linkedDirectory); + const unsafePath = join(linkedDirectory, "auth.yaml"); + + try { + loadAuthenticationConfig(unsafePath); + throw new Error("unsafe auth configuration unexpectedly loaded"); + } catch (error) { + expect((error as Error).message).toBe("authentication configuration is invalid"); + expect(String(error)).not.toContain(unsafePath); + } +}); + +test("rejects auth.yaml when its owner is not the runtime owner", () => { + const geteuid = process.geteuid; + if (!geteuid) return; + const owner = geteuid(); + const file = writeFixture(localConfig()); + const spy = vi.spyOn(process, "geteuid").mockReturnValue(owner + 1); + try { + expect(() => loadAuthenticationConfig(file)).toThrow("authentication configuration is invalid"); + } finally { + spy.mockRestore(); + } +}); + +test("provider redacts an absent canonical path", () => { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-config-absent-")); + chmodSync(directory, 0o700); + directories.push(directory); + const missing = join(directory, "private-path-UNIQUE-4K6.yaml"); + + try { + createAuthenticationConfigProvider(missing).current(); + throw new Error("missing authentication configuration unexpectedly loaded"); + } catch (error) { + expect((error as Error).message).toBe("authentication configuration is invalid"); + expect(String(error)).not.toContain(missing); + } +}); + +test("rejects a path replacement during the bounded auth.yaml read", () => { + const file = writeFixture(localConfig()); + const replacement = `${file}.replacement`; + writeFileSync(replacement, stringify(localConfig({ publicUrl: "http://127.0.0.1:9999" })), { encoding: "utf8", mode: 0o600 }); + chmodSync(replacement, 0o600); + readHook.callback = () => renameSync(replacement, file); + + expect(() => loadAuthenticationConfig(file)).toThrow("authentication configuration is invalid"); +}); + +test("rejects input larger than one MiB", () => { + const file = writeFixture(`${"#".repeat(1024 * 1024)}\n`); + expect(() => loadAuthenticationConfig(file)).toThrow("authentication configuration is invalid"); +}); diff --git a/backend/test/auth-cors.test.ts b/backend/test/auth-cors.test.ts new file mode 100644 index 00000000..79c8bb7a --- /dev/null +++ b/backend/test/auth-cors.test.ts @@ -0,0 +1,54 @@ +import { afterAll, beforeAll, expect, test } from "vitest"; +import { createLocalAuthFixture, type LocalAuthFixture, localPublicUrl } from "./auth-test-fixtures.js"; + +let fixture: LocalAuthFixture; + +beforeAll(async () => { + fixture = await createLocalAuthFixture(); +}); + +afterAll(async () => { + await fixture.close(); +}); + +test("credentialed CORS permits only the current configured public origin", async () => { + const allowedPreflight = await fixture.app.inject({ + method: "OPTIONS", url: "/me", + headers: { origin: localPublicUrl, "access-control-request-method": "GET" }, + }); + expect(allowedPreflight.statusCode).toBe(204); + expect(allowedPreflight.headers["access-control-allow-origin"]).toBe(localPublicUrl); + expect(allowedPreflight.headers["access-control-allow-credentials"]).toBe("true"); + + const canonicalPreflight = await fixture.app.inject({ + method: "OPTIONS", url: "/me", + headers: { origin: "HTTP://127.0.0.1:8787", "access-control-request-method": "GET" }, + }); + expect(canonicalPreflight.statusCode).toBe(204); + expect(canonicalPreflight.headers["access-control-allow-origin"]).toBe(localPublicUrl); + + const attackerPreflight = await fixture.app.inject({ + method: "OPTIONS", url: "/me", + headers: { origin: "https://attacker.example.test", "access-control-request-method": "GET" }, + }); + expect(attackerPreflight.headers["access-control-allow-origin"]).toBeUndefined(); + expect(attackerPreflight.headers["access-control-allow-credentials"]).toBeUndefined(); + + const allowed = await fixture.app.inject({ + method: "GET", url: "/me", headers: { cookie: fixture.cookie, origin: localPublicUrl }, + }); + expect(allowed.statusCode).toBe(200); + expect(allowed.headers["access-control-allow-origin"]).toBe(localPublicUrl); + expect(allowed.headers["access-control-allow-credentials"]).toBe("true"); + + const attacker = await fixture.app.inject({ + method: "GET", url: "/me", headers: { cookie: fixture.cookie, origin: "https://attacker.example.test" }, + }); + expect(attacker.statusCode).toBe(200); + expect(attacker.headers["access-control-allow-origin"]).toBeUndefined(); + expect(attacker.headers["access-control-allow-credentials"]).toBeUndefined(); + + const nonBrowser = await fixture.app.inject({ method: "GET", url: "/me", headers: { cookie: fixture.cookie } }); + expect(nonBrowser.statusCode).toBe(200); + expect(nonBrowser.headers["access-control-allow-origin"]).toBeUndefined(); +}); diff --git a/backend/test/auth-csrf.test.ts b/backend/test/auth-csrf.test.ts new file mode 100644 index 00000000..e926616e --- /dev/null +++ b/backend/test/auth-csrf.test.ts @@ -0,0 +1,79 @@ +import { afterEach, expect, test } from "vitest"; +import { deriveCsrfToken } from "../src/auth/csrf.js"; +import { createLocalAuthFixture, localPublicUrl } from "./auth-test-fixtures.js"; + +const cleanups: Array<() => Promise> = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); + +async function createApp() { + const fixture = await createLocalAuthFixture(); + cleanups.push(() => fixture.close()); + return fixture; +} + +test("derived CSRF tokens are deterministic per opaque cookie and are never the cookie token", async () => { + const { cookie, csrfToken } = await createApp(); + const token = cookie.split("=", 2)[1] ?? ""; + expect(deriveCsrfToken(token)).toBe(csrfToken); + expect(csrfToken).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(csrfToken).not.toBe(token); +}); + +test("cookie-authenticated state changes require an exact Origin, Fetch Metadata when present, and CSRF token", async () => { + const { app, cookie, csrfToken } = await createApp(); + const cases = [ + { headers: { cookie, origin: localPublicUrl, "sec-fetch-site": "same-origin" } }, + { headers: { cookie, origin: "http://127.0.0.1:8788", "sec-fetch-site": "same-origin", "x-thothii-csrf": csrfToken } }, + { headers: { cookie, origin: localPublicUrl, "sec-fetch-site": "cross-site", "x-thothii-csrf": csrfToken } }, + { headers: { cookie, origin: localPublicUrl, "x-thothii-csrf": csrfToken.slice(0, -1) } }, + ]; + + for (const request of cases) { + const response = await app.inject({ method: "POST", url: "/auth/logout", ...request }); + expect(response.statusCode).toBe(403); + expect(response.json()).toEqual({ code: "csrf_failed", error: "Request origin validation failed" }); + } +}); + +test("duplicate or malformed CSRF and session-cookie headers fail closed", async () => { + const { app, cookie, csrfToken } = await createApp(); + const duplicateCsrf = await app.inject({ + method: "POST", url: "/auth/logout", + headers: { cookie, origin: localPublicUrl, "x-thothii-csrf": `${csrfToken}, ${csrfToken}` }, + }); + const duplicateCookie = await app.inject({ + method: "POST", url: "/auth/logout", + headers: { cookie: `${cookie}; ${cookie}`, origin: localPublicUrl, "x-thothii-csrf": csrfToken }, + }); + const malformedCookie = await app.inject({ + method: "POST", url: "/auth/logout", + headers: { cookie: "thothii_session=not-a-token", origin: localPublicUrl, "x-thothii-csrf": csrfToken }, + }); + const oversizedCsrf = await app.inject({ + method: "POST", url: "/auth/logout", + headers: { cookie, origin: localPublicUrl, "x-thothii-csrf": "x".repeat(4097) }, + }); + const oversizedCookie = await app.inject({ + method: "POST", url: "/auth/logout", + headers: { cookie: `${cookie}; padding=${"x".repeat(4097)}`, origin: localPublicUrl, "x-thothii-csrf": csrfToken }, + }); + + expect(duplicateCsrf.statusCode).toBe(403); + expect(duplicateCookie.statusCode).toBe(401); + expect(malformedCookie.statusCode).toBe(401); + expect(oversizedCsrf.statusCode).toBe(403); + expect(oversizedCookie.statusCode).toBe(401); +}); + +test("an unauthenticated state-changing application route cannot bypass the central boundary", async () => { + const { app } = await createApp(); + const response = await app.inject({ + method: "POST", url: "/sessions", headers: { origin: localPublicUrl, "x-thothii-csrf": "x".repeat(43) }, + payload: { question: "must not reach a session handler" }, + }); + expect(response.statusCode).toBe(401); + expect(response.json()).toEqual({ code: "authentication_required", error: "Authentication is required" }); +}); diff --git a/backend/test/auth-diagnostic-command.test.ts b/backend/test/auth-diagnostic-command.test.ts new file mode 100644 index 00000000..8f87f6cb --- /dev/null +++ b/backend/test/auth-diagnostic-command.test.ts @@ -0,0 +1,291 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import type { AuthDiagnoser, AuthDiagnostics } from "../src/auth/diagnostics.js"; +import { + configuredSecretValues, + createConfiguredAuthDiagnoser, + runConfiguredDiagnosticCommand, + runDiagnosticCommand, +} from "../src/auth/diagnostic-command.js"; +import type { AppConfig } from "../src/config.js"; +import type { LoadedAuthConfig } from "../src/auth/types.js"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +function secretBundle(contents: string): string { + const root = mkdtempSync(join(tmpdir(), "thothii-diagnostic-command-")); + roots.push(root); + const file = join(root, "secrets"); + writeFileSync(file, contents, { mode: 0o600 }); + chmodSync(file, 0o600); + return file; +} + +function configuredOidc(groups: readonly string[]): AuthDiagnoser { + const loaded: LoadedAuthConfig = { + revision: "a".repeat(64), + sourcePath: "/safe/auth.yaml", + value: { + version: 1, + mode: "oidc", + publicUrl: "https://thothii.example.test", + session: { regularTtlSeconds: 1, regularIdleSeconds: 1, rememberTtlSeconds: 1, rememberIdleSeconds: 1, oidcTtlSeconds: 1 }, + oidc: { + issuer: "https://issuer.example.test", clientId: "thothii", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", scopes: ["openid"], groupsClaim: "groups", + }, + groupCatalog: { + driver: "authentik", baseUrl: "https://authentik.example.test", + apiTokenRef: "THT_AUTHENTIK_API_TOKEN", + }, + authorization: { groupRoles: { "Thoth Users": ["user"], "Thoth Administrators": ["admin"] } }, + }, + }; + const config = { + authMode: "oidc", + authStateRoot: "/safe/auth-state", + authentication: { current: () => loaded }, + secretsFile: secretBundle("THT_OIDC_CLIENT_SECRET=oidc-secret\nTHT_AUTHENTIK_API_TOKEN=catalog-secret\n"), + secretFiles: {}, + } as AppConfig; + return createConfiguredAuthDiagnoser(config, { + sessionRootValidator: async () => undefined, + groupCatalog: () => ({ verifyConfiguredGroups: async () => [] }), + oidcProtocol: () => ({ + diagnose: async () => undefined, + authorizationUrl: async () => new URL("https://issuer.example.test/authorize"), + callback: async () => { throw new Error("not used"); }, + verifyDeviceFlow: async () => ({ + issuer: "https://issuer.example.test", subject: "subject", groups, + tokenExpiresAt: new Date(Date.now() + 60_000), + }), + }), + } as any); +} + +const failure: AuthDiagnostics = { + ready: false, + mode: "oidc", + checks: [{ + level: "error", + code: "oidc_mapped_group_missing", + field: "Thoth Administrators", + message: "A configured authorization group does not exist: command-secret-sentinel.", + }], +}; + +test("writes one redacted JSON diagnostic report and uses a failing diagnostic exit status", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + const diagnoser: AuthDiagnoser = { inspect: vi.fn(async () => failure) }; + + const exitCode = await runDiagnosticCommand(["--json"], { + diagnoser, + secretValues: ["command-secret-sentinel"], + stdout: (line) => stdout.push(line), + stderr: (line) => stderr.push(line), + }); + + expect(exitCode).toBe(1); + expect(diagnoser.inspect).toHaveBeenCalledWith({ live: true }); + expect(stderr).toEqual([]); + expect(stdout).toHaveLength(1); + expect(JSON.parse(stdout[0])).toEqual({ + ...failure, + checks: [{ + ...failure.checks[0], + message: "A configured authorization group does not exist: [REDACTED].", + }], + }); + expect(stdout.join("\n")).not.toContain("command-secret-sentinel"); +}); + +test("delegates an interactive diagnostic to OIDC and keeps its device prompt on stderr", async () => { + const stdout: string[] = []; + const stderr: string[] = []; + const ready: AuthDiagnostics = { + ready: true, + mode: "oidc", + checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }], + }; + const diagnoser: AuthDiagnoser = { + inspect: vi.fn(async (options) => { + options.presentDeviceCode?.("https://issuer.example.test/device", "ABCD-EFGH"); + return ready; + }), + }; + + const exitCode = await runDiagnosticCommand(["--json", "--interactive"], { + diagnoser, + stdout: (line) => stdout.push(line), + stderr: (line) => stderr.push(line), + }); + + expect(exitCode).toBe(0); + expect(diagnoser.inspect).toHaveBeenCalledWith(expect.objectContaining({ live: true, interactive: true })); + expect(stderr).toEqual(["Open https://issuer.example.test/device and enter code ABCD-EFGH"]); + expect(JSON.parse(stdout.join(""))).toEqual(ready); +}); + +test("fails a direct-group identity that maps to zero Thoth roles without exposing provider groups", async () => { + const unmapped = "Unmapped Provider Group SECRET-SENTINEL"; + const report = await configuredOidc([unmapped]).inspect({ + live: true, + interactive: true, + presentDeviceCode: () => undefined, + }); + + expect(report).toEqual({ + ready: false, + mode: "oidc", + checks: [{ + level: "error", + code: "oidc_groups_claim_invalid", + message: "The OIDC device-flow identity could not be validated.", + }], + }); + expect(JSON.stringify(report)).not.toContain(unmapped); +}); + +test("accepts a direct-group identity with at least one exact configured role mapping", async () => { + const report = await configuredOidc(["Thoth Users", "Unmapped Provider Group"]).inspect({ + live: true, + interactive: true, + presentDeviceCode: () => undefined, + }); + + expect(report).toMatchObject({ ready: true, checks: [{ code: "auth_ready" }] }); + expect(JSON.stringify(report)).not.toContain("Unmapped Provider Group"); +}); + +test("redacts every parsed mounted secret before writing an interactive prompt", async () => { + const modelSecret = "model-secret-SENTINEL"; + const deviceCode = "DEVICE-CODE-SENTINEL"; + const legacyVectorSecret = "legacy-vector-secret-SENTINEL"; + const config = { + secretsFile: secretBundle([ + `THT_MODEL_API_KEY=${modelSecret}`, + `THT_DWH_API_KEY=${deviceCode}`, + "THT_OIDC_CLIENT_SECRET=oidc-secret", + "THT_AUTHENTIK_API_TOKEN=catalog-secret", + "", + ].join("\n")), + secretFiles: { THT_VEC_API_KEY_SECRET_FILE: secretBundle(legacyVectorSecret) }, + } as AppConfig; + const secrets = configuredSecretValues(config); + const stdout: string[] = []; + const stderr: string[] = []; + + await runDiagnosticCommand(["--json", "--interactive"], { + diagnoser: { + inspect: async (request) => { + request.presentDeviceCode?.( + `https://issuer.example.test/device/${modelSecret}?legacy=${legacyVectorSecret}`, + deviceCode, + ); + return { ready: true, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }] }; + }, + }, + secretValues: secrets, + stdout: (line) => stdout.push(line), + stderr: (line) => stderr.push(line), + }); + + expect(secrets).toEqual(expect.arrayContaining([ + modelSecret, deviceCode, legacyVectorSecret, "oidc-secret", "catalog-secret", + ])); + expect(stdout.join("") + stderr.join("")).not.toContain(modelSecret); + expect(stdout.join("") + stderr.join("")).not.toContain(deviceCode); + expect(stdout.join("") + stderr.join("")).not.toContain(legacyVectorSecret); +}); + +test("includes every nested Pi authentication scalar in the direct-command redaction corpus", () => { + const piApiKey = "pi-api-key-SENTINEL"; + const piAccessToken = "pi-access-token-SENTINEL"; + const piAuthFile = secretBundle(JSON.stringify({ + providers: { + anthropic: { + key: piApiKey, + oauth: { access: piAccessToken, expires: 1_800_000_000 }, + }, + }, + })); + const config = { + secretsFile: secretBundle("THT_MODEL_API_KEY=model-secret\n"), + secretFiles: {}, + piAuthFile, + } as AppConfig; + + expect(configuredSecretValues(config)).toEqual(expect.arrayContaining([ + "model-secret", piApiKey, piAccessToken, + ])); +}); + +test("fails closed when any declared direct-command secret source cannot be loaded", () => { + const root = mkdtempSync(join(tmpdir(), "thothii-diagnostic-command-missing-")); + roots.push(root); + const config = { + secretsFile: secretBundle("THT_MODEL_API_KEY=model-secret\n"), + secretFiles: {}, + piAuthFile: join(root, "missing-pi-auth.json"), + } as AppConfig; + + expect(() => configuredSecretValues(config)).toThrow("diagnostic secret corpus is unavailable"); +}); + +test("suppresses interactive prompt forwarding when the production secret preflight is incomplete", async () => { + const root = mkdtempSync(join(tmpdir(), "thothii-diagnostic-command-preflight-")); + roots.push(root); + const stdout: string[] = []; + const stderr: string[] = []; + + const exitCode = await runConfiguredDiagnosticCommand( + ["--json", "--interactive"], + { NODE_ENV: "test", AUTH_MODE: "none", THT_PI_AUTH_FILE: join(root, "missing-auth.json") }, + (line) => stdout.push(line), + (line) => stderr.push(line), + ); + + expect(exitCode).toBe(1); + expect(stderr).toEqual([]); + expect(stdout).toHaveLength(1); + expect(JSON.parse(stdout[0])).toMatchObject({ ready: false, checks: [{ code: "auth_config_invalid" }] }); +}); + +test.each([ + { + ready: true, mode: "oidc", checks: [{ level: "error", code: "oidc_secret_missing", message: "attacker-message" }], + }, + { + ready: false, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "attacker-message" }], + }, + { + ready: false, mode: "oidc", checks: [ + { level: "error", code: "oidc_secret_missing", message: "one" }, + { level: "error", code: "oidc_secret_missing", message: "duplicate-attacker" }, + ], + }, + { + ready: false, mode: "oidc", checks: [{ + level: "error", code: "oidc_secret_missing", message: "failure", field: "attacker-field-SENTINEL", + }], + }, +])("fails closed on a hostile semantic report %#", async (hostile) => { + const stdout: string[] = []; + const exitCode = await runDiagnosticCommand(["--json"], { + diagnoser: { inspect: async () => hostile as AuthDiagnostics }, + stdout: (line) => stdout.push(line), + stderr: () => undefined, + }); + + expect(exitCode).toBe(1); + expect(JSON.parse(stdout.join(""))).toEqual({ + ready: false, + mode: "none", + checks: [{ level: "error", code: "auth_config_invalid", message: "Authentication configuration is unavailable." }], + }); + expect(stdout.join("")).not.toMatch(/attacker|duplicate/i); +}); diff --git a/backend/test/auth-diagnostics.test.ts b/backend/test/auth-diagnostics.test.ts new file mode 100644 index 00000000..894b589f --- /dev/null +++ b/backend/test/auth-diagnostics.test.ts @@ -0,0 +1,486 @@ +import { chmodSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { createAuthDiagnoser } from "../src/auth/diagnostics.js"; +import { createAuthenticationConfigProvider } from "../src/auth/config.js"; +import { createAuthentikGroupCatalog } from "../src/auth/authentik-group-catalog.js"; +import { createLocalUserRegistry } from "../src/auth/local-registry.js"; +import { createOidcProtocol, OidcJwksUnavailableError } from "../src/auth/oidc-client.js"; +import type { LoadedAuthConfig } from "../src/auth/types.js"; + +const sentinels = [ + "oidc-client-secret-UNIQUE-7P3", "authentik-api-token-UNIQUE-9Q7", + "cookie-UNIQUE-5M1", "$argon2id$v=19$password-hash-UNIQUE-2T8", "/private/path-UNIQUE-4K6", +]; +const roots: string[] = []; +const acceptSessionRoot = () => undefined; +const validPasswordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function privateRoot(): string { + const root = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-diagnostics-")); + chmodSync(root, 0o700); + roots.push(root); + return root; +} + +function oidcConfig(): LoadedAuthConfig { + return { + revision: "a".repeat(64), sourcePath: "/safe/auth.yaml", + value: { + version: 1, mode: "oidc", publicUrl: "https://thothii.example.test", + session: { regularTtlSeconds: 1, regularIdleSeconds: 1, rememberTtlSeconds: 1, rememberIdleSeconds: 1, oidcTtlSeconds: 1 }, + oidc: { issuer: "https://issuer.example.test/application/o/thothii/", clientId: "thothii", clientSecretRef: "THT_OIDC_CLIENT_SECRET", scopes: ["openid"], groupsClaim: "groups" }, + groupCatalog: { driver: "authentik", baseUrl: "https://authentik.example.test", apiTokenRef: "THT_AUTHENTIK_API_TOKEN" }, + authorization: { groupRoles: { "TOT Users": ["user"], "TOT Admin": ["admin"] } }, + }, + }; +} + +function localConfig(sourcePath: string): LoadedAuthConfig { + return { + revision: "b".repeat(64), sourcePath, + value: { + version: 1, mode: "local", publicUrl: "http://127.0.0.1:8080", + session: { regularTtlSeconds: 1, regularIdleSeconds: 1, rememberTtlSeconds: 1, rememberIdleSeconds: 1, oidcTtlSeconds: 1 }, + local: { usersFile: "users.yaml" }, + }, + }; +} + +function registryYaml(role: "user" | "admin", passwordHash = validPasswordHash): string { + return [ + "version: 1", "users:", " - id: 6ba7b810-9dad-4ed1-80b4-00c04fd430c8", + " username: Admin", " displayName: Admin", ` passwordHash: ${passwordHash}`, + " roles:", ` - ${role}`, " enabled: true", " authRevision: 1", "", + ].join("\n"); +} + +test("reports deterministic live OIDC checks and silently ignores unrelated groups", async () => { + const oidcDiagnose = vi.fn(async () => undefined); + const fetch = vi.fn(async (input) => { + const requested = new URL(String(input)).searchParams.get("name"); + return Response.json({ + pagination: { next: null }, + results: [{ name: requested }, { name: "Unmapped Corporate Group" }], + }); + }); + const groupCatalog = createAuthentikGroupCatalog({ + baseUrl: "https://authentik.example.test", apiToken: sentinels[1]!, fetch, + }); + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", sentinels[0]!], ["THT_AUTHENTIK_API_TOKEN", sentinels[1]!]]), + oidcProtocol: { diagnose: oidcDiagnose }, groupCatalog, + }).inspect({ live: true }); + + expect(report).toEqual({ ready: true, mode: "oidc", checks: [expect.objectContaining({ level: "info", code: "auth_ready" })] }); + expect(oidcDiagnose).toHaveBeenCalledOnce(); + expect(fetch).toHaveBeenCalledTimes(2); + expect(fetch.mock.calls.map(([input]) => new URL(String(input)).searchParams.get("name"))) + .toEqual(["TOT Admin", "TOT Users"]); + expect(report.checks).not.toContainEqual(expect.objectContaining({ level: "warning" })); + expect(JSON.stringify(report)).not.toContain("Unmapped Corporate Group"); + expect(report.checks.map((item) => item.message).join("\n")).not.toContain("Unmapped Corporate Group"); +}); + +test("requires both fixed OIDC and Authentik secrets only in OIDC mode", async () => { + const oidc = createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", secrets: new Map(), + sessionRootValidator: acceptSessionRoot, + }); + const local = createAuthDiagnoser({ + authMode: "local", authStateRoot: "/safe/auth-state", secrets: new Map(), + sessionRootValidator: acceptSessionRoot, + authentication: { current: () => localConfig("/safe/auth.yaml") }, + localUserRegistry: { hasEnabledAdmin: async () => false } as never, + }); + + await expect(oidc.inspect({ live: false })).resolves.toMatchObject({ ready: false, checks: [ + expect.objectContaining({ code: "oidc_secret_missing" }), + ] }); + await expect(local.inspect({ live: false })).resolves.toMatchObject({ ready: false, checks: [ + expect.objectContaining({ code: "local_admin_missing" }), + ] }); +}); + +test.each([ + ["OIDC maximum", "THT_OIDC_CLIENT_SECRET", 4096, true], + ["OIDC overflow", "THT_OIDC_CLIENT_SECRET", 4097, false], + ["Authentik maximum", "THT_AUTHENTIK_API_TOKEN", 16 * 1024, true], + ["Authentik overflow", "THT_AUTHENTIK_API_TOKEN", 16 * 1024 + 1, false], +])("uses the runtime $s secret boundary in static diagnosis", async (_label, name, length, ready) => { + const secrets = new Map([ + ["THT_OIDC_CLIENT_SECRET", "o"], + ["THT_AUTHENTIK_API_TOKEN", "a"], + ]); + secrets.set(name, "x".repeat(length)); + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, secrets, + }).inspect({ live: false }); + + expect(report.ready).toBe(ready); + expect(report.checks.map((item) => item.code)).toEqual(ready ? ["auth_ready"] : ["oidc_secret_missing"]); +}); + +test("rejects control characters in either required secret during static diagnosis", async () => { + for (const name of ["THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN"] as const) { + const secrets = new Map([ + ["THT_OIDC_CLIENT_SECRET", "oidc-secret"], + ["THT_AUTHENTIK_API_TOKEN", "authentik-token"], + ]); + secrets.set(name, "invalid\u0000secret"); + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, secrets, + }).inspect({ live: false }); + expect(report.checks).toEqual([expect.objectContaining({ code: "oidc_secret_missing" })]); + } +}); + +test("distinguishes a valid registry without an enabled admin from a malformed registry", async () => { + const validRoot = privateRoot(); + const validUsers = join(validRoot, "users.yaml"); + writeFileSync(validUsers, registryYaml("user"), { encoding: "utf8", mode: 0o600 }); + chmodSync(validUsers, 0o600); + const validReport = await createAuthDiagnoser({ + authMode: "local", authStateRoot: validRoot, + sessionRootValidator: acceptSessionRoot, + authentication: { current: () => localConfig(join(validRoot, "auth.yaml")) }, + localUserRegistry: createLocalUserRegistry(validUsers), + }).inspect({ live: false }); + expect(validReport.checks).toEqual([expect.objectContaining({ code: "local_admin_missing" })]); + + const malformedRoot = privateRoot(); + const malformedUsers = join(malformedRoot, "users.yaml"); + writeFileSync(malformedUsers, registryYaml("admin", sentinels[3]), { encoding: "utf8", mode: 0o600 }); + chmodSync(malformedUsers, 0o600); + const malformedReport = await createAuthDiagnoser({ + authMode: "local", authStateRoot: malformedRoot, + sessionRootValidator: acceptSessionRoot, + authentication: { current: () => localConfig(join(malformedRoot, "auth.yaml")) }, + localUserRegistry: createLocalUserRegistry(malformedUsers), + }).inspect({ live: false }); + expect(malformedReport.checks).toEqual([expect.objectContaining({ code: "local_user_registry_invalid" })]); + expect(JSON.stringify(malformedReport)).not.toContain(sentinels[3]); + expect(JSON.stringify(malformedReport)).not.toContain(malformedUsers); +}); + +test("maps unsafe auth.yaml storage from the real provider to a redacted config failure", async () => { + const root = privateRoot(); + const unsafePath = join(root, basename(sentinels[4]!)); + writeFileSync(unsafePath, [ + "version: 1", "mode: local", "publicUrl: http://127.0.0.1:8080", "local:", " usersFile: users.yaml", "", + ].join("\n"), { encoding: "utf8", mode: 0o640 }); + chmodSync(unsafePath, 0o640); + const report = await createAuthDiagnoser({ + authMode: "local", authStateRoot: root, + sessionRootValidator: acceptSessionRoot, + authentication: createAuthenticationConfigProvider(unsafePath), + }).inspect({ live: false }); + + expect(report.checks).toEqual([expect.objectContaining({ code: "auth_config_invalid" })]); + expect(JSON.stringify(report)).not.toContain(unsafePath); + expect(JSON.stringify(report)).not.toContain(sentinels[4]); +}); + +test("maps OIDC and group catalog failures to only the closed diagnostics code union", async () => { + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", sentinels[0]!], ["THT_AUTHENTIK_API_TOKEN", sentinels[1]!]]), + oidcProtocol: { diagnose: async () => { throw new Error("issuer unavailable"); } }, + groupCatalog: { verifyConfiguredGroups: async () => [{ level: "error", code: "oidc_mapped_group_missing", message: "configured group is missing", field: "TOT Users" }] }, + }).inspect({ live: true }); + + expect(report.ready).toBe(false); + expect(report.checks.map((check) => check.code)).toEqual(["oidc_discovery_unreachable", "oidc_mapped_group_missing"]); +}); + +test("reports a JWKS validation failure through its closed diagnostic code", async () => { + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", sentinels[0]!], ["THT_AUTHENTIK_API_TOKEN", sentinels[1]!]]), + oidcProtocol: { diagnose: async () => { throw new OidcJwksUnavailableError(); } }, + groupCatalog: { verifyConfiguredGroups: async () => [] }, + }).inspect({ live: true }); + + expect(report.checks).toEqual([expect.objectContaining({ code: "oidc_jwks_unreachable" })]); +}); + +test("bounds a live diagnosis whose OIDC dependency ignores abort and starts no later checks", async () => { + vi.useFakeTimers(); + try { + let rejectLate: ((error: Error) => void) | undefined; + const oidcDiagnose = vi.fn(() => new Promise((_resolve, reject) => { rejectLate = reject; })); + const verifyConfiguredGroups = vi.fn(async () => []); + const completion = createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", "oidc"], ["THT_AUTHENTIK_API_TOKEN", "authentik"]]), + oidcProtocol: { diagnose: oidcDiagnose }, groupCatalog: { verifyConfiguredGroups }, + }).inspect({ live: true }); + const outcome = completion.then((report) => report, () => undefined); + + await vi.advanceTimersByTimeAsync(29_999); + await expect(Promise.race([outcome, Promise.resolve("pending")])).resolves.toBe("pending"); + await vi.advanceTimersByTimeAsync(1); + await expect(outcome).resolves.toMatchObject({ + ready: false, + checks: [expect.objectContaining({ code: "oidc_discovery_unreachable" })], + }); + expect(oidcDiagnose).toHaveBeenCalledOnce(); + expect(verifyConfiguredGroups).not.toHaveBeenCalled(); + rejectLate?.(new Error("late-secret-detail")); + await Promise.resolve(); + } finally { + vi.useRealTimers(); + } +}); + +test("composes caller cancellation with the overall live-diagnostic deadline", async () => { + const caller = new AbortController(); + const verifyConfiguredGroups = vi.fn(async () => []); + const completion = createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", "oidc"], ["THT_AUTHENTIK_API_TOKEN", "authentik"]]), + oidcProtocol: { diagnose: async () => await new Promise(() => undefined) }, + groupCatalog: { verifyConfiguredGroups }, + }).inspect({ live: true, signal: caller.signal }); + caller.abort(); + + await expect(completion).resolves.toMatchObject({ + ready: false, + checks: [expect.objectContaining({ code: "oidc_discovery_unreachable" })], + }); + expect(verifyConfiguredGroups).not.toHaveBeenCalled(); +}); + +test("never reports auth_ready when cancellation lands during OIDC completion", async () => { + const caller = new AbortController(); + const verifyConfiguredGroups = vi.fn(async () => []); + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: oidcConfig }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", "oidc"], ["THT_AUTHENTIK_API_TOKEN", "authentik"]]), + oidcProtocol: { diagnose: async () => { caller.abort(); } }, + groupCatalog: { verifyConfiguredGroups }, + }).inspect({ live: true, signal: caller.signal }); + + expect(report).toMatchObject({ + ready: false, + checks: [expect.objectContaining({ code: "oidc_discovery_unreachable" })], + }); + expect(verifyConfiguredGroups).not.toHaveBeenCalled(); +}); + +test("stops starting mapped-group requests when the overall live deadline expires", async () => { + vi.useFakeTimers(); + try { + const loaded = oidcConfig(); + if (loaded.value.mode !== "oidc") throw new Error("test configuration is not OIDC"); + loaded.value.authorization.groupRoles = Object.fromEntries(Array.from({ length: 10 }, (_unused, index) => [ + `Mapped Group ${String(index).padStart(2, "0")}`, + index === 0 ? ["admin"] : ["user"], + ])); + const fetch = vi.fn((input, init) => new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const name = new URL(String(input)).searchParams.get("name"); + resolve(Response.json({ pagination: { next: null }, results: [{ name }] })); + }, 4_000); + init?.signal?.addEventListener("abort", () => { + clearTimeout(timer); + reject(new DOMException("aborted", "AbortError")); + }, { once: true }); + })); + const completion = createAuthDiagnoser({ + authMode: "oidc", authentication: { current: () => loaded }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", "oidc"], ["THT_AUTHENTIK_API_TOKEN", "authentik"]]), + oidcProtocol: { diagnose: async () => undefined }, + groupCatalog: createAuthentikGroupCatalog({ + baseUrl: "https://authentik.example.test", apiToken: "authentik", fetch, + }), + }).inspect({ live: true }); + + await vi.advanceTimersByTimeAsync(30_000); + await expect(completion).resolves.toMatchObject({ + ready: false, + checks: [expect.objectContaining({ code: "oidc_group_catalog_unreachable" })], + }); + expect(fetch).toHaveBeenCalledTimes(8); + await vi.advanceTimersByTimeAsync(30_000); + expect(fetch).toHaveBeenCalledTimes(8); + } finally { + vi.useRealTimers(); + } +}); + +test("maps a concrete discovery adapter issuer mismatch to its dedicated code", async () => { + const loaded = oidcConfig(); + const fetch = vi.fn(async (input) => { + const url = new URL(String(input)); + if (!url.pathname.includes(".well-known")) throw new Error("JWKS must not be requested after issuer mismatch"); + return Response.json({ + issuer: "https://different-issuer.example.test", + authorization_endpoint: "https://issuer.example.test/authorize", + token_endpoint: "https://issuer.example.test/token", + jwks_uri: "https://issuer.example.test/jwks", + response_types_supported: ["code"], + grant_types_supported: ["authorization_code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }); + }); + const oidcProtocol = createOidcProtocol({ + issuer: loaded.value.mode === "oidc" ? loaded.value.oidc.issuer : "", + clientId: "thothii", clientSecret: sentinels[0]!, + callbackUrl: "https://thothii.example.test/api/auth/oidc/callback", + scopes: ["openid"], groupsClaim: "groups", fetch, + }); + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: () => loaded }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", sentinels[0]!], ["THT_AUTHENTIK_API_TOKEN", sentinels[1]!]]), + oidcProtocol, groupCatalog: { verifyConfiguredGroups: async () => [] }, + }).inspect({ live: true }); + + expect(report.checks).toEqual([expect.objectContaining({ code: "oidc_issuer_mismatch" })]); + expect(fetch).toHaveBeenCalledOnce(); + expect(JSON.stringify(report)).not.toContain("different-issuer"); +}); + +test.each([ + ["an upstream error", 503, { + issuer: "https://different-issuer.example.test", + authorization_endpoint: "https://issuer.example.test/authorize", + token_endpoint: "https://issuer.example.test/token", + jwks_uri: "https://issuer.example.test/jwks", + response_types_supported: ["code"], + grant_types_supported: ["authorization_code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }], + ["schema-invalid metadata", 200, { issuer: "https://different-issuer.example.test" }], + ["an unsafe foreign issuer", 200, { + issuer: "http://different-issuer.example.test", + authorization_endpoint: "https://issuer.example.test/authorize", + token_endpoint: "https://issuer.example.test/token", + jwks_uri: "https://issuer.example.test/jwks", + response_types_supported: ["code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }], + ["an unsafe authorization endpoint", 200, { + issuer: "https://different-issuer.example.test", + authorization_endpoint: "http://127.0.0.1/authorize", + token_endpoint: "https://issuer.example.test/token", + jwks_uri: "https://issuer.example.test/jwks", + response_types_supported: ["code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }], + ["an unsafe token endpoint", 200, { + issuer: "https://different-issuer.example.test", + authorization_endpoint: "https://issuer.example.test/authorize", + token_endpoint: "https://operator:secret@issuer.example.test/token", + jwks_uri: "https://issuer.example.test/jwks", + response_types_supported: ["code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }], + ["an unsafe JWKS endpoint", 200, { + issuer: "https://different-issuer.example.test", + authorization_endpoint: "https://issuer.example.test/authorize", + token_endpoint: "https://issuer.example.test/token", + jwks_uri: "https://issuer.example.test/jwks#fragment", + response_types_supported: ["code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }], +])("does not classify %s containing an issuer as an issuer mismatch", async (_label, status, body) => { + const loaded = oidcConfig(); + const fetch = vi.fn(async () => Response.json(body, { status })); + const oidcProtocol = createOidcProtocol({ + issuer: loaded.value.mode === "oidc" ? loaded.value.oidc.issuer : "", + clientId: "thothii", clientSecret: sentinels[0]!, + callbackUrl: "https://thothii.example.test/api/auth/oidc/callback", + scopes: ["openid"], groupsClaim: "groups", fetch, + }); + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: () => loaded }, authStateRoot: "/safe/auth-state", + sessionRootValidator: acceptSessionRoot, + secrets: new Map([["THT_OIDC_CLIENT_SECRET", sentinels[0]!], ["THT_AUTHENTIK_API_TOKEN", sentinels[1]!]]), + oidcProtocol, groupCatalog: { verifyConfiguredGroups: async () => [] }, + }).inspect({ live: true }); + + expect(report.checks).toEqual([expect.objectContaining({ code: "oidc_discovery_unreachable" })]); + expect(report.checks).not.toContainEqual(expect.objectContaining({ code: "oidc_issuer_mismatch" })); + expect(fetch).toHaveBeenCalledOnce(); + expect(JSON.stringify(report)).not.toContain("different-issuer"); +}); + +test("redacts exceptional configuration, registry, protocol, and catalog errors", async () => { + const detail = sentinels.join(" "); + const report = await createAuthDiagnoser({ + authMode: "oidc", authentication: { current: () => { throw new Error(detail); } }, authStateRoot: sentinels[4]!, + secrets: new Map(), oidcProtocol: { diagnose: async () => { throw new Error(detail); } }, + groupCatalog: { verifyConfiguredGroups: async () => { throw new Error(detail); } }, + }).inspect({ live: true }); + + const rendered = JSON.stringify(report); + for (const sentinel of sentinels) expect(rendered).not.toContain(sentinel); + expect(report.checks.every((check) => check.level === "error" || check.level === "info")).toBe(true); +}); + +test.skipIf(process.platform === "win32")("routes production POSIX static validation through the native auth-storage bridge", async () => { + const validateRoot = vi.fn(async () => undefined); + const report = await createAuthDiagnoser({ + authMode: "none", + authStateRoot: "/var/lib/thothii/auth", + posixStorageBridge: { validateRoot }, + }).inspect({ live: false }); + + expect(report).toMatchObject({ ready: true }); + expect(validateRoot).toHaveBeenCalledOnce(); + expect(validateRoot).toHaveBeenCalledWith("/var/lib/thothii/auth"); +}); + +test("routes native Windows static session-root validation through the auth-storage bridge", async () => { + const originalPlatform = process.platform; + const validateRoot = vi.fn(async () => undefined); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const report = await createAuthDiagnoser({ + authMode: "none", + authStateRoot: "C:\\ProgramData\\ThothII\\auth", + windowsStorageBridge: { validateRoot } as never, + }).inspect({ live: false }); + + expect(report).toMatchObject({ ready: true }); + expect(validateRoot).toHaveBeenCalledOnce(); + expect(validateRoot).toHaveBeenCalledWith("C:\\ProgramData\\ThothII\\auth"); + } finally { + Object.defineProperty(process, "platform", { configurable: true, value: originalPlatform }); + } +}); + +test("accepts a platform storage validator without exposing its root or failure", async () => { + const platformRoot = "C:\\private-path-UNIQUE-6R2\\auth"; + const sessionRootValidator = vi.fn(async () => { throw new Error(`${platformRoot} denied`); }); + const report = await createAuthDiagnoser({ + authMode: "none", authStateRoot: platformRoot, sessionRootValidator, + }).inspect({ live: false }); + + expect(sessionRootValidator).toHaveBeenCalledWith(platformRoot); + expect(report.checks).toEqual([expect.objectContaining({ code: "auth_session_store_invalid" })]); + expect(JSON.stringify(report)).not.toContain(platformRoot); +}); diff --git a/backend/test/auth-dynamic-registry.test.ts b/backend/test/auth-dynamic-registry.test.ts new file mode 100644 index 00000000..67eb01fa --- /dev/null +++ b/backend/test/auth-dynamic-registry.test.ts @@ -0,0 +1,91 @@ +import { afterEach, expect, test } from "vitest"; +import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; +import { buildApp, type AppWithAuthSessionStore } from "../src/app.js"; +import { loadAuthenticationConfig } from "../src/auth/config.js"; +import { loadConfig } from "../src/config.js"; +import { createFixtureAuthStorageBridge, prepareAuthStateRoot } from "./auth-test-fixtures.js"; + +const password = "correct horse battery staple"; +const passwordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"; +const userA = { id: "6ba7b810-9dad-4ed1-80b4-00c04fd430c8", username: "AdminA" }; +const userB = { id: "6ba7b811-9dad-4ed1-80b4-00c04fd430c8", username: "AdminB" }; +const publicUrl = "http://127.0.0.1:8787"; +const cleanups: Array<() => Promise> = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); + +function usersYaml(user: typeof userA): string { + return [ + "version: 1", "users:", ` - id: ${user.id}`, ` username: ${user.username}`, + ` passwordHash: ${passwordHash}`, " roles:", " - admin", " enabled: true", " authRevision: 1", "", + ].join("\n"); +} + +function configYaml(usersFile: string) { + return stringify({ version: 1, mode: "local", publicUrl, local: { usersFile } }); +} + +function cookiePair(response: { headers: Record }): string { + const header = response.headers["set-cookie"]; + const first = Array.isArray(header) ? header[0] : header; + return first?.split(";", 1)[0] ?? ""; +} + +test("each login and session resolve uses the current config snapshot users file", async () => { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-dynamic-")); + chmodSync(directory, 0o700); + const authFile = join(directory, "auth.yaml"); + const usersAFile = join(directory, "users-a.yaml"); + const usersBFile = join(directory, "users-bbbbb.yaml"); + writeFileSync(usersAFile, usersYaml(userA), { encoding: "utf8", mode: 0o600 }); + writeFileSync(usersBFile, usersYaml(userB), { encoding: "utf8", mode: 0o600 }); + writeFileSync(authFile, configYaml("users-a.yaml"), { encoding: "utf8", mode: 0o600 }); + chmodSync(authFile, 0o600); + chmodSync(usersAFile, 0o600); + chmodSync(usersBFile, 0o600); + const authStateRoot = join(directory, "auth-state"); + prepareAuthStateRoot(authStateRoot); + const app = buildApp(loadConfig({ + THT_AUTH_CONFIG_FILE: authFile, + THT_AUTH_STATE_ROOT: authStateRoot, + THT_HARNESS_DIR: "/tmp/h", + }), { authStorageBridgeForTest: createFixtureAuthStorageBridge() }); + cleanups.push(async () => { + await app.close(); + rmSync(directory, { recursive: true, force: true }); + }); + const signIn = (username: string) => app.inject({ + method: "POST", url: "/auth/local/login", + headers: { origin: publicUrl, "sec-fetch-site": "same-origin" }, + payload: { username, password }, + }); + + const first = await signIn(userA.username); + expect(first.statusCode).toBe(200); + const aCookie = cookiePair(first); + expect((await app.inject({ method: "GET", url: "/me", headers: { cookie: aCookie } })).json()) + .toMatchObject({ subject: userA.id }); + + writeFileSync(authFile, configYaml("users-bbbbb.yaml"), { encoding: "utf8", mode: 0o600 }); + chmodSync(authFile, 0o600); + expect(readFileSync(usersAFile, "utf8")).toContain(userA.username); + + const removedUser = await signIn(userA.username); + expect(removedUser.statusCode).toBe(401); + expect(removedUser.json()).toEqual({ code: "invalid_credentials", error: "Invalid username or password" }); + expect((await app.inject({ method: "GET", url: "/me", headers: { cookie: aCookie } })).statusCode).toBe(401); + + const second = await signIn(userB.username); + expect(second.statusCode).toBe(200); + const bCookie = cookiePair(second); + expect(await app.inject({ method: "GET", url: "/me", headers: { cookie: bCookie } }).then((response) => response.json())) + .toMatchObject({ subject: userB.id }); + const token = bCookie.split("=", 2)[1] ?? ""; + const record = await (app as AppWithAuthSessionStore).thothiiAuthSessionStore?.resolve(token); + expect(record).toMatchObject({ subject: userB.id, authConfigRevision: loadAuthenticationConfig(authFile).revision }); +}); diff --git a/backend/test/auth-password.test.ts b/backend/test/auth-password.test.ts new file mode 100644 index 00000000..5dd95a46 --- /dev/null +++ b/backend/test/auth-password.test.ts @@ -0,0 +1,118 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, test, vi } from "vitest"; + +const { argon2Spy } = vi.hoisted(() => ({ argon2Spy: vi.fn() })); +vi.mock("node:crypto", async (importOriginal) => { + const actual = await importOriginal(); + argon2Spy.mockImplementation(actual.argon2); + return { ...actual, argon2: argon2Spy }; +}); + +import { isValidPasswordHash, verifyPassword } from "../src/auth/password.js"; + +interface Argon2Vector { + password: string; + phc: string; +} + +const vectors = JSON.parse(readFileSync( + resolve(import.meta.dirname, "fixtures/argon2id-vectors.json"), + "utf8", +)) as Argon2Vector[]; + +describe("local Argon2id password verification", () => { + test("accepts every committed Go-generated vector", async () => { + expect(vectors.length).toBeGreaterThan(0); + for (const vector of vectors) { + await expect(verifyPassword(vector.password, vector.phc)).resolves.toBe(true); + } + }); + + test("rejects a one-byte password change", async () => { + for (const vector of vectors) { + await expect(verifyPassword(`${vector.password}!`, vector.phc)).resolves.toBe(false); + } + }); + + test("rejects ill-formed Unicode instead of authenticating as U+FFFD", async () => { + const replacementPassword = "correct horse battery stap\uFFFD"; + const loneSurrogatePassword = "correct horse battery stap\uD800"; + const replacementPasswordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$+tAXzgaQVnNaonNvgevyG6UKlaKcwyRMi1mESNk0BvQ"; + + await expect(verifyPassword(replacementPassword, replacementPasswordHash)).resolves.toBe(true); + await expect(verifyPassword(loneSurrogatePassword, replacementPasswordHash)).resolves.toBe(false); + }); + + test("accepts a multibyte password at exactly the 1024-byte boundary", async () => { + const password = "é".repeat(512); + const passwordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$LfO3M3JBKeXf1knenCjQ6m9z4B7nedb0As2uc522I1I"; + + expect(Buffer.byteLength(password, "utf8")).toBe(1024); + await expect(verifyPassword(password, passwordHash)).resolves.toBe(true); + await expect(verifyPassword(`${password}é`, passwordHash)).resolves.toBe(false); + }); + + test("rejects an over-limit password before converting it with Buffer.from", async () => { + const password = "é".repeat(513); + const fromSpy = vi.spyOn(Buffer, "from"); + + try { + expect(Buffer.byteLength(password, "utf8")).toBe(1026); + await expect(verifyPassword(password, vectors[0].phc)).resolves.toBe(false); + expect(fromSpy.mock.calls.some(([value, encoding]) => value === password && encoding === "utf8")).toBe(false); + } finally { + fromSpy.mockRestore(); + } + }); + + test("rejects malformed and oversized PHC parameters before Argon2 allocation", async () => { + const password = vectors[0].password; + const digest = vectors[0].phc.split("$")[5]; + const salt = vectors[0].phc.split("$")[4]; + const cases = [ + `$argon2id$v=19$m=262145,t=1,p=1$${salt}$${digest}`, + `$argon2id$v=19$m=65536,t=11,p=1$${salt}$${digest}`, + `$argon2id$v=19$m=65536,t=3,p=5$${salt}$${digest}`, + `$argon2id$v=19$m=65536,t=3,p=1$${salt}$${"A".repeat(88)}`, + `$argon2id$v=19$m=65536,t=3,p=1$${salt}=$${digest}`, + `$argon2id$v=19$m=8,t=1,p=1$${salt}$${digest}`, + ]; + + for (const phc of cases) { + argon2Spy.mockClear(); + await expect(verifyPassword(password, phc)).resolves.toBe(false); + expect(argon2Spy).not.toHaveBeenCalled(); + } + }); + + test("accepts only the exact Go Argon2id policy in registry PHCs", () => { + const [empty, algorithm, version, parameters, salt, digest] = vectors[0].phc.split("$"); + expect(empty).toBe(""); + expect(algorithm).toBe("argon2id"); + expect(version).toBe("v=19"); + expect(isValidPasswordHash(`$${algorithm}$${version}$m=8,t=1,p=1$${salt}$${digest}`)).toBe(false); + expect(isValidPasswordHash(`$${algorithm}$${version}$m=65536,t=1,p=1$${salt}$${digest}`)).toBe(false); + expect(isValidPasswordHash(`$${algorithm}$${version}$m=65536,t=3,p=2$${salt}$${digest}`)).toBe(false); + expect(isValidPasswordHash(`$${algorithm}$${version}$${parameters}$${salt}$${digest}`)).toBe(true); + }); + + test("rejects raw-base64 PHCs with non-zero trailing bits", async () => { + const nonCanonicalSalt = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODx$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"; + const nonCanonicalDigest = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC5"; + + for (const phc of [nonCanonicalSalt, nonCanonicalDigest]) { + expect(isValidPasswordHash(phc)).toBe(false); + await expect(verifyPassword(vectors[0].password, phc)).resolves.toBe(false); + } + }); + + test("surfaces a sanitized operational error when native Argon2 fails", async () => { + argon2Spy.mockImplementationOnce((_algorithm, _parameters, callback) => { + callback(new Error("native details must not leave the verifier")); + }); + + await expect(verifyPassword(vectors[0].password, vectors[0].phc)) + .rejects.toThrow("local_password_verification_failed"); + }); +}); diff --git a/backend/test/auth-pi-management-local.test.ts b/backend/test/auth-pi-management-local.test.ts new file mode 100644 index 00000000..606513c6 --- /dev/null +++ b/backend/test/auth-pi-management-local.test.ts @@ -0,0 +1,70 @@ +import { expect, test, vi } from "vitest"; +import type { PiManagementService } from "../src/pi/management.js"; +import { createLocalAuthFixture } from "./auth-test-fixtures.js"; + +function fakeService(): PiManagementService { + return { + status: vi.fn(async () => ({ ready: true })), + options: vi.fn(async () => ({ providers: [], models: [], reasoning: [], checkedAt: "2026-08-17T00:00:00.000Z" })), + configure: vi.fn(async (value) => ({ ...value, updatedAt: "2026-08-17T00:00:00.000Z" })), + test: vi.fn(async () => ({ ready: true, checkedAt: "2026-08-17T00:00:00.000Z" })), + logs: vi.fn(async () => ({ lines: [] })), + }; +} + +test("a local HTTPS cookie session authorizes Pi writes through an untrusted internal HTTP hop", async () => { + const service = fakeService(); + const fixture = await createLocalAuthFixture( + { piManagement: service }, + { publicUrl: "HTTPS://thothii.example.test" }, + ); + try { + // The fixture performs the real login and /me request through the production hooks. + expect(fixture.loginStatus).toBe(200); + expect(fixture.meStatus).toBe(200); + expect(fixture.publicUrl).toBe("HTTPS://thothii.example.test"); + + const proxyHeaders = fixture.sessionHeaders({ host: "127.0.0.1:8080" }); + const configured = await fixture.app.inject({ + method: "PUT", + url: "/pi-management/config", + headers: proxyHeaders, + payload: { provider: "zai", model: "glm-5.2", reasoning: "high" }, + }); + const smoke = await fixture.app.inject({ + method: "POST", + url: "/pi-management/test", + headers: proxyHeaders, + }); + + expect(configured.statusCode).toBe(200); + expect(smoke.statusCode).toBe(200); + expect(service.configure).toHaveBeenCalledTimes(1); + expect(service.test).toHaveBeenCalledTimes(1); + + fixture.resetDownstreamHits(); + vi.mocked(service.configure).mockClear(); + vi.mocked(service.test).mockClear(); + const wrongOrigin = await fixture.app.inject({ + method: "PUT", + url: "/pi-management/config", + headers: fixture.sessionHeaders({ host: "127.0.0.1:8080", origin: "https://evil.example" }), + payload: { provider: "zai", model: "glm-5.2", reasoning: "high" }, + }); + const wrongCsrf = await fixture.app.inject({ + method: "POST", + url: "/pi-management/test", + headers: fixture.sessionHeaders({ host: "127.0.0.1:8080", "x-thothii-csrf": "wrong" }), + }); + + for (const response of [wrongOrigin, wrongCsrf]) { + expect(response.statusCode).toBe(403); + expect(response.json()).toEqual({ code: "csrf_failed", error: "Request origin validation failed" }); + } + expect(fixture.downstreamHits()).toBe(0); + expect(service.configure).not.toHaveBeenCalled(); + expect(service.test).not.toHaveBeenCalled(); + } finally { + await fixture.close(); + } +}); diff --git a/backend/test/auth-production-csrf.test.ts b/backend/test/auth-production-csrf.test.ts new file mode 100644 index 00000000..dec9ae6d --- /dev/null +++ b/backend/test/auth-production-csrf.test.ts @@ -0,0 +1,104 @@ +import { afterAll, beforeAll, expect, test } from "vitest"; +import { createLocalAuthFixture, type LocalAuthFixture, localPublicUrl } from "./auth-test-fixtures.js"; + +interface StateChangingRoute { + family: string; + method: "POST" | "PUT" | "DELETE"; + url: string; +} + +const stateChangingRoutes: readonly StateChangingRoute[] = [ + { family: "auth logout", method: "POST", url: "/auth/logout" }, + { family: "runtime prewarm", method: "POST", url: "/runtime/prewarm" }, + { family: "session create", method: "POST", url: "/sessions" }, + { family: "session mutation", method: "POST", url: "/sessions/session-1/response" }, + { family: "session mutation", method: "POST", url: "/sessions/session-1/steer" }, + { family: "session mutation", method: "POST", url: "/sessions/session-1/resume" }, + { family: "session mutation", method: "POST", url: "/sessions/session-1/close" }, + { family: "session mutation", method: "POST", url: "/sessions/session-1/rename" }, + { family: "session mutation", method: "POST", url: "/sessions/session-1/group" }, + { family: "session archive", method: "POST", url: "/sessions/session-1/archive" }, + { family: "session archive", method: "POST", url: "/sessions/session-1/unarchive" }, + { family: "session delete", method: "DELETE", url: "/sessions/session-1" }, + { family: "SQL preview", method: "POST", url: "/sessions/session-1/sql/preview" }, + { family: "SQL export", method: "POST", url: "/sessions/session-1/sql/export" }, + { family: "workspace registry", method: "POST", url: "/workspace-registry/pull" }, + { family: "workspace validation", method: "POST", url: "/workspaces/validate" }, + { family: "workspace secrets", method: "PUT", url: "/workspaces/workspace-1/secrets" }, + { family: "workspace secrets", method: "DELETE", url: "/workspaces/workspace-1/secrets/secret-1" }, + { family: "workspace diagnostics", method: "POST", url: "/workspaces/workspace-1/test" }, + { family: "settings", method: "PUT", url: "/settings" }, + { family: "Pi management", method: "PUT", url: "/pi-management/config" }, + { family: "Pi management", method: "POST", url: "/pi-management/test" }, + { family: "maintenance", method: "POST", url: "/internal/maintenance/activate" }, + { family: "maintenance", method: "POST", url: "/internal/maintenance/deactivate" }, +]; + +let fixture: LocalAuthFixture; + +beforeAll(async () => { + fixture = await createLocalAuthFixture(); +}); + +afterAll(async () => { + await fixture.close(); +}); + +test.each(stateChangingRoutes)( + "$family $method $url rejects every production CSRF/session-boundary failure before downstream code", + async ({ method, url }) => { + const failures: Array<{ expectedStatus: number; headers: Record }> = [ + { expectedStatus: 403, headers: fixture.sessionHeaders({ "x-thothii-csrf": undefined }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ "x-thothii-csrf": "malformed" }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ "x-thothii-csrf": `${fixture.csrfToken}, ${fixture.csrfToken}` }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ "x-thothii-csrf": "x".repeat(4097) }) }, + { expectedStatus: 401, headers: fixture.sessionHeaders({ cookie: undefined }) }, + { expectedStatus: 401, headers: fixture.sessionHeaders({ cookie: "thothii_session=not-a-token" }) }, + { expectedStatus: 401, headers: fixture.sessionHeaders({ cookie: `${fixture.cookie}; ${fixture.cookie}` }) }, + { expectedStatus: 401, headers: fixture.sessionHeaders({ cookie: `${fixture.cookie}; padding=${"x".repeat(4097)}` }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ origin: undefined }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ origin: "http://wrong.example.test" }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ origin: `${localPublicUrl}, ${localPublicUrl}` }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ origin: `${localPublicUrl}${"x".repeat(4097)}` }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ "sec-fetch-site": "cross-site" }) }, + { expectedStatus: 403, headers: fixture.sessionHeaders({ "sec-fetch-site": "same-origin, same-origin" }) }, + ]; + + for (const { expectedStatus, headers } of failures) { + fixture.resetDownstreamHits(); + const response = await fixture.app.inject({ method, url, headers, payload: {} }); + expect(response.statusCode).toBe(expectedStatus); + expect(response.json()).toEqual(expectedStatus === 403 + ? { code: "csrf_failed", error: "Request origin validation failed" } + : { code: "authentication_required", error: "Authentication is required" }); + expect(fixture.downstreamHits()).toBe(0); + } + }, +); + +test("a valid real local session cookie and derived CSRF token cross the same production boundary", async () => { + fixture.resetDownstreamHits(); + const response = await fixture.app.inject({ + method: "PUT", + url: "/settings", + headers: fixture.sessionHeaders(), + payload: {}, + }); + + expect(response.statusCode).toBe(200); + expect(fixture.downstreamHits()).toBe(1); + expect(localPublicUrl).toBe("http://127.0.0.1:8787"); +}); + +test("a valid control may omit optional Fetch Metadata while retaining its exact Origin and CSRF pair", async () => { + fixture.resetDownstreamHits(); + const response = await fixture.app.inject({ + method: "PUT", + url: "/settings", + headers: fixture.sessionHeaders({ "sec-fetch-site": undefined }), + payload: {}, + }); + + expect(response.statusCode).toBe(200); + expect(fixture.downstreamHits()).toBe(1); +}); diff --git a/backend/test/auth-request-snapshot.test.ts b/backend/test/auth-request-snapshot.test.ts new file mode 100644 index 00000000..548723a9 --- /dev/null +++ b/backend/test/auth-request-snapshot.test.ts @@ -0,0 +1,184 @@ +import { afterEach, expect, test } from "vitest"; +import { chmodSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; +import { buildApp, type AppWithAuthSessionStore } from "../src/app.js"; +import { loadAuthenticationConfig } from "../src/auth/config.js"; +import type { AuthenticationConfigProvider, LoadedAuthConfig } from "../src/auth/types.js"; +import { loadConfig } from "../src/config.js"; +import { createFixtureAuthStorageBridge, prepareAuthStateRoot } from "./auth-test-fixtures.js"; + +const password = "correct horse battery staple"; +const passwordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"; +const originA = "http://127.0.0.1:8787"; +const originB = "http://127.0.0.1:8788"; +const userA = { id: "6ba7b810-9dad-4ed1-80b4-00c04fd430c8", username: "AdminA" }; +const userB = { id: "6ba7b811-9dad-4ed1-80b4-00c04fd430c8", username: "AdminB" }; +const cleanups: Array<() => Promise> = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); + +function localYaml(publicUrl: string, usersFile: string): string { + return stringify({ version: 1, mode: "local", publicUrl, local: { usersFile } }); +} + +function oidcYaml(publicUrl: string): string { + return stringify({ + version: 1, + mode: "oidc", + publicUrl, + oidc: { + issuer: "https://issuer.example.test/application/o/thothii/", + clientId: "thothii", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", + scopes: ["openid"], + groupsClaim: "groups", + }, + groupCatalog: { + driver: "authentik", + baseUrl: "https://issuer.example.test", + apiTokenRef: "THT_AUTHENTIK_API_TOKEN", + }, + authorization: { groupRoles: { Users: ["user"], Admins: ["admin"] } }, + }); +} + +function usersYaml(user: typeof userA): string { + return [ + "version: 1", "users:", ` - id: ${user.id}`, ` username: ${user.username}`, + ` passwordHash: ${passwordHash}`, " roles:", " - admin", " enabled: true", " authRevision: 1", "", + ].join("\n"); +} + +function firstCookie(response: { headers: Record }): string { + const header = response.headers["set-cookie"]; + return (Array.isArray(header) ? header[0] : header)?.split(";", 1)[0] ?? ""; +} + +async function createFixture(first: "A" | "B", later: "A" | "B" | "oidc") { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-request-snapshot-")); + chmodSync(directory, 0o700); + const authA = join(directory, "auth-a.yaml"); + const authB = join(directory, "auth-b.yaml"); + const usersA = join(directory, "users-a.yaml"); + const usersB = join(directory, "users-b.yaml"); + writeFileSync(usersA, usersYaml(userA), { encoding: "utf8", mode: 0o600 }); + writeFileSync(usersB, usersYaml(userB), { encoding: "utf8", mode: 0o600 }); + writeFileSync(authA, localYaml(originA, "users-a.yaml"), { encoding: "utf8", mode: 0o600 }); + writeFileSync(authB, later === "oidc" ? oidcYaml(originB) : localYaml(originB, "users-b.yaml"), { encoding: "utf8", mode: 0o600 }); + for (const path of [authA, authB, usersA, usersB]) chmodSync(path, 0o600); + + const snapshots = { A: loadAuthenticationConfig(authA), B: loadAuthenticationConfig(authB) }; + let calls = 0; + const provider: AuthenticationConfigProvider = { + current: () => { + calls += 1; + return calls === 1 ? snapshots[first] : snapshots[later === "oidc" ? "B" : later]; + }, + }; + const authStateRoot = join(directory, "auth-state"); + prepareAuthStateRoot(authStateRoot); + const config = loadConfig({ + THT_AUTH_CONFIG_FILE: authA, + THT_AUTH_STATE_ROOT: authStateRoot, + THT_HARNESS_DIR: "/tmp/h", + }); + config.authentication = provider; + const app = buildApp(config, { authStorageBridgeForTest: createFixtureAuthStorageBridge() }) as AppWithAuthSessionStore; + cleanups.push(async () => { + await app.close(); + rmSync(directory, { recursive: true, force: true }); + }); + return { + app, + snapshots, + calls: () => calls, + resetCalls: () => { calls = 0; }, + userFor(snapshot: LoadedAuthConfig) { + return snapshot.sourcePath === authA ? userA : userB; + }, + }; +} + +test.each([ + { first: "A" as const, later: "B" as const }, + { first: "B" as const, later: "A" as const }, +])("a $later session cannot yield data under the replacement $first CORS snapshot", async ({ first, later }) => { + const fixture = await createFixture(first, later); + const requestSnapshot = fixture.snapshots[first]; + const replacementSnapshot = fixture.snapshots[later]; + const user = fixture.userFor(replacementSnapshot); + const created = await fixture.app.thothiiAuthSessionStore?.create({ + principal: { + issuer: "local", subject: user.id, displayName: user.username, roles: ["admin"], + permissions: ["session.use", "session.read_all", "session.manage_all", "settings.manage", "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read"], + isAdmin: true, + }, + method: "local", + remembered: false, + userAuthRevision: 1, + authConfigRevision: replacementSnapshot.revision, + idleTtlMs: 60_000, + absoluteTtlMs: 60_000, + }); + expect(created).toBeDefined(); + + fixture.resetCalls(); + const response = await fixture.app.inject({ + method: "GET", + url: "/me", + headers: { cookie: `thothii_session=${created?.token}`, origin: requestSnapshot.value.publicUrl }, + }); + + expect(fixture.calls()).toBe(1); + expect(response.headers["access-control-allow-origin"]).toBe(new URL(requestSnapshot.value.publicUrl).origin); + expect(response.statusCode).toBe(401); + expect(response.json()).toEqual({ code: "authentication_required", error: "Authentication is required" }); + expect(response.body).not.toContain(user.id); +}); + +test.each([ + { first: "A" as const, later: "B" as const }, + { first: "B" as const, later: "A" as const }, +])("local login uses and stamps its one $first request snapshot despite $later replacement", async ({ first, later }) => { + const fixture = await createFixture(first, later); + const snapshot = fixture.snapshots[first]; + const user = fixture.userFor(snapshot); + fixture.resetCalls(); + const response = await fixture.app.inject({ + method: "POST", + url: "/auth/local/login", + headers: { origin: snapshot.value.publicUrl, "sec-fetch-site": "same-origin" }, + payload: { username: user.username, password }, + }); + + expect(response.statusCode).toBe(200); + expect(fixture.calls()).toBe(1); + const token = firstCookie(response).split("=", 2)[1] ?? ""; + fixture.resetCalls(); + const record = await fixture.app.thothiiAuthSessionStore?.resolve(token); + expect(record).toMatchObject({ subject: user.id, authConfigRevision: snapshot.revision }); +}); + +test("auth config and valid preflight use the same first snapshot when the provider is replaced", async () => { + const fixture = await createFixture("A", "oidc"); + fixture.resetCalls(); + const preflight = await fixture.app.inject({ + method: "OPTIONS", + url: "/me", + headers: { origin: originA, "access-control-request-method": "GET" }, + }); + expect(preflight.statusCode).toBe(204); + expect(preflight.headers["access-control-allow-origin"]).toBe(originA); + expect(fixture.calls()).toBe(1); + + fixture.resetCalls(); + const response = await fixture.app.inject({ method: "GET", url: "/auth/config", headers: { origin: originA } }); + expect(response.statusCode).toBe(200); + expect(response.headers["access-control-allow-origin"]).toBe(originA); + expect(response.json()).toEqual({ mode: "local", localLogin: true, oidcLogin: false }); + expect(fixture.calls()).toBe(1); +}); diff --git a/backend/test/auth-routes-local.test.ts b/backend/test/auth-routes-local.test.ts new file mode 100644 index 00000000..8ff97319 --- /dev/null +++ b/backend/test/auth-routes-local.test.ts @@ -0,0 +1,519 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { chmodSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { LoginFailureLimiter } from "../src/auth/routes.js"; +import { createFixtureAuthStorageBridge, prepareAuthStateRoot } from "./auth-test-fixtures.js"; + +const password = "correct horse battery staple"; +const passwordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"; +const adminId = "6ba7b810-9dad-4ed1-80b4-00c04fd430c8"; +const publicUrl = "http://127.0.0.1:8787"; +const cleanups: Array<() => Promise> = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); + +function localConfig(url = publicUrl) { + return { + version: 1, + mode: "local", + publicUrl: url, + local: { usersFile: "users.yaml" }, + }; +} + +function usersYaml(options: { enabled?: boolean; username?: string } = {}): string { + const users = [ + "version: 1", + "users:", + ` - id: ${adminId}`, + ` username: ${options.username ?? "Admin"}`, + " displayName: Local administrator", + ` passwordHash: ${passwordHash}`, + " roles:", + " - admin", + ` enabled: ${options.enabled ?? true}`, + " authRevision: 1", + ]; + if (options.enabled === false) { + users.push( + " - id: 6ba7b811-9dad-4ed1-80b4-00c04fd430c8", + " username: BackupAdmin", + ` passwordHash: ${passwordHash}`, + " roles:", + " - admin", + " enabled: true", + " authRevision: 1", + ); + } + return [...users, ""].join("\n"); +} + +function firstSetCookie(response: { headers: Record }): string { + const header = response.headers["set-cookie"]; + if (Array.isArray(header)) return header[0] ?? ""; + return header ?? ""; +} + +function cookiePair(setCookie: string): string { + return setCookie.split(";", 1)[0] ?? ""; +} + +async function createLocalApp(options: { + publicUrl?: string; + enabled?: boolean; + stateRoot?: string; + registry?: { + findByUsername(username: string): Promise; + findBySubject(subject: string): Promise; + verify(user: unknown, suppliedPassword: string): Promise; + }; +} = {}) { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-routes-")); + chmodSync(directory, 0o700); + const authConfigFile = join(directory, "auth.yaml"); + const usersFile = join(directory, "users.yaml"); + const authStateRoot = options.stateRoot ?? join(directory, "auth-state"); + writeFileSync(authConfigFile, stringify(localConfig(options.publicUrl)), { encoding: "utf8", mode: 0o600 }); + writeFileSync(usersFile, usersYaml({ enabled: options.enabled }), { encoding: "utf8", mode: 0o600 }); + chmodSync(authConfigFile, 0o600); + chmodSync(usersFile, 0o600); + prepareAuthStateRoot(authStateRoot); + const app = buildApp(loadConfig({ + THT_AUTH_CONFIG_FILE: authConfigFile, + THT_AUTH_STATE_ROOT: authStateRoot, + THT_HARNESS_DIR: "/tmp/h", + }), { + ...(options.registry === undefined ? {} : { localUserRegistry: options.registry }), + authStorageBridgeForTest: createFixtureAuthStorageBridge(), + } as any); + cleanups.push(async () => { + await app.close(); + rmSync(directory, { recursive: true, force: true }); + }); + return { app, authConfigFile, usersFile, authStateRoot, directory, publicUrl: options.publicUrl ?? publicUrl }; +} + +async function login(app: Awaited>["app"], body: Record = {}) { + return app.inject({ + method: "POST", + url: "/auth/local/login", + headers: { origin: publicUrl, "sec-fetch-site": "same-origin" }, + payload: { username: "Admin", password, ...body }, + }); +} + +test("local login sets a non-persistent opaque session cookie and exposes only a safe /me DTO", async () => { + const { app } = await createLocalApp(); + + const signedIn = await login(app); + expect(signedIn.statusCode).toBe(200); + const setCookie = firstSetCookie(signedIn); + expect(setCookie).toMatch(/^thothii_session=[A-Za-z0-9_-]{43}; /); + expect(setCookie).toContain("HttpOnly"); + expect(setCookie).toContain("SameSite=Lax"); + expect(setCookie).toContain("Path=/"); + expect(setCookie).not.toMatch(/Max-Age=/i); + expect(setCookie).not.toContain("Secure"); + + const me = await app.inject({ method: "GET", url: "/me", headers: { cookie: cookiePair(setCookie) } }); + expect(me.statusCode).toBe(200); + expect(me.json()).toEqual({ + issuer: "local", + subject: adminId, + displayName: "Local administrator", + roles: ["admin"], + permissions: [ + "session.use", "session.read_all", "session.manage_all", "settings.manage", + "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + ], + isAdmin: true, + csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/), + session: { + method: "local", + remembered: false, + idleExpiresAt: expect.any(String), + absoluteExpiresAt: expect.any(String), + }, + }); + expect(JSON.stringify(me.json())).not.toContain("authConfigRevision"); + expect(JSON.stringify(me.json())).not.toContain("authRevision"); + expect(JSON.stringify(me.json())).not.toContain(cookiePair(setCookie).split("=", 2)[1] ?? ""); +}); + +test("remembered login uses a persistent secure cookie under an HTTPS public URL and survives app recreation", async () => { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-remembered-")); + chmodSync(directory, 0o700); + const authConfigFile = join(directory, "auth.yaml"); + const usersFile = join(directory, "users.yaml"); + const authStateRoot = join(directory, "auth-state"); + writeFileSync(authConfigFile, stringify(localConfig("https://thothii.example.test")), { encoding: "utf8", mode: 0o600 }); + writeFileSync(usersFile, usersYaml(), { encoding: "utf8", mode: 0o600 }); + chmodSync(authConfigFile, 0o600); + chmodSync(usersFile, 0o600); + prepareAuthStateRoot(authStateRoot); + const config = () => loadConfig({ THT_AUTH_CONFIG_FILE: authConfigFile, THT_AUTH_STATE_ROOT: authStateRoot, THT_HARNESS_DIR: "/tmp/h" }); + const first = buildApp(config(), { authStorageBridgeForTest: createFixtureAuthStorageBridge() }); + try { + const signedIn = await first.inject({ + method: "POST", + url: "/auth/local/login", + headers: { origin: "https://thothii.example.test", "sec-fetch-site": "same-origin" }, + payload: { username: "Admin", password, remember: true }, + }); + const setCookie = firstSetCookie(signedIn); + expect(signedIn.statusCode).toBe(200); + expect(setCookie).toContain("Max-Age=2592000"); + expect(setCookie).toContain("Secure"); + await first.close(); + + const restarted = buildApp(config(), { authStorageBridgeForTest: createFixtureAuthStorageBridge() }); + cleanups.push(async () => { + await restarted.close(); + rmSync(directory, { recursive: true, force: true }); + }); + const me = await restarted.inject({ method: "GET", url: "/me", headers: { cookie: cookiePair(setCookie) } }); + expect(me.statusCode).toBe(200); + expect(me.json()).toMatchObject({ subject: adminId, session: { remembered: true, method: "local" } }); + } catch (error) { + await first.close(); + rmSync(directory, { recursive: true, force: true }); + throw error; + } +}); + +test("unknown, disabled, and wrong-password logins share one generic failure contract", async () => { + const enabled = await createLocalApp(); + const disabled = await createLocalApp({ enabled: false }); + const attempts = await Promise.all([ + login(enabled.app, { username: "Unknown" }), + login(disabled.app), + login(enabled.app, { password: `${password}!` }), + ]); + + for (const response of attempts) { + expect(response.statusCode).toBe(401); + expect(response.json()).toEqual({ code: "invalid_credentials", error: "Invalid username or password" }); + expect(response.headers["set-cookie"]).toBeUndefined(); + } +}); + +test("invalid password input still reaches the local verifier with a bounded Argon2-safe surrogate", async () => { + const user = { + id: adminId, username: "Admin", normalizedUsername: "admin", displayName: "Local administrator", + passwordHash, roles: ["admin"], enabled: true, authRevision: 1, + }; + const verify = vi.fn(async () => false); + const { app } = await createLocalApp({ + registry: { findByUsername: async () => user, findBySubject: async () => user, verify }, + }); + + const response = await login(app, { password: "short" }); + expect(response.statusCode).toBe(401); + const verifierPassword = verify.mock.calls[0]?.[1]; + expect(verifierPassword).not.toBe("short"); + expect(Buffer.byteLength(verifierPassword ?? "", "utf8")).toBeGreaterThanOrEqual(12); +}); + +test("local login requires the exact configured Origin and same-origin Fetch Metadata", async () => { + const { app } = await createLocalApp(); + const missingOrigin = await app.inject({ method: "POST", url: "/auth/local/login", payload: { username: "Admin", password } }); + const wrongOrigin = await app.inject({ + method: "POST", url: "/auth/local/login", headers: { origin: "http://127.0.0.1:8788" }, payload: { username: "Admin", password }, + }); + const crossSite = await app.inject({ + method: "POST", url: "/auth/local/login", headers: { origin: publicUrl, "sec-fetch-site": "cross-site" }, payload: { username: "Admin", password }, + }); + + for (const response of [missingOrigin, wrongOrigin, crossSite]) { + expect(response.statusCode).toBe(403); + expect(response.json()).toEqual({ code: "csrf_failed", error: "Request origin validation failed" }); + } +}); + +test("logout revokes the session and clears the cookie with the production attributes", async () => { + const { app } = await createLocalApp(); + const signedIn = await login(app); + const setCookie = firstSetCookie(signedIn); + const me = await app.inject({ method: "GET", url: "/me", headers: { cookie: cookiePair(setCookie) } }); + + const loggedOut = await app.inject({ + method: "POST", + url: "/auth/logout", + headers: { + cookie: cookiePair(setCookie), origin: publicUrl, "sec-fetch-site": "same-origin", + "x-thothii-csrf": me.json().csrfToken, + }, + }); + expect(loggedOut.statusCode).toBe(204); + const cleared = firstSetCookie(loggedOut); + expect(cleared).toMatch(/^thothii_session=;/); + expect(cleared).toContain("Max-Age=0"); + expect(cleared).toContain("HttpOnly"); + expect(cleared).toContain("SameSite=Lax"); + expect(cleared).toContain("Path=/"); + expect(cleared).toContain("Expires="); + expect((await app.inject({ method: "GET", url: "/me", headers: { cookie: cookiePair(setCookie) } })).statusCode).toBe(401); +}); + +test.each([false, true])( + "an uppercase HTTPS public URL sets and clears the secure cookie for remembered=%s", + async (remember) => { + const configuredPublicUrl = "HTTPS://thothii.example.test"; + const origin = new URL(configuredPublicUrl).origin; + const { app } = await createLocalApp({ publicUrl: configuredPublicUrl }); + + const signedIn = await app.inject({ + method: "POST", + url: "/auth/local/login", + headers: { origin, "sec-fetch-site": "same-origin" }, + payload: { username: "Admin", password, remember }, + }); + const setCookie = firstSetCookie(signedIn); + expect(signedIn.statusCode).toBe(200); + expect(setCookie).toContain("Secure"); + if (remember) expect(setCookie).toContain("Max-Age=2592000"); + else expect(setCookie).not.toMatch(/Max-Age=/i); + + const me = await app.inject({ method: "GET", url: "/me", headers: { cookie: cookiePair(setCookie) } }); + const loggedOut = await app.inject({ + method: "POST", + url: "/auth/logout", + headers: { + cookie: cookiePair(setCookie), + origin, + "sec-fetch-site": "same-origin", + "x-thothii-csrf": me.json().csrfToken, + }, + }); + + expect(loggedOut.statusCode).toBe(204); + expect(firstSetCookie(loggedOut)).toContain("Secure"); + }, +); + +test("failed logins are limited by normalized username and source address", async () => { + const user = { + id: adminId, username: "Admin", normalizedUsername: "admin", displayName: "Local administrator", + passwordHash, roles: ["admin"], enabled: true, authRevision: 1, + }; + const registry = { + findByUsername: async () => user, + findBySubject: async () => user, + verify: async () => false, + }; + const { app } = await createLocalApp({ registry }); + for (let attempt = 0; attempt < 10; attempt += 1) { + const response = await login(app, { username: "aDmIn" }); + expect(response.statusCode).toBe(401); + } + const limited = await login(app, { username: "ADMIN" }); + expect(limited.statusCode).toBe(429); + expect(limited.json()).toEqual({ code: "login_rate_limited", error: "Too many login attempts" }); + + const addressLimited = await createLocalApp({ registry }); + for (let attempt = 0; attempt < 20; attempt += 1) { + const response = await login(addressLimited.app, { username: `User${attempt}` }); + expect(response.statusCode).toBe(401); + } + expect((await login(addressLimited.app, { username: "A-new-username" })).statusCode).toBe(429); +}); + +test("failed-attempt limiter keeps exact bounded windows without check-time mutation or active-key eviction", () => { + const now = 1_000_000; + const limiter = new LoginFailureLimiter({ maximumEntries: 2 }); + + expect(limiter.isLimited("checked-only", "127.0.0.1", now)).toBe(false); + expect(limiter.recordFailure("victim", "127.0.0.1", now)).toBe(true); + expect(limiter.recordFailure("attacker", "127.0.0.1", now)).toBe(true); + expect(limiter.recordFailure("flood", "127.0.0.1", now)).toBe(false); + + for (let attempt = 1; attempt < 10; attempt += 1) { + expect(limiter.recordFailure("victim", "127.0.0.1", now)).toBe(true); + } + expect(limiter.isLimited("victim", "127.0.0.1", now)).toBe(true); + expect(limiter.recordFailure("victim", "127.0.0.1", now)).toBe(false); + expect(limiter.isLimited("victim", "127.0.0.1", now + 10 * 60 * 1000 - 1)).toBe(true); + expect(limiter.isLimited("victim", "127.0.0.1", now + 10 * 60 * 1000)).toBe(false); + expect(limiter.recordFailure("victim", "127.0.0.1", now + 10 * 60 * 1000)).toBe(true); +}); + +test("failed-attempt limiter allows twenty source-address failures, then rejects the twenty-first", () => { + const limiter = new LoginFailureLimiter(); + const now = 1_000_000; + + for (let attempt = 0; attempt < 20; attempt += 1) { + expect(limiter.recordFailure(`user-${attempt}`, "127.0.0.1", now)).toBe(true); + } + expect(limiter.isLimited("new-user", "127.0.0.1", now)).toBe(true); + expect(limiter.recordFailure("new-user", "127.0.0.1", now)).toBe(false); +}); + +test("successful and pre-authentication failures never reset or consume failed-login counters", async () => { + let calls = 0; + const user = { + id: adminId, username: "Admin", normalizedUsername: "admin", displayName: "Local administrator", + passwordHash, roles: ["admin"], enabled: true, authRevision: 1, + }; + const { app } = await createLocalApp({ + registry: { + findByUsername: async () => user, + findBySubject: async () => user, + verify: async () => { + calls += 1; + return calls === 10; + }, + }, + }); + + const originRejected = await app.inject({ + method: "POST", url: "/auth/local/login", headers: { origin: "http://wrong.example.test" }, + payload: { username: "Admin", password }, + }); + expect(originRejected.statusCode).toBe(403); + expect(calls).toBe(0); + for (let attempt = 0; attempt < 9; attempt += 1) expect((await login(app)).statusCode).toBe(401); + expect((await login(app)).statusCode).toBe(200); + expect((await login(app)).statusCode).toBe(401); + expect((await login(app)).statusCode).toBe(429); + expect((await login(app)).statusCode).toBe(429); +}); + +test("only two Argon2 verifications run concurrently and excess login attempts fail immediately", async () => { + let calls = 0; + let release!: () => void; + const blocked = new Promise((resolve) => { release = resolve; }); + let entered!: () => void; + const twoEntered = new Promise((resolve) => { entered = resolve; }); + const user = { + id: adminId, username: "Admin", normalizedUsername: "admin", displayName: "Local administrator", + passwordHash, roles: ["admin"], enabled: true, authRevision: 1, + }; + const registry = { + findByUsername: async () => user, + findBySubject: async () => user, + verify: async () => { + calls += 1; + if (calls === 2) entered(); + await blocked; + return false; + }, + }; + const { app } = await createLocalApp({ registry }); + const first = login(app); + const second = login(app); + await twoEntered; + const excess = await login(app); + expect(excess.statusCode).toBe(429); + expect(calls).toBe(2); + release(); + expect((await first).statusCode).toBe(401); + expect((await second).statusCode).toBe(401); +}); + +test("the real native asynchronous Argon2 verifier holds two permits and releases them after completion", async () => { + const { app } = await createLocalApp(); + const first = login(app, { password: `${password}!` }); + const second = login(app, { password: `${password}!` }); + await new Promise((resolve) => setImmediate(resolve)); + + const excess = await login(app, { password: `${password}!` }); + expect(excess.statusCode).toBe(429); + await expect(first).resolves.toMatchObject({ statusCode: 401 }); + await expect(second).resolves.toMatchObject({ statusCode: 401 }); + await expect(login(app, { password: `${password}!` })).resolves.toMatchObject({ statusCode: 401 }); +}); + +test("a verifier failure is sanitized and releases its concurrency permit", async () => { + let attempts = 0; + const user = { + id: adminId, username: "Admin", normalizedUsername: "admin", displayName: "Local administrator", + passwordHash, roles: ["admin"], enabled: true, authRevision: 1, + }; + const { app } = await createLocalApp({ + registry: { + findByUsername: async () => user, + findBySubject: async () => user, + verify: async () => { + attempts += 1; + if (attempts === 1) throw new Error("fixture verifier failure"); + return false; + }, + }, + }); + + const failed = await login(app); + expect(failed.statusCode).toBe(503); + expect(failed.json()).toEqual({ code: "auth_unavailable", error: "Authentication is unavailable" }); + expect((await login(app)).statusCode).toBe(401); + expect(attempts).toBe(2); +}); + +test("operational registry failures do dummy work, return 503, and never consume login-failure capacity", async () => { + let available = false; + let verificationCalls = 0; + const user = { + id: adminId, username: "Admin", normalizedUsername: "admin", displayName: "Local administrator", + passwordHash, roles: ["admin"], enabled: true, authRevision: 1, + }; + const { app } = await createLocalApp({ + registry: { + findByUsername: async () => { + if (!available) throw new Error("registry file is unavailable"); + return user; + }, + findBySubject: async () => user, + verify: async () => { + verificationCalls += 1; + return false; + }, + }, + }); + + for (let attempt = 0; attempt < 11; attempt += 1) { + const response = await login(app); + expect(response.statusCode).toBe(503); + expect(response.json()).toEqual({ code: "auth_unavailable", error: "Authentication is unavailable" }); + } + expect(verificationCalls).toBe(0); + + available = true; + for (let attempt = 0; attempt < 10; attempt += 1) expect((await login(app)).statusCode).toBe(401); + expect((await login(app)).statusCode).toBe(429); +}); + +test("operational config failures return 503 and never consume login-failure capacity", async () => { + const { app, authConfigFile } = await createLocalApp(); + writeFileSync(authConfigFile, "version: 1\nmode: unsupported\n", { encoding: "utf8", mode: 0o600 }); + chmodSync(authConfigFile, 0o600); + + for (let attempt = 0; attempt < 11; attempt += 1) { + const response = await login(app); + expect(response.statusCode).toBe(503); + expect(response.json()).toEqual({ code: "auth_unavailable", error: "Authentication is unavailable" }); + } + + writeFileSync(authConfigFile, stringify(localConfig()), { encoding: "utf8", mode: 0o600 }); + chmodSync(authConfigFile, 0o600); + for (let attempt = 0; attempt < 10; attempt += 1) { + expect((await login(app, { password: `${password}!` })).statusCode).toBe(401); + } + expect((await login(app, { password: `${password}!` })).statusCode).toBe(429); +}); + +test("public auth configuration is safe and unavailable OIDC login fails closed", async () => { + const { app } = await createLocalApp(); + const configuration = await app.inject({ method: "GET", url: "/auth/config" }); + expect(configuration.statusCode).toBe(200); + expect(configuration.json()).toEqual({ mode: "local", localLogin: true, oidcLogin: false }); + expect(JSON.stringify(configuration.json())).not.toContain("users.yaml"); + + const placeholder = await app.inject({ method: "GET", url: "/auth/oidc/login" }); + expect(placeholder.statusCode).toBe(503); + expect(placeholder.json()).toEqual({ code: "auth_unavailable", error: "Authentication is unavailable" }); +}); diff --git a/backend/test/auth-routes-oidc.test.ts b/backend/test/auth-routes-oidc.test.ts new file mode 100644 index 00000000..8e4fe4c7 --- /dev/null +++ b/backend/test/auth-routes-oidc.test.ts @@ -0,0 +1,584 @@ +import Fastify from "fastify"; +import cookie from "@fastify/cookie"; +import { afterEach, expect, test, vi } from "vitest"; +import { registerAuthRoutes } from "../src/auth/routes.js"; +import type { LoadedAuthConfig, OidcStateRecord } from "../src/auth/types.js"; +import type { AuthSessionStore } from "../src/auth/session-store.js"; +import { createOidcProtocol, type OidcProtocol } from "../src/auth/oidc-client.js"; + +const revision = "a".repeat(64); +const issuer = "https://issuer.example.test"; +const state = "s".repeat(43); +const nonce = "n".repeat(43); +const verifier = "v".repeat(43); +const transactionCookieName = "__Host-thothii_oidc_tx"; +const loopbackTransactionCookieName = "thothii_oidc_tx"; +const createdApps: Array> = []; + +function setCookieHeaders(response: { headers: Record }): string[] { + const header = response.headers["set-cookie"]; + return header === undefined ? [] : Array.isArray(header) ? header : [header]; +} + +function transactionCookie( + response: { headers: Record }, + name = transactionCookieName, +): string | undefined { + return setCookieHeaders(response) + .find((header) => header.startsWith(`${name}=`) && !header.includes("Max-Age=0")) + ?.split(";", 1)[0]; +} + +function expectTransactionCleared( + response: { headers: Record }, + name = transactionCookieName, + secure = true, +): void { + const header = setCookieHeaders(response).find((candidate) => + candidate.startsWith(`${name}=`) && candidate.includes("Max-Age=0")); + expect(header).toBeDefined(); + expect(header).toContain("HttpOnly"); + expect(header).toContain("SameSite=Lax"); + expect(header).toContain("Path=/"); + expect(header).not.toContain("Domain="); + if (secure) expect(header).toContain("Secure"); + else expect(header).not.toContain("Secure"); +} + +function expectBothTransactionVariantsCleared( + response: { headers: Record }, + activeName: string, + activeSecure: boolean, +): void { + expectTransactionCleared(response, activeName, activeSecure); + expectTransactionCleared( + response, + activeName === transactionCookieName ? loopbackTransactionCookieName : transactionCookieName, + activeName !== transactionCookieName, + ); +} + +function config(overrides: Partial = {}): LoadedAuthConfig { + return { + revision, + sourcePath: "/private/auth.yaml", + value: { + version: 1, + mode: "oidc", + publicUrl: "https://thothii.example.test", + session: { + regularTtlSeconds: 3600, regularIdleSeconds: 300, + rememberTtlSeconds: 3600, rememberIdleSeconds: 300, oidcTtlSeconds: 3600, + }, + oidc: { + issuer, clientId: "thothii", clientSecretRef: "THT_OIDC_CLIENT_SECRET", + scopes: ["openid", "profile"], groupsClaim: "groups", + }, + groupCatalog: { driver: "authentik", baseUrl: issuer, apiTokenRef: "THT_AUTHENTIK_API_TOKEN" }, + authorization: { groupRoles: { Users: ["user"], Admins: ["admin"] } }, + ...overrides, + }, + } as LoadedAuthConfig; +} + +function stateRecord(extra: Partial = {}): OidcStateRecord { + return { + version: 1, nonce, codeVerifier: verifier, returnTo: "/", + authConfigRevision: revision, issuer, + browserTransactionDigest: "b".repeat(64), + browserTransactionTransport: "https", + createdAt: "2030-01-01T00:00:00.000Z", expiresAt: "2030-01-01T00:10:00.000Z", + ...extra, + } as OidcStateRecord; +} + +function fixture(options: { + loaded?: LoadedAuthConfig; + identity?: Awaited>; + protocol?: OidcProtocol; + callbackFailure?: boolean; + stateReturnTo?: string; +} = {}) { + let loaded = options.loaded ?? config(); + let protocolAvailable = true; + let storedState: OidcStateRecord | undefined; + const stateInputs: Array> = []; + const creates: Array> = []; + const createTimes: Array = []; + const callbacks: URL[] = []; + let authorizationRequests = 0; + const defaultProtocol: OidcProtocol = { + authorizationUrl: async ({ state: received, nonce: receivedNonce, codeVerifier }) => { + authorizationRequests += 1; + expect(received).toBe(state); + expect(receivedNonce).toHaveLength(43); + expect(codeVerifier).toHaveLength(43); + return new URL(`https://issuer.example.test/authorize?state=${received}`); + }, + callback: async ({ currentUrl }) => { + callbacks.push(currentUrl); + if (options.callbackFailure) throw new Error("provider failure with access-token-must-not-leak"); + return options.identity ?? { + issuer, subject: "user-123", displayName: "Ada", groups: ["Users", "Admins", "Unmapped"], + tokenExpiresAt: new Date(Date.now() + 120_000), + }; + }, + diagnose: async () => undefined, + }; + const protocol = options.protocol ?? defaultProtocol; + const store = { + createOidcState: async (input: Record) => { + stateInputs.push(input); + const record = stateRecord({ + nonce: input.nonce as string, + codeVerifier: input.codeVerifier as string, + authConfigRevision: input.authConfigRevision as string, + issuer: input.issuer as string, + }); + (record as OidcStateRecord & { browserTransactionDigest: string }).browserTransactionDigest = + input.browserTransactionDigest as string; + (record as OidcStateRecord & { browserTransactionTransport?: string }).browserTransactionTransport = + input.browserTransactionTransport as string | undefined ?? "https"; + if (options.stateReturnTo) (record as { returnTo: string }).returnTo = options.stateReturnTo; + storedState = record; + return { state, record: storedState }; + }, + consumeOidcState: async (received: string) => { + if (received !== state) return undefined; + const consumed = storedState; + storedState = undefined; + return consumed; + }, + create: async (input: Record, now?: Date) => { + creates.push(input); + createTimes.push(now); + return { token: "opaque-session-token", csrfToken: "c".repeat(43), record: {} }; + }, + } as unknown as AuthSessionStore; + const app = Fastify(); + app.decorateRequest("authConfigSnapshot", undefined); + app.decorateRequest("authConfigSnapshotCaptured", false); + app.decorateRequest("authConfigSnapshotUnavailable", false); + app.register(cookie); + registerAuthRoutes(app, { + authMode: "oidc", + authentication: { current: () => loaded }, + sessionStore: store, + resolveOidcProtocol: () => protocolAvailable ? protocol : undefined, + }); + createdApps.push(app); + return { + app, creates, createTimes, callbacks, stateInputs, + authorizationRequests: () => authorizationRequests, + setConfig(next: LoadedAuthConfig) { loaded = next; }, + setProtocolAvailable(available: boolean) { protocolAvailable = available; }, + stateWasConsumed: () => storedState === undefined, + }; +} + +afterEach(async () => { + await Promise.all(createdApps.splice(0).map((app) => app.close())); +}); + +async function beginOidcLogin(subject: ReturnType) { + const response = await subject.app.inject({ method: "GET", url: "/auth/oidc/login" }); + return { + response, + cookie: transactionCookie(response) ?? transactionCookie(response, loopbackTransactionCookieName), + }; +} + +async function finishOidcLogin( + subject: ReturnType, + cookie: string | undefined, + query = `code=good&state=${state}`, +) { + return await subject.app.inject({ + method: "GET", + url: `/auth/oidc/callback?${query}`, + ...(cookie ? { headers: { cookie } } : {}), + }); +} + +async function completesWithin(operation: Promise, timeoutMs = 150): Promise { + return await new Promise((resolve) => { + const timeout = setTimeout(() => resolve(undefined), timeoutMs); + operation.then( + (value) => { clearTimeout(timeout); resolve(value); }, + () => { clearTimeout(timeout); resolve(undefined); }, + ); + }); +} + +test("rate limits OIDC initiation before state creation and provider discovery", async () => { + const subject = fixture(); + + for (let attempt = 0; attempt < 20; attempt += 1) { + expect((await beginOidcLogin(subject)).response.statusCode).toBe(302); + } + const limited = await beginOidcLogin(subject); + + expect(limited.response.statusCode).toBe(429); + expect(limited.response.json()).toEqual({ + code: "login_rate_limited", + error: "Too many login attempts", + }); + expect(subject.stateInputs).toHaveLength(20); + expect(subject.authorizationRequests()).toBe(20); +}); + +test("OIDC initiation rate-limit capacity expires after ten minutes", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-02T03:04:05.000Z")); + const subject = fixture(); + try { + for (let attempt = 0; attempt < 20; attempt += 1) { + expect((await beginOidcLogin(subject)).response.statusCode).toBe(302); + } + expect((await beginOidcLogin(subject)).response.statusCode).toBe(429); + + vi.advanceTimersByTime(10 * 60_000 + 1); + + expect((await beginOidcLogin(subject)).response.statusCode).toBe(302); + expect(subject.authorizationRequests()).toBe(21); + } finally { + vi.useRealTimers(); + } +}); + +test("creates digest-only bound state, maps exact groups, creates a cookie session, and redirects safely", async () => { + const subject = fixture(); + const { response: start, cookie } = await beginOidcLogin(subject); + expect(start.statusCode).toBe(302); + expect(new URL(start.headers.location ?? "").searchParams.get("state")).toBe(state); + expect(subject.stateInputs[0]).toMatchObject({ + returnTo: "/", authConfigRevision: revision, issuer, + browserTransactionDigest: expect.stringMatching(/^[a-f0-9]{64}$/), + browserTransactionTransport: "https", + }); + expect(cookie).toMatch(new RegExp(`^${transactionCookieName}=[A-Za-z0-9_-]{43}$`)); + const issued = setCookieHeaders(start) + .find((header) => header.startsWith(`${transactionCookieName}=`) && !header.includes("Max-Age=0")); + expect(issued).toContain("HttpOnly"); + expect(issued).toContain("SameSite=Lax"); + expect(issued).toContain("Secure"); + expect(issued).toContain("Path=/"); + expect(issued).not.toContain("Domain="); + expect(JSON.stringify(subject.stateInputs)).not.toContain(cookie?.split("=", 2)[1] ?? "missing-cookie"); + + const callback = await subject.app.inject({ + method: "GET", + url: `/auth/oidc/callback?code=good&state=${state}`, + headers: { host: "attacker.example.test", cookie: cookie ?? "" }, + }); + expect(callback.statusCode).toBe(302); + expect(callback.headers.location).toBe("/"); + expect(setCookieHeaders(callback).join("\n")).toContain("HttpOnly"); + expect(setCookieHeaders(callback).join("\n")).toContain("SameSite=Lax"); + expect(setCookieHeaders(callback).join("\n")).toContain("Secure"); + expectBothTransactionVariantsCleared(callback, transactionCookieName, true); + expect(subject.callbacks[0]?.href).toBe(`https://thothii.example.test/api/auth/oidc/callback?code=good&state=${state}`); + expect(subject.creates).toHaveLength(1); + expect(subject.creates[0]).toMatchObject({ + method: "oidc", remembered: false, authConfigRevision: revision, + principal: { issuer, subject: "user-123", roles: ["user", "admin"] }, + idleTtlMs: 300_000, + }); + const absoluteTtlMs = subject.creates[0]?.absoluteTtlMs; + expect(typeof absoluteTtlMs).toBe("number"); + expect(absoluteTtlMs as number).toBeGreaterThan(0); + expect(absoluteTtlMs as number).toBeLessThanOrEqual(120_000); + expect(JSON.stringify(subject.creates)).not.toContain("access-token-must-not-leak"); + expect(JSON.stringify(subject.creates)).not.toContain("refresh-token-must-not-leak"); +}); + +test.each([ + "http://127.42.0.1:8787", + "http://[::1]:8787", +])("uses the non-prefixed, non-Secure transaction cookie for literal loopback OIDC %s", async (publicUrl) => { + const subject = fixture({ loaded: config({ publicUrl }) }); + const { response: start, cookie } = await beginOidcLogin(subject); + + expect(start.statusCode).toBe(302); + expect(subject.stateInputs[0]).toMatchObject({ browserTransactionTransport: "loopback_http" }); + expect(cookie).toMatch(new RegExp(`^${loopbackTransactionCookieName}=[A-Za-z0-9_-]{43}$`)); + const issued = setCookieHeaders(start).find((header) => header.startsWith(`${loopbackTransactionCookieName}=`)); + expect(issued).toContain("HttpOnly"); + expect(issued).toContain("SameSite=Lax"); + expect(issued).toContain("Path=/"); + expect(issued).not.toContain("Secure"); + expect(issued).not.toContain("Domain="); + + const callback = await finishOidcLogin(subject, cookie); + expect(callback.statusCode).toBe(302); + expect(subject.callbacks[0]?.href).toBe(`${publicUrl}/api/auth/oidc/callback?code=good&state=${state}`); + expectBothTransactionVariantsCleared(callback, loopbackTransactionCookieName, false); +}); + +test("uses the consumed state transport to reject cross-mode and shadowed transaction cookies", async () => { + const subject = fixture({ loaded: config({ publicUrl: "http://127.0.0.1:8787" }) }); + const { cookie } = await beginOidcLogin(subject); + expect(cookie).toBeDefined(); + + const failed = await finishOidcLogin(subject, [ + cookie, + `${transactionCookieName}=${"x".repeat(43)}`, + ].join("; ")); + + expect(failed.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); + expectBothTransactionVariantsCleared(failed, loopbackTransactionCookieName, false); +}); + +test("rejects a transaction presented only under the wrong transport cookie name", async () => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + const wrongTransportCookie = cookie?.replace(transactionCookieName, loopbackTransactionCookieName); + + const failed = await finishOidcLogin(subject, wrongTransportCookie); + + expect(failed.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); +}); + +test("rejects duplicate same-mode transaction cookies instead of trusting a parser-selected value", async () => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + expect(cookie).toBeDefined(); + + const failed = await finishOidcLogin(subject, [ + cookie, + `${transactionCookieName}=${"x".repeat(43)}`, + ].join("; ")); + + expect(failed.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); + expect(subject.stateWasConsumed()).toBe(true); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); +}); + +test("pins the callback transaction transport to its captured configuration snapshot", async () => { + const subject = fixture({ loaded: config({ publicUrl: "http://127.42.0.1:8787" }) }); + const { cookie } = await beginOidcLogin(subject); + subject.setConfig(config({ publicUrl: "https://thothii.example.test" })); + + const failed = await finishOidcLogin(subject, cookie); + + expect(failed.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); + expect(subject.stateWasConsumed()).toBe(true); + expectBothTransactionVariantsCleared(failed, loopbackTransactionCookieName, false); +}); + +test("clears the state-pinned loopback transaction variant after a terminal callback failure", async () => { + const subject = fixture({ + loaded: config({ publicUrl: "http://127.0.0.1:8787" }), + callbackFailure: true, + }); + const { cookie } = await beginOidcLogin(subject); + const failed = await finishOidcLogin(subject, cookie); + + expect(failed.statusCode).toBe(401); + expectBothTransactionVariantsCleared(failed, loopbackTransactionCookieName, false); +}); + +test("consumes state on callback failure and refuses replay", async () => { + const subject = fixture({ callbackFailure: true }); + const { cookie } = await beginOidcLogin(subject); + const failed = await finishOidcLogin(subject, cookie); + expect(failed.statusCode).toBe(401); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); + expect(subject.creates).toEqual([]); + const failedOutput = JSON.stringify({ body: failed.json(), cookies: setCookieHeaders(failed) }); + expect(failedOutput).not.toContain("access-token-must-not-leak"); + expect(failedOutput).not.toContain("opaque-session-token"); + const replay = await finishOidcLogin(subject, cookie); + expect(replay.statusCode).toBe(401); + expect(subject.callbacks).toHaveLength(1); +}); + +test("fails a bounded concrete-provider callback without a session, cookie, or token persistence", async () => { + let tokenAborted = false; + const protocol = createOidcProtocol({ + issuer, + clientId: "thothii", + clientSecret: "client-secret-must-not-persist", + callbackUrl: "https://thothii.example.test/api/auth/oidc/callback", + scopes: ["openid", "profile"], + groupsClaim: "groups", + httpTimeoutMs: 20, + fetch: async (input, init) => { + const url = new URL(input instanceof Request ? input.url : input.toString()); + if (url.pathname.includes(".well-known/")) { + return Response.json({ + issuer, + authorization_endpoint: `${issuer}/authorize`, + token_endpoint: `${issuer}/token`, + jwks_uri: `${issuer}/jwks`, + response_types_supported: ["code"], + grant_types_supported: ["authorization_code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }); + } + if (url.pathname === "/token") { + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + tokenAborted = true; + reject(new Error("provider access-token-must-not-persist")); + }, { once: true }); + }); + } + return new Response(null, { status: 404 }); + }, + }); + const subject = fixture({ protocol }); + const { cookie } = await beginOidcLogin(subject); + expect(cookie).toBeDefined(); + + const failed = await completesWithin(finishOidcLogin(subject, cookie)); + + expect(failed?.statusCode).toBe(401); + expect(tokenAborted).toBe(true); + expect(subject.creates).toEqual([]); + if (!failed) return; + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); + const failedOutput = JSON.stringify({ body: failed.json(), cookies: setCookieHeaders(failed) }); + expect(failedOutput).not.toContain("access-token-must-not-persist"); + expect(failedOutput).not.toContain("thothii_session"); +}); + +test("consumes state when the protocol becomes unavailable before callback", async () => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + subject.setProtocolAvailable(false); + const failed = await finishOidcLogin(subject, cookie); + expect(failed.statusCode).toBe(401); + expect(subject.stateWasConsumed()).toBe(true); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); +}); + +test("rejects a consumed state with a non-root return target", async () => { + const subject = fixture({ stateReturnTo: "https://attacker.example.test" }); + const { cookie } = await beginOidcLogin(subject); + const callback = await finishOidcLogin(subject, cookie); + expect(callback.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); + expect(subject.creates).toEqual([]); + expectBothTransactionVariantsCleared(callback, transactionCookieName, true); +}); + +test("rejects an OIDC state when its configuration revision changes before callback", async () => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + subject.setConfig({ ...config(), revision: "b".repeat(64) }); + const callback = await finishOidcLogin(subject, cookie); + expect(callback.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); + expectBothTransactionVariantsCleared(callback, transactionCookieName, true); +}); + +test("rejects an OIDC state when its issuer changes before callback", async () => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + const previous = config(); + subject.setConfig({ + ...previous, + value: { ...previous.value, oidc: { ...previous.value.oidc, issuer: "https://other.example.test" } }, + } as LoadedAuthConfig); + const callback = await finishOidcLogin(subject, cookie); + expect(callback.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); +}); + +test("creates an authenticated but forbidden principal for extra unmapped groups", async () => { + const subject = fixture({ identity: { + issuer, subject: "user-123", groups: ["Unmapped"], tokenExpiresAt: new Date(Date.now() + 60_000), + } }); + const { cookie } = await beginOidcLogin(subject); + const callback = await finishOidcLogin(subject, cookie); + expect(callback.statusCode).toBe(302); + expect(subject.creates[0]).toMatchObject({ principal: { roles: [], permissions: [], isAdmin: false } }); +}); + +test("rejects a callback from a different browser and clears the transaction cookie", async () => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + expect(cookie).toBeDefined(); + + const failed = await finishOidcLogin(subject, `${transactionCookieName}=${"x".repeat(43)}`); + expect(failed.statusCode).toBe(401); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); + expect(subject.stateWasConsumed()).toBe(true); + expect(subject.creates).toEqual([]); + + const replay = await finishOidcLogin(subject, cookie); + expect(replay.statusCode).toBe(401); + expect(subject.creates).toEqual([]); +}); + +test.each([ + ["unknown", `code=good&state=${state}&unexpected=value`], + ["duplicate", `code=good&code=other&state=${state}`], + ["malformed", `code=good&state=${state}&error_description=%00bad`], +])("burns canonical state before rejecting %s callback parameters", async (_label, query) => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + const failed = await finishOidcLogin(subject, cookie, query); + expect(failed.statusCode).toBe(401); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); + expect(subject.stateWasConsumed()).toBe(true); + expect(subject.callbacks).toEqual([]); + + const replay = await finishOidcLogin(subject, cookie); + expect(replay.statusCode).toBe(401); + expect(subject.callbacks).toEqual([]); +}); + +test("boundedly burns a canonical state from an oversized callback URL", async () => { + const subject = fixture(); + const { cookie } = await beginOidcLogin(subject); + const oversized = `state=${state}&code=good&padding=${"x".repeat(4096)}`; + + const failed = await finishOidcLogin(subject, cookie, oversized); + + expect(failed.statusCode).toBe(401); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); + expect(subject.stateWasConsumed()).toBe(true); + expect(subject.callbacks).toEqual([]); + expect((await finishOidcLogin(subject, cookie)).statusCode).toBe(401); +}); + +test("rejects an empty direct groups claim without creating a session cookie", async () => { + const subject = fixture({ identity: { + issuer, subject: "user-123", groups: [], tokenExpiresAt: new Date(Date.now() + 60_000), + } }); + const { cookie } = await beginOidcLogin(subject); + const callback = await finishOidcLogin(subject, cookie); + expect(callback.statusCode).toBe(401); + expectBothTransactionVariantsCleared(callback, transactionCookieName, true); + expect(subject.creates).toEqual([]); + expect(setCookieHeaders(callback).join("\n")).not.toContain("opaque-session-token"); +}); + +test("clears both transaction variants when the callback state cannot be consumed", async () => { + const subject = fixture(); + const failed = await finishOidcLogin(subject, `${transactionCookieName}=${"x".repeat(43)}`); + + expect(failed.statusCode).toBe(401); + expectBothTransactionVariantsCleared(failed, transactionCookieName, true); +}); + +test("uses one captured instant for token TTL derivation and session creation", async () => { + const tokenExpiresAt = new Date(Date.now() + 120_000); + const subject = fixture({ identity: { + issuer, subject: "user-123", groups: ["Users"], tokenExpiresAt, + } }); + const { cookie } = await beginOidcLogin(subject); + const callback = await finishOidcLogin(subject, cookie); + expect(callback.statusCode).toBe(302); + expect(subject.createTimes[0]).toBeInstanceOf(Date); + expect((subject.createTimes[0] as Date).getTime() + (subject.creates[0]?.absoluteTtlMs as number)) + .toBe(tokenExpiresAt.getTime()); +}); diff --git a/backend/test/auth-session-store.test.ts b/backend/test/auth-session-store.test.ts new file mode 100644 index 00000000..7531ef2f --- /dev/null +++ b/backend/test/auth-session-store.test.ts @@ -0,0 +1,1487 @@ +import { createHash } from "node:crypto"; +import { fork } from "node:child_process"; +import { + chmodSync, + existsSync, + linkSync, + lstatSync, + mkdirSync, + mkdtempSync, + opendirSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +const fsHooks = vi.hoisted(() => ({ + afterRead: undefined as undefined | (() => void), + afterWrite: undefined as undefined | (() => void), + beforeLstat: undefined as undefined | ((path: string) => boolean), + afterLstat: undefined as undefined | ((path: string) => boolean), + beforeReaddir: undefined as undefined | ((path: string) => void), + transformLstat: undefined as undefined | ((path: string, info: import("node:fs").Stats) => import("node:fs").Stats), +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readSync: (...args: Parameters) => { + const result = actual.readSync(...args); + const callback = fsHooks.afterRead; + fsHooks.afterRead = undefined; + callback?.(); + return result; + }, + writeSync: (...args: Parameters) => { + const result = actual.writeSync(...args); + const callback = fsHooks.afterWrite; + fsHooks.afterWrite = undefined; + callback?.(); + return result; + }, + lstatSync: (...args: Parameters) => { + const before = fsHooks.beforeLstat; + if (before?.(String(args[0]))) fsHooks.beforeLstat = undefined; + const original = actual.lstatSync(...args); + const result = fsHooks.transformLstat?.(String(args[0]), original) ?? original; + const callback = fsHooks.afterLstat; + if (callback?.(String(args[0]))) fsHooks.afterLstat = undefined; + return result; + }, + readdirSync: (...args: Parameters) => { + fsHooks.beforeReaddir?.(String(args[0])); + return actual.readdirSync(...args); + }, + }; +}); +import { + createFileAuthSessionStore, + deriveCsrfToken, + type AuthSessionStore, + type FileAuthSessionStoreOptions, + type SessionCreateInput, +} from "../src/auth/session-store.js"; +import { + createWindowsAuthStorageBridge, + type WindowsAuthStorageBridge, + type WindowsAuthStorageDirectory, +} from "../src/auth/windows-auth-storage.js"; + +const roots: string[] = []; +const base = new Date("2030-01-02T03:04:05.000Z"); +const revision = "a".repeat(64); +const validLocalUser = { enabled: true, authRevision: 7, roles: ["admin"] as const }; + +function oidcInput(nonce: string, codeVerifier: string) { + return { + nonce, + codeVerifier, + returnTo: "/" as const, + authConfigRevision: revision, + issuer: "https://issuer.example.test", + browserTransactionDigest: "b".repeat(64), + browserTransactionTransport: "https" as const, + }; +} + +function sessionFilename(index: number): string { + return `${index.toString(16).padStart(64, "0")}.json`; +} + +afterEach(() => { + fsHooks.afterRead = undefined; + fsHooks.afterWrite = undefined; + fsHooks.beforeLstat = undefined; + fsHooks.afterLstat = undefined; + fsHooks.beforeReaddir = undefined; + fsHooks.transformLstat = undefined; + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function root(): string { + const path = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-session-")); + chmodSync(path, 0o700); + for (const child of ["sessions", "oidc"]) { + mkdirSync(join(path, child), { mode: 0o700 }); + chmodSync(join(path, child), 0o700); + } + roots.push(path); + return path; +} + +function digestPath(rootPath: string, directory: "sessions" | "oidc", rawValue: string): string { + return join(rootPath, directory, `${createHash("sha256").update(rawValue).digest("hex")}.json`); +} + +function claimPath(rootPath: string, rawState: string): string { + return join(rootPath, "oidc", `${createHash("sha256").update(rawState).digest("hex")}.claim`); +} + +// This fixture adapter keeps session-store behavior tests self-contained. Production has no +// Node filesystem fallback: it always uses the hidden Go bridge. The Go authstorage suite owns +// descriptor-pinning/race assertions; this adapter only supplies ordinary fixture semantics. +function testPosixStorageBridge( + ensureOverride?: (rootPath: string) => Promise, +): WindowsAuthStorageBridge { + let overrideUsed = false; + const ensure = async (rootPath: string): Promise => { + if (ensureOverride && !overrideUsed) { + overrideUsed = true; + await ensureOverride(rootPath); + return; + } + if (!existsSync(rootPath)) { + if (realpathSync(dirname(rootPath)) !== dirname(rootPath)) throw new Error("invalid"); + mkdirSync(rootPath, { mode: 0o700 }); + chmodSync(rootPath, 0o700); + } + const rootInfo = lstatSync(rootPath); + if (!rootInfo.isDirectory() || rootInfo.isSymbolicLink() || (rootInfo.mode & 0o7777) !== 0o700) throw new Error("invalid"); + for (const child of ["sessions", "oidc"]) { + const path = join(rootPath, child); + if (!existsSync(path)) { + mkdirSync(path, { mode: 0o700 }); + chmodSync(path, 0o700); + } + const info = lstatSync(path); + if (!info.isDirectory() || info.isSymbolicLink() || (info.mode & 0o7777) !== 0o700) throw new Error("invalid"); + } + }; + const directory = async (rootPath: string, name: WindowsAuthStorageDirectory): Promise => { + await ensure(rootPath); + return join(rootPath, name); + }; + const record = (path: string, links: readonly number[]): import("node:fs").Stats => { + const info = lstatSync(path); + if (!info.isFile() || info.isSymbolicLink() || !links.includes(info.nlink) || (info.mode & 0o7777) !== 0o600 + || (typeof process.geteuid === "function" && info.uid !== process.geteuid())) { + throw new Error("invalid"); + } + return info; + }; + const same = (left: import("node:fs").Stats, right: import("node:fs").Stats): boolean => + left.dev === right.dev && left.ino === right.ino && left.nlink === right.nlink; + const paths = async (rootPath: string, name: WindowsAuthStorageDirectory, filename: string): Promise => + join(await directory(rootPath, name), filename); + const listNames = async ( + rootPath: string, + name: WindowsAuthStorageDirectory, + maximumEntries: number, + ): Promise<{ name: string; modifiedUnixMs: number }[]> => { + const handle = opendirSync(await directory(rootPath, name)); + const entries: string[] = []; + try { + for (;;) { + const entry = handle.readSync(); + if (entry === null) break; + entries.push(entry.name); + if (entries.length > maximumEntries) throw new Error("invalid"); + } + } finally { + handle.closeSync(); + } + return entries.sort().map((filename) => { + const links = name === "oidc" ? [1, 2] : [1]; + const info = record(join(rootPath, name, filename), links); + return { name: filename, modifiedUnixMs: info.mtimeMs }; + }); + }; + return { + validateRoot: async (rootPath) => { await ensure(rootPath); }, + ensureLayout: ensure, + readAuthConfig: (path) => readFileSync(path), + readLocalUsers: async (path) => readFileSync(path), + create: async (rootPath, name, filename, contents) => { + const path = await paths(rootPath, name, filename); + try { + writeFileSync(path, contents, { flag: "wx", mode: 0o600 }); + chmodSync(path, 0o600); + return true; + } catch (error: any) { + if (error?.code !== "EEXIST") throw error; + record(path, [1]); + return false; + } + }, + read: async (rootPath, name, filename) => { + const path = await paths(rootPath, name, filename); + try { + record(path, [1]); + } catch (error: any) { + if (error?.code === "ENOENT") return undefined; + throw error; + } + return readFileSync(path); + }, + replace: async (rootPath, name, filename, contents) => { + const path = await paths(rootPath, name, filename); + record(path, [1]); + const temporary = `${path}.test-replacement`; + writeFileSync(temporary, contents, { flag: "wx", mode: 0o600 }); + chmodSync(temporary, 0o600); + renameSync(temporary, path); + }, + remove: async (rootPath, name, filename) => { + const path = await paths(rootPath, name, filename); + try { + record(path, [1]); + } catch (error: any) { + if (error?.code === "ENOENT") return false; + throw error; + } + unlinkSync(path); + return true; + }, + list: async (rootPath, name, maximumEntries = 256) => await listNames(rootPath, name, maximumEntries), + listPage: async (rootPath, name, afterName, maximumEntries) => { + if (name !== "sessions") throw new Error("invalid"); + const handle = opendirSync(await directory(rootPath, name)); + const all: string[] = []; + try { + for (;;) { + const entry = handle.readSync(); + if (entry === null) break; + all.push(entry.name); + } + } finally { + handle.closeSync(); + } + all.sort(); + for (const filename of all) record(join(rootPath, name, filename), [1]); + const selected = all.filter((filename) => afterName === undefined || filename > afterName).slice(0, maximumEntries + 1); + return { + entries: selected.slice(0, maximumEntries).map((filename) => { + const info = statSync(join(rootPath, name, filename)); + return { name: filename, modifiedUnixMs: info.mtimeMs }; + }), + more: selected.length > maximumEntries, + }; + }, + claimConsume: async (rootPath, filename) => { + const source = await paths(rootPath, "oidc", filename); + const claim = source.replace(/\.json$/, ".claim"); + try { + record(source, [1]); + } catch (error: any) { + if (error?.code === "ENOENT") return undefined; + return undefined; + } + try { + linkSync(source, claim); + } catch (error: any) { + if (error?.code === "EEXIST") return undefined; + throw error; + } + const contents = readFileSync(source); + unlinkSync(source); + unlinkSync(claim); + return contents; + }, + readClaim: async (rootPath, filename) => { + const source = await paths(rootPath, "oidc", filename); + const claim = source.replace(/\.json$/, ".claim"); + try { + const sourceInfo = record(source, [2]); + const claimInfo = record(claim, [2]); + if (!same(sourceInfo, claimInfo)) throw new Error("invalid"); + return readFileSync(source); + } catch (error: any) { + if (error?.code === "ENOENT") return undefined; + return undefined; + } + }, + removeClaim: async (rootPath, filename) => { + const source = await paths(rootPath, "oidc", filename); + const claim = source.replace(/\.json$/, ".claim"); + try { + const sourceInfo = record(source, [2]); + const claimInfo = record(claim, [2]); + if (!same(sourceInfo, claimInfo)) throw new Error("invalid"); + } catch (error: any) { + if (error?.code === "ENOENT") return false; + return false; + } + unlinkSync(source); + unlinkSync(claim); + return true; + }, + }; +} + +function validStore(storageRoot: string, options: FileAuthSessionStoreOptions = {}): AuthSessionStore { + const posixStorageBridge = options.posixStorageBridge ?? testPosixStorageBridge(); + return createFileAuthSessionStore(storageRoot, { + currentAuthConfigRevision: () => revision, + findLocalUser: async () => validLocalUser, + }, { ...options, posixStorageBridge }); +} + +async function create( + store: AuthSessionStore, + overrides: Partial = {}, + now = base, +) { + return store.create({ + principal: { + issuer: "local", + subject: "6ba7b810-9dad-4ed1-80b4-00c04fd430c8", + displayName: "Local administrator", + roles: ["admin"], + permissions: ["session.use", "settings.manage"], + isAdmin: true, + }, + method: "local", + remembered: true, + userAuthRevision: 7, + authConfigRevision: revision, + idleTtlMs: 10 * 60_000, + absoluteTtlMs: 30 * 60_000, + ...overrides, + }, now); +} + +async function expectStoreInvalid(operation: Promise): Promise { + await expect(operation).rejects.toThrow("auth_session_store_invalid"); +} + +async function isolatedOidcConsumer(storageRoot: string, state: string): Promise<{ + start(): void; + result: Promise; +}> { + const child = fork(new URL("./fixtures/oidc-state-consumer.mts", import.meta.url), [], { + cwd: process.cwd(), + execArgv: ["--import", "tsx"], + env: { + ...process.env, + THT_TEST_SESSION_ROOT: storageRoot, + THT_TEST_OIDC_STATE: state, + }, + silent: true, + }); + const ready = new Promise((resolve, reject) => { + child.once("message", (message) => { + if (message === "ready") resolve(); + else reject(new Error("OIDC consumer did not become ready")); + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code !== null && code !== 0) reject(new Error("OIDC consumer exited before ready")); + }); + }); + const result = new Promise((resolve, reject) => { + child.on("message", (message) => { + if (message && typeof message === "object" && "consumed" in message) { + const outcome = message as { consumed: unknown; failed?: unknown }; + if (outcome.failed === true) reject(new Error("OIDC consumer failed")); + else resolve(outcome.consumed === true); + } + }); + child.on("error", reject); + child.on("exit", (code) => { + if (code !== 0) reject(new Error("OIDC consumer exited without a result")); + }); + }); + await ready; + return { start: () => child.send("consume"), result }; +} + +async function isolatedOidcCreator(storageRoot: string, attempts: number): Promise<{ + start(): void; + result: Promise; +}> { + const child = fork(new URL("./fixtures/oidc-state-creator.mts", import.meta.url), [], { + cwd: process.cwd(), + execArgv: ["--import", "tsx"], + env: { + ...process.env, + THT_TEST_SESSION_ROOT: storageRoot, + THT_TEST_OIDC_ATTEMPTS: String(attempts), + THT_TEST_OIDC_NOW: base.toISOString(), + THT_TEST_OIDC_CAPACITY: "8", + }, + silent: true, + }); + const ready = new Promise((resolve, reject) => { + child.once("message", (message) => { + if (message === "ready") resolve(); + else reject(new Error("OIDC creator did not become ready")); + }); + child.once("error", reject); + child.once("exit", (code) => { + if (code !== null && code !== 0) reject(new Error("OIDC creator exited before ready")); + }); + }); + const result = new Promise((resolve, reject) => { + let outcome: { created: unknown; failed?: unknown; error?: unknown } | undefined; + child.on("message", (message) => { + if (message && typeof message === "object" && "created" in message) { + outcome = message as { created: unknown; failed?: unknown; error?: unknown }; + } + }); + child.on("error", reject); + child.on("exit", (code) => { + if (code !== 0 || outcome?.failed === true || typeof outcome?.created !== "number") { + reject(new Error(`OIDC creator failed: ${String(outcome?.error)}`)); + } else { + resolve(outcome.created); + } + }); + }); + await ready; + return { start: () => child.send("create"), result }; +} + +describe("file-backed auth session store", () => { + test.skipIf(process.platform === "win32")("delegates missing layout creation to the retained native bridge", async () => { + const storageRoot = join(root(), "auth"); + const ensureLayout = vi.fn(async (requestedRoot: string) => { + expect(requestedRoot).toBe(storageRoot); + mkdirSync(requestedRoot, { mode: 0o700 }); + chmodSync(requestedRoot, 0o700); + for (const child of ["sessions", "oidc"]) { + mkdirSync(join(requestedRoot, child), { mode: 0o700 }); + chmodSync(join(requestedRoot, child), 0o700); + } + }); + const store = validStore(storageRoot, { posixStorageBridge: testPosixStorageBridge(ensureLayout) }); + + await expect(create(store)).resolves.toMatchObject({ record: { method: "local" } }); + expect(ensureLayout).toHaveBeenCalledOnce(); + + chmodSync(join(storageRoot, "oidc"), 0o750); + await expectStoreInvalid(store.createOidcState(oidcInput("n".repeat(16), "v".repeat(43)), base)); + }); + + test.skipIf(process.platform === "win32")("does not perform a path-based fallback when bridge layout creation loses an ancestor race", async () => { + const outer = root(); + const outside = root(); + const parent = join(outer, "parent"); + const movedParent = join(outer, "parent-original"); + mkdirSync(parent, { mode: 0o700 }); + chmodSync(parent, 0o700); + const storageRoot = join(parent, "auth"); + const ensureLayout = vi.fn(async () => { + renameSync(parent, movedParent); + symlinkSync(outside, parent); + throw new Error(`${storageRoot} rejected`); + }); + const store = validStore(storageRoot, { posixStorageBridge: testPosixStorageBridge(ensureLayout) }); + + await expectStoreInvalid(create(store)); + expect(ensureLayout).toHaveBeenCalledOnce(); + expect(existsSync(join(outside, "auth"))).toBe(false); + expect(existsSync(join(movedParent, "auth"))).toBe(false); + }); + + test.skipIf(process.platform === "win32")("never follows a symlinked ancestor while creating a missing session root", async () => { + const outer = root(); + const outside = root(); + const linkedParent = join(outer, "linked-parent"); + symlinkSync(outside, linkedParent); + const storageRoot = join(linkedParent, "auth"); + + await expectStoreInvalid(create(validStore(storageRoot))); + expect(existsSync(join(outside, "auth"))).toBe(false); + }); + + test.skipIf(process.platform === "win32")("rejects an uncreatable missing root without side effects", async () => { + const outer = root(); + const lockedParent = join(outer, "locked-parent"); + mkdirSync(lockedParent, { mode: 0o700 }); + chmodSync(lockedParent, 0o500); + const storageRoot = join(lockedParent, "auth"); + try { + await expectStoreInvalid(create(validStore(storageRoot))); + expect(existsSync(storageRoot)).toBe(false); + } finally { + chmodSync(lockedParent, 0o700); + } + }); + + test("does not fall back to Node paths when the retained POSIX bridge rejects creation", async () => { + const storageRoot = join(root(), "auth"); + const bridge = { + ...testPosixStorageBridge(), + create: async () => { throw new Error("native create rejected"); }, + } as WindowsAuthStorageBridge; + + await expectStoreInvalid(create(validStore(storageRoot, { posixStorageBridge: bridge }))); + expect(existsSync(storageRoot)).toBe(false); + }); + + test("fails closed and revokes a session when constructed without validity dependencies", async () => { + const storageRoot = root(); + const store = createFileAuthSessionStore(storageRoot, undefined, { posixStorageBridge: testPosixStorageBridge() }); + const created = await create(store); + + await expect(store.resolve(created.token)).resolves.toBeUndefined(); + expect(existsSync(digestPath(storageRoot, "sessions", created.token))).toBe(false); + }); + + test("revokes a session when a validity dependency throws", async () => { + const storageRoot = root(); + const store = createFileAuthSessionStore(storageRoot, { + currentAuthConfigRevision: () => { throw new Error("dependency unavailable"); }, + findLocalUser: async () => validLocalUser, + }, { posixStorageBridge: testPosixStorageBridge() }); + const created = await create(store); + + await expectStoreInvalid(store.resolve(created.token)); + expect(existsSync(digestPath(storageRoot, "sessions", created.token))).toBe(false); + }); + + test("creates 256-bit opaque tokens, digest-only files, and derived CSRF values", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const first = await create(store); + const second = await create(store); + const path = digestPath(storageRoot, "sessions", first.token); + const contents = readFileSync(path, "utf8"); + + expect(first.token).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(Buffer.from(first.token, "base64url")).toHaveLength(32); + expect(second.token).not.toBe(first.token); + expect(lstatSync(path).isFile()).toBe(true); + expect(path).toMatch(/[a-f0-9]{64}\.json$/); + expect(contents).not.toContain(first.token); + expect(contents).not.toContain(first.csrfToken); + expect(deriveCsrfToken(first.token)).toBe(first.csrfToken); + expect(first.csrfToken).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(first.csrfToken).not.toBe(first.token); + + if (process.platform !== "win32") { + expect(lstatSync(storageRoot).mode & 0o7777).toBe(0o700); + expect(lstatSync(join(storageRoot, "sessions")).mode & 0o7777).toBe(0o700); + expect(lstatSync(join(storageRoot, "oidc")).mode & 0o7777).toBe(0o700); + expect(lstatSync(path).mode & 0o7777).toBe(0o600); + } + }); + + test("survives a backend restart and respects idle and absolute expiry", async () => { + const storageRoot = root(); + const firstStore = validStore(storageRoot); + const created = await create(firstStore); + const restartedStore = validStore(storageRoot); + + await expect(restartedStore.resolve(created.token, new Date(base.getTime() + 9 * 60_000))) + .resolves.toMatchObject({ subject: created.record.subject, remembered: true }); + await expect(restartedStore.resolve(created.token, new Date(base.getTime() + 10 * 60_000))) + .resolves.toBeUndefined(); + expect(existsSync(digestPath(storageRoot, "sessions", created.token))).toBe(false); + + const absolute = await create(restartedStore, { idleTtlMs: 30 * 60_000, absoluteTtlMs: 10 * 60_000 }); + await expect(restartedStore.resolve(absolute.token, new Date(base.getTime() + 10 * 60_000))) + .resolves.toBeUndefined(); + }); + + test("touches at most once per five minutes and never extends absolute expiry", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await create(store); + const path = digestPath(storageRoot, "sessions", created.token); + const before = readFileSync(path, "utf8"); + + await store.touch(created.token, new Date(base.getTime() + 4 * 60_000)); + expect(readFileSync(path, "utf8")).toBe(before); + + await store.touch(created.token, new Date(base.getTime() + 5 * 60_000)); + await expect(store.resolve(created.token, new Date(base.getTime() + 5 * 60_000))).resolves.toMatchObject({ + lastSeenAt: "2030-01-02T03:09:05.000Z", + idleExpiresAt: "2030-01-02T03:19:05.000Z", + absoluteExpiresAt: "2030-01-02T03:34:05.000Z", + }); + }); + + test("revokes sessions and prunes expired session and OIDC-state records", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const revoked = await create(store); + const expired = await create(store, { idleTtlMs: 60_000, absoluteTtlMs: 60_000 }); + const oidc = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + + await store.revoke(revoked.token); + await expect(store.resolve(revoked.token)).resolves.toBeUndefined(); + expect(existsSync(digestPath(storageRoot, "sessions", revoked.token))).toBe(false); + + await expect(store.prune(new Date(base.getTime() + 11 * 60_000))).resolves.toBe(2); + expect(existsSync(digestPath(storageRoot, "sessions", expired.token))).toBe(false); + expect(existsSync(digestPath(storageRoot, "oidc", oidc.state))).toBe(false); + }); + + test.skipIf(process.platform === "win32")("allows startup maintenance past 512 live records and keeps OIDC available", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const liveSeed = await create(store, { idleTtlMs: 30 * 60_000, absoluteTtlMs: 30 * 60_000 }); + const liveContents = readFileSync(digestPath(storageRoot, "sessions", liveSeed.token)); + unlinkSync(digestPath(storageRoot, "sessions", liveSeed.token)); + const expiredSeed = await create(store, { idleTtlMs: 60_000, absoluteTtlMs: 60_000 }); + const expiredContents = readFileSync(digestPath(storageRoot, "sessions", expiredSeed.token)); + unlinkSync(digestPath(storageRoot, "sessions", expiredSeed.token)); + + for (let index = 0; index < 512; index += 1) { + writeFileSync(join(storageRoot, "sessions", sessionFilename(index)), liveContents, { mode: 0o600 }); + } + const expiredName = sessionFilename(512); + writeFileSync(join(storageRoot, "sessions", expiredName), expiredContents, { mode: 0o600 }); + + await expect(store.prune(new Date(base.getTime() + 2 * 60_000))).resolves.toBe(0); + expect(existsSync(join(storageRoot, "sessions", expiredName))).toBe(true); + expect(existsSync(join(storageRoot, "sessions", sessionFilename(0)))).toBe(true); + await expect(store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base)) + .resolves.toMatchObject({ record: { nonce: "n".repeat(43) } }); + + await expect(store.prune(new Date(base.getTime() + 2 * 60_000))).resolves.toBe(1); + expect(existsSync(join(storageRoot, "sessions", expiredName))).toBe(false); + expect(existsSync(join(storageRoot, "sessions", sessionFilename(0)))).toBe(true); + }); + + test.skipIf(process.platform === "win32")("serializes concurrent continuation passes without stranding a later page", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const liveSeed = await create(store, { idleTtlMs: 30 * 60_000, absoluteTtlMs: 30 * 60_000 }); + const liveContents = readFileSync(digestPath(storageRoot, "sessions", liveSeed.token)); + unlinkSync(digestPath(storageRoot, "sessions", liveSeed.token)); + const expiredSeed = await create(store, { idleTtlMs: 60_000, absoluteTtlMs: 60_000 }); + const expiredContents = readFileSync(digestPath(storageRoot, "sessions", expiredSeed.token)); + unlinkSync(digestPath(storageRoot, "sessions", expiredSeed.token)); + for (let index = 0; index < 512; index += 1) { + writeFileSync(join(storageRoot, "sessions", sessionFilename(index)), liveContents, { mode: 0o600 }); + } + const expiredName = sessionFilename(512); + writeFileSync(join(storageRoot, "sessions", expiredName), expiredContents, { mode: 0o600 }); + + await expect(Promise.all([ + store.prune(new Date(base.getTime() + 2 * 60_000)), + store.prune(new Date(base.getTime() + 2 * 60_000)), + ])).resolves.toEqual([0, 1]); + expect(existsSync(join(storageRoot, "sessions", expiredName))).toBe(false); + expect(existsSync(join(storageRoot, "sessions", sessionFilename(0)))).toBe(true); + }); + + test.skipIf(process.platform === "win32")("fails closed on an unsafe ordinary record beyond the first continuation page", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const seed = await create(store, { idleTtlMs: 30 * 60_000, absoluteTtlMs: 30 * 60_000 }); + const contents = readFileSync(digestPath(storageRoot, "sessions", seed.token)); + unlinkSync(digestPath(storageRoot, "sessions", seed.token)); + for (let index = 0; index < 512; index += 1) { + writeFileSync(join(storageRoot, "sessions", sessionFilename(index)), contents, { mode: 0o600 }); + } + symlinkSync(join(storageRoot, "missing-target"), join(storageRoot, "sessions", sessionFilename(512))); + + await expect(store.prune(new Date(base.getTime() + 2 * 60_000))) + .rejects.toThrow("auth_session_store_invalid"); + }); + + test("fails closed when the retained POSIX bridge rejects a session continuation page", async () => { + const storageRoot = root(); + const bridge = { + ...testPosixStorageBridge(), + listPage: async () => { throw new Error("native page rejected"); }, + } as WindowsAuthStorageBridge; + const store = validStore(storageRoot, { posixStorageBridge: bridge }); + + await expect(store.prune(new Date(base.getTime() + 2 * 60_000))) + .rejects.toThrow("auth_session_store_invalid"); + }); + + test("persists the OIDC transaction transport as consumed state metadata", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await store.createOidcState({ + ...oidcInput("n".repeat(43), "v".repeat(43)), + browserTransactionTransport: "loopback_http", + }); + + await expect(store.consumeOidcState(created.state)).resolves.toMatchObject({ + browserTransactionTransport: "loopback_http", + }); + }); + + test("creates bounded OIDC state records that expire and are single-use", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + const path = digestPath(storageRoot, "oidc", created.state); + + expect(created.state).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(readFileSync(path, "utf8")).not.toContain(created.state); + await expect(store.consumeOidcState(created.state, new Date(base.getTime() + 9 * 60_000))) + .resolves.toMatchObject({ + nonce: "n".repeat(43), codeVerifier: "v".repeat(43), returnTo: "/", + authConfigRevision: revision, issuer: "https://issuer.example.test", + browserTransactionDigest: "b".repeat(64), + browserTransactionTransport: "https", + }); + await expect(store.consumeOidcState(created.state)).resolves.toBeUndefined(); + + const expired = await store.createOidcState(oidcInput("x".repeat(43), "y".repeat(43)), base); + await expect(store.consumeOidcState(expired.state, new Date(base.getTime() + 10 * 60_000))) + .resolves.toBeUndefined(); + }); + + test("rejects an OIDC-state flood without deleting any valid live state", async () => { + const storageRoot = root(); + const store = validStore(storageRoot, { oidcStateCapacity: 8 }); + const created = await Promise.all(Array.from({ length: 8 }, (_unused, index) => + store.createOidcState(oidcInput(`n${String(index).padStart(42, "0")}`, `v${String(index).padStart(42, "0")}`), base))); + + await expect(store.createOidcState(oidcInput("x".repeat(43), "y".repeat(43)), base)) + .rejects.toThrow("auth_oidc_state_capacity"); + await expect(store.consumeOidcState(created[0].state, new Date(base.getTime() + 60_000))) + .resolves.toMatchObject({ nonce: "n" + "0".repeat(42) }); + expect(created.slice(1).every(({ state }) => existsSync(digestPath(storageRoot, "oidc", state)))).toBe(true); + }); + + test("prunes expired OIDC states before admitting a new transaction at capacity", async () => { + const storageRoot = root(); + const store = validStore(storageRoot, { oidcStateCapacity: 8 }); + const expired = await Promise.all(Array.from({ length: 8 }, (_unused, index) => + store.createOidcState(oidcInput(`n${String(index).padStart(42, "0")}`, `v${String(index).padStart(42, "0")}`), base))); + + const admitted = await store.createOidcState( + oidcInput("x".repeat(43), "y".repeat(43)), + new Date(base.getTime() + 10 * 60_000), + ); + + expect(expired.every(({ state }) => !existsSync(digestPath(storageRoot, "oidc", state)))).toBe(true); + expect(existsSync(digestPath(storageRoot, "oidc", admitted.state))).toBe(true); + }); + + test.skipIf(process.platform === "win32")("starts OIDC without scanning more than 256 ordinary session records", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const liveSession = await create(store); + for (let index = 0; index < 300; index += 1) { + writeFileSync(join(storageRoot, "sessions", `${index.toString(16).padStart(64, "0")}.json`), "{}", { + encoding: "utf8", + mode: 0o600, + }); + } + + const oidc = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + + expect(oidc.state).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(existsSync(digestPath(storageRoot, "sessions", liveSession.token))).toBe(true); + }); + + test.skipIf(process.platform === "win32")("uses bounded directory reads for ordinary and OIDC pruning", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const expiredSession = await create(store, { idleTtlMs: 60_000, absoluteTtlMs: 60_000 }); + const expiredOidc = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + fsHooks.beforeReaddir = (path) => { + if (path === join(storageRoot, "sessions") || path === join(storageRoot, "oidc")) { + throw new Error("unbounded directory read"); + } + }; + + await expect(store.prune(new Date(base.getTime() + 11 * 60_000))).resolves.toBe(2); + expect(existsSync(digestPath(storageRoot, "sessions", expiredSession.token))).toBe(false); + expect(existsSync(digestPath(storageRoot, "oidc", expiredOidc.state))).toBe(false); + }); + + test.skipIf(process.platform === "win32")("rejects an overfull OIDC directory without deleting a live state", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const live = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + for (let index = 0; index < 193; index += 1) { + writeFileSync(join(storageRoot, "oidc", `${(index + 1_000).toString(16).padStart(64, "0")}.json`), "{}", { + encoding: "utf8", + mode: 0o600, + }); + } + + await expect(store.createOidcState(oidcInput("x".repeat(43), "y".repeat(43)), base)) + .rejects.toThrow("auth_session_store_invalid"); + expect(existsSync(digestPath(storageRoot, "oidc", live.state))).toBe(true); + }); + + test("OIDC-only cleanup removes expired state while preserving a live transaction", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const expired = await store.createOidcState(oidcInput("e".repeat(43), "f".repeat(43)), base); + const live = await store.createOidcState( + oidcInput("l".repeat(43), "m".repeat(43)), + new Date(base.getTime() + 60_000), + ); + + const admitted = await store.createOidcState( + oidcInput("n".repeat(43), "v".repeat(43)), + new Date(base.getTime() + 10 * 60_000), + ); + + expect(existsSync(digestPath(storageRoot, "oidc", expired.state))).toBe(false); + expect(existsSync(digestPath(storageRoot, "oidc", admitted.state))).toBe(true); + await expect(store.consumeOidcState(live.state, new Date(base.getTime() + 10 * 60_000))) + .resolves.toMatchObject({ nonce: "l".repeat(43) }); + }); + + test("bounds OIDC state creation across concurrent Node processes", async () => { + const storageRoot = root(); + const creators = await Promise.all([ + isolatedOidcCreator(storageRoot, 6), + isolatedOidcCreator(storageRoot, 6), + ]); + + creators.forEach(({ start }) => start()); + const winners = await Promise.all(creators.map(({ result }) => result)); + + expect(winners.reduce((sum, value) => sum + value, 0)).toBe(8); + const names = readdirSync(join(storageRoot, "oidc")); + expect(names.filter((name) => /^[a-f0-9]{64}\.json$/.test(name))).toHaveLength(8); + }); + + test("fails closed when an OIDC state already has an atomic filesystem claim", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + const statePath = digestPath(storageRoot, "oidc", created.state); + linkSync(statePath, claimPath(storageRoot, created.state)); + + await expect(store.consumeOidcState(created.state)).resolves.toBeUndefined(); + expect(existsSync(statePath)).toBe(true); + }); + + test("treats a competing OIDC claim installed between availability and state checks as unavailable", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + const statePath = digestPath(storageRoot, "oidc", created.state); + const stateClaimPath = claimPath(storageRoot, created.state); + fsHooks.beforeLstat = (observed) => { + if (observed !== statePath) return false; + linkSync(statePath, stateClaimPath); + return true; + }; + + await expect(store.consumeOidcState(created.state)).resolves.toBeUndefined(); + expect(existsSync(statePath)).toBe(true); + expect(existsSync(stateClaimPath)).toBe(true); + }); + + test("prunes an expired OIDC state abandoned after an atomic claim", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + const statePath = digestPath(storageRoot, "oidc", created.state); + const stateClaimPath = claimPath(storageRoot, created.state); + linkSync(statePath, stateClaimPath); + + await expect(store.prune(new Date(base.getTime() + 10 * 60_000))).resolves.toBe(1); + expect(existsSync(statePath)).toBe(false); + expect(existsSync(stateClaimPath)).toBe(false); + }); + + test("retains an in-flight orphan claim but removes it after the bounded recovery window", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + const statePath = digestPath(storageRoot, "oidc", created.state); + const stateClaimPath = claimPath(storageRoot, created.state); + linkSync(statePath, stateClaimPath); + unlinkSync(statePath); + + await expect(store.consumeOidcState(created.state)).resolves.toBeUndefined(); + expect(existsSync(stateClaimPath)).toBe(true); + await expect(store.prune(new Date("2031-01-02T03:04:05.000Z"))).resolves.toBe(1); + expect(existsSync(stateClaimPath)).toBe(false); + }); + + test("allows exactly one separate Node isolate to consume an OIDC state", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + const [first, second] = await Promise.all([ + isolatedOidcConsumer(storageRoot, created.state), + isolatedOidcConsumer(storageRoot, created.state), + ]); + + first.start(); + second.start(); + const outcomes = await Promise.all([first.result, second.result]); + expect(outcomes.filter(Boolean)).toHaveLength(1); + expect(existsSync(digestPath(storageRoot, "oidc", created.state))).toBe(false); + }); + + test.skipIf(process.platform === "win32")("refuses symlinked and hard-linked session records", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const symlinked = await create(store); + const symlinkPath = digestPath(storageRoot, "sessions", symlinked.token); + const target = `${symlinkPath}.target`; + renameSync(symlinkPath, target); + symlinkSync(target, symlinkPath); + await expectStoreInvalid(store.resolve(symlinked.token)); + + const hardLinked = await create(store); + const hardLinkPath = digestPath(storageRoot, "sessions", hardLinked.token); + linkSync(hardLinkPath, `${hardLinkPath}.link`); + await expectStoreInvalid(store.resolve(hardLinked.token)); + }); + + test("refuses malformed and oversized session records without disclosing their contents", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const malformed = await create(store); + const malformedPath = digestPath(storageRoot, "sessions", malformed.token); + writeFileSync(malformedPath, "{}", { encoding: "utf8", mode: 0o600 }); + chmodSync(malformedPath, 0o600); + await expectStoreInvalid(store.resolve(malformed.token)); + + const oversized = await create(store); + const oversizedPath = digestPath(storageRoot, "sessions", oversized.token); + writeFileSync(oversizedPath, "#".repeat(20_000), { encoding: "utf8", mode: 0o600 }); + chmodSync(oversizedPath, 0o600); + await expectStoreInvalid(store.resolve(oversized.token)); + }); + + test.skipIf(process.platform === "win32")("refuses unsafe existing roots and storage subdirectories", async () => { + const unsafeRoot = root(); + chmodSync(unsafeRoot, 0o755); + await expectStoreInvalid(create(validStore(unsafeRoot))); + + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await create(store); + chmodSync(join(storageRoot, "sessions"), 0o755); + await expectStoreInvalid(store.resolve(created.token)); + + const outer = root(); + const realRoot = join(outer, "real-auth"); + mkdirSync(realRoot, { mode: 0o700 }); + chmodSync(realRoot, 0o700); + const linkedRoot = join(outer, "linked-auth"); + symlinkSync(realRoot, linkedRoot); + await expectStoreInvalid(create(validStore(linkedRoot))); + }); + + test("fails closed when the retained POSIX bridge rejects a session read", async () => { + const storageRoot = root(); + const bridge = { + ...testPosixStorageBridge(), + read: async () => { throw new Error("native read rejected"); }, + } as WindowsAuthStorageBridge; + const store = validStore(storageRoot, { posixStorageBridge: bridge }); + const created = await create(store); + + await expectStoreInvalid(store.resolve(created.token)); + }); + + test("does not bypass a retained POSIX bridge read failure", async () => { + const storageRoot = root(); + const bridge = { + ...testPosixStorageBridge(), + read: async () => { throw new Error("native root replacement rejected"); }, + } as WindowsAuthStorageBridge; + const store = validStore(storageRoot, { posixStorageBridge: bridge }); + const created = await create(store); + + await expectStoreInvalid(store.resolve(created.token)); + }); + + test("fails closed when the retained POSIX bridge rejects a session replacement", async () => { + const storageRoot = root(); + const bridge = { + ...testPosixStorageBridge(), + replace: async () => { throw new Error("native replace rejected"); }, + } as WindowsAuthStorageBridge; + const store = validStore(storageRoot, { posixStorageBridge: bridge }); + const created = await create(store); + + await expectStoreInvalid(store.touch(created.token, new Date(base.getTime() + 5 * 60_000))); + }); + + test.skipIf(process.platform === "win32")("refuses directory replacement during a touch", async () => { + const storageRoot = root(); + const store = validStore(storageRoot); + const created = await create(store); + const path = digestPath(storageRoot, "sessions", created.token); + const sessions = join(storageRoot, "sessions"); + const replacement = join(storageRoot, "sessions-replacement"); + fsHooks.afterLstat = (observed) => { + if (observed !== path) return false; + renameSync(sessions, replacement); + symlinkSync(replacement, sessions); + return true; + }; + + await expectStoreInvalid(store.touch(created.token, new Date(base.getTime() + 5 * 60_000))); + }); + + test("fails closed when the retained POSIX bridge rejects a session removal", async () => { + const storageRoot = root(); + const bridge = { + ...testPosixStorageBridge(), + remove: async () => { throw new Error("native remove rejected"); }, + } as WindowsAuthStorageBridge; + const store = validStore(storageRoot, { posixStorageBridge: bridge }); + const created = await create(store); + + await expectStoreInvalid(store.revoke(created.token)); + }); + + test("does not bypass a retained POSIX bridge removal failure", async () => { + const storageRoot = root(); + const bridge = { + ...testPosixStorageBridge(), + remove: async () => { throw new Error("native root replacement rejected"); }, + } as WindowsAuthStorageBridge; + const store = validStore(storageRoot, { posixStorageBridge: bridge }); + const created = await create(store); + + await expectStoreInvalid(store.revoke(created.token)); + }); + + test("routes native Windows session creation through an injected storage bridge", async () => { + const storageRoot = root(); + const store = validStore(join(storageRoot, "windows-auth-state")); + const createRecord = vi.fn(async () => true); + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const created = await create( + createFileAuthSessionStore(join(storageRoot, "windows-auth-state"), { + currentAuthConfigRevision: () => revision, + findLocalUser: async () => validLocalUser, + }, { windowsStorageBridge: { create: createRecord } } as never), + ); + expect(created.token).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(createRecord).toHaveBeenCalledTimes(1); + expect(existsSync(join(storageRoot, "windows-auth-state"))).toBe(false); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("routes native Windows lifecycle and OIDC operations only through the storage bridge", async () => { + const records = new Map(); + const calls: string[] = []; + const key = (directory: string, filename: string) => `${directory}/${filename}`; + const bridge = { + create: async (_root: string, directory: string, filename: string, contents: Buffer) => { + calls.push("create"); + const entry = key(directory, filename); + if (records.has(entry)) return false; + records.set(entry, Buffer.from(contents)); + return true; + }, + read: async (_root: string, directory: string, filename: string) => { + calls.push("read"); + const value = records.get(key(directory, filename)); + return value === undefined ? undefined : Buffer.from(value); + }, + replace: async (_root: string, directory: string, filename: string, contents: Buffer) => { + calls.push("replace"); + records.set(key(directory, filename), Buffer.from(contents)); + }, + remove: async (_root: string, directory: string, filename: string) => { + calls.push("remove"); + return records.delete(key(directory, filename)); + }, + list: async (_root: string, directory: string) => { + calls.push("list"); + return [...records.keys()] + .filter((entry) => entry.startsWith(`${directory}/`)) + .map((entry) => ({ name: entry.slice(directory.length + 1), modifiedUnixMs: base.getTime() })); + }, + listPage: async (_root: string, directory: string, after: string | undefined, maximumEntries: number) => { + calls.push("list"); + if (directory !== "sessions") throw new Error("only ordinary sessions use continuation pages"); + const names = [...records.keys()] + .filter((entry) => entry.startsWith("sessions/")) + .map((entry) => entry.slice("sessions/".length)) + .filter((name) => after === undefined || name > after) + .sort(); + return { + entries: names.slice(0, maximumEntries).map((name) => ({ name, modifiedUnixMs: base.getTime() })), + more: names.length > maximumEntries, + }; + }, + claimConsume: async (_root: string, filename: string) => { + calls.push("claim-consume"); + const entry = key("oidc", filename); + const value = records.get(entry); + records.delete(entry); + return value === undefined ? undefined : Buffer.from(value); + }, + readClaim: async () => undefined, + removeClaim: async () => false, + }; + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const store = createFileAuthSessionStore("C:\\ProgramData\\ThothII\\auth", { + currentAuthConfigRevision: () => revision, + findLocalUser: async () => validLocalUser, + }, { windowsStorageBridge: bridge } as never); + const session = await create(store); + await expect(store.resolve(session.token)).resolves.toMatchObject({ subject: session.record.subject }); + await store.touch(session.token, new Date(base.getTime() + 5 * 60_000)); + await store.revoke(session.token); + await expect(store.resolve(session.token)).resolves.toBeUndefined(); + + const oidc = await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + await expect(store.consumeOidcState(oidc.state, new Date(base.getTime() + 9 * 60_000))) + .resolves.toMatchObject({ nonce: "n".repeat(43) }); + await expect(store.consumeOidcState(oidc.state)).resolves.toBeUndefined(); + await create(store, { idleTtlMs: 60_000, absoluteTtlMs: 60_000 }); + await store.createOidcState(oidcInput("x".repeat(43), "y".repeat(43)), base); + await expect(store.prune(new Date(base.getTime() + 11 * 60_000))).resolves.toBe(2); + expect(calls).toEqual(expect.arrayContaining(["create", "read", "replace", "remove", "claim-consume", "list"])); + expect(records).toHaveLength(0); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("continues Windows ordinary-session maintenance with a strict 512-record page cursor", async () => { + const records = new Map(); + const pages: Array<{ after: string | undefined; maximumEntries: number }> = []; + const key = (directory: string, filename: string) => `${directory}/${filename}`; + const bridge = { + create: async (_root: string, directory: string, filename: string, contents: Buffer) => { + const entry = key(directory, filename); + if (records.has(entry)) return false; + records.set(entry, Buffer.from(contents)); + return true; + }, + read: async (_root: string, directory: string, filename: string) => { + const value = records.get(key(directory, filename)); + return value === undefined ? undefined : Buffer.from(value); + }, + remove: async (_root: string, directory: string, filename: string) => records.delete(key(directory, filename)), + list: async (_root: string, directory: string) => { + if (directory === "sessions") throw new Error("ordinary maintenance must use a continuation page"); + return []; + }, + listPage: async (_root: string, directory: string, after: string | undefined, maximumEntries: number) => { + if (directory !== "sessions") throw new Error("only ordinary sessions use continuation pages"); + pages.push({ after, maximumEntries }); + const names = [...records.keys()] + .filter((entry) => entry.startsWith("sessions/")) + .map((entry) => entry.slice("sessions/".length)) + .filter((name) => after === undefined || name > after) + .sort(); + return { + entries: names.slice(0, maximumEntries).map((name) => ({ name, modifiedUnixMs: base.getTime() })), + more: names.length > maximumEntries, + }; + }, + }; + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const store = createFileAuthSessionStore("C:\\ProgramData\\ThothII\\auth", { + currentAuthConfigRevision: () => revision, + findLocalUser: async () => validLocalUser, + }, { windowsStorageBridge: bridge as never }); + await create(store, { idleTtlMs: 30 * 60_000, absoluteTtlMs: 30 * 60_000 }); + const liveContents = [...records.values()][0]; + records.clear(); + const expiredSeed = await create(store, { idleTtlMs: 60_000, absoluteTtlMs: 60_000 }); + const expiredContents = [...records.values()][0]; + records.clear(); + if (!liveContents || !expiredContents) throw new Error("seed records missing"); + for (let index = 0; index < 512; index += 1) { + records.set(key("sessions", sessionFilename(index)), Buffer.from(liveContents)); + } + const expiredName = sessionFilename(512); + records.set(key("sessions", expiredName), Buffer.from(expiredContents)); + + await expect(store.prune(new Date(base.getTime() + 2 * 60_000))).resolves.toBe(0); + expect(records.has(key("sessions", expiredName))).toBe(true); + await expect(store.prune(new Date(base.getTime() + 2 * 60_000))).resolves.toBe(1); + expect(records.has(key("sessions", expiredName))).toBe(false); + expect(records.has(key("sessions", sessionFilename(0)))).toBe(true); + expect(pages).toEqual([ + { after: undefined, maximumEntries: 512 }, + { after: sessionFilename(511), maximumEntries: 512 }, + ]); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("starts OIDC on Windows without listing more than 256 ordinary session records", async () => { + const records = new Map(); + for (let index = 0; index < 300; index += 1) { + records.set(`sessions/${index.toString(16).padStart(64, "0")}.json`, Buffer.from("{}")); + } + const key = (directory: string, filename: string) => `${directory}/${filename}`; + const listed: string[] = []; + const bridge = { + create: async (_root: string, directory: string, filename: string, contents: Buffer) => { + const entry = key(directory, filename); + if (records.has(entry)) return false; + records.set(entry, Buffer.from(contents)); + return true; + }, + read: async (_root: string, directory: string, filename: string) => records.get(key(directory, filename)), + remove: async (_root: string, directory: string, filename: string) => records.delete(key(directory, filename)), + list: async (_root: string, directory: string) => { + listed.push(directory); + if (directory === "sessions") throw new Error("ordinary sessions must not be listed during OIDC start"); + return [...records.keys()] + .filter((entry) => entry.startsWith("oidc/")) + .map((entry) => ({ name: entry.slice("oidc/".length), modifiedUnixMs: base.getTime() })); + }, + }; + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const store = createFileAuthSessionStore("C:\\ProgramData\\ThothII\\auth", undefined, { + windowsStorageBridge: bridge as never, + }); + + await expect(store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base)) + .resolves.toMatchObject({ record: { nonce: "n".repeat(43) } }); + expect(listed).toEqual(["oidc", "oidc"]); + expect([...records.keys()].filter((entry) => entry.startsWith("sessions/"))).toHaveLength(300); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("rejects an overfull Windows OIDC directory before reading or deleting live entries", async () => { + const entries = Array.from({ length: 193 }, (_unused, index) => ({ + name: `${index.toString(16).padStart(64, "0")}.json`, + modifiedUnixMs: base.getTime(), + })); + const read = vi.fn(async () => undefined); + const remove = vi.fn(async () => false); + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const store = createFileAuthSessionStore("C:\\ProgramData\\ThothII\\auth", undefined, { + windowsStorageBridge: { + list: async () => entries, + read, + remove, + } as never, + }); + + await expect(store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base)) + .rejects.toThrow("auth_session_store_invalid"); + expect(read).not.toHaveBeenCalled(); + expect(remove).not.toHaveBeenCalled(); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("prunes nonempty Windows directories through the Go lower-camel list DTO", async () => { + const records = new Map(); + const key = (directory: string, entry: string) => `${directory}/${entry}`; + const response = (value: Record) => ({ + code: 0, + stdout: Buffer.from(`${JSON.stringify({ version: 1, ok: true, ...value })}\n`), + stderr: Buffer.alloc(0), + }); + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async ({ input }) => { + const request = JSON.parse(input.toString("utf8")) as { + operation: string; + directory: "sessions" | "oidc"; + filename?: string; + contentBase64?: string; + continuation?: boolean; + afterName?: string; + }; + const entry = request.filename === undefined ? undefined : key(request.directory, request.filename); + switch (request.operation) { + case "create": + if (entry === undefined || request.contentBase64 === undefined || records.has(entry)) return response({ created: false }); + records.set(entry, Buffer.from(request.contentBase64, "base64")); + return response({ created: true }); + case "read": + return entry === undefined || !records.has(entry) + ? response({}) + : response({ found: true, contentBase64: records.get(entry)?.toString("base64") }); + case "remove": + return response({ removed: entry !== undefined && records.delete(entry) }); + case "list": + if (request.continuation === true) { + const names = [...records.keys()] + .filter((value) => value.startsWith(`${request.directory}/`)) + .map((value) => value.slice(request.directory.length + 1)) + .filter((name) => request.afterName === undefined || name > request.afterName) + .sort(); + return response({ + entries: names.slice(0, 512).map((name) => ({ name, modifiedUnixMs: base.getTime() })), + more: names.length > 512, + }); + } + return response({ + // Raw lower-camel entry objects, exactly as authstorage's Go response emits them. + entries: [...records.keys()] + .filter((value) => value.startsWith(`${request.directory}/`)) + .map((value) => ({ name: value.slice(request.directory.length + 1), modifiedUnixMs: base.getTime() })), + }); + default: + throw new Error("unexpected bridge operation"); + } + }, + }); + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const store = createFileAuthSessionStore("C:\\ProgramData\\ThothII\\auth", { + currentAuthConfigRevision: () => revision, + findLocalUser: async () => validLocalUser, + }, { windowsStorageBridge: bridge }); + await create(store, { idleTtlMs: 60_000, absoluteTtlMs: 60_000 }); + await store.createOidcState(oidcInput("n".repeat(43), "v".repeat(43)), base); + + await expect(store.prune(new Date(base.getTime() + 11 * 60_000))).resolves.toBe(2); + expect(records).toHaveLength(0); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("prunes empty Windows directories through the required Go entries array", async () => { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async ({ input }) => { + const request = JSON.parse(input.toString("utf8")) as { continuation?: boolean }; + return { + code: 0, + stdout: Buffer.from(`${JSON.stringify({ + version: 1, + ok: true, + entries: [], + ...(request.continuation === true ? { more: false } : {}), + })}\n`), + stderr: Buffer.alloc(0), + }; + }, + }); + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const store = createFileAuthSessionStore("C:\\ProgramData\\ThothII\\auth", { + currentAuthConfigRevision: () => revision, + findLocalUser: async () => validLocalUser, + }, { windowsStorageBridge: bridge }); + + await expect(store.prune(base)).resolves.toBe(0); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("prunes a stale Windows OIDC orphan claim through the Go DTO path", async () => { + const claim = `${"d".repeat(64)}.claim`; + let removed = false; + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async ({ input }) => { + const request = JSON.parse(input.toString("utf8")) as { + operation: string; + directory: string; + filename?: string; + continuation?: boolean; + }; + const response = (value: Record) => ({ + code: 0, + stdout: Buffer.from(`${JSON.stringify({ version: 1, ok: true, ...value })}\n`), + stderr: Buffer.alloc(0), + }); + if (request.operation === "list") { + return response({ + entries: request.directory === "oidc" + ? [{ name: claim, modifiedUnixMs: base.getTime() }] + : [], + ...(request.continuation === true ? { more: false } : {}), + }); + } + if (request.operation === "remove" && request.directory === "oidc" && request.filename === claim) { + removed = true; + return response({ removed: true }); + } + throw new Error("unexpected bridge request"); + }, + }); + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const store = createFileAuthSessionStore("C:\\ProgramData\\ThothII\\auth", { + currentAuthConfigRevision: () => revision, + findLocalUser: async () => validLocalUser, + }, { windowsStorageBridge: bridge }); + + await expect(store.prune(new Date(base.getTime() + 11 * 60_000))).resolves.toBe(1); + expect(removed).toBe(true); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); + + test("revokes on config, local-user, revision, enabled, or role mismatch before returning", async () => { + const storageRoot = root(); + let currentRevision = revision; + let localUser: { enabled: boolean; authRevision: number; roles: readonly ("user" | "admin")[] } | undefined = { + enabled: true, + authRevision: 7, + roles: ["admin"], + }; + const store = createFileAuthSessionStore(storageRoot, { + currentAuthConfigRevision: () => currentRevision, + findLocalUser: async () => localUser, + }, { posixStorageBridge: testPosixStorageBridge() }); + + const configChanged = await create(store); + currentRevision = "b".repeat(64); + await expect(store.resolve(configChanged.token)).resolves.toBeUndefined(); + expect(existsSync(digestPath(storageRoot, "sessions", configChanged.token))).toBe(false); + + currentRevision = revision; + for (const mismatch of [ + undefined, + { enabled: false, authRevision: 7, roles: ["admin"] as const }, + { enabled: true, authRevision: 8, roles: ["admin"] as const }, + { enabled: true, authRevision: 7, roles: ["user"] as const }, + ]) { + localUser = mismatch; + const session = await create(store); + await expect(store.resolve(session.token)).resolves.toBeUndefined(); + expect(existsSync(digestPath(storageRoot, "sessions", session.token))).toBe(false); + } + }); + + test("serializes concurrent resolve and revoke without resurrecting a record", async () => { + const storageRoot = root(); + const firstStore = validStore(storageRoot); + const secondStore = validStore(storageRoot); + const created = await create(firstStore); + + await Promise.all([ + ...Array.from({ length: 8 }, () => firstStore.resolve(created.token)), + ...Array.from({ length: 8 }, () => secondStore.revoke(created.token)), + ]); + + await expect(firstStore.resolve(created.token)).resolves.toBeUndefined(); + expect(existsSync(digestPath(storageRoot, "sessions", created.token))).toBe(false); + }); +}); diff --git a/backend/test/auth-test-fixtures.ts b/backend/test/auth-test-fixtures.ts new file mode 100644 index 00000000..60ce878b --- /dev/null +++ b/backend/test/auth-test-fixtures.ts @@ -0,0 +1,151 @@ +import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { FastifyInstance } from "fastify"; +import { stringify } from "yaml"; +import { buildApp, type BuildAppDeps } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { createFixturePosixAuthStorageBridge } from "./fixtures/posix-auth-storage-bridge.mjs"; + +export const localPassword = "correct horse battery staple"; +export const localPasswordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"; +export const localAdminId = "6ba7b810-9dad-4ed1-80b4-00c04fd430c8"; +export const localPublicUrl = "http://127.0.0.1:8787"; + +export interface LocalAuthFixture { + app: FastifyInstance; + publicUrl: string; + cookie: string; + csrfToken: string; + loginStatus: number; + meStatus: number; + downstreamHits(): number; + resetDownstreamHits(): void; + sessionHeaders(overrides?: Record): Record; + close(): Promise; +} + +export interface LocalAuthFixtureOptions { + publicUrl?: string; +} + +function firstSetCookie(response: { headers: Record }): string { + const header = response.headers["set-cookie"]; + return Array.isArray(header) ? header[0] ?? "" : header ?? ""; +} + +function cookiePair(setCookie: string): string { + return setCookie.split(";", 1)[0] ?? ""; +} + +/** Unit fixtures which bypass the installed tht binary start from an already-safe native layout. */ +export function prepareAuthStateRoot(root: string): void { + mkdirSync(root, { mode: 0o700 }); + chmodSync(root, 0o700); + for (const child of ["sessions", "oidc"]) { + mkdirSync(join(root, child), { mode: 0o700 }); + chmodSync(join(root, child), 0o700); + } +} + +/** Explicit test-only substitute for the production tht-backed POSIX storage bridge. */ +export function createFixtureAuthStorageBridge() { + return createFixturePosixAuthStorageBridge(); +} + +/** Creates a production-local app and authenticates through the real login/session boundary. */ +export async function createLocalAuthFixture( + deps?: BuildAppDeps, + options: LocalAuthFixtureOptions = {}, +): Promise { + const publicUrl = options.publicUrl ?? localPublicUrl; + const publicOrigin = new URL(publicUrl).origin; + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-fixture-")); + chmodSync(directory, 0o700); + const authConfigFile = join(directory, "auth.yaml"); + const usersFile = join(directory, "users.yaml"); + writeFileSync(authConfigFile, stringify({ + version: 1, + mode: "local", + publicUrl, + local: { usersFile: "users.yaml" }, + }), { encoding: "utf8", mode: 0o600 }); + writeFileSync(usersFile, [ + "version: 1", + "users:", + ` - id: ${localAdminId}`, + " username: Admin", + " displayName: Local administrator", + ` passwordHash: ${localPasswordHash}`, + " roles:", + " - admin", + " enabled: true", + " authRevision: 1", + "", + ].join("\n"), { encoding: "utf8", mode: 0o600 }); + chmodSync(authConfigFile, 0o600); + chmodSync(usersFile, 0o600); + + const authStateRoot = join(directory, "auth-state"); + prepareAuthStateRoot(authStateRoot); + const app = buildApp( + loadConfig({ + THT_AUTH_CONFIG_FILE: authConfigFile, + THT_AUTH_STATE_ROOT: authStateRoot, + THT_HARNESS_DIR: "/tmp/h", + }), + { + ...deps, + authStorageBridgeForTest: deps?.authStorageBridgeForTest ?? createFixtureAuthStorageBridge(), + }, + ); + let downstream = 0; + app.addHook("preHandler", async () => { downstream += 1; }); + + try { + const signedIn = await app.inject({ + method: "POST", + url: "/auth/local/login", + headers: { origin: publicOrigin, "sec-fetch-site": "same-origin" }, + payload: { username: "Admin", password: localPassword }, + }); + if (signedIn.statusCode !== 200) throw new Error("local_auth_fixture_login_failed"); + const cookie = cookiePair(firstSetCookie(signedIn)); + const me = await app.inject({ method: "GET", url: "/me", headers: { cookie } }); + const csrfToken = (me.json() as { csrfToken?: unknown }).csrfToken; + if (me.statusCode !== 200 || typeof csrfToken !== "string") throw new Error("local_auth_fixture_session_failed"); + downstream = 0; + + return { + app, + publicUrl, + cookie, + csrfToken, + loginStatus: signedIn.statusCode, + meStatus: me.statusCode, + downstreamHits: () => downstream, + resetDownstreamHits: () => { downstream = 0; }, + sessionHeaders(overrides = {}) { + const headers: Record = { + cookie, + origin: publicOrigin, + "sec-fetch-site": "same-origin", + "x-thothii-csrf": csrfToken, + }; + for (const [key, value] of Object.entries(overrides)) { + if (value === undefined) delete headers[key]; + else headers[key] = value; + } + return headers; + }, + async close() { + await app.close(); + rmSync(directory, { recursive: true, force: true }); + }, + }; + } catch (error) { + await app.close(); + rmSync(directory, { recursive: true, force: true }); + throw error; + } +} diff --git a/backend/test/auth.test.ts b/backend/test/auth.test.ts index 8285eb47..531d0243 100644 --- a/backend/test/auth.test.ts +++ b/backend/test/auth.test.ts @@ -1,10 +1,26 @@ -import { test, expect } from "vitest"; +import { test, expect, vi } from "vitest"; import Fastify from "fastify"; -import { authPreHandler, getPrincipal } from "../src/auth/auth.js"; -import { chmodSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { authenticateSession, authPreHandler, getPrincipal } from "../src/auth/auth.js"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { expandLocalHome, localPrincipal } from "../src/auth/principal.js"; +import { expandLocalHome, localPrincipal, upstreamPrincipal } from "../src/auth/principal.js"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; + +test("server smoke rejects retired trusted claims under OIDC authentication", () => { + const smoke = readFileSync("../scripts/unified-deployment-smoke.sh", "utf8"); + for (const header of [ + "x-thoth-trusted-principal-issuer", + "x-thoth-trusted-principal-subject", + "x-thoth-trusted-principal-display-name", + "x-thoth-trusted-is-admin", + ]) { + expect(smoke).toContain(`-H '${header}:`); + } + expect(smoke).toContain('[[ "$trusted_header_status" == 401 ]]'); + expect(smoke).toContain("server accepted retired trusted identity headers"); +}); test("local mode resolves a stable local principal", async () => { const app = Fastify(); @@ -13,7 +29,12 @@ test("local mode resolves a stable local principal", async () => { expect((await app.inject({ method: "GET", url: "/me" })).json()).toMatchObject({ issuer: "local", subject: expect.any(String), - isAdmin: false, + roles: ["admin"], + permissions: [ + "session.use", "session.read_all", "session.manage_all", "settings.manage", + "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + ], + isAdmin: true, }); }); @@ -26,7 +47,10 @@ test("mock mode makes a principal from the test header", async () => { url: "/me", headers: { "x-mock-user": "alice" }, }); - expect(res.json()).toEqual({ issuer: "mock", subject: "alice", displayName: "alice", isAdmin: false }); + expect(res.json()).toEqual({ + issuer: "mock", subject: "alice", displayName: "alice", + roles: ["user"], permissions: ["session.use"], isAdmin: false, + }); }); test("upstream mode accepts only normalized proxy principal headers", async () => { @@ -47,7 +71,12 @@ test("upstream mode accepts only normalized proxy principal headers", async () = }, }); expect(authenticated.json()).toEqual({ - issuer: "portal", subject: "42", displayName: "Alice", isAdmin: true, + issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"], + permissions: [ + "session.use", "session.read_all", "session.manage_all", "settings.manage", + "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + ], + isAdmin: true, }); }); @@ -65,6 +94,165 @@ test("upstream mode rejects legacy client identity headers without proxy princip } }); +test("buildApp exposes safe nullable session metadata for authenticated legacy modes", async () => { + const configurations = [ + { + name: "none", + config: loadConfig({ NODE_ENV: "test", AUTH_MODE: "none", THT_HARNESS_DIR: "/tmp/h" }), + headers: {}, + expected: { issuer: "local", roles: ["admin"] }, + }, + { + name: "mock", + config: loadConfig({ NODE_ENV: "test", AUTH_MODE: "mock", THT_HARNESS_DIR: "/tmp/h" }), + headers: { "x-mock-user": "legacy-mock" }, + expected: { issuer: "mock", subject: "legacy-mock", roles: ["user"] }, + }, + { + name: "upstream", + config: loadConfig({ NODE_ENV: "production", AUTH_MODE: "upstream", THT_HARNESS_DIR: "/tmp/h" }), + headers: { + "x-thoth-principal-issuer": "portal", + "x-thoth-principal-subject": "legacy-upstream", + "x-thoth-is-admin": "0", + }, + expected: { issuer: "portal", subject: "legacy-upstream", roles: ["user"] }, + }, + ]; + + for (const legacy of configurations) { + const app = buildApp(legacy.config, { thtRunner: {} as any, listModels: async () => [] }); + try { + const response = await app.inject({ method: "GET", url: "/me", headers: legacy.headers }); + expect(response.statusCode, legacy.name).toBe(200); + expect(response.json()).toMatchObject({ + ...legacy.expected, + csrfToken: null, + session: null, + }); + } finally { + await app.close(); + } + } +}); + +test("the session boundary exposes only exact health and authentication protocol paths", async () => { + const app = Fastify(); + app.addHook("preHandler", authenticateSession({ + mode: "local", + authentication: { + current: () => ({ + sourcePath: "/private/auth.yaml", + revision: "a".repeat(64), + value: { + version: 1, + mode: "local", + publicUrl: "http://127.0.0.1:8787", + session: { + regularTtlSeconds: 43_200, regularIdleSeconds: 7_200, + rememberTtlSeconds: 2_592_000, rememberIdleSeconds: 604_800, oidcTtlSeconds: 28_800, + }, + local: { usersFile: "users.yaml" }, + }, + }), + }, + sessionStore: { resolve: async () => undefined } as any, + })); + app.get("/health", async () => ({ ok: true })); + app.get("/auth/config", async () => ({ mode: "local" })); + app.get("/healthz", async () => ({ ok: true })); + app.get("/auth/configured", async () => ({ mode: "local" })); + + expect((await app.inject({ method: "GET", url: "/health?probe=1" })).statusCode).toBe(200); + expect((await app.inject({ method: "GET", url: "/auth/config?ui=1" })).statusCode).toBe(200); + expect((await app.inject({ method: "GET", url: "/healthz" })).statusCode).toBe(401); + expect((await app.inject({ method: "GET", url: "/auth/configured" })).statusCode).toBe(401); +}); + +test("loopback maintenance headers can never mint an administrator in configured auth modes", async () => { + const app = Fastify(); + app.addHook("preHandler", authenticateSession({ mode: "local" })); + app.get("/private", async (request) => getPrincipal(request)); + app.post("/private", async (request) => getPrincipal(request)); + const headers = { + "x-thoth-principal-issuer": "tht", + "x-thoth-principal-subject": "tht-maintenance", + "x-thoth-principal-display-name": "Tht maintenance", + "x-thoth-is-admin": "1", + }; + + for (const method of ["GET", "POST"] as const) { + const response = await app.inject({ method, url: "/private", headers, remoteAddress: "127.0.0.1" }); + expect(response.statusCode).toBe(503); + expect(response.body).not.toContain("tht-maintenance"); + } +}); + +test("the session boundary rejects tht maintenance headers outside exact loopback provenance", async () => { + const app = Fastify(); + app.addHook("preHandler", authenticateSession({ mode: "local" })); + app.get("/private", async (request) => getPrincipal(request)); + const exact = { + "x-thoth-principal-issuer": "tht", + "x-thoth-principal-subject": "tht-maintenance", + "x-thoth-principal-display-name": "Tht maintenance", + "x-thoth-is-admin": "1", + }; + + expect((await app.inject({ method: "GET", url: "/private", headers: exact, remoteAddress: "172.30.0.9" })).statusCode).toBe(503); + expect((await app.inject({ + method: "GET", url: "/private", remoteAddress: "127.0.0.1", + headers: { ...exact, "x-thoth-principal-subject": "not-maintenance" }, + })).statusCode).toBe(503); +}); + +test("the session boundary touches a valid cookie session through the bounded Task 7 store operation", async () => { + const sessions = { + resolve: vi.fn(async () => ({ + version: 1, + issuer: "local", + subject: "user-1", + method: "local", + roles: ["user"], + permissions: ["session.use"], + userAuthRevision: 1, + authConfigRevision: "b".repeat(64), + remembered: false, + createdAt: "2026-08-16T00:00:00.000Z", + lastSeenAt: "2026-08-16T00:00:00.000Z", + idleExpiresAt: "2026-08-16T02:00:00.000Z", + absoluteExpiresAt: "2026-08-16T12:00:00.000Z", + })), + touch: vi.fn(async () => {}), + }; + const app = Fastify(); + app.addHook("preHandler", authenticateSession({ + mode: "local", + authentication: { + current: () => ({ + sourcePath: "/private/auth.yaml", + revision: "b".repeat(64), + value: { + version: 1, + mode: "local", + publicUrl: "http://127.0.0.1:8787", + session: { + regularTtlSeconds: 43_200, regularIdleSeconds: 7_200, + rememberTtlSeconds: 2_592_000, rememberIdleSeconds: 604_800, oidcTtlSeconds: 28_800, + }, + local: { usersFile: "users.yaml" }, + }, + }), + }, + sessionStore: sessions as any, + })); + app.get("/private", async (request) => getPrincipal(request)); + + const token = "z".repeat(43); + expect((await app.inject({ method: "GET", url: "/private", headers: { cookie: `thothii_session=${token}` } })).statusCode).toBe(200); + expect(sessions.touch).toHaveBeenCalledWith(token); +}); + test("local identity expands tilde homes and restores private POSIX permissions", () => { expect(expandLocalHome("~/thoth-test", "/home/tester")).toBe("/home/tester/thoth-test"); expect(expandLocalHome("~", "/home/tester")).toBe("/home/tester"); diff --git a/backend/test/authentik-group-catalog.test.ts b/backend/test/authentik-group-catalog.test.ts new file mode 100644 index 00000000..e7dc6067 --- /dev/null +++ b/backend/test/authentik-group-catalog.test.ts @@ -0,0 +1,193 @@ +import { expect, test, vi } from "vitest"; +import { createAuthentikGroupCatalog } from "../src/auth/authentik-group-catalog.js"; + +const apiToken = "authentik-api-token-UNIQUE-9Q7"; +const clientSecret = "oidc-client-secret-UNIQUE-7P3"; +const passwordHash = "$argon2id$v=19$password-hash-UNIQUE-2T8"; +const cookie = "cookie-UNIQUE-5M1"; +const filePath = "/private/path-UNIQUE-4K6"; + +function catalog(fetch: typeof globalThis.fetch) { + return createAuthentikGroupCatalog({ + baseUrl: "https://authentik.example.test", + apiToken, + fetch, + }); +} + +function groups(...names: string[]): Response { + return Response.json({ pagination: { next: null }, results: names.map((name) => ({ name })) }); +} + +test("looks up only each configured group by its exact encoded name", async () => { + const fetch = vi.fn(async () => groups("TOT Users")); + const result = await catalog(fetch).verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + expect(result).toEqual([]); + expect(fetch).toHaveBeenCalledOnce(); + const [input, init] = fetch.mock.calls[0]!; + expect(String(input)).toBe("https://authentik.example.test/api/v3/core/groups/?name=TOT+Users&include_users=false&page_size=2"); + expect(init).toMatchObject({ redirect: "error" }); + expect(new Headers(init?.headers).get("authorization")).toBe(`Bearer ${apiToken}`); + expect(init?.signal).toBeInstanceOf(AbortSignal); +}); + +test.each([ + ["missing", groups(), "oidc_mapped_group_missing"], + ["duplicate exact results", groups("TOT Users", "TOT Users"), "oidc_mapped_group_ambiguous"], + ["non-exact result", groups("tot users"), "oidc_mapped_group_missing"], +])("reports configured group %s without exposing an upstream body", async (_caseName, response, code) => { + const result = await catalog(vi.fn(async () => response)) + .verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + expect(result).toEqual([expect.objectContaining({ level: "error", code, field: "TOT Users" })]); +}); + +test("rejects more than the requested two bounded group results", async () => { + const response = groups("TOT Users", "Unrelated One", "Unrelated Two"); + const result = await catalog(vi.fn(async () => response)) + .verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + expect(result).toEqual([expect.objectContaining({ code: "oidc_group_catalog_unreachable" })]); + expect(JSON.stringify(result)).not.toContain("Unrelated"); +}); + +test("treats a pagination continuation as an ambiguous configured group", async () => { + const response = Response.json({ pagination: { next: "https://authentik.example.test/api/v3/core/groups/?page=2" }, results: [{ name: "TOT Users" }] }); + const result = await catalog(vi.fn(async () => response)) + .verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + expect(result).toEqual([expect.objectContaining({ code: "oidc_mapped_group_ambiguous", field: "TOT Users" })]); +}); + +test.each([ + ["missing next", { pagination: {}, results: [{ name: "TOT Users" }] }], + ["numeric next", { pagination: { next: 2 }, results: [{ name: "TOT Users" }] }], + ["boolean next", { pagination: { next: false }, results: [{ name: "TOT Users" }] }], + ["malformed continuation", { pagination: { next: "not a URL" }, results: [{ name: "TOT Users" }] }], +])("rejects a group response with %s as unreachable", async (_label, body) => { + const result = await catalog(vi.fn(async () => Response.json(body))) + .verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + expect(result).toEqual([expect.objectContaining({ code: "oidc_group_catalog_unreachable" })]); + expect(result).not.toContainEqual(expect.objectContaining({ code: "oidc_mapped_group_ambiguous" })); +}); + +test("enforces the exact Authentik token boundary before making a request", async () => { + const fetch = vi.fn(async () => groups("TOT Users")); + const accepted = createAuthentikGroupCatalog({ + baseUrl: "https://authentik.example.test", apiToken: "a".repeat(16 * 1024), fetch, + }); + const rejected = createAuthentikGroupCatalog({ + baseUrl: "https://authentik.example.test", apiToken: "a".repeat(16 * 1024 + 1), fetch, + }); + + await expect(accepted.verifyConfiguredGroups(["TOT Users"], new AbortController().signal)).resolves.toEqual([]); + await expect(rejected.verifyConfiguredGroups(["TOT Users"], new AbortController().signal)) + .resolves.toEqual([expect.objectContaining({ code: "oidc_group_catalog_unreachable" })]); + expect(fetch).toHaveBeenCalledOnce(); +}); + +test.each([ + ["unauthorized", new Response("upstream body must not escape", { status: 401 }), "oidc_group_catalog_unauthorized"], + ["forbidden", new Response("upstream body must not escape", { status: 403 }), "oidc_group_catalog_unauthorized"], + ["redirect", new Response(null, { status: 302, headers: { location: "https://elsewhere.invalid" } }), "oidc_group_catalog_unreachable"], + ["invalid json", new Response("not-json"), "oidc_group_catalog_unreachable"], +])("returns a stable diagnostic for %s without parsing or leaking response data", async (_caseName, response, code) => { + const result = await catalog(vi.fn(async () => response)) + .verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + expect(result).toEqual([expect.objectContaining({ level: "error", code })]); + expect(JSON.stringify(result)).not.toContain("upstream body must not escape"); +}); + +test("refuses declared and streamed group catalog bodies larger than one MiB", async () => { + const declared = new Response(new ReadableStream({ pull() { throw new Error("must not read"); } }), { + headers: { "content-length": String(1024 * 1024 + 1) }, + }); + const streamed = new Response(new ReadableStream({ + type: "bytes", + pull(controller) { + controller.enqueue(new Uint8Array(1024 * 1024)); + controller.enqueue(new Uint8Array(1)); + controller.close(); + }, + })); + + for (const response of [declared, streamed]) { + const result = await catalog(vi.fn(async () => response)) + .verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + expect(result).toEqual([expect.objectContaining({ code: "oidc_group_catalog_unreachable" })]); + } +}); + +test.each([ + ["malformed", "not-a-number"], + ["oversized", String(1024 * 1024 + 1)], +])("cancels a %s declared-size body without waiting for hanging cancellation", async (_caseName, contentLength) => { + let cancelled = false; + const body = new ReadableStream({ + pull() { /* early declared-size rejection must not read */ }, + cancel() { + cancelled = true; + return new Promise(() => { /* cancellation remains advisory */ }); + }, + }); + const completion = catalog(vi.fn(async () => new Response(body, { + headers: { "content-length": contentLength }, + }))).verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + await expect(Promise.race([ + completion, + new Promise((resolve) => setTimeout(() => resolve("timed-out"), 100)), + ])).resolves.toEqual([expect.objectContaining({ code: "oidc_group_catalog_unreachable" })]); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("contains a rejected declared-size cancellation without an unhandled rejection", async () => { + let cancelled = false; + const body = new ReadableStream({ + pull() { /* early declared-size rejection must not read */ }, + cancel() { + cancelled = true; + return Promise.reject(new Error("cancellation-detail-must-stay-contained")); + }, + }); + + await expect(catalog(vi.fn(async () => new Response(body, { + headers: { "content-length": "invalid" }, + }))).verifyConfiguredGroups(["TOT Users"], new AbortController().signal)) + .resolves.toEqual([expect.objectContaining({ code: "oidc_group_catalog_unreachable" })]); + expect(cancelled).toBe(true); +}); + +test("aborts a hanging request at five seconds", async () => { + vi.useFakeTimers(); + try { + let aborted = false; + const fetch = vi.fn((_input, init) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + aborted = true; + reject(new DOMException("aborted", "AbortError")); + }, { once: true }); + })); + const completion = catalog(fetch).verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + await vi.advanceTimersByTimeAsync(5_000); + + await expect(completion).resolves.toEqual([expect.objectContaining({ code: "oidc_group_catalog_unreachable" })]); + expect(aborted).toBe(true); + } finally { + vi.useRealTimers(); + } +}); + +test("never puts secrets or upstream details in catalog diagnostics", async () => { + const upstreamDetail = `${apiToken} ${clientSecret} ${passwordHash} ${cookie} ${filePath}`; + const result = await catalog(vi.fn(async () => { + throw new Error(upstreamDetail); + })).verifyConfiguredGroups(["TOT Users"], new AbortController().signal); + + const rendered = JSON.stringify(result); + for (const sentinel of [apiToken, clientSecret, passwordHash, cookie, filePath]) expect(rendered).not.toContain(sentinel); +}); diff --git a/backend/test/authorization.test.ts b/backend/test/authorization.test.ts new file mode 100644 index 00000000..cacfed39 --- /dev/null +++ b/backend/test/authorization.test.ts @@ -0,0 +1,74 @@ +import { expect, test } from "vitest"; +import Fastify from "fastify"; +import { authPreHandler, getPrincipal } from "../src/auth/auth.js"; +import { hasPermission, requireSameOriginOrNonBrowser } from "../src/auth/authorization.js"; +import type { PrincipalContext } from "../src/auth/principal.js"; +import type { Permission } from "../src/auth/types.js"; + +const noRole: PrincipalContext = { + issuer: "oidc", subject: "no-role", roles: [], permissions: [], isAdmin: false, +}; +const user: PrincipalContext = { + issuer: "oidc", subject: "user", roles: ["user"], permissions: ["session.use"], isAdmin: false, +}; +const admin: PrincipalContext = { + issuer: "oidc", subject: "admin", roles: ["admin"], + permissions: [ + "session.use", "session.read_all", "session.manage_all", "settings.manage", + "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + ], + isAdmin: true, +}; + +const catalog: readonly Permission[] = [ + "session.use", "session.read_all", "session.manage_all", "settings.manage", + "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", +]; + +test("permission matrix gives role-less identities no access, users session use, and admins every catalog permission", () => { + for (const permission of catalog) { + expect(hasPermission(noRole, permission)).toBe(false); + expect(hasPermission(user, permission)).toBe(permission === "session.use"); + expect(hasPermission(admin, permission)).toBe(true); + } +}); + +test("compatibility adapters derive roles and permissions before routes inspect a principal", async () => { + const app = Fastify(); + app.addHook("preHandler", authPreHandler("mock")); + app.get("/me", async (request) => getPrincipal(request)); + + const response = await app.inject({ method: "GET", url: "/me", headers: { "x-mock-user": "alice" } }); + + expect(response.json()).toEqual({ + issuer: "mock", subject: "alice", displayName: "alice", + roles: ["user"], permissions: ["session.use"], isAdmin: false, + }); + + const elevated = await app.inject({ + method: "GET", url: "/me", headers: { "x-mock-user": "operator", "x-thoth-is-admin": "true" }, + }); + const malformed = await app.inject({ + method: "GET", url: "/me", headers: { "x-mock-user": "mallory", "x-thoth-is-admin": "yes" }, + }); + expect(elevated.json()).toMatchObject({ roles: ["admin"], isAdmin: true }); + expect(malformed.json()).toMatchObject({ roles: ["user"], isAdmin: false }); +}); + +test("same-origin guard permits non-browser and same-origin calls but rejects a cross-origin browser", async () => { + const app = Fastify(); + app.get("/guard", async (request, reply) => { + const denied = requireSameOriginOrNonBrowser(request, reply); + return denied ?? { ok: true }; + }); + + expect((await app.inject({ method: "GET", url: "/guard" })).statusCode).toBe(200); + expect((await app.inject({ + method: "GET", url: "/guard", headers: { host: "127.0.0.1:8080", origin: "http://127.0.0.1:8080" }, + })).statusCode).toBe(200); + const denied = await app.inject({ + method: "GET", url: "/guard", headers: { host: "127.0.0.1:8080", origin: "https://evil.example" }, + }); + expect(denied.statusCode).toBe(403); + expect(denied.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); +}); diff --git a/backend/test/config.test.ts b/backend/test/config.test.ts index 7db92b3c..fc4920b1 100644 --- a/backend/test/config.test.ts +++ b/backend/test/config.test.ts @@ -1,6 +1,31 @@ import { expect, test } from "vitest"; +import { chmodSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; import { loadConfig } from "../src/config.js"; +function authFile(value: unknown): { directory: string; file: string } { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-app-auth-config-")); + chmodSync(directory, 0o700); + const file = join(directory, "auth.yaml"); + writeFileSync(file, stringify(value), { encoding: "utf8", mode: 0o600 }); + chmodSync(file, 0o600); + return { directory, file }; +} + +function oidcAuthConfig(): Record { + return { + version: 1, mode: "oidc", publicUrl: "https://thothii.example.org", + oidc: { + issuer: "https://authentik.example.org/application/o/thothii/", clientId: "thothii", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", scopes: ["openid"], groupsClaim: "groups", + }, + groupCatalog: { driver: "authentik", baseUrl: "https://authentik.example.org", apiTokenRef: "THT_AUTHENTIK_API_TOKEN" }, + authorization: { groupRoles: { "TOT Users": ["user"], "TOT Admin": ["admin"] } }, + }; +} + test("loadConfig accepts container listening and runtime paths", () => { expect(loadConfig({ HOST: "0.0.0.0", @@ -17,10 +42,18 @@ test("loadConfig accepts container listening and runtime paths", () => { thtBin: "/opt/venv/bin/tht", piBin: "/usr/local/bin/pi", settingsFile: "/data/settings/settings.json", + maintenanceFile: "/data/settings/maintenance.json", dataRoot: "/data", }); }); +test("loadConfig accepts only an absolute mounted Pi authentication source", () => { + expect(loadConfig({ THT_PI_AUTH_FILE: "/home/thoth/.pi/agent/auth.json" }).piAuthFile) + .toBe("/home/thoth/.pi/agent/auth.json"); + expect(() => loadConfig({ THT_PI_AUTH_FILE: "relative/auth.json" })) + .toThrow("Pi authentication source configuration is invalid"); +}); + test("loadConfig keeps local development defaults", () => { expect(loadConfig({})).toMatchObject({ host: "127.0.0.1", @@ -29,10 +62,124 @@ test("loadConfig keeps local development defaults", () => { thtBin: "tht", piBin: "pi", settingsFile: "data/settings.json", + maintenanceFile: "data/maintenance.json", + workspaceRegistry: { + root: "/data/workspace-registry", + branch: "main", + }, + workspaceSecretStoreRoot: "/data/workspace-secrets", + workspaceSecretRuntimeRoot: "/tmp/thothii-workspace-secrets", + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, + authMode: "none", + authConfigFile: "/run/thothii-auth/auth.yaml", + authStateRoot: "/data/auth", }); expect(loadConfig({}).dataRoot).toBeUndefined(); }); +test("loadConfig allows none and mock only outside production when auth.yaml is absent", () => { + const originalNodeEnvironment = process.env.NODE_ENV; + delete process.env.NODE_ENV; + try { + expect(() => loadConfig({ AUTH_MODE: "none" })).toThrow("production requires auth.yaml or AUTH_MODE=upstream"); + } finally { + if (originalNodeEnvironment === undefined) delete process.env.NODE_ENV; + else process.env.NODE_ENV = originalNodeEnvironment; + } + expect(loadConfig({ NODE_ENV: "test", AUTH_MODE: "mock" }).authMode).toBe("mock"); + expect(loadConfig({ NODE_ENV: "development", AUTH_MODE: "none" }).authMode).toBe("none"); + expect(loadConfig({ NODE_ENV: "production", AUTH_MODE: "upstream" }).authMode).toBe("upstream"); + expect(() => loadConfig({ NODE_ENV: "production" })).toThrow("production requires auth.yaml or AUTH_MODE=upstream"); + expect(() => loadConfig({ NODE_ENV: "production", AUTH_MODE: "mock" })) + .toThrow("production requires auth.yaml or AUTH_MODE=upstream"); +}); + +test("local Compose profiles explicitly select the development auth environment", () => { + for (const profile of ["../../deploy/compose.local.yaml", "../../docker-compose.dev.yml"]) { + expect(readFileSync(new URL(profile, import.meta.url), "utf8")).toMatch(/NODE_ENV:\s*development/); + } +}); + +test("loadConfig makes an existing auth.yaml authoritative and rejects AUTH_MODE split-brain", () => { + const { directory, file } = authFile(oidcAuthConfig()); + try { + const config = loadConfig({ THT_AUTH_CONFIG_FILE: file, THT_AUTH_STATE_ROOT: "/state/auth" }); + expect(config.authMode).toBe("oidc"); + expect(config.authStateRoot).toBe("/state/auth"); + expect(config.authentication?.current().sourcePath).toBe(file); + expect(() => loadConfig({ THT_AUTH_CONFIG_FILE: file, AUTH_MODE: "upstream" })) + .toThrow("authentication configuration and AUTH_MODE cannot both be set"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("loadConfig rejects an auth config path that exists but is not a regular file", () => { + const directory = mkdtempSync(join(realpathSync(tmpdir()), "thothii-app-auth-config-directory-")); + try { + expect(() => loadConfig({ THT_AUTH_CONFIG_FILE: directory })) + .toThrow("authentication configuration is invalid"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("public exposure accepts configured OIDC and the upstream migration mode only", () => { + const { directory, file } = authFile(oidcAuthConfig()); + try { + expect(loadConfig({ + THOTH_PUBLIC_EXPOSURE: "true", THT_AUTH_CONFIG_FILE: file, THT_SESSION_STORAGE: "postgres", + THT_SESSION_DB_HOST: "db.internal", THT_SESSION_DB_NAME: "thoth", THT_SESSION_RUNTIME_USER: "thoth_sessions_app", + THT_SESSION_RUNTIME_PASSWORD_FILE: "/run/secrets/session_runtime_password", THT_SESSION_DB_SSLMODE: "verify-full", + THT_SESSION_DB_SSLROOTCERT: "/run/secrets/session_ca.pem", + }).authMode).toBe("oidc"); + expect(() => loadConfig({ THOTH_PUBLIC_EXPOSURE: "true", AUTH_MODE: "mock" })) + .toThrow("public exposure requires AUTH_MODE=upstream or configured OIDC"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("loadConfig accepts only the allowed internal semantic runtime hosts", () => { + expect(loadConfig({ + THT_INTERNAL_QDRANT_URL: "http://localhost:6333", + THT_INTERNAL_EMBEDDING_URL: "http://127.0.0.1:11434", + })).toMatchObject({ + internalQdrantUrl: "http://localhost:6333", + internalEmbeddingUrl: "http://127.0.0.1:11434", + }); + + expect(() => loadConfig({ THT_INTERNAL_QDRANT_URL: "http://qdrant.internal:6333" })) + .toThrow(/internal.*qdrant|host validation|invalid/i); + expect(() => loadConfig({ THT_INTERNAL_EMBEDDING_URL: "http://example.com:11434" })) + .toThrow(/internal.*embedding|host validation|invalid/i); + expect(() => loadConfig({ THT_INTERNAL_QDRANT_URL: "https://qdrant:6333" })) + .toThrow(/internal.*qdrant|invalid/i); + expect(() => loadConfig({ THT_INTERNAL_EMBEDDING_URL: "https://embedding:11434" })) + .toThrow(/internal.*embedding|invalid/i); +}); + +test("loadConfig enables the legacy workspace request only through explicit local mode", () => { + expect(loadConfig({ THT_LEGACY_WORKSPACE_MODE: "local" }).legacyWorkspaceMode).toBe(true); + + expect(() => loadConfig({ + THT_LEGACY_WORKSPACE_MODE: "local", + AUTH_MODE: "upstream", + THT_SESSION_STORAGE: "postgres", + THT_SESSION_DB_HOST: "db.internal", + THT_SESSION_DB_NAME: "thoth", + THT_SESSION_RUNTIME_USER: "thoth_sessions_app", + THT_SESSION_RUNTIME_PASSWORD_FILE: "/run/secrets/session_runtime_password", + THT_SESSION_DB_SSLMODE: "verify-full", + THT_SESSION_DB_SSLROOTCERT: "/run/secrets/session_ca.pem", + })).toThrow(/legacy workspace mode requires local session storage/); + expect(() => loadConfig({ THT_LEGACY_WORKSPACE_MODE: "true" })) + .toThrow(/legacy workspace mode configuration is invalid/); +}); + test("loadConfig rejects unauthenticated public exposure", () => { expect(() => loadConfig({ THOTH_PUBLIC_EXPOSURE: "true", diff --git a/backend/test/e2e-f1.test.ts b/backend/test/e2e-f1.test.ts index daa5f355..cf3df606 100644 --- a/backend/test/e2e-f1.test.ts +++ b/backend/test/e2e-f1.test.ts @@ -29,7 +29,9 @@ async function readUntil( } test("loop F1: crea sessione → SSE riceve il widget → risponde → il modello riparte (follow-up)", async () => { - const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + const app = buildApp(loadConfig({ + THT_HARNESS_DIR: "../harness", THT_LEGACY_WORKSPACE_MODE: "local", + }), { thtRunner: { ollamaEnsure: async () => ({ ok: true }), searchPack: async () => {}, @@ -43,11 +45,12 @@ test("loop F1: crea sessione → SSE riceve il widget → risponde → il modell const base = `http://127.0.0.1:${(app.server.address() as any).port}`; // 1. Create session — spawns fake-pi, sends prompt, fake-pi emits widget - await fetch(`${base}/sessions`, { + const created = await fetch(`${base}/sessions`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ workspace: "w", question: "q" }), }); + expect(created.status).toBe(200); // 2. Small delay to let fake-pi process the prompt and fill pendingWidget await new Promise((r) => setTimeout(r, 50)); diff --git a/backend/test/effective-config.test.ts b/backend/test/effective-config.test.ts new file mode 100644 index 00000000..ea60a30b --- /dev/null +++ b/backend/test/effective-config.test.ts @@ -0,0 +1,197 @@ +import { createHash } from "node:crypto"; +import { expect, test } from "vitest"; +import { + buildCanonicalEffectiveConfig, + canonicalEffectiveConfigJson, + configFingerprint, + effectiveConfigIdentity, + inputFingerprint, +} from "../src/workspaces/effective-config.js"; + +const semanticRuntime = { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, +}; + +function directRendered(): Record { + return { + runtime_identity: { + workspace_id: "psd-clinical", + workspace_revision: "a".repeat(40), + source_identity: "workspace://psd-clinical", + }, + session_storage: { mode: "local" }, + profile: "server", + language: "it", + database: { + host: "dwh.internal", + port: 5432, + database: "postgres", + schema: "datawarehouse", + user: "thoth_reader", + password_file: "/run/secrets/dwh-password", + ssl_ca_file: "/run/secrets/dwh-ca.pem", + transport: "direct", + }, + dwh: { type: "postgres_direct" }, + resources: { + vector: { + engine: "qdrant", + base_url: "http://qdrant:6333", + collection: "psd-clinical", + dimensions: 1024, + distance: "cosine", + collection_lifecycle: "require_existing", + }, + embeddings: { + provider: "ollama_internal", + base_url: "http://embedding:11434", + model: "qwen3-embedding:0.6b", + dimensions: 1024, + }, + }, + roots: { + sessions: "/data/sessions/psd-clinical/sessions", + artifacts: "/data/sessions/psd-clinical/artifacts", + indexes: "/data/sessions/psd-clinical/indexes", + }, + paths: { + sessions: "/data/sessions/psd-clinical/sessions", + artifacts: "/data/sessions/psd-clinical/artifacts", + indexes: "/data/sessions/psd-clinical/indexes", + memory: "/data/sessions/psd-clinical/memory", + }, + evidence: { + sources: [{ + type: "filesystem", + root: "/srv/registry/snapshots/rev/psd-clinical/evidence", + patterns: ["**/*.md"], + max_bytes: 10_485_760, + }], + }, + }; +} + +function restRendered(): Record { + return { + ...directRendered(), + database: { + host: "localhost", + port: 5432, + database: "postgres", + schema: "datawarehouse", + user: "rest", + password: "", + transport: "rest", + }, + dwh: { + type: "thoth_rest", + database: { database: "postgres", schema: "datawarehouse" }, + endpoint: { + base_url: "https://dwh.example.test", + api_key_file: "/run/secrets/dwh-api-key", + }, + }, + rest: { + base_url: "https://dwh.example.test", + api_key_file: "/run/secrets/dwh-api-key", + }, + }; +} + +const directCanonical = + `{"schemaVersion":1,"dwh":{` + + `"engine":"postgres","database":"postgres","schema":"datawarehouse",` + + `"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` + + `"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` + + `"embedding":{"model":"qwen3-embedding:0.6b","dimensions":1024},` + + `"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` + + `"indexes":"/data/sessions/psd-clinical/indexes"}}`; + +test("canonical effective config is deterministic and contains the expected key order", () => { + const rendered = directRendered(); + const canonical = buildCanonicalEffectiveConfig(rendered); + expect(canonicalEffectiveConfigJson(canonical)).toBe(directCanonical); + expect(buildCanonicalEffectiveConfig(rendered)).toEqual(canonical); +}); + +test("canonical effective config excludes secrets, evidence, session storage, and runtime identity", () => { + const json = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(directRendered())); + expect(json).not.toContain("password_file"); + expect(json).not.toContain("ssl_ca_file"); + expect(json).not.toContain("session_storage"); + expect(json).not.toContain("runtime_identity"); + expect(json).not.toContain("evidence"); + expect(json).not.toContain("sources"); + expect(json).not.toContain("collection_lifecycle"); + expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity + expect(json).not.toContain("memory"); + expect(json).not.toContain('"sessions"'); +}); + +test("REST transport canonicalizes to transport rest_api with baseUrl and no host/port", () => { + const canonical = buildCanonicalEffectiveConfig(restRendered()); + const json = canonicalEffectiveConfigJson(canonical); + expect(json).toContain(`"transport":"rest_api"`); + expect(json).toContain(`"baseUrl":"https://dwh.example.test"`); + expect(json).not.toContain(`"host":`); + expect(json).not.toContain(`"port":`); + expect(json).not.toContain("api_key_file"); +}); + +test("identity and fingerprint helpers produce stable prefixed hex values", () => { + const rendered = directRendered(); + const identity = effectiveConfigIdentity("psd-clinical", rendered); + const cfg = configFingerprint(rendered); + const input = inputFingerprint("psd-clinical", rendered); + + expect(identity).toMatch(/^workspace:\/\/psd-clinical@v1:[0-9a-f]{64}$/); + expect(cfg).toBe("sha256:" + createHash("sha256").update(directCanonical).digest("hex")); + expect(input).toBe("sha256:" + createHash("sha256").update(identity).digest("hex")); + expect(input).not.toBe(cfg); +}); + +test("content-only or session_storage changes keep the same effective config identity", () => { + const base = directRendered(); + const identityBefore = effectiveConfigIdentity("psd-clinical", base); + const fingerprintBefore = configFingerprint(base); + + const contentOnly = { + ...base, + runtime_identity: { + ...base.runtime_identity, + workspace_revision: "b".repeat(40), + }, + session_storage: { mode: "remote", url: "http://example.test" }, + evidence: { + sources: [{ + type: "filesystem", + root: "/srv/registry/snapshots/other/psd-clinical/evidence", + patterns: ["**/*.txt"], + max_bytes: 999, + }], + }, + }; + + expect(effectiveConfigIdentity("psd-clinical", contentOnly)).toBe(identityBefore); + expect(configFingerprint(contentOnly)).toBe(fingerprintBefore); +}); + +test("DWH-affecting changes alter the effective config identity", () => { + const base = directRendered(); + const identityBefore = effectiveConfigIdentity("psd-clinical", base); + + const changedHost = { ...base, database: { ...(base.database as object), host: "dwh-two.internal" } }; + expect(effectiveConfigIdentity("psd-clinical", changedHost)).not.toBe(identityBefore); + + const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } }; + expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore); + + const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record).vector, collection: "other" } } }; + expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore); + + const changedTransport = restRendered(); + expect(effectiveConfigIdentity("psd-clinical", changedTransport)).not.toBe(identityBefore); +}); diff --git a/backend/test/evidence-materialization.test.ts b/backend/test/evidence-materialization.test.ts new file mode 100644 index 00000000..2bbfce5e --- /dev/null +++ b/backend/test/evidence-materialization.test.ts @@ -0,0 +1,134 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js"; +import { materializeEvidenceTree } from "../src/workspaces/evidence-materialization.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Evidence Materializer Test", + gitAuthorEmail: "evidence-materializer@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +async function fixture(): Promise<{ root: string; remote: string; commit: string }> { + const root = mkdtempSync(join(tmpdir(), "thoth-evidence-materializer-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Evidence Materializer Test"]); + await git(source, ["config", "user.email", "evidence-materializer@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n"); + mkdirSync(join(source, "research", "evidence", "nested"), { recursive: true }); + writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n schema_version: 3\n id: research\n"); + writeFileSync(join(source, "research", "evidence", "guide.md"), "# guide\n"); + writeFileSync(join(source, "research", "evidence", "nested", "deep.md"), "# deep\n"); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + return { root, remote, commit }; +} + +async function repo(fixture: { root: string; remote: string }): Promise { + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + return repository; +} + +test("materializes the tree, hashes every file, and writes a bounded manifest", async () => { + const fixtureValue = await fixture(); + const repository = await repo(fixtureValue); + const target = mkdtempSync(join(tmpdir(), "thoth-evidence-target-")); + temporaryRoots.push(target); + + const result = await materializeEvidenceTree({ + repository, + revision: fixtureValue.commit, + id: "research", + targetDirectory: target, + }); + + expect(readFileSync(join(result.root, "guide.md"), "utf8")).toBe("# guide\n"); + expect(readFileSync(join(result.root, "nested", "deep.md"), "utf8")).toBe("# deep\n"); + expect(result.manifest).toMatchObject({ + schemaVersion: 1, + workspace: "research", + commit: fixtureValue.commit, + entryCount: 2, + }); + expect(Object.keys(result.manifest.files).sort()).toEqual(["guide.md", "nested/deep.md"]); + expect(result.manifest.files["guide.md"]!.digest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(result.manifestDigest).toMatch(/^[0-9a-f]{64}$/); + expect(readFileSync(result.manifestPath, "utf8")).toContain('"entryCount":2'); +}); + +test("refuses symlink-containing trees and bound violations without publishing", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "E"]); + await git(source, ["config", "user.email", "e@e.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n"); + mkdirSync(join(source, "research", "evidence"), { recursive: true }); + writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n schema_version: 3\n id: research\n"); + writeFileSync(join(source, "research", "outside.md"), "# outside\n"); + symlinkSync("../outside.md", join(source, "research", "evidence", "link.md")); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "symlink"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + const repository = await repo({ root, remote }); + const target = mkdtempSync(join(tmpdir(), "thoth-evidence-unsafe-target-")); + temporaryRoots.push(target); + + await expect(materializeEvidenceTree({ repository, revision: commit, id: "research", targetDirectory: target })) + .rejects.toThrow(); + expect(readdirSync(target)).toEqual([]); + + // A valid tree but a per-file bound of 1 byte must also refuse. + const fixtureValue = await fixture(); + const repository2 = await repo(fixtureValue); + const target2 = mkdtempSync(join(tmpdir(), "thoth-evidence-bound-target-")); + temporaryRoots.push(target2); + await expect(materializeEvidenceTree({ + repository: repository2, + revision: fixtureValue.commit, + id: "research", + targetDirectory: target2, + limits: { maxFileBytes: 1 }, + })).rejects.toThrow(); + expect(readdirSync(target2)).toEqual([]); +}); diff --git a/backend/test/fixtures/argon2id-vectors.json b/backend/test/fixtures/argon2id-vectors.json new file mode 100644 index 00000000..c04a149a --- /dev/null +++ b/backend/test/fixtures/argon2id-vectors.json @@ -0,0 +1 @@ +[{"password":"correct horse battery staple","saltHex":"000102030405060708090a0b0c0d0e0f","memoryKiB":65536,"passes":3,"parallelism":1,"keyLength":32,"phc":"$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"}] diff --git a/backend/test/fixtures/oidc-provider.mjs b/backend/test/fixtures/oidc-provider.mjs new file mode 100644 index 00000000..86fc58f9 --- /dev/null +++ b/backend/test/fixtures/oidc-provider.mjs @@ -0,0 +1,463 @@ +#!/usr/bin/env node +/** + * Hermetic loopback OIDC/AuthentiK-shaped provider for browser smoke tests. + * + * It deliberately has no network dependency and binds only to 127.0.0.1. Its + * ephemeral TLS and signing keys are test-scoped; callers receive the CA path + * needed to trust the provider from a spawned backend process. + */ +import { spawnSync } from "node:child_process"; +import { + createHash, + generateKeyPairSync, + randomBytes, + sign as signRsa, + timingSafeEqual, +} from "node:crypto"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:https"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const LOOPBACK_HOST = "127.0.0.1"; +const ISSUER_PATH = "/application/o/thothii"; +const MAX_BODY_BYTES = 32 * 1024; +const MAX_STATE_TTL_MS = 5 * 60 * 1_000; +const MAX_STATE_LIMIT = 1_024; +const MAX_DEVICE_POLLS = 32; +const VALID_IDENTITIES = new Set([ + "ordinary", + "admin", + "missing-groups", + "malformed-groups", + "unmapped", + "expired", +]); + +const identityClaims = Object.freeze({ + ordinary: { subject: "fixture-ordinary", displayName: "Fixture ordinary", groups: ["fixture-users"] }, + admin: { subject: "fixture-admin", displayName: "Fixture administrator", groups: ["fixture-admin"] }, + "missing-groups": { subject: "fixture-missing-groups", displayName: "Fixture missing groups" }, + "malformed-groups": { subject: "fixture-malformed-groups", displayName: "Fixture malformed groups", groups: ["fixture-users", 7] }, + unmapped: { subject: "fixture-unmapped", displayName: "Fixture unmapped", groups: ["fixture-unmapped"] }, + expired: { subject: "fixture-expired", displayName: "Fixture expired", groups: ["fixture-users"], expired: true }, +}); + +function safeError(code) { + return new Error(code); +} + +function boundedInteger(value, fallback, maximum, code) { + const selected = value ?? fallback; + if (!Number.isSafeInteger(selected) || selected < 1 || selected > maximum) throw safeError(code); + return selected; +} + +function boundedNonNegativeInteger(value, fallback, maximum, code) { + const selected = value ?? fallback; + if (!Number.isSafeInteger(selected) || selected < 0 || selected > maximum) throw safeError(code); + return selected; +} + +function controlledString(value, code) { + if (typeof value !== "string" || value.length < 1 || value.length > 512 || /[\r\n]/u.test(value)) { + throw safeError(code); + } + return value; +} + +function exactParameter(values, name) { + const matches = values.getAll(name); + return matches.length === 1 && matches[0].length > 0 ? matches[0] : undefined; +} + +function secretMatches(actual, expected) { + if (typeof actual !== "string") return false; + const actualDigest = createHash("sha256").update(actual).digest(); + const expectedDigest = createHash("sha256").update(expected).digest(); + return timingSafeEqual(actualDigest, expectedDigest); +} + +function pruneExpired(records, currentTime) { + for (const [key, record] of records) { + if (record.expiresAt <= currentTime) records.delete(key); + } +} + +function ensurePrivateDirectory(directory) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + chmodSync(directory, 0o700); +} + +function createCertificate(directory) { + const keyFile = join(directory, "provider-key.pem"); + const certificateFile = join(directory, "provider-ca.pem"); + const result = spawnSync("openssl", [ + "req", "-x509", "-newkey", "rsa:2048", "-sha256", "-nodes", + "-keyout", keyFile, + "-out", certificateFile, + "-subj", "/CN=127.0.0.1", + "-addext", "subjectAltName=IP:127.0.0.1", + "-days", "1", + ], { stdio: "ignore" }); + if (result.status !== 0) throw safeError("oidc_fixture_certificate_generation_failed"); + chmodSync(keyFile, 0o600); + chmodSync(certificateFile, 0o600); + return { key: readFileSync(keyFile), cert: readFileSync(certificateFile), certificateFile }; +} + +function sendJson(reply, status, value) { + reply.writeHead(status, { + "cache-control": "no-store", + "content-type": "application/json; charset=utf-8", + }); + reply.end(JSON.stringify(value)); +} + +function redirect(reply, location) { + reply.writeHead(302, { "cache-control": "no-store", location }); + reply.end(); +} + +async function requestBody(request) { + let size = 0; + const chunks = []; + for await (const chunk of request) { + size += chunk.length; + if (size > MAX_BODY_BYTES) throw safeError("oidc_fixture_request_too_large"); + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function jwt(privateKey, issuer, audience, nonce, identity, currentTime) { + const claims = identityClaims[identity]; + const now = Math.floor(currentTime / 1_000); + const payload = { + iss: issuer, + sub: claims.subject, + aud: audience, + exp: now + (claims.expired ? -30 : 60), + iat: now - 1, + nonce, + name: claims.displayName, + ...(Object.hasOwn(claims, "groups") ? { groups: claims.groups } : {}), + }; + const header = { alg: "RS256", typ: "JWT", kid: "fixture-rs256" }; + const protectedPart = Buffer.from(JSON.stringify(header)).toString("base64url"); + const payloadPart = Buffer.from(JSON.stringify(payload)).toString("base64url"); + const signingInput = `${protectedPart}.${payloadPart}`; + const signature = signRsa("RSA-SHA256", Buffer.from(signingInput), privateKey).toString("base64url"); + return `${signingInput}.${signature}`; +} + +function authorizationIdentity(identity) { + if (!VALID_IDENTITIES.has(identity)) throw safeError("oidc_fixture_identity_invalid"); + return identity; +} + +/** + * Start an HTTPS test provider. The returned telemetry intentionally excludes + * transient authorization codes, browser state, nonces, and token material. + */ +export async function startFakeOidcProvider(options = {}) { + const host = options.host ?? LOOPBACK_HOST; + if (host !== LOOPBACK_HOST) throw safeError("oidc_fixture_loopback_required"); + const registration = options.registration; + if (!registration || typeof registration !== "object") throw safeError("oidc_fixture_registration_required"); + const clientId = controlledString(registration.clientId, "oidc_fixture_client_id_invalid"); + const clientSecret = controlledString(registration.clientSecret, "oidc_fixture_client_secret_invalid"); + const redirectUri = controlledString(registration.redirectUri, "oidc_fixture_redirect_invalid"); + let parsedRedirect; + try { + parsedRedirect = new URL(redirectUri); + } catch { + throw safeError("oidc_fixture_redirect_invalid"); + } + if (parsedRedirect.protocol !== "http:" || parsedRedirect.hostname !== LOOPBACK_HOST + || parsedRedirect.username || parsedRedirect.password || parsedRedirect.hash) { + throw safeError("oidc_fixture_redirect_invalid"); + } + const apiToken = controlledString(options.apiToken, "oidc_fixture_api_token_required"); + const now = options.now ?? Date.now; + if (typeof now !== "function") throw safeError("oidc_fixture_clock_invalid"); + const authorizationStateTtlMs = boundedInteger( + options.authorizationStateTtlMs, 60_000, MAX_STATE_TTL_MS, "oidc_fixture_authorization_ttl_invalid", + ); + const authorizationStateLimit = boundedInteger( + options.authorizationStateLimit, 64, MAX_STATE_LIMIT, "oidc_fixture_authorization_limit_invalid", + ); + const deviceStateTtlMs = boundedInteger( + options.deviceStateTtlMs, 60_000, MAX_STATE_TTL_MS, "oidc_fixture_device_ttl_invalid", + ); + const deviceStateLimit = boundedInteger( + options.deviceStateLimit, 32, MAX_STATE_LIMIT, "oidc_fixture_device_limit_invalid", + ); + const devicePendingPolls = boundedNonNegativeInteger( + options.devicePendingPolls, 0, MAX_DEVICE_POLLS, "oidc_fixture_device_pending_polls_invalid", + ); + const devicePollLimit = boundedInteger( + options.devicePollLimit, 5, MAX_DEVICE_POLLS, "oidc_fixture_device_poll_limit_invalid", + ); + const ownsDirectory = options.directory === undefined; + const directory = options.directory ?? mkdtempSync(join(tmpdir(), "thothii-oidc-fixture-")); + ensurePrivateDirectory(directory); + const certificate = createCertificate(directory); + const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const publicJwk = publicKey.export({ format: "jwk" }); + const jwks = { + keys: [{ ...publicJwk, alg: "RS256", kid: "fixture-rs256", use: "sig" }], + }; + const authorizations = new Map(); + const deviceCodes = new Map(); + let activeIdentity = authorizationIdentity(options.identity ?? "ordinary"); + let lastAuthorization = undefined; + let issuer = undefined; + let baseUrl = undefined; + + function currentTime() { + const value = now(); + if (!Number.isSafeInteger(value) || value < 0) throw safeError("oidc_fixture_clock_invalid"); + return value; + } + + function authenticateClient(request, values) { + if (request.headers.authorization !== undefined) return false; + const suppliedClientId = exactParameter(values, "client_id"); + const suppliedClientSecret = exactParameter(values, "client_secret"); + return secretMatches(suppliedClientId, clientId) && secretMatches(suppliedClientSecret, clientSecret); + } + + const server = createServer({ key: certificate.key, cert: certificate.cert }, async (request, reply) => { + try { + if (!issuer || !baseUrl || !request.url) { + sendJson(reply, 503, { error: "temporarily_unavailable" }); + return; + } + const url = new URL(request.url, baseUrl); + const path = url.pathname; + const discoveryPath = `${ISSUER_PATH}/.well-known/openid-configuration`; + const rfc8414Path = `/.well-known/openid-configuration${ISSUER_PATH}`; + if (request.method === "GET" && (path === discoveryPath || path === rfc8414Path)) { + sendJson(reply, 200, { + issuer, + authorization_endpoint: `${issuer}authorize`, + token_endpoint: `${issuer}token`, + jwks_uri: `${issuer}jwks`, + device_authorization_endpoint: `${issuer}device_authorization`, + response_types_supported: ["code"], + subject_types_supported: ["public"], + grant_types_supported: ["authorization_code", "urn:ietf:params:oauth:grant-type:device_code"], + token_endpoint_auth_methods_supported: ["client_secret_post"], + code_challenge_methods_supported: ["S256"], + id_token_signing_alg_values_supported: ["RS256"], + }); + return; + } + if (request.method === "GET" && path === `${ISSUER_PATH}/jwks`) { + sendJson(reply, 200, jwks); + return; + } + if (request.method === "GET" && path === `${ISSUER_PATH}/authorize`) { + const suppliedRedirectUri = exactParameter(url.searchParams, "redirect_uri"); + const suppliedClientId = exactParameter(url.searchParams, "client_id"); + const state = exactParameter(url.searchParams, "state"); + const nonce = exactParameter(url.searchParams, "nonce"); + const challenge = exactParameter(url.searchParams, "code_challenge"); + if (exactParameter(url.searchParams, "response_type") !== "code" + || !secretMatches(suppliedRedirectUri, redirectUri) + || !secretMatches(suppliedClientId, clientId) || !state || !nonce || !challenge + || exactParameter(url.searchParams, "code_challenge_method") !== "S256") { + sendJson(reply, 400, { error: "invalid_request" }); + return; + } + const reservationTime = currentTime(); + pruneExpired(authorizations, reservationTime); + if (authorizations.size >= authorizationStateLimit) { + sendJson(reply, 503, { error: "temporarily_unavailable" }); + return; + } + const callback = new URL(redirectUri); + const code = randomBytes(32).toString("base64url"); + authorizations.set(code, { + challenge, + clientId, + redirectUri, + identity: activeIdentity, + nonce, + expiresAt: reservationTime + authorizationStateTtlMs, + }); + lastAuthorization = { identity: activeIdentity, codeChallengeMethod: "S256", pkceVerified: false }; + callback.searchParams.set("code", code); + callback.searchParams.set("state", state); + redirect(reply, callback.href); + return; + } + if (request.method === "POST" && path === `${ISSUER_PATH}/device_authorization`) { + const values = new URLSearchParams(await requestBody(request)); + if (!authenticateClient(request, values)) { + sendJson(reply, 401, { error: "invalid_client" }); + return; + } + const reservationTime = currentTime(); + pruneExpired(deviceCodes, reservationTime); + if (deviceCodes.size >= deviceStateLimit) { + sendJson(reply, 503, { error: "temporarily_unavailable" }); + return; + } + const deviceCode = randomBytes(32).toString("base64url"); + const userCode = "FIXTURE-CODE"; + deviceCodes.set(deviceCode, { + clientId, + identity: activeIdentity, + nonce: "device", + expiresAt: reservationTime + deviceStateTtlMs, + polls: 0, + }); + sendJson(reply, 200, { + device_code: deviceCode, + user_code: userCode, + verification_uri: `${issuer}device`, + verification_uri_complete: `${issuer}device?user_code=${userCode}`, + expires_in: Math.max(1, Math.floor(deviceStateTtlMs / 1_000)), + interval: 1, + }); + return; + } + if (request.method === "POST" && path === `${ISSUER_PATH}/token`) { + const values = new URLSearchParams(await requestBody(request)); + if (!authenticateClient(request, values)) { + sendJson(reply, 401, { error: "invalid_client" }); + return; + } + const tokenTime = currentTime(); + pruneExpired(authorizations, tokenTime); + pruneExpired(deviceCodes, tokenTime); + const grantType = exactParameter(values, "grant_type"); + let record; + if (grantType === "authorization_code") { + const code = exactParameter(values, "code") ?? ""; + record = authorizations.get(code); + if (record !== undefined) authorizations.delete(code); + const verifier = exactParameter(values, "code_verifier") ?? ""; + const suppliedRedirectUri = exactParameter(values, "redirect_uri"); + const suppliedClientId = exactParameter(values, "client_id"); + const verified = record !== undefined + && secretMatches(suppliedClientId, record.clientId) + && secretMatches(suppliedRedirectUri, record.redirectUri) + && secretMatches(createHash("sha256").update(verifier).digest("base64url"), record.challenge); + if (!verified) { + sendJson(reply, 400, { error: "invalid_grant" }); + return; + } + if (lastAuthorization) lastAuthorization = { ...lastAuthorization, pkceVerified: true }; + } else if (grantType === "urn:ietf:params:oauth:grant-type:device_code") { + const deviceCode = exactParameter(values, "device_code") ?? ""; + record = deviceCodes.get(deviceCode); + if (record === undefined) { + sendJson(reply, 400, { error: "invalid_grant" }); + return; + } + record.polls += 1; + if (record.polls >= devicePollLimit && record.polls <= devicePendingPolls) { + deviceCodes.delete(deviceCode); + sendJson(reply, 400, { error: "expired_token" }); + return; + } + if (record.polls <= devicePendingPolls) { + sendJson(reply, 400, { error: "authorization_pending" }); + return; + } + deviceCodes.delete(deviceCode); + } else { + sendJson(reply, 400, { error: "unsupported_grant_type" }); + return; + } + sendJson(reply, 200, { + access_token: randomBytes(32).toString("base64url"), + token_type: "Bearer", + expires_in: 60, + id_token: jwt(privateKey, issuer, record.clientId, record.nonce, record.identity, tokenTime), + }); + return; + } + if (request.method === "GET" && path === "/api/v3/core/groups/") { + const authorization = request.headers.authorization; + if (!secretMatches(authorization, `Bearer ${apiToken}`)) { + sendJson(reply, 401, { detail: "authentication required" }); + return; + } + const name = url.searchParams.get("name") ?? ""; + const present = name === "fixture-users" || name === "fixture-admin"; + sendJson(reply, 200, { + pagination: { next: null }, + results: present ? [{ name }] : [], + }); + return; + } + sendJson(reply, 404, { error: "not_found" }); + } catch { + if (!reply.headersSent) sendJson(reply, 400, { error: "invalid_request" }); + else reply.end(); + } + }); + + try { + await new Promise((resolveListen, rejectListen) => { + const onError = (error) => rejectListen(error); + server.once("error", onError); + server.listen({ host, port: options.port ?? 0 }, () => { + server.off("error", onError); + resolveListen(); + }); + }); + } catch (error) { + server.close(); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + throw error; + } + const address = server.address(); + if (!address || typeof address === "string") { + await new Promise((resolveClose) => server.close(resolveClose)); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + throw safeError("oidc_fixture_listen_failed"); + } + baseUrl = `https://${LOOPBACK_HOST}:${address.port}`; + issuer = `${baseUrl}${ISSUER_PATH}/`; + + return { + baseUrl, + issuer, + caFile: certificate.certificateFile, + setIdentity(identity) { + activeIdentity = authorizationIdentity(identity); + }, + lastAuthorization() { + return lastAuthorization === undefined ? undefined : { ...lastAuthorization }; + }, + async close() { + await new Promise((resolveClose) => server.close(resolveClose)); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + }, + }; +} + +const currentFile = fileURLToPath(import.meta.url); +if (process.argv[1] && resolve(process.argv[1]) === currentFile) { + const provider = await startFakeOidcProvider({ + registration: { + clientId: "fixture-standalone-client", + clientSecret: "fixture-standalone-secret-not-production", + redirectUri: "http://127.0.0.1:8787/api/auth/oidc/callback", + }, + apiToken: "fixture-standalone-api-token-not-production", + }); + process.stdout.write('{"status":"ready"}\n'); + const close = async () => { + await provider.close(); + process.exit(0); + }; + process.once("SIGINT", () => { void close(); }); + process.once("SIGTERM", () => { void close(); }); +} diff --git a/backend/test/fixtures/oidc-state-consumer.mts b/backend/test/fixtures/oidc-state-consumer.mts new file mode 100644 index 00000000..f1eb0ab0 --- /dev/null +++ b/backend/test/fixtures/oidc-state-consumer.mts @@ -0,0 +1,22 @@ +import { createFileAuthSessionStore } from "../../src/auth/session-store.js"; +import { createFixturePosixAuthStorageBridge } from "./posix-auth-storage-bridge.mjs"; + +const root = process.env.THT_TEST_SESSION_ROOT; +const state = process.env.THT_TEST_OIDC_STATE; + +if (!root || !state || !process.send) process.exit(2); + +process.send("ready"); +process.once("message", async (message) => { + if (message !== "consume") process.exit(3); + try { + const record = await createFileAuthSessionStore(root, undefined, { + posixStorageBridge: createFixturePosixAuthStorageBridge(), + }).consumeOidcState(state); + process.send?.({ consumed: record !== undefined }); + process.exit(0); + } catch { + process.send?.({ consumed: false, failed: true }); + process.exit(1); + } +}); diff --git a/backend/test/fixtures/oidc-state-creator.mts b/backend/test/fixtures/oidc-state-creator.mts new file mode 100644 index 00000000..ab3848bf --- /dev/null +++ b/backend/test/fixtures/oidc-state-creator.mts @@ -0,0 +1,46 @@ +import { createFileAuthSessionStore } from "../../src/auth/session-store.js"; +import { createFixturePosixAuthStorageBridge } from "./posix-auth-storage-bridge.mjs"; + +const root = process.env.THT_TEST_SESSION_ROOT; +const attempts = Number(process.env.THT_TEST_OIDC_ATTEMPTS); +const now = new Date(process.env.THT_TEST_OIDC_NOW ?? "invalid"); +const capacity = Number(process.env.THT_TEST_OIDC_CAPACITY); +if (!root || !Number.isSafeInteger(attempts) || attempts < 1 || Number.isNaN(now.getTime()) + || !Number.isSafeInteger(capacity) || capacity < 1) process.exit(2); + +const store = createFileAuthSessionStore(root, undefined, { + oidcStateCapacity: capacity, + posixStorageBridge: createFixturePosixAuthStorageBridge(), +}); +process.send?.("ready"); +process.once("message", async (message) => { + if (message !== "create") process.exit(3); + let created = 0; + try { + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + await store.createOidcState({ + nonce: `n${String(process.pid).padStart(10, "0")}${String(attempt).padStart(32, "0")}`, + codeVerifier: `v${String(process.pid).padStart(10, "0")}${String(attempt).padStart(32, "0")}`, + returnTo: "/", + authConfigRevision: "a".repeat(64), + issuer: "https://issuer.example.test", + browserTransactionDigest: "b".repeat(64), + browserTransactionTransport: "https", + }, now); + created += 1; + } catch (error) { + if (!(error instanceof Error) || error.message !== "auth_oidc_state_capacity") throw error; + } + } + process.send?.({ created }); + process.exit(0); + } catch (error) { + process.send?.({ + created, + failed: true, + error: error instanceof Error ? error.message : "non-error failure", + }); + process.exit(4); + } +}); diff --git a/backend/test/fixtures/posix-auth-storage-bridge.mts b/backend/test/fixtures/posix-auth-storage-bridge.mts new file mode 100644 index 00000000..3dda1545 --- /dev/null +++ b/backend/test/fixtures/posix-auth-storage-bridge.mts @@ -0,0 +1,150 @@ +import { + chmodSync, + existsSync, + linkSync, + lstatSync, + mkdirSync, + opendirSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import type { + WindowsAuthStorageBridge, + WindowsAuthStorageDirectory, +} from "../../src/auth/windows-auth-storage.js"; + +// Test-process fixture only. The production POSIX implementation is the Go auth-storage bridge; +// the Go package covers retained-descriptor adversarial races separately. +export function createFixturePosixAuthStorageBridge(): WindowsAuthStorageBridge { + const ensure = async (root: string): Promise => { + if (!existsSync(root)) { + mkdirSync(root, { mode: 0o700 }); + chmodSync(root, 0o700); + } + for (const name of ["sessions", "oidc"]) { + const path = join(root, name); + if (!existsSync(path)) { + mkdirSync(path, { mode: 0o700 }); + chmodSync(path, 0o700); + } + } + }; + const path = async (root: string, directory: WindowsAuthStorageDirectory, filename: string): Promise => { + await ensure(root); + return join(root, directory, filename); + }; + const names = async (root: string, directory: WindowsAuthStorageDirectory): Promise => { + await ensure(root); + const handle = opendirSync(join(root, directory)); + const result: string[] = []; + try { + for (;;) { + const entry = handle.readSync(); + if (entry === null) break; + result.push(entry.name); + } + } finally { + handle.closeSync(); + } + return result.sort(); + }; + const missing = (error: unknown): boolean => (error as { code?: unknown })?.code === "ENOENT"; + return { + validateRoot: ensure, + ensureLayout: ensure, + readAuthConfig: (value) => readFileSync(value), + readLocalUsers: async (value) => readFileSync(value), + create: async (root, directory, filename, contents) => { + try { + const value = await path(root, directory, filename); + writeFileSync(value, contents, { flag: "wx", mode: 0o600 }); + chmodSync(value, 0o600); + return true; + } catch (error) { + if (missing(error) || (error as { code?: unknown })?.code === "EEXIST") return false; + throw error; + } + }, + read: async (root, directory, filename) => { + try { return readFileSync(await path(root, directory, filename)); } catch (error) { + if (missing(error)) return undefined; + throw error; + } + }, + replace: async (root, directory, filename, contents) => { + const value = await path(root, directory, filename); + const temporary = `${value}.fixture-replacement`; + writeFileSync(temporary, contents, { flag: "wx", mode: 0o600 }); + renameSync(temporary, value); + }, + remove: async (root, directory, filename) => { + try { + unlinkSync(await path(root, directory, filename)); + return true; + } catch (error) { + if (missing(error)) return false; + throw error; + } + }, + list: async (root, directory, maximumEntries = 256) => { + const result = await names(root, directory); + if (result.length > maximumEntries) throw new Error("fixture list overflow"); + return result.map((name) => ({ name, modifiedUnixMs: lstatSync(join(root, directory, name)).mtimeMs })); + }, + listPage: async (root, directory, afterName, maximumEntries) => { + if (directory !== "sessions") throw new Error("fixture directory invalid"); + const selected = (await names(root, directory)).filter((name) => afterName === undefined || name > afterName); + return { + entries: selected.slice(0, maximumEntries).map((name) => ({ + name, modifiedUnixMs: lstatSync(join(root, directory, name)).mtimeMs, + })), + more: selected.length > maximumEntries, + }; + }, + claimConsume: async (root, filename) => { + const source = await path(root, "oidc", filename); + const claim = source.replace(/\.json$/, ".claim"); + try { + linkSync(source, claim); + } catch (error) { + if (missing(error) || (error as { code?: unknown })?.code === "EEXIST") return undefined; + throw error; + } + const contents = readFileSync(source); + unlinkSync(source); + unlinkSync(claim); + return contents; + }, + readClaim: async (root, filename) => { + const source = await path(root, "oidc", filename); + const claim = source.replace(/\.json$/, ".claim"); + try { + const left = lstatSync(source); + const right = lstatSync(claim); + if (left.ino !== right.ino || left.dev !== right.dev || left.nlink !== 2 || right.nlink !== 2) return undefined; + return readFileSync(source); + } catch (error) { + if (missing(error)) return undefined; + throw error; + } + }, + removeClaim: async (root, filename) => { + const source = await path(root, "oidc", filename); + const claim = source.replace(/\.json$/, ".claim"); + try { + const left = lstatSync(source); + const right = lstatSync(claim); + if (left.ino !== right.ino || left.dev !== right.dev || left.nlink !== 2 || right.nlink !== 2) return false; + unlinkSync(source); + unlinkSync(claim); + return true; + } catch (error) { + if (missing(error)) return false; + throw error; + } + }, + }; +} diff --git a/backend/test/fixtures/sessions-scope-all.json b/backend/test/fixtures/sessions-scope-all.json new file mode 100644 index 00000000..267d85af --- /dev/null +++ b/backend/test/fixtures/sessions-scope-all.json @@ -0,0 +1,14 @@ +[ + { + "id": "open-session", + "status": "open", + "archived": false, + "active": false + }, + { + "id": "finished-session", + "status": "finalized", + "archived": false, + "active": false + } +] diff --git a/backend/test/fixtures/windows-auth-storage-real-child.mjs b/backend/test/fixtures/windows-auth-storage-real-child.mjs new file mode 100644 index 00000000..0fe8adc0 --- /dev/null +++ b/backend/test/fixtures/windows-auth-storage-real-child.mjs @@ -0,0 +1,21 @@ +import { appendFileSync, closeSync, writeFileSync } from "node:fs"; + +const [, , mode, marker] = process.argv; + +writeFileSync(marker, `started:${process.pid}\n`); +process.on("exit", () => appendFileSync(marker, "exited\n")); +process.on("SIGTERM", () => { + appendFileSync(marker, "terminated\n"); + if (mode !== "timeout") process.exit(0); +}); + +if (mode === "stdin") { + closeSync(0); + appendFileSync(marker, "stdin-closed\n"); +} else if (mode === "stdout") { + process.stdout.write(Buffer.alloc(64 * 1024 + 1)); +} else if (mode === "stderr") { + process.stderr.write(Buffer.alloc(64 * 1024 + 1)); +} + +setInterval(() => {}, 1_000); diff --git a/backend/test/health.test.ts b/backend/test/health.test.ts index 6c2008ac..1be89fb1 100644 --- a/backend/test/health.test.ts +++ b/backend/test/health.test.ts @@ -2,6 +2,10 @@ import { test, expect } from "vitest"; import { buildApp } from "../src/app.js"; import { loadConfig } from "../src/config.js"; +test("runtime exposes the Node 24 compatibility contract", () => { + expect(Number(process.versions.node.split(".")[0])).toBe(24); +}); + test("GET /health reports process readiness without external services", async () => { const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/tmp/h" })); const res = await app.inject({ method: "GET", url: "/health" }); diff --git a/backend/test/list-models.test.ts b/backend/test/list-models.test.ts index 84699025..3f615a0c 100644 --- a/backend/test/list-models.test.ts +++ b/backend/test/list-models.test.ts @@ -20,6 +20,9 @@ function enabled(...ids: string[]) { return () => ({ ids, warnings: [], source: "/test/settings.json" }); } +const noManagedModels = { readModelsStore: () => undefined }; +const MANAGED_CONFIG_ERROR = "Pi provider/model configuration is invalid"; + test("createPiModelLister returns mapped PiModel[] from get_available_models", async () => { const script = scriptWith([ { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true, extra: "ignored" }, @@ -27,6 +30,7 @@ test("createPiModelLister returns mapped PiModel[] from get_available_models", a ]); try { const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + ...noManagedModels, loadEnabledModels: enabled("zai/glm-5.2", "anthropic/claude-opus-4-8"), spawnFn: () => spawn("node", [FAKE, script]) as any, }); @@ -45,6 +49,7 @@ test("createPiModelLister caches within ttl (spawns once for two calls)", async try { let spawns = 0; const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + ...noManagedModels, loadEnabledModels: enabled("zai/glm-5.2"), spawnFn: () => { spawns++; return spawn("node", [FAKE, script]) as any; }, ttlMs: 10_000, @@ -69,6 +74,7 @@ test("production model-list spawn preserves PATH and passes the portable data ro PI_BIN: "/usr/local/bin/pi", THT_DATA_ROOT: "/data", }), { + ...noManagedModels, loadEnabledModels: enabled("test/unavailable"), spawnFn: (...args: any[]) => { calls.push(args); @@ -101,6 +107,7 @@ test("model-list spawn scrubs ambient provider credentials and generic secret me process.env.CLOUDFLARE_ACCOUNT_ID = "must-not-leak"; try { const lister = createPiModelLister(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), { + ...noManagedModels, loadEnabledModels: enabled("test/unavailable"), spawnFn: (...args: any[]) => { calls.push(args); @@ -135,6 +142,7 @@ test("model listing does not require PI_PROVIDER or read the generic credential" PI_BIN: "/usr/local/bin/pi", THT_MODEL_API_KEY_FILE: "/missing-and-must-not-be-read", }), { + ...noManagedModels, loadEnabledModels: enabled("zai/glm-5.2"), spawnFn: (...args: any[]) => { calls.push(args); @@ -158,6 +166,7 @@ test("returns only enabled available models in enabledModels order", async () => ]); try { const lister = createPiModelLister(loadConfig({}), { + ...noManagedModels, loadEnabledModels: enabled( "zai/glm-5.2", "deepseek/deepseek-v4-flash", @@ -179,6 +188,7 @@ test("empty enabled model scope fails closed without spawning Pi", async () => { let spawns = 0; const warnings: string[] = []; const lister = createPiModelLister(loadConfig({}), { + ...noManagedModels, loadEnabledModels: () => ({ ids: [], warnings: ["scope invalid"] }), warn: (message) => warnings.push(message), spawnFn: () => { spawns += 1; throw new Error("must not spawn"); }, @@ -195,6 +205,7 @@ test("warns and returns empty when enabled identifiers are unavailable", async ( const warnings: string[] = []; try { const lister = createPiModelLister(loadConfig({}), { + ...noManagedModels, loadEnabledModels: enabled("zai/glm-5.2"), warn: (message) => warnings.push(message), spawnFn: () => spawn("node", [FAKE, script]) as any, @@ -205,3 +216,123 @@ test("warns and returns empty when enabled identifiers are unavailable", async ( rmSync(path.dirname(script), { recursive: true, force: true }); } }); + +const executableModelsConfigCases: Array<[string, unknown]> = [ + ["nested provider headers", { + providers: { + selected: { + headers: { Authorization: "!sensitive-header-command /private/header-path" }, + }, + }, + }], + ["provider apiKey", { + providers: { + selected: { apiKey: "!sensitive-api-key-command /private/key-path" }, + }, + }], + ["selected model objects", { + providers: { + selected: { + models: [{ id: "model", name: "!sensitive-model-command /private/model-path" }], + }, + }, + }], + ["selected model overrides", { + providers: { + selected: { + modelOverrides: { + model: { headers: { "X-Override": "!sensitive-override-command /private/override-path" } }, + }, + }, + }, + }], + ["nested arrays", { + providers: { + selected: { + compat: { nested: ["literal", { value: "!sensitive-array-command /private/array-path" }] }, + }, + }, + }], +]; + +// Catches Task 8 model discovery delegating raw managed models.json values to Pi. Pi 0.80.3 +// executes leading-! values at request time, so the complete managed store must be rejected before +// it can become an authoritative source of API choices. +test.each(executableModelsConfigCases)( + "managed models ingestion rejects executable strings in %s", + async (_name, modelsConfig) => { + let spawns = 0; + const lister = createPiModelLister(loadConfig({}), { + loadEnabledModels: enabled("selected/model"), + readModelsStore: () => JSON.stringify(modelsConfig), + spawnFn: () => { + spawns += 1; + throw new Error("unsafe model-list spawn"); + }, + }); + + let caught: unknown; + try { + await lister(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toBe(MANAGED_CONFIG_ERROR); + expect(String(caught)).not.toMatch(/sensitive|private|command|path/i); + expect(spawns).toBe(0); + }, +); + +// Selection-time smoke filtering intentionally ignores unrelated providers, but the full +// installation-owned models.json is invalid at the model-choice ingestion boundary. +test("managed models ingestion rejects an executable string in an unrelated provider", async () => { + let spawns = 0; + const lister = createPiModelLister(loadConfig({}), { + loadEnabledModels: enabled("selected/model"), + readModelsStore: () => JSON.stringify({ + providers: { + selected: { models: [{ id: "model" }] }, + unrelated: { apiKey: "!sensitive-unrelated-command /private/unrelated-path" }, + }, + }), + spawnFn: () => { + spawns += 1; + throw new Error("unsafe model-list spawn"); + }, + }); + + await expect(lister()).rejects.toThrow(MANAGED_CONFIG_ERROR); + expect(spawns).toBe(0); +}); + +test("managed models ingestion revalidates the store before serving a cached choice", async () => { + const script = scriptWith([ + { provider: "selected", id: "model", name: "Selected model", reasoning: true }, + ]); + let managedModels = JSON.stringify({ + providers: { selected: { models: [{ id: "model" }] } }, + }); + let spawns = 0; + try { + const lister = createPiModelLister(loadConfig({}), { + loadEnabledModels: enabled("selected/model"), + readModelsStore: () => managedModels, + spawnFn: () => { + spawns += 1; + return spawn("node", [FAKE, script]) as any; + }, + ttlMs: 10_000, + }); + + await expect(lister()).resolves.toHaveLength(1); + managedModels = JSON.stringify({ + providers: { selected: { headers: { Authorization: "!new-unsafe-value" } } }, + }); + + await expect(lister()).rejects.toThrow(MANAGED_CONFIG_ERROR); + expect(spawns).toBe(1); + } finally { + rmSync(path.dirname(script), { recursive: true, force: true }); + } +}); diff --git a/backend/test/local-registry.test.ts b/backend/test/local-registry.test.ts new file mode 100644 index 00000000..6a7718b5 --- /dev/null +++ b/backend/test/local-registry.test.ts @@ -0,0 +1,304 @@ +import { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + renameSync, + realpathSync, + symlinkSync, + unlinkSync, + utimesSync, + writeFileSync, + linkSync, +} from "node:fs"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +type OwnershipObservation = + | "file-lstat" + | "file-fstat" + | "directory-lstat" + | "directory-fstat"; + +const ownershipOverride = vi.hoisted(() => ({ + observation: undefined as OwnershipObservation | undefined, + uid: undefined as number | undefined, +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const replaceUid = (value: T, uid: number): T => new Proxy(value, { + get(target, property) { + if (property === "uid") return uid; + const member = Reflect.get(target, property, target); + return typeof member === "function" ? member.bind(target) : member; + }, + }); + const maybeReplace = ( + value: T, + source: "lstat" | "fstat", + ): T => { + const kind = value.isDirectory() ? "directory" : "file"; + return ownershipOverride.observation === `${kind}-${source}` && ownershipOverride.uid !== undefined + ? replaceUid(value, ownershipOverride.uid) + : value; + }; + return { + ...actual, + lstatSync(path: import("node:fs").PathLike) { + return maybeReplace(actual.lstatSync(path), "lstat"); + }, + fstatSync(fd: number) { + return maybeReplace(actual.fstatSync(fd), "fstat"); + }, + }; +}); + +import { createLocalUserRegistry } from "../src/auth/local-registry.js"; + +const password = "correct horse battery staple"; +const passwordHash = "$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"; +const adminId = "6ba7b810-9dad-4ed1-80b4-00c04fd430c8"; +const userId = "7ba7b810-9dad-4ed1-80b4-00c04fd430c8"; + +const createdRoots: string[] = []; + +afterEach(() => { + ownershipOverride.observation = undefined; + ownershipOverride.uid = undefined; + for (const root of createdRoots.splice(0)) { + for (const name of ["users.yaml", "users-link.yaml", "users-target.yaml", "replacement.yaml"]) { + const path = join(root, name); + if (existsSync(path) || lstatMaybe(path)) unlinkSync(path); + } + } +}); + +function lstatMaybe(path: string): boolean { + try { + lstatSync(path); + return true; + } catch { + return false; + } +} + +function root(): string { + const path = mkdtempSync(join(realpathSync(tmpdir()), "thothii-local-registry-")); + chmodSync(path, 0o700); + if ((lstatSync(path).mode & 0o7777) !== 0o700) throw new Error("test root is not private"); + createdRoots.push(path); + return path; +} + +function userYaml(options: { + id?: string; + username?: string; + displayName?: string; + enabled?: boolean; + role?: "user" | "admin"; +} = {}): string { + return [ + ` - id: ${options.id ?? adminId}`, + ` username: ${options.username ?? "Admin"}`, + ` displayName: ${options.displayName ?? "Admin"}`, + ` passwordHash: ${passwordHash}`, + ` roles:`, + ` - ${options.role ?? "admin"}`, + ` enabled: ${options.enabled ?? true}`, + ` authRevision: 1`, + ].join("\n") + "\n"; +} + +function registryYaml(users: string): string { + return `version: 1\nusers:\n${users}`; +} + +function writeRegistry(contents: string, file = "users.yaml"): { root: string; path: string } { + const directory = root(); + const path = join(directory, file); + writeFileSync(path, contents, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); + return { root: directory, path }; +} + +async function expectInvalid(operation: Promise, secrets: string[] = []): Promise { + try { + await operation; + throw new Error("operation unexpectedly succeeded"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + expect(message).toBe("local_user_registry_invalid"); + for (const secret of secrets) expect(message).not.toContain(secret); + } +} + +describe("local user registry", () => { + test("reads known fields, performs case-insensitive lookup, and verifies passwords", async () => { + const fixture = writeRegistry(registryYaml(userYaml({ displayName: "Local administrator" }))); + const registry = createLocalUserRegistry(fixture.path); + + await expect(registry.findByUsername("aDmIn")).resolves.toMatchObject({ + id: adminId, + username: "Admin", + normalizedUsername: "admin", + displayName: "Local administrator", + passwordHash, + roles: ["admin"], + enabled: true, + authRevision: 1, + }); + await expect(registry.findBySubject(adminId)).resolves.toMatchObject({ username: "Admin" }); + await expect(registry.verify(await registry.findByUsername("admin"), password)).resolves.toBe(true); + await expect(registry.verify(await registry.findByUsername("admin"), `${password}!`)).resolves.toBe(false); + }); + + test("uses a dummy verification path for unknown and disabled users", async () => { + const fixture = writeRegistry(registryYaml(userYaml() + userYaml({ + id: userId, + username: "operator", + role: "user", + enabled: false, + }))); + const registry = createLocalUserRegistry(fixture.path); + + await expect(registry.verify(undefined, password)).resolves.toBe(false); + await expect(registry.verify(await registry.findByUsername("operator"), password)).resolves.toBe(false); + }); + + test("reports whether a structurally valid registry has an enabled administrator", async () => { + const admin = createLocalUserRegistry(writeRegistry(registryYaml(userYaml())).path); + const usersOnly = createLocalUserRegistry(writeRegistry(registryYaml(userYaml({ role: "user" }))).path); + const disabledAdmin = createLocalUserRegistry(writeRegistry(registryYaml(userYaml({ enabled: false }))).path); + + await expect(admin.hasEnabledAdmin()).resolves.toBe(true); + await expect(usersOnly.hasEnabledAdmin()).resolves.toBe(false); + await expect(disabledAdmin.hasEnabledAdmin()).resolves.toBe(false); + await expectInvalid(usersOnly.findByUsername("admin")); + await expectInvalid(disabledAdmin.findBySubject(adminId)); + }); + + test("rejects a valid registry under a non-private authentication directory", async () => { + const fixture = writeRegistry(registryYaml(userYaml())); + chmodSync(fixture.root, 0o750); + expect(lstatSync(fixture.root).mode & 0o7777).toBe(0o750); + + await expectInvalid(createLocalUserRegistry(fixture.path).findByUsername("admin"), ["admin", passwordHash, fixture.path]); + }); + + test("rejects a valid registry under a symlinked authentication directory", async () => { + const outer = root(); + const realDirectory = join(outer, "real-auth"); + const linkedDirectory = join(outer, "linked-auth"); + mkdirSync(realDirectory, { mode: 0o700 }); + chmodSync(realDirectory, 0o700); + const path = join(realDirectory, "users.yaml"); + writeFileSync(path, registryYaml(userYaml()), { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); + symlinkSync(realDirectory, linkedDirectory); + + await expectInvalid(createLocalUserRegistry(join(linkedDirectory, "users.yaml")).findByUsername("admin"), ["admin", passwordHash]); + }); + + test.each([ + ["duplicate normalized usernames", registryYaml(userYaml() + userYaml({ id: userId, username: "admin" }))], + ["duplicate IDs", registryYaml(userYaml() + userYaml({ username: "operator" }))], + ["unknown YAML fields", `${registryYaml(userYaml())}unexpected: true\n`], + ["duplicate roles", registryYaml(userYaml().replace(" - admin", " - admin\n - admin"))], + ["invalid password hash", registryYaml(userYaml().replace(passwordHash, "not-a-password-hash"))], + ["control character in display name", registryYaml(userYaml().replace("displayName: Admin", 'displayName: "Admin\\tUser"'))], + ])("rejects %s", async (_name, contents) => { + const fixture = writeRegistry(contents); + await expectInvalid(createLocalUserRegistry(fixture.path).findByUsername("admin"), ["admin", passwordHash, fixture.path]); + }); + + test.each(["symlink", "hard link", "mode wider than 0600", "file larger than 1 MiB"])( + "rejects unsafe %s registry metadata", + async (kind) => { + const fixture = writeRegistry(registryYaml(userYaml())); + if (kind === "symlink") { + const target = join(fixture.root, "users-target.yaml"); + renameSync(fixture.path, target); + symlinkSync(target, fixture.path); + } else if (kind === "hard link") { + linkSync(fixture.path, join(fixture.root, "users-link.yaml")); + } else if (kind === "mode wider than 0600") { + chmodSync(fixture.path, 0o640); + } else { + writeFileSync(fixture.path, "#".repeat((1 << 20) + 1), { encoding: "utf8", mode: 0o600 }); + } + await expectInvalid(createLocalUserRegistry(fixture.path).findByUsername("admin"), ["admin", passwordHash, fixture.path]); + }, + ); + + test.runIf(process.platform !== "win32").each([ + "file-lstat", + "file-fstat", + "directory-lstat", + "directory-fstat", + ] as const)("rejects foreign ownership at the %s boundary", async (observation) => { + const fixture = writeRegistry(registryYaml(userYaml())); + const owner = process.geteuid(); + ownershipOverride.observation = observation; + ownershipOverride.uid = owner === 0 ? 1 : owner - 1; + + await expectInvalid( + createLocalUserRegistry(fixture.path).findByUsername("admin"), + ["admin", passwordHash, fixture.path], + ); + }); + + test.runIf(process.platform !== "win32")("fails closed when the effective UID is invalid", async () => { + const fixture = writeRegistry(registryYaml(userYaml())); + const getuid = vi.spyOn(process, "geteuid").mockReturnValue(-1); + try { + await expectInvalid( + createLocalUserRegistry(fixture.path).findByUsername("admin"), + ["admin", passwordHash, fixture.path], + ); + } finally { + getuid.mockRestore(); + } + }); + + test("reloads a same-size atomic replacement with changed metadata", async () => { + const fixture = writeRegistry(registryYaml(userYaml({ displayName: "Admin" }))); + const registry = createLocalUserRegistry(fixture.path); + await expect(registry.findByUsername("admin")).resolves.toMatchObject({ displayName: "Admin" }); + + const replacement = join(fixture.root, "replacement.yaml"); + writeFileSync(replacement, registryYaml(userYaml({ displayName: "Owner" })), { encoding: "utf8", mode: 0o600 }); + chmodSync(replacement, 0o600); + utimesSync(replacement, new Date("2035-01-01T00:00:00Z"), new Date("2035-01-01T00:00:00Z")); + expect(lstatSync(replacement).size).toBe(lstatSync(fixture.path).size); + renameSync(replacement, fixture.path); + + await expect(registry.findByUsername("admin")).resolves.toMatchObject({ displayName: "Owner" }); + }); + + test("routes native Windows users.yaml loading only through the bounded auth-storage bridge", async () => { + const usersPath = "C:\\ProgramData\\ThothII\\auth\\users.yaml"; + const readLocalUsers = vi.fn(async (path: string) => { + expect(path).toBe(usersPath); + return Buffer.from(registryYaml(userYaml({ displayName: "Bridge administrator" })), "utf8"); + }); + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + if (!originalPlatform) throw new Error("platform descriptor unavailable"); + Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); + try { + const registry = createLocalUserRegistry(usersPath, { windowsStorageBridge: { readLocalUsers } } as never); + await expect(registry.findByUsername("ADMIN")).resolves.toMatchObject({ + id: adminId, + displayName: "Bridge administrator", + }); + await expect(registry.hasEnabledAdmin()).resolves.toBe(true); + // Windows reloads from the bridge on every registry observation so an atomic host + // replacement cannot be missed between authorization checks. + expect(readLocalUsers).toHaveBeenCalledTimes(2); + } finally { + Object.defineProperty(process, "platform", originalPlatform); + } + }); +}); diff --git a/backend/test/maintenance-gate.test.ts b/backend/test/maintenance-gate.test.ts new file mode 100644 index 00000000..b086a983 --- /dev/null +++ b/backend/test/maintenance-gate.test.ts @@ -0,0 +1,129 @@ +import { test, expect } from "vitest"; +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { MaintenanceBarrier } from "../src/runtime/maintenance-gate.js"; + +test("activation waits for an in-flight admission lease and rejects later admissions", async () => { + const gate = new MaintenanceBarrier(); + const release = gate.acquire(); + expect(release).toBeTypeOf("function"); + let acknowledged = false; + const activation = gate.activate().then(() => { acknowledged = true; }); + await Promise.resolve(); + expect(acknowledged).toBe(false); + expect(gate.acquire()).toBeUndefined(); + release?.(); + await activation; + expect(acknowledged).toBe(true); + expect(gate.status()).toEqual({ active: true, admissions: 0 }); + gate.deactivate(); + expect(gate.acquire()).toBeTypeOf("function"); +}); + +test("durable activation survives recreation and deactivation removes the marker first", async () => { + const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-gate-")); + const marker = join(directory, "maintenance.json"); + try { + const first = new MaintenanceBarrier(marker); + const release = first.acquire(); + expect(release).toBeTypeOf("function"); + const activation = first.activate(); + expect(existsSync(marker)).toBe(true); + expect(first.acquire()).toBeUndefined(); + expect(first.status()).toEqual({ active: true, admissions: 1 }); + release?.(); + await activation; + expect(first.status()).toEqual({ active: true, admissions: 0 }); + + const recreated = new MaintenanceBarrier(marker); + expect(recreated.status()).toEqual({ active: true, admissions: 0 }); + recreated.deactivate(); + expect(existsSync(marker)).toBe(false); + expect(recreated.status()).toEqual({ active: false, admissions: 0 }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("activation reconciles active state when directory fsync fails after marker rename", async () => { + const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-activate-fsync-")); + const marker = join(directory, "maintenance.json"); + try { + const gate = new MaintenanceBarrier(marker, { + syncDirectory() { throw new Error("injected post-rename fsync failure"); }, + }); + await expect(gate.activate()).rejects.toThrow(/post-rename fsync failure/); + expect(existsSync(marker)).toBe(true); + expect(gate.status()).toEqual({ active: true, admissions: 0, recoveryRequired: true }); + expect(gate.acquire()).toBeUndefined(); + + const recovered = new MaintenanceBarrier(marker); + expect(recovered.status()).toEqual({ active: true, admissions: 0 }); + expect(recovered.acquire()).toBeUndefined(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test.each(["write", "fsync"] as const)( + "activation cleans its marker temp file after a pre-rename %s failure", + async (failure) => { + const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-temp-cleanup-")); + const marker = join(directory, "maintenance.json"); + try { + const gate = new MaintenanceBarrier(marker, { + writeFile(descriptor, contents) { + if (failure === "write") throw new Error("injected marker write failure"); + writeFileSync(descriptor, contents, "utf8"); + }, + syncFile() { + if (failure === "fsync") throw new Error("injected marker fsync failure"); + }, + syncDirectory() {}, + }); + + await expect(gate.activate()).rejects.toThrow(`injected marker ${failure} failure`); + expect(existsSync(marker)).toBe(false); + expect(readdirSync(directory).filter((entry) => entry.includes(".tmp-"))).toEqual([]); + expect(gate.status()).toEqual({ active: false, admissions: 0, recoveryRequired: true }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, +); + +test("deactivation reconciles inactive state when directory fsync fails after marker removal", async () => { + const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-deactivate-fsync-")); + const marker = join(directory, "maintenance.json"); + let failSync = false; + try { + const gate = new MaintenanceBarrier(marker, { + syncDirectory() { + if (failSync) throw new Error("injected post-remove fsync failure"); + }, + }); + await gate.activate(); + failSync = true; + expect(() => gate.deactivate()).toThrow(/post-remove fsync failure/); + expect(existsSync(marker)).toBe(true); + expect(gate.status()).toEqual({ active: true, admissions: 0, recoveryRequired: true }); + expect(gate.acquire()).toBeUndefined(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("status and admission recover from a persistent marker even when memory started inactive", () => { + const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-reconcile-")); + const marker = join(directory, "maintenance.json"); + try { + const gate = new MaintenanceBarrier(marker); + expect(gate.status().active).toBe(false); + writeFileSync(marker, '{"version":1,"active":true}\n', { mode: 0o600 }); + expect(gate.status()).toEqual({ active: true, admissions: 0 }); + expect(gate.acquire()).toBeUndefined(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/backend/test/oidc-client.test.ts b/backend/test/oidc-client.test.ts new file mode 100644 index 00000000..c3da421a --- /dev/null +++ b/backend/test/oidc-client.test.ts @@ -0,0 +1,607 @@ +import { createSign, generateKeyPairSync } from "node:crypto"; +import { expect, test, vi } from "vitest"; +import { + createOidcProtocol, + OidcIssuerMismatchError, + OidcJwksUnavailableError, + OidcDeviceFlowUnavailableError, + OidcProtocolError, + OidcProviderUnavailableError, +} from "../src/auth/oidc-client.js"; + +const issuer = "https://issuer.example.test"; +const clientId = "thothii"; +const callbackUrl = "https://thothii.example.test/api/auth/oidc/callback"; +const verifier = "v".repeat(43); +const nonce = "n".repeat(43); +const state = "s".repeat(43); + +const keys = generateKeyPairSync("rsa", { modulusLength: 2048 }); +const jwk = { ...keys.publicKey.export({ format: "jwk" }), kid: "test-key", use: "sig", alg: "RS256" }; + +function token(claims: Record, invalidSignature = false): string { + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url"); + const input = `${encode({ alg: "RS256", kid: "test-key", typ: "JWT" })}.${encode(claims)}`; + const signer = createSign("RSA-SHA256"); + signer.update(input); + signer.end(); + const signature = signer.sign(keys.privateKey).toString("base64url"); + const corruptedSignature = signature.startsWith("A") ? `B${signature.slice(1)}` : `A${signature.slice(1)}`; + return `${input}.${invalidSignature ? corruptedSignature : signature}`; +} + +function protocol(options: { + claims?: Record; + discoveryIssuer?: string; + invalidSignature?: boolean; + seen?: URL[]; + discoveryResponse?: (init?: RequestInit) => Response | Promise; + tokenResponse?: (init?: RequestInit) => Response | Promise; + deviceResponse?: (init?: RequestInit) => Response | Promise; + jwksResponse?: (init?: RequestInit) => Response | Promise; + httpTimeoutMs?: number; + jwksTimeoutMs?: number; + discoveryMetadata?: Record; +} = {}) { + const now = Math.floor(Date.now() / 1000); + const claims = { + iss: issuer, + sub: "user-123", + aud: clientId, + exp: now + 300, + iat: now, + nonce, + name: "Ada Lovelace", + groups: ["TOT Users", "Unmapped group"], + ...options.claims, + }; + const fetch = async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : typeof input === "string" ? input : input.toString()); + options.seen?.push(url); + if (url.pathname.includes(".well-known/")) { + if (options.discoveryResponse) return await options.discoveryResponse(init); + return Response.json({ + issuer: options.discoveryIssuer ?? issuer, + authorization_endpoint: `${issuer}/authorize`, + token_endpoint: `${issuer}/token`, + jwks_uri: `${issuer}/jwks`, + response_types_supported: ["code"], + grant_types_supported: ["authorization_code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + ...options.discoveryMetadata, + }); + } + if (url.pathname === "/jwks") return options.jwksResponse ? await options.jwksResponse(init) : Response.json({ keys: [jwk] }); + if (url.pathname === "/device") { + if (options.deviceResponse) return await options.deviceResponse(init); + return Response.json({ + device_code: "device-code-must-not-be-persisted", + user_code: "ABCD-EFGH", + verification_uri: `${issuer}/device`, + expires_in: 60, + interval: 1, + }); + } + if (url.pathname === "/token") { + if (options.tokenResponse) return await options.tokenResponse(init); + return Response.json({ + token_type: "Bearer", + access_token: "access-token-must-not-be-persisted", + refresh_token: "refresh-token-must-not-be-persisted", + id_token: token(claims, options.invalidSignature), + }); + } + return new Response(null, { status: 404 }); + }; + const protocolOptions = { + issuer, + clientId, + clientSecret: "client-secret-must-not-be-persisted", + callbackUrl, + scopes: ["openid", "profile"], + groupsClaim: "groups", + fetch, + ...(options.httpTimeoutMs === undefined ? {} : { httpTimeoutMs: options.httpTimeoutMs }), + ...(options.jwksTimeoutMs === undefined ? {} : { jwksTimeoutMs: options.jwksTimeoutMs }), + } as Parameters[0]; + return createOidcProtocol(protocolOptions); +} + +async function callback(subject = protocol()) { + return subject.callback({ + currentUrl: new URL(`${callbackUrl}?code=good&state=${state}`), state, nonce, codeVerifier: verifier, + }); +} + +async function settlesWithin(operation: Promise, timeoutMs = 150): Promise<"resolved" | "rejected" | "timed-out"> { + return await new Promise((resolve) => { + const timeout = setTimeout(() => resolve("timed-out"), timeoutMs); + operation.then( + () => { clearTimeout(timeout); resolve("resolved"); }, + () => { clearTimeout(timeout); resolve("rejected"); }, + ); + }); +} + +test("uses HTTPS discovery, Authorization Code, and PKCE S256 without external network", async () => { + const seen: URL[] = []; + const subject = protocol({ seen }); + const authorization = await subject.authorizationUrl({ state, nonce, codeVerifier: verifier }); + + expect(authorization.origin).toBe(issuer); + expect(authorization.pathname).toBe("/authorize"); + expect(Object.fromEntries(authorization.searchParams)).toMatchObject({ + response_type: "code", client_id: clientId, redirect_uri: callbackUrl, state, nonce, + code_challenge_method: "S256", scope: "openid profile", + }); + expect(authorization.searchParams.get("code_challenge")).not.toBe(verifier); + await expect(callback(subject)).resolves.toEqual({ + issuer, subject: "user-123", displayName: "Ada Lovelace", + groups: ["TOT Users", "Unmapped group"], tokenExpiresAt: expect.any(Date), + }); + expect(seen.map((url) => url.origin)).toEqual([issuer, issuer, issuer]); +}); + +test("uses a validated bounded OIDC device flow and returns only a verified direct-group identity", async () => { + const seen: URL[] = []; + const subject = protocol({ + seen, + discoveryMetadata: { device_authorization_endpoint: `${issuer}/device` }, + }); + const presented: Array<[string, string]> = []; + + await expect(subject.verifyDeviceFlow!(new AbortController().signal, (uri, code) => { + presented.push([uri, code]); + })).resolves.toMatchObject({ + issuer, + subject: "user-123", + groups: ["TOT Users", "Unmapped group"], + }); + + expect(presented).toEqual([[`${issuer}/device`, "ABCD-EFGH"]]); + expect(seen.map((url) => url.pathname)).toEqual([ + "/.well-known/openid-configuration", "/device", "/token", "/jwks", + ]); +}); + +test("refuses device flow when discovery has no safe device authorization endpoint", async () => { + await expect(protocol().verifyDeviceFlow!(new AbortController().signal, () => undefined)) + .rejects.toBeInstanceOf(OidcDeviceFlowUnavailableError); +}); + +test("stops authorization-pending polling at the provider device expiry", async () => { + const caller = new AbortController(); + const timeout = vi.spyOn(AbortSignal, "timeout"); + try { + const tokenResponse = vi.fn(async () => Response.json( + { error: "authorization_pending", error_description: "pending" }, + { status: 400 }, + )); + const subject = protocol({ + discoveryMetadata: { device_authorization_endpoint: `${issuer}/device` }, + deviceResponse: async () => Response.json({ + device_code: "ephemeral-device-code", + user_code: "ABCD-EFGH", + verification_uri: `${issuer}/device`, + expires_in: 0.05, + interval: 0.01, + }), + tokenResponse, + }); + const completion = subject.verifyDeviceFlow!(caller.signal, () => undefined); + + expect(await settlesWithin(completion, 250)).toBe("rejected"); + expect(tokenResponse.mock.calls.length).toBeLessThanOrEqual(5); + expect(timeout).toHaveBeenCalledWith(10 * 60_000); + expect(timeout).toHaveBeenCalledWith(50); + await expect(completion).rejects.toThrow(OidcProtocolError); + } finally { + caller.abort(); + timeout.mockRestore(); + } +}); + +test("combines the caller cancellation with device-flow deadlines before provider requests", async () => { + const seen: URL[] = []; + const caller = new AbortController(); + caller.abort(new DOMException("caller deadline", "AbortError")); + const subject = protocol({ + seen, + discoveryMetadata: { device_authorization_endpoint: `${issuer}/device` }, + }); + + await expect(subject.verifyDeviceFlow!(caller.signal, () => undefined)).rejects.toThrow(OidcProtocolError); + expect(seen).toEqual([]); +}); + +test("rejects a hanging discovery request at the provider transport deadline", async () => { + let aborted = false; + const subject = protocol({ + httpTimeoutMs: 20, + discoveryResponse: (init) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + aborted = true; + reject(new DOMException("aborted", "AbortError")); + }, { once: true }); + }), + }); + + const completion = subject.authorizationUrl({ state, nonce, codeVerifier: verifier }); + expect(await settlesWithin(completion)).toBe("rejected"); + const error = await completion.catch((reason: unknown) => reason); + expect(error).toBeInstanceOf(OidcProviderUnavailableError); + expect((error as Error).message).toBe("oidc_provider_unavailable"); + expect(aborted).toBe(true); +}); + +test("rejects oversized discovery Content-Length before reading or buffering its body", async () => { + let pulls = 0; + let cancelled = false; + const body = new ReadableStream({ + type: "bytes", + pull(controller) { + pulls += 1; + controller.enqueue(new TextEncoder().encode("{}")); + controller.close(); + }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ + discoveryResponse: () => new Response(body, { headers: { "content-length": String(1024 * 1024 + 1) } }), + }); + + await expect(subject.authorizationUrl({ state, nonce, codeVerifier: verifier })).rejects.toThrow(OidcProtocolError); + expect(pulls).toBe(0); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("stops chunked discovery streaming at the provider response limit", async () => { + let pulls = 0; + let cancelled = false; + const body = new ReadableStream({ + type: "bytes", + pull(controller) { + pulls += 1; + if (pulls === 1) controller.enqueue(new Uint8Array(700_000)); + else if (pulls === 2) controller.enqueue(new Uint8Array(400_000)); + else { + controller.enqueue(new Uint8Array([1])); + controller.close(); + } + }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ discoveryResponse: () => new Response(body) }); + + await expect(subject.authorizationUrl({ state, nonce, codeVerifier: verifier })).rejects.toThrow(OidcProtocolError); + expect(pulls).toBe(2); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("rejects a hanging token exchange at the provider transport deadline", async () => { + let aborted = false; + const subject = protocol({ + httpTimeoutMs: 20, + tokenResponse: (init) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + aborted = true; + reject(new DOMException("aborted", "AbortError")); + }, { once: true }); + }), + }); + await subject.authorizationUrl({ state, nonce, codeVerifier: verifier }); + + const completion = callback(subject); + expect(await settlesWithin(completion)).toBe("rejected"); + const error = await completion.catch((reason: unknown) => reason); + expect(error).toBeInstanceOf(OidcProviderUnavailableError); + expect((error as Error).message).toBe("oidc_provider_unavailable"); + expect(aborted).toBe(true); +}); + +test("rejects oversized token Content-Length before reading or buffering its body", async () => { + let pulls = 0; + let cancelled = false; + const body = new ReadableStream({ + type: "bytes", + pull(controller) { + pulls += 1; + controller.enqueue(new TextEncoder().encode("{}")); + controller.close(); + }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ + tokenResponse: () => new Response(body, { headers: { "content-length": String(1024 * 1024 + 1) } }), + }); + await subject.authorizationUrl({ state, nonce, codeVerifier: verifier }); + + await expect(callback(subject)).rejects.toThrow(OidcProtocolError); + expect(pulls).toBe(0); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("stops chunked token streaming at the provider response limit", async () => { + let pulls = 0; + let cancelled = false; + const body = new ReadableStream({ + type: "bytes", + pull(controller) { + pulls += 1; + if (pulls === 1) controller.enqueue(new Uint8Array(700_000)); + else if (pulls === 2) controller.enqueue(new Uint8Array(400_000)); + else { + controller.enqueue(new Uint8Array([1])); + controller.close(); + } + }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ tokenResponse: () => new Response(body) }); + await subject.authorizationUrl({ state, nonce, codeVerifier: verifier }); + + await expect(callback(subject)).rejects.toThrow(OidcProtocolError); + expect(pulls).toBe(2); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("fails promptly and releases a provider response whose cancellation never settles", async () => { + let cancelled = false; + const body = new ReadableStream({ + pull() { return new Promise(() => { /* no body bytes are ever delivered */ }); }, + cancel() { + cancelled = true; + return new Promise(() => { /* cancellation remains advisory */ }); + }, + }); + const subject = protocol({ + discoveryResponse: () => new Response(body, { headers: { "content-length": String(1024 * 1024 + 1) } }), + }); + + const completion = subject.authorizationUrl({ state, nonce, codeVerifier: verifier }); + expect(await settlesWithin(completion)).toBe("rejected"); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("rejects non-HTTPS issuer configuration and a discovery issuer mismatch", async () => { + expect(() => createOidcProtocol({ + issuer: "http://issuer.example.test", clientId, clientSecret: "secret", callbackUrl, + scopes: ["openid"], groupsClaim: "groups", + })).toThrow(OidcProtocolError); + try { + await protocol({ discoveryIssuer: "https://other.example.test" }) + .authorizationUrl({ state, nonce, codeVerifier: verifier }); + throw new Error("issuer mismatch unexpectedly accepted"); + } catch (error) { + expect(error).toBeInstanceOf(OidcIssuerMismatchError); + expect((error as Error).message).toBe("oidc_issuer_mismatch"); + } +}); + +test("enforces the exact shared OIDC client-secret boundary", () => { + expect(() => createOidcProtocol({ + issuer, clientId, clientSecret: "s".repeat(4096), callbackUrl, + scopes: ["openid"], groupsClaim: "groups", + })).not.toThrow(); + expect(() => createOidcProtocol({ + issuer, clientId, clientSecret: "s".repeat(4097), callbackUrl, + scopes: ["openid"], groupsClaim: "groups", + })).toThrow(OidcProtocolError); +}); + +test.each([ + ["authorization", { authorization_endpoint: "http://127.0.0.1/authorize" }], + ["token", { token_endpoint: "http://127.0.0.1/token" }], + ["JWKS", { jwks_uri: "http://127.0.0.1/jwks" }], +])("keeps the discovered %s endpoint HTTPS-only", async (_label, discoveryMetadata) => { + const subject = protocol({ discoveryMetadata }); + await expect(subject.authorizationUrl({ state, nonce, codeVerifier: verifier })) + .rejects.toThrow(OidcProtocolError); +}); + +test.each([ + "http://127.0.0.1:3000/api/auth/oidc/callback", + "http://127.255.255.254/api/auth/oidc/callback", + "http://[::1]:3000/api/auth/oidc/callback", + "https://thothii.example.test/api/auth/oidc/callback", +])("accepts the configured callback URL %s", (configuredCallbackUrl) => { + expect(() => createOidcProtocol({ + issuer, clientId, clientSecret: "secret", callbackUrl: configuredCallbackUrl, + scopes: ["openid"], groupsClaim: "groups", + })).not.toThrow(); +}); + +test.each([ + "http://localhost/api/auth/oidc/callback", + "http://loopback.example.test/api/auth/oidc/callback", + "http://user@127.0.0.1/api/auth/oidc/callback", + "http://127.1/api/auth/oidc/callback", + "http://127.0.0.01/api/auth/oidc/callback", + "http://0177.0.0.1/api/auth/oidc/callback", + "http://0x7f000001/api/auth/oidc/callback", + "http://2130706433/api/auth/oidc/callback", + "http://[::ffff:127.0.0.1]/api/auth/oidc/callback", + "http://128.0.0.1/api/auth/oidc/callback", + "http://192.168.1.1/api/auth/oidc/callback", +])("rejects the non-canonical or non-loopback HTTP callback URL %s", (configuredCallbackUrl) => { + expect(() => createOidcProtocol({ + issuer, clientId, clientSecret: "secret", callbackUrl: configuredCallbackUrl, + scopes: ["openid"], groupsClaim: "groups", + })).toThrow(OidcProtocolError); +}); + +test.each([ + ["state", new URL(`${callbackUrl}?code=good&state=wrong`), {}], + ["nonce", new URL(`${callbackUrl}?code=good&state=${state}`), { nonce: "wrong" }], + ["audience", new URL(`${callbackUrl}?code=good&state=${state}`), { aud: "someone-else" }], + ["issuer", new URL(`${callbackUrl}?code=good&state=${state}`), { iss: "https://other.example.test" }], + ["expiry", new URL(`${callbackUrl}?code=good&state=${state}`), { exp: Math.floor(Date.now() / 1000) - 1 }], + ["subject", new URL(`${callbackUrl}?code=good&state=${state}`), { sub: undefined }], +])("rejects invalid %s claims or callback bindings", async (_label, currentUrl, claims) => { + const subject = protocol({ claims }); + await expect(subject.callback({ currentUrl, state, nonce, codeVerifier: verifier })).rejects.toThrow(OidcProtocolError); +}); + +test("rejects invalid ID-token signatures", async () => { + await expect(callback(protocol({ invalidSignature: true }))).rejects.toThrow(OidcProtocolError); +}); + +test("aborts a hanging JWKS request at the configured timeout", async () => { + let aborted = false; + const subject = protocol({ + jwksTimeoutMs: 20, + jwksResponse: (init) => new Promise((_resolve, reject) => { + const fallback = setTimeout(() => reject(new Error("JWKS fixture was not aborted")), 200); + init?.signal?.addEventListener("abort", () => { + aborted = true; + clearTimeout(fallback); + reject(new DOMException("aborted", "AbortError")); + }, { once: true }); + }), + }); + await expect(callback(subject)).rejects.toThrow(OidcProtocolError); + expect(aborted).toBe(true); +}); + +test("diagnose validates the bounded JWKS endpoint after discovery", async () => { + const subject = protocol({ jwksResponse: () => new Response("upstream JWKS body", { status: 503 }) }); + + await expect(subject.diagnose(new AbortController().signal)).rejects.toBeInstanceOf(OidcJwksUnavailableError); +}); + +test("rejects an oversized JWKS Content-Length before reading the body", async () => { + let pulls = 0; + let cancelled = false; + const body = new ReadableStream({ + type: "bytes", + pull(controller) { + pulls += 1; + controller.enqueue(new TextEncoder().encode("{}")); + controller.close(); + }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ + jwksResponse: () => new Response(body, { headers: { "content-length": String(1024 * 1024 + 1) } }), + }); + await expect(callback(subject)).rejects.toThrow(OidcProtocolError); + expect(pulls).toBe(0); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("cancels and releases a JWKS stream with an invalid Content-Length", async () => { + let cancelled = false; + const body = new ReadableStream({ + pull() { /* remains pending until the response is rejected and cancelled */ }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ + jwksResponse: () => new Response(body, { headers: { "content-length": "not-a-number" } }), + }); + + await expect(callback(subject)).rejects.toThrow(OidcProtocolError); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("cancels and releases a non-success JWKS response body", async () => { + let cancelled = false; + const body = new ReadableStream({ + pull() { /* remains pending until the response is rejected and cancelled */ }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ jwksResponse: () => new Response(body, { status: 503 }) }); + + await expect(callback(subject)).rejects.toThrow(OidcProtocolError); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("rejects promptly and releases the JWKS reader when stream cancellation never settles", async () => { + let cancelled = false; + const body = new ReadableStream({ + pull() { /* a non-success response is rejected before any body read */ }, + cancel() { + cancelled = true; + return new Promise(() => { /* deliberately never settles */ }); + }, + }); + const subject = protocol({ jwksResponse: () => new Response(body, { status: 503 }) }); + const completion = callback(subject).then(() => "resolved" as const, () => "rejected" as const); + const outcome = await Promise.race([ + completion, + new Promise<"timed-out">((resolve) => setTimeout(() => resolve("timed-out"), 100)), + ]); + + expect(outcome).toBe("rejected"); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("preserves the JWKS body timeout when cancellation never settles", async () => { + let cancelled = false; + const body = new ReadableStream({ + pull() { return new Promise(() => { /* body read deliberately hangs */ }); }, + cancel() { + cancelled = true; + return new Promise(() => { /* deliberately never settles */ }); + }, + }); + const subject = protocol({ + jwksTimeoutMs: 20, + jwksResponse: () => new Response(body), + }); + const completion = callback(subject).then(() => "resolved" as const, () => "rejected" as const); + const outcome = await Promise.race([ + completion, + new Promise<"timed-out">((resolve) => setTimeout(() => resolve("timed-out"), 100)), + ]); + + expect(outcome).toBe("rejected"); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test("stops streaming a JWKS response as soon as the byte limit is exceeded", async () => { + let pulls = 0; + let cancelled = false; + const body = new ReadableStream({ + type: "bytes", + pull(controller) { + pulls += 1; + if (pulls === 1) controller.enqueue(new Uint8Array(700_000)); + else if (pulls === 2) controller.enqueue(new Uint8Array(400_000)); + else { + controller.enqueue(new Uint8Array([1])); + controller.close(); + } + }, + cancel() { cancelled = true; }, + }); + const subject = protocol({ jwksResponse: () => new Response(body) }); + await expect(callback(subject)).rejects.toThrow(OidcProtocolError); + expect(pulls).toBe(2); + expect(cancelled).toBe(true); + expect(body.locked).toBe(false); +}); + +test.each([ + ["absent", { groups: undefined }], + ["non-array", { groups: "TOT Users" }], + ["empty array", { groups: [] }], + ["empty", { groups: [""] }], + ["duplicate", { groups: ["TOT Users", "TOT Users"] }], + ["control", { groups: ["TOT\u0000Users"] }], + ["oversized", { groups: ["x".repeat(257)] }], + ["distributed", { _claim_names: { groups: "source" }, _claim_sources: { source: { endpoint: "https://issuer.example.test/claims" } } }], + ["overage", { hasgroups: true }], +])("rejects %s mandatory groups claims", async (_label, claims) => { + await expect(callback(protocol({ claims }))).rejects.toThrow(OidcProtocolError); +}); diff --git a/backend/test/oidc-provider-fixture.test.ts b/backend/test/oidc-provider-fixture.test.ts new file mode 100644 index 00000000..a127fa07 --- /dev/null +++ b/backend/test/oidc-provider-fixture.test.ts @@ -0,0 +1,254 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { request as httpsRequest } from "node:https"; +import { afterEach, describe, expect, test } from "vitest"; +import { startFakeOidcProvider } from "./fixtures/oidc-provider.mjs"; + +const registration = Object.freeze({ + clientId: "fixture-client", + clientSecret: "fixture-client-secret-not-production", + redirectUri: "http://127.0.0.1:8787/api/auth/oidc/callback", +}); +const apiToken = "fixture-api-token-not-production"; +const verifier = "fixture-pkce-verifier-0123456789-abcdefghijklmnopqrstuvwxyz"; +const challenge = createHash("sha256").update(verifier).digest("base64url"); + +let provider; + +afterEach(async () => { + await provider?.close(); + provider = undefined; +}); + +async function start(options = {}) { + provider = await startFakeOidcProvider({ registration, apiToken, ...options }); + return provider; +} + +function exchange(target, options = {}) { + return new Promise((resolve, reject) => { + const body = options.body ?? ""; + const request = httpsRequest(target, { + method: options.method ?? "GET", + ca: readFileSync(provider.caFile), + headers: { + accept: "application/json", + ...(body.length === 0 ? {} : { + "content-length": String(Buffer.byteLength(body)), + "content-type": "application/x-www-form-urlencoded", + }), + ...options.headers, + }, + }, (response) => { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.once("error", reject); + response.once("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + const parsed = text.length === 0 ? {} : JSON.parse(text); + if (parsed && typeof parsed === "object") { + if (Object.hasOwn(parsed, "access_token")) parsed.access_token = "[redacted]"; + if (Object.hasOwn(parsed, "id_token")) parsed.id_token = "[redacted]"; + } + resolve({ + status: response.statusCode ?? 0, + location: response.headers.location, + body: parsed, + }); + }); + }); + request.once("error", reject); + request.end(body); + }); +} + +function form(entries) { + return new URLSearchParams(entries).toString(); +} + +async function authorize(overrides = {}) { + const target = new URL(`${provider.issuer}authorize`); + const values = { + response_type: "code", + client_id: registration.clientId, + redirect_uri: registration.redirectUri, + state: "fixture-state", + nonce: "fixture-nonce", + code_challenge: challenge, + code_challenge_method: "S256", + ...overrides, + }; + for (const [name, value] of Object.entries(values)) target.searchParams.set(name, value); + return exchange(target); +} + +function codeFrom(response) { + return new URL(response.location).searchParams.get("code"); +} + +function tokenBody(code, overrides = {}) { + return form({ + grant_type: "authorization_code", + client_id: registration.clientId, + client_secret: registration.clientSecret, + code, + redirect_uri: registration.redirectUri, + code_verifier: verifier, + ...overrides, + }); +} + +function deviceAuthorizationBody(overrides = {}) { + return form({ + client_id: registration.clientId, + client_secret: registration.clientSecret, + ...overrides, + }); +} + +function deviceTokenBody(deviceCode, overrides = {}) { + return form({ + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + client_id: registration.clientId, + client_secret: registration.clientSecret, + device_code: deviceCode, + ...overrides, + }); +} + +describe("loopback OIDC fixture security contract", () => { + test("advertises only production client_secret_post and rejects missing, wrong, basic, or duplicate client credentials", async () => { + await start(); + const discovery = await exchange(`${provider.issuer}.well-known/openid-configuration`); + expect(discovery.body.token_endpoint_auth_methods_supported).toEqual(["client_secret_post"]); + + const endpoint = `${provider.issuer}token`; + const requests = [ + form({ grant_type: "authorization_code", code: "unknown" }), + form({ grant_type: "authorization_code", code: "unknown", client_id: registration.clientId, client_secret: "wrong" }), + `${form({ grant_type: "authorization_code", code: "unknown", client_id: registration.clientId, client_secret: registration.clientSecret })}&client_secret=duplicate`, + `${form({ grant_type: "authorization_code", code: "unknown", client_id: registration.clientId, client_secret: registration.clientSecret })}&client_id=duplicate`, + form({ grant_type: "authorization_code", code: "unknown", client_id: "wrong", client_secret: registration.clientSecret }), + ]; + for (const body of requests) { + const response = await exchange(endpoint, { method: "POST", body }); + expect(response).toMatchObject({ status: 401, body: { error: "invalid_client" } }); + } + const basic = Buffer.from(`${registration.clientId}:${registration.clientSecret}`).toString("base64"); + await expect(exchange(endpoint, { + method: "POST", + body: form({ grant_type: "authorization_code", code: "unknown" }), + headers: { authorization: `Basic ${basic}` }, + })).resolves.toMatchObject({ status: 401, body: { error: "invalid_client" } }); + + await expect(exchange(endpoint, { + method: "POST", + body: tokenBody("unknown"), + })).resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + }); + + test("requires the exact Authentik bearer token", async () => { + await start(); + const target = `${provider.baseUrl}/api/v3/core/groups/?name=fixture-users`; + await expect(exchange(target)).resolves.toMatchObject({ status: 401 }); + await expect(exchange(target, { headers: { authorization: "Bearer wrong" } })) + .resolves.toMatchObject({ status: 401 }); + await expect(exchange(target, { headers: { authorization: `Bearer ${apiToken}` } })) + .resolves.toMatchObject({ status: 200, body: { results: [{ name: "fixture-users" }] } }); + }); + + test("binds authorization codes to registration, redirect URI, and PKCE and consumes terminal attempts", async () => { + await start(); + await expect(authorize({ client_id: "wrong" })).resolves.toMatchObject({ status: 400 }); + await expect(authorize({ redirect_uri: "http://127.0.0.1:8787/wrong" })).resolves.toMatchObject({ status: 400 }); + + const redirectCode = codeFrom(await authorize()); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", + body: tokenBody(redirectCode, { redirect_uri: "http://127.0.0.1:8787/wrong" }), + })).resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + await expect(exchange(`${provider.issuer}token`, { method: "POST", body: tokenBody(redirectCode) })) + .resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + + const pkceCode = codeFrom(await authorize()); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", + body: tokenBody(pkceCode, { code_verifier: "wrong-verifier" }), + })).resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + await expect(exchange(`${provider.issuer}token`, { method: "POST", body: tokenBody(pkceCode) })) + .resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + + const successfulCode = codeFrom(await authorize()); + await expect(exchange(`${provider.issuer}token`, { method: "POST", body: tokenBody(successfulCode) })) + .resolves.toMatchObject({ status: 200 }); + await expect(exchange(`${provider.issuer}token`, { method: "POST", body: tokenBody(successfulCode) })) + .resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + }); + + test("prunes expired authorization state before enforcing a fail-closed cardinality cap", async () => { + let now = 1_000; + await start({ now: () => now, authorizationStateTtlMs: 1_000, authorizationStateLimit: 1 }); + const first = await authorize(); + expect(first.status).toBe(302); + await expect(authorize({ state: "capacity" })) + .resolves.toMatchObject({ status: 503, body: { error: "temporarily_unavailable" } }); + now += 1_001; + await expect(authorize({ state: "after-expiry" })).resolves.toMatchObject({ status: 302 }); + await expect(exchange(`${provider.issuer}token`, { method: "POST", body: tokenBody(codeFrom(first)) })) + .resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + }); + + test("bounds device state, polling, expiry, and replay", async () => { + let now = 5_000; + await start({ + now: () => now, + deviceStateTtlMs: 1_000, + deviceStateLimit: 1, + devicePendingPolls: 1, + devicePollLimit: 3, + }); + const device = await exchange(`${provider.issuer}device_authorization`, { + method: "POST", body: deviceAuthorizationBody(), + }); + expect(device.status).toBe(200); + await expect(exchange(`${provider.issuer}device_authorization`, { + method: "POST", body: deviceAuthorizationBody(), + })).resolves.toMatchObject({ status: 503, body: { error: "temporarily_unavailable" } }); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", body: deviceTokenBody(device.body.device_code), + })).resolves.toMatchObject({ status: 400, body: { error: "authorization_pending" } }); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", body: deviceTokenBody(device.body.device_code), + })).resolves.toMatchObject({ status: 200 }); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", body: deviceTokenBody(device.body.device_code), + })).resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + + const expired = await exchange(`${provider.issuer}device_authorization`, { + method: "POST", body: deviceAuthorizationBody(), + }); + now += 1_001; + await expect(exchange(`${provider.issuer}device_authorization`, { + method: "POST", body: deviceAuthorizationBody(), + })).resolves.toMatchObject({ status: 200 }); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", body: deviceTokenBody(expired.body.device_code), + })).resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + }); + + test("deletes a device grant when its polling limit is exhausted", async () => { + await start({ devicePendingPolls: 10, devicePollLimit: 2 }); + const device = await exchange(`${provider.issuer}device_authorization`, { + method: "POST", body: deviceAuthorizationBody(), + }); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", body: deviceTokenBody(device.body.device_code), + })).resolves.toMatchObject({ status: 400, body: { error: "authorization_pending" } }); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", body: deviceTokenBody(device.body.device_code), + })).resolves.toMatchObject({ status: 400, body: { error: "expired_token" } }); + await expect(exchange(`${provider.issuer}token`, { + method: "POST", body: deviceTokenBody(device.body.device_code), + })).resolves.toMatchObject({ status: 400, body: { error: "invalid_grant" } }); + }); +}); diff --git a/backend/test/pi-management.test.ts b/backend/test/pi-management.test.ts new file mode 100644 index 00000000..452deae9 --- /dev/null +++ b/backend/test/pi-management.test.ts @@ -0,0 +1,335 @@ +import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test, vi } from "vitest"; +import { loadConfig } from "../src/config.js"; +import { + PiManagementError, + createPiManagement, + type PiExecFile, +} from "../src/pi/management.js"; + +function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-management-")), "settings.json")) { + return loadConfig({ + THT_HARNESS_DIR: "../harness", + SETTINGS_FILE: settingsFile, + PI_BIN: "/usr/local/bin/pi", + PI_MANAGEMENT_TIMEOUT_MS: "750", + }); +} + +const supportedModels = [ + { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }, + { provider: "deepseek", id: "deepseek-v4", name: "DeepSeek V4", reasoning: true }, +]; + +function successfulExec(calls: Array<{ command: string; args: string[]; timeout: number }>): PiExecFile { + return async (command, args, options) => { + calls.push({ command, args, timeout: options.timeout }); + return { stdout: "pi 0.80.3\n", stderr: "" }; + }; +} + +// Catches a Pi executable that emits unexpected text or is invoked through a shell, which could +// turn a version display into a command-injection or information-disclosure surface. +test("status parses only a Pi version from a fixed execFile argument array", async () => { + const calls: Array<{ command: string; args: string[]; timeout: number }> = []; + const service = createPiManagement(configFor(), { + execute: successfulExec(calls), + listModels: async () => supportedModels, + readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }), + credentialStatus: () => "missing", + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + await expect(service.status()).resolves.toEqual({ + version: "0.80.3", + ready: true, + credentials: "missing", + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-05T10:00:00.000Z", + }); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ command: "/usr/local/bin/pi", args: ["--version"] }); + expect(calls[0].timeout).toBeGreaterThan(0); + expect(calls[0].timeout).toBeLessThanOrEqual(750); +}); + +// Catches credential presence being inferred from smoke success/failure or exposing any +// credential material instead of the installation's explicit sanitized presence state. +test.each(["present", "missing"] as const)( + "status reports configured-provider credentials only as %s", + async (credentials) => { + const checkedProviders: Array = []; + const service = createPiManagement(configFor(), { + execute: successfulExec([]), + listModels: async () => supportedModels, + readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }), + credentialStatus: (provider) => { + checkedProviders.push(provider); + return credentials; + }, + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + const status = await service.status(); + expect(status).toEqual({ + version: "0.80.3", + ready: true, + credentials, + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-05T10:00:00.000Z", + }); + expect(checkedProviders).toEqual(["zai"]); + expect(JSON.stringify(status)).not.toMatch(/api.?key|token|password|secret/i); + }, +); + +// Catches an options response that leaks provider metadata or lets callers choose model IDs that +// Pi did not explicitly enable for this installation. +test("options expose only closed provider, model, and reasoning choices", async () => { + const service = createPiManagement(configFor(), { + execute: successfulExec([]), + listModels: async () => supportedModels, + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + await expect(service.options()).resolves.toEqual({ + providers: ["zai", "deepseek"], + models: [ + { provider: "zai", id: "glm-5.2" }, + { provider: "deepseek", id: "deepseek-v4" }, + ], + reasoning: ["low", "medium", "high"], + checkedAt: "2026-08-05T10:00:00.000Z", + }); +}); + +// Catches raw managed models.json validation details being collapsed into an ambiguous model-list +// failure or escaping through the Pi Management options API. +test("options report invalid managed model configuration with a stable sanitized error", async () => { + const service = createPiManagement(configFor(), { + execute: successfulExec([]), + listModels: async () => { + throw Object.assign( + new Error("!sensitive-command /private/models.json raw-secret"), + { code: "PI_MANAGED_CONFIG_INVALID" }, + ); + }, + }); + + let caught: unknown; + try { + await service.options(); + } catch (error) { + caught = error; + } + expect(caught).toMatchObject({ + code: "pi_management_unavailable", + message: "Pi provider/model configuration is invalid", + }); + expect(String(caught)).not.toMatch(/sensitive|private|models\.json|secret/i); +}); + +// Catches configuration writes that accept whitespace, unknown choices, or extra free-form fields +// before reaching the durable installation settings file. +test("config rejects invalid free-form values before writing settings", async () => { + const directory = mkdtempSync(join(tmpdir(), "tht-pi-management-invalid-")); + try { + let writes = 0; + const service = createPiManagement(configFor(join(directory, "settings.json")), { + execute: successfulExec([]), + listModels: async () => supportedModels, + readSettings: () => ({}), + saveSettings: () => { writes += 1; return {}; }, + }); + + await expect(service.configure({ + provider: "zai ", model: "glm-5.2", reasoning: "medium", unexpected: "value", + } as any)).rejects.toMatchObject({ code: "pi_management_invalid_config" }); + expect(writes).toBe(0); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +// Catches a non-atomic implementation that can leave partial settings or temporary files after a +// normal installation-default update. +test("config validates closed choices and atomically persists non-secret defaults", async () => { + const directory = mkdtempSync(join(tmpdir(), "tht-pi-management-write-")); + const settingsFile = join(directory, "settings.json"); + try { + const service = createPiManagement(configFor(settingsFile), { + execute: successfulExec([]), + listModels: async () => supportedModels, + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + await expect(service.configure({ + provider: "zai", model: "glm-5.2", reasoning: "high", + })).resolves.toEqual({ + provider: "zai", model: "glm-5.2", reasoning: "high", updatedAt: "2026-08-05T10:00:00.000Z", + }); + expect(JSON.parse(readFileSync(settingsFile, "utf8"))).toEqual({ + provider: "zai", model: "glm-5.2", thinking: "high", + }); + expect(readdirSync(directory)).toEqual(["settings.json"]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +// Catches a hung Pi smoke check that leaves an operator waiting indefinitely or returns raw child +// diagnostics containing provider credentials. +test("smoke uses the configured timeout and reports a sanitized timeout", async () => { + const calls: Array<{ command: string; args: string[]; timeout: number }> = []; + const service = createPiManagement(configFor(), { + execute: async (command, args, options) => { + calls.push({ command, args, timeout: options.timeout }); + throw Object.assign(new Error("provider token=raw-provider-token"), { code: "ETIMEDOUT" }); + }, + listModels: async () => supportedModels, + readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }), + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + await expect(service.test()).resolves.toEqual({ + ready: false, + message: "Pi smoke check timed out", + checkedAt: "2026-08-05T10:00:00.000Z", + }); + expect(calls).toEqual([{ command: "/usr/local/bin/pi", args: ["--version"], timeout: 750 }]); +}); + +// Catches a smoke endpoint that validates only the Pi binary/model catalogue and never makes a +// request through the configured provider and model. +test("smoke exercises the configured provider and model", async () => { + const providerChecks: unknown[] = []; + const service = createPiManagement(configFor(), { + execute: successfulExec([]), + listModels: async () => supportedModels, + readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }), + smokeProvider: async (request) => { providerChecks.push(request); }, + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + await expect(service.test()).resolves.toEqual({ + ready: true, + checkedAt: "2026-08-05T10:00:00.000Z", + }); + expect(providerChecks).toEqual([{ + provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: expect.any(Number), + }]); +}); + +// Catches expired provider credentials being treated as ready or raw provider diagnostics being +// reflected through the management API. +test("smoke fails closed and sanitizes configured-provider authentication errors", async () => { + const service = createPiManagement(configFor(), { + execute: successfulExec([]), + listModels: async () => supportedModels, + readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }), + smokeProvider: async () => { + throw new Error('401 {"token":"raw-expired-token","output":"raw-provider-output"}'); + }, + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + const result = await service.test(); + expect(result).toEqual({ + ready: false, + message: "Pi provider smoke check failed", + checkedAt: "2026-08-05T10:00:00.000Z", + }); + expect(JSON.stringify(result)).not.toMatch(/raw-expired-token|raw-provider-output/); +}); + +// Catches selected auth/models validation failures being downgraded to a generic provider error +// or exposing the rejected command, path, or secret through POST /pi-management/test. +test("smoke reports invalid managed provider configuration with a stable sanitized error", async () => { + const service = createPiManagement(configFor(), { + execute: successfulExec([]), + listModels: async () => supportedModels, + readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }), + smokeProvider: async () => { + throw Object.assign( + new Error("!sensitive-command /private/models.json raw-secret"), + { code: "PI_MANAGED_CONFIG_INVALID" }, + ); + }, + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + const result = await service.test(); + expect(result).toEqual({ + ready: false, + message: "Pi provider/model configuration is invalid", + checkedAt: "2026-08-05T10:00:00.000Z", + }); + expect(JSON.stringify(result)).not.toMatch(/sensitive|private|models\.json|secret/i); +}); + +// Catches separate per-phase timeouts that allow a later provider turn to exceed the one +// end-to-end Pi Management smoke budget. +test("smoke applies one deadline across version and a hung provider turn", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-05T10:00:00.000Z")); + try { + const providerTimeouts: number[] = []; + const service = createPiManagement(configFor(), { + execute: async () => await new Promise((resolve) => setTimeout( + () => resolve({ stdout: "pi 0.80.3\n", stderr: "" }), + 500, + )), + listModels: async () => supportedModels, + readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }), + smokeProvider: async ({ timeoutMs }) => { + providerTimeouts.push(timeoutMs); + await new Promise(() => {}); + }, + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + let settled = false; + const pending = service.test().finally(() => { settled = true; }); + await vi.advanceTimersByTimeAsync(500); + expect(providerTimeouts).toEqual([250]); + await vi.advanceTimersByTimeAsync(249); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toEqual({ + ready: false, + message: "Pi smoke check timed out", + checkedAt: "2026-08-05T10:00:00.000Z", + }); + } finally { + vi.useRealTimers(); + } +}); + +// Catches an unbounded diagnostics endpoint or one that returns bearer tokens and connection +// passwords captured in Pi output. +test("logs keep only the latest 200 redacted lines", async () => { + const source = Array.from({ length: 205 }, (_, index) => `line-${index + 1}`); + source[203] = "Authorization: Bearer raw-bearer-token"; + source[204] = "database_url=postgres://thoth:raw-db-password@example.invalid/db"; + source[202] = '{"token":"raw-json-secret","password":"raw-json-password"}'; + source[201] = "THT_MODEL_API_KEY=raw-env-secret"; + const service = createPiManagement(configFor(), { + execute: successfulExec([]), + listModels: async () => supportedModels, + readLogs: () => source.join("\n"), + now: () => new Date("2026-08-05T10:00:00.000Z"), + }); + + const logs = await service.logs(); + expect(logs.checkedAt).toBe("2026-08-05T10:00:00.000Z"); + expect(logs.lines).toHaveLength(200); + expect(logs.lines[0]).toBe("line-6"); + expect(logs.lines.join("\n")).not.toContain("raw-bearer-token"); + expect(logs.lines.join("\n")).not.toContain("raw-db-password"); + expect(logs.lines.join("\n")).not.toContain("raw-json-secret"); + expect(logs.lines.join("\n")).not.toContain("raw-json-password"); + expect(logs.lines.join("\n")).not.toContain("raw-env-secret"); + expect(logs.lines.join("\n")).toContain("[REDACTED]"); +}); diff --git a/backend/test/pi-process-manager.test.ts b/backend/test/pi-process-manager.test.ts index 27c4b71a..47fc5104 100644 --- a/backend/test/pi-process-manager.test.ts +++ b/backend/test/pi-process-manager.test.ts @@ -3,14 +3,36 @@ import { spawn } from "node:child_process"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { EventEmitter } from "node:events"; -import { chmodSync, writeFileSync } from "node:fs"; +import { + chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; import { PiProcessManager } from "../src/pi/pi-process-manager.js"; import { loadConfig } from "../src/config.js"; +import { + PI_MANAGED_CONFIG_ERROR_MESSAGE, + validateDeclarativePiConfig, +} from "../src/pi/managed-config.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const FAKE = path.resolve(__dirname, "../../harness/tests/fake_pi/fake_pi_rpc.mjs"); const SCRIPT = path.resolve(__dirname, "../../harness/tests/fake_pi/scripts/f1_disambiguation.json"); +const SAFE_AUTH = '{"deepseek":{"type":"api_key","key":"safe-token"}}\n'; +const SAFE_MODELS = [ + "{", + ' "providers": {', + ' "local-qwen": {"baseUrl":"http://model.invalid/v1","models":[{"id":"qwen"}]}', + " }", + "}", + "", +].join("\n"); + +function writeSafeAgentConfig(agentDir: string): void { + writeFileSync(path.join(agentDir, "auth.json"), SAFE_AUTH, { mode: 0o600 }); + writeFileSync(path.join(agentDir, "models.json"), SAFE_MODELS, { mode: 0o600 }); +} + test("spawnFor avvia un runtime e il bridge emette il widget F1", async () => { const cfg = loadConfig({ THT_HARNESS_DIR: "../harness" }); const mgr = new PiProcessManager(cfg, { spawnFn: () => spawn("node", [FAKE, SCRIPT]) as any }); @@ -102,8 +124,8 @@ test("teardownForPrincipal stops only runtimes owned by that user", () => { bobChild.kill = vi.fn(); const children = [aliceChild, bobChild]; const mgr = new PiProcessManager(loadConfig({}), { spawnFn: () => children.shift() as any }); - const alice = { issuer: "portal", subject: "alice", isAdmin: false }; - const bob = { issuer: "portal", subject: "bob", isAdmin: false }; + const alice = { issuer: "portal", subject: "alice", roles: ["user"] as const, permissions: ["session.use"] as const, isAdmin: false }; + const bob = { issuer: "portal", subject: "bob", roles: ["user"] as const, permissions: ["session.use"] as const, isAdmin: false }; mgr.createFor("alice-session", { principal: alice }); mgr.createFor("bob-session", { principal: bob }); @@ -123,7 +145,7 @@ test("createFor keeps at most one runtime for the same user", () => { secondChild.kill = vi.fn(); const children = [firstChild, secondChild]; const mgr = new PiProcessManager(loadConfig({}), { spawnFn: () => children.shift() as any }); - const principal = { issuer: "portal", subject: "alice", isAdmin: false }; + const principal = { issuer: "portal", subject: "alice", roles: ["user"] as const, permissions: ["session.use"] as const, isAdmin: false }; mgr.createFor("first", { principal }); const second = mgr.createFor("second", { principal }); @@ -146,16 +168,175 @@ function recordingChild() { return ch; } +test("Pi receives the leased workspace runtime config and releases it on direct teardown", () => { + const child = recordingChild(); + let spawnEnv: NodeJS.ProcessEnv | undefined; + const release = vi.fn(); + const mgr = new PiProcessManager(loadConfig({}), { + spawnFn: (_command, _args, options) => { + spawnEnv = options.env; + return child as any; + }, + }); + + mgr.createFor("canonical-runtime", { + runtimeConfig: { path: "/trusted/runtime-uuid.yaml", release }, + } as any); + expect(spawnEnv?.THT_CONFIG).toBe("/trusted/runtime-uuid.yaml"); + + // The child deliberately emits neither exit nor close. Ownership cleanup must not depend on it. + mgr.teardown("canonical-runtime"); + expect(release).toHaveBeenCalledOnce(); +}); + +test("a close-only child event releases its temporary Pi agent snapshot", () => { + const child = recordingChild(); + let snapshotDir: string | undefined; + const mgr = new PiProcessManager(loadConfig({}), { + spawnFn: (_command, _args, options) => { + snapshotDir = options.env.PI_CODING_AGENT_DIR; + return child as any; + }, + }); + + mgr.createFor("close-only-snapshot", {}); + expect(snapshotDir).toBeTruthy(); + expect(existsSync(snapshotDir!)).toBe(true); + + child.emit("close", 0); + expect(existsSync(snapshotDir!)).toBe(false); + mgr.teardown("close-only-snapshot"); +}); + +test.each([ + ["new", "auth.json", '{"deepseek":{"key":"!runtime-auth-command runtime-secret /private/runtime-auth"}}\n'], + ["new", "models.json", '{"providers":{"local-qwen":{"headers":["!runtime-model-command runtime-secret /private/runtime-model"]}}}\n'], + ["resume", "auth.json", '{"deepseek":{"key":"!resume-auth-command runtime-secret /private/resume-auth"}}\n'], + ["resume", "models.json", '{"providers":{"local-qwen":{"models":[{"apiKey":"!resume-model-command runtime-secret /private/resume-model"}]}}}\n'], +] as const)( + "%s runtime rejects post-admission executable %s before auth resolution or child spawn", + async (mode, changedFile, unsafeRaw) => { + const root = mkdtempSync(path.join(tmpdir(), "tht-runtime-managed-config-")); + const agentDir = path.join(root, "agent"); + mkdirSync(agentDir, { mode: 0o700 }); + writeSafeAgentConfig(agentDir); + vi.stubEnv("PI_CODING_AGENT_DIR", agentDir); + let authResolutions = 0; + let spawns = 0; + const mgr = new PiProcessManager(loadConfig({}), { + authProviders: () => { authResolutions += 1; return new Set(); }, + spawnFn: () => { spawns += 1; throw new Error("SPAWN_BOUNDARY_REACHED"); }, + }); + + try { + // Admission/model validation succeeded while the mounted files were still safe, and the + // session was then persisted. The operator-controlled mount changes before runtime start. + validateDeclarativePiConfig(readFileSync(path.join(agentDir, "auth.json"), "utf8")); + validateDeclarativePiConfig(readFileSync(path.join(agentDir, "models.json"), "utf8")); + writeFileSync(path.join(root, "session-created"), `${mode}\n`); + writeFileSync(path.join(agentDir, changedFile), unsafeRaw, { mode: 0o600 }); + + let failure: unknown; + try { + if (mode === "new") { + mgr.createFor("post-admission-new", { provider: "local-qwen" }); + } else { + await mgr.spawnFor("post-admission-resume", { + provider: "local-qwen", mode: "resume", + }); + } + } catch (error) { + failure = error; + } + const message = failure instanceof Error ? failure.message : String(failure); + + expect({ message, authResolutions, spawns, runtimes: mgr.count() }).toEqual({ + message: PI_MANAGED_CONFIG_ERROR_MESSAGE, + authResolutions: 0, + spawns: 0, + runtimes: 0, + }); + expect(message).not.toMatch(/runtime-secret|\/private\/|runtime-(?:auth|model)-command|resume-(?:auth|model)-command/); + } finally { + mgr.teardown("post-admission-new"); + mgr.teardown("post-admission-resume"); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); + } + }, +); + +test("runtime Pi consumes exact validated auth/models snapshots and keeps persistent agent resources", async () => { + const root = mkdtempSync(path.join(tmpdir(), "tht-runtime-agent-snapshot-")); + const agentDir = path.join(root, "agent"); + const sessionsDir = path.join(agentDir, "sessions"); + const extensionDir = path.join(agentDir, "extensions"); + mkdirSync(sessionsDir, { recursive: true, mode: 0o700 }); + mkdirSync(extensionDir, { recursive: true, mode: 0o700 }); + writeSafeAgentConfig(agentDir); + const settings = '{"quietStartup":true}\n'; + writeFileSync(path.join(agentDir, "settings.json"), settings, { mode: 0o600 }); + writeFileSync(path.join(extensionDir, "runtime-extension.js"), "export default {};\n"); + vi.stubEnv("PI_CODING_AGENT_DIR", agentDir); + vi.stubEnv("PI_CODING_AGENT_SESSION_DIR", ""); + const child = recordingChild(); + let spawnEnv: NodeJS.ProcessEnv | undefined; + const mgr = new PiProcessManager(loadConfig({}), { + spawnFn: (_command, _args, options) => { + spawnEnv = options.env; + // This mutation happens after validation but before the child can open either source file. + writeFileSync(path.join(agentDir, "auth.json"), '{"deepseek":{"key":"!late-auth-command"}}\n'); + writeFileSync(path.join(agentDir, "models.json"), '{"providers":{"late":{"apiKey":"!late-model-command"}}}\n'); + return child as any; + }, + }); + let snapshotDir: string | undefined; + let childExited = false; + + try { + await mgr.spawnFor("snapshot-session", { provider: "local-qwen" }); + snapshotDir = spawnEnv?.PI_CODING_AGENT_DIR; + + expect(snapshotDir).toBeTruthy(); + expect(snapshotDir).not.toBe(agentDir); + expect(readFileSync(path.join(snapshotDir!, "auth.json"), "utf8")).toBe(SAFE_AUTH); + expect(readFileSync(path.join(snapshotDir!, "models.json"), "utf8")).toBe(SAFE_MODELS); + expect(readFileSync(path.join(snapshotDir!, "settings.json"), "utf8")).toBe(settings); + expect(readFileSync(path.join(snapshotDir!, "extensions", "runtime-extension.js"), "utf8")) + .toBe("export default {};\n"); + expect(spawnEnv?.PI_CODING_AGENT_SESSION_DIR).toBe(sessionsDir); + + mgr.teardown("snapshot-session"); + child.emit("exit", 0); + childExited = true; + expect(existsSync(snapshotDir!)).toBe(false); + } finally { + mgr.teardown("snapshot-session"); + if (!childExited) child.emit("exit", 0); + vi.unstubAllEnvs(); + if (snapshotDir && snapshotDir !== agentDir) rmSync(snapshotDir, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true }); + } +}); + test("createFor kills a spawned child when post-spawn initialization throws", () => { const child = recordingChild(); child.kill = vi.fn(); child.stdout = { on: () => { throw new Error("READER_INIT_SENTINEL"); }, }; - const mgr = new PiProcessManager(loadConfig({}), { spawnFn: () => child as any }); + let snapshotDir: string | undefined; + const mgr = new PiProcessManager(loadConfig({}), { + spawnFn: (_command, _args, options) => { + snapshotDir = options.env.PI_CODING_AGENT_DIR; + return child as any; + }, + }); expect(() => mgr.createFor("broken-init", {})).toThrow("READER_INIT_SENTINEL"); + expect(snapshotDir).toBeTruthy(); + expect(existsSync(snapshotDir!)).toBe(false); expect(child.kill).toHaveBeenCalledOnce(); expect(mgr.get("broken-init")).toBeUndefined(); expect(mgr.count()).toBe(0); diff --git a/backend/test/pi-provider-smoke.test.ts b/backend/test/pi-provider-smoke.test.ts new file mode 100644 index 00000000..b96499cc --- /dev/null +++ b/backend/test/pi-provider-smoke.test.ts @@ -0,0 +1,394 @@ +import { EventEmitter } from "node:events"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { loadConfig } from "../src/config.js"; +import { createPiProviderSmoke } from "../src/pi/provider-smoke.js"; + +afterEach(() => vi.unstubAllEnvs()); + +function rpcChild(onCommand: (command: any, emit: (message: unknown) => void) => void) { + const child: any = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = { resume: vi.fn() }; + child.kill = vi.fn(); + const emit = (message: unknown) => queueMicrotask(() => { + child.stdout.emit("data", `${JSON.stringify(message)}\n`); + }); + child.stdin = { + write: (data: unknown) => { + onCommand(JSON.parse(String(data)), emit); + return true; + }, + }; + return child; +} + +function successfulProviderChild() { + return rpcChild((command, emit) => { + if (command.type === "set_model" || command.type === "set_thinking_level") { + emit({ type: "response", id: command.id, success: true }); + } + if (command.type === "prompt") { + emit({ + type: "message_end", + message: { role: "assistant", stopReason: "stop", content: "must-not-be-returned" }, + }); + emit({ + type: "agent_end", + messages: [{ role: "assistant", stopReason: "stop", content: "must-not-be-returned" }], + }); + } + }); +} + +const MANAGED_CONFIG_ERROR = "Pi provider/model configuration is invalid"; + +// Catches an isolated smoke agent that copies auth.json but drops the selected custom +// provider/model from models.json, causing set_model to fail before the real request. +test("provider smoke reaches the selected custom provider from an isolated models.json", async () => { + let isolatedAgentDir: string | undefined; + let providerRequests = 0; + const child = rpcChild((command, emit) => { + if (command.type === "set_model") { + const models = JSON.parse(readFileSync(`${isolatedAgentDir}/models.json`, "utf8")); + const selectedProvider = models.providers?.[command.provider]; + const selectedModel = selectedProvider?.models?.find( + (candidate: { id?: unknown }) => candidate.id === command.modelId, + ); + emit({ type: "response", id: command.id, success: Boolean(selectedModel) }); + } + if (command.type === "set_thinking_level") { + emit({ type: "response", id: command.id, success: true }); + } + if (command.type === "prompt") { + providerRequests++; + emit({ + type: "message_end", + message: { role: "assistant", stopReason: "stop", content: "must-not-be-returned" }, + }); + emit({ + type: "agent_end", + messages: [{ role: "assistant", stopReason: "stop", content: "must-not-be-returned" }], + }); + } + }); + const smoke = createPiProviderSmoke(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), { + spawnFn: (_command, _args, options) => { + isolatedAgentDir = options.env.PI_CODING_AGENT_DIR; + expect(readdirSync(isolatedAgentDir)).toEqual(["auth.json", "models.json"]); + expect(JSON.parse(readFileSync(`${isolatedAgentDir}/models.json`, "utf8"))).toEqual({ + providers: { + "custom-openai": { + baseUrl: "https://selected.invalid/v1", + apiKey: "$CUSTOM_OPENAI_API_KEY", + api: "openai-completions", + headers: { "X-Literal-Bang": "$!literal-value" }, + models: [{ id: "selected-model", name: "Selected model", reasoning: true }], + }, + }, + }); + expect(JSON.parse(readFileSync(`${isolatedAgentDir}/auth.json`, "utf8"))).toEqual({ + "custom-openai": { type: "api_key", key: "${CUSTOM_OPENAI_API_KEY}" }, + }); + return child; + }, + authProviders: () => new Set(["custom-openai"]), + readAuthStore: () => JSON.stringify({ + "custom-openai": { type: "api_key", key: "${CUSTOM_OPENAI_API_KEY}" }, + unrelated: { type: "api_key", key: "!must-not-run-or-enter-isolated-context" }, + }), + readModelsStore: () => JSON.stringify({ + providers: { + "custom-openai": { + baseUrl: "https://selected.invalid/v1", + apiKey: "$CUSTOM_OPENAI_API_KEY", + api: "openai-completions", + headers: { "X-Literal-Bang": "$!literal-value" }, + models: [ + { id: "selected-model", name: "Selected model", reasoning: true }, + { id: "unrelated-model", name: "Must not enter isolated context" }, + ], + }, + unrelated: { + baseUrl: "https://unrelated.invalid/v1", + api: "openai-completions", + apiKey: "!must-not-run", + models: [{ id: "unrelated-model" }], + }, + }, + }), + }); + + await expect(smoke({ + provider: "custom-openai", model: "selected-model", reasoning: "medium", timeoutMs: 750, + })).resolves.toBeUndefined(); + expect(providerRequests).toBe(1); + expect(child.kill).toHaveBeenCalledOnce(); + expect(isolatedAgentDir && existsSync(dirname(isolatedAgentDir))).toBe(false); +}); + +const selectedExecutableConfigCases: Array<{ + name: string; + selectedAuth?: unknown; + selectedProvider: Record; +}> = [ + { + name: "selected auth credential", + selectedAuth: { + type: "api_key", + key: "!sensitive-credential-command /private/credential-path", + }, + selectedProvider: {}, + }, + { + name: "selected provider apiKey", + selectedProvider: { + apiKey: "!sensitive-api-key-command /private/key-path", + }, + }, + { + name: "nested selected-provider headers", + selectedProvider: { + headers: { Authorization: "!sensitive-header-command /private/header-path" }, + }, + }, + { + name: "selected model object", + selectedProvider: { + models: [{ + id: "selected-model", + name: "!sensitive-model-command /private/model-path", + }], + }, + }, + { + name: "selected model override", + selectedProvider: { + modelOverrides: { + "selected-model": { + headers: { "X-Override": "!sensitive-override-command /private/override-path" }, + }, + }, + }, + }, + { + name: "array nested in selected provider configuration", + selectedProvider: { + compat: { + nested: ["literal", { value: "!sensitive-array-command /private/array-path" }], + }, + }, + }, +]; + +// Catches a defense that validates only known top-level fields or waits until after the isolated +// Pi process starts. Every selected value crossing into auth.json/models.json must be declarative. +test.each(selectedExecutableConfigCases)( + "provider smoke rejects executable config in $name before isolated Pi spawn", + async ({ selectedAuth, selectedProvider }) => { + const child = successfulProviderChild(); + const spawnFn = vi.fn(() => child); + const smoke = createPiProviderSmoke(loadConfig({}), { + spawnFn, + authProviders: () => new Set(["custom-openai"]), + readAuthStore: () => JSON.stringify({ + "custom-openai": selectedAuth ?? { type: "api_key", key: "test-only" }, + unrelated: { type: "api_key", key: "!must-not-contaminate-selected-provider" }, + }), + readModelsStore: () => JSON.stringify({ + providers: { + "custom-openai": { + baseUrl: "https://selected.invalid/v1", + api: "openai-completions", + models: [{ id: "selected-model", name: "Selected model" }], + ...selectedProvider, + }, + unrelated: { + apiKey: "!must-not-contaminate-selected-provider", + models: [{ id: "unrelated-model" }], + }, + }, + }), + }); + + let caught: unknown; + try { + await smoke({ + provider: "custom-openai", model: "selected-model", reasoning: "medium", timeoutMs: 750, + }); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toBe(MANAGED_CONFIG_ERROR); + expect(String(caught)).not.toMatch(/sensitive|private|command|path/i); + expect(spawnFn).not.toHaveBeenCalled(); + }, +); + +// Catches a provider smoke process that runs from the trusted harness or leaves Pi tools, +// extensions, skills, context files, templates, themes, or session persistence enabled. +test("provider smoke makes one configured request from an isolated no-capability Pi process", async () => { + vi.stubEnv("THT_DATA_ROOT", "/mounted-workflow-state"); + vi.stubEnv("THT_SESSION", "mounted-session-id"); + vi.stubEnv("THT_AUTHOR", "mounted-author"); + vi.stubEnv("THT_CONFIG", "/mounted-workflow-state/config.yaml"); + vi.stubEnv("PI_CODING_AGENT_DIR", "/home/thoth/.pi/agent"); + vi.stubEnv("PI_CODING_AGENT_SESSION_DIR", "/mounted-session-state"); + const commands: any[] = []; + const spawns: any[][] = []; + const child = rpcChild((command, emit) => { + commands.push(command); + if (command.type === "set_model" || command.type === "set_thinking_level") { + emit({ type: "response", id: command.id, success: true }); + } + if (command.type === "prompt") { + emit({ + type: "message_end", + message: { role: "assistant", stopReason: "stop", content: "raw-provider-output" }, + }); + emit({ + type: "agent_end", + messages: [{ role: "assistant", stopReason: "stop", content: "raw-provider-output" }], + }); + } + }); + const smoke = createPiProviderSmoke(loadConfig({ + THT_HARNESS_DIR: "/app/harness", + PI_BIN: "/usr/local/bin/pi", + THT_DATA_ROOT: "/mounted-workflow-state", + }), { + spawnFn: (...args) => { + spawns.push(args); + expect(args[2].cwd).not.toBe("/app/harness"); + expect(readdirSync(args[2].cwd)).toEqual([]); + expect(args[2].env.PI_CODING_AGENT_DIR).not.toBe("/home/thoth/.pi/agent"); + expect(readdirSync(args[2].env.PI_CODING_AGENT_DIR)).toEqual(["auth.json"]); + expect(JSON.parse(readFileSync(`${args[2].env.PI_CODING_AGENT_DIR}/auth.json`, "utf8"))) + .toEqual({ zai: { type: "api_key", key: "test-only" } }); + return child; + }, + authProviders: () => new Set(["zai"]), + readAuthStore: () => JSON.stringify({ + zai: { type: "api_key", key: "test-only" }, + deepseek: { type: "api_key", key: "must-not-enter-isolated-context" }, + }), + readModelsStore: () => undefined, + }); + + await expect(smoke({ + provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750, + })).resolves.toBeUndefined(); + expect(spawns).toHaveLength(1); + expect(spawns[0][0]).toBe("/usr/local/bin/pi"); + expect(spawns[0][1]).toEqual([ + "--mode", "rpc", + "--no-session", + "--no-tools", + "--no-extensions", + "--no-skills", + "--no-prompt-templates", + "--no-themes", + "--no-context-files", + "--no-approve", + ]); + expect(spawns[0][2].env).not.toHaveProperty("THT_DATA_ROOT"); + expect(spawns[0][2].env).not.toHaveProperty("THT_SESSION"); + expect(spawns[0][2].env).not.toHaveProperty("THT_AUTHOR"); + expect(spawns[0][2].env).not.toHaveProperty("THT_CONFIG"); + expect(spawns[0][2].env).not.toHaveProperty("PI_CODING_AGENT_SESSION_DIR"); + expect(existsSync(dirname(spawns[0][2].cwd))).toBe(false); + expect(commands.map(({ id: _id, ...command }) => command)).toEqual([ + { type: "set_model", provider: "zai", modelId: "glm-5.2" }, + { type: "set_thinking_level", level: "medium" }, + { type: "prompt", message: expect.stringMatching(/health check/i) }, + ]); + expect(child.kill).toHaveBeenCalledOnce(); +}); + +const unexpectedToolEvents = [ + { + name: "streamed tool call", + event: { + type: "message_update", + assistantMessageEvent: { type: "toolcall_start", contentIndex: 0 }, + }, + }, + { + name: "tool execution", + event: { type: "tool_execution_start", toolCallId: "tool-1", toolName: "read" }, + }, + { + name: "completed message tool call", + event: { + type: "message_end", + message: { role: "assistant", stopReason: "toolUse", content: [{ type: "toolCall" }] }, + }, + }, + { + name: "turn tool result", + event: { type: "turn_end", toolResults: [{ role: "toolResult" }] }, + }, +]; + +// Catches Pi/provider regressions that surface a tool capability despite the fixed no-tools argv; +// accepting agent_end after any such event could hide a mounted-state read or mutation. +test.each(unexpectedToolEvents)("provider smoke fails closed on an unexpected $name event", async ({ event }) => { + const child = rpcChild((command, emit) => { + if (command.type === "set_model" || command.type === "set_thinking_level") { + emit({ type: "response", id: command.id, success: true }); + } + if (command.type === "prompt") { + emit(event); + emit({ type: "agent_end", messages: [] }); + } + }); + const smoke = createPiProviderSmoke(loadConfig({}), { + spawnFn: () => child, + authProviders: () => new Set(["zai"]), + readAuthStore: () => '{"zai":{"type":"api_key","key":"test-only"}}', + readModelsStore: () => undefined, + }); + + await expect(smoke({ + provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750, + })).rejects.toThrow("Pi provider smoke check failed"); + expect(child.kill).toHaveBeenCalledOnce(); +}); + +// Catches provider errors that are accepted as a successful health check or returned with raw +// credential/output diagnostics. +test("provider smoke rejects a failed model turn with a stable non-secret error", async () => { + const child = rpcChild((command, emit) => { + if (command.type === "set_model" || command.type === "set_thinking_level") { + emit({ type: "response", id: command.id, success: true }); + } + if (command.type === "prompt") { + emit({ + type: "message_end", + message: { + role: "assistant", stopReason: "error", + errorMessage: '401 {"token":"raw-provider-secret"}', + }, + }); + emit({ type: "agent_end", messages: [] }); + } + }); + const smoke = createPiProviderSmoke(loadConfig({}), { + spawnFn: () => child, + authProviders: () => new Set(["zai"]), + readAuthStore: () => '{"zai":{"type":"api_key","key":"test-only"}}', + readModelsStore: () => undefined, + }); + + let caught: unknown; + try { + await smoke({ provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750 }); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toBe("Pi provider smoke check failed"); + expect(String(caught)).not.toContain("raw-provider-secret"); +}); diff --git a/backend/test/pi-spawn-args.test.ts b/backend/test/pi-spawn-args.test.ts index 45ed3621..15e3c4c6 100644 --- a/backend/test/pi-spawn-args.test.ts +++ b/backend/test/pi-spawn-args.test.ts @@ -36,18 +36,22 @@ test("production spawnFn launches `pi --mode rpc` with no --approve (pi 0.73 dro test("Pi child replaces stale principal env and omits absent display names", async () => { const saved = Object.fromEntries([ "THT_PRINCIPAL_ISSUER", "THT_PRINCIPAL_SUBJECT", "THT_PRINCIPAL_DISPLAY_NAME", "THT_PRINCIPAL_IS_ADMIN", + "THT_PRINCIPAL_PERMISSIONS", ].map((key) => [key, process.env[key]])); Object.assign(process.env, { THT_PRINCIPAL_ISSUER: "stale", THT_PRINCIPAL_SUBJECT: "stale", THT_PRINCIPAL_DISPLAY_NAME: "stale", - THT_PRINCIPAL_IS_ADMIN: "true", + THT_PRINCIPAL_IS_ADMIN: "true", THT_PRINCIPAL_PERMISSIONS: "pi.manage", }); try { (nodeSpawn as any).mockClear(); const mgr = new PiProcessManager(loadConfig({})); - await mgr.spawnFor("s-principal", { principal: { issuer: "portal", subject: "42", isAdmin: false } }); + await mgr.spawnFor("s-principal", { + principal: { issuer: "portal", subject: "42", roles: ["user"], permissions: ["session.use"], isAdmin: false }, + }); const env = (nodeSpawn as any).mock.calls[0][2].env; expect(env).toMatchObject({ THT_PRINCIPAL_ISSUER: "portal", THT_PRINCIPAL_SUBJECT: "42", THT_PRINCIPAL_IS_ADMIN: "false", + THT_PRINCIPAL_PERMISSIONS: "session.use", }); expect(env).not.toHaveProperty("THT_PRINCIPAL_DISPLAY_NAME"); mgr.teardown("s-principal"); diff --git a/backend/test/provider-credentials.test.ts b/backend/test/provider-credentials.test.ts index ff7414ae..a3b06cb9 100644 --- a/backend/test/provider-credentials.test.ts +++ b/backend/test/provider-credentials.test.ts @@ -5,6 +5,7 @@ import { PI_0803_CREDENTIAL_ENV_NAMES, buildPiChildEnv, canonicalPiProvider, + piProviderCredentialStatus, } from "../src/pi/provider-credentials.js"; test("canonical provider aliases resolve to packaged Pi 0.80.3 IDs", () => { @@ -130,6 +131,74 @@ test("local-qwen is an explicit local provider and needs no generic key", () => expect(env).not.toHaveProperty("THT_MODEL_API_KEY_FILE"); }); +test("credential status reports only present or missing without treating local providers as credentialed", () => { + expect(piProviderCredentialStatus({ + provider: "deepseek", + authProviders: new Set(["deepseek"]), + resolveCredentialValue: () => "must-not-be-returned", + })).toBe("present"); + expect(piProviderCredentialStatus({ provider: "deepseek" })).toBe("missing"); + expect(piProviderCredentialStatus({ + provider: "local-qwen", + resolveCredentialValue: () => "must-not-be-returned", + })).toBe("missing"); +}); + +// Catches the generic managed secret being read before providers that self-authenticate or need +// no credential have been classified. +test("credential status never resolves the generic secret for auth-store or local providers", () => { + let secretReads = 0; + const unreadableSecret = () => { + secretReads += 1; + throw new Error("unrelated generic secret is unreadable"); + }; + + expect(piProviderCredentialStatus({ + provider: "deepseek", + authProviders: new Set(["deepseek"]), + resolveCredentialValue: unreadableSecret, + })).toBe("present"); + expect(piProviderCredentialStatus({ + provider: "local-qwen", + resolveCredentialValue: unreadableSecret, + })).toBe("missing"); + expect(secretReads).toBe(0); +}); + +// Catches generic hosted providers skipping their managed-secret source or treating an absent or +// unreadable source as credentialed. +test("credential status resolves the generic secret only for providers that require it", () => { + let presentReads = 0; + expect(piProviderCredentialStatus({ + provider: "openai", + resolveCredentialValue: () => { + presentReads += 1; + return "managed-openai-key"; + }, + })).toBe("present"); + expect(presentReads).toBe(1); + + let missingReads = 0; + expect(piProviderCredentialStatus({ + provider: "openai", + resolveCredentialValue: () => { + missingReads += 1; + return undefined; + }, + })).toBe("missing"); + expect(missingReads).toBe(1); + + let unreadableReads = 0; + expect(piProviderCredentialStatus({ + provider: "openai", + resolveCredentialValue: () => { + unreadableReads += 1; + throw new Error("generic secret is unreadable"); + }, + })).toBe("missing"); + expect(unreadableReads).toBe(1); +}); + test("bundle value is injected without exposing bundle metadata to Pi", () => { const env = buildPiChildEnv({ ambient: { diff --git a/backend/test/qdrant-collection.test.ts b/backend/test/qdrant-collection.test.ts new file mode 100644 index 00000000..917423f4 --- /dev/null +++ b/backend/test/qdrant-collection.test.ts @@ -0,0 +1,88 @@ +import { expect, test } from "vitest"; +import { QDRANT_REQUIRED_INDEXES, reconcileCollection } from "../src/workspaces/qdrant-collection.js"; + +function fakeRequest(info: any | undefined, { create = true, index = true } = {}) { + let current = info; + let created = false; + return async (url: string, init?: any) => { + if (init?.method === "PUT" && /\/index$/.test(url)) { + if (!index) return { status: 409, ok: false, json: async () => ({}) } as any; + // Simulate the index being created: the collection becomes fully compatible. + current = compatible(); + return { status: 200, ok: true, json: async () => ({}) } as any; + } + if (init?.method === "PUT") { + if (!create) return { status: 409, ok: false, json: async () => ({}) } as any; + created = true; + current = compatible(); + return { status: 200, ok: true, json: async () => ({}) } as any; + } + if (current === undefined) return { status: 404, ok: false, json: async () => ({}) } as any; + return { status: 200, ok: true, json: async () => ({ result: current }) } as any; + }; +} + +const payloadSchema = Object.fromEntries(QDRANT_REQUIRED_INDEXES.map((f) => [f, { data_type: "keyword" }])); +const compatible = (size = 1024, distance = "Cosine", schema = payloadSchema) => ({ + config: { params: { vectors: { size, distance } } }, + payload_schema: schema, +}); + +test("self-heal creates a missing compatible collection", async () => { + const r = await reconcileCollection({ + baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine", + mode: "self_heal", request: fakeRequest(undefined), + }); + expect(r.ok).toBe(true); +}); + +test("require_existing refuses a missing collection", async () => { + const r = await reconcileCollection({ + baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine", + mode: "require_existing", request: fakeRequest(undefined), + }); + expect(r).toEqual({ ok: false, code: "semantic_index_incompatible" }); +}); + +test("self-heal adds missing keyword indexes", async () => { + const schema = { ...payloadSchema }; + delete schema["workspace_revision"]; + const r = await reconcileCollection({ + baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine", + mode: "self_heal", request: fakeRequest(compatible(1024, "Cosine", schema)), + }); + expect(r).toMatchObject({ ok: true, state: "repaired" }); +}); + +test("refuses incompatible dimensions or distance without mutating", async () => { + for (const info of [compatible(768, "Cosine"), compatible(1024, "Dot")]) { + const r = await reconcileCollection({ + baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine", + mode: "self_heal", request: fakeRequest(info), + }); + expect(r).toEqual({ ok: false, code: "semantic_index_incompatible" }); + } +}); + +test("self-heal creates with the Qdrant-valid distance enum", async () => { + let createdBody: any; + const base = fakeRequest(undefined); + const request = async (url: string, init?: any) => { + if (init?.method === "PUT" && !/\/index$/.test(url)) createdBody = JSON.parse(String(init.body)); + return base(url, init); + }; + const r = await reconcileCollection({ + baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine", + mode: "self_heal", request, + }); + expect(r.ok).toBe(true); + expect(createdBody.vectors.distance).toBe("Cosine"); +}); + +test("ready compatible collection passes", async () => { + const r = await reconcileCollection({ + baseUrl: "http://qdrant:6333", collection: "c", dimensions: 1024, distance: "cosine", + mode: "require_existing", request: fakeRequest(compatible()), + }); + expect(r).toMatchObject({ ok: true, state: "ready" }); +}); diff --git a/backend/test/readiness-manager.test.ts b/backend/test/readiness-manager.test.ts index be0cb039..5eb22fcc 100644 --- a/backend/test/readiness-manager.test.ts +++ b/backend/test/readiness-manager.test.ts @@ -1,6 +1,21 @@ import { expect, test } from "vitest"; import { ReadinessManager } from "../src/runtime/readiness-manager.js"; +const workspace = { + workspace: { schema_version: 3, id: "psd", name: "PSD", language: "it" }, + dwh: { + engine: "postgres", database: "warehouse", schema: "public", + supported_transports: ["postgres_direct"], + }, + semantic_index: { + vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" }, + embedding: { + provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024, + }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, +} as const; + function deferred() { let resolve!: (value: T) => void; const promise = new Promise((r) => { resolve = r; }); @@ -60,3 +75,31 @@ test("readiness does not cache failed results", async () => { await expect(readiness.ensure("psd")).resolves.toMatchObject({ ok: true }); expect(calls).toBe(2); }); + +test("readiness checks Qdrant before Ollama and skips Ollama on semantic incompatibility", async () => { + let ollamaCalls = 0; + const tht = { + qdrantEnsure: async () => ({ ok: false, code: "semantic_index_incompatible" }), + ollamaEnsure: async () => { ollamaCalls += 1; return { ok: true }; }, + } as any; + const readiness = new ReadinessManager(tht, 60); + + await expect(readiness.ensure("/registry/psd.yaml", undefined, workspace as any)).resolves.toEqual({ + ok: false, + code: "semantic_index_incompatible", + }); + expect(ollamaCalls).toBe(0); +}); + +test("readiness returns a sanitized activation code when a semantic probe throws", async () => { + const tht = { + qdrantEnsure: async () => { throw new Error("dial http://qdrant:6333/private"); }, + ollamaEnsure: async () => ({ ok: true }), + } as any; + const readiness = new ReadinessManager(tht, 60); + + await expect(readiness.ensure("/registry/psd.yaml", undefined, workspace as any)).resolves.toEqual({ + ok: false, + code: "workspace_not_activatable", + }); +}); diff --git a/backend/test/registry-annotations.test.ts b/backend/test/registry-annotations.test.ts new file mode 100644 index 00000000..094d3152 --- /dev/null +++ b/backend/test/registry-annotations.test.ts @@ -0,0 +1,135 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { WorkspaceRegistry } from "../src/workspaces/registry.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +const validYaml = `workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`; + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Registry Annotations Test", + gitAuthorEmail: "registry-annotations@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +type AnnotationsLayout = "absent" | "valid" | "malformed" | "dir"; + +async function fixture(layout: AnnotationsLayout): Promise<{ root: string; remote: string; commit: string }> { + const root = mkdtempSync(join(tmpdir(), "thoth-registry-annotations-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Registry Annotations Test"]); + await git(source, ["config", "user.email", "registry-annotations@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), + "schema_version: 1\nworkspaces:\n - id: psd-clinical\n name: Policlinico San Donato\n"); + mkdirSync(join(source, "psd-clinical", "schema"), { recursive: true }); + writeFileSync(join(source, "psd-clinical", "workspace.yaml"), validYaml); + const annotationsPath = join(source, "psd-clinical", "schema", "annotations.yaml"); + if (layout === "valid") writeFileSync(annotationsPath, "tables: {}\n"); + if (layout === "malformed") writeFileSync(annotationsPath, "tables: [not, a, mapping]\n"); + if (layout === "dir") { + mkdirSync(annotationsPath, { recursive: true }); + writeFileSync(join(annotationsPath, "child.txt"), "nested\n"); + } + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + return { root, remote, commit }; +} + +test("activation accepts a valid curated annotation blob", async () => { + const fixtureValue = await fixture("valid"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).resolves.toMatchObject({ degraded: false }); +}); + +test("activation accepts an absent annotation blob", async () => { + const fixtureValue = await fixture("absent"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).resolves.toMatchObject({ degraded: false }); +}); + +test("activation rejects malformed annotations", async () => { + const fixtureValue = await fixture("malformed"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("activation rejects a tree at the annotations path", async () => { + const fixtureValue = await fixture("dir"); + const registry = new WorkspaceRegistry(config(join(fixtureValue.root, "registry"), fixtureValue.remote)); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("activation syncs the curated annotations to the revision root when a data root is set", async () => { + const fixtureValue = await fixture("valid"); + const dataRoot = mkdtempSync(join(tmpdir(), "thoth-registry-annotations-data-")); + temporaryRoots.push(dataRoot); + const registry = new WorkspaceRegistry({ + ...config(join(fixtureValue.root, "registry"), fixtureValue.remote), + dataRoot, + }); + + await registry.bootstrap(); + + const annotationsPath = join(dataRoot, "sessions", "psd-clinical", "revisions", fixtureValue.commit, "artifacts", "mschema", "annotations.yaml"); + const manifestPath = join(dataRoot, "sessions", "psd-clinical", "revisions", fixtureValue.commit, "artifacts", "mschema", "annotations.ownership.json"); + expect(readFileSync(annotationsPath, "utf8")).toBe("tables: {}\n"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + expect(manifest).toMatchObject({ workspace: "psd-clinical", commit: fixtureValue.commit }); +}); diff --git a/backend/test/registry-evidence.test.ts b/backend/test/registry-evidence.test.ts new file mode 100644 index 00000000..66e17e19 --- /dev/null +++ b/backend/test/registry-evidence.test.ts @@ -0,0 +1,115 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { WorkspaceRegistry } from "../src/workspaces/registry.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +const descriptor = `workspace: + schema_version: 3 + id: research + name: Research + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: research + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +evidence: + source: + type: filesystem + uri: research/evidence +`; + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Evidence Registry Test", + gitAuthorEmail: "evidence-registry@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +async function fixture(): Promise<{ root: string; remote: string; commit: string }> { + const root = mkdtempSync(join(tmpdir(), "thoth-registry-evidence-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Evidence Registry Test"]); + await git(source, ["config", "user.email", "evidence-registry@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces:\n - id: research\n name: Research\n"); + mkdirSync(join(source, "research", "evidence"), { recursive: true }); + writeFileSync(join(source, "research", "workspace.yaml"), descriptor); + writeFileSync(join(source, "research", "evidence", "guide.md"), "# guide\n"); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + return { root, remote, commit }; +} + +test("activation materializes filesystem Evidence and chains its manifest into snapshot.json", async () => { + const fixtureValue = await fixture(); + const registryRoot = join(fixtureValue.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, fixtureValue.remote)); + + await registry.bootstrap(); + + const evidence = join(registryRoot, "snapshots", fixtureValue.commit, "research", "evidence"); + const manifestPath = join(registryRoot, "snapshots", fixtureValue.commit, "research", "evidence.manifest.json"); + expect(readFileSync(join(evidence, "guide.md"), "utf8")).toBe("# guide\n"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + expect(manifest).toMatchObject({ schemaVersion: 1, workspace: "research", commit: fixtureValue.commit, entryCount: 1 }); + const snapshotManifest = JSON.parse(readFileSync(join(registryRoot, "snapshots", fixtureValue.commit, "snapshot.json"), "utf8")); + expect(snapshotManifest.files["research/evidence.manifest.json"]).toMatch(/^[0-9a-f]{64}$/); + + // Re-activation verifies the existing materialized root. + await expect(registry.bootstrap()).resolves.toMatchObject({ degraded: false }); +}); + +test("activation fails closed when the materialized Evidence manifest is tampered", async () => { + const fixtureValue = await fixture(); + const registryRoot = join(fixtureValue.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, fixtureValue.remote)); + await registry.bootstrap(); + + const manifestPath = join(registryRoot, "snapshots", fixtureValue.commit, "research", "evidence.manifest.json"); + writeFileSync(manifestPath, `${JSON.stringify({ schemaVersion: 1, workspace: "research", commit: fixtureValue.commit, tree: "0".repeat(40), entryCount: 0, totalBytes: 0, files: {} })}\n`); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); diff --git a/backend/test/routes-pi-management.test.ts b/backend/test/routes-pi-management.test.ts new file mode 100644 index 00000000..d1d98a9a --- /dev/null +++ b/backend/test/routes-pi-management.test.ts @@ -0,0 +1,177 @@ +import { expect, test, vi } from "vitest"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import type { PiManagementService } from "../src/pi/management.js"; +import { piManagementRoutes } from "../src/routes/pi-management.js"; + +void piManagementRoutes; + +function fakeService(): PiManagementService { + return { + status: vi.fn(async () => ({ + version: "0.80.3", ready: true, + credentials: "present", + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-05T10:00:00.000Z", + })), + options: vi.fn(async () => ({ + providers: ["zai"], models: [{ provider: "zai", id: "glm-5.2" }], + reasoning: ["low", "medium", "high"], checkedAt: "2026-08-05T10:00:00.000Z", + })), + configure: vi.fn(async (value) => ({ ...value, updatedAt: "2026-08-05T10:00:00.000Z" })), + test: vi.fn(async () => ({ ready: true, checkedAt: "2026-08-05T10:00:00.000Z" })), + logs: vi.fn(async () => ({ lines: ["Pi smoke check succeeded"], checkedAt: "2026-08-05T10:00:00.000Z" })), + }; +} + +function appWith(service: PiManagementService, env: Record = {}) { + return buildApp(loadConfig({ THT_HARNESS_DIR: "../harness", ...env }), { + thtRunner: {} as any, + piManagement: service, + }); +} + +const adminHeaders = { + "x-thoth-principal-issuer": "portal", + "x-thoth-principal-subject": "operator", + "x-thoth-is-admin": "1", +}; + +const exposedServerEnv = { + AUTH_MODE: "upstream", + THOTH_PUBLIC_EXPOSURE: "true", + THT_SESSION_STORAGE: "postgres", + THT_SESSION_DB_HOST: "db.example.invalid", + THT_SESSION_DB_NAME: "thoth_sessions", + THT_SESSION_RUNTIME_USER: "thoth_runtime", + THT_SESSION_RUNTIME_PASSWORD_FILE: "/run/secrets/session-password", + THT_SESSION_DB_SSLMODE: "verify-full", + THT_SESSION_DB_SSLROOTCERT: "/run/secrets/session-ca.pem", +}; + +// Catches a server deployment that lets an ordinary authenticated user inspect or mutate +// installation-wide Pi configuration without the trusted upstream admin claim. +test("exposed upstream deployments reject Pi Management without a trusted admin identity", async () => { + const service = fakeService(); + const app = appWith(service, exposedServerEnv); + try { + const response = await app.inject({ + method: "GET", url: "/pi-management/status", + headers: { ...adminHeaders, "x-thoth-is-admin": "0" }, + }); + + expect(response.statusCode).toBe(403); + expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); + expect(service.status).not.toHaveBeenCalled(); + } finally { + await app.close(); + } +}); + +test("Pi Management test requires pi.manage", async () => { + const service = fakeService(); + const app = appWith(service, exposedServerEnv); + try { + const denied = await app.inject({ + method: "POST", url: "/pi-management/test", + headers: { ...adminHeaders, "x-thoth-is-admin": "0" }, + }); + const allowed = await app.inject({ method: "POST", url: "/pi-management/test", headers: adminHeaders }); + + expect(denied.statusCode).toBe(403); + expect(denied.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); + expect(allowed.statusCode).toBe(200); + } finally { + await app.close(); + } +}); + +// Catches an accidental privilege regression that blocks safe loopback-only installations or +// returns fields beyond the sanctioned Pi Management status contract. +test("loopback-only AUTH_MODE=none may read the sanitized Pi status", async () => { + const app = appWith(fakeService()); + try { + const response = await app.inject({ method: "GET", url: "/pi-management/status" }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual({ + version: "0.80.3", ready: true, + credentials: "present", + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-05T10:00:00.000Z", + }); + } finally { + await app.close(); + } +}); + +// A local implicit administrator has pi.manage, but a browser origin still cannot borrow that +// authority to mutate local configuration or trigger provider work. +test("loopback-only management rejects cross-origin writes for its local administrator", async () => { + const service = fakeService(); + const app = appWith(service); + try { + const configured = await app.inject({ + method: "PUT", url: "/pi-management/config", + headers: { host: "127.0.0.1:8080", origin: "https://evil.example" }, + payload: { provider: "zai", model: "glm-5.2", reasoning: "high" }, + }); + const smoke = await app.inject({ + method: "POST", url: "/pi-management/test", + headers: { host: "127.0.0.1:8080", origin: "https://evil.example" }, + }); + + expect(configured.statusCode).toBe(403); + expect(smoke.statusCode).toBe(403); + expect(service.configure).not.toHaveBeenCalled(); + expect(service.test).not.toHaveBeenCalled(); + } finally { + await app.close(); + } +}); + +// Catches an origin guard that also blocks the same-origin Docker frontend or non-browser local +// lifecycle clients that do not send Origin. +test("loopback-only management preserves same-origin frontend and origin-less local writes", async () => { + const service = fakeService(); + const app = appWith(service); + try { + const sameOrigin = await app.inject({ + method: "PUT", url: "/pi-management/config", + headers: { host: "127.0.0.1:8080", origin: "http://127.0.0.1:8080" }, + payload: { provider: "zai", model: "glm-5.2", reasoning: "high" }, + }); + const lifecycleClient = await app.inject({ method: "POST", url: "/pi-management/test" }); + + expect(sameOrigin.statusCode).toBe(200); + expect(lifecycleClient.statusCode).toBe(200); + } finally { + await app.close(); + } +}); + +// Catches route wiring that bypasses closed service validation or gives the browser a Docker/image +// lifecycle endpoint rather than only installation-default configuration and diagnostics. +test("trusted admins receive only configuration, smoke, options, and log endpoints", async () => { + const service = fakeService(); + const app = appWith(service, exposedServerEnv); + try { + const options = await app.inject({ method: "GET", url: "/pi-management/options", headers: adminHeaders }); + const configured = await app.inject({ + method: "PUT", url: "/pi-management/config", headers: adminHeaders, + payload: { provider: "zai", model: "glm-5.2", reasoning: "high" }, + }); + const smoke = await app.inject({ method: "POST", url: "/pi-management/test", headers: adminHeaders }); + const logs = await app.inject({ method: "GET", url: "/pi-management/logs", headers: adminHeaders }); + + expect(options.statusCode).toBe(200); + expect(configured.statusCode).toBe(200); + expect(configured.json()).toMatchObject({ provider: "zai", model: "glm-5.2", reasoning: "high" }); + expect(smoke.statusCode).toBe(200); + expect(logs.statusCode).toBe(200); + expect(app.printRoutes()).not.toContain("update"); + expect(app.printRoutes()).not.toContain("rollback"); + } finally { + await app.close(); + } +}); diff --git a/backend/test/routes-sessions.test.ts b/backend/test/routes-sessions.test.ts index 9f019541..60e23cdb 100644 --- a/backend/test/routes-sessions.test.ts +++ b/backend/test/routes-sessions.test.ts @@ -1,15 +1,68 @@ -import { test, expect } from "vitest"; +import { test, expect, vi } from "vitest"; import { spawn as nodeSpawn } from "node:child_process"; import path from "node:path"; import os from "node:os"; -import { chmodSync, unlinkSync, writeFileSync } from "node:fs"; -import { buildApp } from "../src/app.js"; +import { chmodSync, readFileSync, unlinkSync, writeFileSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { buildApp as buildRealApp } from "../src/app.js"; import { loadConfig } from "../src/config.js"; import { SseHub } from "../src/sse/sse-hub.js"; +import { MaintenanceBarrier } from "../src/runtime/maintenance-gate.js"; +import { PiProcessManager } from "../src/pi/pi-process-manager.js"; +import { validateDeclarativePiConfig } from "../src/pi/managed-config.js"; +import Fastify from "fastify"; +import { sessionRoutes } from "../src/routes/sessions.js"; const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs"); const SCRIPT = path.resolve("../harness/tests/fake_pi/scripts/f1_disambiguation.json"); +function operationalWorkspace(id = "default") { + return { + workspace: { schema_version: 3, id, name: id, language: "en" }, + dwh: { + engine: "postgres", database: "warehouse", schema: "public", + supported_transports: ["postgres_direct"], + }, + semantic_index: { + vector_store: { + engine: "qdrant", collection: id, dimensions: 1024, distance: "cosine", + }, + embedding: { + provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024, + }, + }, + llm_policy: { + allowed: ["zai/glm-5.2", "deepseek/deepseek-v4-pro", "local-qwen/qwen3.6-35b-a3b"], + }, + } as const; +} + +const defaultWorkspaceRegistry = { + list: vi.fn(async () => [{ + id: "default", commit: "e".repeat(40), blob: "f".repeat(40), + snapshotPath: `/data/workspace-registry/snapshots/${"e".repeat(40)}/default.yaml`, + }]), + read: vi.fn(async (id: string) => ({ + workspace: operationalWorkspace(id), + revision: { + id, commit: "e".repeat(40), blob: "f".repeat(40), + snapshotPath: `/data/workspace-registry/snapshots/${"e".repeat(40)}/${id}.yaml`, + }, + })), +}; + +function buildApp(config: Parameters[0], deps: Record = {}) { + const thtRunner = deps.thtRunner + ? { qdrantEnsure: async () => ({ ok: true }), ...(deps.thtRunner as object) } + : undefined; + return buildRealApp(config, { + workspaceRuntimeSupport: () => true, + ...deps, + ...(thtRunner ? { thtRunner } : {}), + workspaceRegistry: { ...defaultWorkspaceRegistry, ...(deps.workspaceRegistry as object | undefined) }, + } as any); +} + function mutApp(thtRunner: any) { return buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { thtRunner: { ollamaEnsure: async () => ({ ok: true }), ...thtRunner }, @@ -48,6 +101,170 @@ test("upstream requests without a principal fail before a Pi runtime can be crea expect(created).toBe(false); }); +test("GET /sessions with a query still requires session.use", async () => { + const app = Fastify(); + app.addHook("preHandler", async (request) => { + request.principal = { issuer: "oidc", subject: "no-role", roles: [], permissions: [], isAdmin: false }; + }); + sessionRoutes(app, { + mgr: { get: () => undefined } as any, + tht: { sessionList: async () => [] } as any, + hub: {} as any, + getSettings: async () => ({}), + readiness: {} as any, + listModels: async () => [], + workspaceRegistry: { list: async () => [] } as any, + workspaceRuntimeSupport: () => true, + maintenanceBarrier: new MaintenanceBarrier(), + }); + + const response = await app.inject({ method: "GET", url: "/sessions?scope=mine" }); + + expect(response.statusCode).toBe(403); + expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); +}); + +test("maintenance rejects new and resumed session admission without interrupting running sessions", async () => { + const maintenanceBarrier = new MaintenanceBarrier(); + await maintenanceBarrier.activate(); + const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), { + maintenanceBarrier, + thtRunner: { withPrincipal: () => ({ sessionShow: async () => ({ id: "open", status: "open" }) }) } as any, + }); + + const create = await app.inject({ + method: "POST", url: "/sessions", headers: aliceHeaders, payload: { question: "q" }, + }); + const resume = await app.inject({ method: "POST", url: "/sessions/open/resume", headers: aliceHeaders }); + + expect(create.statusCode).toBe(503); + expect(resume.statusCode).toBe(503); + expect(create.json()).toEqual({ + code: "maintenance", error: "Session admission is temporarily paused for maintenance. Try again shortly.", + }); + expect(resume.json()).toEqual(create.json()); +}); + +test("a durable maintenance marker initializes admission closed after backend recreation", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "tht-maintenance-")); + try { + const marker = path.join(dir, "maintenance.json"); + writeFileSync(marker, '{"transaction":"test"}\n'); + const app = buildApp(loadConfig({ + AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness", THT_MAINTENANCE_FILE: marker, + }), { thtRunner: {} as any }); + const response = await app.inject({ + method: "POST", url: "/sessions", headers: aliceHeaders, payload: { question: "q" }, + }); + expect(response.statusCode).toBe(503); + expect(response.json()).toMatchObject({ code: "maintenance" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("maintenance control requires an upstream identity and remains loopback-only", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "tht-maintenance-control-")); + const marker = path.join(dir, "maintenance.json"); + try { + const app = buildApp(loadConfig({ + AUTH_MODE: "upstream", + THT_HARNESS_DIR: "../harness", + THT_MAINTENANCE_FILE: marker, + }), { thtRunner: {} as any }); + + expect((await app.inject({ method: "POST", url: "/internal/maintenance/activate" })).statusCode).toBe(401); + const activated = await app.inject({ + method: "POST", url: "/internal/maintenance/activate", headers: aliceHeaders, + }); + expect(activated.statusCode).toBe(200); + expect(activated.json()).toEqual({ active: true, admissions: 0 }); + + const spoofedProxy = await app.inject({ + method: "POST", + url: "/internal/maintenance/deactivate", + remoteAddress: "172.30.0.9", + headers: { + "x-thoth-principal-subject": "thothctl-maintenance", + "x-thoth-is-admin": "1", + }, + }); + expect(spoofedProxy.statusCode).toBe(403); + + const status = await app.inject({ method: "GET", url: "/internal/maintenance/status", headers: aliceHeaders }); + expect(status.json()).toEqual({ active: true, admissions: 0 }); + const deactivated = await app.inject({ + method: "POST", url: "/internal/maintenance/deactivate", headers: aliceHeaders, + }); + expect(deactivated.json()).toEqual({ active: false, admissions: 0 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("maintenance endpoints report marker-derived state after post-rename and post-remove fsync failures", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "tht-maintenance-endpoint-fsync-")); + const marker = path.join(dir, "maintenance.json"); + let failSync = true; + try { + const maintenanceBarrier = new MaintenanceBarrier(marker, { + syncDirectory() { + if (failSync) throw new Error("injected maintenance fsync failure"); + }, + }); + const app = buildApp(loadConfig({ + AUTH_MODE: "none", + THT_HARNESS_DIR: "../harness", + THT_MAINTENANCE_FILE: marker, + }), { thtRunner: {} as any, maintenanceBarrier }); + + const activated = await app.inject({ method: "POST", url: "/internal/maintenance/activate" }); + expect(activated.statusCode).toBe(500); + expect(activated.json()).toMatchObject({ + active: true, + admissions: 0, + recoveryRequired: true, + code: "maintenance_durability_failed", + }); + + failSync = false; + expect((await app.inject({ method: "GET", url: "/internal/maintenance/status" })).json()) + .toEqual({ active: true, admissions: 0, recoveryRequired: true }); + failSync = true; + const deactivated = await app.inject({ method: "POST", url: "/internal/maintenance/deactivate" }); + expect(deactivated.statusCode).toBe(500); + expect(deactivated.json()).toMatchObject({ + active: true, + admissions: 0, + recoveryRequired: true, + code: "maintenance_durability_failed", + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("admin all-sessions response matches the authenticated lifecycle wire fixture", async () => { + const fixture = JSON.parse(readFileSync( + path.join(import.meta.dirname, "fixtures", "sessions-scope-all.json"), + "utf8", + )); + const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + withPrincipal: () => ({ sessionList: async () => fixture.map(({ active: _active, ...row }: any) => row) }), + } as any, + mgr: { get: () => undefined } as any, + workspaceRegistry: defaultWorkspaceRegistry as any, + }); + const response = await app.inject({ + method: "GET", + url: "/sessions?scope=all", + headers: { ...aliceHeaders, "x-thoth-is-admin": "1" }, + }); + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual(fixture); +}); + test("session routes conceal foreign or missing sessions and deny SSE before it subscribes", async () => { let subscribed = false; const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), { @@ -82,11 +299,86 @@ test("session listing permits all scope only to admins", async () => { }); expect(regularAll.statusCode).toBe(403); + expect(regularAll.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); expect(mine.statusCode).toBe(200); expect(adminAll.statusCode).toBe(200); expect(seen).toEqual([false, true]); }); +test("an administrator session listing retains revisions referenced by resumable manifests", async () => { + const retained = vi.fn(async () => {}); + const retainedRevision = "a".repeat(40); + const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + withPrincipal: () => ({ sessionList: async () => [ + { id: "open", status: "open", archived: false, workspace_revision: retainedRevision }, + { id: "finalized", status: "finalized", archived: false, workspace_revision: "b".repeat(40) }, + { id: "archived", status: "closed", archived: true, workspace_revision: "c".repeat(40) }, + ] }), + } as any, + workspaceRegistry: { reconcileSnapshotRetention: retained } as any, + }); + + const response = await app.inject({ + method: "GET", url: "/sessions?scope=all", + headers: { ...aliceHeaders, "x-thoth-is-admin": "1" }, + }); + + expect(response.statusCode).toBe(200); + expect(retained).toHaveBeenCalledWith([retainedRevision]); +}); + +test("retention scans a removed workspace's retained snapshot", async () => { + const retained = vi.fn(async () => {}); + const removedRevision = "e".repeat(40); + const activeSnapshot = "/registry/snapshots/a/other.yaml"; + const removedSnapshot = "/registry/snapshots/e/removed.yaml"; + const listRetainedSnapshots = vi.fn(async () => [ + { id: "other", commit: "a".repeat(40), snapshotPath: activeSnapshot }, + { id: "removed", commit: removedRevision, snapshotPath: removedSnapshot }, + ]); + const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + withPrincipal: () => ({ + sessionList: async (snapshotPath: string) => snapshotPath === removedSnapshot + ? [{ id: "resumable", status: "closed", archived: false, workspace_revision: removedRevision }] + : [], + }), + } as any, + workspaceRegistry: { + list: async () => [{ id: "other", commit: "a".repeat(40), snapshotPath: activeSnapshot }], + listRetainedSnapshots, + reconcileSnapshotRetention: retained, + } as any, + }); + + const response = await app.inject({ + method: "GET", url: "/sessions?scope=all", + headers: { ...aliceHeaders, "x-thoth-is-admin": "1" }, + }); + + expect(response.statusCode).toBe(200); + expect(listRetainedSnapshots).toHaveBeenCalledOnce(); + expect(retained).toHaveBeenCalledWith([removedRevision]); + expect(response.json()).toEqual([expect.objectContaining({ id: "resumable" })]); +}); + +test("the single local installation listing reconciles its resumable workspace pins", async () => { + const retained = vi.fn(async () => {}); + const retainedRevision = "d".repeat(40); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + sessionList: async () => [{ id: "open", status: "closed", archived: false, workspace_revision: retainedRevision }], + } as any, + workspaceRegistry: { reconcileSnapshotRetention: retained } as any, + }); + + const response = await app.inject({ method: "GET", url: "/sessions" }); + + expect(response.statusCode).toBe(200); + expect(retained).toHaveBeenCalledWith([retainedRevision]); +}); + test("A, B, and admin requests preserve owner isolation through session route mutations", async () => { const owners = new Map([["a", "alice"], ["b", "bob"]]); const closed: Array<{ id: string; subject: string }> = []; @@ -148,6 +440,302 @@ test("new sessions are created through the authenticated principal, not a client expect(principal).toMatchObject({ issuer: "portal", subject: "alice" }); }); +test("new sessions reject the client legacy workspace field unless local legacy mode is explicit", async () => { + const sessionNew = vi.fn(async () => ({ id: "legacy" })); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { sessionNew, searchPack: async () => {} } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + mgr: { get: () => undefined, createFor: () => ({ bridge: { onClientEvent: () => {} } }), configure: async () => {}, start: () => {} } as any, + getSettings: () => ({}) as any, + }); + + const response = await app.inject({ + method: "POST", url: "/sessions", payload: { question: "q", workspace: "legacy" }, + }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toMatchObject({ code: "workspace_revision_unavailable" }); + expect(sessionNew).not.toHaveBeenCalled(); +}); + +test("explicit local legacy mode permits the unpinned client workspace request", async () => { + const sessionNew = vi.fn(async () => ({ id: "legacy" })); + const app = buildApp(loadConfig({ + THT_HARNESS_DIR: "../harness", THT_LEGACY_WORKSPACE_MODE: "local", + }), { + thtRunner: { sessionNew, searchPack: async () => {} } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + mgr: { get: () => undefined, createFor: () => ({ bridge: { onClientEvent: () => {} } }), configure: async () => {}, start: () => {} } as any, + getSettings: () => ({}) as any, + }); + + const response = await app.inject({ + method: "POST", url: "/sessions", payload: { question: "q", workspace: "legacy" }, + }); + + expect(response.statusCode).toBe(200); + expect(sessionNew).toHaveBeenCalledWith(expect.objectContaining({ + workspaceConfigPath: undefined, workspaceId: undefined, workspaceRevision: undefined, + })); +}); + +test("creates a session from the active immutable workspace revision", async () => { + const sessionNew = vi.fn(async () => ({ id: "pinned" })); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { sessionNew, searchPack: async () => {} } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + mgr: { + get: () => undefined, + createFor: () => ({ bridge: { onClientEvent: () => {} } }), + configure: async () => {}, + start: () => {}, + } as any, + getSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "low" }) as any, + listModels: async () => [{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }], + workspaceRegistry: { + read: vi.fn(async () => ({ + workspace: { + workspace: { schema_version: 2, id: "psd-clinical", name: "PSD", language: "it" }, + dwh: {}, semantic_index: {}, llm_policy: { allowed: ["zai/glm-5.2"] }, + }, + revision: { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), + snapshotPath: "/data/workspace-registry/snapshots/abc/psd-clinical.yaml", + }, + })), + } as any, + }); + + await app.inject({ + method: "POST", url: "/sessions", + payload: { question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" }, + }); + + expect(sessionNew).toHaveBeenCalledWith(expect.objectContaining({ + workspaceConfigPath: "/data/workspace-registry/snapshots/abc/psd-clinical.yaml", + workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40), + })); +}); + +test("rejects an SSH-only workspace before persisting or starting a session", async () => { + const sessionNew = vi.fn(async () => ({ id: "must-not-exist" })); + const ensure = vi.fn(async () => ({ ok: true })); + const createFor = vi.fn(); + const abort = vi.fn(async () => {}); + const markPersisted = vi.fn(async () => {}); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { sessionNew, searchPack: async () => {} } as any, + readiness: { ensure } as any, + mgr: { get: () => undefined, createFor } as any, + getSettings: () => ({ workspace: "ssh-workspace" }) as any, + workspaceRuntimeSupport: vi.fn(() => false), + workspaceRegistry: { + acquireSessionRevision: vi.fn(async () => ({ + workspace: { + workspace: { schema_version: 2, id: "ssh-workspace", name: "SSH", language: "en" }, + dwh: { + engine: "postgres", database: "postgres", schema: "public", + supported_transports: ["ssh_tunnel"], + }, + semantic_index: { + vector_store: { + engine: "pgvector", database: "postgres", schema: "vectors", + collection: "documents", dimensions: 768, distance: "cosine", + supported_transports: ["ssh_tunnel"], + }, + embedding: { + provider: "ollama_compatible", model: "nomic-embed-text", dimensions: 768, + }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + }, + revision: { + id: "ssh-workspace", commit: "a".repeat(40), blob: "b".repeat(40), + snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/ssh-workspace.yaml`, + }, + abort, + markPersisted, + })), + } as any, + }); + + const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toMatchObject({ code: "workspace_not_activatable" }); + expect(ensure).not.toHaveBeenCalled(); + expect(sessionNew).not.toHaveBeenCalled(); + expect(createFor).not.toHaveBeenCalled(); + expect(abort).toHaveBeenCalledOnce(); + expect(markPersisted).not.toHaveBeenCalled(); +}); + +test("hands a revision lease to retention only after the session manifest is durable", async () => { + const persisted = deferred<{ id: string }>(); + const markPersisted = vi.fn(async () => {}); + const abort = vi.fn(async () => {}); + const acquireSessionRevision = vi.fn(async () => ({ + workspace: { llm_policy: { allowed: ["zai/glm-5.2"] } }, + revision: { + id: "leased", commit: "a".repeat(40), blob: "b".repeat(40), + snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/leased.yaml`, + }, + markPersisted, + abort, + })); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { sessionNew: () => persisted.promise, searchPack: async () => {} } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + mgr: { + get: () => undefined, + createFor: () => ({ bridge: { onClientEvent: () => {} } }), + configure: async () => {}, + start: () => {}, + } as any, + getSettings: () => ({ workspace: "leased", provider: "zai", model: "glm-5.2" }) as any, + listModels: async () => [{ provider: "zai", id: "glm-5.2", name: "GLM", reasoning: true }], + workspaceRuntimeSupport: () => true, + workspaceRegistry: { acquireSessionRevision } as any, + }); + + const request = app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } }); + await new Promise((resolve) => setImmediate(resolve)); + expect(markPersisted).not.toHaveBeenCalled(); + expect(abort).not.toHaveBeenCalled(); + + persisted.resolve({ id: "leased-session" }); + expect((await request).statusCode).toBe(200); + expect(markPersisted).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); +}); + +test("creates a session from the configured default workspace revision when workspaceId is omitted", async () => { + const sessionNew = vi.fn(async () => ({ id: "default-pinned" })); + const registry = { + read: vi.fn(async (id: string) => ({ + workspace: { llm_policy: { allowed: ["zai/glm-5.2"] } }, + revision: { + id, commit: "c".repeat(40), blob: "d".repeat(40), + snapshotPath: `/data/workspace-registry/snapshots/${"c".repeat(40)}/${id}.yaml`, + }, + })), + }; + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { sessionNew, searchPack: async () => {} } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + mgr: { get: () => undefined, createFor: () => ({ bridge: { onClientEvent: () => {} } }), configure: async () => {}, start: () => {} } as any, + getSettings: () => ({ workspace: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" }) as any, + listModels: async () => [{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }], + workspaceRegistry: registry as any, + }); + + await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } }); + + expect(registry.read).toHaveBeenCalledWith("psd-clinical"); + expect(sessionNew).toHaveBeenCalledWith(expect.objectContaining({ + workspaceId: "psd-clinical", workspaceRevision: "c".repeat(40), + workspaceConfigPath: `/data/workspace-registry/snapshots/${"c".repeat(40)}/psd-clinical.yaml`, + })); +}); + +test("session lifecycle locates a B session when installation default is A", async () => { + const aPath = "/registry/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/a-workspace.yaml"; + const bPath = "/registry/snapshots/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/b-workspace.yaml"; + const bPinnedPath = "/registry/snapshots/cccccccccccccccccccccccccccccccccccccccc/b-workspace.yaml"; + const bManifest = { + id: "session-b", status: "open", archived: false, + workspace_id: "b-workspace", workspace_revision: "c".repeat(40), + provider: "zai", model: "glm-5.2", thinking: "low", + }; + const calls: string[] = []; + const runtimeSources: string[] = []; + const runtimeOptions: string[] = []; + let active: any; + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + sessionNew: async (input: any) => { + calls.push(`new:${input.workspaceConfigPath}`); + return { id: "session-b" }; + }, + searchPack: async () => {}, + sessionList: async (workspace: string) => { + calls.push(`list:${workspace}`); + return workspace === bPath ? [{ id: "session-b", status: "open", question: "B question" }] : []; + }, + sessionShow: async (id: string, workspace: string) => { + calls.push(`show:${workspace}`); + if (id === "session-b" && workspace === bPath) return bManifest; + throw new Error("session not found"); + }, + reopenSession: async (id: string, workspace: string) => { + calls.push(`reopen:${workspace}`); + expect(id).toBe("session-b"); + }, + acquireWorkspaceRuntime: (workspace: string) => { + runtimeSources.push(workspace); + return { + path: `/runtime/${runtimeSources.length}.yaml`, + workspaceId: "b-workspace", + workspaceRevision: "c".repeat(40), + release: vi.fn(), + }; + }, + } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + mgr: { + get: () => active, + createFor: (_id: string, options: any) => { + runtimeOptions.push(options.runtimeConfig?.path ?? "missing"); + active = { bridge: { onClientEvent: () => {}, respond: () => true, turnState: () => "idle" } }; + return active; + }, + configure: async () => {}, start: () => {}, + teardownForPrincipal: () => [], + teardownIfCurrent: (_id: string, expected: any) => { + if (active !== expected) return false; + active = undefined; + return true; + }, + } as any, + getSettings: () => ({ workspace: "a-workspace", provider: "zai", model: "glm-5.2", thinking: "low" }) as any, + listModels: async () => [{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }], + workspaceRegistry: { + read: async (id: string) => ({ + workspace: { llm_policy: { allowed: ["zai/glm-5.2"] } }, + revision: { id, commit: "b".repeat(40), blob: "d".repeat(40), snapshotPath: bPath }, + }), + list: async () => [ + { id: "a-workspace", commit: "a".repeat(40), blob: "a".repeat(40), snapshotPath: aPath }, + { id: "b-workspace", commit: "b".repeat(40), blob: "b".repeat(40), snapshotPath: bPath }, + ], + readPinned: vi.fn(async (id: string, revision: string) => { + expect([id, revision]).toEqual(["b-workspace", "c".repeat(40)]); + return { workspace: operationalWorkspace(id), workspaceConfigPath: bPinnedPath }; + }), + } as any, + }); + + expect((await app.inject({ method: "POST", url: "/sessions", payload: { + question: "B question", workspaceId: "b-workspace", provider: "zai", model: "glm-5.2", thinking: "low", + } })).statusCode).toBe(200); + expect((await app.inject({ method: "GET", url: "/sessions" })).json()).toEqual([ + expect.objectContaining({ id: "session-b", active: true }), + ]); + expect((await app.inject({ method: "GET", url: "/sessions/session-b" })).json()).toMatchObject(bManifest); + expect((await app.inject({ method: "POST", url: "/sessions/session-b/response", payload: { ui_response: {} } })).statusCode) + .toBe(204); + + active = undefined; + expect((await app.inject({ method: "POST", url: "/sessions/session-b/resume" })).json()) + .toEqual({ id: "session-b", alreadyActive: false }); + expect(calls).toContain(`new:${bPath}`); + expect(calls).toContain(`list:${bPath}`); + expect(calls).toContain(`show:${bPath}`); + expect(calls).toContain(`reopen:${bPinnedPath}`); + expect(runtimeSources).toEqual([bPath, bPinnedPath]); + expect(runtimeOptions).toEqual(["/runtime/1.yaml", "/runtime/2.yaml"]); +}); + test("POST /sessions usa i settings (workspace/provider/model/thinking) e crea+avvia", async () => { const modelKey = path.join(os.tmpdir(), `thoth-model-key-${process.pid}`); writeFileSync(modelKey, "test-model-key", { mode: 0o600 }); @@ -171,7 +759,7 @@ test("POST /sessions usa i settings (workspace/provider/model/thinking) e crea+a }); const created = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } }); expect(created.json()).toEqual({ id: "s1" }); - expect(sessionNewArg.workspace).toBe("w"); + expect(sessionNewArg.workspaceConfigPath).toContain(`/snapshots/${"e".repeat(40)}/w.yaml`); expect(sessionNewArg.provider).toBe("zai"); expect(sessionNewArg.model).toBe("glm-5.2"); expect(sessionNewArg.thinking).toBe("high"); @@ -373,6 +961,200 @@ test("POST /sessions/:id/resume configura Pi con il thinking persistito", async expect(configured.thinking).toBe("medium"); }); +test("POST /sessions/:id/resume uses the manifest's retained workspace revision", async () => { + const reopenSession = vi.fn(async () => {}); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + mgr: { + get: () => undefined, + createFor: () => ({ bridge: { onClientEvent: () => {} } }), + configure: async () => {}, start: () => {}, + } as any, + thtRunner: { + sessionShow: async () => ({ + status: "open", archived: false, workspace_id: "psd-clinical", workspace_revision: "a".repeat(40), + }), + reopenSession, + } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + getSettings: () => ({ workspace: "legacy" }) as any, + workspaceRegistry: { + readPinned: vi.fn(async () => ({ + workspace: operationalWorkspace("psd-clinical"), + revision: { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), + snapshotPath: "/data/workspace-registry/snapshots/aaaaaaaa/psd-clinical.yaml", + }, + })), + } as any, + }); + + const response = await app.inject({ method: "POST", url: "/sessions/pinned/resume" }); + + expect(response.statusCode).toBe(200); + expect(reopenSession).toHaveBeenCalledWith( + "pinned", "/data/workspace-registry/snapshots/aaaaaaaa/psd-clinical.yaml", + ); +}); + +test("POST /sessions/:id/resume returns a sanitized error when its retained revision is unavailable", async () => { + const rawFailure = "cannot read /data/workspace-registry/snapshots/secret-revision"; + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + sessionShow: async () => ({ + status: "open", archived: false, workspace_id: "psd-clinical", workspace_revision: "a".repeat(40), + }), + } as any, + getSettings: () => ({ workspace: "legacy" }) as any, + workspaceRegistry: { readPinned: async () => { throw new Error(rawFailure); } } as any, + }); + + const response = await app.inject({ method: "POST", url: "/sessions/pinned/resume" }); + + expect(response.statusCode).toBe(409); + expect(response.body).not.toContain(rawFailure); + expect(response.json()).toMatchObject({ code: "workspace_revision_unavailable" }); +}); + +test("POST /sessions/:id/resume rejects a pinned schema-v2 workspace before readiness or runtime", async () => { + const readiness = vi.fn(async () => ({ ok: true })); + const reopenSession = vi.fn(async () => {}); + const acquireWorkspaceRuntime = vi.fn(); + const createFor = vi.fn(); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + sessionShow: async () => ({ + status: "open", archived: false, + workspace_id: "psd-clinical", workspace_revision: "a".repeat(40), + }), + reopenSession, + acquireWorkspaceRuntime, + } as any, + readiness: { ensure: readiness } as any, + mgr: { get: () => undefined, createFor } as any, + getSettings: () => ({ workspace: "legacy" }) as any, + workspaceRegistry: { + readPinned: vi.fn(async () => ({ + workspace: { + workspace: { schema_version: 2, id: "psd-clinical", name: "PSD", language: "it" }, + dwh: { + engine: "postgres", database: "warehouse", schema: "public", + supported_transports: ["rest_api"], + }, + semantic_index: { + vector_store: { + engine: "pgvector", database: "warehouse", schema: "vectors", + collection: "documents", dimensions: 768, distance: "cosine", + supported_transports: ["rest_api"], + }, + embedding: { + provider: "ollama_compatible", model: "nomic-embed-text", dimensions: 768, + }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + }, + workspaceConfigPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/psd-clinical.yaml`, + })), + } as any, + }); + + const response = await app.inject({ method: "POST", url: "/sessions/pinned-v2/resume" }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toEqual({ + code: "workspace_revision_unavailable", + error: "Session workspace configuration is unavailable. Check configuration and try again.", + }); + expect(readiness).not.toHaveBeenCalled(); + expect(reopenSession).not.toHaveBeenCalled(); + expect(acquireWorkspaceRuntime).not.toHaveBeenCalled(); + expect(createFor).not.toHaveBeenCalled(); +}); + +test("a pruned pin blocks Resume but not active or mutation lifecycle routes", async () => { + const activePath = "/registry/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/b-workspace.yaml"; + const prunedError = "cannot read /registry/snapshots/secret-pruned-revision/b-workspace.yaml"; + const calls: string[] = []; + const readPinned = vi.fn(async () => { throw new Error(prunedError); }); + let active: any = { bridge: { respond: () => true } }; + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + sessionShow: async (_id: string, workspace: string) => { + expect(workspace).toBe(activePath); + return { + id: "pruned", status: "open", archived: false, + workspace_id: "b-workspace", workspace_revision: "b".repeat(40), + }; + }, + closeSession: async (_id: string, workspace: string) => { calls.push(`close:${workspace}`); }, + deleteSession: async (_id: string, workspace: string) => { calls.push(`delete:${workspace}`); }, + } as any, + mgr: { + get: () => active, + teardownIfCurrent: (_id: string, expected: any) => { + if (active !== expected) return false; + active = undefined; + return true; + }, + } as any, + workspaceRegistry: { + list: async () => [{ + id: "b-workspace", commit: "a".repeat(40), blob: "a".repeat(40), snapshotPath: activePath, + }], + readPinned, + } as any, + }); + + expect((await app.inject({ method: "POST", url: "/sessions/pruned/response", payload: { ui_response: {} } })).statusCode) + .toBe(204); + expect((await app.inject({ method: "POST", url: "/sessions/pruned/close" })).statusCode).toBe(200); + expect((await app.inject({ method: "DELETE", url: "/sessions/pruned" })).statusCode).toBe(204); + expect(calls).toEqual([`close:${activePath}`, `delete:${activePath}`]); + expect(readPinned).not.toHaveBeenCalled(); + + const resume = await app.inject({ method: "POST", url: "/sessions/pruned/resume" }); + expect(resume.statusCode).toBe(409); + expect(resume.body).not.toContain(prunedError); + expect(resume.json()).toMatchObject({ code: "workspace_revision_unavailable" }); + expect(readPinned).toHaveBeenCalledWith("b-workspace", "b".repeat(40)); +}); + +test("POST /sessions/:id/resume refuses a pinned finalized session before reading its snapshot", async () => { + const readPinned = vi.fn(async () => { throw new Error("must not resolve"); }); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + sessionShow: async () => ({ + status: "finalized", archived: false, workspace_id: "psd-clinical", workspace_revision: "a".repeat(40), + }), + } as any, + getSettings: () => ({ workspace: "legacy" }) as any, + workspaceRegistry: { readPinned } as any, + }); + + const response = await app.inject({ method: "POST", url: "/sessions/pinned-final/resume" }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toMatchObject({ error: expect.stringMatching(/sola lettura/i) }); + expect(readPinned).not.toHaveBeenCalled(); +}); + +test("GET /sessions/:id warns when a legacy manifest has no workspace revision", async () => { + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + mgr: { + get: () => undefined, + createFor: () => ({ bridge: { onClientEvent: () => {} } }), + configure: async () => {}, start: () => {}, + } as any, + thtRunner: { sessionShow: async () => ({ status: "open", archived: false }), reopenSession: async () => {} } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + getSettings: () => ({ workspace: "legacy" }) as any, + }); + + const response = await app.inject({ method: "GET", url: "/sessions/legacy" }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ warning: expect.stringMatching(/legacy/i) }); +}); + test("POST /sessions/:id/resume usa il thinking globale se manca nel manifest", async () => { let configured: any; const bridge = { onClientEvent: () => {}, emitClientEvent: () => {} }; @@ -1531,8 +2313,9 @@ test("POST /sessions/:id/rename calls setName", async () => { expect(arg).toEqual({ id: "s1", name: "N" }); }); -test("rename authorizes and mutates through the same selected workspace", async () => { +test("rename authorizes and mutates through the same registry snapshot", async () => { const workspaces: string[] = []; + const tenantPath = "/registry/snapshots/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/tenant-a.yaml"; const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), { thtRunner: { withPrincipal: () => ({ @@ -1540,7 +2323,11 @@ test("rename authorizes and mutates through the same selected workspace", async setName: async (_id: string, _name: string, workspace: string) => { workspaces.push(`set:${workspace}`); }, }), } as any, - getSettings: () => ({ workspace: "tenant-a" }) as any, + workspaceRegistry: { + list: async () => [{ + id: "tenant-a", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: tenantPath, + }], + } as any, }); const res = await app.inject({ method: "POST", url: "/sessions/s1/rename", payload: { name: "N" }, @@ -1549,7 +2336,7 @@ test("rename authorizes and mutates through the same selected workspace", async }, }); expect(res.statusCode).toBe(204); - expect(workspaces).toEqual(["show:tenant-a", "set:tenant-a"]); + expect(workspaces).toEqual([`show:${tenantPath}`, `set:${tenantPath}`]); }); test("POST /sessions/:id/group calls setGroup", async () => { @@ -1655,11 +2442,35 @@ test("POST /sessions readiness failure returns one fixed public message without expect(res.statusCode).toBe(503); expect(res.json()).toEqual({ error: "Session services are not ready. Check configuration and connectivity, then try again.", + code: "workspace_not_activatable", }); expect(res.body).not.toMatch(/secret\.invalid|DO_NOT_LEAK|\/srv\/private\/model-key/); expect(createdCalled).toBe(false); }); +test.each(["semantic_index_incompatible", "workspace_not_activatable"] as const)( + "POST /sessions does not persist when Qdrant readiness returns %s", + async (code) => { + const sessionNew = vi.fn(async () => ({ id: "must-not-exist" })); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { sessionNew } as any, + readiness: { ensure: async () => ({ ok: false, code }) } as any, + getSettings: () => ({ workspace: "psd" }) as any, + }); + + const response = await app.inject({ + method: "POST", url: "/sessions", payload: { question: "q" }, + }); + + expect(response.statusCode).toBe(503); + expect(response.json()).toEqual({ + error: "Session services are not ready. Check configuration and connectivity, then try again.", + code, + }); + expect(sessionNew).not.toHaveBeenCalled(); + }, +); + test("POST /sessions returns storage 503 before creating a Pi runtime when session persistence fails", async () => { let piCreated = false; const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { @@ -1680,8 +2491,10 @@ test("POST /sessions returns storage 503 before creating a Pi runtime when sessi test("POST /sessions proceeds when ollamaEnsure succeeds", async () => { let ensureWs: string | undefined; + const qdrantEnsure = vi.fn(async () => ({ ok: true })); const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { thtRunner: { + qdrantEnsure, ollamaEnsure: async (ws: string) => { ensureWs = ws; return { ok: true }; }, searchPack: async () => {}, sessionNew: async () => ({ id: "s1" }), @@ -1691,7 +2504,8 @@ test("POST /sessions proceeds when ollamaEnsure succeeds", async () => { }); const res = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } }); expect(res.json()).toEqual({ id: "s1" }); - expect(ensureWs).toBe("psd"); + expect(qdrantEnsure).toHaveBeenCalledWith(operationalWorkspace("psd"), 60, "self_heal"); + expect(ensureWs).toContain(`/snapshots/${"e".repeat(40)}/psd.yaml`); }); test("POST /sessions rejects an unavailable saved model before persisting a session", async () => { @@ -1732,7 +2546,7 @@ test("POST /sessions marks a persisted session failed when runtime construction sessionNew: async () => ({ id: "s-runtime-failure" }), failSession: async (id: string, workspace: string) => { expect(id).toBe("s-runtime-failure"); - expect(workspace).toBe("psd"); + expect(workspace).toContain(`/snapshots/${"e".repeat(40)}/psd.yaml`); failed += 1; }, } as any, @@ -1757,6 +2571,112 @@ test("POST /sessions marks a persisted session failed when runtime construction expect(failed).toBe(1); }); +test.each([ + [ + "new", + "auth.json", + '{"unrelated":{"credential":{"nested":["!post-session-auth-command route-secret /private/route-auth"]}}}\n', + "Session startup failed. Check configuration and connectivity, then Resume the session.", + ], + [ + "resume", + "models.json", + '{"providers":{"local-qwen":{"models":[{"id":"qwen3.6-35b-a3b","headers":{"x":"!post-session-model-command route-secret /private/route-model"}}]}}}\n', + "Session could not be resumed. Check configuration and connectivity, then try again.", + ], +] as const)( + "POST %s refuses executable %s changed after session persistence without spawning or leaking", + async (flow, changedFile, unsafeRaw, publicMessage) => { + const agentDir = mkdtempSync(path.join(tmpdir(), "tht-route-runtime-config-")); + const safeAuth = '{"deepseek":{"type":"api_key","key":"safe-token"}}\n'; + const safeModels = '{"providers":{"local-qwen":{"baseUrl":"http://model.invalid/v1","models":[{"id":"qwen3.6-35b-a3b"}]}}}\n'; + writeFileSync(path.join(agentDir, "auth.json"), safeAuth, { mode: 0o600 }); + writeFileSync(path.join(agentDir, "models.json"), safeModels, { mode: 0o600 }); + vi.stubEnv("PI_CODING_AGENT_DIR", agentDir); + const cfg = loadConfig({ THT_HARNESS_DIR: "../harness" }); + let authResolutions = 0; + let spawns = 0; + const mgr = new PiProcessManager(cfg, { + authProviders: () => { authResolutions += 1; return new Set(); }, + spawnFn: () => { spawns += 1; throw new Error("ROUTE_SPAWN_BOUNDARY_REACHED"); }, + }); + const hub = new SseHub(); + const events: Array<{ event: string; data: object }> = []; + const sessionId = `post-persistence-${flow}`; + hub.subscribe(sessionId, (event, data) => events.push({ event, data })); + const failSession = vi.fn(async () => {}); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + const validateEarlierState = () => { + validateDeclarativePiConfig(readFileSync(path.join(agentDir, "auth.json"), "utf8")); + validateDeclarativePiConfig(readFileSync(path.join(agentDir, "models.json"), "utf8")); + }; + + if (flow === "resume") { + // This session was admitted and persisted while both mounted files were safe. + validateEarlierState(); + writeFileSync(path.join(agentDir, changedFile), unsafeRaw, { mode: 0o600 }); + } + + const app = buildApp(cfg, { + mgr, + hub, + thtRunner: { + sessionNew: async () => { + // Model admission completed immediately above this persistence boundary. + writeFileSync(path.join(agentDir, changedFile), unsafeRaw, { mode: 0o600 }); + return { id: sessionId }; + }, + failSession, + searchPack: async () => {}, + sessionShow: async () => ({ + id: sessionId, + status: "open", + archived: false, + provider: "local-qwen", + model: "qwen3.6-35b-a3b", + thinking: "low", + }), + reopenSession: async () => {}, + } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + getSettings: () => ({ + workspace: "w", + provider: "local-qwen", + model: "qwen3.6-35b-a3b", + thinking: "low", + }) as any, + listModels: async () => { + validateEarlierState(); + return [{ + provider: "local-qwen", id: "qwen3.6-35b-a3b", name: "Qwen", reasoning: true, + }]; + }, + }); + + try { + const response = flow === "new" + ? await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } }) + : await app.inject({ method: "POST", url: `/sessions/${sessionId}/resume` }); + const logs = consoleError.mock.calls.flat().map(String).join(" "); + + expect(response.statusCode).toBe(503); + expect(response.json()).toEqual({ error: publicMessage }); + expect({ authResolutions, spawns, runtimes: mgr.count() }).toEqual({ + authResolutions: 0, spawns: 0, runtimes: 0, + }); + expect(failSession).toHaveBeenCalledTimes(flow === "new" ? 1 : 0); + expect(events).toEqual([]); + expect(`${response.body}\n${logs}`).not.toContain(unsafeRaw.trim()); + expect(`${response.body}\n${logs}`).not.toMatch(/route-secret|post-session-(?:auth|model)-command|\/private\/route-|tht-route-runtime-config/); + } finally { + await app.close(); + consoleError.mockRestore(); + vi.unstubAllEnvs(); + rmSync(agentDir, { recursive: true, force: true }); + } + }, +); + test("POST /sessions/:id/resume returns 409 for a read-only session without calling ollamaEnsure", async () => { let ensureCalled = false; const app = mutApp({ @@ -1783,6 +2703,7 @@ test("POST /sessions/:id/resume readiness failure returns the same fixed public expect(res.statusCode).toBe(503); expect(res.json()).toEqual({ error: "Session services are not ready. Check configuration and connectivity, then try again.", + code: "workspace_not_activatable", }); expect(res.body).not.toMatch(/secret\.invalid|DO_NOT_LEAK|\/srv\/private\/resume-key/); }); @@ -1902,3 +2823,105 @@ test("POST /sessions bootstrap failure emits only a fixed recovery message", asy expect(clientOutput).not.toContain("DO_NOT_LEAK"); expect(clientOutput).not.toContain("/srv/private/model-key"); }); + + +test.each([ + { mode: "missing signed Evidence file", evidence: true, binding: "missing", expectedStatus: 409, reachesReadiness: false }, + { mode: "unsafe signed Evidence file", evidence: true, binding: "unsafe", expectedStatus: 409, reachesReadiness: false }, + { mode: "safe signed Evidence file", evidence: true, binding: "safe", expectedStatus: 503, reachesReadiness: true }, + { mode: "no Evidence descriptor", evidence: false, binding: "missing", expectedStatus: 503, reachesReadiness: true }, +])("real buildApp admission handles $mode before Pi spawn", async ({ + evidence, binding, expectedStatus, reachesReadiness, +}) => { + const root = mkdtempSync(path.join(tmpdir(), "thoth-evidence-admission-")); + const unsafeRoot = mkdtempSync(path.join(tmpdir(), "thoth-evidence-unsafe-")); + const signedFile = path.join(root, "signed-urls.json"); + const unsafeFile = path.join(unsafeRoot, "signed-urls.json"); + writeFileSync(signedFile, '["CANARY-SIGNED-QUERY"]'); + writeFileSync(unsafeFile, '["CANARY-UNSAFE-SIGNED-QUERY"]'); + const variable = "THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE"; + const previous = { + transport: process.env.THT_WS_PSD_CLINICAL_DWH_TRANSPORT, + baseUrl: process.env.THT_WS_PSD_CLINICAL_DWH_BASE_URL, + signed: process.env[variable], + }; + process.env.THT_WS_PSD_CLINICAL_DWH_TRANSPORT = "rest_api"; + process.env.THT_WS_PSD_CLINICAL_DWH_BASE_URL = "https://dwh.example.test"; + if (binding === "safe") process.env[variable] = signedFile; + else if (binding === "unsafe") process.env[variable] = unsafeFile; + else delete process.env[variable]; + + const descriptor = { + ...operationalWorkspace("psd-clinical"), + dwh: { + ...operationalWorkspace("psd-clinical").dwh, + supported_transports: ["rest_api"], + }, + diagnostics: { + dwh_rest: { + method: "GET", path: "/health", auth: "none", + response: { database: "database", schema: "schema" }, + }, + }, + ...(evidence ? { + evidence: { + source: { + type: "http", + uris: ["https://evidence.example.test/guide.md"], + authentication: "signed_urls_file", + connect_timeout_ms: 5_000, + read_timeout_ms: 30_000, + max_bytes: 10 * 1024 * 1024, + max_redirects: 5, + allow_private_hosts: false, + max_cache_bytes: 64 * 1024 * 1024, + }, + policy: { max_chunk_chars: 4_000, retain_published_generations: 3 }, + }, + } : {}), + } as any; + const canonicalBefore = JSON.stringify(descriptor); + const ensure = vi.fn(async () => ({ ok: false, code: "workspace_not_activatable" as const })); + const createFor = vi.fn(); + const abort = vi.fn(async () => {}); + const revision = { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), + snapshotPath: `/data/workspace-registry/snapshots/${"a".repeat(40)}/psd-clinical.yaml`, + }; + + try { + const app = buildRealApp(loadConfig({ + THT_HARNESS_DIR: "../harness", + THT_WORKSPACE_SECRET_ROOTS: root, + }), { + thtRunner: { sessionNew: vi.fn(), searchPack: async () => {} } as any, + readiness: { ensure } as any, + mgr: { get: () => undefined, createFor } as any, + getSettings: () => ({ workspace: "psd-clinical" }) as any, + workspaceRegistry: { + acquireSessionRevision: vi.fn(async () => ({ + workspace: descriptor, revision, abort, markPersisted: vi.fn(async () => {}), + })), + } as any, + }); + const response = await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } }); + + expect(response.statusCode).toBe(expectedStatus); + expect(ensure).toHaveBeenCalledTimes(reachesReadiness ? 1 : 0); + expect(createFor).not.toHaveBeenCalled(); + expect(JSON.stringify(descriptor)).toBe(canonicalBefore); + expect(revision.commit).toBe("a".repeat(40)); + expect(response.body).not.toContain("CANARY-SIGNED-QUERY"); + expect(response.body).not.toContain("CANARY-UNSAFE-SIGNED-QUERY"); + } finally { + const restore = (name: string, value: string | undefined) => { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + }; + restore("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", previous.transport); + restore("THT_WS_PSD_CLINICAL_DWH_BASE_URL", previous.baseUrl); + restore(variable, previous.signed); + rmSync(root, { recursive: true, force: true }); + rmSync(unsafeRoot, { recursive: true, force: true }); + } +}); diff --git a/backend/test/routes-settings.test.ts b/backend/test/routes-settings.test.ts index f25a4964..9a10d7b0 100644 --- a/backend/test/routes-settings.test.ts +++ b/backend/test/routes-settings.test.ts @@ -5,6 +5,13 @@ import { join } from "node:path"; import { buildApp } from "../src/app.js"; import { loadConfig } from "../src/config.js"; +const userHeaders = { + "x-thoth-principal-issuer": "portal", + "x-thoth-principal-subject": "alice", + "x-thoth-is-admin": "0", +}; +const adminHeaders = { ...userHeaders, "x-thoth-principal-subject": "admin", "x-thoth-is-admin": "1" }; + function appWithTmpSettings(extraEnv: Record = {}, deps = {}) { const dir = mkdtempSync(join(tmpdir(), "tht-set-route-")); const app = buildApp( @@ -31,8 +38,21 @@ test("GET /settings returns effective defaults (env provider/model/thinking, fir } }); -test("PUT /settings persists and GET reads it back", async () => { - const { app, dir } = appWithTmpSettings({}, { +test("GET /settings uses the stored installation workspace default before the harness fallback", async () => { + const { app, dir } = appWithTmpSettings({}); + try { + writeFileSync(join(dir, "settings.json"), JSON.stringify({ workspace: "psd-clinical" })); + + const response = await app.inject({ method: "GET", url: "/settings" }); + + expect(response.json()).toMatchObject({ workspace: "psd-clinical" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("PUT /settings does not persist personal workspace or LLM choices", async () => { + const { app, dir } = appWithTmpSettings({ PI_PROVIDER: "zai", PI_MODEL: "glm-5.2", PI_THINKING: "medium" }, { listModels: async () => [{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }], }); try { @@ -42,13 +62,30 @@ test("PUT /settings persists and GET reads it back", async () => { }); expect(put.statusCode).toBe(200); const got = await app.inject({ method: "GET", url: "/settings" }); - expect(got.json()).toMatchObject({ workspace: "psd", provider: "zai", model: "glm-5.2", thinking: "high" }); + expect(got.json()).toMatchObject({ provider: "zai", model: "glm-5.2", thinking: "medium" }); + expect(got.json()).not.toMatchObject({ workspace: "psd", thinking: "high" }); } finally { rmSync(dir, { recursive: true, force: true }); } }); -test("settings are isolated by the authenticated repository principal", async () => { +test("PUT /settings requires settings.manage", async () => { + const { app, dir } = appWithTmpSettings({ AUTH_MODE: "upstream" }, { listModels: async () => [] }); + try { + const body = { workspace: "psd", provider: "zai", model: "glm-5.2", thinking: "high" }; + const denied = await app.inject({ method: "PUT", url: "/settings", headers: userHeaders, payload: body }); + const allowed = await app.inject({ method: "PUT", url: "/settings", headers: adminHeaders, payload: body }); + + expect(denied.statusCode).toBe(403); + expect(denied.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); + expect(allowed.statusCode).toBe(200); + } finally { + await app.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("settings no longer read or write principal-specific preferences", async () => { const preferences = new Map(); const runner = { withPrincipal: (principal: any) => ({ @@ -73,11 +110,11 @@ test("settings are isolated by the authenticated repository principal", async () const alice = await app.inject({ method: "GET", url: "/settings", headers: headers("alice") }); const bob = await app.inject({ method: "GET", url: "/settings", headers: headers("bob") }); - expect(alice.json()).toMatchObject({ workspace: "psd", thinking: "high" }); - expect(bob.json()).not.toMatchObject({ workspace: "psd", thinking: "high" }); + expect(alice.json()).toEqual(bob.json()); + expect(preferences.size).toBe(0); }); -test("GET /settings seeds an empty private profile from complete legacy settings once", async () => { +test("GET /settings retains complete legacy installation defaults without seeding a private profile", async () => { let preferences: Record = {}; const writes: Record[] = []; const runner = { @@ -104,13 +141,13 @@ test("GET /settings seeds an empty private profile from complete legacy settings expect(first.statusCode).toBe(200); expect(first.json()).toEqual(expected); expect(second.json()).toEqual(expected); - expect(writes).toEqual([expected]); + expect(writes).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }); -test("GET /settings does not overwrite an existing private profile with legacy settings", async () => { +test("GET /settings ignores stale private preferences in favor of installation defaults", async () => { const privateSettings = { workspace: "private", provider: "zai", model: "glm-5.2", thinking: "high", }; @@ -130,7 +167,9 @@ test("GET /settings does not overwrite an existing private profile with legacy s const response = await app.inject({ method: "GET", url: "/settings" }); expect(response.statusCode).toBe(200); - expect(response.json()).toEqual(privateSettings); + expect(response.json()).toEqual({ + workspace: "local", provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low", + }); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/backend/test/routes-sql-meta.test.ts b/backend/test/routes-sql-meta.test.ts index 8d56f65e..f21dfefb 100644 --- a/backend/test/routes-sql-meta.test.ts +++ b/backend/test/routes-sql-meta.test.ts @@ -1,6 +1,8 @@ import { test, expect, vi } from "vitest"; +import Fastify from "fastify"; import { buildApp } from "../src/app.js"; import { loadConfig } from "../src/config.js"; +import { sqlRoutes } from "../src/routes/sql.js"; // --------------------------------------------------------------------------- // SQL routes @@ -29,6 +31,35 @@ test("POST /sessions/:id/sql/preview returns rows from injected thtRunner stub", expect(res.json()).toEqual(previewResult); }); +test("SQL lookup and execution derive harness admin compatibility from session.read_all", async () => { + const seen: boolean[] = []; + const app = Fastify(); + app.addHook("preHandler", async (request) => { + request.principal = { + issuer: "portal", subject: "inconsistent", roles: ["user"], permissions: ["session.use"], isAdmin: true, + }; + }); + const runner = { + sessionShow: async () => ({ id: "s1" }), + sqlPreview: async () => ({ columns: [], rows: [], execution_ms: 0, truncated: false }), + }; + sqlRoutes(app, { + tht: { + withPrincipal: (principal: { isAdmin: boolean }) => { + seen.push(principal.isAdmin); + return runner; + }, + } as any, + getSettings: async () => ({ workspace: "legacy" }), + workspaceRegistry: { list: async () => [] } as any, + }); + + const response = await app.inject({ method: "POST", url: "/sessions/s1/sql/preview", payload: {} }); + + expect(response.statusCode).toBe(200); + expect(seen).toEqual([false, false]); +}); + test("POST /sessions/:id/sql/preview passes limit and offset to thtRunner", async () => { let captured: any; const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { @@ -85,41 +116,46 @@ test("POST /sessions/:id/sql/preview returns 500 when thtRunner throws", async ( expect(res.json()).toMatchObject({ error: /boom/ }); }); -// --------------------------------------------------------------------------- -// Meta routes -// --------------------------------------------------------------------------- - -test("GET /workspaces lists yaml files from ../harness/workspaces", async () => { - // The real ../harness/workspaces directory contains *.yaml files. +test("registry-backed SQL preview resolves and uses the session's pinned runtime revision", async () => { + const activePath = `/registry/snapshots/${"a".repeat(40)}/psd-clinical.yaml`; + const pinnedPath = `/registry/snapshots/${"b".repeat(40)}/psd-clinical.yaml`; + const calls: string[] = []; + const runner = { + sessionShow: async (_id: string, workspace: string) => { + calls.push(`show:${workspace}`); + if (workspace === activePath) return { + id: "s1", workspace_id: "psd-clinical", workspace_revision: "b".repeat(40), + }; + throw new Error("session not found"); + }, + sqlPreview: async (_id: string, _page: unknown, workspace: string) => { + calls.push(`preview:${workspace}`); + return { columns: [], rows: [], execution_ms: 0, truncated: false }; + }, + }; const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { - thtRunner: {} as any, + thtRunner: { ...runner, withPrincipal: () => runner } as any, + getSettings: () => ({ workspace: "legacy-default" }) as any, + workspaceRegistry: { + list: async () => [{ + id: "psd-clinical", commit: "a".repeat(40), blob: "c".repeat(40), + snapshotPath: activePath, + }], + readPinned: async () => ({ workspace: {}, workspaceConfigPath: pinnedPath }), + } as any, }); - const res = await app.inject({ method: "GET", url: "/workspaces" }); - - expect(res.statusCode).toBe(200); - const body = res.json() as { name: string; file: string }[]; - expect(Array.isArray(body)).toBe(true); - // ../harness/workspaces has at least one yaml (tht-test.yaml / tht.example.yaml) - expect(body.length).toBeGreaterThan(0); - for (const w of body) { - expect(typeof w.name).toBe("string"); - expect(w.name).not.toContain(".yaml"); // name strips extension - expect(w.file).toMatch(/\.ya?ml$/); - } -}); - -test("GET /workspaces returns [] when harnessDir has no workspaces subdir", async () => { - const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/nonexistent-harness-dir" }), { - thtRunner: {} as any, + const response = await app.inject({ + method: "POST", url: "/sessions/s1/sql/preview", payload: { limit: 10 }, }); - const res = await app.inject({ method: "GET", url: "/workspaces" }); - - expect(res.statusCode).toBe(200); - expect(res.json()).toEqual([]); + expect(response.statusCode).toBe(200); + expect(calls).toEqual([`show:${activePath}`, `preview:${pinnedPath}`]); }); +// Workspace registry route coverage lives in routes-workspaces.test.ts. `/workspaces` no longer +// reads legacy harness files: the Git registry is the single shared source of truth. + test("GET /models returns {models:[...]} from injected listModels stub", async () => { const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { thtRunner: {} as any, diff --git a/backend/test/routes-workspaces.test.ts b/backend/test/routes-workspaces.test.ts new file mode 100644 index 00000000..92368e82 --- /dev/null +++ b/backend/test/routes-workspaces.test.ts @@ -0,0 +1,512 @@ +import { execFile } from "node:child_process"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test, vi } from "vitest"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { createProductionWorkspaceDiagnoser } from "../src/workspaces/diagnostics.js"; +import { WorkspaceRegistry, type WorkspaceRevision } from "../src/workspaces/registry.js"; +import { serializeWorkspaceYaml, type CanonicalWorkspace } from "../src/workspaces/schema.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; +import type { AuthDiagnoser, AuthDiagnostics } from "../src/auth/diagnostics.js"; + +const workspace: CanonicalWorkspace = { + workspace: { + schema_version: 3, + id: "psd-clinical", + name: "Policlinico San Donato", + description: "Clinical analytics workspace", + language: "it", + }, + dwh: { + engine: "postgres", + database: "warehouse", + schema: "datawarehouse", + supported_transports: ["postgres_direct"], + }, + semantic_index: { + vector_store: { + engine: "qdrant", + collection: "psd-clinical", + dimensions: 1024, + distance: "cosine", + }, + embedding: { + provider: "ollama_internal", + model: "qwen3-embedding:0.6b", + dimensions: 1024, + }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, +}; + +const revision: WorkspaceRevision = { + id: workspace.workspace.id, + commit: "a".repeat(40), + blob: "b".repeat(40), + snapshotPath: "/registry/snapshots/psd-clinical.yaml", +}; + +type RegistryFake = Pick; + +function registryFake(overrides: Partial = {}): RegistryFake { + return { + bootstrap: vi.fn(async () => ({ + branch: "main", head: revision.commit, ahead: 0, behind: 0, degraded: false, + })), + pull: vi.fn(async () => ({ + branch: "main", head: revision.commit, ahead: 0, behind: 0, degraded: false, + })), + list: vi.fn(async () => [revision]), + listCatalog: vi.fn(async () => [{ + id: "psd-clinical", + name: "Policlinico San Donato", + configurationState: "ready" as const, + revision, + }]), + read: vi.fn(async () => ({ workspace, revision })), + ...overrides, + }; +} + +const readyAuthentication: AuthDiagnostics = { + ready: true, + mode: "none", + checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }], +}; + +function appFor( + registry: RegistryFake, + diagnose = vi.fn(async () => ({ activatable: true, diagnostics: [] })), + secretStore = testSecretStore(), + env: Record = {}, + authDiagnoser: AuthDiagnoser = { inspect: vi.fn(async () => readyAuthentication) }, +) { + return buildApp(loadConfig({ + THT_HARNESS_DIR: "/missing-harness", + THT_WORKSPACE_REGISTRY_ROOT: "/tmp/thoth-route-test-registry", + ...env, + }), { + thtRunner: {} as any, + workspaceRegistry: registry as WorkspaceRegistry, + workspaceDiagnoser: diagnose, + workspaceSecretStore: secretStore, + authDiagnoser, + } as any); +} + +const userHeaders = { + "x-thoth-principal-issuer": "portal", + "x-thoth-principal-subject": "alice", + "x-thoth-is-admin": "0", +}; +const adminHeaders = { ...userHeaders, "x-thoth-principal-subject": "admin", "x-thoth-is-admin": "1" }; + +const secretStoreRoots: string[] = []; + +function testSecretStore(): WorkspaceSecretStore { + const root = mkdtempSync(join(tmpdir(), "thoth-route-secret-store-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "thoth-route-secret-runtime-")); + secretStoreRoots.push(root, runtimeRoot); + return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "route-test" }); +} + +test("returns a redacted registry status and pulls without Git credential details", async () => { + const registry = registryFake({ + bootstrap: vi.fn(async () => ({ + branch: "main", + head: revision.commit, + ahead: 0, + behind: 0, + degraded: true, + lastError: "git_auth_failed" as const, + })), + }); + const app = appFor(registry); + + const status = await app.inject({ method: "GET", url: "/workspace-registry/status" }); + const pull = await app.inject({ method: "POST", url: "/workspace-registry/pull" }); + + expect(status.statusCode).toBe(200); + expect(status.json()).toMatchObject({ branch: "main", degraded: true, lastError: "git_auth_failed" }); + expect(status.body).not.toMatch(/token|credential|private.?key/i); + expect(pull.statusCode).toBe(200); + expect(registry.pull).toHaveBeenCalledTimes(1); +}); + +test("workspace mutations and secret writes require their catalog permissions", async () => { + const registry = registryFake(); + const app = appFor(registry, undefined, testSecretStore(), { AUTH_MODE: "upstream" }); + try { + const deniedPull = await app.inject({ method: "POST", url: "/workspace-registry/pull", headers: userHeaders }); + const allowedPull = await app.inject({ method: "POST", url: "/workspace-registry/pull", headers: adminHeaders }); + const deniedBootstrap = await app.inject({ method: "GET", url: "/workspace-registry/status", headers: userHeaders }); + const allowedBootstrap = await app.inject({ method: "GET", url: "/workspace-registry/status", headers: adminHeaders }); + const deniedSecret = await app.inject({ + method: "PUT", url: "/workspaces/psd-clinical/secrets", headers: userHeaders, + payload: { values: { "dwh.password": "secret" } }, + }); + const allowedSecret = await app.inject({ + method: "PUT", url: "/workspaces/psd-clinical/secrets", headers: adminHeaders, + payload: { values: { "dwh.password": "secret" } }, + }); + + expect(deniedPull.statusCode).toBe(403); + expect(deniedPull.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); + expect(allowedPull.statusCode).toBe(200); + expect(deniedBootstrap.statusCode).toBe(403); + expect(deniedBootstrap.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); + expect(allowedBootstrap.statusCode).toBe(200); + expect(deniedSecret.statusCode).toBe(403); + expect(deniedSecret.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); + expect(allowedSecret.statusCode).toBe(200); + } finally { + await app.close(); + } +}); + +test.each([ + ["POST", "/workspaces/publish"], + ["GET", "/workspaces/psd-clinical/export"], + ["POST", "/workspaces/import"], +] as const)("does not register the removed %s %s mutation or bundle route", async (method, url) => { + const response = await appFor(registryFake()).inject({ method, url }); + + expect(response.statusCode).toBe(404); +}); + +test("lists workspace summaries and reads a validated immutable workspace", async () => { + const app = appFor(registryFake()); + + const list = await app.inject({ method: "GET", url: "/workspaces" }); + const read = await app.inject({ method: "GET", url: "/workspaces/psd-clinical" }); + + expect(list.statusCode).toBe(200); + expect(list.json()).toEqual([expect.objectContaining({ + id: "psd-clinical", + displayName: "Policlinico San Donato", + configurationState: "configuration_required", + revision, + })]); + expect(read.statusCode).toBe(200); + expect(read.json()).toEqual({ workspace, revision }); +}); + +test("validates a schema v3 workspace without mutating the repository", async () => { + const app = appFor(registryFake()); + + const response = await app.inject({ + method: "POST", url: "/workspaces/validate", payload: { workspace }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ workspace }); +}); + +test("aggregates one static and one live authentication report without reordering connector diagnostics", async () => { + const connectorDiagnostics = [{ + level: "info" as const, + code: "binding_ok" as const, + message: "Installation bindings and diagnostics succeeded.", + }]; + const diagnose = vi.fn(async () => ({ activatable: true, diagnostics: connectorDiagnostics })); + const authentication: AuthDiagnostics = { + ready: false, + mode: "oidc", + checks: [{ + level: "error", + code: "oidc_mapped_group_missing", + field: "Thoth Administrators", + message: "A configured authorization group does not exist.", + }], + }; + const authDiagnoser: AuthDiagnoser = { inspect: vi.fn(async () => authentication) }; + const app = appFor(registryFake(), diagnose, testSecretStore(), {}, authDiagnoser); + + const validation = await app.inject({ + method: "POST", url: "/workspaces/validate", payload: { workspace }, + }); + const connection = await app.inject({ + method: "POST", url: "/workspaces/psd-clinical/test", payload: {}, + }); + + expect(validation.statusCode).toBe(200); + expect(validation.json()).toMatchObject({ + workspace, + activatable: false, + diagnostics: [], + authentication, + }); + expect(connection.statusCode).toBe(200); + expect(connection.json()).toEqual({ + activatable: false, + diagnostics: connectorDiagnostics, + authentication, + }); + expect(diagnose).toHaveBeenCalledTimes(1); + expect(authDiagnoser.inspect).toHaveBeenNthCalledWith(1, { live: false }); + expect(authDiagnoser.inspect).toHaveBeenNthCalledWith(2, { live: true }); +}); + +test("fails closed without reflecting a hostile authentication report", async () => { + const attacker = "attacker-field-SENTINEL"; + const authDiagnoser: AuthDiagnoser = { inspect: vi.fn(async () => ({ + ready: false, + mode: "oidc", + checks: [{ + level: "error", code: "oidc_secret_missing", message: "failure", field: attacker, + }], + } as AuthDiagnostics)) }; + const app = appFor(registryFake(), undefined, testSecretStore(), {}, authDiagnoser); + + const response = await app.inject({ + method: "POST", url: "/workspaces/validate", payload: { workspace }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json()).toEqual({ code: "workspace_invalid", message: "Workspace request is invalid." }); + expect(response.body).not.toContain(attacker); +}); + +test.each([1, 2])("rejects schema v%s at the validation boundary with a sanitized error", async (version) => { + const legacy = { + ...workspace, + workspace: { ...workspace.workspace, schema_version: version }, + }; + const response = await appFor(registryFake()).inject({ + method: "POST", url: "/workspaces/validate", payload: { workspace: legacy }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json()).toEqual({ code: "workspace_invalid", message: "Workspace request is invalid." }); + expect(response.body).not.toMatch(/migration_required|schema version/i); +}); + +test("runs diagnostics for a schema v3 workspace", async () => { + const diagnose = vi.fn(async () => ({ activatable: true, diagnostics: [] })); + const app = appFor(registryFake(), diagnose); + + const response = await app.inject({ + method: "POST", url: "/workspaces/psd-clinical/test", payload: {}, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual({ activatable: true, diagnostics: [], authentication: readyAuthentication }); + expect(diagnose).toHaveBeenCalledWith(workspace, { + dwh: expect.objectContaining({ transport: "postgres_direct" }), + evidence: { missing: [], values: {} }, + }, { writeProbe: false }); +}); + +test("reports runtime secret requirements without returning stored values", async () => { + const secretStore = testSecretStore(); + const app = appFor(registryFake(), undefined, secretStore); + + const missing = await app.inject({ + method: "GET", url: "/workspaces/psd-clinical/runtime-configuration", + }); + expect(missing.statusCode).toBe(200); + expect(missing.json()).toMatchObject({ + workspaceId: "psd-clinical", + revision, + configurationState: "configuration_required", + requirements: [{ + id: "dwh.password", + connector: "dwh", + label: "Data warehouse password", + required: true, + configured: false, + }], + }); + + const secret = "never-return-this-password"; + const save = await app.inject({ + method: "PUT", + url: "/workspaces/psd-clinical/secrets", + payload: { values: { "dwh.password": secret } }, + }); + expect(save.statusCode).toBe(200); + expect(save.body).not.toContain(secret); + expect(save.json()).toMatchObject({ + configurationState: "ready", + requirements: [{ id: "dwh.password", configured: true }], + }); + + const configured = await app.inject({ + method: "GET", url: "/workspaces/psd-clinical/runtime-configuration", + }); + expect(configured.body).not.toContain(secret); + expect(configured.json()).toMatchObject({ configurationState: "ready" }); +}); + +test("rejects undeclared secret identifiers and supports forgetting a configured secret", async () => { + const secretStore = testSecretStore(); + const app = appFor(registryFake(), undefined, secretStore); + + const unknown = await app.inject({ + method: "PUT", + url: "/workspaces/psd-clinical/secrets", + payload: { values: { "evidence.secret_key": "not-applicable" } }, + }); + expect(unknown.statusCode).toBe(400); + expect(secretStore.configured("psd-clinical")).toEqual([]); + + secretStore.put("psd-clinical", "dwh.password", "temporary-password"); + const forget = await app.inject({ + method: "DELETE", + url: "/workspaces/psd-clinical/secrets/dwh.password", + }); + expect(forget.statusCode).toBe(200); + expect(forget.json()).toMatchObject({ configurationState: "configuration_required" }); + expect(secretStore.has("psd-clinical", "dwh.password")).toBe(false); +}); + +test("materializes stored secrets only for the diagnostic lease", async () => { + const secretStore = testSecretStore(); + secretStore.put("psd-clinical", "dwh.password", "diagnostic-password"); + let materializedPath = ""; + const diagnose = vi.fn(async (_workspace, bindings) => { + materializedPath = bindings.dwh.values.THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE; + expect(readFileSync(materializedPath, "utf8")).toBe("diagnostic-password"); + return { activatable: true, diagnostics: [] }; + }); + const app = appFor(registryFake(), diagnose, secretStore); + + const response = await app.inject({ + method: "POST", url: "/workspaces/psd-clinical/test", payload: {}, + }); + + expect(response.statusCode).toBe(200); + expect(materializedPath).not.toBe(""); + expect(existsSync(materializedPath)).toBe(false); +}); + +test("reports a missing Evidence credential without changing the registry revision", async () => { + const evidenceWorkspace: CanonicalWorkspace = { + ...workspace, + evidence: { + source: { + type: "http", + uris: ["https://evidence.example.test/guide.md"], + authentication: "signed_urls_file", + connect_timeout_ms: 5_000, + read_timeout_ms: 30_000, + max_bytes: 10 * 1024 * 1024, + max_redirects: 5, + allow_private_hosts: false, + max_cache_bytes: 64 * 1024 * 1024, + }, + policy: { max_chunk_chars: 4_000, retain_published_generations: 3 }, + }, + }; + const read = vi.fn(async () => ({ workspace: evidenceWorkspace, revision })); + const app = appFor(registryFake({ read }), createProductionWorkspaceDiagnoser(100)); + const variable = "THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE"; + const previous = process.env[variable]; + delete process.env[variable]; + + try { + const response = await app.inject({ + method: "POST", url: "/workspaces/psd-clinical/test", payload: {}, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + activatable: false, + diagnostics: expect.arrayContaining([expect.objectContaining({ + code: "binding_missing", + field: "evidence.source.authentication", + variable, + })]), + }); + expect(read).toHaveBeenCalledTimes(1); + } finally { + if (previous === undefined) delete process.env[variable]; + else process.env[variable] = previous; + } +}); + +const runFile = promisify(execFile); +const realRouteRoots: string[] = []; + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +async function createRealRouteFixture() { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-route-")); + realRouteRoots.push(root); + const remote = join(root, "remote.git"); + const author = join(root, "author"); + const registryRoot = join(root, "registry"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(author); + await git(author, ["init", "--initial-branch=main"]); + await git(author, ["config", "user.name", "Workspace Route Test"]); + await git(author, ["config", "user.email", "workspace-route@example.invalid"]); + const descriptor: CanonicalWorkspace = { + ...workspace, + evidence: { + source: { + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: ["**/*.md"], + max_bytes: 1024 * 1024, + }, + policy: { max_chunk_chars: 4_000, retain_published_generations: 3 }, + }, + }; + writeFileSync(join(author, "thoth-workspaces.yaml"), [ + "schema_version: 1", + "workspaces:", + " - id: psd-clinical", + " name: Policlinico San Donato", + " description: Clinical analytics workspace", + "", + ].join("\n")); + mkdirSync(join(author, "psd-clinical", "evidence"), { recursive: true }); + writeFileSync(join(author, "psd-clinical", "workspace.yaml"), serializeWorkspaceYaml(descriptor)); + writeFileSync(join(author, "psd-clinical", "evidence", "guide.md"), "Evidence bytes\n"); + await git(author, ["add", "."]); + await git(author, ["commit", "-m", "Initial workspace"]); + await git(author, ["remote", "add", "origin", remote]); + await git(author, ["push", "origin", "main"]); + const initialCommit = await git(author, ["rev-parse", "HEAD"]); + const config = loadConfig({ + THT_HARNESS_DIR: "/missing-harness", + THT_WORKSPACE_REGISTRY_ROOT: registryRoot, + THT_WORKSPACE_GIT_REMOTE: remote, + }); + const registry = new WorkspaceRegistry(config.workspaceRegistry); + const app = buildApp(config, { + thtRunner: {} as any, + workspaceRegistry: registry, + workspaceDiagnoser: vi.fn(async () => ({ activatable: true, diagnostics: [] })), + }); + return { author, initialCommit, app, registry }; +} + +afterEach(() => { + realRouteRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); + secretStoreRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +test("a failed candidate pull keeps the last valid active workspace", async () => { + const fixture = await createRealRouteFixture(); + await fixture.registry.bootstrap(); + rmSync(join(fixture.author, "psd-clinical", "evidence"), { recursive: true }); + await git(fixture.author, ["add", "-A"]); + await git(fixture.author, ["commit", "-m", "Remove required Evidence tree"]); + await git(fixture.author, ["push", "origin", "main"]); + + const pull = await fixture.app.inject({ method: "POST", url: "/workspace-registry/pull" }); + + expect(pull.statusCode).toBe(400); + expect(pull.json()).toEqual({ code: "workspace_invalid", message: "Workspace request is invalid." }); + await expect(fixture.registry.read("psd-clinical")).resolves.toMatchObject({ + revision: { commit: fixture.initialCommit }, + }); +}); diff --git a/backend/test/secret-bundle.test.ts b/backend/test/secret-bundle.test.ts index a3f2b550..a0580da8 100644 --- a/backend/test/secret-bundle.test.ts +++ b/backend/test/secret-bundle.test.ts @@ -22,6 +22,25 @@ test("parses comments, blank lines and values containing equals", () => { ])); }); +test("accepts the fixed OIDC and Authentik secret references", () => { + const file = bundle("THT_OIDC_CLIENT_SECRET=oidc-secret\nTHT_AUTHENTIK_API_TOKEN=authentik-token\n"); + expect(loadSecretBundle(file)).toEqual(new Map([ + ["THT_OIDC_CLIENT_SECRET", "oidc-secret"], + ["THT_AUTHENTIK_API_TOKEN", "authentik-token"], + ])); +}); + +test.each([ + ["THT_OIDC_CLIENT_SECRET", 4096], + ["THT_AUTHENTIK_API_TOKEN", 16 * 1024], +])("enforces the shared exact value boundary for %s", (name, maximum) => { + expect(loadSecretBundle(bundle(`${name}=${"x".repeat(maximum)}\n`)).get(name)).toHaveLength(maximum); + expect(() => loadSecretBundle(bundle(`${name}=${"x".repeat(maximum + 1)}\n`))) + .toThrow("secret bundle is unavailable"); + expect(() => loadSecretBundle(bundle(`${name}=invalid\u0000secret\n`))) + .toThrow("secret bundle is unavailable"); +}); + test.each([ ["duplicate", "THT_MODEL_API_KEY=a\nTHT_MODEL_API_KEY=b\n"], ["unknown", "UNKNOWN_KEY=x\n"], diff --git a/backend/test/session-bridge.test.ts b/backend/test/session-bridge.test.ts index fbfe1d50..f0a41341 100644 --- a/backend/test/session-bridge.test.ts +++ b/backend/test/session-bridge.test.ts @@ -105,6 +105,30 @@ test("assistant provider errors are sanitized and leave the turn failed", () => expect(JSON.stringify(seen)).not.toContain("DO_NOT_LEAK"); }); +test("subscription model errors become a safe actionable client message", () => { + const { rpc, fire } = fakeRpc(); + const bridge = new SessionBridge(rpc); + const seen: any[] = []; + bridge.onClientEvent((event) => seen.push(event)); + + fire({ + type: "message_end", + message: { + role: "assistant", + stopReason: "error", + errorMessage: "429: {\"code\":\"1311\",\"message\":\"Your current subscription plan does not yet include access to GLM-5.3\",\"secret\":\"DO_NOT_LEAK\"}", + }, + }); + + expect(seen).toContainEqual({ + type: "info", + level: "error", + text: "The selected model is unavailable for the current subscription. Choose another model and start a new session.", + }); + expect(JSON.stringify(seen)).not.toContain("GLM-5.3"); + expect(JSON.stringify(seen)).not.toContain("DO_NOT_LEAK"); +}); + test("markFailed records backend-detected failure without emitting raw detail", () => { const { rpc } = fakeRpc(); const bridge = new SessionBridge(rpc); diff --git a/backend/test/settings-store.test.ts b/backend/test/settings-store.test.ts index cfdb090d..a17ef315 100644 --- a/backend/test/settings-store.test.ts +++ b/backend/test/settings-store.test.ts @@ -1,8 +1,13 @@ import { test, expect } from "vitest"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { closeSync, existsSync, fsyncSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { loadSettings, saveSettings } from "../src/settings/settings-store.js"; +import { + captureSettingsSnapshot, + loadSettings, + restoreSettingsSnapshot, + saveSettings, +} from "../src/settings/settings-store.js"; import { loadConfig } from "../src/config.js"; function cfgWith(file: string) { @@ -45,3 +50,50 @@ test("loadConfig sets settingsFile from SETTINGS_FILE, default data/settings.jso expect(loadConfig({}).settingsFile).toBe("data/settings.json"); expect(loadConfig({ SETTINGS_FILE: "/x/y.json" }).settingsFile).toBe("/x/y.json"); }); + +test("saveSettings restores the previous file when post-rename directory durability fails", () => { + const dir = mkdtempSync(join(tmpdir(), "tht-set-transaction-")); + try { + const cfg = cfgWith(join(dir, "settings.json")); + saveSettings(cfg, { provider: "old", model: "old-model", thinking: "low" }); + let syncs = 0; + expect(() => saveSettings( + cfg, + { provider: "new", model: "new-model", thinking: "high" }, + { + syncDirectory(directory: string) { + syncs += 1; + if (syncs === 1) throw new Error("injected directory fsync failure"); + const fd = openSync(directory, "r"); + try { fsyncSync(fd); } finally { closeSync(fd); } + }, + }, + )).toThrow(/directory fsync failure/); + expect(loadSettings(cfg)).toEqual({ provider: "old", model: "old-model", thinking: "low" }); + expect(syncs).toBeGreaterThanOrEqual(2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("settings snapshots restore exact absent and empty-file states", () => { + const dir = mkdtempSync(join(tmpdir(), "tht-settings-snapshot-")); + try { + const file = join(dir, "settings.json"); + const cfg = cfgWith(file); + const absent = captureSettingsSnapshot(cfg); + expect(absent).toEqual({ exists: false, rawBase64: "" }); + saveSettings(cfg, { provider: "new", model: "model", thinking: "high" }); + restoreSettingsSnapshot(cfg, absent); + expect(existsSync(file)).toBe(false); + + writeFileSync(file, Buffer.alloc(0), { mode: 0o600 }); + const empty = captureSettingsSnapshot(cfg); + expect(empty.exists).toBe(true); + saveSettings(cfg, { provider: "new", model: "model", thinking: "high" }); + restoreSettingsSnapshot(cfg, empty); + expect(readFileSync(file)).toEqual(Buffer.alloc(0)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/backend/test/startup-error.test.ts b/backend/test/startup-error.test.ts new file mode 100644 index 00000000..8faed34c --- /dev/null +++ b/backend/test/startup-error.test.ts @@ -0,0 +1,62 @@ +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { formatStartupFailure } from "../src/startup-error.js"; + +describe("formatStartupFailure", () => { + it.each([ + [new Error("auth_session_store_invalid"), "backend startup failed: auth_session_store_invalid"], + [new Error("auth_config_invalid"), "backend startup failed: auth_config_invalid"], + [new Error("workspace_registry_invalid"), "backend startup failed: workspace_registry_invalid"], + ])("emits only an allowlisted startup cause", (error, expected) => { + expect(formatStartupFailure(error)).toBe(expected); + }); + + it("collapses unknown errors without exposing their message, stack, token, or path", () => { + const error = new Error( + "EACCES password=plain-secret token=token-secret at /run/secrets/private-token", + ); + error.stack = "Error: raw failure\n at /app/backend/dist/server.js:42:1"; + + const formatted = formatStartupFailure(error); + + expect(formatted).toBe("backend startup failed: startup_unknown"); + for (const leaked of [ + "EACCES", "plain-secret", "token-secret", "/run/secrets", "raw failure", "server.js", + ]) { + expect(formatted).not.toContain(leaked); + } + }); + + it("sanitizes synchronous configuration failures from the real server subprocess", () => { + const root = mkdtempSync(join(tmpdir(), "thothii-startup-secret-")); + const secret = "startup-password-do-not-log"; + const authDirectory = join(root, `auth-${secret}`); + mkdirSync(authDirectory, { mode: 0o700 }); + const authFile = join(authDirectory, "auth.yaml"); + writeFileSync(authFile, `version: 1\nmode: local\npassword: ${secret}\n`, { mode: 0o600 }); + chmodSync(authDirectory, 0o700); + const entrypoint = resolve(process.cwd(), "src/server.ts"); + + const result = spawnSync(process.execPath, ["--import", "tsx", entrypoint], { + cwd: process.cwd(), + encoding: "utf8", + timeout: 15_000, + env: { + ...process.env, + NODE_ENV: "test", + THT_AUTH_CONFIG_FILE: authFile, + THT_AUTH_STATE_ROOT: join(root, "auth-state"), + }, + }); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr.trim()).toBe("backend startup failed: auth_config_invalid"); + expect(`${result.stdout}${result.stderr}`).not.toContain(secret); + expect(`${result.stdout}${result.stderr}`).not.toContain(authDirectory); + expect(`${result.stdout}${result.stderr}`).not.toContain("server.ts"); + }); +}); diff --git a/backend/test/tht-qdrant-readiness.test.ts b/backend/test/tht-qdrant-readiness.test.ts new file mode 100644 index 00000000..1cfc998a --- /dev/null +++ b/backend/test/tht-qdrant-readiness.test.ts @@ -0,0 +1,100 @@ +import { expect, test, vi } from "vitest"; +import { ThtRunner } from "../src/tht/tht-runner.js"; +import type { CanonicalWorkspace } from "../src/workspaces/schema.js"; + +const keywordIndexes = [ + "content_hash", "document_id", "kind", "record_key", "record_kind", + "vector_generation", "workspace_id", "workspace_revision", +]; + +const workspace: CanonicalWorkspace = { + workspace: { schema_version: 3, id: "psd", name: "PSD", language: "it" }, + dwh: { + engine: "postgres", database: "warehouse", schema: "public", + supported_transports: ["postgres_direct"], + }, + semantic_index: { + vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" }, + embedding: { + provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024, + }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, +}; + +function runner(request: (...args: any[]) => Promise) { + return new ThtRunner({ + thtBin: "tht", + harnessDir: "/harness", + configPath: "config/tht.yaml", + semanticRuntime: { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, + }, + qdrantRequest: request, + }); +} + +function response(status: number, body: unknown) { + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + }; +} + +function collection(overrides: Record = {}) { + return { + result: { + config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, + payload_schema: Object.fromEntries(keywordIndexes.map((field) => [field, { data_type: "keyword" }])), + ...overrides, + }, + }; +} + +test("Qdrant readiness uses only the internal URL and accepts the exact collection contract", async () => { + const request = vi.fn(async () => response(200, collection())); + + await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ ok: true, state: "ready" }); + expect(request).toHaveBeenCalledOnce(); + expect(request.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd"); + expect(request.mock.calls[0][1]).toMatchObject({ method: "GET", signal: expect.any(AbortSignal) }); +}); + +test("Qdrant readiness classifies a missing collection as semantic incompatibility", async () => { + const request = vi.fn(async () => response(404, { status: "error", detail: "secret" })); + + await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ + ok: false, + code: "semantic_index_incompatible", + }); +}); + +test.each([ + ["dimensions", collection({ + config: { params: { vectors: { size: 768, distance: "Cosine" } } }, + })], + ["distance", collection({ + config: { params: { vectors: { size: 1024, distance: "Dot" } } }, + })], + ["payload indexes", collection({ payload_schema: { workspace_id: { data_type: "keyword" } } })], +])("Qdrant readiness rejects incompatible %s", async (_label, body) => { + const request = vi.fn(async () => response(200, body)); + + await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ + ok: false, + code: "semantic_index_incompatible", + }); +}); + +test("Qdrant readiness sanitizes unreachable internal service failures", async () => { + const request = vi.fn(async () => { throw new Error("connect http://qdrant:6333/private"); }); + + await expect(runner(request).qdrantEnsure(workspace, 3)).resolves.toEqual({ + ok: false, + code: "workspace_not_activatable", + }); +}); diff --git a/backend/test/tht-runner.test.ts b/backend/test/tht-runner.test.ts index f07a8b4a..e5c9f1ec 100644 --- a/backend/test/tht-runner.test.ts +++ b/backend/test/tht-runner.test.ts @@ -1,24 +1,31 @@ import { test, expect, vi } from "vitest"; import { EventEmitter } from "node:events"; -import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { ThtRunner } from "../src/tht/tht-runner.js"; // Spy on child_process.spawn so we can capture the resolved argv (incl. -c config) // that ThtRunner.run() builds, without launching a real process. -vi.mock("node:child_process", () => ({ - spawn: vi.fn(() => { - const ch: any = new EventEmitter(); - ch.stdout = new EventEmitter(); - ch.stderr = new EventEmitter(); - queueMicrotask(() => { - ch.stdout.emit("data", Buffer.from('{"id":"x"}')); - ch.emit("close", 0); - }); - return ch; - }), -})); +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + spawn: vi.fn(() => { + const ch: any = new EventEmitter(); + ch.stdout = new EventEmitter(); + ch.stderr = new EventEmitter(); + queueMicrotask(() => { + ch.stdout.emit("data", Buffer.from('{"id":"x"}')); + ch.emit("close", 0); + }); + return ch; + }), + }; +}); import { spawn } from "node:child_process"; test("sessionNew parses id from JSON", async () => { @@ -124,19 +131,24 @@ test("run omits ambient THT_DATA_ROOT when config does not provide one", async ( test("principal-bound tht child replaces stale principal env and omits an absent display name", async () => { const saved = Object.fromEntries([ "THT_PRINCIPAL_ISSUER", "THT_PRINCIPAL_SUBJECT", "THT_PRINCIPAL_DISPLAY_NAME", "THT_PRINCIPAL_IS_ADMIN", + "THT_PRINCIPAL_PERMISSIONS", ].map((key) => [key, process.env[key]])); Object.assign(process.env, { THT_PRINCIPAL_ISSUER: "stale-issuer", THT_PRINCIPAL_SUBJECT: "stale-subject", THT_PRINCIPAL_DISPLAY_NAME: "Stale Name", THT_PRINCIPAL_IS_ADMIN: "true", + THT_PRINCIPAL_PERMISSIONS: "pi.manage,unknown.permission", }); try { (spawn as any).mockClear(); const runner = new ThtRunner({ thtBin: "tht", harnessDir: "/h", configPath: "config/tht.yaml" }) - .withPrincipal({ issuer: "portal", subject: "42", isAdmin: false }); + .withPrincipal({ + issuer: "portal", subject: "42", roles: ["user"], permissions: ["session.use"], isAdmin: false, + }); await runner.run(["session", "list", "--json"]); const env = (spawn as any).mock.calls[0][2].env; expect(env).toMatchObject({ THT_PRINCIPAL_ISSUER: "portal", THT_PRINCIPAL_SUBJECT: "42", THT_PRINCIPAL_IS_ADMIN: "false", + THT_PRINCIPAL_PERMISSIONS: "session.use", }); expect(env).not.toHaveProperty("THT_PRINCIPAL_DISPLAY_NAME"); } finally { @@ -169,6 +181,113 @@ test("buildArgv appends -c AFTER the subcommand (never a global -c)", () => { ]); }); +test("buildArgv passes an absolute immutable snapshot after the tht subcommand", () => { + const root = mkdtempSync(join(tmpdir(), "tht-runner-snapshot-")); + const snapshotRoot = join(root, "snapshots", "runtime"); + mkdirSync(snapshotRoot, { recursive: true, mode: 0o700 }); + const r = new ThtRunner({ + thtBin: "tht", harnessDir: "/nope", configPath: "config/tht.yaml", runtimeSnapshotRoot: snapshotRoot, + }); + try { + const snapshot = r.createRuntimeSnapshot("language: en\n"); + expect(lstatSync(snapshot).isFile()).toBe(true); + expect(lstatSync(snapshot).mode & 0o777).toBe(0o400); + expect(r.buildArgv(["session", "new"], snapshot)).toEqual([ + "session", "new", "-c", snapshot, + ]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("absolute config paths must be unmodified runner-created snapshots", () => { + const root = mkdtempSync(join(tmpdir(), "tht-runner-snapshot-")); + const snapshotRoot = join(root, "snapshots", "runtime"); + const outside = join(root, "outside.yaml"); + mkdirSync(snapshotRoot, { recursive: true, mode: 0o700 }); + writeFileSync(outside, "language: en\n"); + const r = new ThtRunner({ + thtBin: "tht", harnessDir: "/nope", configPath: "config/tht.yaml", runtimeSnapshotRoot: snapshotRoot, + }); + try { + expect(() => r.buildArgv(["session", "new"], "/tmp/untrusted.yaml")) + .toThrow(/trusted runtime snapshot/i); + expect(() => r.buildArgv(["session", "new"], outside)) + .toThrow(/trusted runtime snapshot/i); + + const snapshot = r.createRuntimeSnapshot("language: en\n"); + chmodSync(snapshot, 0o600); + writeFileSync(snapshot, "language: it\n"); + chmodSync(snapshot, 0o400); + expect(() => r.buildArgv(["session", "new"], snapshot)) + .toThrow(/trusted runtime snapshot/i); + + const symlink = join(snapshotRoot, "symlink.yaml"); + symlinkSync(outside, symlink); + expect(() => r.buildArgv(["session", "new"], symlink)) + .toThrow(/trusted runtime snapshot/i); + + const directory = join(snapshotRoot, "directory.yaml"); + mkdirSync(directory); + expect(() => r.buildArgv(["session", "new"], directory)) + .toThrow(/trusted runtime snapshot/i); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("runtime snapshots require an absolute configured root", () => { + const relativeRoot = `tht-runner-relative-${Date.now()}`; + const r = new ThtRunner({ + thtBin: "tht", harnessDir: "/nope", configPath: "config/tht.yaml", runtimeSnapshotRoot: relativeRoot, + }); + try { + expect(() => r.createRuntimeSnapshot("language: en\n")).toThrow(/runtime snapshot root/i); + } finally { + rmSync(join(process.cwd(), relativeRoot), { recursive: true, force: true }); + } +}); + +test("runtime snapshots are consumed through a read-only descriptor and cleaned after success", async () => { + const root = mkdtempSync(join(tmpdir(), "tht-runner-snapshot-")); + const snapshotRoot = join(root, "snapshots", "runtime"); + const r = new ThtRunner({ + thtBin: "tht", harnessDir: "/nope", configPath: "config/tht.yaml", runtimeSnapshotRoot: snapshotRoot, + }); + try { + (spawn as any).mockClear(); + await r.runWithRuntimeSnapshot(["session", "list", "--json"], "language: en\n"); + const [, argv, options] = (spawn as any).mock.calls[0]; + expect(argv.slice(-2)).toEqual(["-c", "/dev/fd/3"]); + expect(options.stdio).toHaveLength(4); + expect(readdirSync(snapshotRoot)).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("runtime snapshots are cleaned after a failed child", async () => { + const root = mkdtempSync(join(tmpdir(), "tht-runner-snapshot-")); + const snapshotRoot = join(root, "snapshots", "runtime"); + const r = new ThtRunner({ + thtBin: "tht", harnessDir: "/nope", configPath: "config/tht.yaml", runtimeSnapshotRoot: snapshotRoot, + }); + try { + (spawn as any).mockImplementationOnce(() => { + const ch: any = new EventEmitter(); + ch.stdout = new EventEmitter(); + ch.stderr = new EventEmitter(); + queueMicrotask(() => ch.emit("close", 1)); + return ch; + }); + const result = await r.runWithRuntimeSnapshot(["session", "list", "--json"], "language: en\n"); + expect(result.code).toBe(1); + expect(readdirSync(snapshotRoot)).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("sqlPreview argv has no positional file — uses --session to resolve path", async () => { // The harness preview_cmd now resolves sql_final.sql from the session workspace; // the backend must NOT pass a sessions//sql_final.sql positional arg. diff --git a/backend/test/windows-auth-storage.test.ts b/backend/test/windows-auth-storage.test.ts new file mode 100644 index 00000000..ac8fed9a --- /dev/null +++ b/backend/test/windows-auth-storage.test.ts @@ -0,0 +1,590 @@ +import { appendFileSync, chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { + createPosixAuthStorageBridge, + createWindowsAuthStorageBridge, +} from "../src/auth/windows-auth-storage.js"; + +const root = "C:\\ProgramData\\ThothII\\auth"; +const posixRoot = "/var/lib/thothii/auth"; +const filename = "a".repeat(64) + ".json"; +const realChildFixture = fileURLToPath(new URL("./fixtures/windows-auth-storage-real-child.mjs", import.meta.url)); +const fixtureRoots: string[] = []; + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'`; +} + +async function waitForMarker(marker: string, expected: string): Promise { + const deadline = Date.now() + 3_000; + while (Date.now() < deadline) { + if (existsSync(marker) && readFileSync(marker, "utf8").includes(expected)) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`real helper marker did not contain ${expected}`); +} + +function realChildBridge( + mode: "timeout" | "stdout" | "stderr" | "stdin", + pathStyle: "windows" | "posix", +) { + const directory = mkdtempSync(join(tmpdir(), "thothii-auth-bridge-child-")); + fixtureRoots.push(directory); + const marker = join(directory, "marker.txt"); + const launcher = join(directory, "tht.exe"); + writeFileSync(launcher, `#!/bin/sh\nexec ${shellQuote(process.execPath)} ${shellQuote(realChildFixture)} ${shellQuote(mode)} ${shellQuote(marker)} "$@"\n`, { mode: 0o700 }); + chmodSync(launcher, 0o700); + const factory = pathStyle === "windows" ? createWindowsAuthStorageBridge : createPosixAuthStorageBridge; + return { + marker, + bridge: factory({ + thtExecutable: pathStyle === "windows" ? "C:\\tht.exe" : launcher, + spawnChild: (_executable, args, options) => spawn(launcher, [...args], options), + // Leave enough startup headroom for a real child under a busy CI host while retaining a + // sub-1.5-second bound from request start through final settlement. + deadlinesForTest: { timeoutMs: 750, terminationGraceMs: 50, finalSettlementMs: 500 }, + ...(mode === "stdin" ? { + beforeInputForTest: async () => { + await waitForMarker(marker, "stdin-closed"); + appendFileSync(marker, "before-input\n"); + }, + } : {}), + } as never), + }; +} + +afterEach(() => { + for (const directory of fixtureRoots.splice(0)) { + const marker = join(directory, "marker.txt"); + try { + const pid = Number(/^started:(\d+)$/m.exec(readFileSync(marker, "utf8"))?.[1]); + if (Number.isSafeInteger(pid) && pid > 0) process.kill(pid, "SIGKILL"); + } catch { /* the test-owned child already exited or did not start */ } + rmSync(directory, { recursive: true, force: true }); + } +}); + +class FakeBridgeChild extends EventEmitter { + readonly stdin = new PassThrough(); + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); + readonly kill = vi.fn(() => true); + readonly unref = vi.fn(); + + close(code = 0, signal: NodeJS.Signals | null = null): void { + this.emit("close", code, signal); + } +} + +function bridgeForChild(child: FakeBridgeChild, pathStyle: "windows" | "posix" = "windows") { + const spawnChild = vi.fn(() => child); + const factory = pathStyle === "windows" ? createWindowsAuthStorageBridge : createPosixAuthStorageBridge; + const bridge = factory({ + thtExecutable: pathStyle === "windows" ? "C:\\tht.exe" : "/opt/thothii/bin/tht", + spawnChild, + } as never); + return { bridge, spawnChild }; +} + +describe("Windows auth-storage bridge", () => { + test("uses a dedicated native storage executable without replacing the harness tht", async () => { + vi.stubEnv("THT_BIN", "/opt/venv/bin/tht"); + vi.stubEnv("THT_AUTH_STORAGE_BIN", "/usr/local/bin/tht-auth-storage"); + const calls: Array<{ executable: string }> = []; + const bridge = createPosixAuthStorageBridge({ + invoke: async (call) => { + calls.push(call); + return { + code: 0, + stdout: Buffer.from('{"version":1,"ok":true,"prepared":true}\n'), + stderr: Buffer.alloc(0), + }; + }, + }); + + await bridge.ensureLayout("/data/auth"); + + expect(calls).toHaveLength(1); + expect(calls[0]!.executable).toBe("/usr/local/bin/tht-auth-storage"); + expect(process.env.THT_BIN).toBe("/opt/venv/bin/tht"); + vi.unstubAllEnvs(); + }); + + test("uses the same bounded hidden bridge to ensure a POSIX session layout", async () => { + const calls: Array<{ executable: string; args: readonly string[]; input: Buffer; timeoutMs: number }> = []; + const bridge = createPosixAuthStorageBridge({ + thtExecutable: "/opt/thothii/bin/tht", + invoke: async (call) => { + calls.push(call); + return { + code: 0, + stdout: Buffer.from('{"version":1,"ok":true,"prepared":true}\n'), + stderr: Buffer.alloc(0), + }; + }, + }); + + await expect(bridge.ensureLayout("/var/lib/thothii/auth")).resolves.toBeUndefined(); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ + executable: "/opt/thothii/bin/tht", + args: ["_auth-storage"], + timeoutMs: 5_000, + }); + expect(JSON.parse(calls[0]!.input.toString("utf8"))).toEqual({ + version: 1, + operation: "ensure-layout", + root: "/var/lib/thothii/auth", + }); + expect(JSON.stringify(calls[0]!.args)).not.toContain("/var/lib/thothii/auth"); + await expect(bridge.ensureLayout("/var/lib/thothii/../auth")) + .rejects.toThrow("auth_session_store_invalid"); + }); + + test("permits reservation slots only for OIDC record operations", async () => { + const requests: Array> = []; + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async ({ input }) => { + const request = JSON.parse(input.toString("utf8")) as Record; + requests.push(request); + const operation = request.operation; + const body = operation === "create" + ? { created: true } + : operation === "read" + ? { found: true, contentBase64: Buffer.from("slot").toString("base64") } + : operation === "remove" + ? { removed: true } + : { entries: [{ name: "slot-00.json", modifiedUnixMs: 1 }] }; + return { + code: 0, + stdout: Buffer.from(`${JSON.stringify({ version: 1, ok: true, ...body })}\n`), + stderr: Buffer.alloc(0), + }; + }, + }); + + await expect(bridge.create("C:\\auth", "oidc", "slot-00.json", Buffer.from("slot"))).resolves.toBe(true); + await expect(bridge.read("C:\\auth", "oidc", "slot-00.json")).resolves.toEqual(Buffer.from("slot")); + await expect(bridge.list("C:\\auth", "oidc")).resolves.toEqual([ + { name: "slot-00.json", modifiedUnixMs: 1 }, + ]); + await expect(bridge.remove("C:\\auth", "oidc", "slot-00.json")).resolves.toBe(true); + await expect(bridge.create("C:\\auth", "sessions", "slot-00.json", Buffer.from("slot"))) + .rejects.toThrow("auth_session_store_invalid"); + await expect(bridge.create("C:\\auth", "oidc", "slot-64.json", Buffer.from("slot"))) + .rejects.toThrow("auth_session_store_invalid"); + expect(requests).toHaveLength(4); + }); + + test("uses hidden tht argv and sends record bytes only over bounded stdin", async () => { + const calls: Array<{ executable: string; args: readonly string[]; input: Buffer; timeoutMs: number }> = []; + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\Program Files\\ThothII\\tht.exe", + invoke: async (call) => { + calls.push(call); + return { code: 0, stdout: Buffer.from('{"version":1,"ok":true,"created":true}\n'), stderr: Buffer.alloc(0) }; + }, + }); + + await expect(bridge.create(root, "sessions", filename, Buffer.from('{"subject":"record-data"}'))).resolves.toBe(true); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ + executable: "C:\\Program Files\\ThothII\\tht.exe", + args: ["_auth-storage"], + }); + expect(JSON.stringify(calls[0].args)).not.toContain("record-data"); + expect(JSON.parse(calls[0].input.toString("utf8"))).toMatchObject({ + version: 1, + operation: "create", + root, + directory: "sessions", + filename, + contentBase64: Buffer.from('{"subject":"record-data"}').toString("base64"), + }); + expect(calls[0].timeoutMs).toBeGreaterThan(0); + }); + + test("validates Windows roots and reads auth.yaml through the same bounded hidden bridge", async () => { + const asyncCalls: Array> = []; + const syncCalls: Array<{ args: readonly string[]; input: Buffer; timeoutMs: number; maximumOutputBytes: number }> = []; + const config = Buffer.from("version: 1\nmode: local\n"); + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\Program Files\\ThothII\\tht.exe", + invoke: async ({ input }) => { + asyncCalls.push(JSON.parse(input.toString("utf8")) as Record); + return { code: 0, stdout: Buffer.from('{"version":1,"ok":true,"validated":true}\n'), stderr: Buffer.alloc(0) }; + }, + invokeSync: (call: { args: readonly string[]; input: Buffer; timeoutMs: number; maximumOutputBytes: number }) => { + syncCalls.push(call); + return { + code: 0, + stdout: Buffer.from(`${JSON.stringify({ version: 1, ok: true, found: true, contentBase64: config.toString("base64") })}\n`), + stderr: Buffer.alloc(0), + }; + }, + } as never) as unknown as { + validateRoot(root: string): Promise; + readAuthConfig(path: string): Buffer; + }; + + await expect(bridge.validateRoot(root)).resolves.toBeUndefined(); + expect(bridge.readAuthConfig(`${root}\\auth.yaml`)).toEqual(config); + expect(asyncCalls).toEqual([{ version: 1, operation: "validate-root", root }]); + expect(JSON.parse(syncCalls[0]!.input.toString("utf8"))).toEqual({ + version: 1, operation: "read-auth-config", root, filename: "auth.yaml", + }); + expect(syncCalls[0]!.args).toEqual(["_auth-storage"]); + expect(syncCalls[0]!.timeoutMs).toBe(5_000); + expect(syncCalls[0]!.maximumOutputBytes).toBeGreaterThan(1024 * 1024); + expect(JSON.stringify(syncCalls[0]!.args)).not.toContain(root); + expect(JSON.stringify(syncCalls[0]!.args)).not.toContain(config.toString("utf8")); + }); + + test("reads native Windows users.yaml only through a bounded hidden bridge request", async () => { + const users = Buffer.from("version: 1\nusers:\n - passwordHash: not-in-argv\n", "utf8"); + const calls: Array<{ args: readonly string[]; input: Buffer; maximumOutputBytes: number }> = []; + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\Program Files\\ThothII\\tht.exe", + invoke: async (call) => { + calls.push(call); + return { + code: 0, + stdout: Buffer.from(`${JSON.stringify({ + version: 1, ok: true, found: true, contentBase64: users.toString("base64"), + })}\n`), + stderr: Buffer.alloc(0), + }; + }, + }); + + await expect(bridge.readLocalUsers(`${root}\\users.yaml`)).resolves.toEqual(users); + expect(calls).toHaveLength(1); + expect(calls[0]!.args).toEqual(["_auth-storage"]); + expect(calls[0]!.maximumOutputBytes).toBeGreaterThan(1024 * 1024); + expect(JSON.parse(calls[0]!.input.toString("utf8"))).toEqual({ + version: 1, operation: "read-local-users", root, filename: "users.yaml", + }); + expect(JSON.stringify(calls[0]!.args)).not.toContain("not-in-argv"); + expect(calls[0]!.input.toString("utf8")).not.toContain("not-in-argv"); + }); + + test.each([ + { label: "nonzero", result: { code: 1, stdout: Buffer.from('{"version":1,"ok":true}\n'), stderr: Buffer.from("secret") } }, + { label: "malformed stdout", result: { code: 0, stdout: Buffer.from("not-json"), stderr: Buffer.alloc(0) } }, + { label: "unexpected stdout", result: { code: 0, stdout: Buffer.from('{"version":1,"ok":true}\nextra'), stderr: Buffer.alloc(0) } }, + { label: "unexpected JSON field", result: { code: 0, stdout: Buffer.from('{"version":1,"ok":true,"created":true,"detail":"secret"}\n'), stderr: Buffer.alloc(0) } }, + ])("fails closed on $label bridge output", async ({ result }) => { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => result, + }); + + await expect(bridge.create(root, "sessions", filename, Buffer.from("record"))) + .rejects.toThrow("auth_session_store_invalid"); + }); + + test("fails closed on a bridge timeout without disclosing request content", async () => { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => { throw new Error("timeout secret-record"); }, + }); + + await expect(bridge.create(root, "sessions", filename, Buffer.from("secret-record"))) + .rejects.toThrow("auth_session_store_invalid"); + }); + + test("rejects a claimed-read response without bounded record bytes", async () => { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => ({ code: 0, stdout: Buffer.from('{"version":1,"ok":true,"found":true}\n'), stderr: Buffer.alloc(0) }), + }); + + await expect(bridge.readClaim(root, filename)).rejects.toThrow("auth_session_store_invalid"); + }); + + test("rejects an executable value that would require shell parsing", () => { + expect(() => createWindowsAuthStorageBridge({ thtExecutable: "tht.exe && unexpected" })) + .toThrow("auth_session_store_invalid"); + }); + + test("parses the lower-camel list DTO emitted by the Go helper for a nonempty directory", async () => { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => ({ + code: 0, + // This is the raw JSON object emitted by authstorage.response after Go's DTO encoding. + stdout: Buffer.from(`{"version":1,"ok":true,"entries":[{"name":"${filename}","modifiedUnixMs":1893456245000}]}\n`), + stderr: Buffer.alloc(0), + }), + }); + + await expect(bridge.list(root, "oidc")).resolves.toEqual([ + { name: filename, modifiedUnixMs: 1_893_456_245_000 }, + ]); + }); + + test("passes an explicit bounded directory limit to the Go helper", async () => { + const invoke = vi.fn(async () => ({ + code: 0, + stdout: Buffer.from('{"version":1,"ok":true,"entries":[]}\n'), + stderr: Buffer.alloc(0), + })); + const bridge = createWindowsAuthStorageBridge({ thtExecutable: "C:\\tht.exe", invoke }); + + await expect(bridge.list(root, "oidc", 192)).resolves.toEqual([]); + expect(JSON.parse(invoke.mock.calls[0][0].input.toString("utf8"))).toMatchObject({ + operation: "list", + directory: "oidc", + maximumEntries: 192, + }); + }); + + test("uses a strict, bounded continuation page for ordinary Windows session maintenance", async () => { + const invoke = vi.fn(async () => ({ + code: 0, + stdout: Buffer.from(`{"version":1,"ok":true,"entries":[{"name":"${filename}","modifiedUnixMs":1}],"more":false}\n`), + stderr: Buffer.alloc(0), + })); + const bridge = createWindowsAuthStorageBridge({ thtExecutable: "C:\\tht.exe", invoke }) as unknown as { + listPage(root: string, directory: "sessions", after: string | undefined, maximumEntries: number): Promise<{ + entries: Array<{ name: string; modifiedUnixMs: number }>; + more: boolean; + }>; + }; + + await expect(bridge.listPage(root, "sessions", "0".repeat(64) + ".json", 512)).resolves.toEqual({ + entries: [{ name: filename, modifiedUnixMs: 1 }], + more: false, + }); + expect(JSON.parse(invoke.mock.calls[0][0].input.toString("utf8"))).toMatchObject({ + operation: "list", + directory: "sessions", + maximumEntries: 512, + continuation: true, + afterName: "0".repeat(64) + ".json", + }); + }); + + test("rejects ambiguous ordinary-session continuation responses", async () => { + const after = "f".repeat(64) + ".json"; + const low = "a".repeat(64) + ".json"; + const high = "b".repeat(64) + ".json"; + const cases = [ + { label: "missing more marker", body: { entries: [{ name: filename, modifiedUnixMs: 1 }] } }, + { label: "more without a full page", body: { entries: [{ name: filename, modifiedUnixMs: 1 }], more: true } }, + { label: "non-progressing name", body: { entries: [{ name: low, modifiedUnixMs: 1 }], more: false } }, + { label: "duplicate names", body: { entries: [{ name: high, modifiedUnixMs: 1 }, { name: high, modifiedUnixMs: 2 }], more: false } }, + ]; + + for (const { body } of cases) { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => ({ + code: 0, + stdout: Buffer.from(`${JSON.stringify({ version: 1, ok: true, ...body })}\n`), + stderr: Buffer.alloc(0), + }), + }) as unknown as { + listPage(root: string, directory: "sessions", afterName: string | undefined, maximumEntries: number): Promise; + }; + await expect(bridge.listPage(root, "sessions", after, 512)).rejects.toThrow("auth_session_store_invalid"); + } + }); + + test("accepts a bounded ordinary-session page larger than the legacy 256-entry limit", async () => { + const entries = Array.from({ length: 300 }, (_unused, index) => ({ + name: `${index.toString(16).padStart(64, "0")}.json`, + modifiedUnixMs: 1_893_456_245_000, + })); + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => ({ + code: 0, + stdout: Buffer.from(`${JSON.stringify({ version: 1, ok: true, entries })}\n`), + stderr: Buffer.alloc(0), + }), + }); + + await expect(bridge.list(root, "sessions", 300)).resolves.toHaveLength(300); + }); + + test("parses the Go helper's required empty entries array", async () => { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => ({ + code: 0, + stdout: Buffer.from('{"version":1,"ok":true,"entries":[]}\n'), + stderr: Buffer.alloc(0), + }), + }); + + await expect(bridge.list(root, "sessions")).resolves.toEqual([]); + }); + + test("allows only canonical OIDC claim removal and rejects claims elsewhere", async () => { + const claim = `${"b".repeat(64)}.claim`; + const invoke = vi.fn(async () => ({ + code: 0, + stdout: Buffer.from('{"version":1,"ok":true,"removed":true}\n'), + stderr: Buffer.alloc(0), + })); + const bridge = createWindowsAuthStorageBridge({ thtExecutable: "C:\\tht.exe", invoke }); + + await expect(bridge.remove(root, "oidc", claim)).resolves.toBe(true); + await expect(bridge.remove(root, "sessions", claim)).rejects.toThrow("auth_session_store_invalid"); + await expect(bridge.read(root, "oidc", claim)).rejects.toThrow("auth_session_store_invalid"); + await expect(bridge.create(root, "oidc", claim, Buffer.from("record"))).rejects.toThrow("auth_session_store_invalid"); + await expect(bridge.replace(root, "oidc", claim, Buffer.from("record"))).rejects.toThrow("auth_session_store_invalid"); + await expect(bridge.remove(root, "oidc", `../${claim}`)).rejects.toThrow("auth_session_store_invalid"); + await expect(bridge.remove(root, "oidc", `${claim}.bak`)).rejects.toThrow("auth_session_store_invalid"); + expect(invoke).toHaveBeenCalledTimes(1); + }); + + test("rejects OIDC claim entries returned for a sessions list", async () => { + const claim = `${"c".repeat(64)}.claim`; + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + invoke: async () => ({ + code: 0, + stdout: Buffer.from(`{"version":1,"ok":true,"entries":[{"name":"${claim}","modifiedUnixMs":1}]}\n`), + stderr: Buffer.alloc(0), + }), + }); + + await expect(bridge.list(root, "sessions")).rejects.toThrow("auth_session_store_invalid"); + }); + + test("settles a stdin-closed looping helper by a final deadline when close never arrives", async () => { + vi.useFakeTimers(); + const child = new FakeBridgeChild(); + const { bridge, spawnChild } = bridgeForChild(child); + const pending = bridge.list(root, "sessions"); + const outcome = pending.then(() => "resolved", () => "rejected"); + try { + await vi.advanceTimersByTimeAsync(5_000); + expect(spawnChild).toHaveBeenCalledOnce(); + expect(child.kill).toHaveBeenCalledOnce(); + expect(child.unref).toHaveBeenCalledOnce(); + expect(child.stdin.destroyed).toBe(true); + await expect(Promise.race([outcome, Promise.resolve("pending")])).resolves.toBe("pending"); + + await vi.advanceTimersByTimeAsync(1_000); + expect(child.kill).toHaveBeenCalledTimes(2); + await expect(outcome).resolves.toBe("rejected"); + expect(() => child.emit("error", new Error("late helper failure"))).not.toThrow(); + expect(() => child.stdin.emit("error", new Error("late stdin failure"))).not.toThrow(); + } finally { + child.close(); + vi.useRealTimers(); + } + }); + + test("releases bounded late-error guards when a finally-dead helper closes", async () => { + vi.useFakeTimers(); + const child = new FakeBridgeChild(); + const { bridge } = bridgeForChild(child); + const outcome = bridge.list(root, "sessions").then(() => "resolved", () => "rejected"); + try { + await vi.advanceTimersByTimeAsync(6_000); + await expect(outcome).resolves.toBe("rejected"); + expect(child.listenerCount("error")).toBe(1); + expect(child.stdin.listenerCount("error")).toBe(1); + expect(child.stdout.listenerCount("error")).toBe(1); + expect(child.stderr.listenerCount("error")).toBe(1); + + child.close(); + expect(child.listenerCount("error")).toBe(0); + expect(child.stdin.listenerCount("error")).toBe(0); + expect(child.stdout.listenerCount("error")).toBe(0); + expect(child.stderr.listenerCount("error")).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + + test.each(["stdout", "stderr"] as const)("aborts a %s-flooding helper and waits for close", async (stream) => { + const child = new FakeBridgeChild(); + const { bridge, spawnChild } = bridgeForChild(child); + const pending = bridge.list(root, "sessions"); + const outcome = pending.then(() => "resolved", () => "rejected"); + await Promise.resolve(); + expect(spawnChild).toHaveBeenCalledOnce(); + + child[stream].write(Buffer.alloc(64 * 1024 + 1)); + await Promise.resolve(); + expect(child.kill).toHaveBeenCalledOnce(); + expect(child.stdin.destroyed).toBe(true); + await expect(Promise.race([outcome, Promise.resolve("pending")])).resolves.toBe("pending"); + + child.close(); + await expect(outcome).resolves.toBe("rejected"); + }); + + test("aborts a helper when its stdin errors and waits for termination", async () => { + const child = new FakeBridgeChild(); + const { bridge, spawnChild } = bridgeForChild(child); + const stdinErrored = new Promise((resolve) => child.stdin.once("error", () => resolve())); + child.stdin.once("finish", () => child.stdin.destroy(new Error("stdin failure"))); + const pending = bridge.list(root, "sessions"); + const outcome = pending.then(() => "resolved", () => "rejected"); + await Promise.resolve(); + expect(spawnChild).toHaveBeenCalledOnce(); + + await stdinErrored; + expect(child.kill).toHaveBeenCalledOnce(); + child.close(); + await expect(outcome).resolves.toBe("rejected"); + }); + + test("aborts an errored child exactly once and waits for its close event", async () => { + const child = new FakeBridgeChild(); + const { bridge, spawnChild } = bridgeForChild(child); + const pending = bridge.list(root, "sessions"); + const outcome = pending.then(() => "resolved", () => "rejected"); + await Promise.resolve(); + expect(spawnChild).toHaveBeenCalledOnce(); + + child.emit("error", new Error("helper error")); + child.emit("error", new Error("duplicate helper error")); + expect(child.kill).toHaveBeenCalledOnce(); + await expect(Promise.race([outcome, Promise.resolve("pending")])).resolves.toBe("pending"); + child.close(); + await expect(outcome).resolves.toBe("rejected"); + expect(() => child.emit("error", new Error("late helper failure"))).not.toThrow(); + expect(() => child.stdin.emit("error", new Error("late stdin failure"))).not.toThrow(); + expect(() => child.stdout.emit("error", new Error("late stdout failure"))).not.toThrow(); + expect(() => child.stderr.emit("error", new Error("late stderr failure"))).not.toThrow(); + }); + + test("fails closed when launching the helper throws before a child exists", async () => { + const bridge = createWindowsAuthStorageBridge({ + thtExecutable: "C:\\tht.exe", + spawnChild: () => { throw new Error("launch detail must not escape"); }, + }); + + await expect(bridge.list(root, "sessions")).rejects.toThrow("auth_session_store_invalid"); + }); + + test.each([ + ...(["windows", "posix"] as const).flatMap((pathStyle) => + (["timeout", "stdout", "stderr", "stdin"] as const).map((mode) => ({ pathStyle, mode }))), + ])("settles a real $pathStyle $mode helper within the production deadline", async ({ pathStyle, mode }) => { + const { bridge, marker } = realChildBridge(mode, pathStyle); + const startedAt = Date.now(); + const pending = bridge.list(pathStyle === "windows" ? root : posixRoot, "sessions"); + const outcome = pending.then(() => undefined, (error: unknown) => error); + await waitForMarker(marker, "started"); + if (mode === "stdin") await waitForMarker(marker, "before-input"); + + await expect(outcome).resolves.toMatchObject({ message: "auth_session_store_invalid" }); + await waitForMarker(marker, "terminated"); + expect(Date.now() - startedAt).toBeLessThan(1_500); + }, 5_000); +}); diff --git a/backend/test/workspace-maintenance.test.ts b/backend/test/workspace-maintenance.test.ts new file mode 100644 index 00000000..b9422a34 --- /dev/null +++ b/backend/test/workspace-maintenance.test.ts @@ -0,0 +1,114 @@ +import { expect, test, vi } from "vitest"; +import { + runWorkspaceMaintenanceCli, + type WorkspaceMaintenanceIo, +} from "../src/workspace-maintenance.js"; +import type { WorkspaceOperationResult } from "../src/workspaces/preprocessing-service.js"; + +function io(stdin: string): WorkspaceMaintenanceIo & { stdout: string[]; stderr: string[] } { + const stdout: string[] = []; + const stderr: string[] = []; + return { + stdin, + stdout, + stderr, + writeStdout: (value) => void stdout.push(value), + writeStderr: (value) => void stderr.push(value), + }; +} + +function ok(operation: string): WorkspaceOperationResult { + return { + schemaVersion: 1, + status: "succeeded", + code: "ok", + workspaceId: "psd-clinical", + workspaceRevision: "a".repeat(40), + descriptorBlob: "b".repeat(40), + operation, + completedStages: [], + }; +} + +test("entrypoint emits exactly one pristine JSON document and maps success/block/failure exits", async () => { + const service = { + inspect: vi.fn(async () => ok("inspect")), + preprocessDwh: vi.fn(async () => ({ ...ok("preprocess dwh"), status: "blocked", code: "manual_review_required" as const })), + run: vi.fn(async () => ({ ...ok("preprocess run"), status: "failed", code: "semantic_index_incompatible" as const })), + } as any; + + const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "inspect"], service, inspectIo)).toBe(0); + expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "inspect", code: "ok" }); + expect(inspectIo.stderr.join("")).toBe(""); + + const blockedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-dwh"], service, blockedIo)).toBe(3); + expect(JSON.parse(blockedIo.stdout.join(""))).toMatchObject({ code: "manual_review_required" }); + + const failedIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "preprocess-run"], service, failedIo)).toBe(1); + expect(JSON.parse(failedIo.stdout.join(""))).toMatchObject({ code: "semantic_index_incompatible" }); +}); + +test("malformed stdin, unknown commands, and extra fields fail with exit 2 but still return bounded JSON", async () => { + const service = {} as any; + + const malformedIo = io("not-json"); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "inspect"], service, malformedIo)).toBe(2); + expect(JSON.parse(malformedIo.stdout.join(""))).toMatchObject({ status: "failed", operation: "inspect" }); + + const extraFieldIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", unexpected: true })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "inspect"], service, extraFieldIo)).toBe(2); + expect(JSON.parse(extraFieldIo.stdout.join(""))).toMatchObject({ status: "failed", operation: "inspect" }); + + const unknownIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "explode"], service, unknownIo)).toBe(2); + expect(JSON.parse(unknownIo.stdout.join(""))).toMatchObject({ status: "failed", operation: "explode" }); +}); + +test("raw exception text is redacted from stderr and stdout remains within the public result contract", async () => { + const service = { + inspect: vi.fn(async () => { + throw new Error("https://secret.example.invalid?q=token SELECT * FROM sensitive_table"); + }), + } as any; + const captured = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" })); + + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "inspect"], service, captured)).toBe(1); + expect(JSON.parse(captured.stdout.join(""))).toMatchObject({ + status: "failed", + operation: "inspect", + }); + expect(captured.stderr.join("")).not.toContain("secret.example.invalid"); + expect(captured.stderr.join("")).not.toContain("SELECT *"); +}); + +test("vector-inspect and vector-rebuild dispatch to the service with the exact envelope", async () => { + const service = { + vectorInspect: vi.fn(async () => ok("vector inspect")), + vectorRebuild: vi.fn(async () => ok("vector rebuild")), + } as any; + + const inspectIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical" })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-inspect"], service, inspectIo)).toBe(0); + expect(service.vectorInspect).toHaveBeenCalledWith({ workspaceId: "psd-clinical" }); + expect(JSON.parse(inspectIo.stdout.join(""))).toMatchObject({ operation: "vector inspect", code: "ok" }); + + const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(0); + expect(service.vectorRebuild).toHaveBeenCalledWith({ + workspaceId: "psd-clinical", + collection: "psd-clinical", + confirm: "psd-clinical", + destroy: true, + }); +}); + +test("vector-rebuild without exact confirmation is refused by the service", async () => { + const service = { + vectorRebuild: vi.fn(async () => ({ ...ok("vector rebuild"), status: "failed", code: "semantic_index_incompatible" as const })), + } as any; + const rebuildIo = io(JSON.stringify({ schemaVersion: 1, workspaceId: "psd-clinical", collection: "other", confirm: "other", destroy: true })); + expect(await runWorkspaceMaintenanceCli(["node", "workspace-maintenance", "vector-rebuild"], service, rebuildIo)).toBe(1); +}); diff --git a/backend/test/workspace-preprocessing-service.test.ts b/backend/test/workspace-preprocessing-service.test.ts new file mode 100644 index 00000000..2055b96b --- /dev/null +++ b/backend/test/workspace-preprocessing-service.test.ts @@ -0,0 +1,527 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; +import { PreprocessingStateStore } from "../src/workspaces/preprocessing-state.js"; +import { syncAnnotations } from "../src/workspaces/annotations-sync.js"; +import { + WorkspacePreprocessingService, + type ChildProcessRequest, +} from "../src/workspaces/preprocessing-service.js"; + +const roots: string[] = []; + +afterEach(() => { + roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +const semanticRuntime = { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, +}; + +const baseWorkspace = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Runtime Lease + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`); +const filesystemWorkspace = parseWorkspaceYaml(`${baseWorkspace ? '' : ''}workspace: + schema_version: 3 + id: fs-workspace + name: Filesystem + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: fs-workspace + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +evidence: + source: + type: filesystem + uri: fs-workspace/evidence +`); +const privateHttpWorkspace = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: http-workspace + name: Http + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: http-workspace + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +evidence: + source: + type: http + uris: [http://127.0.0.1/private.md] + authentication: none + connect_timeout_ms: 1000 + read_timeout_ms: 2000 + max_bytes: 100 + max_redirects: 0 + allow_private_hosts: true + max_cache_bytes: 100 +`); + +function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id) { + return { + workspace, + workspaceId, + workspaceRevision: "a".repeat(40), + descriptorBlob: "b".repeat(40), + catalogBlob: "c".repeat(40), + configLease: { + path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}-identitysuffix.yaml`, + workspaceId, + workspaceRevision: "a".repeat(40), + descriptorBlob: "b".repeat(40), + catalogBlob: "c".repeat(40), + configDigest: "sha256:config", + bindingDigest: "sha256:bindings", + semanticQdrantUrl: "http://qdrant:6333", + effectiveConfig: { + schemaVersion: 1, + dwh: { + engine: "postgres", + database: "analytics", + schema: "mart", + transport: "postgres_direct", + host: "dwh.internal", + port: 5432, + user: "reader", + }, + vector: { collection: workspaceId, dimensions: 1024, distance: "cosine" }, + embedding: { model: "qwen3-embedding:0.6b", dimensions: 1024 }, + roots: { artifacts: "/data/artifacts", indexes: "/data/indexes" }, + }, + effectiveConfigIdentity: "workspace://psd-clinical@v1:" + "d".repeat(64), + configFingerprint: "sha256:" + "e".repeat(64), + inputFingerprint: "sha256:" + "f".repeat(64), + release: () => undefined, + }, + }; +} + +function fixture(workspace = baseWorkspace) { + const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-")); + roots.push(dataRoot); + const requests: ChildProcessRequest[] = []; + const runChild = vi.fn(async (request: ChildProcessRequest) => { + requests.push(request); + return { exitCode: 0, stdout: JSON.stringify({ status: "succeeded" }), stderr: "" }; + }); + const service = new WorkspacePreprocessingService({ + dataRoot, + acquireActiveRuntime: async () => runtime(workspace), + runChild, + listSessions: async () => [], + semanticPreflight: async () => ({ ok: true }), + }); + return { dataRoot, runChild, requests, service }; +} + +test("preprocess dwh uses fixed argv and resumes outer state without rerunning a completed stage", async () => { + const f = fixture(); + f.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), + stderr: "", + }); + + const first = await f.service.preprocessDwh({ workspaceId: "psd-clinical" }); + expect(first).toMatchObject({ + status: "succeeded", + code: "ok", + operation: "preprocess dwh", + completedStages: ["dwh"], + childRuns: { dwh: "d".repeat(32) }, + }); + expect((f.runChild.mock.calls[0]![0] as ChildProcessRequest).argv).toEqual([ + "preprocess", "dwh", "--steps", "introspect,lsh", "--json", "-c", "/dev/fd/3", + ]); + + const second = await f.service.preprocessDwh({ + workspaceId: "psd-clinical", + resumeRunId: first.runId!, + }); + expect(second.status).toBe("unchanged"); + expect(f.runChild).toHaveBeenCalledTimes(1); +}); + +test("schema suggest-fks publishes a candidate artifact and blocks full runs for manual review", async () => { + const f = fixture(); + f.runChild + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), + stderr: "", + }) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + candidate_count: 1, + candidate_digest: "sha256:" + "e".repeat(64), + candidate_yaml: "tables: []\n", + }), + stderr: "", + }); + + const result = await f.service.run({ workspaceId: "psd-clinical" }); + expect(result).toMatchObject({ + status: "blocked", + code: "manual_review_required", + completedStages: ["dwh", "fk_suggest"], + }); + expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv[0])).toEqual(["preprocess", "schema"]); +}); + +test("schema check requires the exact candidate digest and stages annotations via a temp file without recording a review", async () => { + const f = fixture(); + f.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + candidate_count: 1, + candidate_digest: "sha256:" + "e".repeat(64), + candidate_yaml: "tables: []\n", + }), + stderr: "", + }); + const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" }); + await expect(f.service.checkSchema({ + workspaceId: "psd-clinical", + annotationsYaml: "tables: {}\n", + reviewedCandidatesDigest: "sha256:" + "f".repeat(64), + })).resolves.toMatchObject({ status: "failed", code: "annotation_invalid" }); + + const reviewedDigest = suggest.artifactIdentities![0]!.digest; + let stagedPath = ""; + f.runChild.mockImplementationOnce(async (request: ChildProcessRequest) => { + stagedPath = request.argv[request.argv.indexOf("--annotations") + 1]!; + expect(readFileSync(stagedPath, "utf8")).toBe("tables: {}\n"); + expect(request.argv).toEqual([ + "schema", "check", "--annotations", stagedPath, + "--reviewed-candidates", reviewedDigest, + "--json", "-c", "/dev/fd/3", + ]); + return { + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + orphan_count: 0, + annotations_digest: "sha256:annotations", + reviewed_candidates_digest: reviewedDigest, + }), + stderr: "", + }; + }); + + const checked = await f.service.checkSchema({ + workspaceId: "psd-clinical", + annotationsYaml: "tables: {}\n", + reviewedCandidatesDigest: reviewedDigest, + }); + expect(checked).toMatchObject({ status: "succeeded", code: "ok" }); + expect(() => readFileSync(stagedPath, "utf8")).toThrow(); + const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" }); + expect(state.readFkReview(suggest.runId!)).toBeUndefined(); +}); + +test("index schema fails closed when semantic preflight refuses the collection", async () => { + const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-")); + roots.push(dataRoot); + const runChild = vi.fn(); + const service = new WorkspacePreprocessingService({ + dataRoot, + acquireActiveRuntime: async () => runtime(baseWorkspace), + runChild, + listSessions: async () => [], + semanticPreflight: async () => ({ ok: false, code: "semantic_index_incompatible" }), + }); + + const result = await service.indexSchema({ workspaceId: "psd-clinical" }); + expect(result).toMatchObject({ status: "failed", code: "semantic_index_incompatible" }); + expect(runChild).not.toHaveBeenCalled(); +}); + +test("filesystem Evidence proceeds after materialization and private HTTP hosts outside the allowlist are refused", async () => { + const filesystem = fixture(filesystemWorkspace); + filesystem.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", counts: { added: 1 } }), + stderr: "", + }); + const materialized = await filesystem.service.preprocessEvidence({ workspaceId: "fs-workspace" }); + expect(materialized).toMatchObject({ status: "succeeded", code: "ok" }); + expect(filesystem.runChild).toHaveBeenCalledTimes(1); + + const httpDataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-service-")); + roots.push(httpDataRoot); + const httpService = new WorkspacePreprocessingService({ + dataRoot: httpDataRoot, + acquireActiveRuntime: async () => runtime(privateHttpWorkspace, "http-workspace"), + runChild: vi.fn(), + listSessions: async () => [], + semanticPreflight: async () => ({ ok: true }), + httpPrivateHostAllowlist: ["metadata.internal"], + }); + + const refused = await httpService.preprocessEvidence({ workspaceId: "http-workspace" }); + expect(refused).toMatchObject({ status: "failed", code: "egress_policy_refused" }); +}); + +test("full runs follow the explicit order and finish unchanged when no Evidence source exists", async () => { + const f = fixture(); + f.runChild + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), + stderr: "", + }) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + candidate_count: 0, + candidate_digest: "sha256:" + "0".repeat(64), + candidate_yaml: "tables: []\n", + }), + stderr: "", + }) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 }, + }), + stderr: "", + }); + + const result = await f.service.run({ workspaceId: "psd-clinical" }); + expect(result).toMatchObject({ + status: "succeeded", + code: "ok", + completedStages: ["dwh", "fk_suggest", "schema_index"], + warnings: ["workspace has no Evidence source"], + }); + expect(f.runChild.mock.calls.map(([request]) => (request as ChildProcessRequest).argv.slice(0, 2).join(" "))).toEqual([ + "preprocess dwh", + "schema suggest-fks", + "vector index-schema", + ]); +}); + + +test("schema accept validates the synced Git blob and records the review", async () => { + const f = fixture(); + f.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + candidate_count: 1, + candidate_digest: "sha256:" + "e".repeat(64), + candidate_yaml: "tables: {}\n", + }), + stderr: "", + }); + const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" }); + const runId = suggest.runId!; + const candidateDigest = suggest.artifactIdentities![0]!.digest; + const synced = syncAnnotations({ + dataRoot: f.dataRoot, + workspaceId: "psd-clinical", + commit: "a".repeat(40), + blobId: "b".repeat(40), + contents: Buffer.from("tables: {}\n"), + }); + + f.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + orphan_count: 0, + annotations_digest: synced.contentDigest, + reviewed_candidates_digest: candidateDigest, + }), + stderr: "", + }); + + const result = await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true }); + expect(result).toMatchObject({ status: "succeeded", code: "ok", operation: "schema accept" }); + const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" }); + expect(state.readFkReview(runId)).toMatchObject({ + reviewedCandidatesDigest: candidateDigest, + annotationsDigest: synced.contentDigest, + workspaceRevision: "a".repeat(40), + blobId: "b".repeat(40), + }); +}); + +test("schema accept fails closed without --yes, for an unknown run, or with no synced annotations", async () => { + const f = fixture(); + f.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ + status: "succeeded", + candidate_count: 1, + candidate_digest: "sha256:" + "e".repeat(64), + candidate_yaml: "tables: {}\n", + }), + stderr: "", + }); + const suggest = await f.service.suggestFks({ workspaceId: "psd-clinical" }); + const runId = suggest.runId!; + + await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: false })) + .resolves.toMatchObject({ status: "failed", code: "annotation_invalid" }); + + await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId: "e".repeat(32), yes: true })) + .resolves.toMatchObject({ status: "failed", code: "annotation_invalid" }); + + await expect(f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true })) + .resolves.toMatchObject({ status: "failed", code: "annotation_invalid" }); + expect(f.runChild).toHaveBeenCalledTimes(1); +}); + +test("full runs continue after schema accept only when the accepted blob matches the current revision", async () => { + const f = fixture(); + const revision = "a".repeat(40); + f.runChild + .mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" }) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }), + stderr: "", + }); + + const blocked = await f.service.run({ workspaceId: "psd-clinical" }); + expect(blocked).toMatchObject({ status: "blocked", code: "manual_review_required" }); + const runId = blocked.runId!; + const state = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" }); + const candidateDigest = state.readFkCandidates(runId)!.digest; + + const synced = syncAnnotations({ + dataRoot: f.dataRoot, + workspaceId: "psd-clinical", + commit: revision, + blobId: "b".repeat(40), + contents: Buffer.from("tables: {}\n"), + }); + + // Accept writes the review; the resume then passes the gate and reaches schema indexing. + f.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", orphan_count: 0, annotations_digest: synced.contentDigest, reviewed_candidates_digest: candidateDigest }), + stderr: "", + }); + await f.service.acceptSchema({ workspaceId: "psd-clinical", runId, yes: true }); + + f.runChild.mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", counts: { added: 1, updated: 0, deleted: 0, unchanged: 0 } }), + stderr: "", + }); + const resumed = await f.service.run({ workspaceId: "psd-clinical", resumeRunId: runId }); + expect(resumed).toMatchObject({ status: "succeeded", code: "ok" }); + + // A review whose accepted blob digest no longer matches the current revision stays blocked. + const second = fixture(); + second.runChild + .mockResolvedValueOnce({ exitCode: 0, stdout: JSON.stringify({ status: "succeeded", run_id: "d".repeat(32) }), stderr: "" }) + .mockResolvedValueOnce({ + exitCode: 0, + stdout: JSON.stringify({ status: "succeeded", candidate_count: 1, candidate_digest: "sha256:" + "e".repeat(64), candidate_yaml: "tables: {}\n" }), + stderr: "", + }); + const secondBlocked = await second.service.run({ workspaceId: "psd-clinical" }); + const secondRunId = secondBlocked.runId!; + const secondState = new PreprocessingStateStore({ dataRoot: second.dataRoot, workspaceId: "psd-clinical" }); + const secondCandidate = secondState.readFkCandidates(secondRunId)!.digest; + secondState.writeFkReview(secondRunId, { + reviewedCandidatesDigest: secondCandidate, + annotationsDigest: "sha256:" + "0".repeat(64), + workspaceRevision: revision, + blobId: "b".repeat(40), + }); + const stillBlocked = await second.service.run({ workspaceId: "psd-clinical", resumeRunId: secondRunId }); + expect(stillBlocked).toMatchObject({ status: "blocked", code: "manual_review_required" }); +}); + +test("vector rebuild recreates the full collection contract including keyword indexes", async () => { + const f = fixture(); + // mock fetch: DELETE ok, then reconcileCollection self-heals create + indexes (real fetch in deps) + const calls: string[] = []; + const fakeFetch = async (url: string, init?: any) => { + calls.push(`${init?.method ?? "GET"} ${url}`); + if ((init?.method ?? "GET") === "DELETE") return new Response("", { status: 200 }); + if (url.endsWith("/collections/psd-clinical") && init?.method === "PUT") return new Response("", { status: 200 }); + if (url.endsWith("/collections/psd-clinical") && init?.method === "GET") { + return new Response(JSON.stringify({ result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, payload_schema: { content_hash: { data_type: "keyword" }, document_id: { data_type: "keyword" }, kind: { data_type: "keyword" }, record_key: { data_type: "keyword" }, record_kind: { data_type: "keyword" }, vector_generation: { data_type: "keyword" }, workspace_id: { data_type: "keyword" }, workspace_revision: { data_type: "keyword" } } } }), { status: 200 }); + } + if (url.endsWith("/collections/psd-clinical/index") && init?.method === "PUT") return new Response("", { status: 200 }); + return new Response(JSON.stringify({ result: {} }), { status: 200 }); + }; + const service = new WorkspacePreprocessingService({ + dataRoot: f.dataRoot, + acquireActiveRuntime: async () => runtime(baseWorkspace), + runChild: vi.fn(), + listSessions: async () => [], + semanticPreflight: async () => ({ ok: true }), + }); + // replace global fetch used by vectorRebuild/reconcileCollection + const original = globalThis.fetch; + globalThis.fetch = fakeFetch as any; + try { + const result = await service.vectorRebuild({ workspaceId: "psd-clinical", collection: "psd-clinical", confirm: "psd-clinical", destroy: true }); + expect(result).toMatchObject({ status: "succeeded", code: "ok" }); + } finally { + globalThis.fetch = original; + } + expect(calls.some((c) => c.startsWith("DELETE "))).toBe(true); +}); diff --git a/backend/test/workspace-preprocessing-state.test.ts b/backend/test/workspace-preprocessing-state.test.ts new file mode 100644 index 00000000..b0e69788 --- /dev/null +++ b/backend/test/workspace-preprocessing-state.test.ts @@ -0,0 +1,119 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { PreprocessingStateStore } from "../src/workspaces/preprocessing-state.js"; + +const roots: string[] = []; + +afterEach(() => { + roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +function fixture() { + const dataRoot = mkdtempSync(join(tmpdir(), "tht-preprocessing-state-")); + roots.push(dataRoot); + return { + dataRoot, + store: new PreprocessingStateStore({ dataRoot, workspaceId: "psd-clinical" }), + }; +} + +test("job state creates durable 0600 JSON and enforces same-revision resume", async () => { + const { store } = fixture(); + const job = await store.beginJob({ + operation: "preprocess dwh", + workspaceRevision: "a".repeat(40), + descriptorBlob: "b".repeat(40), + catalogBlob: "c".repeat(40), + configDigest: "sha256:config", + bindingDigest: "sha256:bindings", + }); + + const path = store.jobPath(job.runId); + expect(existsSync(path)).toBe(true); + expect(statSync(path).mode & 0o777).toBe(0o600); + expect(JSON.parse(readFileSync(path, "utf8"))).toMatchObject({ + schemaVersion: 1, + operation: "preprocess dwh", + workspaceId: "psd-clinical", + workspaceRevision: "a".repeat(40), + descriptorBlob: "b".repeat(40), + catalogBlob: "c".repeat(40), + configDigest: "sha256:config", + bindingDigest: "sha256:bindings", + }); + + await expect(store.beginJob({ + operation: "preprocess dwh", + runId: job.runId, + workspaceRevision: "d".repeat(40), + descriptorBlob: "b".repeat(40), + catalogBlob: "c".repeat(40), + configDigest: "sha256:config", + bindingDigest: "sha256:bindings", + })).rejects.toMatchObject({ code: "preprocessing_resume_mismatch" }); + + const resumed = await store.beginJob({ + operation: "preprocess dwh", + runId: job.runId, + workspaceRevision: "a".repeat(40), + descriptorBlob: "b".repeat(40), + catalogBlob: "c".repeat(40), + configDigest: "sha256:config", + bindingDigest: "sha256:bindings", + }); + expect(resumed.runId).toBe(job.runId); +}); + +test("writer lock rejects a concurrent contender and the kernel releases it after holder death", async () => { + const f = fixture(); + const other = new PreprocessingStateStore({ dataRoot: f.dataRoot, workspaceId: "psd-clinical" }); + const first = await f.store.acquireWriterLock(); + await expect(other.acquireWriterLock()).rejects.toMatchObject({ code: "preprocessing_conflict" }); + + process.kill(first.holderPid, "SIGKILL"); + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + try { + const recovered = await other.acquireWriterLock(); + await recovered.release(); + return; + } catch (error) { + if ((error as { code?: string }).code !== "preprocessing_conflict") throw error; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + throw new Error("writer lock was not released after holder death"); +}); + +test("session inventory guard blocks resumable sessions pinned to a different revision", async () => { + const { store } = fixture(); + + await expect(store.assertSessionInventoryCompatible("a".repeat(40), [ + { id: "open-other", status: "closed", archived: false, workspaceRevision: "b".repeat(40) }, + ])).rejects.toMatchObject({ code: "preprocessing_conflict" }); + + await expect(store.assertSessionInventoryCompatible("a".repeat(40), [ + { id: "current", status: "open", archived: false, workspaceRevision: "a".repeat(40) }, + { id: "finalized", status: "finalized", archived: false, workspaceRevision: "b".repeat(40) }, + { id: "archived", status: "closed", archived: true, workspaceRevision: "c".repeat(40) }, + ])).resolves.toBeUndefined(); +}); + +test("candidate and review artifacts are digest-bound durable files", async () => { + const { store } = fixture(); + const runId = "1".repeat(32); + const candidate = await store.writeFkCandidates(runId, `tables: [] +`); + const review = await store.writeFkReview(runId, { + reviewedCandidatesDigest: candidate.digest, + annotationsDigest: "sha256:annotations", + workspaceRevision: "a".repeat(40), + }); + + expect(candidate.digest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(review.digest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(store.readFkCandidates(runId)?.digest).toBe(candidate.digest); + expect(store.readFkReview(runId)?.reviewedCandidatesDigest).toBe(candidate.digest); +}); diff --git a/backend/test/workspace-registry-deployment.test.ts b/backend/test/workspace-registry-deployment.test.ts new file mode 100644 index 00000000..3c205843 --- /dev/null +++ b/backend/test/workspace-registry-deployment.test.ts @@ -0,0 +1,334 @@ +import { execFile, execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { WorkspaceRegistry } from "../src/workspaces/registry.js"; +import { parseWorkspaceYaml, serializeWorkspaceYaml, type CanonicalWorkspace } from "../src/workspaces/schema.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +function readFixture(name: string): string { + return readFileSync(new URL(`../../scripts/fixtures/${name}`, import.meta.url), "utf8"); +} + +function catalogYaml(workspace: CanonicalWorkspace): string { + const { id, name, description } = workspace.workspace; + return `schema_version: 1 +workspaces: + - id: ${id} + name: ${name}${description ? ` + description: ${description}` : ""} +`; +} + +async function git(cwd: string, args: string[]): Promise { + await runFile("git", args, { cwd }); +} + +async function gitOutput(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +async function commitAll(cwd: string, message: string): Promise { + await git(cwd, ["add", "-A"]); + await git(cwd, [ + "-c", + "user.name=Workspace Registry Deployment Test", + "-c", + "user.email=workspace-registry-deployment@example.invalid", + "commit", + "-m", + message, + ]); + await git(cwd, ["push", "origin", "main"]); + return await gitOutput(cwd, ["rev-parse", "HEAD"]); +} + +function registryConfig(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Workspace Registry Deployment Test", + gitAuthorEmail: "workspace-registry-deployment@example.invalid", + installationId: "deployment-test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 4, + }; +} + +function workspaceVariant( + workspace: CanonicalWorkspace, + changes: Partial, +): CanonicalWorkspace { + const id = changes.id ?? workspace.workspace.id; + return { + ...workspace, + workspace: { ...workspace.workspace, ...changes, id }, + semantic_index: { + ...workspace.semantic_index, + vector_store: { ...workspace.semantic_index.vector_store, collection: id }, + }, + ...(workspace.evidence?.source.type === "filesystem" + ? { + evidence: { + ...workspace.evidence, + source: { ...workspace.evidence.source, uri: `${id}/evidence` }, + }, + } + : {}), + }; +} + +async function createRegistryFixture(workspace: CanonicalWorkspace): Promise<{ + root: string; + source: string; + remote: string; + registry: WorkspaceRegistry; +}> { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-deployment-")); + temporaryRoots.push(root); + const source = join(root, "source"); + const remote = join(root, "remote.git"); + mkdirSync(source, { recursive: true }); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Workspace Registry Deployment Test"]); + await git(source, ["config", "user.email", "workspace-registry-deployment@example.invalid"]); + + const id = workspace.workspace.id; + mkdirSync(join(source, id), { recursive: true }); + writeFileSync(join(source, "thoth-workspaces.yaml"), catalogYaml(workspace)); + writeFileSync(join(source, id, "workspace.yaml"), serializeWorkspaceYaml(workspace)); + if (workspace.evidence?.source.type === "filesystem") { + mkdirSync(join(source, id, "evidence"), { recursive: true }); + writeFileSync(join(source, id, "evidence", "guide.md"), "guide v1\n"); + } + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "Seed workspace registry deployment fixture"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + + return { + root, + source, + remote, + registry: new WorkspaceRegistry(registryConfig(join(root, "registry"), remote)), + }; +} + +test("declares a durable isolated registry volume and installs fixtures under the root catalog contract", () => { + const compose = readFileSync(new URL("../../compose.yaml", import.meta.url), "utf8"); + const development = readFileSync(new URL("../../docker-compose.dev.yml", import.meta.url), "utf8"); + const gitHttps = readFileSync(new URL("../../deploy/compose.git-https.yaml", import.meta.url), "utf8"); + const gitSsh = readFileSync(new URL("../../deploy/compose.git-ssh.yaml", import.meta.url), "utf8"); + const dockerfile = readFileSync(new URL("../../docker/core.Dockerfile", import.meta.url), "utf8"); + const smoke = readFileSync(new URL("../../scripts/workspace-registry-smoke.sh", import.meta.url), "utf8"); + const unified = readFileSync(new URL("../../scripts/unified-deployment-smoke.sh", import.meta.url), "utf8"); + const windows = readFileSync(new URL("../../scripts/test-windows-clone-contract.ps1", import.meta.url), "utf8"); + + for (const source of [compose, development]) { + expect(source).toContain("THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry"); + expect(source).toContain("THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE}"); + expect(source).toContain("workspace-registry:/data/workspace-registry"); + } + expect(compose).not.toMatch(/workspace-registry-git-(?:credentials|ca|ssh-key|known-hosts):ro/); + expect(gitHttps).toMatch(/workspace-registry-git-credentials:ro/); + expect(gitHttps).toMatch(/workspace-registry-git-ca:ro/); + expect(gitSsh).toMatch(/workspace-registry-git-ssh-key:ro/); + expect(gitSsh).toMatch(/workspace-registry-git-known-hosts:ro/); + expect(dockerfile).toMatch(/mkdir -p[^\n]*\/data\/workspace-registry/); + expect(dockerfile).toMatch(/chown -R thoth:thoth \/home\/thoth\/\.pi \/data/); + + expect(smoke).toContain('--fixtures-only'); + expect(smoke).toContain('thoth-workspaces.yaml'); + expect(smoke).toContain('$seed/$workspace_registry_smoke_id/workspace.yaml'); + expect(smoke).toContain('$seed/$workspace_registry_smoke_id/evidence/guide.md'); + expect(smoke).not.toContain('"$seed/workspaces/local.yaml"'); + expect(smoke).not.toContain('workspace-content/local'); + expect(smoke).toContain('core_remote="/fixtures/offline.git"'); + expect(smoke).toContain('"degraded":true'); + expect(smoke).toContain('compose-config-contract)'); + expect(smoke).toContain('cleanup-failure-path)'); + + expect(unified).toContain('--fixtures-only'); + expect(unified).toContain('thoth-workspaces.yaml'); + expect(unified).toContain('$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml'); + expect(unified).toContain('$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md'); + expect(unified).not.toContain('"$TASK13_SEED/workspaces/task13-smoke.yaml"'); + expect(unified).not.toContain('workspace-content/task13-smoke'); + + expect(windows).toContain('thoth-workspaces.yaml'); + expect(windows).toContain('$workspaceDestination = Join-Path $workspaceDirectory "workspace.yaml"'); + expect(windows).toContain('Join-Path $workspaceEvidence "guide.md"'); + expect(windows).toContain('legacy flat descriptor path must not be used'); + expect(windows).not.toContain('workspace-content'); +}); + +test("shared deployment fixtures remain valid standalone descriptors with canonical filesystem Evidence", () => { + const smoke = parseWorkspaceYaml(readFixture("workspace-registry-smoke.yaml")); + const task13 = parseWorkspaceYaml(readFixture("workspace-registry-task13.yaml")); + const windows = parseWorkspaceYaml(readFixture("workspace-registry-windows.yaml")); + + expect(smoke).toMatchObject({ + workspace: { + schema_version: 3, + id: "local", + name: "Local", + description: "Isolated workspace registry smoke fixture.", + }, + evidence: { + source: { type: "filesystem", uri: "local/evidence", patterns: ["**/*.md"] }, + policy: { max_chunk_chars: 4000, retain_published_generations: 3 }, + }, + }); + expect(task13).toMatchObject({ + workspace: { schema_version: 3, id: "task13-smoke", name: "Task 13 Smoke" }, + evidence: { + source: { type: "filesystem", uri: "task13-smoke/evidence", patterns: ["**/*.md"] }, + policy: { max_chunk_chars: 4000, retain_published_generations: 3 }, + }, + }); + expect(windows).toMatchObject({ + workspace: { schema_version: 3, id: "task13-windows", name: "Task 13 Windows" }, + evidence: { + source: { type: "filesystem", uri: "task13-windows/evidence", patterns: ["**/*.md"] }, + policy: { max_chunk_chars: 4000, retain_published_generations: 3 }, + }, + }); +}); + +test("registry boots from the nested catalog layout, accepts co-committed display metadata, and advances on evidence-only commits", async () => { + const workspace = parseWorkspaceYaml(readFixture("workspace-registry-smoke.yaml")); + const fixture = await createRegistryFixture(workspace); + + const bootstrapped = await fixture.registry.bootstrap(); + expect(bootstrapped.head).toMatch(/^[0-9a-f]{40}$/); + expect(await fixture.registry.listCatalog()).toContainEqual( + expect.objectContaining({ + id: "local", + name: "Local", + description: "Isolated workspace registry smoke fixture.", + configurationState: "ready", + }), + ); + + const coCommitted = workspaceVariant(workspace, { name: "Local Updated" }); + writeFileSync( + join(fixture.source, "local", "workspace.yaml"), + serializeWorkspaceYaml(coCommitted), + ); + writeFileSync(join(fixture.source, "thoth-workspaces.yaml"), catalogYaml(coCommitted)); + const metadataCommit = await commitAll(fixture.source, "Update catalog and descriptor together"); + const metadataStatus = await fixture.registry.pull(); + expect(metadataStatus.head).toBe(metadataCommit); + const metadataRevision = await fixture.registry.read("local"); + expect(metadataRevision.workspace.workspace.name).toBe("Local Updated"); + + writeFileSync(join(fixture.source, "local", "evidence", "guide.md"), "guide v2\n"); + const evidenceCommit = await commitAll(fixture.source, "Update curated evidence only"); + const evidenceStatus = await fixture.registry.pull(); + expect(evidenceStatus.head).toBe(evidenceCommit); + expect(evidenceStatus.head).not.toBe(metadataCommit); + const evidenceRevision = await fixture.registry.read("local"); + expect(evidenceRevision.workspace.workspace.name).toBe("Local Updated"); + expect(evidenceRevision.revision.commit).toBe(evidenceCommit); + expect(evidenceRevision.revision.commit).not.toBe(metadataRevision.revision.commit); + expect(evidenceRevision.revision.blob).toBe(metadataRevision.revision.blob); +}); + +test("registry rejects orphan descriptors, metadata mismatches, and the retired flat layout while keeping the last active snapshot", async () => { + const workspace = parseWorkspaceYaml(readFixture("workspace-registry-smoke.yaml")); + + const expectRejectedMutation = async (mutate: (fixture: Awaited>) => Promise | void) => { + const fixture = await createRegistryFixture(workspace); + await fixture.registry.bootstrap(); + const active = await fixture.registry.read("local"); + await mutate(fixture); + await commitAll(fixture.source, "Apply invalid registry mutation"); + await expect(fixture.registry.pull()).rejects.toMatchObject({ code: "workspace_invalid" }); + const retained = await fixture.registry.read("local"); + expect(retained.revision.commit).toBe(active.revision.commit); + expect(retained.workspace.workspace.name).toBe("Local"); + }; + + await expectRejectedMutation((fixture) => { + const mismatched = workspaceVariant(workspace, { name: "Local Mismatched" }); + writeFileSync(join(fixture.source, "local", "workspace.yaml"), serializeWorkspaceYaml(mismatched)); + }); + + await expectRejectedMutation((fixture) => { + const orphan = workspaceVariant(workspace, { id: "orphan", name: "Orphan Workspace" }); + mkdirSync(join(fixture.source, "orphan", "evidence"), { recursive: true }); + writeFileSync(join(fixture.source, "orphan", "workspace.yaml"), serializeWorkspaceYaml(orphan)); + writeFileSync(join(fixture.source, "orphan", "evidence", "guide.md"), "orphan guide\n"); + }); + + await expectRejectedMutation((fixture) => { + mkdirSync(join(fixture.source, "workspaces"), { recursive: true }); + mkdirSync(join(fixture.source, "workspace-content", "local", "evidence"), { recursive: true }); + writeFileSync(join(fixture.source, "workspaces", "local.yaml"), serializeWorkspaceYaml(workspace)); + writeFileSync(join(fixture.source, "workspace-content", "local", "evidence", "guide.md"), "legacy guide\n"); + }); +}); + +test("Windows clone contract copies the shared complete schema v3 descriptor into the nested registry layout", () => { + const descriptor = parseWorkspaceYaml(readFixture("workspace-registry-windows.yaml")); + const windows = readFileSync( + new URL("../../scripts/test-windows-clone-contract.ps1", import.meta.url), + "utf8", + ); + + expect(windows).toContain('"scripts/fixtures/workspace-registry-windows.yaml"'); + expect(windows).toContain('thoth-workspaces.yaml'); + expect(windows).toContain('$workspaceDestination = Join-Path $workspaceDirectory "workspace.yaml"'); + expect(windows).toContain('Join-Path $workspaceEvidence "guide.md"'); + expect(windows).not.toContain('schema_version: 3'); + expect(descriptor).toMatchObject({ + workspace: { + schema_version: 3, + id: "task13-windows", + name: "Task 13 Windows", + language: "en", + }, + evidence: { + source: { type: "filesystem", uri: "task13-windows/evidence", patterns: ["**/*.md"] }, + policy: { max_chunk_chars: 4000, retain_published_generations: 3 }, + }, + }); +}); + +test("workspace registry smoke image cleanup is scoped to the per-run image identity", () => { + const output = execFileSync("bash", ["scripts/workspace-registry-smoke.sh"], { + cwd: new URL("../..", import.meta.url), + env: { ...process.env, WORKSPACE_REGISTRY_SMOKE_SELF_TEST: "image-cleanup-identity" }, + encoding: "utf8", + }); + + expect(output).toContain("workspace registry smoke image cleanup identity self-test passed"); +}); + +test("workspace registry smoke cleanup failure-path self-test preserves status and retention", () => { + const output = execFileSync("bash", ["scripts/workspace-registry-smoke.sh"], { + cwd: new URL("../..", import.meta.url), + env: { ...process.env, WORKSPACE_REGISTRY_SMOKE_SELF_TEST: "cleanup-failure-path" }, + encoding: "utf8", + }); + + expect(output).toContain("workspace registry smoke cleanup failure-path self-test passed"); +}); + +test("workspace migration source modules are absent from the live backend boundary", () => { + expect(existsSync(new URL("../src/workspaces/migrate-legacy.ts", import.meta.url))).toBe(false); + expect(existsSync(new URL("../src/workspaces/migrate-v2-qdrant.ts", import.meta.url))).toBe(false); +}); diff --git a/backend/test/workspace-registry.test.ts b/backend/test/workspace-registry.test.ts new file mode 100644 index 00000000..08b9163b --- /dev/null +++ b/backend/test/workspace-registry.test.ts @@ -0,0 +1,1162 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + chmodSync, existsSync, lstatSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { WorkspaceRepositoryLock } from "../src/workspaces/git-repository.js"; +import { WorkspaceRegistry } from "../src/workspaces/registry.js"; +import { + parseWorkspaceYaml, renderWorkspaceDocs, serializeWorkspaceYaml, type CanonicalWorkspace, +} from "../src/workspaces/schema.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const validYaml = `workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`; + +function withFilesystemEvidence(source: string, id = "psd-clinical"): string { + return source.concat(`evidence: + source: + type: filesystem + uri: ${id}/evidence +`); +} + +function withDwhRestTransport(source: string): string { + return source.replace( + "supported_transports: [postgres_direct]", + "supported_transports: [postgres_direct, rest_api]", + ); +} + +function withDwhRestDiagnostic(source: string): string { + return withDwhRestTransport(source).concat(`diagnostics: + dwh_rest: + method: GET + path: /health + auth: none + response: + database: database + schema: schema +`); +} + +function withEmbeddingDiagnostic(source: string): string { + return source.concat(`diagnostics: + embedding: + method: GET + path: /models + auth: none + response: + model: model + dimensions: dimensions +`); +} + +function withDwhRestAndEmbeddingDiagnostics(source: string): string { + return withDwhRestTransport(source).concat(`diagnostics: + dwh_rest: + method: GET + path: /health + auth: none + response: + database: database + schema: schema + embedding: + method: GET + path: /models + auth: none + response: + model: model + dimensions: dimensions +`); +} + +function withVectorRestTransport(source: string): string { + return source.replace( + "supported_transports: [pgvector_direct]", + "supported_transports: [pgvector_direct, rest_api]", + ); +} + +function withVectorMetadataDiagnostic(source: string): string { + return withVectorRestTransport(source).concat(`diagnostics: + vector_rest: + metadata: + method: GET + path: /metadata + auth: none + response: + collection: collection + dimensions: dimensions + distance: distance +`); +} + +function withReversibleVectorProbe(source: string): string { + return withVectorRestTransport(source).concat(`diagnostics: + vector_rest: + metadata: + method: GET + path: /metadata + auth: none + response: + collection: collection + dimensions: dimensions + distance: distance + reversible_probe: + method: POST + path: /probe + auth: bearer + response: + operation: operation +`); +} + +function legacyV1Yaml(source = validYaml): string { + return source + .replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n") + .replace(" collection: psd-clinical\n", " collection: psd_clinical\n") + .replace(" dimensions: 1024", " dimensions: 768") + .replace(" provider: ollama_internal", " provider: ollama_compatible") + .replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe") + .replace(" dimensions: 1024", " dimensions: 768") + .replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n") + .replace("schema_version: 3", "schema_version: 1"); +} + +function legacyV2Yaml(source = validYaml): string { + return source + .replace(" engine: qdrant\n", " engine: pgvector\n database: postgres\n schema: vectors\n") + .replace(" collection: psd-clinical\n", " collection: psd_clinical\n") + .replace(" dimensions: 1024", " dimensions: 768") + .replace(" provider: ollama_internal", " provider: ollama_compatible") + .replace(" model: qwen3-embedding:0.6b", " model: nomic-embed-text-v2-moe") + .replace(" dimensions: 1024", " dimensions: 768") + .replace("distance: cosine\n", "distance: cosine\n supported_transports: [pgvector_direct]\n") + .replace("schema_version: 3", "schema_version: 2"); +} + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + await runFile("git", args, { cwd }); +} + +async function gitOutput(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function catalogYaml(entries: Array<{ id: string; name: string; description?: string }>): string { + return `schema_version: 1 +workspaces: +${entries.map((entry) => ` - id: ${entry.id} + name: ${entry.name}${entry.description ? `\n description: ${entry.description}` : ""}`).join("\n")} +`; +} + +async function fixture(workspaceSource = validYaml): Promise<{ + root: string; remote: string; source: string; initialCommit: string; +}> { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Workspace Registry Test"]); + await git(source, ["config", "user.email", "workspace-registry@example.invalid"]); + const workspace = workspaceSource.includes("schema_version: 3") + ? parseWorkspaceYaml(workspaceSource) : undefined; + mkdirSync(join(source, "psd-clinical"), { recursive: true }); + writeFileSync(join(source, "thoth-workspaces.yaml"), catalogYaml([ + { id: "psd-clinical", name: workspace?.workspace.name ?? "Policlinico San Donato", ...(workspace?.workspace.description ? { description: workspace.workspace.description } : {}) }, + ])); + writeFileSync(join(source, "psd-clinical", "workspace.yaml"), workspaceSource); + if (workspaceSource.includes("type: filesystem")) { + mkdirSync(join(source, "psd-clinical", "evidence"), { recursive: true }); + writeFileSync(join(source, "psd-clinical", "evidence", "guide.md"), "guide v1\n"); + await git(source, ["add", "-A"]); + } else { + await git(source, ["add", "thoth-workspaces.yaml", "psd-clinical/workspace.yaml"]); + } + await git(source, ["commit", "-m", "Initial workspace"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const { stdout } = await runFile("git", ["rev-parse", "HEAD"], { cwd: source }); + return { root, remote, source, initialCommit: stdout.trim() }; +} + +async function contentOnlyFixture(): Promise<{ + root: string; remote: string; source: string; initialCommit: string; +}> { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-empty-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Workspace Registry Test"]); + await git(source, ["config", "user.email", "workspace-registry@example.invalid"]); + const evidence = join(source, "p1-filesystem", "evidence"); + mkdirSync(evidence, { recursive: true }); + writeFileSync(join(evidence, "guide.md"), "curated content\n"); + writeFileSync(join(source, "thoth-workspaces.yaml"), catalogYaml([{ id: "p1-filesystem", name: "p1-filesystem" }])); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "Bootstrap curated content"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const initialCommit = await gitOutput(source, ["rev-parse", "HEAD"]); + return { root, remote, source, initialCommit }; +} + +async function multiWorkspaceFixture(workspaces: Record): Promise<{ + root: string; remote: string; source: string; initialCommit: string; +}> { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Workspace Registry Test"]); + await git(source, ["config", "user.email", "workspace-registry@example.invalid"]); + const entries = []; + for (const [id, workspaceSource] of Object.entries(workspaces)) { + const workspace = workspaceSource.includes("schema_version: 3") ? parseWorkspaceYaml(workspaceSource) : undefined; + entries.push({ id, name: workspace.workspace.name, ...(workspace.workspace.description ? { description: workspace.workspace.description } : {}) }); + mkdirSync(join(source, id), { recursive: true }); + writeFileSync(join(source, id, "workspace.yaml"), workspaceSource); + if (workspaceSource.includes("type: filesystem")) mkdirSync(join(source, id, "evidence"), { recursive: true }); + } + writeFileSync(join(source, "thoth-workspaces.yaml"), catalogYaml(entries)); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "Initial workspaces"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const { stdout } = await runFile("git", ["rev-parse", "HEAD"], { cwd: source }); + return { root, remote, source, initialCommit: stdout.trim() }; +} + +function config( + root: string, + remoteUrl: string, + overrides: Partial = {}, +): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Workspace Registry Test", + gitAuthorEmail: "workspace-registry@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + ...overrides, + }; +} + +async function pushInvalidWorkspace(source: string): Promise { + writeFileSync(join(source, "psd-clinical", "workspace.yaml"), "workspace: invalid\n"); + await git(source, ["add", "psd-clinical/workspace.yaml"]); + await git(source, ["commit", "-m", "Invalid workspace"]); + await git(source, ["push", "origin", "main"]); +} + +type HistoricalRevisionState = "absent" | "operational" | "migration_required" | "unknown"; + +function revisionWithHistoricalState( + revision: Record, + encoding: HistoricalRevisionState, +): Record { + const { state: _state, ...stateFree } = revision; + return encoding === "absent" ? stateFree : { + ...stateFree, + state: encoding === "unknown" ? "retired" : encoding, + }; +} + +function rewritePersistedRevisionStates( + root: string, + commit: string, + activeEncoding: HistoricalRevisionState, + manifestEncoding: HistoricalRevisionState, +): void { + const activePath = join(root, "state", "active.json"); + const snapshotPath = join(root, "snapshots", commit, "snapshot.json"); + const active = JSON.parse(readFileSync(activePath, "utf8")); + const manifest = JSON.parse(readFileSync(snapshotPath, "utf8")); + active.revisions = active.revisions.map((revision: Record) => ( + revisionWithHistoricalState(revision, activeEncoding) + )); + manifest.revisions = manifest.revisions.map((revision: Record) => ( + revisionWithHistoricalState(revision, manifestEncoding) + )); + writeFileSync(activePath, JSON.stringify(active)); + chmodSync(snapshotPath, 0o600); + writeFileSync(snapshotPath, JSON.stringify(manifest)); +} + +function persistedState(root: string, commit: string): { active: any; manifest: any } { + return { + active: JSON.parse(readFileSync(join(root, "state", "active.json"), "utf8")), + manifest: JSON.parse(readFileSync(join(root, "snapshots", commit, "snapshot.json"), "utf8")), + }; +} + +function filesystemFingerprint(root: string): string { + if (!existsSync(root)) return "absent"; + const records: string[] = []; + const visit = (path: string, relative: string): void => { + const entry = lstatSync(path); + const metadata = [ + entry.dev, entry.ino, entry.mode, entry.uid, entry.gid, + entry.size, entry.mtimeMs, entry.ctimeMs, + ].join(":"); + if (entry.isSymbolicLink()) { + records.push(`link:${relative}:${metadata}`); + return; + } + if (entry.isDirectory()) { + records.push(`directory:${relative}:${metadata}`); + for (const name of readdirSync(path).sort()) visit(join(path, name), relative === "." ? name : `${relative}/${name}`); + return; + } + if (entry.isFile()) { + const contents = readFileSync(path); + records.push(`file:${relative}:${metadata}:${contents.length}:${createHash("sha256").update(contents).digest("hex")}`); + return; + } + records.push(`other:${relative}:${metadata}`); + }; + visit(root, "."); + return createHash("sha256").update(records.join("\n")).digest("hex"); +} + +test("verifies a never-initialized registry without contacting its remote", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-uninitialized-")); + temporaryRoots.push(root); + const registryRoot = join(root, "registry"); + mkdirSync(registryRoot, { mode: 0o700 }); + const registry = new WorkspaceRegistry(config( + registryRoot, + join(root, "missing-remote.git"), + )); + + const before = filesystemFingerprint(registryRoot); + + await expect(registry.verifyStoredState()).resolves.toEqual({ + state: "uninitialized", + workspaces: 0, + fingerprint: `sha256:${before}`, + }); + expect(filesystemFingerprint(registryRoot)).toBe(before); + expect(existsSync(join(registryRoot, "repo"))).toBe(false); + expect(readdirSync(registryRoot)).toEqual([]); +}); + +test("never-initialized inspection refuses an absent or partial root without creating it", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-absent-")); + temporaryRoots.push(root); + const registryRoot = join(root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, join(root, "missing-remote.git"))); + + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(filesystemFingerprint(registryRoot)).toBe("absent"); + + mkdirSync(join(registryRoot, "state"), { recursive: true }); + const partial = filesystemFingerprint(registryRoot); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(filesystemFingerprint(registryRoot)).toBe(partial); +}); + +test("verifies initialized snapshots and rejects partial or malformed persisted state", async () => { + const remote = await fixture(); + const registryRoot = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, remote.remote)); + await registry.bootstrap(); + + const before = filesystemFingerprint(registryRoot); + const savedPath = process.env.PATH; + process.env.PATH = join(remote.root, "no-executables"); + try { + await expect(registry.verifyStoredState()).resolves.toEqual({ + state: "active", + workspaces: 1, + fingerprint: `sha256:${before}`, + }); + } finally { + if (savedPath === undefined) delete process.env.PATH; + else process.env.PATH = savedPath; + } + expect(filesystemFingerprint(registryRoot)).toBe(before); + + const firstIntegrity = await registry.verifyStoredState(); + utimesSync(join(registryRoot, "repo"), new Date(1_000), new Date(1_000)); + const metadataIntegrity = await registry.verifyStoredState(); + expect(metadataIntegrity.fingerprint).not.toBe(firstIntegrity.fingerprint); + + rmSync(join(registryRoot, "state", "active.json")); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + + writeFileSync(join(registryRoot, "state", "active.json"), "{malformed"); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("read-only inspection rejects extra components, links, and ephemeral runtime contents", async () => { + const remote = await fixture(); + const registryRoot = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, remote.remote)); + const status = await registry.bootstrap(); + const assertHostile = async (setup: () => void, cleanup: () => void): Promise => { + setup(); + const before = filesystemFingerprint(registryRoot); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(filesystemFingerprint(registryRoot)).toBe(before); + cleanup(); + }; + + await assertHostile( + () => mkdirSync(join(registryRoot, "unexpected")), + () => rmSync(join(registryRoot, "unexpected"), { recursive: true }), + ); + await assertHostile( + () => rmSync(join(registryRoot, "locks", "empty-hooks"), { recursive: true }), + () => mkdirSync(join(registryRoot, "locks", "empty-hooks")), + ); + await assertHostile( + () => writeFileSync(join(registryRoot, "state", "unexpected.json"), "{}"), + () => rmSync(join(registryRoot, "state", "unexpected.json")), + ); + await assertHostile( + () => writeFileSync(join(registryRoot, "snapshots", status.head!, "unexpected"), "extra"), + () => rmSync(join(registryRoot, "snapshots", status.head!, "unexpected")), + ); + await assertHostile( + () => { + mkdirSync(join(registryRoot, "snapshots", "runtime"), { recursive: true }); + writeFileSync(join(registryRoot, "snapshots", "runtime", "restored-secret.yaml"), "secret: forbidden"); + }, + () => rmSync(join(registryRoot, "snapshots", "runtime"), { recursive: true }), + ); + await assertHostile( + () => symlinkSync(join(registryRoot, "state", "active.json"), join(registryRoot, "linked-active.json")), + () => rmSync(join(registryRoot, "linked-active.json")), + ); + await assertHostile( + () => symlinkSync(join(registryRoot, "state", "active.json"), join(registryRoot, "repo", "linked-active.json")), + () => rmSync(join(registryRoot, "repo", "linked-active.json")), + ); +}); + +test("rejects a catalog entry without a descriptor instead of creating a bootstrap slot", async () => { + const remote = await contentOnlyFixture(); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(existsSync(join(remote.root, "registry", "state", "active.json"))).toBe(false); +}); +test("bootstraps a checkout and activates a validated immutable snapshot", async () => { + const remote = await fixture(); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + + const status = await registry.bootstrap(); + + expect(status.head).toMatch(/^[0-9a-f]{40}$/); + expect(existsSync(registry.snapshotPath(status.head!, "psd-clinical"))).toBe(true); + await expect(registry.read("psd-clinical")).resolves.toMatchObject({ + revision: { commit: remote.initialCommit, id: "psd-clinical" }, + }); +}); + +test("concurrent first lists lazily bootstrap a clean registry once safely", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + + const [first, second] = await Promise.all([registry.list(), registry.list()]); + for (const revisions of [first, second]) { + expect(revisions).toEqual([ + expect.objectContaining({ + id: "psd-clinical", + commit: remote.initialCommit, + }), + ]); + } + expect(existsSync(join(root, "state", "active.json"))).toBe(true); +}); + +test.each(["missing", "blob"])( + "rejects a remote filesystem descriptor with a %s Evidence root and keeps the active snapshot", + async (invalidKind) => { + const remote = await fixture(withFilesystemEvidence(validYaml)); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + await registry.bootstrap(); + const evidenceRoot = join(remote.source, "psd-clinical", "evidence"); + rmSync(evidenceRoot, { recursive: true, force: true }); + if (invalidKind === "blob") writeFileSync(evidenceRoot, "not a tree\n"); + await git(remote.source, ["add", "-A", "psd-clinical/evidence"]); + await git(remote.source, ["commit", "-m", `Make Evidence root ${invalidKind}`]); + await git(remote.source, ["push", "origin", "main"]); + const invalidCommit = await gitOutput(remote.source, ["rev-parse", "HEAD"]); + + await expect(registry.pull()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(invalidCommit).not.toBe(remote.initialCommit); + await expect(registry.read("psd-clinical")).resolves.toMatchObject({ + revision: { commit: remote.initialCommit }, + }); + }, +); + +test("activation validates filesystem Evidence against its exact safeHead rather than checkout HEAD", async () => { + const remote = await fixture(withFilesystemEvidence(validYaml)); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + await registry.bootstrap(); + rmSync(join(remote.source, "psd-clinical", "evidence"), { + recursive: true, force: true, + }); + await git(remote.source, ["add", "-A", "psd-clinical/evidence"]); + await git(remote.source, ["commit", "-m", "Remove current Evidence root"]); + await git(remote.source, ["push", "origin", "main"]); + const invalidHead = await gitOutput(remote.source, ["rev-parse", "HEAD"]); + const internals = registry as unknown as { + repository: { pull(): Promise<{ head?: string }> }; + activate(commit: string): Promise; + }; + + expect((await internals.repository.pull()).head).toBe(invalidHead); + await expect(internals.activate(remote.initialCommit)).resolves.toBeUndefined(); + await expect(registry.read("psd-clinical")).resolves.toMatchObject({ + revision: { commit: remote.initialCommit }, + }); +}); + +test("creates an immutable descriptor revision for a content-only Evidence commit", async () => { + const remote = await fixture(withFilesystemEvidence(validYaml)); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + const initial = await registry.read("psd-clinical"); + const evidencePath = "psd-clinical/evidence"; + const initialTree = await gitOutput(remote.source, ["rev-parse", `${remote.initialCommit}:${evidencePath}`]); + writeFileSync(join(remote.source, evidencePath, "guide.md"), "guide v2\n"); + await git(remote.source, ["add", `${evidencePath}/guide.md`]); + await git(remote.source, ["commit", "-m", "Update Evidence only"]); + await git(remote.source, ["push", "origin", "main"]); + const contentCommit = await gitOutput(remote.source, ["rev-parse", "HEAD"]); + const contentTree = await gitOutput(remote.source, ["rev-parse", `${contentCommit}:${evidencePath}`]); + + await registry.pull(); + const current = await registry.read("psd-clinical"); + + expect(contentTree).not.toBe(initialTree); + expect(current.revision).toMatchObject({ commit: contentCommit, blob: initial.revision.blob }); + expect(current.revision.snapshotPath).not.toBe(initial.revision.snapshotPath); + expect(readFileSync(current.revision.snapshotPath, "utf8")).toBe( + readFileSync(initial.revision.snapshotPath, "utf8"), + ); + await expect(runFile("git", [ + "--git-dir", remote.remote, "cat-file", "-e", `${contentCommit}:${evidencePath}/guide.md`, + ], { cwd: remote.root })).resolves.toBeDefined(); +}); + +test("keeps content-only historical descriptor revisions distinguishable by commit", async () => { + const remote = await fixture(withFilesystemEvidence(validYaml)); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + await registry.bootstrap(); + writeFileSync( + join(remote.source, "psd-clinical", "evidence", "guide.md"), + "historical content\n", + ); + await git(remote.source, ["add", "psd-clinical/evidence/guide.md"]); + await git(remote.source, ["commit", "-m", "Retained Evidence update"]); + await git(remote.source, ["push", "origin", "main"]); + const contentCommit = await gitOutput(remote.source, ["rev-parse", "HEAD"]); + await registry.pull(); + await registry.reconcileSnapshotRetention([remote.initialCommit]); + + const retained = (await registry.listRetainedSnapshots()).filter(({ id }) => id === "psd-clinical"); + expect(retained.map(({ commit }) => commit)).toEqual([contentCommit, remote.initialCommit]); + const oldPinned = await registry.readPinned("psd-clinical", remote.initialCommit); + const newPinned = await registry.readPinned("psd-clinical", contentCommit); + expect(oldPinned.workspaceConfigPath).not.toBe(newPinned.workspaceConfigPath); + expect(oldPinned.workspace).toEqual(newPinned.workspace); +}); + +test.each([ + ["adds", validYaml, withDwhRestDiagnostic(validYaml)], + ["removes", withDwhRestDiagnostic(validYaml), validYaml], +])("pulls a curator change that %s a diagnostics branch without API rewrite", async (_operation, baseSource, remoteSource) => { + const remote = await fixture(baseSource); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + await registry.bootstrap(); + const initial = await registry.read("psd-clinical"); + writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), remoteSource); + await git(remote.source, ["add", "psd-clinical/workspace.yaml"]); + await git(remote.source, ["commit", "-m", `Registry ${_operation} diagnostic branch`]); + await git(remote.source, ["push", "origin", "main"]); + + await registry.pull(); + const updated = await registry.read("psd-clinical"); + expect(updated.revision.commit).not.toBe(initial.revision.commit); +}); +test.each([ + ["v1", legacyV1Yaml()], + ["v2", legacyV2Yaml()], +])("rejects a schema %s descriptor instead of activating it", async (_version, legacyYaml) => { + const remote = await fixture(legacyYaml); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(existsSync(join(remote.root, "registry", "state", "active.json"))).toBe(false); +}); + +test("writes only state-free revisions and never exposes revision state", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + + const initial = persistedState(root, remote.initialCommit); + expect(Object.keys(initial.active).sort()).toEqual(["catalog", "head", "revisions"]); + expect(Object.keys(initial.manifest).sort()).toEqual(["catalog", "files", "head", "revisions"]); + expect(Object.keys(initial.active.revisions[0]).sort()).toEqual(["blob", "commit", "id", "snapshotPath"]); + expect(Object.keys(initial.manifest.revisions[0]).sort()).toEqual(["blob", "commit", "id", "snapshotPath"]); + + const listed = await registry.list(); + const read = await registry.read("psd-clinical"); + expect(listed[0]).not.toHaveProperty("state"); + expect(read.revision).not.toHaveProperty("state"); +}); + +test("accepts historical operational state without leaking it or rewriting the immutable snapshot", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + await new WorkspaceRegistry(config(root, remote.remote)).bootstrap(); + rewritePersistedRevisionStates(root, remote.initialCommit, "operational", "operational"); + const snapshotPath = join(root, "snapshots", remote.initialCommit, "snapshot.json"); + const historicalManifest = readFileSync(snapshotPath, "utf8"); + + const restored = new WorkspaceRegistry(config(root, remote.remote)); + const listed = await restored.list(); + const read = await restored.read("psd-clinical"); + + expect(listed[0]).not.toHaveProperty("state"); + expect(read.revision).not.toHaveProperty("state"); + expect(readFileSync(snapshotPath, "utf8")).toBe(historicalManifest); + + await restored.bootstrap(); + const rewrittenActive = persistedState(root, remote.initialCommit).active; + expect(rewrittenActive.revisions[0]).not.toHaveProperty("state"); + expect(readFileSync(snapshotPath, "utf8")).toBe(historicalManifest); +}); + +test.each([ + ["historical active and state-free snapshot", "operational", "absent"], + ["state-free active and historical snapshot", "absent", "operational"], +] as const)("normalizes mixed persisted revision encodings: %s", async (_name, activeState, manifestState) => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + await new WorkspaceRegistry(config(root, remote.remote)).bootstrap(); + rewritePersistedRevisionStates(root, remote.initialCommit, activeState, manifestState); + const snapshotPath = join(root, "snapshots", remote.initialCommit, "snapshot.json"); + const historicalManifest = readFileSync(snapshotPath, "utf8"); + + const revisions = await new WorkspaceRegistry(config(root, remote.remote)).list(); + + expect(revisions[0]).not.toHaveProperty("state"); + expect(readFileSync(snapshotPath, "utf8")).toBe(historicalManifest); +}); + +test.each([ + ["migration_required in active state", "migration_required", "absent"], + ["migration_required in snapshot manifest", "absent", "migration_required"], + ["unknown state in active state", "unknown", "operational"], + ["unknown state in snapshot manifest", "operational", "unknown"], +] as const)("rejects %s", async (_name, activeState, manifestState) => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + await new WorkspaceRegistry(config(root, remote.remote)).bootstrap(); + rewritePersistedRevisionStates(root, remote.initialCommit, activeState, manifestState); + + await expect(new WorkspaceRegistry(config(root, remote.remote)).list()).rejects.toMatchObject({ + code: "workspace_invalid", + }); +}); + +test.each([ + ["active top level", "active", "top"], + ["active revision", "active", "revision"], + ["snapshot top level", "manifest", "top"], + ["snapshot revision", "manifest", "revision"], +] as const)("rejects unknown fields in %s", async (_name, component, location) => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + await new WorkspaceRegistry(config(root, remote.remote)).bootstrap(); + const path = component === "active" + ? join(root, "state", "active.json") + : join(root, "snapshots", remote.initialCommit, "snapshot.json"); + const persisted = JSON.parse(readFileSync(path, "utf8")); + if (location === "top") persisted.unexpected = true; + else persisted.revisions[0].unexpected = true; + if (component === "manifest") chmodSync(path, 0o600); + writeFileSync(path, JSON.stringify(persisted)); + + await expect(new WorkspaceRegistry(config(root, remote.remote)).list()).rejects.toMatchObject({ + code: "workspace_invalid", + }); +}); + +test("normalizes operational state in retained historical snapshots without rewriting them", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), validYaml.replace( + "name: Policlinico San Donato", "name: Current workspace", + )); + writeFileSync(join(remote.source, "thoth-workspaces.yaml"), catalogYaml([ + { id: "psd-clinical", name: "Current workspace" }, + ])); + await git(remote.source, ["add", "-A"]); + await git(remote.source, ["commit", "-m", "Update active workspace"]); + await git(remote.source, ["push", "origin", "main"]); + await registry.pull(); + rewritePersistedRevisionStates(root, remote.initialCommit, "absent", "operational"); + const snapshotPath = join(root, "snapshots", remote.initialCommit, "snapshot.json"); + const historicalManifest = readFileSync(snapshotPath, "utf8"); + + const retained = await new WorkspaceRegistry(config(root, remote.remote)).listRetainedSnapshots(); + + expect(retained).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: "psd-clinical", commit: remote.initialCommit }), + ])); + expect(retained.every((revision) => !("state" in revision))).toBe(true); + expect(readFileSync(snapshotPath, "utf8")).toBe(historicalManifest); +}); + +test("normalizes historical operational state during offline fallback after restart", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + await new WorkspaceRegistry(config(root, remote.remote)).bootstrap(); + rewritePersistedRevisionStates(root, remote.initialCommit, "operational", "operational"); + rmSync(remote.remote, { recursive: true, force: true }); + + const restored = new WorkspaceRegistry(config(root, remote.remote)); + await expect(restored.pull()).resolves.toMatchObject({ + degraded: true, + head: remote.initialCommit, + repository: { host: "local", repository: "configured-repository", transport: "local" }, + }); + const listed = await restored.list(); + const read = await restored.read("psd-clinical"); + expect(listed[0]).not.toHaveProperty("state"); + expect(read.revision).not.toHaveProperty("state"); +}); + +test("fails closed when a retained snapshot descriptor is not schema v3", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + await new WorkspaceRegistry(config(root, remote.remote)).bootstrap(); + const snapshotDirectory = join(root, "snapshots", remote.initialCommit); + const yamlPath = join(snapshotDirectory, "psd-clinical.yaml"); + chmodSync(yamlPath, 0o600); + const legacy = legacyV2Yaml(); + writeFileSync(yamlPath, legacy); + const manifestPath = join(snapshotDirectory, "snapshot.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + manifest.files["psd-clinical.yaml"] = createHash("sha256").update(legacy).digest("hex"); + chmodSync(manifestPath, 0o600); + writeFileSync(manifestPath, JSON.stringify(manifest)); + + await expect(new WorkspaceRegistry(config(root, remote.remote)).list()).rejects.toMatchObject({ + code: "workspace_invalid", + }); +}); + +test("keeps the last valid snapshot when a pulled commit has invalid YAML", async () => { + const remote = await fixture(); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + await registry.bootstrap(); + await pushInvalidWorkspace(remote.source); + + await expect(registry.pull()).rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(registry.read("psd-clinical")).resolves.toMatchObject({ + revision: { commit: remote.initialCommit }, + }); +}); + +test("rejects duplicate schema v3 collection ownership and keeps the previous active snapshot", async () => { + const v3Yaml = validYaml; + const remote = await multiWorkspaceFixture({ + "psd-clinical": v3Yaml, + "research-clinical": v3Yaml + .replace("id: psd-clinical", "id: research-clinical") + .replace("name: Policlinico San Donato", "name: Research Clinical") + .replace("collection: psd-clinical", "collection: research-clinical"), + }); + const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); + await registry.bootstrap(); + + writeFileSync( + join(remote.source, "research-clinical", "workspace.yaml"), + v3Yaml + .replace("id: psd-clinical", "id: research-clinical") + .replace("name: Policlinico San Donato", "name: Research Clinical") + .replace("collection: psd-clinical", "collection: shared"), + ); + writeFileSync( + join(remote.source, "psd-clinical", "workspace.yaml"), + v3Yaml.replace("collection: psd-clinical", "collection: shared"), + ); + await git(remote.source, ["add", "-A"]); + await git(remote.source, ["commit", "-m", "Duplicate collection ownership"]); + await git(remote.source, ["push", "origin", "main"]); + + await expect(registry.pull()).rejects.toMatchObject({ + code: "workspace_invalid", + message: "Workspace repository content is invalid", + }); + await expect(registry.read("psd-clinical")).resolves.toMatchObject({ + revision: { commit: remote.initialCommit }, + }); + await expect(registry.read("research-clinical")).resolves.toMatchObject({ + revision: { commit: remote.initialCommit }, + }); +}); + +test("retains a historical snapshot while a resumable manifest still references its revision", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + + writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), validYaml.replace( + "name: Policlinico San Donato", "name: Updated Policlinico San Donato", + )); + writeFileSync(join(remote.source, "thoth-workspaces.yaml"), catalogYaml([ + { id: "psd-clinical", name: "Updated Policlinico San Donato" }, + ])); + await git(remote.source, ["add", "-A"]); + await git(remote.source, ["commit", "-m", "Update workspace"]); + await git(remote.source, ["push", "origin", "main"]); + const currentCommit = await gitOutput(remote.source, ["rev-parse", "HEAD"]); + await registry.pull(); + + await registry.reconcileSnapshotRetention([remote.initialCommit]); + expect(existsSync(registry.snapshotPath(remote.initialCommit, "psd-clinical"))).toBe(true); + expect(existsSync(registry.snapshotPath(currentCommit, "psd-clinical"))).toBe(true); + + await registry.reconcileSnapshotRetention([]); + expect(existsSync(registry.snapshotPath(remote.initialCommit, "psd-clinical"))).toBe(false); + expect(existsSync(registry.snapshotPath(currentCommit, "psd-clinical"))).toBe(true); +}); + +test("a session revision lease survives stale retention scans until its manifest is observed", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + const lease = await registry.acquireSessionRevision("psd-clinical"); + + writeFileSync(join(remote.source, "psd-clinical", "workspace.yaml"), validYaml.replace( + "name: Policlinico San Donato", "name: Concurrent revision", + )); + writeFileSync(join(remote.source, "thoth-workspaces.yaml"), catalogYaml([ + { id: "psd-clinical", name: "Concurrent revision" }, + ])); + await git(remote.source, ["add", "-A"]); + await git(remote.source, ["commit", "-m", "Publish while session is starting"]); + await git(remote.source, ["push", "origin", "main"]); + await registry.pull(); + + await registry.reconcileSnapshotRetention([]); + expect(existsSync(registry.snapshotPath(remote.initialCommit, "psd-clinical"))).toBe(true); + + await lease.markPersisted(); + await registry.reconcileSnapshotRetention([]); + expect(existsSync(registry.snapshotPath(remote.initialCommit, "psd-clinical"))).toBe(true); + + await registry.reconcileSnapshotRetention([remote.initialCommit]); + await registry.reconcileSnapshotRetention([]); + expect(existsSync(registry.snapshotPath(remote.initialCommit, "psd-clinical"))).toBe(false); +}); + +test("lists operational descriptors retained after their workspace was removed from the active revision", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + + mkdirSync(join(remote.source, "archive-only"), { recursive: true }); + writeFileSync(join(remote.source, "archive-only", "workspace.yaml"), validYaml.replace( + "id: psd-clinical", "id: archive-only", + ).replace("collection: psd-clinical", "collection: archive-only")); + writeFileSync(join(remote.source, "thoth-workspaces.yaml"), catalogYaml([ + { id: "psd-clinical", name: "Policlinico San Donato" }, + { id: "archive-only", name: "Policlinico San Donato" }, + ])); + await git(remote.source, ["add", "-A"]); + await git(remote.source, ["commit", "-m", "Add retained workspace"]); + await git(remote.source, ["push", "origin", "main"]); + await registry.pull(); + + rmSync(join(remote.source, "psd-clinical", "workspace.yaml")); + writeFileSync(join(remote.source, "thoth-workspaces.yaml"), catalogYaml([ + { id: "archive-only", name: "Policlinico San Donato" }, + ])); + await git(remote.source, ["add", "-u"]); + await git(remote.source, ["commit", "-m", "Remove original workspace"]); + await git(remote.source, ["push", "origin", "main"]); + await registry.pull(); + + const retained = await registry.listRetainedSnapshots(); + expect(retained).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: "psd-clinical", commit: remote.initialCommit }), + expect.objectContaining({ id: "archive-only" }), + ])); +}); + +test("does not bypass an existing live advisory repository lock", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + const lock = new WorkspaceRepositoryLock(join(root, "locks")); + let release!: () => void; + let started!: () => void; + const held = lock.run(async () => { + started(); + await new Promise((resolve) => { release = resolve; }); + }); + await new Promise((resolve) => { started = resolve; }); + + try { + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "workspace_stale" }); + } finally { + release(); + await held; + } +}); + +test("rejects a symbolic-link registry root before creating a lock below it", async () => { + const remote = await fixture(); + const target = join(remote.root, "registry-target"); + const root = join(remote.root, "registry-link"); + mkdirSync(target); + symlinkSync(target, root); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + + await expect(registry.bootstrap()).rejects.toMatchObject({ code: "git_unavailable" }); + expect(existsSync(join(target, "locks"))).toBe(false); +}); + +test("rejects a locally-ahead checkout instead of activating local-only content", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + const checkout = join(root, "repo"); + writeFileSync(join(checkout, "psd-clinical", "workspace.yaml"), validYaml.replace( + "name: Policlinico San Donato", "name: Local only workspace", + )); + await git(checkout, ["config", "user.name", "Workspace Registry Test"]); + await git(checkout, ["config", "user.email", "workspace-registry@example.invalid"]); + await git(checkout, ["add", "psd-clinical/workspace.yaml"]); + await git(checkout, ["commit", "-m", "Local-only workspace"]); + + await expect(registry.pull()).rejects.toMatchObject({ code: "git_non_fast_forward" }); + await expect(registry.read("psd-clinical")).resolves.toMatchObject({ + revision: { commit: remote.initialCommit }, + workspace: { workspace: { name: "Policlinico San Donato" } }, + }); +}); + +test("recovers a dead-process advisory lock while preserving active snapshot safety", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + mkdirSync(join(root, "locks"), { recursive: true }); + writeFileSync(join(root, "locks", "repository.lock"), JSON.stringify({ pid: 999_999_999 })); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + + await expect(registry.bootstrap()).resolves.toMatchObject({ + head: remote.initialCommit, + degraded: false, + }); +}); + +test.each(["manifest", "blob", "workspace", "document"])( + "rejects a corrupted %s snapshot component instead of reporting it active", + async (component) => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + const snapshot = join(root, "snapshots", remote.initialCommit); + + if (component === "manifest") { + const file = join(snapshot, "snapshot.json"); + chmodSync(file, 0o600); + writeFileSync(file, "{"); + } + if (component === "blob") { + const activePath = join(root, "state", "active.json"); + const active = JSON.parse(readFileSync(activePath, "utf8")); + active.revisions[0].blob = "not-a-git-blob"; + writeFileSync(activePath, JSON.stringify(active)); + } + if (component === "workspace") { + const file = join(snapshot, "psd-clinical.yaml"); + chmodSync(file, 0o600); + writeFileSync(file, "truncated"); + } + if (component === "document") rmSync(join(snapshot, "psd-clinical.md")); + + await expect(registry.list()).rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(registry.read("psd-clinical")).rejects.toMatchObject({ code: "workspace_invalid" }); + }, +); + +test("rejects a corrupt fallback snapshot instead of returning degraded active state", async () => { + const remote = await fixture(); + const root = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(root, remote.remote)); + await registry.bootstrap(); + const document = join(root, "snapshots", remote.initialCommit, "psd-clinical.md"); + chmodSync(document, 0o600); + writeFileSync(document, "corrupt"); + rmSync(remote.remote, { recursive: true, force: true }); + + await expect(registry.pull()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); + + +test("snapshots canonical Evidence artifacts at the active commit and materializes filesystem Evidence bytes", async () => { + const remote = await fixture(withFilesystemEvidence(validYaml)); + const registryRoot = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, remote.remote)); + + const status = await registry.bootstrap(); + const active = await registry.read("psd-clinical"); + const snapshotDirectory = join(registryRoot, "snapshots", status.head!); + const descriptor = active.workspace as CanonicalWorkspace; + const docs = renderWorkspaceDocs(descriptor); + const expectedFiles: Record = { + "psd-clinical.yaml": serializeWorkspaceYaml(descriptor), + "psd-clinical.env.example": docs.envExample, + "psd-clinical.md": docs.markdown, + }; + const manifest = JSON.parse(readFileSync(join(snapshotDirectory, "snapshot.json"), "utf8")); + + expect(status.head).toBe(remote.initialCommit); + const committedDescriptor = parseWorkspaceYaml(await gitOutput(remote.source, [ + "show", `${remote.initialCommit}:psd-clinical/workspace.yaml`, + ])) as CanonicalWorkspace; + const committedBlob = await gitOutput(remote.source, [ + "rev-parse", `${remote.initialCommit}:psd-clinical/workspace.yaml`, + ]); + expect(active.revision.blob).toBe(committedBlob); + expect(expectedFiles["psd-clinical.yaml"]).toBe(serializeWorkspaceYaml(committedDescriptor)); + expect(readdirSync(snapshotDirectory).sort()).toEqual([ + "psd-clinical", "psd-clinical.env.example", "psd-clinical.md", "psd-clinical.yaml", "snapshot.json", + ]); + expect(manifest.head).toBe(remote.initialCommit); + expect(manifest.revisions[0]).toMatchObject({ + id: "psd-clinical", commit: remote.initialCommit, blob: committedBlob, + }); + expect(Object.keys(manifest.files).sort()).toEqual([ + "psd-clinical.env.example", "psd-clinical.md", "psd-clinical.yaml", "psd-clinical/evidence.manifest.json", + ]); + for (const [name, contents] of Object.entries(expectedFiles)) { + expect(readFileSync(join(snapshotDirectory, name), "utf8")).toBe(contents); + expect(manifest.files[name]).toBe(createHash("sha256").update(contents).digest("hex")); + } + // P6: the materialized Evidence tree and its digest-chained manifest. + const evidenceManifest = JSON.parse(readFileSync( + join(snapshotDirectory, "psd-clinical", "evidence.manifest.json"), "utf8", + )); + expect(evidenceManifest).toMatchObject({ workspace: "psd-clinical", commit: remote.initialCommit, entryCount: 1 }); + expect(manifest.files["psd-clinical/evidence.manifest.json"]).toBe( + createHash("sha256").update(`${JSON.stringify(evidenceManifest)}\n`).digest("hex"), + ); + expect(readFileSync(join(snapshotDirectory, "psd-clinical", "evidence", "guide.md"), "utf8")) + .toBe("guide v1\n"); + expect(JSON.stringify(manifest)).not.toContain("workspace-content/"); + expect(readdirSync(snapshotDirectory).some((name) => name === "workspace-content")).toBe(false); + expect(readFileSync(join(remote.source, "psd-clinical/evidence/guide.md"), "utf8")) + .toBe("guide v1\n"); +}); + + +test("never copies an installation secret canary into Git, generated artifacts, metadata, or errors", async () => { + const remote = await fixture(validYaml.concat(`evidence: + source: + type: http + uris: [https://evidence.example.test/guide.md] + authentication: signed_urls_file +`)); + const canary = "CANARY-EVIDENCE-SECRET-ONLY-IN-FIXTURE"; + const secretDirectory = join(remote.root, "fixture-secrets"); + mkdirSync(secretDirectory); + const secretFile = join(secretDirectory, "signed-urls"); + writeFileSync(secretFile, canary); + const registryRoot = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, remote.remote, { + secretRoots: [secretDirectory], + })); + const previous = process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE; + process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = secretFile; + try { + const status = await registry.bootstrap(); + const snapshotDirectory = join(registryRoot, "snapshots", status.head!); + let gitBlobText = ""; + try { + gitBlobText = (await runFile( + "git", ["grep", "-I", "-h", "-e", canary, "HEAD", "--", "."], { cwd: remote.source }, + )).stdout; + } catch (error) { + if (!error || typeof error !== "object" || !("code" in error) || error.code !== 1) throw error; + gitBlobText = "stdout" in error ? String(error.stdout ?? "") : ""; + } + expect(gitBlobText).toBe(""); + for (const name of readdirSync(snapshotDirectory)) { + expect(readFileSync(join(snapshotDirectory, name), "utf8")).not.toContain(canary); + } + } finally { + if (previous === undefined) delete process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE; + else process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = previous; + } +}); diff --git a/backend/test/workspace-runtime-config-lease.test.ts b/backend/test/workspace-runtime-config-lease.test.ts new file mode 100644 index 00000000..f7bae6af --- /dev/null +++ b/backend/test/workspace-runtime-config-lease.test.ts @@ -0,0 +1,362 @@ +import { execFile } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test, vi } from "vitest"; +import { parse } from "yaml"; +import { WorkspaceRegistry } from "../src/workspaces/registry.js"; +import { + buildCanonicalEffectiveConfig, + canonicalEffectiveConfigJson, + effectiveConfigIdentity, +} from "../src/workspaces/effective-config.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; +import { + publishDeterministicRuntimeConfigLease, + renderActiveWorkspaceRuntime, + renderWorkspaceRuntimeFromSnapshotPath, +} from "../src/workspaces/runtime-config-lease.js"; + +const runFile = promisify(execFile); +const roots: string[] = []; + +afterEach(() => { + vi.unstubAllEnvs(); + roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + return (await runFile("git", args, { cwd })).stdout.trim(); +} + +async function fixture() { + const root = mkdtempSync(join(tmpdir(), "tht-runtime-lease-")); + roots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + const registryRoot = join(root, "registry"); + const secretRoot = join(root, "secrets"); + const dataRoot = join(root, "data"); + const harnessDir = join(root, "harness"); + mkdirSync(harnessDir, { recursive: true }); + mkdirSync(join(harnessDir, "config"), { recursive: true }); + writeFileSync(join(harnessDir, "config", "tht.yaml"), `session_storage: + mode: local +profile: server +`); + + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Runtime Lease Test"]); + await git(source, ["config", "user.email", "runtime-lease@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), `schema_version: 1 +workspaces: [{id: psd-clinical, name: Runtime Lease}] +`); + mkdirSync(join(source, "psd-clinical", "evidence"), { recursive: true }); + writeFileSync(join(source, "psd-clinical", "workspace.yaml"), `workspace: + schema_version: 3 + id: psd-clinical + name: Runtime Lease + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +evidence: + source: + type: filesystem + uri: psd-clinical/evidence +`); + writeFileSync(join(source, "psd-clinical", "evidence", "guide.md"), `# hello +`); + await git(source, ["add", "."]); + await git(source, ["commit", "-m", "Canonical workspace"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + + mkdirSync(secretRoot); + const passwordFile = join(secretRoot, "dwh-password"); + writeFileSync(passwordFile, "secret", { mode: 0o600 }); + chmodSync(passwordFile, 0o600); + mkdirSync(dataRoot); + + const registryConfig: WorkspaceRegistryConfig = { + root: registryRoot, + remoteUrl: remote, + branch: "main", + gitAuthorName: "Runtime Lease Test", + gitAuthorEmail: "runtime-lease@example.invalid", + installationId: "test", + secretRoots: [secretRoot], + maxImportBytes: 1024 * 1024, + maxImportEntries: 16, + }; + const registry = new WorkspaceRegistry(registryConfig); + await registry.bootstrap(); + const revision = (await registry.list())[0]; + + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct"); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse.internal"); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432"); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "reader"); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE", passwordFile); + + return { + dataRoot, + harnessDir, + source, + registry, + registryConfig, + revision, + }; +} + +const semanticRuntime = { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, +}; + +test("active workspace rendering is byte-identical to direct snapshot rendering", async () => { + const f = await fixture(); + const direct = renderWorkspaceRuntimeFromSnapshotPath({ + snapshotPath: f.revision.snapshotPath, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + const active = await renderActiveWorkspaceRuntime({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + + expect(active.renderedConfig).toBe(direct.renderedConfig); + expect(active.workspaceRevision).toBe(f.revision.commit); + expect(active.descriptorBlob).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/); +}); + +test("renders a revision-qualified annotations root for the active revision", async () => { + const f = await fixture(); + const active = await renderActiveWorkspaceRuntime({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + const rendered = parse(active.renderedConfig) as Record; + + expect(rendered.paths.annotations_root).toBe( + join(f.dataRoot, "sessions", "psd-clinical", "revisions", f.revision.commit, "artifacts"), + ); + expect(rendered.roots.annotations_root).toBe(rendered.paths.annotations_root); + expect(rendered.paths.artifacts).toBe(join(f.dataRoot, "sessions", "psd-clinical", "artifacts")); +}); + +test("deterministic operator leases are keyed by logical identity and stable across calls", async () => { + const f = await fixture(); + const first = await publishDeterministicRuntimeConfigLease({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + const second = await publishDeterministicRuntimeConfigLease({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + + const suffix = first.inputFingerprint.slice(7, 23); + expect(second.path).toBe(first.path); + expect(first.path).toBe(join( + f.dataRoot, + "sessions", + "psd-clinical", + "preprocessing", + "runtime-config", + `${f.revision.commit}-${suffix}.yaml`, + )); + expect(statSync(first.path).mode & 0o777).toBe(0o400); + expect(statSync(first.manifestPath).mode & 0o777).toBe(0o600); + expect(readFileSync(first.path, "utf8")).toContain("collection_lifecycle: require_existing"); + expect(readFileSync(first.path, "utf8")).toContain("memory:"); + expect(existsSync(first.manifestPath)).toBe(true); + expect(first.effectiveConfigIdentity).toMatch(/^workspace:\/\/psd-clinical@v1:[0-9a-f]{64}$/); + expect(first.configFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(first.inputFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(first.inputFingerprint).not.toBe(first.configFingerprint); + const manifest = JSON.parse(readFileSync(first.manifestPath, "utf8")); + expect(manifest).toMatchObject({ + schemaVersion: 1, + workspaceId: "psd-clinical", + workspaceRevision: f.revision.commit, + descriptorBlob: first.descriptorBlob, + catalogBlob: first.catalogBlob, + configDigest: first.configDigest, + bindingDigest: first.bindingDigest, + effectiveConfigIdentity: first.effectiveConfigIdentity, + configFingerprint: first.configFingerprint, + inputFingerprint: first.inputFingerprint, + path: first.path, + }); + + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal"); + const changed = await publishDeterministicRuntimeConfigLease({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + expect(changed.path).not.toBe(first.path); + expect(changed.inputFingerprint).not.toBe(first.inputFingerprint); + expect(changed.configFingerprint).not.toBe(first.configFingerprint); + expect(readFileSync(changed.path, "utf8")).toContain("warehouse-two.internal"); +}); + +test("runtime rendering rejects untrusted snapshot paths and symlinks", async () => { + const f = await fixture(); + const outside = join(f.dataRoot, "outside.yaml"); + writeFileSync(outside, readFileSync(f.revision.snapshotPath, "utf8")); + const symlink = join(f.dataRoot, "alias.yaml"); + symlinkSync(f.revision.snapshotPath, symlink); + + expect(() => renderWorkspaceRuntimeFromSnapshotPath({ + snapshotPath: outside, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + })).toThrow(/trusted runtime snapshot/i); + expect(() => renderWorkspaceRuntimeFromSnapshotPath({ + snapshotPath: symlink, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + })).toThrow(/trusted runtime snapshot/i); +}); + + +test("operator lease and session snapshot produce byte-identical effective DWH bindings", async () => { + const f = await fixture(); + const session = renderWorkspaceRuntimeFromSnapshotPath({ + snapshotPath: f.revision.snapshotPath, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + const lease = await publishDeterministicRuntimeConfigLease({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + + const sessionCanonical = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(parse(session.renderedConfig))); + const operatorCanonical = canonicalEffectiveConfigJson(lease.effectiveConfig); + expect(operatorCanonical).toBe(sessionCanonical); + expect(lease.effectiveConfigIdentity).toBe( + effectiveConfigIdentity("psd-clinical", parse(session.renderedConfig)), + ); +}); + +test("a content-only Evidence commit keeps the same effective config identity with a new revision lease", async () => { + const f = await fixture(); + const firstLease = await publishDeterministicRuntimeConfigLease({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + const firstIdentity = firstLease.effectiveConfigIdentity; + + writeFileSync( + join(f.source, "psd-clinical", "evidence", "guide.md"), + "# updated content only\n", + ); + await git(f.source, ["add", "psd-clinical/evidence/guide.md"]); + await git(f.source, ["commit", "-m", "Evidence content only"]); + await git(f.source, ["push", "origin", "main"]); + await f.registry.pull(); + const current = (await f.registry.list())[0]; + + const secondLease = await publishDeterministicRuntimeConfigLease({ + workspaceId: "psd-clinical", + registry: f.registry, + registryConfig: f.registryConfig, + harnessDir: f.harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + secretRoots: f.registryConfig.secretRoots, + semanticRuntime, + }); + + expect(secondLease.workspaceRevision).toBe(current.commit); + expect(secondLease.workspaceRevision).not.toBe(firstLease.workspaceRevision); + expect(secondLease.effectiveConfigIdentity).toBe(firstIdentity); + expect(secondLease.path).not.toBe(firstLease.path); +}); diff --git a/backend/test/workspace-runtime-handoff.test.ts b/backend/test/workspace-runtime-handoff.test.ts new file mode 100644 index 00000000..5f29c7ab --- /dev/null +++ b/backend/test/workspace-runtime-handoff.test.ts @@ -0,0 +1,420 @@ +import { execFile } from "node:child_process"; +import { + chmodSync, existsSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test, vi } from "vitest"; +import { parse } from "yaml"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { ThtRunner } from "../src/tht/tht-runner.js"; +import { WorkspaceRegistry } from "../src/workspaces/registry.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const harnessDir = resolve("../harness"); +const thtBin = join(harnessDir, ".venv", "bin", "tht"); +const roots: string[] = []; + +const canonicalWorkspace = `workspace: + schema_version: 3 + id: psd-clinical + name: Runtime handoff + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`; + +const filesystemWorkspace = `${canonicalWorkspace}evidence: + source: + type: filesystem + uri: psd-clinical/evidence +`; + +function evidenceWorkspace(source: string, policy = ""): string { + return `${canonicalWorkspace}evidence: + source: +${source}${policy}`; +} + +afterEach(() => { + vi.unstubAllEnvs(); + roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + return (await runFile("git", args, { cwd })).stdout.trim(); +} + +async function fixture(workspaceSource = filesystemWorkspace) { + const root = mkdtempSync(join(tmpdir(), "tht-runtime-handoff-")); + roots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + const registryRoot = join(root, "registry"); + const secretRoot = join(root, "secrets"); + const dataRoot = join(root, "data"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Runtime Handoff Test"]); + await git(source, ["config", "user.email", "runtime-handoff@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: psd-clinical, name: Runtime handoff}]\n"); + mkdirSync(join(source, "psd-clinical"), { recursive: true }); + writeFileSync(join(source, "psd-clinical", "workspace.yaml"), workspaceSource); + const evidenceRoot = join(source, "psd-clinical", "evidence"); + mkdirSync(evidenceRoot, { recursive: true }); + writeFileSync(join(evidenceRoot, "guide.md"), "# Immutable revision evidence\n"); + await git(source, ["add", "."]); + await git(source, ["commit", "-m", "Canonical workspace"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + mkdirSync(secretRoot); + const secretContents: Record = { + "dwh-password": "dwh-password-value", + "evidence-signed-urls.json": JSON.stringify([ + "https://evidence.example.test/guide.md?token=SIGNED-HANDOFF-CANARY", + ]), + "evidence-access": "ACCESS-HANDOFF-CANARY", + "evidence-secret": "SECRET-HANDOFF-CANARY", + "evidence-token": "TOKEN-HANDOFF-CANARY", + }; + for (const [name, contents] of Object.entries(secretContents)) { + const path = join(secretRoot, name); + writeFileSync(path, contents, { mode: 0o600 }); + chmodSync(path, 0o600); + } + mkdirSync(dataRoot); + const registryConfig: WorkspaceRegistryConfig = { + root: registryRoot, + remoteUrl: remote, + branch: "main", + gitAuthorName: "Runtime Handoff Test", + gitAuthorEmail: "runtime-handoff@example.invalid", + installationId: "test", + secretRoots: [secretRoot], + maxImportBytes: 1024 * 1024, + maxImportEntries: 16, + }; + const registry = new WorkspaceRegistry(registryConfig); + await registry.bootstrap(); + const revision = (await registry.list())[0]; + const environment = { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.invalid", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: join(secretRoot, "dwh-password"), + THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE: join(secretRoot, "evidence-signed-urls.json"), + THT_WS_PSD_CLINICAL_EVIDENCE_ACCESS_KEY_FILE: join(secretRoot, "evidence-access"), + THT_WS_PSD_CLINICAL_EVIDENCE_SECRET_KEY_FILE: join(secretRoot, "evidence-secret"), + THT_WS_PSD_CLINICAL_EVIDENCE_SESSION_TOKEN_FILE: join(secretRoot, "evidence-token"), + }; + for (const [name, value] of Object.entries(environment)) vi.stubEnv(name, value); + vi.stubEnv("THT_HOME", join(root, "home")); + return { root, source, dataRoot, secretRoot, registry, registryConfig, revision }; +} + +function runnerFor(f: Awaited>): ThtRunner { + return new ThtRunner({ + thtBin, + harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"), + secretRoots: f.registryConfig.secretRoots, + } as any); +} + +test("real schema-v3 registry revision loads through ThtRunner and the harness contract", async () => { + const f = await fixture(); + const runner = runnerFor(f); + + expect(await runner.sessionList(f.revision.snapshotPath)).toEqual([]); + const created = await runner.sessionNew({ + question: "runtime handoff", + workspaceConfigPath: f.revision.snapshotPath, + workspaceId: f.revision.id, + workspaceRevision: f.revision.commit, + }); + expect(await runner.sessionShow(created.id, f.revision.snapshotPath)).toMatchObject({ + id: created.id, + workspace_id: "psd-clinical", + workspace_revision: f.revision.commit, + }); + expect(existsSync(join( + f.dataRoot, "sessions", "psd-clinical", "sessions", created.id, "session_manifest.yaml", + ))).toBe(true); + expect(readdirSync(join(f.registryConfig.root, "snapshots", "runtime"))).toEqual([]); +}); + +test("ThtRunner uses a vault secret only for the lifetime of its runtime lease", async () => { + const f = await fixture(); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE", ""); + const vaultRoot = join(f.root, "workspace-secrets"); + const runtimeRoot = join(f.root, "workspace-secret-runtime"); + const secretStore = new WorkspaceSecretStore({ + root: vaultRoot, + runtimeRoot, + installationId: "test", + }); + secretStore.put("psd-clinical", "dwh.password", "vault-runtime-password"); + const runner = new ThtRunner({ + thtBin, + harnessDir, + configPath: "config/tht.yaml", + dataRoot: f.dataRoot, + runtimeSnapshotRoot: join(f.registryConfig.root, "snapshots", "runtime"), + secretRoots: f.registryConfig.secretRoots, + workspaceSecretStore: secretStore, + } as any); + + const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath); + const rendered = parse(readFileSync(lease.path, "utf8")) as { + database: { password_file: string }; + }; + expect(readFileSync(rendered.database.password_file, "utf8")).toBe("vault-runtime-password"); + lease.release(); + expect(existsSync(rendered.database.password_file)).toBe(false); +}); + +test("separate runtime leases hand off byte-identical revision Evidence configs accepted by tht", async () => { + const f = await fixture(); + const runner = runnerFor(f); + const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath); + const second = runner.acquireWorkspaceRuntime(f.revision.snapshotPath); + const expectedRoot = join( + f.registryConfig.root, + "snapshots", + f.revision.commit, + "psd-clinical", + "evidence", + ); + + try { + expect(first.path).not.toBe(second.path); + const firstYaml = readFileSync(first.path, "utf8"); + const secondYaml = readFileSync(second.path, "utf8"); + expect(secondYaml).toBe(firstYaml); + expect(parse(firstYaml).runtime_identity).toEqual({ + workspace_id: "psd-clinical", + workspace_revision: f.revision.commit, + source_identity: "workspace://psd-clinical", + }); + expect(parse(firstYaml).evidence).toEqual({ + sources: [{ + type: "filesystem", + root: expectedRoot, + patterns: ["**/*.md"], + max_bytes: 10_485_760, + }], + }); + expect(parse(firstYaml).vector).toEqual({ + max_chunk_chars: 4_000, + retain_published_generations: 3, + }); + expect(expectedRoot).not.toContain(join(f.registryConfig.root, "repo")); + + for (const lease of [first, second]) { + const checked = await runFile(thtBin, ["config", "check", "-c", lease.path], { + cwd: harnessDir, + env: { ...process.env, THT_HOME: join(f.root, "home") }, + }); + expect(`${checked.stdout}${checked.stderr}`).not.toContain("HANDOFF-CANARY"); + } + + first.release(); + expect(existsSync(first.path)).toBe(false); + expect(existsSync(second.path)).toBe(true); + second.release(); + expect(existsSync(second.path)).toBe(false); + } finally { + first.release(); + second.release(); + } +}); + +test("real Evidence-content-only commit changes runtime identity and root with identical descriptor YAML", async () => { + const f = await fixture(); + const runner = runnerFor(f); + const first = runner.acquireWorkspaceRuntime(f.revision.snapshotPath); + const descriptorBefore = readFileSync(f.revision.snapshotPath, "utf8"); + writeFileSync( + join(f.source, "psd-clinical", "evidence", "guide.md"), + "# Content-only revision two\n", + ); + await git(f.source, ["add", "psd-clinical/evidence/guide.md"]); + await git(f.source, ["commit", "-m", "Update Evidence content only"]); + await git(f.source, ["push", "origin", "main"]); + await f.registry.pull(); + const current = (await f.registry.list())[0]; + const second = runner.acquireWorkspaceRuntime(current.snapshotPath); + + try { + expect(current.commit).not.toBe(f.revision.commit); + expect(current.blob).toBe(f.revision.blob); + expect(readFileSync(current.snapshotPath, "utf8")).toBe(descriptorBefore); + const firstRendered = parse(readFileSync(first.path, "utf8")); + const secondRendered = parse(readFileSync(second.path, "utf8")); + expect(firstRendered.runtime_identity.workspace_revision).toBe(f.revision.commit); + expect(secondRendered.runtime_identity.workspace_revision).toBe(current.commit); + expect(secondRendered.evidence.sources[0].root).toBe(join( + f.registryConfig.root, + "snapshots", + current.commit, + "psd-clinical", + "evidence", + )); + expect(secondRendered.evidence.sources[0].root).not.toBe(firstRendered.evidence.sources[0].root); + + for (const lease of [first, second]) { + await expect(runFile(thtBin, ["config", "check", "-c", lease.path], { + cwd: harnessDir, + env: { ...process.env, THT_HOME: join(f.root, "home") }, + })).resolves.toBeDefined(); + } + } finally { + first.release(); + second.release(); + } +}); + +test("signed HTTP Evidence resolves its file binding and config check never captures its contents", async () => { + const f = await fixture(evidenceWorkspace(` type: http + uris: [https://evidence.example.test/guide.md] + authentication: signed_urls_file + connect_timeout_ms: 1250 + read_timeout_ms: 30001 + max_bytes: 12345 + max_redirects: 2 + allow_private_hosts: false + max_cache_bytes: 67890 +`)); + const runner = runnerFor(f); + const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath); + try { + const yaml = readFileSync(lease.path, "utf8"); + expect(parse(yaml).evidence.sources).toEqual([{ + type: "http", + provenance_urls: ["https://evidence.example.test/guide.md"], + signed_urls_file: realpathSync(join(f.secretRoot, "evidence-signed-urls.json")), + connect_timeout: 1.25, + read_timeout: 30.001, + max_bytes: 12_345, + max_redirects: 2, + allow_private_hosts: false, + max_cache_bytes: 67_890, + }]); + expect(yaml).not.toContain("SIGNED-HANDOFF-CANARY"); + + const checked = await runFile(thtBin, ["config", "check", "-c", lease.path], { + cwd: harnessDir, + env: { ...process.env, THT_HOME: join(f.root, "home") }, + }); + expect(`${checked.stdout}${checked.stderr}`).not.toContain("SIGNED-HANDOFF-CANARY"); + } finally { + lease.release(); + } +}); + +test("static S3 Evidence resolves only configured secret-root file paths", async () => { + const f = await fixture(evidenceWorkspace(` type: s3 + uri: s3://clinical-evidence/published/ + endpoint_url: https://s3.example.test/ + region: eu-west-1 + credentials: static_files + trusted_endpoint: true + allow_private_endpoint: true + allow_insecure_endpoint: false + max_bytes: 222 + max_objects: 33 + max_pages: 4 + page_size: 5 +`, ` policy: + max_chunk_chars: 2500 + retain_published_generations: 7 +`)); + const runner = runnerFor(f); + const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath); + try { + const yaml = readFileSync(lease.path, "utf8"); + expect(parse(yaml).evidence.sources).toEqual([{ + type: "s3", + bucket: "clinical-evidence", + prefix: "published/", + endpoint_url: "https://s3.example.test/", + region: "eu-west-1", + access_key_file: realpathSync(join(f.secretRoot, "evidence-access")), + secret_key_file: realpathSync(join(f.secretRoot, "evidence-secret")), + session_token_file: realpathSync(join(f.secretRoot, "evidence-token")), + trusted_endpoint: true, + allow_private_endpoint: true, + allow_insecure_endpoint: false, + max_bytes: 222, + max_objects: 33, + max_pages: 4, + page_size: 5, + }]); + expect(parse(yaml).vector).toEqual({ + max_chunk_chars: 2_500, + retain_published_generations: 7, + }); + for (const canary of ["ACCESS-HANDOFF-CANARY", "SECRET-HANDOFF-CANARY", "TOKEN-HANDOFF-CANARY"]) { + expect(yaml).not.toContain(canary); + } + + const checked = await runFile(thtBin, ["config", "check", "-c", lease.path], { + cwd: harnessDir, + env: { ...process.env, THT_HOME: join(f.root, "home") }, + }); + const output = `${checked.stdout}${checked.stderr}`; + for (const canary of ["ACCESS-HANDOFF-CANARY", "SECRET-HANDOFF-CANARY", "TOKEN-HANDOFF-CANARY"]) { + expect(output).not.toContain(canary); + } + } finally { + lease.release(); + } +}); + +test("local GET sessions mine uses the real canonical handoff and returns an empty inventory", async () => { + const f = await fixture(); + const app = buildApp(loadConfig({ + AUTH_MODE: "none", + THT_HARNESS_DIR: harnessDir, + THT_BIN: thtBin, + THT_DATA_ROOT: f.dataRoot, + THT_WORKSPACE_REGISTRY_ROOT: f.registryConfig.root, + THT_WORKSPACE_GIT_REMOTE: f.registryConfig.remoteUrl, + THT_WORKSPACE_SECRET_ROOTS: f.registryConfig.secretRoots.join(","), + }), { + thtRunner: runnerFor(f), + workspaceRegistry: f.registry, + mgr: { get: () => undefined } as any, + }); + try { + const response = await app.inject({ method: "GET", url: "/sessions?scope=mine" }); + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual([]); + } finally { + await app.close(); + } +}); diff --git a/backend/test/workspace-runtime-renderer.test.ts b/backend/test/workspace-runtime-renderer.test.ts new file mode 100644 index 00000000..8a151f38 --- /dev/null +++ b/backend/test/workspace-runtime-renderer.test.ts @@ -0,0 +1,431 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { parse } from "yaml"; +import { + renderRuntimeConfig, + type RuntimeBindings, + type RuntimePaths, + type SemanticRuntimeConfig, +} from "../src/workspaces/runtime-renderer.js"; +import { supportsSessionRuntime } from "../src/workspaces/bindings.js"; +import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; + +const workspaceV3 = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct, rest_api, ssh_tunnel] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + default: zai/glm-5.2 + allowed: [zai/glm-5.2] +`); +const paths: RuntimePaths = { + sessions: "/data/workspaces/psd-clinical/sessions", + artifacts: "/data/workspaces/psd-clinical/artifacts", + indexes: "/data/workspaces/psd-clinical/indexes", + memory: "/data/workspaces/psd-clinical/memory", +}; +const semanticRuntime: SemanticRuntimeConfig = { + internalQdrantUrl: "http://qdrant:6333", + internalEmbeddingUrl: "http://embedding:11434", + internalEmbeddingModel: "qwen3-embedding:0.6b", + internalEmbeddingDimensions: 1024, +}; +const directBindings: RuntimeBindings = { + dwh: { + transport: "postgres_direct", + missing: [], + values: { + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "thoth_reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: "/run/secrets/dwh-password", + THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE: "/run/secrets/dwh-ca.pem", + }, + }, + evidence: { missing: [], values: {} }, +}; + +test("renders only the schema-v3 internal Qdrant and Ollama runtime shape", () => { + const rendered = parse(renderRuntimeConfig(workspaceV3, directBindings, paths, { + workspaceId: "psd-clinical", workspaceRevision: "a".repeat(40), + }, {}, semanticRuntime)); + + expect(rendered).toMatchObject({ + runtime_identity: { + workspace_id: "psd-clinical", + workspace_revision: "a".repeat(40), + source_identity: "workspace://psd-clinical", + }, + language: "it", + database: { + host: "dwh.internal", port: 5432, database: "postgres", schema: "datawarehouse", + user: "thoth_reader", password_file: "/run/secrets/dwh-password", + ssl_ca_file: "/run/secrets/dwh-ca.pem", transport: "direct", + }, + dwh: { type: "postgres_direct" }, + resources: { + vector: { engine: "qdrant", base_url: "http://qdrant:6333", collection: "psd-clinical" }, + embeddings: { + provider: "ollama_internal", base_url: "http://embedding:11434", + model: "qwen3-embedding:0.6b", dimensions: 1024, + }, + }, + paths, + }); + expect(rendered).not.toHaveProperty("vector_db"); + expect(rendered).not.toHaveProperty("embeddings"); + expect(rendered).not.toHaveProperty("vector_rest"); +}); + +test("renders schema-v3 DWH REST without exposing secret contents", () => { + const rendered = parse(renderRuntimeConfig(workspaceV3, { + dwh: { + transport: "rest_api", missing: [], values: { + THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test", + THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE: "/run/secrets/dwh-api-key", + }, + }, + evidence: { missing: [], values: {} }, + }, paths)); + + expect(rendered.rest).toEqual({ + base_url: "https://dwh.example.test", api_key_file: "/run/secrets/dwh-api-key", + }); + expect(rendered.dwh).toMatchObject({ + type: "thoth_rest", database: { database: "postgres", schema: "datawarehouse" }, + }); + expect(JSON.stringify(rendered)).not.toContain("api_key:"); +}); + +test("runtime support is fail-closed for DWH SSH and incomplete Evidence", () => { + expect(supportsSessionRuntime(directBindings)).toBe(true); + expect(supportsSessionRuntime({ + ...directBindings, dwh: { ...directBindings.dwh, transport: "ssh_tunnel" }, + })).toBe(false); + expect(supportsSessionRuntime({ + ...directBindings, evidence: { values: {}, missing: ["EVIDENCE_FILE"] }, + })).toBe(false); +}); + +const evidenceSecretRoots: string[] = []; + +afterEach(() => { + evidenceSecretRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +function evidenceSecretFile(name: string, contents: string): string { + const root = mkdtempSync(join(tmpdir(), "tht-renderer-evidence-secret-")); + evidenceSecretRoots.push(root); + const path = join(root, name); + writeFileSync(path, contents, { mode: 0o600 }); + return path; +} + +function evidenceWorkspace(source: Record, policy?: Record) { + return parseWorkspaceYaml(`${canonicalEvidenceWorkspace}\nevidence:\n source: ${JSON.stringify(source)}${ + policy === undefined ? "" : `\n policy: ${JSON.stringify(policy)}` + }\n`); +} + +const canonicalEvidenceWorkspace = `workspace: + schema_version: 3 + id: psd-clinical + name: Runtime Evidence + language: en +dwh: + engine: postgres + database: analytics + schema: mart + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`; + +const evidenceRevision = "1".repeat(40); +const evidenceContext = { + workspaceId: "psd-clinical", + workspaceRevision: evidenceRevision, + revisionContentRoot: `/srv/registry/snapshots/${evidenceRevision}`, +}; + +function evidenceRender( + source: Record, + evidenceBinding: RuntimeBindings["evidence"] = { missing: [], values: {} }, + policy?: Record, +) { + return renderRuntimeConfig( + evidenceWorkspace(source, policy), + { ...directBindings, evidence: evidenceBinding }, + paths, + evidenceContext, + {}, + semanticRuntime, + ); +} + +test("renders filesystem Evidence below the immutable revision content root with default policy", () => { + const yaml = evidenceRender({ + type: "filesystem", + uri: "psd-clinical/evidence", + }); + const rendered = parse(yaml); + + expect(rendered.runtime_identity.workspace_revision).toBe(evidenceRevision); + expect(rendered.evidence).toEqual({ + sources: [{ + type: "filesystem", + root: `/srv/registry/snapshots/${evidenceRevision}/psd-clinical/evidence`, + patterns: ["**/*.md"], + max_bytes: 10_485_760, + }], + }); + expect(rendered.vector).toEqual({ + max_chunk_chars: 4_000, + retain_published_generations: 3, + }); + expect(yaml).not.toContain("/srv/registry/repo"); +}); + +test("renders public HTTP Evidence with exact fractional-second timeouts and every policy limit", () => { + const rendered = parse(evidenceRender({ + type: "http", + uris: ["https://evidence.example.test/guide.md"], + authentication: "none", + connect_timeout_ms: 1_001, + read_timeout_ms: 30_001, + max_bytes: 12_345, + max_redirects: 0, + allow_private_hosts: true, + max_cache_bytes: 67_890, + }, undefined, { + max_chunk_chars: 2_501, + retain_published_generations: 7, + })); + + expect(rendered.evidence).toEqual({ + sources: [{ + type: "http", + urls: ["https://evidence.example.test/guide.md"], + connect_timeout: 1.001, + read_timeout: 30.001, + max_bytes: 12_345, + max_redirects: 0, + allow_private_hosts: true, + max_cache_bytes: 67_890, + }], + }); + expect(rendered.vector).toEqual({ + max_chunk_chars: 2_501, + retain_published_generations: 7, + }); +}); + +test("renders signed HTTP Evidence as provenance plus a validated file path only", () => { + const canary = "SIGNED-URL-CANARY-CONTENT"; + const signedFile = evidenceSecretFile("evidence-signed-urls.json", canary); + const yaml = evidenceRender({ + type: "http", + uris: [ + "https://evidence.example.test/guide.md", + "https://evidence.example.test/runbook.md", + ], + authentication: "signed_urls_file", + }, { + missing: [], + values: { THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE: signedFile }, + }); + + expect(parse(yaml).evidence.sources).toEqual([{ + type: "http", + provenance_urls: [ + "https://evidence.example.test/guide.md", + "https://evidence.example.test/runbook.md", + ], + signed_urls_file: signedFile, + connect_timeout: 5, + read_timeout: 30, + max_bytes: 10_485_760, + max_redirects: 5, + allow_private_hosts: false, + max_cache_bytes: 67_108_864, + }]); + expect(yaml).not.toContain(canary); +}); + +test("renders ambient S3 Evidence without credential keys", () => { + const rendered = parse(evidenceRender({ + type: "s3", + uri: "s3://clinical-evidence/published/guides/", + credentials: "ambient", + region: "eu-west-1", + })); + + expect(rendered.evidence.sources).toEqual([{ + type: "s3", + bucket: "clinical-evidence", + prefix: "published/guides/", + region: "eu-west-1", + trusted_endpoint: false, + allow_private_endpoint: false, + allow_insecure_endpoint: false, + max_bytes: 10_485_760, + max_objects: 10_000, + max_pages: 100, + page_size: 1_000, + }]); + expect(JSON.stringify(rendered.evidence)).not.toMatch(/access_key|secret_key|session_token/); +}); + +test("renders static S3 Evidence with endpoint policy, limits, and file paths but no contents", () => { + const accessCanary = "ACCESS-CANARY-CONTENT"; + const secretCanary = "SECRET-CANARY-CONTENT"; + const tokenCanary = "TOKEN-CANARY-CONTENT"; + const accessFile = evidenceSecretFile("evidence-access", accessCanary); + const secretFile = evidenceSecretFile("evidence-secret", secretCanary); + const tokenFile = evidenceSecretFile("evidence-token", tokenCanary); + const source = { + type: "s3", + uri: "s3://clinical-evidence/published/", + credentials: "static_files", + endpoint_url: "http://minio.internal:9000/", + region: "eu-central-1", + trusted_endpoint: true, + allow_private_endpoint: true, + allow_insecure_endpoint: true, + max_bytes: 222, + max_objects: 33, + max_pages: 4, + page_size: 5, + }; + const values = { + THT_WS_PSD_CLINICAL_EVIDENCE_ACCESS_KEY_FILE: accessFile, + THT_WS_PSD_CLINICAL_EVIDENCE_SECRET_KEY_FILE: secretFile, + THT_WS_PSD_CLINICAL_EVIDENCE_SESSION_TOKEN_FILE: tokenFile, + }; + const yaml = evidenceRender(source, { missing: [], values }); + + expect(parse(yaml).evidence.sources).toEqual([{ + type: "s3", + bucket: "clinical-evidence", + prefix: "published/", + endpoint_url: "http://minio.internal:9000/", + region: "eu-central-1", + access_key_file: accessFile, + secret_key_file: secretFile, + session_token_file: tokenFile, + trusted_endpoint: true, + allow_private_endpoint: true, + allow_insecure_endpoint: true, + max_bytes: 222, + max_objects: 33, + max_pages: 4, + page_size: 5, + }]); + expect(yaml).not.toContain(accessCanary); + expect(yaml).not.toContain(secretCanary); + expect(yaml).not.toContain(tokenCanary); + + const withoutToken = parse(evidenceRender(source, { + missing: [], + values: { + THT_WS_PSD_CLINICAL_EVIDENCE_ACCESS_KEY_FILE: accessFile, + THT_WS_PSD_CLINICAL_EVIDENCE_SECRET_KEY_FILE: secretFile, + }, + })).evidence.sources[0]; + expect(withoutToken).toMatchObject({ access_key_file: accessFile, secret_key_file: secretFile }); + expect(withoutToken).not.toHaveProperty("session_token_file"); +}); + +test("omits Evidence configuration and policy when the descriptor has no Evidence", () => { + const rendered = parse(renderRuntimeConfig( + workspaceV3, + directBindings, + paths, + evidenceContext, + {}, + semanticRuntime, + )); + + expect(rendered).not.toHaveProperty("evidence"); + expect(rendered).not.toHaveProperty("vector"); +}); + +test.each([ + { + source: { + type: "http", uris: ["https://evidence.example.test/guide.md"], + authentication: "signed_urls_file", + }, + missing: "THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE", + }, + { + source: { + type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files", + }, + missing: "THT_WS_PSD_CLINICAL_EVIDENCE_SECRET_KEY_FILE", + }, +])("rejects missing required Evidence binding $missing before rendering", ({ source, missing }) => { + expect(() => evidenceRender(source, { missing: [missing], values: {} })).toThrow( + "runtime configuration requires complete Evidence bindings", + ); +}); + +test("is byte deterministic and revision-bound for descriptor-identical content-only commits", () => { + const source = { + type: "filesystem", + uri: "psd-clinical/evidence", + }; + const first = evidenceRender(source); + expect(evidenceRender(source)).toBe(first); + + const nextRevision = "2".repeat(40); + const next = renderRuntimeConfig( + evidenceWorkspace(source), + directBindings, + paths, + { + workspaceId: "psd-clinical", + workspaceRevision: nextRevision, + revisionContentRoot: `/srv/registry/snapshots/${nextRevision}`, + }, + {}, + semanticRuntime, + ); + const firstParsed = parse(first); + const nextParsed = parse(next); + + expect(next).not.toBe(first); + expect(nextParsed.runtime_identity.workspace_revision).toBe(nextRevision); + expect(nextParsed.evidence.sources[0].root).toBe( + `/srv/registry/snapshots/${nextRevision}/psd-clinical/evidence`, + ); + expect(nextParsed.evidence.sources[0].root).not.toBe(firstParsed.evidence.sources[0].root); +}); diff --git a/backend/test/workspace-secret-requirements.test.ts b/backend/test/workspace-secret-requirements.test.ts new file mode 100644 index 00000000..58d6582a --- /dev/null +++ b/backend/test/workspace-secret-requirements.test.ts @@ -0,0 +1,138 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; + +import { + discoverWorkspaceSecretRequirements, + resolveRuntimeBindingsWithWorkspaceSecrets, +} from "../src/workspaces/secret-requirements.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; +import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; + +const roots: string[] = []; + +function workspace(extra = "") { + return parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: en +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct, rest_api, ssh_tunnel] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: { allowed: [zai/glm-5.2] } +${extra}`); +} + +function store() { + const root = mkdtempSync(join(tmpdir(), "thoth-requirement-vault-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "thoth-requirement-runtime-")); + roots.push(root, runtimeRoot); + return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test-installation" }); +} + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +test("requirements follow the selected DWH transport", () => { + const descriptor = workspace(); + const direct = discoverWorkspaceSecretRequirements(descriptor, { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + }); + expect(direct.map(({ id, required }) => ({ id, required }))).toEqual([ + { id: "dwh.password", required: true }, + ]); + + const rest = discoverWorkspaceSecretRequirements(descriptor, { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api", + }); + expect(rest.map(({ id }) => id)).toEqual(["dwh.api_key"]); + + const ssh = discoverWorkspaceSecretRequirements(descriptor, { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "ssh_tunnel", + }); + expect(ssh.map(({ id }) => id)).toEqual(["dwh.password", "dwh.ssh_private_key"]); +}); + +test("REST without authentication does not request an API key", () => { + const descriptor = workspace(`diagnostics: + dwh_rest: + method: GET + path: /health + auth: none + response: { database: database, schema: schema } +`); + expect(discoverWorkspaceSecretRequirements(descriptor, { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api", + })).toEqual([]); +}); + +test("evidence requirements follow the descriptor authentication mechanism", () => { + const signed = workspace(`evidence: + source: + type: http + uris: [https://evidence.example.test/guide.md] + authentication: signed_urls_file + policy: { max_chunk_chars: 4000, retain_published_generations: 2 } +`); + expect(discoverWorkspaceSecretRequirements(signed, { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + }).map(({ id, required }) => ({ id, required }))).toEqual([ + { id: "dwh.password", required: true }, + { id: "evidence.signed_urls", required: true }, + ]); + + const s3 = workspace(`evidence: + source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files } + policy: { max_chunk_chars: 4000, retain_published_generations: 2 } +`); + expect(discoverWorkspaceSecretRequirements(s3, { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + }).map(({ id, required }) => ({ id, required }))).toEqual([ + { id: "dwh.password", required: true }, + { id: "evidence.access_key", required: true }, + { id: "evidence.secret_key", required: true }, + { id: "evidence.session_token", required: false }, + ]); +}); + +test("vault values are mapped to temporary file bindings and released", () => { + const descriptor = workspace(); + const vault = store(); + vault.put("psd-clinical", "dwh.password", "runtime-password"); + const environment = { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + }; + + const lease = resolveRuntimeBindingsWithWorkspaceSecrets(descriptor, environment, [], vault); + expect(lease.bindings.dwh.missing).toEqual([]); + const secretPath = lease.bindings.dwh.values.THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE!; + expect(readFileSync(secretPath, "utf8")).toBe("runtime-password"); + lease.release(); + expect(() => readFileSync(secretPath, "utf8")).toThrow(); +}); + +test("missing vault values remain missing and materialization never mutates the source environment", () => { + const descriptor = workspace(); + const vault = store(); + const environment = { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + }; + const lease = resolveRuntimeBindingsWithWorkspaceSecrets(descriptor, environment, [], vault); + expect(lease.bindings.dwh.missing).toContain("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"); + expect(environment).not.toHaveProperty("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"); + lease.release(); +}); diff --git a/backend/test/workspace-secret-store.test.ts b/backend/test/workspace-secret-store.test.ts new file mode 100644 index 00000000..bd9e82c0 --- /dev/null +++ b/backend/test/workspace-secret-store.test.ts @@ -0,0 +1,129 @@ +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; + +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; + +const roots: string[] = []; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-secret-store-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "thoth-workspace-secret-runtime-")); + roots.push(root, runtimeRoot); + return { + root, + runtimeRoot, + store: new WorkspaceSecretStore({ + root, + runtimeRoot, + installationId: "installation-test", + }), + }; +} + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("WorkspaceSecretStore", () => { + test("reuses stable materialized paths and removes them after the last lease", () => { + const { store } = fixture(); + store.put("north-star", "dwh.password", "correct horse battery staple"); + + const first = store.materialize("north-star", ["dwh.password"]); + const second = store.materialize("north-star", ["dwh.password"]); + const firstPath = first.files.get("dwh.password")!; + const secondPath = second.files.get("dwh.password")!; + + expect(secondPath).toBe(firstPath); + expect(readFileSync(firstPath, "utf8")).toBe("correct horse battery staple"); + first.release(); + expect(readFileSync(secondPath, "utf8")).toBe("correct horse battery staple"); + second.release(); + expect(existsSync(secondPath)).toBe(false); + }); + + test("persists ciphertext and exposes status without exposing plaintext", () => { + const { root, store } = fixture(); + const secret = "correct horse battery staple"; + + store.put("psd-clinical", "dwh.password", secret); + + expect(store.has("psd-clinical", "dwh.password")).toBe(true); + expect(store.configured("psd-clinical")).toEqual(["dwh.password"]); + const vault = readFileSync(join(root, "vault.json"), "utf8"); + expect(vault).not.toContain(secret); + expect(statSync(join(root, "vault.json")).mode & 0o777).toBe(0o600); + expect(statSync(join(root, "master.key")).mode & 0o777).toBe(0o600); + }); + + test("blind replacement changes the materialized value and forget removes it", () => { + const { store } = fixture(); + store.put("psd-clinical", "dwh.password", "old-value"); + store.put("psd-clinical", "dwh.password", "new-value"); + + const lease = store.materialize("psd-clinical", ["dwh.password"]); + const path = lease.files.get("dwh.password"); + expect(path).toBeDefined(); + expect(readFileSync(path!, "utf8")).toBe("new-value"); + expect(statSync(path!).mode & 0o777).toBe(0o400); + lease.release(); + expect(existsSync(path!)).toBe(false); + + store.forget("psd-clinical", "dwh.password"); + expect(store.has("psd-clinical", "dwh.password")).toBe(false); + }); + + test("materializes only requested secrets and cleans the whole lease directory", () => { + const { runtimeRoot, store } = fixture(); + store.putMany("psd-clinical", { + "dwh.password": "warehouse-password", + "evidence.api_key": "evidence-key", + }); + + const lease = store.materialize("psd-clinical", ["evidence.api_key"]); + expect([...lease.files.keys()]).toEqual(["evidence.api_key"]); + expect(readFileSync(lease.files.get("evidence.api_key")!, "utf8")).toBe("evidence-key"); + expect(statSync(runtimeRoot).mode & 0o777).toBe(0o700); + const directory = join(lease.files.get("evidence.api_key")!, ".."); + lease.release(); + expect(existsSync(directory)).toBe(false); + }); + + test("fails closed with a sanitized error when the encrypted vault is tampered", () => { + const { root, store } = fixture(); + const secret = "must-never-appear-in-errors"; + store.put("psd-clinical", "dwh.password", secret); + + const path = join(root, "vault.json"); + const document = JSON.parse(readFileSync(path, "utf8")) as { + entries: Record; + }; + const record = Object.values(document.entries)[0]!; + record.ciphertext = Buffer.from("tampered").toString("base64"); + writeFileSync(path, JSON.stringify(document), { mode: 0o600 }); + + expect(() => store.materialize("psd-clinical", ["dwh.password"])) + .toThrow("Workspace secret store is unavailable."); + try { + store.materialize("psd-clinical", ["dwh.password"]); + } catch (error) { + expect(String(error)).not.toContain(secret); + } + }); + + test("rejects invalid identifiers and oversized values", () => { + const { store } = fixture(); + expect(() => store.put("../workspace", "dwh.password", "secret")).toThrow(); + expect(() => store.put("psd-clinical", "../password", "secret")).toThrow(); + expect(() => store.put("psd-clinical", "dwh.password", "x".repeat(65_537))).toThrow(); + }); +}); diff --git a/backend/test/workspaces-bindings.test.ts b/backend/test/workspaces-bindings.test.ts new file mode 100644 index 00000000..5b9209e8 --- /dev/null +++ b/backend/test/workspaces-bindings.test.ts @@ -0,0 +1,272 @@ +import { + chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { + resolveBinding, resolveEvidenceBinding, resolveRuntimeBindings, supportsSessionRuntime, +} from "../src/workspaces/bindings.js"; +import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; + +const workspaceV3 = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct, rest_api, ssh_tunnel] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: { allowed: [zai/glm-5.2] } +`); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +function secretPath(name: string): { root: string; path: string } { + const root = mkdtempSync(join(tmpdir(), "thoth-binding-")); + temporaryRoots.push(root); + const secrets = join(root, "secrets"); + mkdirSync(secrets); + const path = join(secrets, name); + writeFileSync(path, ""); + return { root: secrets, path }; +} + +test("resolves schema-v3 direct DWH bindings from the stable namespace", () => { + const password = secretPath("dwh-password"); + const result = resolveBinding(workspaceV3, "DWH", { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: password.path, + }, [password.root]); + + expect(result).toMatchObject({ + transport: "postgres_direct", + missing: [], + values: { + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: realpathSync(password.path), + }, + }); +}); + +test("requires schema-v3 REST credentials unless the DWH diagnostic declares auth none", () => { + expect(resolveBinding(workspaceV3, "DWH", { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api", + THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test", + }, []).missing).toContain("THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE"); + + const noAuth = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: No auth + language: en +dwh: + engine: postgres + database: postgres + schema: public + supported_transports: [rest_api] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +diagnostics: + dwh_rest: + method: GET + path: /health + auth: none + response: { database: database, schema: schema } +llm_policy: { allowed: [zai/glm-5.2] } +`); + expect(resolveBinding(noAuth, "DWH", { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api", + THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test", + }, []).missing).toEqual([]); +}); + +test("rejects unsupported transports and secret paths outside configured roots", () => { + const outside = secretPath("outside-password"); + const allowed = secretPath("allowed-password"); + const directOnly = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Direct only + language: en +dwh: + engine: postgres + database: postgres + schema: public + supported_transports: [postgres_direct] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: { allowed: [zai/glm-5.2] } +`); + expect(resolveBinding(directOnly, "DWH", { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api", + }, []).missing).toContain("THT_WS_PSD_CLINICAL_DWH_TRANSPORT"); + const result = resolveBinding(workspaceV3, "DWH", { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: outside.path, + }, [allowed.root]); + expect(result.missing).toContain("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"); + expect(JSON.stringify(result)).not.toContain(outside.path); +}); + +test("runtime bindings contain only DWH and Evidence roles", () => { + const password = secretPath("dwh-password"); + const bindings = resolveRuntimeBindings(workspaceV3, { + THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct", + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: password.path, + THT_WS_PSD_CLINICAL_VECTOR_BASE_URL: "https://ignored.example.test", + }, [password.root]); + expect(Object.keys(bindings)).toEqual(["dwh", "evidence"]); + expect(bindings.dwh.missing).toEqual([]); + expect(supportsSessionRuntime(bindings)).toBe(true); +}); + +function withEvidence(source: Record) { + return parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: { allowed: [zai/glm-5.2] } +evidence: + source: ${JSON.stringify(source)} +`); +} + +const evidenceVariable = (suffix: string) => `THT_WS_PSD_CLINICAL_EVIDENCE_${suffix}`; + +test.each([ + { type: "filesystem", uri: "psd-clinical/evidence" }, + { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" }, + { type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" }, +])("does not resolve Evidence variables for $type modes without file credentials", (source) => { + expect(resolveEvidenceBinding(withEvidence(source), { + [evidenceVariable("SIGNED_URLS_FILE")]: "/CANARY/http", + [evidenceVariable("ACCESS_KEY_FILE")]: "/CANARY/access", + }, ["/run/secrets"])).toEqual({ values: {}, missing: [] }); +}); + +test("requires only a safe HTTP signed-URL file and never reads its contents", () => { + const signed = secretPath("evidence-signed-urls"); + writeFileSync(signed.path, "CANARY-SIGNED-URL-CONTENT"); + const source = withEvidence({ + type: "http", + uris: ["https://evidence.example.test/guide.md"], + authentication: "signed_urls_file", + }); + const variable = evidenceVariable("SIGNED_URLS_FILE"); + + expect(resolveEvidenceBinding(source, {}, [signed.root]).missing).toEqual([variable]); + const resolved = resolveEvidenceBinding(source, { + [variable]: signed.path, + [evidenceVariable("ACCESS_KEY_FILE")]: signed.path, + }, [signed.root]); + expect(resolved).toEqual({ values: { [variable]: realpathSync(signed.path) }, missing: [] }); + expect(JSON.stringify(resolved)).not.toContain("CANARY-SIGNED-URL-CONTENT"); +}); + +test("canonicalizes an in-root Evidence symlink before passing it to the harness", () => { + const signed = secretPath("evidence-signed-target"); + const link = join(signed.root, "signed-urls-link"); + symlinkSync(signed.path, link); + const source = withEvidence({ + type: "http", + uris: ["https://evidence.example.test/guide.md"], + authentication: "signed_urls_file", + }); + const variable = evidenceVariable("SIGNED_URLS_FILE"); + + expect(resolveEvidenceBinding(source, { [variable]: link }, [signed.root])).toEqual({ + values: { [variable]: realpathSync(signed.path) }, missing: [], + }); +}); + +test("requires S3 access and secret files together while accepting an optional safe session token", () => { + const access = secretPath("evidence-access"); + const secret = secretPath("evidence-secret"); + const token = secretPath("evidence-token"); + const source = withEvidence({ + type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files", + }); + const env = { + [evidenceVariable("ACCESS_KEY_FILE")]: access.path, + [evidenceVariable("SECRET_KEY_FILE")]: secret.path, + [evidenceVariable("SESSION_TOKEN_FILE")]: token.path, + [evidenceVariable("SIGNED_URLS_FILE")]: access.path, + }; + + expect(resolveEvidenceBinding(source, { + [evidenceVariable("ACCESS_KEY_FILE")]: access.path, + }, [access.root]).missing).toEqual([evidenceVariable("SECRET_KEY_FILE")]); + expect(resolveEvidenceBinding(source, env, [access.root, secret.root, token.root])).toEqual({ + values: { + [evidenceVariable("ACCESS_KEY_FILE")]: realpathSync(access.path), + [evidenceVariable("SECRET_KEY_FILE")]: realpathSync(secret.path), + [evidenceVariable("SESSION_TOKEN_FILE")]: realpathSync(token.path), + }, + missing: [], + }); +}); + +test("rejects relative, missing, directory, unreadable, and escaping symlink Evidence paths", () => { + const allowed = secretPath("valid"); + const outside = secretPath("outside"); + const directory = join(allowed.root, "directory"); + mkdirSync(directory); + const link = join(allowed.root, "escape"); + symlinkSync(outside.path, link); + const unreadable = join(allowed.root, "unreadable"); + writeFileSync(unreadable, "secret"); + chmodSync(unreadable, 0o000); + const source = withEvidence({ + type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", + }); + const variable = evidenceVariable("SIGNED_URLS_FILE"); + + for (const path of ["relative", join(allowed.root, "missing"), directory, unreadable, link]) { + expect(resolveEvidenceBinding(source, { [variable]: path }, [allowed.root])).toEqual({ + values: {}, missing: [variable], + }); + } + chmodSync(unreadable, 0o600); +}); + +test("includes Evidence binding completeness in session runtime support without changing v3 compatibility", () => { + const unsigned = resolveRuntimeBindings(workspaceV3, {}, ["/run/secrets"]); + expect(unsigned.evidence).toEqual({ values: {}, missing: [] }); + expect(supportsSessionRuntime(unsigned)).toBe(true); + + const signed = resolveRuntimeBindings(withEvidence({ + type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file", + }), {}, ["/run/secrets"]); + expect(signed.evidence.missing).toEqual([evidenceVariable("SIGNED_URLS_FILE")]); + expect(supportsSessionRuntime(signed)).toBe(false); +}); diff --git a/backend/test/workspaces-catalog.test.ts b/backend/test/workspaces-catalog.test.ts new file mode 100644 index 00000000..23b71a66 --- /dev/null +++ b/backend/test/workspaces-catalog.test.ts @@ -0,0 +1,82 @@ +import { expect, test } from "vitest"; +import { parseWorkspaceYaml } from "../src/workspaces/schema.js"; +import { + CATALOG_PATH, + assertCatalogMatchesDescriptor, + parseWorkspaceCatalogYaml, +} from "../src/workspaces/catalog.js"; + +const descriptor = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd + name: Policlinico San Donato + description: Clinical warehouse + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [rest_api] +semantic_index: + vector_store: { engine: qdrant, collection: psd, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: { allowed: [zai/glm-5.2] } +`); + +test("parses the strict ordered root catalog", () => { + expect(CATALOG_PATH).toBe("thoth-workspaces.yaml"); + expect(parseWorkspaceCatalogYaml(`schema_version: 1 +workspaces: + - id: psd + name: Policlinico San Donato + description: Clinical warehouse + - id: research + name: Research +`)).toEqual({ + schema_version: 1, + workspaces: [ + { id: "psd", name: "Policlinico San Donato", description: "Clinical warehouse" }, + { id: "research", name: "Research" }, + ], + }); +}); + +test("preserves optional description absence and trims metadata", () => { + expect(parseWorkspaceCatalogYaml(`schema_version: 1 +workspaces: + - id: psd + name: " PSD " +`)).toEqual({ schema_version: 1, workspaces: [{ id: "psd", name: "PSD" }] }); +}); + +test.each([ + ["duplicate IDs", `schema_version: 1\nworkspaces: [{id: psd, name: One}, {id: psd, name: Two}]`], + ["invalid ID", `schema_version: 1\nworkspaces: [{id: PSD, name: One}]`], + ["reserved ID", `schema_version: 1\nworkspaces: [{id: workspace-docs, name: One}]`], + ["unknown key", `schema_version: 1\nworkspaces: [{id: psd, name: One, secret: CANARY}]`], + ["duplicate YAML key", `schema_version: 1\nworkspaces:\n - id: psd\n id: research\n name: One`], + ["multiple documents", `schema_version: 1\nworkspaces: []\n---\nschema_version: 1\nworkspaces: []`], +])("rejects %s without exposing unsafe input", (_name, source) => { + expect(() => parseWorkspaceCatalogYaml(source)).toThrow(/catalog|workspace|id|YAML/i); + try { parseWorkspaceCatalogYaml(source); } catch (error) { + expect(String(error)).not.toContain("CANARY"); + } +}); + +test("rejects catalog metadata that differs from its descriptor", () => { + expect(() => assertCatalogMatchesDescriptor( + { id: "psd", name: "Other", description: "Clinical warehouse" }, descriptor, + )).toThrow(/catalog|metadata/i); + expect(() => assertCatalogMatchesDescriptor( + { id: "psd", name: "Policlinico San Donato" }, descriptor, + )).toThrow(/catalog|metadata/i); + expect(() => assertCatalogMatchesDescriptor( + { id: "other", name: "Policlinico San Donato", description: "Clinical warehouse" }, descriptor, + )).toThrow(/catalog|metadata/i); +}); + +test("accepts exact catalog metadata", () => { + expect(() => assertCatalogMatchesDescriptor( + { id: "psd", name: "Policlinico San Donato", description: "Clinical warehouse" }, descriptor, + )).not.toThrow(); +}); diff --git a/backend/test/workspaces-config.test.ts b/backend/test/workspaces-config.test.ts new file mode 100644 index 00000000..65a455e6 --- /dev/null +++ b/backend/test/workspaces-config.test.ts @@ -0,0 +1,64 @@ +import { expect, test } from "vitest"; +import { loadConfig } from "../src/config.js"; + +test("loads a safe Git workspace registry configuration", () => { + const cfg = loadConfig({ + THT_WORKSPACE_REGISTRY_ROOT: "/data/workspace-registry", + THT_WORKSPACE_GIT_REMOTE: "ssh://git@gitea.example/thoth/workspaces.git", + THT_WORKSPACE_GIT_BRANCH: "main", + THT_WORKSPACE_INSTALLATION_ID: "server-psd-1", + THT_WORKSPACE_SECRET_ROOTS: "/run/secrets,/data/secrets", + }); + + expect(cfg.workspaceRegistry).toMatchObject({ + root: "/data/workspace-registry", + remoteUrl: "ssh://git@gitea.example/thoth/workspaces.git", + branch: "main", + installationId: "server-psd-1", + secretRoots: ["/run/secrets", "/data/secrets"], + }); +}); + +test("uses safe workspace registry defaults", () => { + const registry = loadConfig({}).workspaceRegistry; + expect(registry).toMatchObject({ + root: "/data/workspace-registry", + branch: "main", + }); + expect(registry).not.toHaveProperty("gitAuthorName"); + expect(registry).not.toHaveProperty("gitAuthorEmail"); + expect(registry).not.toHaveProperty("maxImportBytes"); + expect(registry).not.toHaveProperty("maxImportEntries"); +}); + +test("rejects a relative registry root", () => { + expect(() => loadConfig({ THT_WORKSPACE_REGISTRY_ROOT: "registry" })).toThrow(/registry/i); +}); + +test("rejects unsafe registry branch, installation ID, and secret roots", () => { + expect(() => loadConfig({ THT_WORKSPACE_GIT_BRANCH: "" })).toThrow(/branch/i); + expect(() => loadConfig({ THT_WORKSPACE_INSTALLATION_ID: "" })).toThrow(/installation/i); + expect(() => loadConfig({ THT_WORKSPACE_SECRET_ROOTS: "/run/secrets,relative" })) + .toThrow(/secret/i); +}); + +test("rejects ref-unsafe Git branches", () => { + for (const branch of ["topic..bad", "--upload-pack=/tmp/x", "release/.hidden", "release.lock"]) { + expect(() => loadConfig({ THT_WORKSPACE_GIT_BRANCH: branch })).toThrow(/branch/i); + } +}); + +test("rejects the reserved HEAD branch without rejecting lowercase head", () => { + expect(() => loadConfig({ THT_WORKSPACE_GIT_BRANCH: "HEAD" })).toThrow(/branch/i); + expect(loadConfig({ THT_WORKSPACE_GIT_BRANCH: "head" }).workspaceRegistry.branch).toBe("head"); +}); + +test("rejects control characters in installation IDs", () => { + for (const codePoint of [...Array(0x20).keys(), ...Array(0x21).keys()].map((code, index) => ( + index < 0x20 ? code : code + 0x7f + ))) { + expect(() => loadConfig({ + THT_WORKSPACE_INSTALLATION_ID: `server-psd-1${String.fromCodePoint(codePoint)}`, + })).toThrow(/installation/i); + } +}); diff --git a/backend/test/workspaces-contracts.test.ts b/backend/test/workspaces-contracts.test.ts new file mode 100644 index 00000000..261a5a90 --- /dev/null +++ b/backend/test/workspaces-contracts.test.ts @@ -0,0 +1,262 @@ +import { expect, test } from "vitest"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildInstallationContract, renderWorkspaceDocs } from "../src/workspaces/contracts.js"; +import { type CanonicalWorkspace, parseWorkspaceYaml } from "../src/workspaces/schema.js"; + +const workspaceV3 = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct, rest_api, ssh_tunnel] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: + allowed: [zai/glm-5.2] +`); + +test("schema-v3 installation contracts expose only DWH bindings and no semantic variables", () => { + const contract = buildInstallationContract(workspaceV3); + const names = contract.variables.map((variable) => variable.name); + const docs = renderWorkspaceDocs(workspaceV3); + + expect(contract.workspaceId).toBe("psd-clinical"); + expect(contract.namespace).toBe("PSD_CLINICAL"); + expect(contract.variables.every((variable) => variable.role === "DWH")).toBe(true); + expect(names).toContain("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"); + expect(names.some((name) => /_VECTOR_|_EMBEDDING_/.test(name))).toBe(false); + expect(docs.envExample).not.toContain("_VECTOR_"); + expect(docs.markdown).toContain("Configurazione dell'installazione"); + expect(docs.markdown).not.toContain("Vector store"); + expect(docs.markdown).not.toContain("Embedding service"); +}); + +test.each([ + ["postgres_direct", "HOST", "BASE_URL"], + ["rest_api", "BASE_URL", "HOST"], + ["ssh_tunnel", "SSH_PRIVATE_KEY_FILE", "BASE_URL"], +] as const)("documents only the DWH fields for %s", (transport, included, excluded) => { + const descriptor = parseWorkspaceYaml(renderWorkspaceWithoutEvidence() + .replace("[postgres_direct]", `[${transport}]`)); + const variables = buildInstallationContract(descriptor).variables; + expect(variables.find(({ suffix }) => suffix === included)?.transports).toEqual([transport]); + expect(variables.some(({ suffix }) => suffix === excluded)).toBe(false); + expect(variables.filter(({ secret }) => secret).every(({ name }) => name.endsWith("_FILE"))) + .toBe(true); +}); + +test("validates public contract and documentation inputs at runtime", () => { + const unsafeWorkspace = { + ...workspaceV3, + workspace: { ...workspaceV3.workspace, id: "psd\nclinical" }, + } as CanonicalWorkspace; + + expect(() => buildInstallationContract(unsafeWorkspace)).toThrow(/id/i); + expect(() => renderWorkspaceDocs(unsafeWorkspace)).toThrow(/id/i); +}); + +test("v3 installation contract omits external vector and embedding bindings", () => { + const contract = buildInstallationContract(workspaceV3); + const names = contract.variables.map((variable) => variable.name); + + expect(names).toContain("THT_WS_PSD_CLINICAL_DWH_TRANSPORT"); + expect(names.some((name) => name.includes("_VECTOR_"))).toBe(false); + expect(names.some((name) => name.includes("_EMBEDDING_"))).toBe(false); + expect(renderWorkspaceDocs(workspaceV3).markdown).not.toContain("Embedding service"); +}); + + +test.each([ + { + mode: "signed HTTP", + source: { + type: "http", uris: ["https://evidence.example.test/guide.md"], + authentication: "signed_urls_file", + }, + expected: ["THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE"], + }, + { + mode: "static S3", + source: { type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files" }, + expected: [ + "THT_WS_PSD_CLINICAL_EVIDENCE_ACCESS_KEY_FILE", + "THT_WS_PSD_CLINICAL_EVIDENCE_SECRET_KEY_FILE", + "THT_WS_PSD_CLINICAL_EVIDENCE_SESSION_TOKEN_FILE", + ], + }, +])("generates source-specific $mode Evidence file bindings", ({ source, expected }) => { + const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`); + const contract = buildInstallationContract(descriptor); + const evidence = contract.variables.filter((variable) => variable.role === "EVIDENCE"); + + expect(contract.namespace).toBe("PSD_CLINICAL"); + expect(evidence.map((variable) => variable.name)).toEqual(expected); + expect(evidence.every((variable) => variable.secret)).toBe(true); + expect(renderWorkspaceDocs(descriptor).envExample).not.toContain("CANARY-SECRET"); +}); + +test.each([ + { type: "filesystem", uri: "psd-clinical/evidence" }, + { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" }, + { type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" }, +])("omits Evidence installation variables for $type modes without file credentials", (source) => { + const descriptor = parseWorkspaceYaml(`${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`); + expect(buildInstallationContract(descriptor).variables.some((variable) => variable.role === "EVIDENCE")) + .toBe(false); +}); + +function renderWorkspaceWithoutEvidence(): string { + return `workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +llm_policy: { allowed: [zai/glm-5.2] } +`; +} + + +const evidenceSources = [ + { + label: "filesystem", + source: { type: "filesystem", uri: "psd-clinical/evidence" }, + variables: [], + }, + { + label: "HTTP signed URL file", + source: { + type: "http", + uris: ["https://evidence.example.test/guide.md", "http://public.example.test/policy.pdf"], + authentication: "signed_urls_file", + }, + variables: ["THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE"], + }, + { + label: "S3 static files", + source: { + type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files", + }, + variables: [ + "THT_WS_PSD_CLINICAL_EVIDENCE_ACCESS_KEY_FILE", + "THT_WS_PSD_CLINICAL_EVIDENCE_SECRET_KEY_FILE", + "THT_WS_PSD_CLINICAL_EVIDENCE_SESSION_TOKEN_FILE", + ], + }, +] as const; + +test.each(evidenceSources)("renders deterministic public Evidence docs for $label", ({ source, variables }) => { + const descriptor = parseWorkspaceYaml( + `${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`, + ); + const firstContract = buildInstallationContract(descriptor); + const secondContract = buildInstallationContract(descriptor); + const firstDocs = renderWorkspaceDocs(descriptor); + const secondDocs = renderWorkspaceDocs(descriptor); + + expect(secondContract).toEqual(firstContract); + expect(secondDocs).toEqual(firstDocs); + expect(firstContract.variables.filter(({ role }) => role === "EVIDENCE").map(({ name }) => name)) + .toEqual(variables); + expect(firstDocs.markdown).toContain("## Evidence source"); + expect(firstDocs.markdown).toContain(`- Type: \`${source.type}\``); + for (const uri of "uris" in source ? source.uris : [source.uri]) { + expect(firstDocs.markdown).toContain(`\`${uri}\``); + } + expect(firstDocs.markdown).toContain("- Maximum source bytes: `10485760`"); + expect(firstDocs.markdown).toContain("- Maximum chunk characters: `4000`"); + expect(firstDocs.markdown).toContain("- Retained published generations: `3`"); + for (const variable of variables) { + expect(firstDocs.markdown).toContain(`\`${variable}\``); + expect(firstDocs.envExample).toContain(`${variable}=`); + } +}); + +test("documents the S3 session token file as optional", () => { + const descriptor = parseWorkspaceYaml( + `${renderWorkspaceWithoutEvidence()}evidence: + source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files } +`, + ); + const markdown = renderWorkspaceDocs(descriptor).markdown; + const required = markdown.slice( + markdown.indexOf("- Required installation file variables:"), + markdown.indexOf("- Optional installation file variables:"), + ); + const optional = markdown.slice(markdown.indexOf("- Optional installation file variables:")); + + expect(required).toContain("EVIDENCE_ACCESS_KEY_FILE"); + expect(required).toContain("EVIDENCE_SECRET_KEY_FILE"); + expect(required).not.toContain("EVIDENCE_SESSION_TOKEN_FILE"); + expect(optional).toContain("EVIDENCE_SESSION_TOKEN_FILE"); +}); + +test("documents same-revision filesystem ownership without claiming P1 materialization", () => { + const descriptor = parseWorkspaceYaml( + `${renderWorkspaceWithoutEvidence()}evidence:\n source: { type: filesystem, uri: psd-clinical/evidence }\n`, + ); + const docs = renderWorkspaceDocs(descriptor).markdown; + + expect(docs).toContain("`psd-clinical/evidence`"); + expect(docs).toMatch(/same Git revision/i); + expect(docs).toMatch(/P6.*materializ/i); + expect(docs).toMatch(/containment.*symlink/i); + expect(docs).toMatch(/does not include Evidence file bytes/i); +}); + +test.each([ + { + source: { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "none" }, + expected: "No credential file is required", + }, + { + source: { type: "http", uris: ["https://evidence.example.test/guide.md"], authentication: "signed_urls_file" }, + expected: "signed URL file", + }, + { + source: { type: "s3", uri: "s3://clinical-evidence/published/", credentials: "ambient" }, + expected: "ambient credentials", + }, + { + source: { type: "s3", uri: "s3://clinical-evidence/published/", credentials: "static_files" }, + expected: "installation file variables", + }, +])("documents $source.type credential mode without reading credential contents", ({ source, expected }) => { + const root = mkdtempSync(join(tmpdir(), "thoth-evidence-doc-secret-")); + const secrets = join(root, "secrets"); + const canary = "CANARY-EVIDENCE-CREDENTIAL-DO-NOT-LEAK"; + mkdirSync(secrets); + const binding = join(secrets, "credential"); + writeFileSync(binding, canary); + const previous = process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE; + process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = binding; + try { + const descriptor = parseWorkspaceYaml( + `${renderWorkspaceWithoutEvidence()}evidence:\n source: ${JSON.stringify(source)}\n`, + ); + const generated = JSON.stringify({ + contract: buildInstallationContract(descriptor), + docs: renderWorkspaceDocs(descriptor), + }); + expect(generated).toContain(expected); + expect(generated).not.toContain(canary); + } finally { + if (previous === undefined) delete process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE; + else process.env.THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE = previous; + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/backend/test/workspaces-diagnostics.test.ts b/backend/test/workspaces-diagnostics.test.ts new file mode 100644 index 00000000..739c45d7 --- /dev/null +++ b/backend/test/workspaces-diagnostics.test.ts @@ -0,0 +1,495 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createConcreteDiagnosticAdapters, + createProductionWorkspaceDiagnoser, + createWorkspaceDiagnoser, + type DiagnosticAdapters, +} from "../src/workspaces/diagnostics.js"; +import type { RuntimeBindings } from "../src/workspaces/runtime-renderer.js"; +import { parseWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js"; + +const workspace = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: warehouse + schema: datawarehouse + timeout_ms: 8000 + supported_transports: [postgres_direct, rest_api, ssh_tunnel] +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] +`); + +const bindings: RuntimeBindings = { + dwh: { + transport: "postgres_direct", + missing: [], + values: { + THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.example.test", + THT_WS_PSD_CLINICAL_DWH_PORT: "5432", + THT_WS_PSD_CLINICAL_DWH_USER: "reader", + THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: "/run/secrets/dwh-password", + THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE: "/run/secrets/dwh-ca", + }, + }, + evidence: { missing: [], values: {} }, +}; + +function successfulAdapters(overrides: Partial = {}): DiagnosticAdapters { + return { + probeConnector: vi.fn(async (request) => ({ + resolved: true, + tlsVerified: true, + authenticated: true, + resource: request.resource, + })), + inspectQdrant: vi.fn(async (request) => ({ + collection: request.collection, + dimensions: 1024, + distance: "cosine", + })), + probeEmbedding: vi.fn(async () => ({ available: true, dimensions: 1024 })), + ...overrides, + }; +} + +function diagnose(adapters = successfulAdapters()) { + return createWorkspaceDiagnoser(adapters, { timeoutMs: 5_000 }); +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +test("diagnoses schema-v3 DWH, internal Qdrant, and internal Ollama without semantic bindings", async () => { + const adapters = successfulAdapters(); + const result = await diagnose(adapters)(workspace, bindings, { writeProbe: false }); + + expect(result).toEqual({ + activatable: true, + diagnostics: [{ + level: "info", code: "binding_ok", + message: "Installation bindings and diagnostics succeeded.", + }], + }); + expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ + role: "dwh", transport: "postgres_direct", + resource: { database: "warehouse", schema: "datawarehouse" }, + })); + expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({ + baseUrl: "http://qdrant:6333", collection: "psd-clinical", + })); + expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ + baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b", + })); +}); + +test("reports incompatible internal Qdrant or Ollama metadata", async () => { + const vector = await diagnose(successfulAdapters({ + inspectQdrant: vi.fn(async () => ({ + collection: "psd-clinical", dimensions: 768, distance: "cosine", + })), + }))(workspace, bindings, { writeProbe: false }); + expect(vector.activatable).toBe(false); + expect(vector.diagnostics).toContainEqual(expect.objectContaining({ + code: "semantic_index_incompatible", + })); + + const embedding = await diagnose(successfulAdapters({ + probeEmbedding: vi.fn(async () => ({ available: true, dimensions: 768 })), + }))(workspace, bindings, { writeProbe: false }); + expect(embedding.activatable).toBe(false); + expect(embedding.diagnostics).toContainEqual(expect.objectContaining({ + code: "semantic_index_incompatible", + })); +}); + +test("reports only sanitized DWH and Evidence binding names before network diagnostics", async () => { + const adapters = successfulAdapters(); + const result = await diagnose(adapters)(workspace, { + dwh: { ...bindings.dwh, missing: ["THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"] }, + evidence: { + values: {}, missing: ["THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE"], + }, + }, { writeProbe: false }); + + expect(result.activatable).toBe(false); + expect(result.diagnostics.map(({ code }) => code)).toEqual(["binding_missing", "binding_missing"]); + expect(result.diagnostics[1]).toMatchObject({ + variable: "THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE", + }); + expect(adapters.probeConnector).not.toHaveBeenCalled(); + expect(adapters.inspectQdrant).not.toHaveBeenCalled(); +}); + +test("keeps schema-v3 DWH SSH diagnostic-only and runtime-inactive", async () => { + const adapters = successfulAdapters(); + const result = await diagnose(adapters)(workspace, { + ...bindings, + dwh: { transport: "ssh_tunnel", missing: [], values: {} }, + }, { writeProbe: false }); + expect(result).toEqual({ + activatable: false, + diagnostics: [expect.objectContaining({ code: "workspace_not_activatable" })], + }); + expect(adapters.probeConnector).not.toHaveBeenCalled(); +}); + +test("uses the schema-v3 declared DWH REST diagnostic and auth policy", async () => { + const restWorkspace = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: REST workspace + language: en +dwh: + engine: postgres + database: warehouse + schema: datawarehouse + supported_transports: [rest_api] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: bearer + response: { database: database, schema: schema } +llm_policy: { allowed: [zai/glm-5.2] } +`); + const adapters = successfulAdapters(); + const result = await diagnose(adapters)(restWorkspace, { + dwh: { transport: "rest_api", missing: [], values: { + THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test", + THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE: "/run/secrets/dwh-api-key", + } }, + evidence: { missing: [], values: {} }, + }, { writeProbe: false }); + expect(result.activatable).toBe(true); + expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ + transport: "rest_api", + baseUrl: "https://dwh.example.test", + credentialFile: "/run/secrets/dwh-api-key", + diagnostic: expect.objectContaining({ path: "/rpc/ping", auth: "bearer" }), + })); +}); + +test("constructs the production diagnoser with its bounded configured timeout", async () => { + const adapters = successfulAdapters(); + await createProductionWorkspaceDiagnoser(1_234, adapters)(workspace, bindings, { + writeProbe: false, + }); + expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1_234 })); + expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1_234 })); +}); + +test("concrete DWH direct diagnostics authenticate, verify resource identity, and close", async () => { + const root = await mkdtemp(join(tmpdir(), "thoth-diagnostic-")); + const passwordFile = join(root, "password"); + await writeFile(passwordFile, "password-value"); + const end = vi.fn(async () => undefined); + const connect = vi.fn(async () => ({ + query: vi.fn(async () => ({ rows: [{ database: "warehouse", schema: "datawarehouse" }] })), + end, + })); + try { + const adapter = createConcreteDiagnosticAdapters({ databaseClient: { connect } }); + const result = await adapter.probeConnector({ + role: "dwh", transport: "postgres_direct", host: "127.0.0.1", port: 5432, + user: "reader", credentialFile: passwordFile, + resource: { database: "warehouse", schema: "datawarehouse" }, + timeoutMs: 1_000, signal: new AbortController().signal, + }); + expect(result).toMatchObject({ resolved: true, authenticated: true, tlsVerified: true }); + expect(connect).toHaveBeenCalledWith(expect.objectContaining({ credentialFile: passwordFile })); + expect(end).toHaveBeenCalledOnce(); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("concrete DWH REST diagnostics honor auth-none without reading credentials", async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ + database: "warehouse", schema: "datawarehouse", + }), { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const result = await createConcreteDiagnosticAdapters().probeConnector({ + role: "dwh", transport: "rest_api", baseUrl: "https://dwh.example.test", + resource: { database: "warehouse", schema: "datawarehouse" }, + timeoutMs: 1_000, signal: new AbortController().signal, + diagnostic: { + method: "GET", path: "/health", auth: "none", + response: { database: "database", schema: "schema" }, + }, + }); + expect(result).toMatchObject({ resolved: true, tlsVerified: true, authenticated: true }); + expect(fetchMock).toHaveBeenCalledWith("https://dwh.example.test/health", expect.objectContaining({ + headers: {}, redirect: "error", + })); +}); + +test("concrete internal semantic diagnostics use only Qdrant and Ollama protocols", async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ + result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } } }, + }), { status: 200 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), { + status: 200, + })); + vi.stubGlobal("fetch", fetchMock); + const adapter = createConcreteDiagnosticAdapters(); + await expect(adapter.inspectQdrant({ + baseUrl: "http://qdrant:6333", collection: "psd-clinical", + timeoutMs: 1_000, signal: new AbortController().signal, + })).resolves.toEqual({ collection: "psd-clinical", dimensions: 1024, distance: "cosine" }); + await expect(adapter.probeEmbedding({ + baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b", + timeoutMs: 1_000, signal: new AbortController().signal, + })).resolves.toEqual({ available: true, dimensions: 1024 }); + expect(fetchMock.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd-clinical"); + expect(fetchMock.mock.calls[1][0]).toBe("http://embedding:11434/api/embed"); + expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual({ + model: "qwen3-embedding:0.6b", input: "diagnostic", + }); + expect(Object.keys(adapter).sort()).toEqual(["inspectQdrant", "probeConnector", "probeEmbedding"]); +}); + +test("preserves a configured production timeout above the default up to the global maximum", async () => { + const adapters = successfulAdapters(); + await createProductionWorkspaceDiagnoser(8_000, adapters)(workspace, bindings, { + writeProbe: false, + }); + expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 })); + expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 })); + expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 })); +}); + +test("bounds descriptor DWH timeout by the configured production timeout", async () => { + const adapters = successfulAdapters(); + await createProductionWorkspaceDiagnoser(10_000, adapters)(workspace, bindings, { + writeProbe: false, + }); + expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 })); + expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 10_000 })); + expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 10_000 })); +}); + +test("aborts a timed-out production diagnostic and returns only a sanitized connector code", async () => { + let aborted = false; + const adapters = successfulAdapters({ + probeConnector: vi.fn((request) => new Promise((_resolve, reject) => { + request.signal.addEventListener("abort", () => { + aborted = true; + reject(new Error("CANARY-TIMEOUT-SECRET")); + }, { once: true }); + })), + }); + const result = await createProductionWorkspaceDiagnoser(5, adapters)(workspace, bindings, { + writeProbe: false, + }); + expect(aborted).toBe(true); + expect(result.activatable).toBe(false); + expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" })); + expect(JSON.stringify(result)).not.toContain("CANARY-TIMEOUT-SECRET"); +}); + +test.each([ + ["rejection", () => Promise.reject(new Error("CANARY-CONNECTOR-SECRET"))], + ["unresolved connector", async (request: Parameters[0]) => ({ + resolved: false, tlsVerified: true, authenticated: true, resource: request.resource, + })], + ["wrong resource", async () => ({ + resolved: true, tlsVerified: true, authenticated: true, + resource: { database: "other", schema: "datawarehouse" }, + })], + ["failed TLS", async (request: Parameters[0]) => ({ + resolved: true, tlsVerified: false, authenticated: true, resource: request.resource, + })], + ["failed authentication", async (request: Parameters[0]) => ({ + resolved: true, tlsVerified: true, authenticated: false, resource: request.resource, + })], +] as const)("sanitizes DWH connector %s", async (_label, probeConnector) => { + const result = await diagnose(successfulAdapters({ probeConnector: vi.fn(probeConnector) }))( + workspace, bindings, { writeProbe: false }, + ); + expect(result.activatable).toBe(false); + expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" })); + expect(JSON.stringify(result)).not.toContain("CANARY-CONNECTOR-SECRET"); +}); + +test("uses a REST secret only as a header and redacts it from failed diagnostics", async () => { + const root = await mkdtemp(join(tmpdir(), "thoth-rest-diagnostic-")); + const credentialFile = join(root, "api-key"); + const canary = "CANARY-REST-AUTH-SECRET"; + await writeFile(credentialFile, canary); + const restDescriptor = parseWorkspaceYaml(`workspace: + schema_version: 3 + id: psd-clinical + name: REST auth + language: en +dwh: + engine: postgres + database: warehouse + schema: datawarehouse + supported_transports: [rest_api] +semantic_index: + vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine } + embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 } +diagnostics: + dwh_rest: + method: GET + path: /health + auth: bearer + response: { database: database, schema: schema } +llm_policy: { allowed: [zai/glm-5.2] } +`); + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response("upstream CANARY-REST-AUTH-SECRET", { status: 503 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ + result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } } }, + }), { status: 200 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), { + status: 200, + })); + vi.stubGlobal("fetch", fetchMock); + try { + const result = await createWorkspaceDiagnoser(createConcreteDiagnosticAdapters())( + restDescriptor, + { + dwh: { transport: "rest_api", missing: [], values: { + THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test", + THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE: credentialFile, + } }, + evidence: { missing: [], values: {} }, + }, + { writeProbe: false }, + ); + expect(fetchMock.mock.calls[0][1].headers).toEqual({ authorization: `Bearer ${canary}` }); + expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" })); + expect(JSON.stringify(result)).not.toContain(canary); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test.each([ + ["Qdrant non-2xx", [ + new Response("CANARY-QDRANT-BODY", { status: 503 }), + new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), { status: 200 }), + ]], + ["Qdrant malformed", [ + new Response(JSON.stringify({ result: "CANARY-QDRANT-BODY" }), { status: 200 }), + new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), { status: 200 }), + ]], + ["Ollama non-2xx", [ + new Response(JSON.stringify({ result: { config: { params: { vectors: { + size: 1024, distance: "Cosine", + } } } } }), { status: 200 }), + new Response("CANARY-OLLAMA-BODY", { status: 503 }), + ]], + ["Ollama malformed", [ + new Response(JSON.stringify({ result: { config: { params: { vectors: { + size: 1024, distance: "Cosine", + } } } } }), { status: 200 }), + new Response(JSON.stringify({ embeddings: "CANARY-OLLAMA-BODY" }), { status: 200 }), + ]], +] as const)("sanitizes %s failures", async (_label, responses) => { + const fetchMock = vi.fn(); + for (const response of responses) fetchMock.mockResolvedValueOnce(response); + vi.stubGlobal("fetch", fetchMock); + const adapters = createConcreteDiagnosticAdapters({ + directProtocol: { + probe: async (request) => ({ + resolved: true, tlsVerified: true, authenticated: true, resource: request.resource, + }), + }, + }); + const result = await createWorkspaceDiagnoser(adapters)(workspace, bindings, { writeProbe: false }); + expect(result.activatable).toBe(false); + expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" })); + expect(JSON.stringify(result)).not.toMatch(/CANARY-(QDRANT|OLLAMA)-BODY/); +}); + +test("closes the concrete PostgreSQL diagnostic client when resource verification fails", async () => { + const root = await mkdtemp(join(tmpdir(), "thoth-diagnostic-close-")); + const passwordFile = join(root, "password"); + await writeFile(passwordFile, "CANARY-DATABASE-SECRET"); + const end = vi.fn(async () => undefined); + const connect = vi.fn(async () => ({ + query: vi.fn(async () => ({ rows: [{ database: "wrong", schema: "datawarehouse" }] })), + end, + })); + try { + const adapter = createConcreteDiagnosticAdapters({ databaseClient: { connect } }); + await expect(adapter.probeConnector({ + role: "dwh", transport: "postgres_direct", host: "127.0.0.1", port: 5432, + user: "reader", credentialFile: passwordFile, + resource: { database: "warehouse", schema: "datawarehouse" }, + timeoutMs: 1_000, signal: new AbortController().signal, + })).rejects.toThrow("direct probe failed"); + expect(end).toHaveBeenCalledOnce(); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("returns observed normalized Qdrant distance for semantic mismatch classification", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ + result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } }, + }), { status: 200 }))); + await expect(createConcreteDiagnosticAdapters().inspectQdrant({ + baseUrl: "http://qdrant:6333", collection: "psd-clinical", + timeoutMs: 1_000, signal: new AbortController().signal, + })).resolves.toEqual({ collection: "psd-clinical", dimensions: 1024, distance: "euclid" }); +}); + + +test("classifies an observed non-cosine Qdrant distance as semantic incompatibility", async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ + result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } }, + }), { status: 200 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), { + status: 200, + })); + vi.stubGlobal("fetch", fetchMock); + const adapters = createConcreteDiagnosticAdapters({ + directProtocol: { + probe: async (request) => ({ + resolved: true, tlsVerified: true, authenticated: true, resource: request.resource, + }), + }, + }); + const result = await createWorkspaceDiagnoser(adapters)(workspace, bindings, { writeProbe: false }); + expect(result.activatable).toBe(false); + expect(result.diagnostics).toContainEqual(expect.objectContaining({ + code: "semantic_index_incompatible", + })); + expect(result.diagnostics).not.toContainEqual(expect.objectContaining({ + code: "connector_unavailable", + })); +}); + +test("resolveDiagnosticUrl appends the path to a base URL with a path prefix", () => { + expect(resolveDiagnosticUrl("https://dwh.example.test/dwh/", "/rpc/ping").toString()) + .toBe("https://dwh.example.test/dwh/rpc/ping"); + expect(resolveDiagnosticUrl("https://dwh.example.test/dwh", "/rpc/ping").toString()) + .toBe("https://dwh.example.test/dwh/rpc/ping"); +}); diff --git a/backend/test/workspaces-git-annotations.test.ts b/backend/test/workspaces-git-annotations.test.ts new file mode 100644 index 00000000..82957e4e --- /dev/null +++ b/backend/test/workspaces-git-annotations.test.ts @@ -0,0 +1,150 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Workspace Registry Test", + gitAuthorEmail: "workspace-registry@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +interface RepoFixture { + root: string; + remote: string; + source: string; + commit: string; +} + +async function makeRepo(id: string, annotations: string | Buffer | "dir" | "symlink"): Promise { + const root = mkdtempSync(join(tmpdir(), "thoth-annotations-git-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Annotations Test"]); + await git(source, ["config", "user.email", "annotations@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), + `schema_version: 1\nworkspaces: [{id: ${id}, name: Workspace}]\n`); + mkdirSync(join(source, id, "schema"), { recursive: true }); + writeFileSync(join(source, id, "workspace.yaml"), `workspace:\n schema_version: 3\n id: ${id}\n`); + const annotationsPath = join(source, id, "schema", "annotations.yaml"); + if (annotations === "dir") { + mkdirSync(annotationsPath, { recursive: true }); + writeFileSync(join(annotationsPath, "child.txt"), "not a blob\n"); + } else if (annotations === "symlink") { + writeFileSync(join(source, id, "target.yaml"), "tables: {}\n"); + symlinkSync("target.yaml", annotationsPath); + } else { + writeFileSync(annotationsPath, annotations); + } + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + return { root, remote, source, commit }; +} + +async function bootstrapped(fixture: RepoFixture): Promise { + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + return repository; +} + +test("reads a regular annotation blob at the exact commit", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + + const object = await repository.annotationsObject(fixture.commit, "research"); + + expect(object).toBeDefined(); + expect(object!.blobId).toMatch(/^[0-9a-f]{40}$/); + expect(object!.contents.toString("utf8")).toBe("tables: {}\n"); +}); + +test("returns undefined when the annotation object is absent", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "absent")).resolves.toBeUndefined(); +}); + +test("refuses a tree at the annotation path", async () => { + const fixture = await makeRepo("clinical", "dir"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "clinical")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("refuses a symlink at the annotation path", async () => { + const fixture = await makeRepo("research", "symlink"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("refuses oversized and non-UTF-8 annotation blobs", async () => { + const oversized = await makeRepo("research", Buffer.concat([ + Buffer.from("tables: {}\n"), + Buffer.alloc(16 * 1024 * 1024, 0x78), + ])); + const repository = await bootstrapped(oversized); + await expect(repository.annotationsObject(oversized.commit, "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + + const nonUtf8 = await makeRepo("research", Buffer.from([0x74, 0x61, 0x62, 0xff, 0xfe, 0x00])); + const repository2 = await bootstrapped(nonUtf8); + await expect(repository2.annotationsObject(nonUtf8.commit, "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("refuses malformed ids and revisions before Git", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + + await expect(repository.annotationsObject(fixture.commit, "workspace-docs")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.annotationsObject(fixture.commit, "../research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.annotationsObject("HEAD", "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("redacts Git failures while reading the annotation blob", async () => { + const fixture = await makeRepo("research", "tables: {}\n"); + const repository = await bootstrapped(fixture); + rmSync(repository.repoPath, { recursive: true, force: true }); + + const error = await repository.annotationsObject(fixture.commit, "research").catch((failure: unknown) => failure); + expect(error).toMatchObject({ code: "git_unavailable", message: "Workspace Git operation failed" }); + expect((error as Error).message).not.toContain(fixture.root); +}); diff --git a/backend/test/workspaces-git-evidence.test.ts b/backend/test/workspaces-git-evidence.test.ts new file mode 100644 index 00000000..9ad14517 --- /dev/null +++ b/backend/test/workspaces-git-evidence.test.ts @@ -0,0 +1,129 @@ +import { execFile } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { GitWorkspaceRepository } from "../src/workspaces/git-repository.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await runFile("git", args, { cwd }); + return stdout.trim(); +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Evidence Test", + gitAuthorEmail: "evidence@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +type EvidenceLayout = "tree" | "empty" | "root-file" | "root-symlink" | "nested-symlink"; + +async function fixture(layout: EvidenceLayout): Promise<{ root: string; remote: string; commit: string }> { + const root = mkdtempSync(join(tmpdir(), "thoth-evidence-git-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Evidence Test"]); + await git(source, ["config", "user.email", "evidence@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: research, name: Research}]\n"); + mkdirSync(join(source, "research"), { recursive: true }); + writeFileSync(join(source, "research", "workspace.yaml"), "workspace:\n schema_version: 3\n id: research\n"); + const evidence = join(source, "research", "evidence"); + if (layout === "tree") { + mkdirSync(join(evidence, "nested"), { recursive: true }); + writeFileSync(join(evidence, "guide.md"), "# guide\n"); + writeFileSync(join(evidence, "nested", "deep.md"), "# deep\n"); + } else if (layout === "empty") { + mkdirSync(evidence, { recursive: true }); + } else if (layout === "root-file") { + writeFileSync(evidence, "not a tree\n"); + } else if (layout === "root-symlink") { + writeFileSync(join(source, "research", "target.md"), "# target\n"); + symlinkSync("target.md", evidence); + } else if (layout === "nested-symlink") { + mkdirSync(evidence, { recursive: true }); + writeFileSync(join(source, "research", "outside.md"), "# outside\n"); + symlinkSync("../outside.md", join(evidence, "link.md")); + } + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "initial"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const commit = await git(source, ["rev-parse", "HEAD"]); + return { root, remote, commit }; +} + +async function bootstrapped(fixture: { root: string; remote: string }): Promise { + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + return repository; +} + +test("enumerates a regular Evidence tree with ordered relative paths", async () => { + const fixtureValue = await fixture("tree"); + const repository = await bootstrapped(fixtureValue); + + const objects = await repository.evidenceTreeObjects(fixtureValue.commit, "research"); + + expect(objects.map((entry) => entry.posixPath)).toEqual(["guide.md", "nested/deep.md"]); + expect(objects.every((entry) => /^[0-9a-f]{40}$/.test(entry.oid))).toBe(true); + expect(objects.every((entry) => entry.mode === "100644" || entry.mode === "100755")).toBe(true); +}); + +test("accepts an empty Evidence tree", async () => { + const fixtureValue = await fixture("empty"); + const repository = await bootstrapped(fixtureValue); + + await expect(repository.evidenceTreeObjects(fixtureValue.commit, "research")).resolves.toEqual([]); +}); + +test("refuses a non-tree Evidence root and symlinks at any depth", async () => { + for (const layout of ["root-file", "root-symlink", "nested-symlink"] as const) { + const fixtureValue = await fixture(layout); + const repository = await bootstrapped(fixtureValue); + await expect(repository.evidenceTreeObjects(fixtureValue.commit, "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + } +}); + +test("refuses malformed revisions and workspace ids", async () => { + const fixtureValue = await fixture("tree"); + const repository = await bootstrapped(fixtureValue); + + await expect(repository.evidenceTreeObjects("HEAD", "research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.evidenceTreeObjects(fixtureValue.commit, "../research")) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("streams bounded Evidence blobs and refuses oversized objects", async () => { + const fixtureValue = await fixture("tree"); + const repository = await bootstrapped(fixtureValue); + const [guide] = await repository.evidenceTreeObjects(fixtureValue.commit, "research"); + + const bytes = await repository.evidenceBlobBytes(guide.oid, 1024); + expect(bytes.toString("utf8")).toBe("# guide\n"); + + await expect(repository.evidenceBlobBytes(guide.oid, 1)) + .rejects.toMatchObject({ code: "workspace_invalid" }); +}); diff --git a/backend/test/workspaces-git-repository.test.ts b/backend/test/workspaces-git-repository.test.ts new file mode 100644 index 00000000..22004a00 --- /dev/null +++ b/backend/test/workspaces-git-repository.test.ts @@ -0,0 +1,338 @@ +import { execFile } from "node:child_process"; +import { existsSync, mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { + GitWorkspaceRepository, + WorkspaceRepositoryLock, + normalizeRepositoryIdentity, +} from "../src/workspaces/git-repository.js"; +import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js"; + +const validYaml = `workspace: + schema_version: 2 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: pgvector + database: postgres + schema: vectors + collection: clinical_documents + dimensions: 768 + distance: cosine + supported_transports: [pgvector_direct] + embedding: + provider: ollama_compatible + model: nomic-embed-text-v2-moe + dimensions: 768 +llm_policy: + allowed: [zai/glm-5.2] +`; + +const runFile = promisify(execFile); +const temporaryRoots: string[] = []; + +afterEach(() => { + temporaryRoots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })); +}); + +async function git(cwd: string, args: string[]): Promise { + await runFile("git", args, { cwd }); +} + +async function temporaryRemote(): Promise<{ root: string; remote: string; source: string; initialCommit: string }> { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-git-")); + temporaryRoots.push(root); + const remote = join(root, "remote.git"); + const source = join(root, "source"); + await git(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(source); + await git(source, ["init", "--initial-branch=main"]); + await git(source, ["config", "user.name", "Workspace Registry Test"]); + await git(source, ["config", "user.email", "workspace-registry@example.invalid"]); + writeFileSync(join(source, "thoth-workspaces.yaml"), "schema_version: 1\nworkspaces: [{id: psd-clinical, name: Policlinico San Donato}, {id: research, name: Policlinico San Donato}]\n"); + mkdirSync(join(source, "psd-clinical"), { recursive: true }); + mkdirSync(join(source, "research"), { recursive: true }); + writeFileSync(join(source, "psd-clinical", "workspace.yaml"), validYaml); + writeFileSync(join(source, "research", "workspace.yaml"), validYaml + .replace("id: psd-clinical", "id: research")); + mkdirSync(join(source, "research", "evidence"), { recursive: true }); + mkdirSync(join(source, "other", "evidence"), { recursive: true }); + mkdirSync(join(source, "blob"), { recursive: true }); + mkdirSync(join(source, "link"), { recursive: true }); + writeFileSync(join(source, "research", "evidence", "guide.md"), "guide v1\n"); + writeFileSync(join(source, "other", "evidence", "other.md"), "other\n"); + writeFileSync(join(source, "blob", "evidence"), "not a tree\n"); + symlinkSync("../research/evidence", join(source, "link", "evidence")); + symlinkSync("guide.md", join(source, "research", "evidence", "nested-link")); + await git(source, ["add", "-A"]); + await git(source, ["commit", "-m", "Initial workspace"]); + await git(source, ["remote", "add", "origin", remote]); + await git(source, ["push", "origin", "main"]); + const { stdout } = await runFile("git", ["rev-parse", "HEAD"], { cwd: source }); + return { root, remote, source, initialCommit: stdout.trim() }; +} + +function config(root: string, remoteUrl: string): WorkspaceRegistryConfig { + return { + root, + remoteUrl, + branch: "main", + gitAuthorName: "Workspace Registry Test", + gitAuthorEmail: "workspace-registry@example.invalid", + installationId: "test", + secretRoots: [], + maxImportBytes: 1024, + maxImportEntries: 1, + }; +} + +test("does not expose repository mutation or publication operations", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + + expect(repository).not.toHaveProperty("createRegistryFile"); + expect(repository).not.toHaveProperty("writeRegistryFile"); + expect(repository).not.toHaveProperty("removeRegistryFile"); + expect(repository).not.toHaveProperty("commitAndPush"); +}); + +test.each([ + ["https://github.com/aritmolab/workspaces.git", { + host: "github.com", repository: "aritmolab/workspaces", transport: "https", + }], + ["ssh://git@gitlab.example.org/clinical/workspaces.git", { + host: "gitlab.example.org", repository: "clinical/workspaces", transport: "ssh", + }], + ["git@gitea.example.org:clinical/workspaces.git", { + host: "gitea.example.org", repository: "clinical/workspaces", transport: "ssh", + }], +] as const)("normalizes the safe repository identity for %s", (remote, expected) => { + expect(normalizeRepositoryIdentity(remote)).toEqual(expected); +}); + +test.each([ + "https://user:secret@github.com/aritmolab/workspaces.git", + "https://github.com/aritmolab/workspaces.git?token=secret", + "ssh://git:secret@gitlab.example.org/clinical/workspaces.git", +])("rejects a remote that could expose embedded credentials: %s", (remote) => { + expect(() => normalizeRepositoryIdentity(remote)).toThrow("Workspace Git remote is invalid"); +}); + +test("bootstraps a persistent checkout from a local bare repository", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + + const status = await repository.bootstrap(); + + expect(status).toMatchObject({ + branch: "main", + head: fixture.initialCommit, + ahead: 0, + behind: 0, + degraded: false, + }); +}); + +test("accepts only a tree at the declared Evidence root for the requested revision", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + "research/evidence", + )).resolves.toBeUndefined(); + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + "missing/evidence", + )).rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + "blob/evidence", + )).rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + "link/evidence", + )).rejects.toMatchObject({ code: "workspace_invalid" }); +}); + +test("redacts Git failures while checking an Evidence tree", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + rmSync(repository.repoPath, { recursive: true, force: true }); + + const error = await repository.assertTreeAtRevision( + fixture.initialCommit, + "research/evidence", + ).catch((failure: unknown) => failure); + expect(error).toMatchObject({ code: "git_unavailable", message: "Workspace Git operation failed" }); + expect((error as Error).message).not.toContain(fixture.root); +}); + +test("classifies repository corruption as unavailable rather than invalid Evidence", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + rmSync(join(repository.repoPath, ".git", "objects"), { recursive: true, force: true }); + + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + "research/evidence", + )).rejects.toMatchObject({ code: "git_unavailable", message: "Workspace Git operation failed" }); +}); + +test("binds Evidence tree validation to old and new content-only commits", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + writeFileSync(join(fixture.source, "research", "evidence", "guide.md"), "guide v2\n"); + await git(fixture.source, ["add", "research/evidence/guide.md"]); + await git(fixture.source, ["commit", "-m", "Update Evidence content"]); + await git(fixture.source, ["push", "origin", "main"]); + const { stdout } = await runFile("git", ["rev-parse", "HEAD"], { cwd: fixture.source }); + const newCommit = stdout.trim(); + await repository.pull(); + const oldTree = (await runFile("git", ["rev-parse", `${fixture.initialCommit}:research/evidence`], { + cwd: fixture.source, + })).stdout.trim(); + const newTree = (await runFile("git", ["rev-parse", `${newCommit}:research/evidence`], { + cwd: fixture.source, + })).stdout.trim(); + + expect(newTree).not.toBe(oldTree); + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + "research/evidence", + )).resolves.toBeUndefined(); + await expect(repository.assertTreeAtRevision( + newCommit, + "research/evidence", + )).resolves.toBeUndefined(); +}); + +test("defers nested Evidence symlink containment to P6", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + + // Task 2 validates only the declared root object. Recursive containment remains a P6 boundary. + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + "research/evidence", + )).resolves.toBeUndefined(); +}); + +test("rejects malformed revisions and shell-like paths without executing them", async () => { + const fixture = await temporaryRemote(); + const repository = new GitWorkspaceRepository(config(join(fixture.root, "registry"), fixture.remote)); + await repository.bootstrap(); + const marker = join(fixture.root, "shell-marker"); + + await expect(repository.assertTreeAtRevision( + "HEAD", + "research/evidence", + )).rejects.toMatchObject({ code: "workspace_invalid" }); + await expect(repository.assertTreeAtRevision( + fixture.initialCommit, + `research/evidence;touch ${marker}`, + )).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(existsSync(marker)).toBe(false); +}); + +test("redacts failed Git checkout details behind a stable error code", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-git-missing-")); + temporaryRoots.push(root); + const remote = join(root, "missing.git"); + const repository = new GitWorkspaceRepository(config(join(root, "registry"), remote)); + + const error = await repository.bootstrap().catch((error: unknown) => error); + expect(error).toMatchObject({ + code: "git_unavailable", + message: "Workspace Git operation failed", + }); + expect((error as Error).message).not.toContain(remote); +}); + +test("maps registry-layout failures to a stable redacted error", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-git-layout-")); + temporaryRoots.push(root); + const file = join(root, "not-a-directory"); + writeFileSync(file, "occupied"); + const repository = new GitWorkspaceRepository(config(file, join(root, "remote.git"))); + + const error = await repository.bootstrap().catch((error: unknown) => error); + + expect(error).toMatchObject({ + code: "git_unavailable", + message: "Workspace registry storage is unavailable", + }); + expect((error as Error).message).not.toContain(file); +}); + +test("maps lock filesystem failures to a stable redacted error", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-git-lock-")); + temporaryRoots.push(root); + const file = join(root, "not-a-directory"); + writeFileSync(file, "occupied"); + const lock = new WorkspaceRepositoryLock(file); + + const error = await lock.run(async () => undefined).catch((error: unknown) => error); + + expect(error).toMatchObject({ + code: "git_unavailable", + message: "Workspace registry lock is unavailable", + }); + expect((error as Error).message).not.toContain(file); +}); + +test("releases its queue after a malformed lock failure so a later attempt can acquire", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-git-queue-")); + temporaryRoots.push(root); + const locks = join(root, "locks"); + mkdirSync(join(locks, "repository.lock"), { recursive: true }); + const lock = new WorkspaceRepositoryLock(locks); + + await expect(lock.run(async () => "unreachable")).rejects.toMatchObject({ code: "git_unavailable" }); + rmSync(join(locks, "repository.lock"), { recursive: true, force: true }); + + const result = await Promise.race([ + lock.run(async () => "recovered"), + new Promise((resolve) => setTimeout(() => resolve("timed out"), 2_000)), + ]); + expect(result).toBe("recovered"); +}); + +test("parallel contenders recover a stale lock file without overlapping critical sections", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-git-contenders-")); + temporaryRoots.push(root); + const locks = join(root, "locks"); + mkdirSync(locks); + writeFileSync(join(locks, "repository.lock"), JSON.stringify({ pid: 999_999_999 })); + const first = new WorkspaceRepositoryLock(locks); + const second = new WorkspaceRepositoryLock(locks); + let active = 0; + let maximum = 0; + const critical = async () => { + active += 1; + maximum = Math.max(maximum, active); + await new Promise((resolve) => setTimeout(resolve, 25)); + active -= 1; + }; + + const results = await Promise.allSettled([first.run(critical), second.run(critical)]); + + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + expect(results.filter((result) => result.status === "rejected")).toHaveLength(1); + expect(maximum).toBe(1); +}); diff --git a/backend/test/workspaces-runtime-v3-boundaries.test.ts b/backend/test/workspaces-runtime-v3-boundaries.test.ts new file mode 100644 index 00000000..022f9015 --- /dev/null +++ b/backend/test/workspaces-runtime-v3-boundaries.test.ts @@ -0,0 +1,70 @@ +import { expect, test, vi } from "vitest"; +import { buildInstallationContract } from "../src/workspaces/contracts.js"; +import { + createProductionWorkspaceDiagnoser, + createWorkspaceDiagnoser, + type DiagnosticAdapters, +} from "../src/workspaces/diagnostics.js"; +import { + resolveBinding, + resolveEvidenceBinding, + resolveRuntimeBindings, +} from "../src/workspaces/bindings.js"; +import { renderRuntimeConfig, type RuntimeBindings } from "../src/workspaces/runtime-renderer.js"; + +const unsupportedWorkspace = { + workspace: { schema_version: 2, id: "legacy-workspace", name: "Legacy", language: "en" }, + dwh: { + engine: "postgres", database: "warehouse", schema: "public", + supported_transports: ["postgres_direct"], + }, + semantic_index: { + vector_store: { + engine: "pgvector", database: "warehouse", schema: "vectors", + collection: "documents", dimensions: 768, distance: "cosine", + supported_transports: ["pgvector_direct"], + }, + embedding: { provider: "ollama_compatible", model: "legacy", dimensions: 768 }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, +}; + +const bindings: RuntimeBindings = { + dwh: { transport: "postgres_direct", values: {}, missing: [] }, + evidence: { values: {}, missing: [] }, +}; + +const adapters: DiagnosticAdapters = { + probeConnector: vi.fn(), + inspectQdrant: vi.fn(), + probeEmbedding: vi.fn(), +}; + +test("renderer rejects callers that bypass the schema-v3 type contract", () => { + expect(() => renderRuntimeConfig(unsupportedWorkspace as never, bindings, { + sessions: "/data/sessions", artifacts: "/data/artifacts", indexes: "/data/indexes", + })).toThrow("Runtime renderer supports only workspace schema version 3"); +}); + +test("installation contract rejects callers that bypass the schema-v3 type contract", () => { + expect(() => buildInstallationContract(unsupportedWorkspace as never)) + .toThrow("Installation contract supports only workspace schema version 3"); +}); + +test("binding entry points reject callers that bypass the schema-v3 type contract", () => { + expect(() => resolveBinding(unsupportedWorkspace as never, "DWH", {}, [])) + .toThrow("Workspace bindings support only workspace schema version 3"); + expect(() => resolveEvidenceBinding(unsupportedWorkspace as never, {}, [])) + .toThrow("Workspace bindings support only workspace schema version 3"); + expect(() => resolveRuntimeBindings(unsupportedWorkspace as never, {}, [])) + .toThrow("Workspace bindings support only workspace schema version 3"); +}); + +test("diagnoser factories reject callers that bypass the schema-v3 type contract", async () => { + await expect(createWorkspaceDiagnoser(adapters)(unsupportedWorkspace as never, bindings, { + writeProbe: false, + })).rejects.toThrow("Workspace diagnoser supports only workspace schema version 3"); + await expect(createProductionWorkspaceDiagnoser(5_000, adapters)( + unsupportedWorkspace as never, bindings, { writeProbe: false }, + )).rejects.toThrow("Workspace diagnoser supports only workspace schema version 3"); +}); diff --git a/backend/test/workspaces-schema.test.ts b/backend/test/workspaces-schema.test.ts new file mode 100644 index 00000000..3e93669e --- /dev/null +++ b/backend/test/workspaces-schema.test.ts @@ -0,0 +1,550 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parse } from "yaml"; +import { expect, test } from "vitest"; +import * as workspaceSchema from "../src/workspaces/schema.js"; +import { + parseWorkspaceYaml, + serializeWorkspaceYaml, + validateWorkspaceDescriptor, +} from "../src/workspaces/schema.js"; + +export const validYaml = `workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + description: Clinical data warehouse workspace + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + port: 5432 + timeout_ms: 5000 + supported_transports: + - postgres_direct + - rest_api + - ssh_tunnel +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + default: zai/glm-5.2 + allowed: + - zai/glm-5.2 + - openai/gpt-5 +`; + +test("rejects a workspace whose embedding dimensions differ from its collection", () => { + expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 1024", "dimensions: 1536"))) + .toThrow(/dimensions/i); +}); + +test("rejects an LLM default outside its allowlist", () => { + expect(() => parseWorkspaceYaml(validYaml.replace("- zai/glm-5.2", "- openai/gpt-5"))) + .toThrow(/allowlist/i); +}); + +test("rejects unknown keys and invalid immutable IDs", () => { + expect(() => parseWorkspaceYaml(validYaml.replace(" language: it", " language: it\n label: PSD"))) + .toThrow(/unrecognized key/i); + expect(() => parseWorkspaceYaml(validYaml.replace("id: psd-clinical", "id: PSD"))) + .toThrow(/id/i); +}); + +test("rejects executable or otherwise custom YAML tags in canonical descriptors", () => { + expect(() => parseWorkspaceYaml(validYaml.replace( + "name: Policlinico San Donato", + "name: !command echo-never-execute", + ))).toThrow(/tag|yaml/i); +}); + +test("accepts optional connection ports and timeouts but rejects unsafe values", () => { + expect(parseWorkspaceYaml(validYaml).dwh.port).toBe(5432); + expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 0"))) + .toThrow(/port/i); + expect(() => parseWorkspaceYaml(validYaml.replace("port: 5432", "port: 65536"))) + .toThrow(/port/i); + expect(() => parseWorkspaceYaml(validYaml.replace("timeout_ms: 5000", "timeout_ms: 0"))) + .toThrow(/timeout/i); + expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 1024", "dimensions: 2048"))) + .toThrow(/1024|dimensions/i); +}); + +test("accepts only the schema v3 internal qdrant semantic shape", () => { + expect(parseWorkspaceYaml(validYaml)).toMatchObject({ + workspace: { schema_version: 3, id: "psd-clinical" }, + semantic_index: { + vector_store: { + engine: "qdrant", + collection: "psd-clinical", + dimensions: 1024, + distance: "cosine", + }, + embedding: { + provider: "ollama_internal", + model: "qwen3-embedding:0.6b", + dimensions: 1024, + }, + }, + }); +}); + +test("committed example descriptors parse as exact schema v3 workspaces", () => { + const example = readFileSync(resolve(process.cwd(), "../deploy/workspaces/example.yaml"), "utf8"); + const psdExample = readFileSync(resolve(process.cwd(), "../deploy/workspaces/psd.yaml.example"), "utf8"); + + expect(() => parseWorkspaceYaml(example)).not.toThrow(); + expect(() => parseWorkspaceYaml(psdExample)).not.toThrow(); + expect(parseWorkspaceYaml(example)).toMatchObject({ + workspace: { schema_version: 3 }, + semantic_index: { + vector_store: { engine: "qdrant", distance: "cosine", dimensions: 1024 }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + }); + expect(parseWorkspaceYaml(psdExample)).toMatchObject({ + workspace: { schema_version: 3 }, + semantic_index: { + vector_store: { engine: "qdrant", distance: "cosine", dimensions: 1024 }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + }); +}); + +test("rejects pgvector semantic stores in schema v3", () => { + expect(() => parseWorkspaceYaml(validYaml.replace("engine: qdrant", "engine: pgvector"))) + .toThrow(/qdrant|pgvector/i); +}); + +test("rejects supported_transports inside schema v3 semantic identity", () => { + const withTransport = validYaml.replace( + " distance: cosine\n", + " distance: cosine\n supported_transports:\n - rest_api\n", + ); + + expect(() => parseWorkspaceYaml(withTransport)).toThrow(/unrecognized key|supported_transports/i); +}); + +test("rejects external embedding providers in schema v3", () => { + expect(() => parseWorkspaceYaml(validYaml.replace("provider: ollama_internal", "provider: openai_compatible"))) + .toThrow(/ollama_internal|provider/i); +}); + +test("rejects non-cosine distance in schema v3", () => { + expect(() => parseWorkspaceYaml(validYaml.replace("distance: cosine", "distance: l2"))) + .toThrow(/cosine|distance/i); +}); + +test("rejects unknown fields in schema v3 semantic identity", () => { + const withUnknownField = validYaml.replace( + " collection: psd-clinical\n", + " collection: psd-clinical\n namespace: psd\n", + ); + + expect(() => parseWorkspaceYaml(withUnknownField)).toThrow(/unrecognized key/i); +}); + +test("rejects legacy semantic connector fields and diagnostics in schema v3", () => { + expect(() => parseWorkspaceYaml(validYaml.replace( + " collection: psd-clinical\n", + " collection: psd-clinical\n database: postgres\n", + ))).toThrow(/unrecognized key|database/i); + + expect(() => parseWorkspaceYaml(validYaml.replace( + "llm_policy:\n", + "diagnostics:\n vector_rest:\n metadata:\n method: GET\n path: /metadata\n auth: bearer\n response:\n collection: collection\n dimensions: dimensions\n distance: distance\nllm_policy:\n", + ))).toThrow(/unrecognized key|vector_rest/i); +}); + +test.each([ + ["v1", `workspace: + schema_version: 1 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: + - postgres_direct +semantic_index: + vector_store: + engine: pgvector + collection: clinical_documents + dimensions: 1024 + distance: cosine + supported_transports: + - pgvector_direct + embedding: + provider: ollama_compatible + model: nomic-embed-text-v2-moe + dimensions: 1024 +llm_policy: + allowed: + - zai/glm-5.2 +`], + ["v2", `workspace: + schema_version: 2 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: + - postgres_direct +semantic_index: + vector_store: + engine: pgvector + database: postgres + schema: vectors + collection: clinical_documents + dimensions: 1024 + distance: cosine + supported_transports: + - pgvector_direct + embedding: + provider: ollama_compatible + model: nomic-embed-text-v2-moe + dimensions: 1024 +llm_policy: + allowed: + - zai/glm-5.2 +`], +])("rejects schema %s descriptors at parser and object-validator boundaries", (_version, yaml) => { + expect(() => parseWorkspaceYaml(yaml)).toThrow(/schema_version|invalid literal|3/i); + expect(() => validateWorkspaceDescriptor(parse(yaml))).toThrow(/schema_version|invalid literal|3/i); +}); + +test("does not expose the redundant canonical validator or v1 migration", () => { + const legacyExports = workspaceSchema as Record; + + expect(legacyExports.validateCanonicalWorkspace).toBeUndefined(); + expect(legacyExports.migrateWorkspaceV1ToV2).toBeUndefined(); +}); + +test("constructs diagnostic URLs only when the resolved URL remains on the service origin", () => { + const resolveDiagnosticUrl = (workspaceSchema as { resolveDiagnosticUrl?: unknown }).resolveDiagnosticUrl; + + expect(resolveDiagnosticUrl).toBeTypeOf("function"); + expect((resolveDiagnosticUrl as (baseUrl: string, path: string) => URL)("https://service.example/base", "/rpc/ping")) + .toMatchObject({ href: "https://service.example/base/rpc/ping" }); + expect(() => (resolveDiagnosticUrl as (baseUrl: string, path: string) => URL)( + "https://service.example/base", "//diagnostic.invalid/rpc", + )).toThrow(/origin/i); +}); + +test("rejects REST diagnostic declarations without their matching connector transport", () => { + const diagnostics = `diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: bearer + response: + database: database + schema: schema +llm_policy: +`; + const declared = validYaml.replace("llm_policy:\n", diagnostics); + + expect(() => parseWorkspaceYaml(declared.replace(" - rest_api\n", ""))).toThrow(/dwh_rest/i); +}); + +test("serializes canonical YAML that parses back to the same workspace", () => { + const workspace = parseWorkspaceYaml(validYaml); + const serialized = serializeWorkspaceYaml(workspace); + + expect(serializeWorkspaceYaml(parseWorkspaceYaml(serialized))).toBe(serialized); + expect(parseWorkspaceYaml(serialized)).toEqual(workspace); +}); + + +function validWorkspaceObject(): Record { + return parseWorkspaceYaml(validYaml) as Record; +} + +function withEvidence(source: Record, policy?: Record): Record { + const workspace = structuredClone(validWorkspaceObject()); + workspace.evidence = policy === undefined ? { source } : { source, policy }; + return workspace; +} + +function expectSafeEvidenceError(workspace: unknown, path: RegExp, canary?: string): void { + let message = ""; + try { + validateWorkspaceDescriptor(workspace); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + expect(message.replace(/\s+/g, " ")).toMatch(path); + if (canary !== undefined) expect(message).not.toContain(canary); +} + +const explicitPolicy = { max_chunk_chars: 8_000, retain_published_generations: 5 }; + +const validEvidenceSources = [ + { + name: "filesystem with explicit values", + source: { + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: ["documents/**/*.pdf", "notes/*.md"], + max_bytes: 12_000_000, + }, + }, + { + name: "HTTP without authentication", + source: { + type: "http", + uris: ["https://evidence.example/manifest.json", "http://evidence.example/files/list.txt"], + authentication: "none", + connect_timeout_ms: 2_000, + read_timeout_ms: 20_000, + max_bytes: 12_000_000, + max_redirects: 2, + allow_private_hosts: false, + max_cache_bytes: 24_000_000, + }, + }, + { + name: "HTTP signed URL manifest", + source: { + type: "http", + uris: ["https://evidence.example/signed-urls.txt"], + authentication: "signed_urls_file", + connect_timeout_ms: 2_000, + read_timeout_ms: 20_000, + max_bytes: 12_000_000, + max_redirects: 2, + allow_private_hosts: true, + max_cache_bytes: 24_000_000, + }, + }, + { + name: "S3 with ambient credentials", + source: { + type: "s3", + uri: "s3://clinical-evidence/published/", + region: "eu-west-1", + credentials: "ambient", + trusted_endpoint: false, + allow_private_endpoint: false, + allow_insecure_endpoint: false, + max_bytes: 12_000_000, + max_objects: 2_000, + max_pages: 20, + page_size: 100, + }, + }, + { + name: "S3 with static-file credentials and a trusted endpoint", + source: { + type: "s3", + uri: "s3://clinical-evidence/published/", + endpoint_url: "https://objects.example", + region: "eu-west-1", + credentials: "static_files", + trusted_endpoint: true, + allow_private_endpoint: false, + allow_insecure_endpoint: false, + max_bytes: 12_000_000, + max_objects: 2_000, + max_pages: 20, + page_size: 100, + }, + }, +] as const; + +test.each(validEvidenceSources)("accepts evidence source: $name", ({ source }) => { + expect(validateWorkspaceDescriptor(withEvidence(source, explicitPolicy))).toMatchObject({ + evidence: { source, policy: explicitPolicy }, + }); +}); + +test("applies filesystem and policy defaults to the canonical descriptor", () => { + const parsed = validateWorkspaceDescriptor(withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + })); + + expect(parsed).toMatchObject({ + evidence: { + source: { + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: ["**/*.md"], + max_bytes: 10 * 1024 * 1024, + }, + policy: { max_chunk_chars: 4_000, retain_published_generations: 3 }, + }, + }); +}); + +test("keeps evidence optional on schema v3", () => { + expect(validateWorkspaceDescriptor(validWorkspaceObject())).not.toHaveProperty("evidence"); +}); + +test("serializes defaulted evidence canonically and parses it without loss", () => { + const canonical = validateWorkspaceDescriptor(withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + })); + if (canonical.workspace.schema_version !== 3) throw new Error("expected schema v3"); + + expect(parseWorkspaceYaml(serializeWorkspaceYaml(canonical))).toEqual(canonical); +}); + +const invalidFilesystemPaths = [ + "/psd-clinical/evidence", + "../psd-clinical/evidence", + "./psd-clinical/evidence", + "/psd-clinical/evidence", + "psd-clinical/evidence/..", + "\\psd-clinical\\evidence", + "psd-clinical/evidence\u0000", + "other-workspace/evidence", + "psd-clinical", + "psd-clinical/evidence/nested", +]; + +test.each(invalidFilesystemPaths)("rejects unsafe or noncanonical filesystem URI %#", (uri) => { + expectSafeEvidenceError(withEvidence({ type: "filesystem", uri }), /evidence.*source.*uri/i); +}); + +const invalidPatterns = ["", "/absolute", "../escape", ".", "folder/./file", "folder//file", "folder/../file", "a\\b", "a\u0007b"]; + +test.each(invalidPatterns)("rejects unsafe evidence glob %#", (pattern) => { + expectSafeEvidenceError(withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + patterns: [pattern], + }), /evidence.*source.*patterns/i); +}); + +test("rejects empty and duplicate filesystem patterns", () => { + const source = { type: "filesystem", uri: "psd-clinical/evidence" }; + expectSafeEvidenceError(withEvidence({ ...source, patterns: [] }), /patterns/i); + expectSafeEvidenceError(withEvidence({ ...source, patterns: ["**/*.pdf", "**/*.pdf"] }), /patterns/i); +}); + +test.each(["ftp", "git", "unknown"])("rejects unsupported evidence discriminator %s", (type) => { + expectSafeEvidenceError(withEvidence({ type, uri: "psd-clinical/evidence" }), /evidence.*source.*type/i); +}); + +test("rejects unknown evidence keys", () => { + expectSafeEvidenceError({ ...withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + }), evidence: { + source: { type: "filesystem", uri: "psd-clinical/evidence", mystery: true }, + policy: explicitPolicy, + mystery: true, + } }, /unrecognized|mystery/i); +}); + +const credentialFields = [ + "password", "api_key", "access_key", "secret_key", "session_token", "signed_url", "headers", "ca_contents", +]; + +test.each(credentialFields)("rejects credential-shaped evidence field %s without leaking it", (field) => { + const canary = `CANARY-${field}-DO-NOT-LEAK`; + expectSafeEvidenceError(withEvidence({ + type: "filesystem", + uri: "psd-clinical/evidence", + [field]: canary, + }), /evidence.*source/i, canary); +}); + +const invalidHttpUris = [ + "https://user:CANARY-HTTP@example.com/manifest", + "https://example.com/manifest?token=CANARY-HTTP", + "https://example.com/manifest#CANARY-HTTP", + "ftp://example.com/manifest/CANARY-HTTP", + "\nhttps://example.com/CANARY-HTTP", +]; + +test.each(invalidHttpUris)("rejects unsafe HTTP descriptor URI %# without leaking it", (uri) => { + expectSafeEvidenceError(withEvidence({ type: "http", uris: [uri] }), /evidence.*source.*uris.*0/i, "CANARY-HTTP"); +}); + +test("rejects empty and canonically duplicate HTTP manifests", () => { + expectSafeEvidenceError(withEvidence({ type: "http", uris: [] }), /uris/i); + expectSafeEvidenceError(withEvidence({ + type: "http", + uris: ["https://EXAMPLE.com:443/manifest", "https://example.com/manifest"], + }), /uris/i); +}); + +const invalidHttpBounds = [ + ["connect_timeout_ms", 0], ["read_timeout_ms", 0], ["max_bytes", 0], + ["max_redirects", -1], ["max_cache_bytes", 0], ["connect_timeout_ms", Number.MAX_SAFE_INTEGER + 1], +] as const; + +test.each(invalidHttpBounds)("rejects invalid HTTP bound %s=%s", (field, value) => { + expectSafeEvidenceError(withEvidence({ + type: "http", + uris: ["https://example.com/manifest"], + [field]: value, + }), new RegExp(field, "i")); +}); + +const invalidS3Uris = [ + "https://bucket/prefix", "s3:///prefix", "s3://user:CANARY-S3@bucket/prefix", + "s3://bucket/prefix?token=CANARY-S3", "s3://bucket/prefix#CANARY-S3", + "s3://bucket:123/CANARY-S3", "s3://bucket/%2e%2e/CANARY-S3", + "s3://127.0.0.1/CANARY-S3", "s3://UPPERCASE/CANARY-S3", +]; + +test.each(invalidS3Uris)("rejects invalid S3 URI %# without leaking it", (uri) => { + expectSafeEvidenceError(withEvidence({ type: "s3", uri }), /evidence.*source.*uri/i, "CANARY-S3"); +}); + +const invalidS3Endpoints = [ + { endpoint_url: "ftp://objects.example", trusted_endpoint: true }, + { endpoint_url: "https://user:CANARY-S3@objects.example", trusted_endpoint: true }, + { endpoint_url: "https://objects.example/path", trusted_endpoint: true }, + { endpoint_url: "https://objects.example?token=CANARY-S3", trusted_endpoint: true }, + { endpoint_url: "https://objects.example#CANARY-S3", trusted_endpoint: true }, + { endpoint_url: "https://objects.example", trusted_endpoint: false }, + { endpoint_url: "http://objects.example", trusted_endpoint: true, allow_insecure_endpoint: false }, + { endpoint_url: "HTTP://objects.example", trusted_endpoint: true, allow_insecure_endpoint: false }, + { trusted_endpoint: true }, + { allow_private_endpoint: true }, + { allow_insecure_endpoint: true }, +]; + +test.each(invalidS3Endpoints)("rejects unsafe or inconsistent S3 endpoint %#", (endpoint) => { + expectSafeEvidenceError(withEvidence({ type: "s3", uri: "s3://bucket/prefix", ...endpoint }), /evidence.*source/i, "CANARY-S3"); +}); + +const invalidS3Bounds = [ + ["max_bytes", 0], ["max_objects", 0], ["max_pages", 0], ["page_size", 0], + ["max_objects", Number.MAX_SAFE_INTEGER + 1], +] as const; + +test.each(invalidS3Bounds)("rejects invalid S3 bound %s=%s", (field, value) => { + expectSafeEvidenceError(withEvidence({ + type: "s3", uri: "s3://bucket/prefix", [field]: value, + }), new RegExp(field, "i")); +}); + +test.each([ + ["max_chunk_chars", 0], + ["max_chunk_chars", Number.MAX_SAFE_INTEGER + 1], + ["retain_published_generations", 0], + ["retain_published_generations", Number.MAX_SAFE_INTEGER + 1], +] as const)("rejects invalid evidence policy bound %s=%s", (field, value) => { + expectSafeEvidenceError(withEvidence({ + type: "filesystem", uri: "psd-clinical/evidence", + }, { ...explicitPolicy, [field]: value }), new RegExp(field, "i")); +}); diff --git a/compose.yaml b/compose.yaml index fdbc1e42..f395f575 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,14 +1,3 @@ -# ThothII — deploy embedded nel portale omics_portal (PRODUZIONE). -# core + frontend sulla rete esterna del portale (omics_portal_omics_network, -# creata da Compose col prefisso project). Alias thothii-core/thothii-frontend -# per il DNS usato dagli upstream nginx del portale. -# NESSUNA porta host esposta: il backend è invisibile dall'esterno. -# -# Prereq: devono esistere entrambe le reti esterne: il portale crea -# omics_portal_omics_network e lo stack vLLM crea localllm_default. -# Avvia il portale con: -# cd /home/chirone/omics_portal && docker compose up -d -# Poi: docker compose up -d --build name: thothii services: @@ -17,45 +6,210 @@ services: context: . dockerfile: docker/core.Dockerfile image: thothii-core:local - env_file: [deploy/thothii.env] environment: HOST: 0.0.0.0 PORT: "8787" THT_HARNESS_DIR: /app/harness THT_BIN: /opt/venv/bin/tht + THT_DATA_ROOT: /data SETTINGS_FILE: /data/settings/settings.json - THT_CONFIG: /app/harness/workspaces/local.yaml # configPath di default per i route tht senza workspace esplicito - THT_MODEL_API_KEY_FILE: /data/secrets/model_api_key # provider key per buildPiChildEnv (codex) - AUTH_MODE: ${AUTH_MODE:-none} + THT_MAINTENANCE_FILE: /data/settings/maintenance.json + THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry + THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE} + THT_WORKSPACE_GIT_BRANCH: ${THT_WORKSPACE_GIT_BRANCH:-main} + THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-local} + THT_WORKSPACE_SECRET_STORE_ROOT: /data/workspace-secrets + THT_WORKSPACE_SECRET_RUNTIME_ROOT: /tmp/thothii-workspace-secrets + THT_WORKSPACE_SECRET_ROOTS: /run/secrets + THT_SECRETS_FILE: /run/secrets/thothii.secrets + THT_PI_AUTH_FILE: /home/thoth/.pi/agent/auth.json + THT_AUTH_CONFIG_FILE: /run/thothii-auth/auth.yaml + THT_AUTH_STATE_ROOT: /data/auth + THT_DB_NAME: ${THT_DB_NAME:-} + THT_DWH_REST_URL: ${THT_DWH_REST_URL:-} + THT_LLM_URL: ${THT_LLM_URL:-} + THT_INTERNAL_QDRANT_URL: http://qdrant:6333 + THT_INTERNAL_EMBEDDING_URL: http://embedding:11434 + THT_INTERNAL_EMBEDDING_MODEL: qwen3-embedding:0.6b + THT_INTERNAL_EMBEDDING_DIMENSIONS: "1024" MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4} - extra_hosts: - - "host.docker.internal:host-gateway" # Supabase :5438 + Ollama :11434 sull'host volumes: - - /home/chirone/thothii-data:/data - - /home/chirone/thothii-data/pi-config:/home/thoth/.pi + - settings:/data/settings + - pi-state:/home/thoth/.pi - ${PI_AUTH_FILE:?set PI_AUTH_FILE}:/home/thoth/.pi/agent/auth.json:ro - - /home/chirone/chirone/etl/docs/evidence:/data/evidence:ro - restart: unless-stopped + - ./deploy/pi/models.json:/home/thoth/.pi/agent/models.json:ro + - ./deploy/pi/settings.json:/home/thoth/.pi/agent/settings.json:ro + - workspace-registry:/data/workspace-registry + - workspace-secrets:/data/workspace-secrets + - sessions:/data/sessions + - ${THT_AUTH_CONFIG_ROOT:?set THT_AUTH_CONFIG_ROOT}:/run/thothii-auth:ro + - auth-state:/data/auth + secrets: + - source: thothii_secrets + target: thothii.secrets + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8787/health"] + interval: 15s + timeout: 3s + retries: 5 + start_period: 30s + depends_on: + qdrant: + condition: service_healthy + embedding-model-init: + condition: service_completed_successfully networks: - omics_portal_omics_network: - aliases: ["thothii-core"] - localllm_default: {} + - thothii + + workspace-maintenance: + image: thothii-core:local + profiles: [workspace-maintenance] + pull_policy: never + entrypoint: ["/usr/bin/tini", "--", "/app/docker/workspace-maintenance-entrypoint.sh"] + environment: + THT_HARNESS_DIR: /app/harness + THT_BIN: /opt/venv/bin/tht + THT_DATA_ROOT: /data + THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry + THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE} + THT_WORKSPACE_GIT_BRANCH: ${THT_WORKSPACE_GIT_BRANCH:-main} + THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-local} + THT_WORKSPACE_SECRET_STORE_ROOT: /data/workspace-secrets + THT_WORKSPACE_SECRET_RUNTIME_ROOT: /tmp/thothii-workspace-secrets + THT_WORKSPACE_SECRET_ROOTS: /run/secrets + THT_SECRETS_FILE: /run/secrets/thothii.secrets + THT_DB_NAME: ${THT_DB_NAME:-} + THT_DWH_REST_URL: ${THT_DWH_REST_URL:-} + THT_LLM_URL: ${THT_LLM_URL:-} + THT_INTERNAL_QDRANT_URL: http://qdrant:6333 + THT_INTERNAL_EMBEDDING_URL: http://embedding:11434 + THT_INTERNAL_EMBEDDING_MODEL: qwen3-embedding:0.6b + THT_INTERNAL_EMBEDDING_DIMENSIONS: "1024" + HOME: /tmp/thoth + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_SESSION_TOKEN: "" + AWS_PROFILE: "" + AWS_DEFAULT_PROFILE: "" + AWS_CONFIG_FILE: /dev/null + AWS_SHARED_CREDENTIALS_FILE: /dev/null + volumes: + - type: volume + source: workspace-registry + target: /data/workspace-registry + read_only: true + - type: volume + source: sessions + target: /data/sessions + - type: volume + source: workspace-secrets + target: /data/workspace-secrets + secrets: + - source: thothii_secrets + target: thothii.secrets + user: "10001:10001" + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=1g,mode=1777 + - /var/tmp:rw,noexec,nosuid,nodev,size=128m,mode=1777 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + restart: "no" + networks: + - thothii frontend: build: context: . dockerfile: docker/frontend.Dockerfile args: - VITE_BASE: /datamart-builder/assets/ - VITE_BACKEND_URL: /datamart-builder/api + VITE_BASE: / + VITE_BACKEND_URL: /api image: thothii-frontend:local - restart: unless-stopped + depends_on: + core: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/ || exit 1"] + interval: 15s + timeout: 3s + retries: 5 + start_period: 10s networks: - omics_portal_omics_network: - aliases: ["thothii-frontend"] + - thothii + + qdrant: + image: qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c + expose: + - "6333" + volumes: + - qdrant-data:/qdrant/storage + healthcheck: + test: + - CMD-SHELL + - > + /usr/bin/bash -lc "exec 3<>/dev/tcp/127.0.0.1/6333 && + printf 'GET /healthz HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' >&3 && + grep -q '200 OK' <&3" + interval: 15s + timeout: 3s + retries: 10 + start_period: 10s + networks: + - thothii + + embedding: + image: ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a + command: ["serve"] + expose: + - "11434" + volumes: + - embedding-models:/root/.ollama + healthcheck: + test: + - CMD-SHELL + - > + /usr/bin/bash -lc "exec 3<>/dev/tcp/127.0.0.1/11434 && + printf 'GET /api/tags HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' >&3 && + grep -q '200 OK' <&3" + interval: 15s + timeout: 5s + retries: 20 + start_period: 10s + networks: + - thothii + + embedding-model-init: + image: ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a + entrypoint: ["/usr/bin/bash", "/opt/thoth/embedding-model-init.sh"] + environment: + OLLAMA_BASE_URL: http://embedding:11434 + OLLAMA_MODEL: qwen3-embedding:0.6b + OLLAMA_WAIT_TIMEOUT_SEC: "180" + volumes: + - embedding-models:/root/.ollama + - ./docker/embedding-model-init.sh:/opt/thoth/embedding-model-init.sh:ro + depends_on: + embedding: + condition: service_healthy + networks: + - thothii networks: - omics_portal_omics_network: - external: true - localllm_default: - external: true + thothii: + +volumes: + settings: + pi-state: + workspace-registry: + workspace-secrets: + sessions: + qdrant-data: + embedding-models: + auth-state: + +secrets: + thothii_secrets: + file: "${THT_SECRETS_FILE:?set THT_SECRETS_FILE}" diff --git a/deploy/compose.embedding-gpu.yaml b/deploy/compose.embedding-gpu.yaml new file mode 100644 index 00000000..8b7731b8 --- /dev/null +++ b/deploy/compose.embedding-gpu.yaml @@ -0,0 +1,7 @@ +services: + embedding: + deploy: + resources: + reservations: + devices: + - capabilities: ["gpu"] diff --git a/deploy/compose.git-https.yaml b/deploy/compose.git-https.yaml new file mode 100644 index 00000000..7e4fc6d5 --- /dev/null +++ b/deploy/compose.git-https.yaml @@ -0,0 +1,16 @@ +# Select this override only for an HTTPS Git remote. The separate CA mount keeps TLS validation +# explicit; neither host-only source file nor its contents belongs in the base Compose contract. +# Active-snapshot workspace-maintenance operations intentionally receive no Git credential mounts. +x-thoth-git-transport: https + +services: + core: + environment: + GIT_CONFIG_COUNT: "2" + GIT_CONFIG_KEY_0: credential.helper + GIT_CONFIG_VALUE_0: store --file=/run/secrets/workspace-registry-git-credentials + GIT_CONFIG_KEY_1: http.sslCAInfo + GIT_CONFIG_VALUE_1: /run/secrets/workspace-registry-git-ca + volumes: + - ${THT_WORKSPACE_GIT_CREDENTIALS_FILE:?set THT_WORKSPACE_GIT_CREDENTIALS_FILE}:/run/secrets/workspace-registry-git-credentials:ro + - ${THT_WORKSPACE_GIT_CA_FILE:?set THT_WORKSPACE_GIT_CA_FILE}:/run/secrets/workspace-registry-git-ca:ro diff --git a/deploy/compose.git-ssh.yaml b/deploy/compose.git-ssh.yaml new file mode 100644 index 00000000..6c33b336 --- /dev/null +++ b/deploy/compose.git-ssh.yaml @@ -0,0 +1,12 @@ +# Select this override only for an SSH Git remote. The host-only source files must be absolute, +# normalized paths; strict host-key checking is mandatory for read-only repository fetch and pull. +# Active-snapshot workspace-maintenance operations intentionally receive no Git credential mounts. +x-thoth-git-transport: ssh + +services: + core: + environment: + GIT_SSH_COMMAND: ssh -i /run/secrets/workspace-registry-git-ssh-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/run/secrets/workspace-registry-git-known-hosts + volumes: + - ${THT_WORKSPACE_GIT_SSH_KEY_FILE:?set THT_WORKSPACE_GIT_SSH_KEY_FILE}:/run/secrets/workspace-registry-git-ssh-key:ro + - ${THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE:?set THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE}:/run/secrets/workspace-registry-git-known-hosts:ro diff --git a/deploy/compose.local-vector.yaml b/deploy/compose.local-vector.yaml deleted file mode 100644 index 0bf41337..00000000 --- a/deploy/compose.local-vector.yaml +++ /dev/null @@ -1,90 +0,0 @@ -services: - core: - profiles: [local-vector] - environment: - THT_VECTOR_DATABASE: "${THT_VECTOR_DATABASE:-thoth}" - THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}" - THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - depends_on: - vector-migrate: - condition: service_completed_successfully - - frontend: - profiles: [local-vector] - - vector-db: - image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb - profiles: [local-vector] - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} - environment: - POSTGRES_DB: "${THT_VECTOR_DATABASE:-thoth}" - POSTGRES_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}" - THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}" - THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - entrypoint: [/opt/thoth/vector-db-entrypoint.sh] - volumes: - - vector_data:/var/lib/postgresql/data - - ./deploy/vector/vector-db-entrypoint.sh:/opt/thoth/vector-db-entrypoint.sh:ro - - ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro - healthcheck: - test: [CMD-SHELL, "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - interval: 5s - timeout: 3s - retries: 20 - start_period: 10s - restart: unless-stopped - - vector-reconcile: - image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb - profiles: [local-vector] - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} - environment: - PGHOST: vector-db - PGPORT: 5432 - PGDATABASE: "${THT_VECTOR_DATABASE:-thoth}" - PGUSER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" - THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}" - THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}" - THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets - entrypoint: [/opt/thoth/reconcile-roles.sh] - secrets: [{source: thothii_secrets, target: thothii.secrets}] - volumes: - - ./deploy/vector/reconcile-roles.sh:/opt/thoth/reconcile-roles.sh:ro - - ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro - depends_on: - vector-db: {condition: service_healthy} - restart: "no" - - vector-migrate: - image: thothii-core:local - profiles: [local-vector] - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} - build: - context: . - dockerfile: docker/core.Dockerfile - entrypoint: [sh, -ec] - command: - - | - . /opt/thoth/secret-policy.sh - export PGPASSWORD=$$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_MIGRATOR_PASSWORD) - exec /opt/venv/bin/tht vector migrate --database-url "postgresql+psycopg2://${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}@vector-db:5432/${THT_VECTOR_DATABASE:-thoth}" --json - secrets: [{source: thothii_secrets, target: thothii.secrets}] - environment: - THT_SECRETS_FILE: /run/secrets/thothii.secrets - volumes: - - ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro - depends_on: - vector-reconcile: {condition: service_completed_successfully} - restart: "no" - -volumes: - vector_data: - labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"} diff --git a/deploy/compose.local.yaml b/deploy/compose.local.yaml index 9c4ebbf9..75d3ae6f 100644 --- a/deploy/compose.local.yaml +++ b/deploy/compose.local.yaml @@ -1,6 +1,25 @@ services: core: environment: - # Non-secret settings come from the root .env interpolation file. - AUTH_MODE: "${AUTH_MODE:-none}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets + NODE_ENV: development + THT_WORKSPACE_INSTALLATION_ID: local + ports: + - "127.0.0.1:${THOTH_CORE_HTTP_PORT:-8787}:8787" + restart: "no" + + frontend: + ports: + - "127.0.0.1:${THOTH_HTTP_PORT:-8080}:8080" + restart: "no" + + workspace-maintenance: + environment: + THT_WORKSPACE_INSTALLATION_ID: local + restart: "no" + + # Local development only: expose the built-in Qdrant web dashboard on loopback. + # The server profile keeps Qdrant private on the Compose network. + qdrant: + ports: + - "127.0.0.1:6333:6333" + restart: "no" diff --git a/deploy/compose.preprocess-local-vector.yaml b/deploy/compose.preprocess-local-vector.yaml deleted file mode 100644 index 8ce48d61..00000000 --- a/deploy/compose.preprocess-local-vector.yaml +++ /dev/null @@ -1,14 +0,0 @@ -services: - preprocess-evidence: - environment: - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - depends_on: - vector-migrate: {condition: service_completed_successfully} - - preprocess-dwh: - environment: - THT_SECRETS_FILE: /run/secrets/thothii.secrets - secrets: [{source: thothii_secrets, target: thothii.secrets}] - depends_on: - vector-migrate: {condition: service_completed_successfully} diff --git a/deploy/compose.preprocess.yaml b/deploy/compose.preprocess.yaml index 0b18e140..4873ba99 100644 --- a/deploy/compose.preprocess.yaml +++ b/deploy/compose.preprocess.yaml @@ -1,3 +1,5 @@ +# Retired for operator use: this profile remains only as a non-public engine-fixture path. +# It exercises the legacy preprocessing fixtures and must not become a second operator interface. services: preprocess-evidence: image: thothii-core:local @@ -9,13 +11,17 @@ services: command: ["mkdir -p /data/workspaces/preprocess-evidence && exec /app/docker/core-entrypoint.sh preprocess evidence --json -c /app/harness/workspaces/preprocess-evidence.yaml"] environment: THT_DATA_ROOT: /data - THT_OLLAMA_URL: "${THT_OLLAMA_URL:-http://host.docker.internal:11434}" THT_SECRETS_FILE: /run/secrets/thothii.secrets secrets: [{source: thothii_secrets, target: thothii.secrets}] volumes: - thoth_data:/data - ./deploy/workspaces:/app/harness/workspaces:ro restart: "no" + depends_on: + qdrant: + condition: service_healthy + embedding-model-init: + condition: service_completed_successfully preprocess-dwh: image: thothii-core:local @@ -33,3 +39,6 @@ services: - thoth_data:/data - ./deploy/workspaces:/app/harness/workspaces:ro restart: "no" + +volumes: + thoth_data: diff --git a/deploy/compose.production.yaml b/deploy/compose.production.yaml deleted file mode 100644 index ab419743..00000000 --- a/deploy/compose.production.yaml +++ /dev/null @@ -1,11 +0,0 @@ -services: - core: - environment: - AUTH_MODE: upstream - THOTH_PUBLIC_EXPOSURE: "true" - THT_DB_NAME: ${THT_DB_NAME:?set THT_DB_NAME} - THT_DWH_REST_URL: ${THT_DWH_REST_URL:?set THT_DWH_REST_URL} - THT_VEC_REST_URL: ${THT_VEC_REST_URL:?set THT_VEC_REST_URL} - THT_OLLAMA_URL: ${THT_OLLAMA_URL:?set THT_OLLAMA_URL} - THT_DOCS_ROOT: ${THT_DOCS_ROOT:-/data/workspaces/example/evidence-source} - THT_SECRETS_FILE: /run/secrets/thothii.secrets diff --git a/deploy/compose.psd-local.yaml.example b/deploy/compose.psd-local.yaml.example deleted file mode 100644 index 2a52b893..00000000 --- a/deploy/compose.psd-local.yaml.example +++ /dev/null @@ -1,52 +0,0 @@ -services: - core: - environment: - AUTH_MODE: ${AUTH_MODE:-none} - THOTH_PUBLIC_EXPOSURE: ${THOTH_PUBLIC_EXPOSURE:-false} - MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4} - PI_PROVIDER: ${PI_PROVIDER:?set PI_PROVIDER} - PI_MODEL: ${PI_MODEL:?set PI_MODEL} - PI_THINKING: ${PI_THINKING:-medium} - THT_PROFILE: ${THT_PROFILE:-workstation} - THT_DB_NAME: ${THT_DB_NAME:?set THT_DB_NAME} - THT_DWH_REST_URL: ${THT_DWH_REST_URL:?set THT_DWH_REST_URL} - THT_VEC_REST_URL: ${THT_VEC_REST_URL:?set THT_VEC_REST_URL} - THT_VEC_WRITE_REST_URL: ${THT_VEC_WRITE_REST_URL:?set THT_VEC_WRITE_REST_URL} - THT_OLLAMA_URL: ${THT_OLLAMA_URL:?set THT_OLLAMA_URL} - THT_SECRETS_FILE: /run/secrets/thothii.secrets - THT_DOCS_ROOT: /data/workspaces/psd - THT_CONFIG: /app/harness/config/tht.yaml - extra_hosts: - - host.docker.internal:host-gateway - networks: !override - default: - aliases: [core, thothii-core] - volumes: - - thoth_data:/data - - thoth_pi_config:/home/thoth/.pi - - ./deploy/pi/models.json:/home/thoth/.pi/agent/models.json:ro - - ./deploy/pi/settings.json:/home/thoth/.pi/agent/settings.json:ro - - ${THT_SECRETS_FILE:?set THT_SECRETS_FILE}:/run/secrets/thothii.secrets:ro - - ${THT_PSD_WORKSPACE_HOST_PATH:?set THT_PSD_WORKSPACE_HOST_PATH}/evidence:/data/evidence:ro - - ./deploy/workspaces/psd.yaml:/app/harness/config/tht.yaml:ro - - ${THT_PSD_WORKSPACE_HOST_PATH:?set THT_PSD_WORKSPACE_HOST_PATH}:/data/workspaces/psd - - frontend: - build: - args: - VITE_BASE: / - VITE_BACKEND_URL: /api - ports: - - "127.0.0.1:8099:8080" - networks: !override - default: - aliases: [frontend, thothii-frontend] - -volumes: - thoth_data: - thoth_pi_config: - -networks: - default: - external: true - name: thothii_default diff --git a/deploy/compose.server.yaml b/deploy/compose.server.yaml new file mode 100644 index 00000000..4bde7877 --- /dev/null +++ b/deploy/compose.server.yaml @@ -0,0 +1,60 @@ +services: + core: + environment: + THOTH_PUBLIC_EXPOSURE: "true" + THT_DATA_ROOT: /data + THT_WORKSPACE_INSTALLATION_ID: server + # prepare-server-pi-state.sh creates the regular child targets before this parent bind is used. + # The real configuration sources still remain separate read-only mounts. + volumes: !override + - type: bind + source: ${THT_DATA_ROOT:?set THT_DATA_ROOT} + target: /data + - type: bind + source: ${THT_AUTH_CONFIG_ROOT:?set THT_AUTH_CONFIG_ROOT} + target: /run/thothii-auth + read_only: true + - type: bind + source: ${THT_PI_STATE_ROOT:?set THT_PI_STATE_ROOT} + target: /home/thoth/.pi + - type: bind + source: ${PI_AUTH_FILE:?set PI_AUTH_FILE} + target: /home/thoth/.pi/agent/auth.json + read_only: true + - type: bind + source: ./deploy/pi/models.json + target: /home/thoth/.pi/agent/models.json + read_only: true + - type: bind + source: ./deploy/pi/settings.json + target: /home/thoth/.pi/agent/settings.json + read_only: true + - type: bind + source: ${THT_WORKSPACE_REGISTRY_ROOT:?set THT_WORKSPACE_REGISTRY_ROOT} + target: /data/workspace-registry + restart: unless-stopped + # Deprecated migration adapter: only use this when no auth.yaml is mounted yet. + # AUTH_MODE: upstream + + frontend: + ports: + - "${THOTH_SERVER_BIND:-127.0.0.1}:${THOTH_HTTP_PORT:-8080}:8080" + restart: unless-stopped + + + workspace-maintenance: + environment: + THT_DATA_ROOT: /data + THT_WORKSPACE_INSTALLATION_ID: server + volumes: !override + - type: bind + source: ${THT_DATA_ROOT:?set THT_DATA_ROOT}/sessions + target: /data/sessions + - type: bind + source: ${THT_WORKSPACE_REGISTRY_ROOT:?set THT_WORKSPACE_REGISTRY_ROOT} + target: /data/workspace-registry + read_only: true + - type: bind + source: ${THT_DATA_ROOT:?set THT_DATA_ROOT}/workspace-secrets + target: /data/workspace-secrets + restart: "no" diff --git a/deploy/compose.session-server.yaml.example b/deploy/compose.session-server.yaml.example index fbb1bc4f..43821be6 100644 --- a/deploy/compose.session-server.yaml.example +++ b/deploy/compose.session-server.yaml.example @@ -3,7 +3,6 @@ services: core: environment: - AUTH_MODE: upstream THOTH_PUBLIC_EXPOSURE: "true" THT_SESSION_STORAGE: postgres THT_CONFIG: /app/harness/workspaces/server-sessions.yaml @@ -20,7 +19,7 @@ services: - source: session_ca target: session_ca.pem volumes: - - ./deploy/workspaces:/app/harness/workspaces:ro + - ${THT_SERVER_WORKSPACE_CONFIG:?set THT_SERVER_WORKSPACE_CONFIG}:/app/harness/workspaces/server-sessions.yaml:ro # Run manually during the maintenance window. It is not a dependency of core, # so the application never gains the schema-changing migrator credential. diff --git a/deploy/env.example b/deploy/env.example deleted file mode 100644 index 881dfca1..00000000 --- a/deploy/env.example +++ /dev/null @@ -1,40 +0,0 @@ -# Deprecated compatibility template; it is not loaded by Docker Compose automatically. -# New installations must copy ../.env.example to ../.env and run -# `docker compose up --build -d` from the repository root. Keep this file only for -# staged upgrades that still invoke `--env-file deploy/env.example` explicitly. -# Never put secret values in this file. - -COMPOSE_FILE=compose.yaml -COMPOSE_PROFILES= -THT_SECRETS_FILE=deploy/secrets/thothii.secrets - -PI_PROVIDER= -PI_MODEL= -PI_THINKING= -MAX_PI_PROCESSES=4 -AUTH_MODE=none - -# User-owned session storage. Keep local for the loopback-only development stack. -# The server-session overlay requires every THT_SESSION_* value below. -THT_SESSION_STORAGE=local -THT_SESSION_DB_HOST= -THT_SESSION_DB_PORT=5432 -THT_SESSION_DB_NAME= -THT_SESSION_RUNTIME_USER= -THT_SESSION_RUNTIME_PASSWORD_SOURCE= -THT_SESSION_MIGRATOR_USER= -THT_SESSION_MIGRATOR_PASSWORD_SOURCE= -THT_SESSION_DB_SSLMODE=verify-full -THT_SESSION_CA_SOURCE= - -THT_DB_NAME= -THT_DWH_REST_URL= -THT_VEC_REST_URL= -THT_OLLAMA_URL= -THT_DOCS_ROOT=/data/workspaces/example/evidence-source - -THT_VECTOR_DATABASE=thoth -THT_VECTOR_BOOTSTRAP_USER=postgres -THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator -THT_VECTOR_READER_USER=thoth_vector_reader -THT_VECTOR_WRITER_USER=thoth_vector_writer diff --git a/deploy/env/local.env.example b/deploy/env/local.env.example new file mode 100644 index 00000000..d7627d2a --- /dev/null +++ b/deploy/env/local.env.example @@ -0,0 +1,19 @@ +# Local profile defaults. `tht setup` writes the active non-secret file to +# deploy//operator.env; this tracked file is only an example. +# Values are locations and non-secret defaults, never credentials. +THOTH_HTTP_PORT=8080 +THOTH_CORE_HTTP_PORT=8787 +MAX_PI_PROCESSES=4 +PI_AUTH_FILE=/absolute/path/to/pi-auth.json +THT_SECRETS_FILE=/absolute/path/to/thothii.secrets +THT_AUTH_CONFIG_ROOT=/absolute/path/to/thothii-auth + +THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git +THT_WORKSPACE_GIT_BRANCH=main + +THT_DB_NAME=warehouse +THT_DWH_REST_URL=https://dwh.example.invalid +THT_LLM_URL=https://llm.example.invalid + +# Optional explicit GPU override for Linux hosts that expose a Docker-compatible GPU device. +# THOTH_ENABLE_EMBEDDING_GPU=1 diff --git a/deploy/env/server.env.example b/deploy/env/server.env.example new file mode 100644 index 00000000..152c3c2e --- /dev/null +++ b/deploy/env/server.env.example @@ -0,0 +1,34 @@ +# Server profile defaults. Copy this file to a reviewed, untracked server.env and pass it with --env-file. +# Values are non-secret documentation values only. +THOTH_SERVER_BIND=127.0.0.1 +THOTH_HTTP_PORT=8080 +MAX_PI_PROCESSES=4 +PI_AUTH_FILE=/absolute/path/to/pi-auth.json +THT_SECRETS_FILE=/absolute/path/to/thothii.secrets +THT_AUTH_CONFIG_ROOT=/absolute/path/to/thothii-auth + +THT_DATA_ROOT=/srv/thothii/data +THT_PI_STATE_ROOT=/srv/thothii/pi-state +THT_WORKSPACE_REGISTRY_ROOT=/srv/thothii/workspace-registry +THT_BACKUP_ROOT=/srv/thothii-backups +THT_SERVER_WORKSPACE_CONFIG=/absolute/path/to/server-sessions.yaml +THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git +THT_WORKSPACE_GIT_BRANCH=main + +THT_DB_NAME=warehouse +THT_DWH_REST_URL=https://dwh.example.invalid +THT_LLM_URL=https://llm.example.invalid + +# Optional explicit GPU override for Linux hosts that expose a Docker-compatible GPU device. +# THOTH_ENABLE_EMBEDDING_GPU=1 + +# Public server session storage. Values are endpoints, roles, or protected source-file paths. +THT_SESSION_DB_HOST=sessions-db.example.invalid +THT_SESSION_DB_PORT=5432 +THT_SESSION_DB_NAME=thoth_sessions +THT_SESSION_RUNTIME_USER=thoth_sessions_app +THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate +THT_SESSION_DB_SSLMODE=verify-full +THT_SESSION_RUNTIME_PASSWORD_SOURCE=/absolute/path/to/session-runtime-password +THT_SESSION_MIGRATOR_PASSWORD_SOURCE=/absolute/path/to/session-migrator-password +THT_SESSION_CA_SOURCE=/absolute/path/to/session-ca.pem diff --git a/deploy/nginx-authenticated-proxy.conf.example b/deploy/nginx-authenticated-proxy.conf.example index 86c9b502..cb48c0f2 100644 --- a/deploy/nginx-authenticated-proxy.conf.example +++ b/deploy/nginx-authenticated-proxy.conf.example @@ -1,5 +1,5 @@ -# Host nginx example. The auth service MUST authenticate every request and return a stable -# identity in X-Authenticated-User. ThothII itself remains on 127.0.0.1:8080. +# Host nginx example. The auth service MUST authenticate every request and return only the +# normalized X-Thoth-* identity/admin claims below. ThothII remains on 127.0.0.1:8080. server { listen 443 ssl; server_name thoth.example.test; @@ -13,12 +13,33 @@ server { proxy_pass_request_body off; proxy_set_header Content-Length ""; proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Authenticated-User ""; + proxy_set_header X-Thoth-Principal-Issuer ""; + proxy_set_header X-Thoth-Principal-Subject ""; + proxy_set_header X-Thoth-Principal-Display-Name ""; + proxy_set_header X-Thoth-Is-Admin ""; + proxy_set_header X-Thoth-Trusted-Principal-Issuer ""; + proxy_set_header X-Thoth-Trusted-Principal-Subject ""; + proxy_set_header X-Thoth-Trusted-Principal-Display-Name ""; + proxy_set_header X-Thoth-Trusted-Is-Admin ""; } location / { auth_request /_authenticate; - auth_request_set $authenticated_user $upstream_http_x_authenticated_user; - proxy_set_header X-Authenticated-User $authenticated_user; + auth_request_set $thoth_principal_issuer $upstream_http_x_thoth_principal_issuer; + auth_request_set $thoth_principal_subject $upstream_http_x_thoth_principal_subject; + auth_request_set $thoth_principal_display_name $upstream_http_x_thoth_principal_display_name; + auth_request_set $thoth_is_admin $upstream_http_x_thoth_is_admin; + # Clear public normalized claims and carry auth_request results over the private hop. + proxy_set_header X-Authenticated-User ""; + proxy_set_header X-Thoth-Principal-Issuer ""; + proxy_set_header X-Thoth-Principal-Subject ""; + proxy_set_header X-Thoth-Principal-Display-Name ""; + proxy_set_header X-Thoth-Is-Admin ""; + proxy_set_header X-Thoth-Trusted-Principal-Issuer $thoth_principal_issuer; + proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject; + proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name; + proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin; proxy_set_header X-Forwarded-Proto https; proxy_set_header Host $host; proxy_pass http://127.0.0.1:8080; diff --git a/deploy/pi/models.json b/deploy/pi/models.json index 1b49dac8..75e48255 100644 --- a/deploy/pi/models.json +++ b/deploy/pi/models.json @@ -6,8 +6,8 @@ "apiKey": "$ZAI_API_KEY", "models": [ { - "id": "glm-5.2", - "name": "GLM-5.2", + "id": "glm-5.3", + "name": "GLM-5.3", "reasoning": true, "contextWindow": 200000, "maxTokens": 131072 diff --git a/deploy/pi/settings.json b/deploy/pi/settings.json index 31a18ce0..08f6d1b4 100644 --- a/deploy/pi/settings.json +++ b/deploy/pi/settings.json @@ -1,7 +1,7 @@ { "defaultProjectTrust": "always", "enabledModels": [ - "zai/glm-5.2", + "zai/glm-5.3", "deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-pro", "aritmolab/qwen3.6-35b-a3b" diff --git a/deploy/psd/.gitignore b/deploy/psd/.gitignore new file mode 100644 index 00000000..1a939266 --- /dev/null +++ b/deploy/psd/.gitignore @@ -0,0 +1,8 @@ +# File operatore reali (contengono o referenziano segreti): non tracciare. +operator.env +workspace-bindings.env +workspace-bindings.yaml +thothii-installation.yaml +connector-secrets.yaml +secrets/* +!secrets/.gitkeep diff --git a/deploy/psd/operator.env.example b/deploy/psd/operator.env.example new file mode 100644 index 00000000..ebdd2a64 --- /dev/null +++ b/deploy/psd/operator.env.example @@ -0,0 +1,28 @@ +# Esempio soltanto: `tht setup` genera deploy//operator.env (non tracciato). +# Solo path non-segreti: i valori delle credenziali restano nei file protetti indicati qui sotto. +THT_WORKSPACE_GIT_REMOTE=git@github.com:mptyl/tht-workspace-psd.git +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_INSTALLATION_ID=psd-local +THT_WORKSPACE_GIT_SSH_KEY_FILE=/deploy/psd/secrets/git-ssh-key +THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/deploy/psd/secrets/git-known-hosts + +# App +THT_SECRETS_FILE=/deploy/psd/secrets/thothii.secrets +PI_AUTH_FILE=/deploy/psd/secrets/pi-auth.json +THT_AUTH_CONFIG_ROOT=/deploy/psd/auth +# DWH and Evidence credentials are entered later in Workspace management and stored encrypted +# by the backend. They do not depend on host filesystem paths. + +# Pi (LLM) +PI_PROVIDER=zai +PI_MODEL=glm-5.3 +PI_THINKING=medium + +# App defaults +THT_DWH_PRECHECK=true +THOTH_PUBLIC_EXPOSURE=false +MAX_PI_PROCESSES=4 +THOTH_HTTP_PORT=8080 +THOTH_CORE_HTTP_PORT=8787 +THT_DB_NAME=postgres +THT_DWH_REST_URL=https://supabase-aritmolab.policlinicosandonato.it/dwh/ diff --git a/deploy/psd/secrets/.gitkeep b/deploy/psd/secrets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/psd/thothii-installation.yaml.example b/deploy/psd/thothii-installation.yaml.example new file mode 100644 index 00000000..d4afc2c3 --- /dev/null +++ b/deploy/psd/thothii-installation.yaml.example @@ -0,0 +1,14 @@ +# Esempio soltanto: `tht setup` genera deploy//thothii-installation.yaml. +# Sostituisci i path assoluti se usi questo riferimento per una configurazione avanzata. +# Seleziona UN solo override Git (https o ssh). +profile: local +projectDirectory: "/projects/ThothII" +envFile: "/projects/ThothII/deploy/psd/operator.env" +workspaceRepository: + remote: git@github.com:mptyl/tht-workspace-psd.git + branch: main + access: ssh +authentication: + configDirectory: "/projects/ThothII/deploy/psd/auth" +overrides: + - "/projects/ThothII/deploy/compose.git-ssh.yaml" diff --git a/deploy/psd/workspace-bindings.env.example b/deploy/psd/workspace-bindings.env.example new file mode 100644 index 00000000..b8754a2d --- /dev/null +++ b/deploy/psd/workspace-bindings.env.example @@ -0,0 +1,8 @@ +# Copia in un file operatore non tracciato (workspace-bindings.env). +# Contiene SOLO bindings THT_WS_* non segreti. I *_FILE sono path DI CONTENITORE +# (/run/secrets/...), popolati dal connector override generato dai *_SOURCE dell'operatore env. +THT_WS_PSD_CLINICAL_DWH_TRANSPORT=rest_api +THT_WS_PSD_CLINICAL_DWH_BASE_URL=https://supabase-aritmolab.policlinicosandonato.it/dwh/ +THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE=/run/secrets/psd-clinical-dwh-api-key +# Opzionale: solo se il DWH REST presenta una CA interna/privata. +THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE=/run/secrets/psd-clinical-dwh-ca.pem diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 8ea67f50..23252411 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -9,10 +9,15 @@ chmod 600 deploy/secrets/thothii.secrets ``` The file uses strict `KEY=VALUE` lines (comments and blank lines are allowed). The supported -keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_VEC_API_KEY`, -`THT_VEC_WRITE_API_KEY`, and the four `THT_VECTOR_*_PASSWORD` role passwords. Values must be -non-empty and contain no whitespace. Do not put secrets in the root `.env`, workspace YAML, -URLs, logs, or `docker compose config` output. +keys are `THT_MODEL_API_KEY`, `THT_DWH_API_KEY`, `THT_CA`, `THT_SSL_CA`, +`THT_OIDC_CLIENT_SECRET`, and `THT_AUTHENTIK_API_TOKEN`. The two authentication keys are fixed +empty entries for local authentication and must be populated only in a protected OIDC installation. +Other configured values must be non-empty and contain no whitespace. Do not put secrets +in the root `.env`, workspace YAML, URLs, logs, or rendered Compose output. + +Do not add vector or embedding endpoint credentials to the bundle. Active operator manuals use +internal Qdrant and Ollama services, so vector/embedding runtime endpoint secrets are not part of +the supported installation contract. Compose mounts the bundle read-only as `/run/secrets/thothii.secrets`. The host file must be a regular non-symlink file with mode `0600` or `0400`; Docker's normal `0444` mode is accepted @@ -20,7 +25,9 @@ only for the runtime mount beneath `/run/secrets`. The core runs as UID 10001. V without printing its contents: ```sh -docker compose run --rm core sh -c 'id && test -r /run/secrets/thothii.secrets' +docker compose --env-file deploy/env/local.env \ + -f compose.yaml -f deploy/compose.local.yaml \ + run --rm core sh -c 'id && test -r /run/secrets/thothii.secrets' ``` A private CA PEM chain is not a bundle value: PEM whitespace is rejected by the strict parser. @@ -31,18 +38,14 @@ Compose files intentionally do not create this mount. ## Migration from separate secret files Older installations used `THT_*_SECRET_FILE` variables and one file per value. Migrate by -copying each value to its bundle key, validating with `docker compose config --quiet`, and only -then deleting the old files. The old variables remain a compatibility path for staged upgrades, -but the documented and tested default is `THT_SECRETS_FILE=deploy/secrets/thothii.secrets`. +copying each retained value to its bundle key, validating with the complete base+profile command, +and only then deleting the old files. The old variables remain a compatibility path for staged +upgrades, but the documented and tested default is an absolute `THT_SECRETS_FILE` path to the +protected bundle. -The local-vector bootstrap rotation helper still accepts an old/new password file as its -maintenance interface. Run it only with files protected by `0600`, then copy the resulting -password into `THT_VECTOR_BOOTSTRAP_PASSWORD` in the bundle before restarting -`vector-reconcile`/the application. The helper never prints password contents. - -Hosted Pi providers must use a single provider key. Compound providers (Bedrock, Azure OpenAI -Responses, Cloudflare Workers AI/Gateway) fail closed until a provider-specific credential -adapter is implemented. +Hosted Pi providers must use a single model key through `THT_MODEL_API_KEY`. Compound providers +(Bedrock, Azure OpenAI Responses, Cloudflare Workers AI/Gateway) fail closed until a +provider-specific credential adapter is implemented. ## User-owned session database secrets diff --git a/deploy/secrets/thothii.secrets.example b/deploy/secrets/thothii.secrets.example index c6598060..40c633db 100644 --- a/deploy/secrets/thothii.secrets.example +++ b/deploy/secrets/thothii.secrets.example @@ -5,16 +5,12 @@ # Hosted model provider (single-key providers only). # THT_MODEL_API_KEY=replace-me -# External DWH and vector adapters. +# External DWH adapter. # THT_DWH_API_KEY=replace-me -# THT_VEC_API_KEY=replace-me -# THT_VEC_WRITE_API_KEY=replace-me - -# Optional local-vector roles. -# THT_VECTOR_BOOTSTRAP_PASSWORD=replace-me -# THT_VECTOR_MIGRATOR_PASSWORD=replace-me -# THT_VECTOR_READER_PASSWORD=replace-me -# THT_VECTOR_WRITER_PASSWORD=replace-me # Optional CA material/path understood by the configured adapter. # THT_CA=/run/secrets/ca-chain.pem + +# OIDC/Authentik references. Keep these fixed keys empty until OIDC is configured. +THT_OIDC_CLIENT_SECRET= +THT_AUTHENTIK_API_TOKEN= diff --git a/deploy/sql/20-vector-roles.sql b/deploy/sql/20-vector-roles.sql deleted file mode 100644 index 9d4d5730..00000000 --- a/deploy/sql/20-vector-roles.sql +++ /dev/null @@ -1,25 +0,0 @@ --- ThothII — ruolo vector read+write (schema vectors). --- Stessa istanza del DWH (porta 5438). ThothII indicizza (write) + ricerca (read) direttamente. --- La separazione reader/writer resta rilevante solo per il path REST (non usato in Profile A). --- psql -h localhost -p 5438 -U postgres -d postgres -v PWD='' -f 20-vector-roles.sql -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'thoth_vector_rw') THEN - CREATE ROLE thoth_vector_rw LOGIN; - END IF; -END $$; --- :'PWD' va fuori dal DO (psql non interpola nelle stringhe dollar-quoted) -ALTER ROLE thoth_vector_rw PASSWORD :'PWD'; - -CREATE SCHEMA IF NOT EXISTS vectors; - --- L'estensione pgvector deve esistere (già presente nell'istanza di produzione). --- CREATE EXTENSION IF NOT EXISTS vector; - -GRANT USAGE, CREATE ON SCHEMA vectors TO thoth_vector_rw; -GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA vectors TO thoth_vector_rw; -GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA vectors TO thoth_vector_rw; -ALTER DEFAULT PRIVILEGES IN SCHEMA vectors - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO thoth_vector_rw; -ALTER DEFAULT PRIVILEGES IN SCHEMA vectors - GRANT USAGE, SELECT ON SEQUENCES TO thoth_vector_rw; diff --git a/deploy/thothii.env.example b/deploy/thothii.env.example deleted file mode 100644 index 2abcfeba..00000000 --- a/deploy/thothii.env.example +++ /dev/null @@ -1,22 +0,0 @@ -# ThothII core — env di runtime (compose env_file). -# Copiare in deploy/thothii.env e completare. NON committare thothii.env. - -# --- DWH (direct, ruolo read-only su schema datawarehouse) --- -THT_DB_HOST=host.docker.internal -THT_DB_PORT=5438 -THT_DB_NAME=postgres -THT_DB_USER=thoth_dwh_reader -THT_DB_PASSWORD=__CHANGE_ME__ - -# --- Vector (direct, ruolo read+write su schema vectors; stessa istanza del DWH) --- -THT_VEC_HOST=host.docker.internal -THT_VEC_PORT=5438 -THT_VEC_USER=thoth_vector_rw -THT_VEC_PASSWORD=__CHANGE_ME__ - -# --- Embeddings (Ollama sull'host, modello nomic-embed-text-v2-moe) --- -THT_OLLAMA_URL=http://host.docker.internal:11434 - -# --- Backend --- -AUTH_MODE=none # none | mock | oidc (in embedded l'auth è al bordo del portale) -MAX_PI_PROCESSES=4 diff --git a/deploy/vector/reconcile-roles.sh b/deploy/vector/reconcile-roles.sh deleted file mode 100755 index 74f14afe..00000000 --- a/deploy/vector/reconcile-roles.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -set -eu - -. /opt/thoth/secret-policy.sh - -bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets} -export PGPASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD) -migrator_password=$(read_bundle_secret "$bundle" THT_VECTOR_MIGRATOR_PASSWORD) -reader_password=$(read_bundle_secret "$bundle" THT_VECTOR_READER_PASSWORD) -writer_password=$(read_bundle_secret "$bundle" THT_VECTOR_WRITER_PASSWORD) - -psql --set=ON_ERROR_STOP=1 \ - --set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \ - --set=migrator_password="$migrator_password" \ - --set=reader_user="$THT_VECTOR_READER_USER" \ - --set=reader_password="$reader_password" \ - --set=writer_user="$THT_VECTOR_WRITER_USER" \ - --set=writer_password="$writer_password" <<'SQL' -SELECT 'CREATE ROLE vector_reader NOLOGIN' -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') \gexec -SELECT 'CREATE ROLE vector_writer NOLOGIN' -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') \gexec -ALTER ROLE vector_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; -ALTER ROLE vector_writer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; - -SELECT format('CREATE ROLE %I LOGIN', :'migrator_user') -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'migrator_user') \gexec -SELECT format('CREATE ROLE %I LOGIN', :'reader_user') -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'reader_user') \gexec -SELECT format('CREATE ROLE %I LOGIN', :'writer_user') -WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'writer_user') \gexec - -SELECT format( - 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', - :'migrator_user', :'migrator_password' -) \gexec -SELECT format( - 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', - :'reader_user', :'reader_password' -) \gexec -SELECT format( - 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', - :'writer_user', :'writer_password' -) \gexec - -SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec -SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec -SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec -SELECT format('CREATE SCHEMA IF NOT EXISTS vectors AUTHORIZATION %I', :'migrator_user') \gexec -SELECT format('ALTER SCHEMA vectors OWNER TO %I', :'migrator_user') \gexec -REVOKE ALL ON SCHEMA vectors FROM PUBLIC; -GRANT USAGE ON SCHEMA vectors TO vector_reader, vector_writer; -CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors; -SQL diff --git a/deploy/vector/rotate-bootstrap-password.py b/deploy/vector/rotate-bootstrap-password.py deleted file mode 100755 index 042c49d7..00000000 --- a/deploy/vector/rotate-bootstrap-password.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -"""Rotate the initialized PostgreSQL bootstrap role and verify before returning success.""" - -from __future__ import annotations - -import os -import sys -from pathlib import Path - -import psycopg2 -from psycopg2 import sql - - -def read_secret(path: str) -> str: - value = Path(path).read_text() - if not value or "\x00" in value or any(character.isspace() for character in value): - raise ValueError("secret must be non-empty and contain no whitespace or NUL bytes") - return value - - -def connect(password: str): - return psycopg2.connect( - host=os.environ.get("THT_VECTOR_HOST", "vector-db"), - port=int(os.environ.get("THT_VECTOR_PORT", "5432")), - dbname=os.environ.get("THT_VECTOR_DATABASE", "thoth"), - user=os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres"), - password=password, - connect_timeout=5, - ) - - -def alter_current_role(connection, password: str) -> None: - with connection.cursor() as cursor: - cursor.execute("SELECT current_user") - current_user = cursor.fetchone()[0] - expected = os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres") - if current_user != expected: - raise RuntimeError("authenticated role does not match THT_VECTOR_BOOTSTRAP_USER") - cursor.execute( - sql.SQL("ALTER ROLE {} PASSWORD {}").format( - sql.Identifier(current_user), sql.Literal(password) - ) - ) - connection.commit() - - -def main() -> int: - if len(sys.argv) != 3: - print("usage: rotate-bootstrap-password.py OLD_SECRET NEW_SECRET", file=sys.stderr) - return 2 - try: - old_password = read_secret(sys.argv[1]) - new_password = read_secret(sys.argv[2]) - if old_password == new_password: - raise ValueError("old and new bootstrap passwords must differ") - old_connection = connect(old_password) - except Exception as exc: - print(f"bootstrap rotation refused before change: {type(exc).__name__}", file=sys.stderr) - return 1 - - try: - alter_current_role(old_connection, new_password) - try: - verification = connect(new_password) - verification.close() - except Exception as verify_exc: - try: - alter_current_role(old_connection, old_password) - except Exception as restore_exc: - print( - "bootstrap rotation verification failed and password restore failed: " - f"{type(verify_exc).__name__}/{type(restore_exc).__name__}", - file=sys.stderr, - ) - return 3 - print( - f"bootstrap rotation verification failed; old password restored: " - f"{type(verify_exc).__name__}", - file=sys.stderr, - ) - return 1 - except Exception as exc: - print(f"bootstrap rotation failed: {type(exc).__name__}", file=sys.stderr) - return 1 - finally: - old_connection.close() - - print("bootstrap database password rotated and new login verified") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/deploy/vector/secret-policy.sh b/deploy/vector/secret-policy.sh deleted file mode 100755 index 01de8eab..00000000 --- a/deploy/vector/secret-policy.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/sh - -validate_secret_file() { - secret_path=$1 - secret_name=$2 - if [ -L "$secret_path" ] || [ ! -f "$secret_path" ] || [ ! -r "$secret_path" ] || [ ! -s "$secret_path" ]; then - echo "$secret_name must be a readable, non-empty regular file" >&2 - return 2 - fi - if LC_ALL=C grep -q '[[:space:]]' "$secret_path"; then - echo "$secret_name must contain no whitespace" >&2 - return 2 - fi - mode=$(stat -c '%a' "$secret_path" 2>/dev/null || stat -f '%Lp' "$secret_path" 2>/dev/null) || return 2 - case "$secret_path:$mode" in - /run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;; - *) echo "$secret_name must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;; - esac -} - -read_secret_file() { - validate_secret_file "$1" "$2" || return - cat "$1" -} - -# Validate the bundle without printing any value. Keep this parser aligned with -# the backend loader: comments/blank lines are allowed, while syntax, allowlist, -# duplicates, empty values, file size, and line size are fail-closed. -validate_bundle() { - bundle_path=$1 - if [ -L "$bundle_path" ] || [ ! -f "$bundle_path" ] || [ ! -r "$bundle_path" ] || [ ! -s "$bundle_path" ]; then - echo "secret bundle must be a readable, non-empty regular file" >&2 - return 2 - fi - mode=$(stat -c '%a' "$bundle_path" 2>/dev/null || stat -f '%Lp' "$bundle_path" 2>/dev/null) || return 2 - case "$bundle_path:$mode" in - /run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;; - *) echo "secret bundle must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;; - esac - size=$(stat -c '%s' "$bundle_path" 2>/dev/null || stat -f '%z' "$bundle_path" 2>/dev/null) || return 2 - if [ "$size" -gt 65536 ]; then - echo "secret bundle exceeds the 64KiB limit" >&2 - return 2 - fi - awk ' - { sub(/\r$/, "", $0) } - length($0) > 16384 { exit 9 } - /^[[:space:]]*$/ || /^[[:space:]]*#/ { next } - /^[A-Z][A-Z0-9_]*=/ { - key=$0; sub(/=.*/, "", key) - val=$0; sub(/^[^=]*=/, "", val) - if (key !~ /^(THT_MODEL_API_KEY|THT_DWH_API_KEY|THT_VEC_API_KEY|THT_VEC_WRITE_API_KEY|THT_CA|THT_SSL_CA|THT_VECTOR_BOOTSTRAP_PASSWORD|THT_VECTOR_MIGRATOR_PASSWORD|THT_VECTOR_READER_PASSWORD|THT_VECTOR_WRITER_PASSWORD|PI_PROVIDER_API_KEY)$/) exit 6 - if (val == "" || ++seen[key] > 1) exit 7 - next - } - { exit 4 } - ' "$bundle_path" || { - echo "secret bundle syntax is invalid" >&2 - return 2 - } -} - -# Read one value from the deployment bundle without putting the bundle itself in -# a service environment. Values selected for credentials must contain no spaces. -read_bundle_secret() { - bundle_path=$1 - bundle_key=$2 - validate_bundle "$bundle_path" || return - case "$bundle_key" in - THT_[A-Z0-9_]*|PI_PROVIDER_API_KEY) ;; - *) echo "invalid secret bundle key" >&2; return 2 ;; - esac - value=$(awk -v wanted="$bundle_key" ' - { sub(/\r$/, "", $0) } - /^[[:space:]]*$/ || /^[[:space:]]*#/ { next } - /^[A-Z][A-Z0-9_]*=/ { - key=$0; sub(/=.*/, "", key) - val=$0; sub(/^[^=]*=/, "", val) - if (key == wanted) { - found=1; print val - } - next - } - { exit 4 } - END { if (!found) exit 5 } - ' "$bundle_path") || { - echo "$bundle_key is unavailable in secret bundle" >&2 - return 3 - } - if [ -z "$value" ] || printf '%s' "$value" | LC_ALL=C grep -q '[[:space:]]'; then - echo "$bundle_key must contain no whitespace" >&2 - return 2 - fi - printf '%s' "$value" -} diff --git a/deploy/vector/vector-db-entrypoint.sh b/deploy/vector/vector-db-entrypoint.sh deleted file mode 100755 index 75cd0e40..00000000 --- a/deploy/vector/vector-db-entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -eu - -. /opt/thoth/secret-policy.sh - -bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets} -export POSTGRES_PASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD) -unset THT_SECRETS_FILE -exec /usr/local/bin/docker-entrypoint.sh postgres diff --git a/deploy/workspace-registry.env.example b/deploy/workspace-registry.env.example new file mode 100644 index 00000000..835d63f3 --- /dev/null +++ b/deploy/workspace-registry.env.example @@ -0,0 +1,19 @@ +# Copy these non-secret workspace repository settings into the installation environment. +# Select at most one Git transport override. Every host path below must be absolute and normalized. +# Their contents are never committed, emitted by the API, or stored in the repository checkout. +THT_WORKSPACE_REGISTRY_ROOT=/data/workspace-registry +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_INSTALLATION_ID=local + +# Set the remote for this installation; use its own SSH/HTTPS address, never an application endpoint. +# THT_WORKSPACE_GIT_REMOTE=ssh://git@your-git-host/your-org/thoth-workspaces.git +# THT_WORKSPACE_GIT_CREDENTIALS_FILE=/absolute/path/to/git-credentials +# THT_WORKSPACE_GIT_CA_FILE=/absolute/path/to/git-ca.pem +# THT_WORKSPACE_GIT_SSH_KEY_FILE=/absolute/path/to/git-ssh-key +# THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/absolute/path/to/git-known-hosts + +# Workspace connector credentials are entered after installation in Workspace management. +# ThothII encrypts them in its workspace-secrets volume and never returns their values to the GUI. +# Git credentials remain installation-only and are selected with one transport override below. +# Run Compose through scripts/compose-with-preflight.sh so relative, non-normalized, and mixed +# SSH/HTTPS selections are rejected before Docker receives the invocation. diff --git a/deploy/workspaces/example.yaml b/deploy/workspaces/example.yaml index 10c1a186..c625015f 100644 --- a/deploy/workspaces/example.yaml +++ b/deploy/workspaces/example.yaml @@ -1,69 +1,50 @@ -language: en +workspace: + schema_version: 3 + id: example + name: Example workspace + description: Generic example WorkspaceV3 descriptor. + language: en dwh: - type: thoth_rest - database: - database: ${THT_DB_NAME} - schema: datawarehouse - endpoint: - base_url: ${THT_DWH_REST_URL} - api_key: ${THT_DWH_API_KEY} - ssl_ca: ${THT_SSL_CA} + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: + - postgres_direct + - rest_api -# Relative logical roots are resolved beneath /data/workspaces/example. -roots: - artifacts: artifacts - indexes: indexes - sessions: sessions - -examples: - max_per_column: 10 - -lsh: - signature_size: 64 - n_gram: 3 - threshold: 0.5 - max_values_per_column: 1000 - -eligibility: - max_declared_len: 128 - max_avg_length: 40 - max_sampled_len: 200 - ignore_columns: [etl_last_update] +semantic_index: + vector_store: + engine: qdrant + collection: example + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 evidence: - source_root: ${THT_DOCS_ROOT} - evidence_dir: evidence + source: + type: filesystem + uri: example/evidence + patterns: + - "**/*.md" + max_bytes: 10485760 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 +llm_policy: + default: zai/glm-5.2 + allowed: + - zai/glm-5.2 -vectors: - type: thoth_vector_http - reader: - base_url: ${THT_VEC_REST_URL} - api_key: ${THT_VEC_API_KEY} - ssl_ca: ${THT_SSL_CA} - writer: - base_url: ${THT_VEC_REST_URL} - api_key: ${THT_VEC_WRITE_API_KEY} - ssl_ca: ${THT_SSL_CA} - -vector: - max_chunk_chars: 4000 - -search: - rrf_k: 60 - top_schema_tables: 12 - schema_chunk_pool: 150 - -execution: - allow: [cte_test, explain, preview, aggregate, export] - max_preview_rows: 10 - max_export_rows: 100000 - statement_timeout_ms: 30000 - warn_execution_ms: 5000 - max_aggregate_cells: 20 +diagnostics: + dwh_rest: + method: GET + path: /health + auth: none + response: + database: database + schema: schema diff --git a/deploy/workspaces/local-vector.yaml b/deploy/workspaces/local-vector.yaml deleted file mode 100644 index b8a46556..00000000 --- a/deploy/workspaces/local-vector.yaml +++ /dev/null @@ -1,48 +0,0 @@ -language: en - -dwh: - type: thoth_rest - database: - database: ${THT_DB_NAME} - schema: datawarehouse - endpoint: - base_url: ${THT_DWH_REST_URL} - api_key: ${THT_DWH_API_KEY} - -vectors: - type: pgvector_direct - reader: - host: vector-db - port: 5432 - database: ${THT_VECTOR_DATABASE} - schema: vectors - user: ${THT_VECTOR_READER_USER} - password_file: ${THT_VECTOR_READER_PASSWORD_FILE} - writer: - host: vector-db - port: 5432 - database: ${THT_VECTOR_DATABASE} - schema: vectors - user: ${THT_VECTOR_WRITER_USER} - password_file: ${THT_VECTOR_WRITER_PASSWORD_FILE} - -roots: - artifacts: artifacts - indexes: indexes - sessions: sessions - -evidence: - source_root: ${THT_DOCS_ROOT} - evidence_dir: evidence - -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 - -execution: - allow: [cte_test, explain, preview, aggregate, export] - max_preview_rows: 10 - max_export_rows: 100000 - statement_timeout_ms: 30000 diff --git a/deploy/workspaces/preprocess-dwh.yaml b/deploy/workspaces/preprocess-dwh.yaml index 59079e7c..d2f3edff 100644 --- a/deploy/workspaces/preprocess-dwh.yaml +++ b/deploy/workspaces/preprocess-dwh.yaml @@ -2,6 +2,10 @@ language: en dwh: type: postgres_direct connection: - {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader, - password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"} + host: "${THT_PREPROCESS_DWH_HOST:-dwh}" + port: "${THT_PREPROCESS_DWH_PORT:-5432}" + database: "${THT_PREPROCESS_DWH_DATABASE:-warehouse}" + schema: "${THT_PREPROCESS_DWH_SCHEMA:-public}" + user: "${THT_PREPROCESS_DWH_USER:-thoth_reader}" + password_file: "${THT_PREPROCESS_DWH_PASSWORD_FILE:-/run/secrets/preprocess-dwh-password}" roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} diff --git a/deploy/workspaces/preprocess-evidence.yaml b/deploy/workspaces/preprocess-evidence.yaml index 638f547d..444b4844 100644 --- a/deploy/workspaces/preprocess-evidence.yaml +++ b/deploy/workspaces/preprocess-evidence.yaml @@ -1,15 +1,22 @@ language: en dwh: type: postgres_direct - connection: {host: unused, database: unused, schema: public, user: unused, password: unused} + connection: + host: "${THT_PREPROCESS_DWH_HOST:-unused}" + port: "${THT_PREPROCESS_DWH_PORT:-5432}" + database: "${THT_PREPROCESS_DWH_DATABASE:-unused}" + schema: "${THT_PREPROCESS_DWH_SCHEMA:-public}" + user: "${THT_PREPROCESS_DWH_USER:-unused}" + password_file: "${THT_PREPROCESS_DWH_PASSWORD_FILE:-/run/secrets/preprocess-dwh-password}" vectors: - type: pgvector_direct - reader: - {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_reader, - password_file: "${THT_VECTOR_READER_PASSWORD_FILE}"} - writer: - {host: vector-db, database: thoth, schema: vectors, user: thoth_vector_writer, - password_file: "${THT_VECTOR_WRITER_PASSWORD_FILE}"} + type: qdrant + base_url: http://qdrant:6333 + collection: preprocess-evidence roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} evidence: {source_root: /data/source, evidence_dir: evidence} -embeddings: {base_url: "${THT_OLLAMA_URL}", model: smoke, dim: 768, batch_size: 32} +embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: qwen3-embedding:0.6b + dim: 1024 + batch_size: 32 diff --git a/deploy/workspaces/psd.yaml.example b/deploy/workspaces/psd.yaml.example index 4b9e52f3..39eea0d0 100644 --- a/deploy/workspaces/psd.yaml.example +++ b/deploy/workspaces/psd.yaml.example @@ -1,43 +1,50 @@ -language: it +workspace: + schema_version: 3 + id: example-workspace + name: Example Workspace + description: Example WorkspaceV3 descriptor. + language: en dwh: - type: thoth_rest - database: - database: ${THT_DB_NAME} - schema: datawarehouse - endpoint: - base_url: ${THT_DWH_REST_URL} - api_key: ${THT_DWH_API_KEY} - ssl_ca: ${THT_SSL_CA} + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: + - postgres_direct + - rest_api -vectors: - type: thoth_vector_http - reader: - base_url: ${THT_VEC_REST_URL} - api_key: ${THT_VEC_API_KEY} - ssl_ca: ${THT_SSL_CA} - writer: - base_url: ${THT_VEC_WRITE_REST_URL} - api_key: ${THT_VEC_WRITE_API_KEY} - ssl_ca: ${THT_SSL_CA} - -roots: - artifacts: /data/workspaces/psd/runtime-v2/artifacts - indexes: /data/workspaces/psd/runtime-v2/indexes - sessions: /data/workspaces/psd/sessions +semantic_index: + vector_store: + engine: qdrant + collection: example-workspace + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 evidence: - source_root: ${THT_DOCS_ROOT} - evidence_dir: evidence + source: + type: filesystem + uri: example-workspace/evidence + patterns: + - "**/*.md" + max_bytes: 10485760 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 +llm_policy: + default: zai/glm-5.2 + allowed: + - zai/glm-5.2 -execution: - allow: [cte_test, explain, preview, aggregate, export] - max_preview_rows: 10 - max_export_rows: 100000 - statement_timeout_ms: 30000 +diagnostics: + dwh_rest: + method: GET + path: /health + auth: none + response: + database: database + schema: schema diff --git a/deploy/workspaces/server-sessions.yaml.example b/deploy/workspaces/server-sessions.yaml.example index 69402679..616c11fa 100644 --- a/deploy/workspaces/server-sessions.yaml.example +++ b/deploy/workspaces/server-sessions.yaml.example @@ -29,8 +29,13 @@ roots: indexes: indexes sessions: sessions -embeddings: - base_url: ${THT_OLLAMA_URL} - model: nomic-embed-text-v2-moe - dim: 768 - batch_size: 32 +resources: + vector: + engine: qdrant + base_url: http://qdrant:6333 + collection: server-sessions + embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: qwen3-embedding:0.6b + dimensions: 1024 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5a51d0f5..4cc28918 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,7 +1,7 @@ -# ThothII — deploy STANDALONE locale (dev / smoke test, senza portale). -# Rete propria + porte host per ispezione diretta. -# docker compose -f docker-compose.dev.yml up -d --build -# frontend: http://localhost:8090 backend: http://localhost:8787 +# ThothII standalone development/smoke stack. +# Run with the canonical local env file: +# docker compose --env-file deploy/env/local.env -f docker-compose.dev.yml up -d --build +# frontend: http://localhost:8090 backend: http://localhost:8787 name: thothii-dev services: @@ -10,27 +10,63 @@ services: context: . dockerfile: docker/core.Dockerfile image: thothii-core:local - env_file: [deploy/thothii.env] environment: HOST: 0.0.0.0 PORT: "8787" THT_HARNESS_DIR: /app/harness THT_BIN: /opt/venv/bin/tht PI_BIN: pi - AUTH_MODE: ${AUTH_MODE:-none} + AUTH_MODE: none + NODE_ENV: development THT_SESSION_STORAGE: local THT_HOME: /data/local-home + THT_DATA_ROOT: /data SETTINGS_FILE: /data/settings/settings.json + THT_MAINTENANCE_FILE: /data/settings/maintenance.json + THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry + THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE} + THT_WORKSPACE_GIT_BRANCH: ${THT_WORKSPACE_GIT_BRANCH:-main} + THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-local} + THT_WORKSPACE_SECRET_STORE_ROOT: /data/workspace-secrets + THT_WORKSPACE_SECRET_RUNTIME_ROOT: /tmp/thothii-workspace-secrets + THT_WORKSPACE_SECRET_ROOTS: /run/secrets + THT_SECRETS_FILE: /run/secrets/thothii.secrets + THT_DB_NAME: ${THT_DB_NAME:-} + THT_DWH_REST_URL: ${THT_DWH_REST_URL:-} + THT_LLM_URL: ${THT_LLM_URL:-} + THT_INTERNAL_QDRANT_URL: http://qdrant:6333 + THT_INTERNAL_EMBEDDING_URL: http://embedding:11434 + THT_INTERNAL_EMBEDDING_MODEL: qwen3-embedding:0.6b + THT_INTERNAL_EMBEDDING_DIMENSIONS: "1024" MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4} extra_hosts: - "host.docker.internal:host-gateway" volumes: - - /home/chirone/thothii-data:/data - - /home/chirone/thothii-data/pi-config:/home/thoth/.pi - - /home/chirone/chirone/etl/docs/evidence:/data/evidence:ro + - dev-data:/data + - dev-pi-state:/home/thoth/.pi + - ${PI_AUTH_FILE:?set PI_AUTH_FILE}:/home/thoth/.pi/agent/auth.json:ro + - ./deploy/pi/models.json:/home/thoth/.pi/agent/models.json:ro + - ./deploy/pi/settings.json:/home/thoth/.pi/agent/settings.json:ro + - workspace-registry:/data/workspace-registry + - workspace-secrets:/data/workspace-secrets + - ${THT_DEV_EVIDENCE_HOST_PATH:-./evidence}:/data/evidence:ro + secrets: + - source: thothii_secrets + target: thothii.secrets ports: - "127.0.0.1:8787:8787" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8787/health"] + interval: 15s + timeout: 3s + retries: 5 + start_period: 30s restart: "no" + depends_on: + qdrant: + condition: service_healthy + embedding-model-init: + condition: service_completed_successfully networks: [thothii-net] frontend: @@ -44,10 +80,77 @@ services: ports: - "127.0.0.1:8090:8080" depends_on: - - core + core: + condition: service_healthy restart: "no" networks: [thothii-net] + qdrant: + image: qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c + expose: + - "6333" + volumes: + - qdrant-data:/qdrant/storage + healthcheck: + test: + - CMD-SHELL + - > + /usr/bin/bash -lc "exec 3<>/dev/tcp/127.0.0.1/6333 && + printf 'GET /healthz HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' >&3 && + grep -q '200 OK' <&3" + interval: 15s + timeout: 3s + retries: 10 + start_period: 10s + networks: [thothii-net] + + embedding: + image: ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a + command: ["serve"] + expose: + - "11434" + volumes: + - embedding-models:/root/.ollama + healthcheck: + test: + - CMD-SHELL + - > + /usr/bin/bash -lc "exec 3<>/dev/tcp/127.0.0.1/11434 && + printf 'GET /api/tags HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n' >&3 && + grep -q '200 OK' <&3" + interval: 15s + timeout: 5s + retries: 20 + start_period: 10s + networks: [thothii-net] + + embedding-model-init: + image: ollama/ollama:0.32.0@sha256:57f573b47f1f71ebb445789f279fe3e596a8beab182f7cf486db9205bad87c5a + entrypoint: ["/usr/bin/bash", "/opt/thoth/embedding-model-init.sh"] + environment: + OLLAMA_BASE_URL: http://embedding:11434 + OLLAMA_MODEL: qwen3-embedding:0.6b + OLLAMA_WAIT_TIMEOUT_SEC: "180" + volumes: + - embedding-models:/root/.ollama + - ./docker/embedding-model-init.sh:/opt/thoth/embedding-model-init.sh:ro + depends_on: + embedding: + condition: service_healthy + networks: [thothii-net] + networks: thothii-net: driver: bridge + +volumes: + dev-data: + dev-pi-state: + workspace-registry: + workspace-secrets: + qdrant-data: + embedding-models: + +secrets: + thothii_secrets: + file: "${THT_SECRETS_FILE:?set THT_SECRETS_FILE}" diff --git a/docker/core-entrypoint.sh b/docker/core-entrypoint.sh index 6f9049cc..df2280e1 100755 --- a/docker/core-entrypoint.sh +++ b/docker/core-entrypoint.sh @@ -13,6 +13,10 @@ node /app/docker/ensure-pi-trust.mjs "${THT_HARNESS_DIR:-/app/harness}" cmd="${1:-server}" case "$cmd" in server) + [[ "${THT_AUTH_STATE_ROOT:-/data/auth}" == /data/auth ]] \ + || { printf '%s\n' 'authentication state root is invalid' >&2; exit 1; } + printf '%s\n' '{"version":1,"operation":"ensure-layout","root":"/data/auth"}' \ + | "${THT_AUTH_STORAGE_BIN:-/usr/local/bin/tht-auth-storage}" _auth-storage >/dev/null exec node /app/backend/dist/server.js ;; check) diff --git a/docker/core.Dockerfile b/docker/core.Dockerfile index 7eb8be45..410eeff4 100644 --- a/docker/core.Dockerfile +++ b/docker/core.Dockerfile @@ -1,35 +1,77 @@ # syntax=docker/dockerfile:1.7 -# thothii-core: Fastify (Node 22) + harness Python 3.12 (tht CLI) + runtime Pi. +# thothii-core: Fastify (Node 24.16) + harness Python 3.12 (tht CLI) + runtime Pi. # Singolo container, entrypoint logico "server" (default). ARG PI_VERSION=0.80.3 +ARG IMAGE_VERSION=local + +# ---- Pinned Node source for the runtime binary and npm ---- +FROM node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS node-runtime + +# ---- Pinned native storage helper used by the authenticated backend ---- +FROM golang:1.26.5-bookworm@sha256:1ecb7edf62a0408027bd5729dfd6b1b8766e578e8df93995b225dfd0944eb651 AS tht-auth-storage-build +WORKDIR /src/tools/tht +COPY tools/tht/go.mod tools/tht/go.sum ./ +RUN go mod download +COPY tools/tht ./ +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/tht-auth-storage ./cmd/tht + +# ---- Stage 0: locked Pi runtime ---- +FROM node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS pi-runtime-build +ARG PI_VERSION +ARG PI_RUNTIME_PACKAGE_VERSION +ARG PI_PACKAGE_NAME=@earendil-works/pi-coding-agent +WORKDIR /opt/pi-runtime +COPY docker/pi-runtime/package.json docker/pi-runtime/package-lock.json ./ +RUN if [ -n "$PI_RUNTIME_PACKAGE_VERSION" ]; then \ + node -e 'const fs=require("node:fs"); const [name,version]=process.argv.slice(1); const manifest=JSON.parse(fs.readFileSync("package.json","utf8")); manifest.dependencies[name]=version; fs.writeFileSync("package.json",JSON.stringify(manifest,null,2)+"\\n");' "$PI_PACKAGE_NAME" "$PI_RUNTIME_PACKAGE_VERSION"; \ + npm install --package-lock-only --ignore-scripts --omit=dev "$PI_PACKAGE_NAME@$PI_RUNTIME_PACKAGE_VERSION"; \ + fi \ + && npm ci --omit=dev \ + && test "$(./node_modules/.bin/pi --version)" = "$PI_VERSION" # ---- Stage 1: backend TypeScript -> dist ---- -FROM node:22-bookworm AS backend-build +FROM node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS backend-build WORKDIR /src/backend COPY backend/package*.json ./ RUN npm ci COPY backend/ ./ RUN npm run build -# ---- Stage 2: runtime (Python 3.12 nativo + Node 22 copiato, stesso glibc bookworm) ---- -FROM python:3.12-slim-bookworm AS runtime +# ---- Stage 2: runtime (Python 3.12 nativo + Node 24.16 copiato, stesso glibc bookworm) ---- +FROM python:3.12-slim-bookworm@sha256:d50fb7611f86d04a3b0471b46d7557818d88983fc3136726336b2a4c657aa30b AS runtime ARG PI_VERSION +ARG IMAGE_VERSION +LABEL org.opencontainers.image.title="thothii-core" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.description="ThothII core with its embedded Pi runtime" \ + io.thothii.pi.version="${PI_VERSION}" # Runtime tools -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates ripgrep fd-find tini \ - && rm -rf /var/lib/apt/lists/* \ - && ln -s /usr/bin/fdfind /usr/local/bin/fd +RUN set -eux; \ + runtime_packages="curl ca-certificates ripgrep fd-find tini git openssh-client"; \ + if ! command -v flock >/dev/null 2>&1; then \ + runtime_packages="$runtime_packages util-linux"; \ + fi; \ + apt-get update; \ + apt-get install -y --no-install-recommends $runtime_packages; \ + rm -rf /var/lib/apt/lists/*; \ + command -v flock >/dev/null 2>&1; \ + ln -s /usr/bin/fdfind /usr/local/bin/fd -# Node 22 + npm copiati dall'immagine ufficiale (stesso Debian bookworm → binario compatibile) -COPY --from=node:22-bookworm /usr/local/bin/node /usr/local/bin/node -COPY --from=node:22-bookworm /usr/local/lib/node_modules /usr/local/lib/node_modules +# Node 24.16 + npm copiati dall'immagine ufficiale (stesso Debian bookworm → binario compatibile) +COPY --from=node-runtime /usr/local/bin/node /usr/local/bin/node +COPY --from=node-runtime /usr/local/lib/node_modules /usr/local/lib/node_modules RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ && ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx # Utente non-root RUN useradd --create-home --uid 10001 --shell /bin/bash thoth -RUN mkdir -p /home/thoth/.pi/agent && chown -R thoth:thoth /home/thoth/.pi +# Docker copies these owned directories into newly-created named volumes, allowing the non-root +# runtime user to create application settings, sessions, registry snapshots, state, and locks. +RUN mkdir -p /home/thoth/.pi/agent /data/settings /data/sessions /data/workspace-registry /data/workspace-secrets \ + /data/auth/sessions /data/auth/oidc \ + && chown -R thoth:thoth /home/thoth/.pi /data \ + && chmod 0700 /data/auth /data/auth/sessions /data/auth/oidc COPY harness/ /app/harness/ # Pi scrive lock/settings in .pi: ownership thoth per sopravvivere al rebuild @@ -44,31 +86,40 @@ RUN python -m venv /opt/venv \ && /opt/venv/bin/pip install --no-cache-dir --upgrade pip \ && (cd /app/harness && /opt/venv/bin/pip install --no-cache-dir .) \ && cp /app/harness/workflow.yaml /opt/venv/lib/python3.12/site-packages/workflow.yaml -# Default locale e alias PSD convergono sul file canonico. Il CLI onora anche -# THT_CONFIG, quindi cambiare CWD non cambia l'identita' dello workspace. +# Il workspace locale predefinito converge sul file canonico. Il CLI onora anche THT_CONFIG, +# quindi cambiare CWD non cambia l'identita' dello workspace. RUN mkdir -p /app/harness/config \ - && cp --remove-destination /app/harness/workspaces/local.yaml /app/harness/config/tht.yaml \ - && ln -sfn /app/harness/config/tht.yaml /app/harness/workspaces/psd.yaml + && cp --remove-destination /app/harness/workspaces/local.yaml /app/harness/config/tht.yaml # PiProcessManager (backend) prepende harnessDir/.venv/bin al PATH del child Pi → symlink al venv reale RUN ln -s /opt/venv /app/harness/.venv -# Backend: dist + node_modules (stesso Node major 22 + glibc bookworm → compatibili) +# Backend: dist + node_modules (stesso Node major 24 + glibc bookworm → compatibili) COPY --from=backend-build /src/backend/dist /app/backend/dist COPY --from=backend-build /src/backend/node_modules /app/backend/node_modules COPY backend/package*.json /app/backend/ -# Runtime Pi (pacchetto npm puro JS, dipendenze prebuilt). Installato come root, eseguibile da thoth. -RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION} +# Runtime Pi is installed only from the committed lockfile. The image exposes its immutable +# executable directly, so no host Pi installation or writable global npm directory is needed. +COPY --from=pi-runtime-build /opt/pi-runtime/node_modules /opt/pi-runtime/node_modules +COPY --from=tht-auth-storage-build /out/tht-auth-storage /usr/local/bin/tht-auth-storage +RUN ln -s /opt/pi-runtime/node_modules/.bin/pi /usr/local/bin/pi \ + && test "$(pi --version)" = "$PI_VERSION" \ + && test -x /usr/local/bin/tht-auth-storage ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH" \ + PI_VERSION="${PI_VERSION}" \ HOST=0.0.0.0 PORT=8787 \ THT_HARNESS_DIR=/app/harness \ THT_BIN=/opt/venv/bin/tht \ + THT_AUTH_STORAGE_BIN=/usr/local/bin/tht-auth-storage \ PI_BIN=pi \ HOME=/home/thoth -COPY docker/core-entrypoint.sh docker/session-migrate.sh docker/ensure-pi-trust.mjs /app/docker/ -RUN chmod +x /app/docker/core-entrypoint.sh /app/docker/session-migrate.sh +COPY scripts/verify-line-endings.sh /usr/local/bin/verify-line-endings +COPY docker/core-entrypoint.sh docker/workspace-maintenance-entrypoint.sh docker/session-migrate.sh docker/ensure-pi-trust.mjs docker/embedding-model-init.sh /app/docker/ +COPY docker/smoke/core-smoke.sh /app/docker/smoke/core-smoke.sh +RUN /usr/local/bin/verify-line-endings /app/docker \ + && chmod +x /app/docker/core-entrypoint.sh /app/docker/workspace-maintenance-entrypoint.sh /app/docker/session-migrate.sh /app/docker/embedding-model-init.sh /app/docker/smoke/core-smoke.sh WORKDIR /app/backend USER thoth diff --git a/docker/embedding-model-init.sh b/docker/embedding-model-init.sh new file mode 100755 index 00000000..312794f1 --- /dev/null +++ b/docker/embedding-model-init.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +ollama_base_url=${OLLAMA_BASE_URL:-http://embedding:11434} +ollama_model=${OLLAMA_MODEL:-qwen3-embedding:0.6b} +wait_timeout_sec=${OLLAMA_WAIT_TIMEOUT_SEC:-180} + +case "$wait_timeout_sec" in + ''|*[!0-9]*) + echo "OLLAMA_WAIT_TIMEOUT_SEC must be an integer number of seconds" >&2 + exit 1 + ;; +esac + +case "$ollama_base_url" in + http://*) + host_and_path=${ollama_base_url#http://} + ;; + *) + echo "OLLAMA_BASE_URL must use http://" >&2 + exit 1 + ;; +esac + +host_port=${host_and_path%%/*} +ollama_host=${host_port%%:*} +ollama_port=${host_port##*:} +if [[ "$host_port" == "$ollama_host" ]]; then + ollama_port=80 +fi + +deadline=$((SECONDS + wait_timeout_sec)) +export OLLAMA_HOST="$ollama_base_url" + +fetch_tags() { + local response body + response=$( + exec 3<>"/dev/tcp/$ollama_host/$ollama_port" + printf 'GET /api/tags HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n' "$ollama_host" >&3 + cat <&3 + ) || return 1 + [[ "$response" == *$' 200 '* || "$response" == HTTP/1.1$' 200'* || "$response" == HTTP/1.0$' 200'* ]] || return 1 + body=${response#*$'\r\n\r\n'} + if [[ "$body" == "$response" ]]; then + body=${response#*$'\n\n'} + fi + printf '%s' "$body" +} + +model_present() { + local compact_json + compact_json=$(printf '%s' "$1" | tr -d '[:space:]') + grep -Fq "\"name\":\"$ollama_model\"" <<<"$compact_json" +} + +wait_for_tags() { + local tags_json + while (( SECONDS <= deadline )); do + if tags_json=$(fetch_tags 2>/dev/null); then + printf '%s' "$tags_json" + return 0 + fi + sleep 1 + done + echo "timed out waiting for Ollama tags at $ollama_base_url/api/tags" >&2 + return 1 +} + +tags_json=$(wait_for_tags) +if model_present "$tags_json"; then + echo "embedding model already cached: $ollama_model" + exit 0 +fi + +ollama pull "$ollama_model" +tags_json=$(fetch_tags) +model_present "$tags_json" || { + echo "embedding model missing after pull: $ollama_model" >&2 + exit 1 +} +echo "embedding model ready: $ollama_model" diff --git a/docker/frontend-entrypoint.sh b/docker/frontend-entrypoint.sh index 15554e4a..f42b0f7b 100644 --- a/docker/frontend-entrypoint.sh +++ b/docker/frontend-entrypoint.sh @@ -1,15 +1,19 @@ #!/bin/sh set -eu -backend_base_url=${BACKEND_BASE_URL-/api} -if ! /usr/local/bin/validate-backend-url "$backend_base_url"; then - echo "Invalid BACKEND_BASE_URL: use empty/root, /api, or a valid http(s) base without credentials, query, or fragment" >&2 +THT_FRONTEND_API_UPSTREAM=${THT_FRONTEND_API_UPSTREAM:-http://core:8787} +if ! /usr/local/bin/validate-frontend-api-upstream "$THT_FRONTEND_API_UPSTREAM"; then + echo "Invalid THT_FRONTEND_API_UPSTREAM: expected internal http://core:8787" >&2 + exit 2 +fi +export THT_FRONTEND_API_UPSTREAM + +if ! envsubst '${THT_FRONTEND_API_UPSTREAM}' \ + < /etc/nginx/templates/default.conf.template \ + > /etc/nginx/conf.d/default.conf; then + echo "Unable to render nginx API upstream configuration" >&2 exit 2 fi -runtime_config=$(jq -cn --arg backend_base_url "$backend_base_url" \ - '{backendBaseUrl: $backend_base_url}') -printf 'window.__THOTHII_CONFIG__ = %s;\n' "$runtime_config" \ - > /usr/share/nginx/html/config.js if [ "$#" -gt 0 ]; then exec "$@" diff --git a/docker/frontend.Dockerfile b/docker/frontend.Dockerfile index f3a09b71..f1324f2d 100644 --- a/docker/frontend.Dockerfile +++ b/docker/frontend.Dockerfile @@ -1,21 +1,24 @@ # syntax=docker/dockerfile:1.7 # thothii-frontend: build Vite (React) + nginx-unprivileged (porta 8080). -# Build args: -# VITE_BASE prefisso asset ("/" standalone, "/datamart-builder/assets/" embedded) -# VITE_BACKEND_URL base API ("http://localhost:8787" standalone, "/datamart-builder/api" embedded) -FROM node:22-bookworm AS build +ARG IMAGE_VERSION=local +FROM node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS build WORKDIR /src COPY frontend/package*.json ./ RUN npm ci COPY frontend/ ./ -ARG VITE_BASE=/ -ARG VITE_BACKEND_URL=http://localhost:8787 -ENV VITE_BASE=$VITE_BASE VITE_BACKEND_URL=$VITE_BACKEND_URL +ENV VITE_BASE=/ VITE_BACKEND_URL=/api RUN npm run build -# typecheck opzionale (non bloccante nella build dell'immagine) -RUN npx tsc -b 2>/dev/null || true -FROM nginxinc/nginx-unprivileged:1.27-alpine AS runtime +FROM nginxinc/nginx-unprivileged:1.27-alpine@sha256:65e3e85dbaed8ba248841d9d58a899b6197106c23cb0ff1a132b7bfe0547e4c0 AS runtime +ARG IMAGE_VERSION +LABEL org.opencontainers.image.title="thothii-frontend" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.description="ThothII standalone frontend" COPY --from=build /src/dist /usr/share/nginx/html -COPY docker/nginx.conf /etc/nginx/conf.d/default.conf +COPY docker/nginx.conf.template /etc/nginx/templates/default.conf.template +COPY --chmod=755 docker/frontend-entrypoint.sh /usr/local/bin/frontend-entrypoint +COPY --chmod=755 docker/validate-frontend-api-upstream.sh /usr/local/bin/validate-frontend-api-upstream +COPY --chmod=755 docker/smoke/frontend-smoke.sh /usr/local/bin/frontend-config-smoke +ENTRYPOINT ["/usr/local/bin/frontend-entrypoint"] +CMD ["nginx", "-g", "daemon off;"] EXPOSE 8080 diff --git a/docker/nginx.conf b/docker/nginx.conf index dcb83245..deb349d2 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -1,6 +1,4 @@ -# nginx per thothii-frontend: serve la SPA (modalità standalone) e reverse-proxy /api -> core. -# In modalità embedded il portale proxya /datamart-builder/assets/ qui (solo asset statici); -# il blocco /api non è usato in embedded (il portale hita core direttamente). +# nginx per thothii-frontend: serve la SPA e inoltra /api al core sulla rete Compose. server { listen 8080; server_name _; @@ -29,7 +27,7 @@ server { chunked_transfer_encoding on; } - # manifest.json servito (lo legge il template tag Django in embedded) + # manifest.json è servito come JSON. location = /manifest.json { default_type application/json; } diff --git a/docker/nginx.conf.template b/docker/nginx.conf.template index 83226fe6..291b5c6c 100644 --- a/docker/nginx.conf.template +++ b/docker/nginx.conf.template @@ -3,31 +3,35 @@ server { server_name _; root /usr/share/nginx/html; - location = /config.js { - add_header Cache-Control "no-store"; - try_files $uri =404; - } - location = /health { - proxy_pass http://core:8787/health; + proxy_pass ${THT_FRONTEND_API_UPSTREAM}/health; proxy_http_version 1.1; - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_cache off; } location /api/ { - proxy_pass http://core:8787/; + # The trailing slash replaces the matched /api/ prefix before the private hop. + proxy_pass ${THT_FRONTEND_API_UPSTREAM}/; proxy_http_version 1.1; - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - # Trusted only when AUTH_MODE=upstream and this frontend port is reachable solely - # from the authenticated host proxy documented in deploy/. - proxy_set_header X-Authenticated-User $http_x_authenticated_user; + # Public normalized claims are discarded by mapping only the private-hop values from the + # authenticated host proxy. Private-hop headers are then cleared before reaching core. + proxy_set_header X-Authenticated-User ""; + proxy_set_header X-Thoth-Principal-Issuer $http_x_thoth_trusted_principal_issuer; + proxy_set_header X-Thoth-Principal-Subject $http_x_thoth_trusted_principal_subject; + proxy_set_header X-Thoth-Principal-Display-Name $http_x_thoth_trusted_principal_display_name; + proxy_set_header X-Thoth-Is-Admin $http_x_thoth_trusted_is_admin; + proxy_set_header X-Thoth-Trusted-Principal-Issuer ""; + proxy_set_header X-Thoth-Trusted-Principal-Subject ""; + proxy_set_header X-Thoth-Trusted-Principal-Display-Name ""; + proxy_set_header X-Thoth-Trusted-Is-Admin ""; proxy_buffering off; proxy_cache off; - proxy_read_timeout 1h; + proxy_read_timeout 3600s; } location / { diff --git a/docker/pi-runtime/package-lock.json b/docker/pi-runtime/package-lock.json index 5143b147..1d3c33f8 100644 --- a/docker/pi-runtime/package-lock.json +++ b/docker/pi-runtime/package-lock.json @@ -9,6 +9,9 @@ "version": "1.0.0", "dependencies": { "@earendil-works/pi-coding-agent": "0.80.3" + }, + "engines": { + "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-coding-agent": { diff --git a/docker/pi-runtime/package.json b/docker/pi-runtime/package.json index 7bd4812e..dca0e017 100644 --- a/docker/pi-runtime/package.json +++ b/docker/pi-runtime/package.json @@ -3,6 +3,9 @@ "version": "1.0.0", "private": true, "description": "Locked Pi runtime dependency for the ThothII core image", + "engines": { + "node": ">=22.19.0" + }, "dependencies": { "@earendil-works/pi-coding-agent": "0.80.3" } diff --git a/docker/smoke/core-smoke.sh b/docker/smoke/core-smoke.sh index 77c968d7..82b8c745 100755 --- a/docker/smoke/core-smoke.sh +++ b/docker/smoke/core-smoke.sh @@ -1,13 +1,14 @@ #!/bin/sh set -eu -test "$(id -u)" != "0" +test "$(id -u)" = "10001" +test -n "${PI_VERSION:-}" node_version="$(node --version)" python_version="$(python --version 2>&1)" case "$node_version" in - v22.19.*|v22.2[0-9].*|v2[3-9].*|v[3-9][0-9].*) ;; - *) echo "Node 22.19+ required, found $node_version" >&2; exit 1 ;; + v24.16.*) ;; + *) echo "Node 24.16 required, found $node_version" >&2; exit 1 ;; esac case "$python_version" in "Python 3.1"[1-9].*|"Python 3."[2-9][0-9].*) ;; @@ -15,7 +16,10 @@ case "$python_version" in esac tht --help >/dev/null -pi --version >/dev/null +test "$(pi --version)" = "$PI_VERSION" +test ! -e /var/run/docker.sock +touch /data/.core-smoke-writable +rm /data/.core-smoke-writable /app/docker/core-entrypoint.sh server & server_pid=$! diff --git a/docker/smoke/frontend-policy-smoke.sh b/docker/smoke/frontend-policy-smoke.sh index 309009d4..2c3d5d1f 100755 --- a/docker/smoke/frontend-policy-smoke.sh +++ b/docker/smoke/frontend-policy-smoke.sh @@ -1,21 +1,66 @@ #!/bin/sh set -eu -corpus=/etc/thothii/backend-url-cases.json +cd "$(dirname "$0")/../.." -jq -c '.[]' "$corpus" | while IFS= read -r case_json; do - value=$(printf '%s' "$case_json" | jq -r '.value') - valid=$(printf '%s' "$case_json" | jq -r '.valid') - if BACKEND_BASE_URL="$value" /usr/local/bin/frontend-entrypoint true \ - >/dev/null 2>&1; then - actual=true - else - actual=false - fi - if [ "$actual" != "$valid" ]; then - echo "entrypoint policy mismatch for BACKEND_BASE_URL=$value: expected $valid" >&2 +nginx_config=docker/nginx.conf.template +for setting in \ + 'proxy_pass ${THT_FRONTEND_API_UPSTREAM}/;' \ + 'proxy_http_version 1.1;' \ + 'proxy_set_header Host $http_host;' \ + 'proxy_buffering off;' \ + 'proxy_read_timeout 3600s;'; do + if ! grep -Fq "$setting" "$nginx_config"; then + echo "missing required nginx API/SSE setting: $setting" >&2 exit 1 fi done -echo "frontend entrypoint canonical URL corpus: ok" +if ! grep -Fqx 'THT_FRONTEND_API_UPSTREAM=${THT_FRONTEND_API_UPSTREAM:-http://core:8787}' \ + docker/frontend-entrypoint.sh; then + echo "frontend entrypoint is missing the private core default" >&2 + exit 1 +fi + +if ! grep -Fq "envsubst '\${THT_FRONTEND_API_UPSTREAM}'" docker/frontend-entrypoint.sh; then + echo "frontend entrypoint does not render the private upstream" >&2 + exit 1 +fi + +if rg -n 'BACKEND_BASE_URL|VITE_BACKEND_URL' docker/frontend-entrypoint.sh docker/nginx.conf.template; then + echo "frontend runtime routing still accepts a browser-facing backend URL" >&2 + exit 1 +fi + +upstream_validator=docker/validate-frontend-api-upstream.sh +for upstream in http://core:8787; do + if ! "$upstream_validator" "$upstream"; then + echo "frontend upstream validator rejected $upstream" >&2 + exit 1 + fi +done + +if grep -Fq 'THT_FRONTEND_API_UPSTREAM=${THT_FRONTEND_API_UPSTREAM%/}' docker/frontend-entrypoint.sh; then + echo "frontend entrypoint must not normalize an upstream path component" >&2 + exit 1 +fi + +for upstream in \ + https://core:8787 \ + http://core:8080 \ + http://core:8787/ \ + http://core:8787// \ + http://core:8787/// \ + http://core:8787/api \ + http://user:pass@core:8787 \ + 'http://core:8787?next=evil' \ + 'http://core:8787#fragment' \ + 'http://core:8787 injected' \ + 'http://core:8787;proxy_pass http://evil'; do + if "$upstream_validator" "$upstream" >/dev/null 2>&1; then + echo "frontend upstream validator accepted unsafe upstream: $upstream" >&2 + exit 1 + fi +done + +echo "frontend same-origin proxy policy: ok" diff --git a/docker/smoke/frontend-smoke.sh b/docker/smoke/frontend-smoke.sh index 0a159f9b..8b84d1db 100644 --- a/docker/smoke/frontend-smoke.sh +++ b/docker/smoke/frontend-smoke.sh @@ -1,14 +1,6 @@ #!/bin/sh set -eu -assignment=$(sed \ - -e 's/^window\.__THOTHII_CONFIG__ = //' \ - -e 's/;$//' \ - /usr/share/nginx/html/config.js) - -printf '%s\n' "$assignment" \ - | jq -e --arg expected "${BACKEND_BASE_URL-/api}" \ - 'type == "object" and keys == ["backendBaseUrl"] and .backendBaseUrl == $expected' \ - >/dev/null +test "$(cat /usr/share/nginx/html/config.js)" = 'window.__THOTHII_CONFIG__ = {};' printf '%s\n' "frontend runtime config smoke: ok" diff --git a/docker/tht.Dockerfile b/docker/tht.Dockerfile new file mode 100644 index 00000000..6d6544c9 --- /dev/null +++ b/docker/tht.Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.26.5-bookworm@sha256:1ecb7edf62a0408027bd5729dfd6b1b8766e578e8df93995b225dfd0944eb651 AS build + +WORKDIR /src/tools/tht +COPY tools/tht/go.mod tools/tht/go.sum ./ +RUN go mod download +COPY tools/tht ./ + +RUN mkdir -p /out \ + && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /out/tht-windows-amd64.exe ./cmd/tht \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /out/tht-darwin-amd64 ./cmd/tht \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags='-s -w' -o /out/tht-darwin-arm64 ./cmd/tht \ + && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o /out/tht-linux-amd64 ./cmd/tht \ + && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -trimpath -ldflags='-s -w' -o /out/tht-linux-arm64 ./cmd/tht + +FROM scratch AS export +COPY --from=build /out/ / diff --git a/docker/validate-backend-url.sh b/docker/validate-backend-url.sh index 374e581b..ed139578 100755 --- a/docker/validate-backend-url.sh +++ b/docker/validate-backend-url.sh @@ -4,27 +4,5 @@ set -eu value=${1-} policy_file=${BACKEND_URL_POLICY_FILE:-/etc/thothii/backend-url-policy.json} -if jq -e --arg value "$value" '.relativeBases | index($value) != null' \ - "$policy_file" >/dev/null; then - exit 0 -fi - -if ! jq -e --arg value "$value" \ - '.absolutePattern as $pattern | $value | test($pattern)' \ - "$policy_file" >/dev/null; then - exit 2 -fi - -authority=${value#*://} -authority=${authority%%/*} -port="" -case "$authority" in - *]:*) port=${authority##*:} ;; - *]) ;; - *:*) port=${authority##*:} ;; -esac - -if [ -n "$port" ]; then - max_port=$(jq -r '.maxPort' "$policy_file") - if [ "${#port}" -gt 5 ] || [ "$port" -gt "$max_port" ]; then exit 2; fi -fi +jq -e --arg value "$value" '.relativeBases | index($value) != null' \ + "$policy_file" >/dev/null diff --git a/docker/validate-frontend-api-upstream.sh b/docker/validate-frontend-api-upstream.sh new file mode 100755 index 00000000..4e2243c5 --- /dev/null +++ b/docker/validate-frontend-api-upstream.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +[ "${1-}" = "http://core:8787" ] diff --git a/docker/workspace-maintenance-entrypoint.sh b/docker/workspace-maintenance-entrypoint.sh new file mode 100644 index 00000000..aecbbcf6 --- /dev/null +++ b/docker/workspace-maintenance-entrypoint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec node /app/backend/dist/workspace-maintenance.js "$@" diff --git a/docs/architecture/authentication.md b/docs/architecture/authentication.md new file mode 100644 index 00000000..9e3b3e5a --- /dev/null +++ b/docs/architecture/authentication.md @@ -0,0 +1,101 @@ +# Authentication architecture + +ThothII has two production authentication modes: `local` and generic `oidc`. The host operator +surface is one CLI, `tht`; there is no separate authentication executable. The backend owns +opaque browser sessions and authorization, while `tht` owns protected configuration and local-user +files. + +## Configuration and trust boundaries + +The installation descriptor points to an operator-controlled authentication directory. It contains +non-secret `auth.yaml` and, for local mode, `users.yaml`. POSIX installations use a private +directory and owner-only regular files; Windows uses equivalent owner-only ACLs. Secret values are +read from the mounted secret bundle and are never placed in YAML, command arguments, logs, JSON +diagnostics, or browser storage. + +Local users have Argon2id password hashes, stable IDs, enabled state, roles, and `authRevision`. +The production role expansion from `backend/src/auth/config.ts` is exact: + +| Role | Permissions | +|---|---| +| `user` | `session.use` | +| `admin` | `session.use`, `session.read_all`, `session.manage_all`, `settings.manage`, `workspace.manage`, `workspace.secrets.manage`, `pi.manage`, `auth.diagnostics.read` | + +`admin` therefore includes the ordinary `session.use` permission. No other role or permission +label is part of the production catalog. + +OIDC is provider-neutral at the browser protocol boundary. Authorization Code + PKCE, issuer, +signature, audience, expiry, state, and nonce are validated before a principal is created. +Authentik is the first certified group-catalog adapter, not a special browser login mode. + +## Group authorization + +OIDC must return a direct, non-empty `groups` claim whose value is a JSON array of strings. +Missing, malformed, indirect, or overage-style claims fail closed. The browser callback returns +HTTP 401 with the generic code `oidc_callback_failed`; it does not expose the internal reason. +Authentication diagnostics and interactive device-flow validation use +`oidc_groups_claim_invalid` for invalid group-claim/identity results. Exact configured external +group names map to Thoth roles and then to permissions. A user with no mapped group is +authenticated but receives no role and gets `403` from protected routes. Unmapped upstream groups +are ignored silently, without an error or warning. + +Every configured mapping is checked by the configured group-catalog adapter. Authentik checks the +exact group name and reports `oidc_mapped_group_missing` when it cannot find it. The dedicated +Authentik API service account has group-view-only privilege; it is separate from the OIDC client. + +## Diagnostics and ordering + +The closed production diagnostic-code union is: + +```text +auth_ready +auth_config_incomplete +auth_config_invalid +auth_session_store_invalid +local_user_registry_invalid +local_admin_missing +oidc_secret_missing +oidc_discovery_unreachable +oidc_issuer_mismatch +oidc_jwks_unreachable +oidc_group_catalog_unreachable +oidc_group_catalog_unauthorized +oidc_mapped_group_missing +oidc_mapped_group_ambiguous +oidc_groups_claim_invalid +oidc_device_flow_unavailable +``` + +Workspace Validate performs static authentication validation without provider connectivity. +`tht auth check` performs live, non-interactive diagnosis: static safety plus OIDC discovery, +issuer/JWKS, group-catalog authentication, and exact configured-group existence. Adding +`--interactive` runs that same live diagnosis and then validates a device-flow identity when the +provider supports Device Authorization. Workspace Test is the aggregate live workspace and +authentication validation. + +The ordered `tht doctor` report is exactly: `descriptor`, `files`, `docker`, `compose`, +`configuration`, `authentication`, `services`, `core-http`, `frontend-http`, +`workspace-registry`, `workflow`, `pi`. Its `authentication` entry is the live non-interactive +diagnosis against the healthy running core; later checks may be skipped when an earlier +prerequisite fails. + +## Browser sessions + +The browser receives only an opaque `HttpOnly`, `SameSite=Lax` cookie named `thothii_session`. +State-changing cookie requests require the in-memory CSRF token, same-origin `Origin`, and Fetch +Metadata checks when present. The frontend never stores bearer tokens or session secrets in Web +Storage. + +- Ordinary local sessions: 2-hour idle and 12-hour absolute expiry; closing the browser removes + the non-persistent cookie. +- **Remember me** local sessions: 7-day idle and 30-day absolute expiry; they survive browser and + ThothII restarts. +- OIDC sessions: at most 8 hours and never beyond the validated ID-token expiry. + +Local password, role, enabled-state, and logout-all changes increment `authRevision` and invalidate +affected sessions. Authentication configuration revision changes invalidate all sessions after +reload. Logout deletes the server record. Backup restore excludes active sessions and OIDC state, +recreates empty private auth-state directories, and therefore forces reauthentication. + +See the [local guide](../install/authentication-local.md), [generic OIDC guide](../install/authentication-oidc.md), +and [Authentik guide](../install/authentik.md) for operator procedures. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 695f82d1..3d8c9f6e 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -4,6 +4,9 @@ ThothII è un **datamart builder human-in-the-loop**: trasforma una domanda in linguaggio naturale in SQL validato (ed eventualmente un datamart dbt) attraverso un **workflow deterministico a 8 fasi NL→SQL**, in cui il modello *propone* e un revisore umano *decide* ai gate. +L'autenticazione di produzione usa local oppure OIDC generico; il solo CLI operatore è tht. +Per sessioni, ruoli, gruppi, diagnostica e ripristino vedere la [documentazione autenticazione](authentication.md). + ## I tre progetti indipendenti ``` @@ -54,6 +57,8 @@ Il frontend renderizza questi widget-descriptor (registro in `src/widgets/`); il ## Come si lancia lo stack -Lo **stack completo** (Pi reale + DWH reale, serve VPN + `harness/.env` + `pi` sul PATH) si avvia con `./scripts/run-stack.sh` (frontend `:5173` → backend `:8787`). +Lo stack locale si avvia con `./scripts/run-stack.sh`, dopo aver creato +`deploy/env/local.env` da `deploy/env/local.env.example`. Il core Compose include Pi; DWH, +vector DB, embedding e LLM sono endpoint esterni configurati nel file locale. Comandi per singolo layer, test, lint: vedi il file `CLAUDE.md` nella radice del repo (guida operativa per Claude Code, tenuta sincronizzata con questa pagina). diff --git a/docs/contracts/tht-dwh.md b/docs/contracts/tht-dwh.md new file mode 100644 index 00000000..fea794a3 --- /dev/null +++ b/docs/contracts/tht-dwh.md @@ -0,0 +1,128 @@ +# `.tht-dwh` — DWH generations, `OWNER.json`, ACTIVE, and fingerprints + +> Operator contract. P3 makes the effective DWH/preprocessing configuration reproducible and +> versioned across the operator CLI and the application sessions, and documents what `.tht-dwh` +> is so operators can reason about why a rerun is instant or why it takes minutes. + +## What `.tht-dwh` is + +`.tht-dwh` is the workspace-local directory that stores the **prepared snapshots of the data +warehouse structure** (the catalog `physical.yaml` plus the LSH hashes used for fuzzy search). +ThothII does not re-read the whole database for every question: it prepares it once, stores the +result here, and reuses it. The directory lives under the workspace runtime root, for example: + +```text +/data/sessions//.tht-dwh/ +``` + +## Immutable generations + +Each preparation run produces a **generation**: an immutable directory containing the catalog and +the LSH artifacts for one exact "effective configuration" (see fingerprints below). Generations +are never modified in place; a new run writes a new generation, and an `ACTIVE` pointer selects +which generation the workspace currently uses. Keeping the old generations makes rollback and +diagnosis safe. + +## `OWNER.json` + +Every generation root contains an `OWNER.json` that records who owns it: + +```json +{ + "workspace_id": "", + "config_fingerprint": "sha256:<64 hex>", + "input_fingerprint": "sha256:<64 hex>" +} +``` + +- `config_fingerprint` is the digest of the **canonical effective configuration** (see below). +- `input_fingerprint` is the digest of the **logical configuration identity**. + +Before reusing a generation, the harness compares the current canonical identity with the one in +`OWNER.json`. If they differ, the generation is **refused** (never silently reused) and a new one +is produced. This is what protects ThothII from using artifacts prepared for a different database, +endpoint, user, schema, or index contract. + +The reader is compatible with the historical schema-v1 `OWNER.json` (same three keys, `sha256:` +values) so existing installations keep working; new writes use the versioned computation. There is +no automatic in-place reinterpretation: operators regenerate explicitly when a root is old. + +## The canonical effective configuration and the logical identity + +The **canonical effective configuration** is the non-secret subset of the rendered runtime +configuration that determines whether a prepared DWH generation is still valid: + +```json +{ + "schemaVersion": 1, + "dwh": { + "engine": "postgres", + "database": "", + "schema": "", + "transport": "postgres_direct | rest_api | ...", + "host": "", + "port": 5432, + "baseUrl": "", + "user": "" + }, + "vector": { "collection": "", "dimensions": 1024, "distance": "cosine" }, + "embedding": { "model": "", "dimensions": 1024 }, + "roots": { "artifacts": "", "indexes": "" } +} +``` + +Deliberately **excluded** (their change must not invalidate a DWH generation): + +- `session_storage` and `runtime_identity` (a content-only Git commit or an Evidence-only change + must not force a full database re-introspection); +- Evidence source/policy (P6 materialization and Evidence preprocessing are separate); +- memory, search, and execution settings; +- **all credentials** (passwords, API keys, signed URLs, and secret-file paths). + +The **logical configuration identity** is: + +```text +workspace://@v1: +``` + +It is the same for the operator CLI and for application sessions, because both derive it from the +same rendered configuration. That is the guarantee that the work prepared by `tht` is exactly +what the sessions will consume. + +## Why a rerun can be instant or take minutes + +- Same canonical identity (e.g., only Evidence files changed) → the generation is reused → the + DWH step is `unchanged` and fast. +- Changed canonical identity (different database, address, user, schema, collection, model, or + artifact/index roots) → the old generation is refused → ThothII re-introspects and writes a new + generation → the step takes as long as the first preparation. + +## Safe migration, regeneration, and recovery + +- **Migration**: existing schema-v1 `OWNER.json` roots are readable; to switch them to the + versioned identity, run a normal regeneration (explicit `--refresh`/new run). No automatic + in-place rewrite. +- **Regeneration**: a new run produces a new immutable generation and moves `ACTIVE`; the previous + generations remain for rollback. +- **Recovery**: if the active generation is corrupt or owned by another configuration, ThothII + fails closed (never mixes artifacts) and tells the operator to regenerate; the old generations + are still available for inspection. + +## Memory root + +P3 also gives each workspace an explicit **workspace-global memory root**: + +```text +/data/sessions//memory/ +``` + +All memory commands, locks, the canonical JSONL registry, and the Qdrant projection use this root +when present. A guarded migration copies and verifies exactly one legacy canonical JSONL from the +old `artifacts/memory` location under the workspace lock and rebuilds the projection; conflicting +legacy registries fail closed. There is no in-place reinterpretation. + +## Revision-scoped search records + +Schema and Evidence records in the Qdrant collection include the pinned `workspace_revision`, so +searches never mix descriptions or documents from different versions of the workspace. Memory and +solved-question records remain workspace-wide on purpose. diff --git a/docs/contracts/tht-pi.md b/docs/contracts/tht-pi.md new file mode 100644 index 00000000..bc577cdd --- /dev/null +++ b/docs/contracts/tht-pi.md @@ -0,0 +1,245 @@ +# `tht pi` lifecycle contract + +`tht` is the only component that drives Docker lifecycle operations. The `core` container +does not mount a Docker socket, and Pi is never updated in a running container. + +## Inspection and configuration + +```text +tht pi status +tht pi doctor +tht pi test +tht pi logs +tht pi configure +``` + +When `--installation` is omitted, `tht` first uses `THOTHII_INSTALLATION` and otherwise +discovers one valid `thothii-installation.yaml` in the current project tree, including an immediate +`deploy/*` directory. Use `--installation /absolute/path/thothii-installation.yaml` as an explicit +override when the descriptor is outside that tree or more than one installation is available. + +`status` executes the image-bundled `pi --version`. `doctor` compares that value with both the +container's `PI_VERSION` contract and the `io.thothii.pi.version` image label; a merely nonempty +version is not sufficient. `doctor` and `test` also require a healthy core, a successful Pi smoke, +valid settings, and an exact selected provider/model pair from the backend's available model +entries. `pi check` remains an alias for `pi test`. Logs are always a bounded, sanitized 200-line +snapshot; there is no follow mode. + +On a TTY, `pi configure` presents numbered provider, model, and thinking choices. Providers and +models come from the backend's closed model list, and the model choices are restricted to the +selected provider. In non-interactive use, all choices must be explicit: + +```text +tht pi configure \ + --provider zai --model glm-5.2 --thinking medium +``` + +The helper snapshots exact settings-file existence and raw bytes, applies the new values atomically, +and verifies the readback and rendered-configuration digest. A helper, readback, or digest failure +restores those exact bytes when the prior file existed; on a clean installation it removes the new +file and verifies the absent/default state. Empty prior files are supported. The command reports +the actual host path from `PI_AUTH_FILE`; credentials remain in that protected host file and must +never be passed as flags. + +Installation-managed Pi provider/model configuration is declarative only. Any JSON value beginning +with `!` is rejected recursively in the complete `models.json` before it can supply management +choices, and the exact selected provider/model and credential payload is checked again before the +isolated smoke files are written. The API returns only the fixed +`Pi provider/model configuration is invalid` message; rejected commands, paths, and secrets are +never included. Use `$NAME`/`${NAME}` environment references in `models.json`, or omit `apiKey` and +provide the selected credential through the protected `PI_AUTH_FILE`, `THT_MODEL_API_KEY_FILE`, or +`THT_SECRETS_FILE` contract. A literal leading exclamation mark uses Pi's `$!` escape. Direct +secret-file references are not a `models.json` feature: ThothII converts its managed key source to +the provider-native child environment, while `PI_AUTH_FILE` is mounted as Pi's protected credential +store. + +## Supported Compose entry points and current image + +Use `tht start`, `stop`, `status`, `logs`, and `doctor` for ordinary installation lifecycle +operations. All `tht` Compose commands automatically include the installation-specific +durable selector when it exists: + +```text +/.tht//current-image.yaml +``` + +This selector is part of the supported installation state: it keeps a verified Pi image selected +across a fresh `tht` process, stop/start, reconcile, and source checkout whose base image is +digest-pinned. Do not delete or hand-edit it. Direct raw `docker compose` lifecycle commands bypass +this protection and are unsupported. Advanced documented Compose rendering must use +`scripts/compose-with-preflight.sh` and include the same selector with `-f` when present; connector +secret overrides must never bypass that preflight wrapper. + +## Reloading Pi configuration + +Configuration reload is a separate lifecycle operation from an image update: + +```text +tht pi restart --yes [--drain] +``` + +`--yes` is required after reviewing the planned core recreation. Restart activates the durable +maintenance gate before it checks sessions. Without `--drain`, active open sessions refuse the +command. With `--drain`, the command polls the authenticated session inventory until no active +sessions remain; it never terminates sessions and the wait is bounded. + +Restart retains the exact current image and never builds, pulls, or upgrades an image. Before any +core mutation, it tags the captured running image ID with a transaction-scoped reference and +selects that reference through a lifecycle-only Compose override. A configured mutable tag moving +after capture therefore cannot change the restarted image. It recreates only `core` with +`--no-deps --force-recreate --no-build --pull never`; `frontend` and named volumes are not +recreated. Before reopening admission, it verifies health, the unchanged Pi version, the +provider/model/settings smoke, unchanged non-secret rendered configuration, the captured image +identity, and the complete persistence-mount fingerprint. + +Restart and update keep separate recovery state: + +```text +/.tht//restart-state.json +/.tht//update-state.json +``` + +The files are mode `0600` and share one installation lifecycle lock, so restart, update, and +rollback cannot race. Every mutating lifecycle command checks both files. Malformed or non-terminal +restart recovery state blocks update and rollback; malformed or incomplete update recovery state +blocks restart. A verified terminal restart state is cleaned up safely before a later mutation. +After core mutation, a restart failure leaves admission gated and preserves both +`restart-state.json` and its exact-image override; the operator must use status/logs and maintenance +recovery rather than deleting recovery material. + +## Updating Pi + +The normal update uses the repository's pinned version and build source automatically: + +```text +tht pi update +tht pi update --version 0.81.0 +``` + +With no `--version`, the command reads the single default `ARG PI_VERSION=` from +`docker/core.Dockerfile` in the selected project. The normal path confirms the explicit update +command, drains active sessions without terminating them, builds the candidate, recreates only +`core`, verifies it, and promotes it transactionally. + +Advanced registry updates remain available and require an immutable digest: + +```text +tht pi update \ + --version 0.81.0 --source build --yes --drain + +tht pi update \ + --version 0.81.0 --source pull \ + --image registry.example.invalid/thothii-core@sha256:<64-lowercase-hex-digits> --yes +``` + +`--source build` rebuilds only `core` with `PI_VERSION=`. `--source pull` requires an +immutable digest reference; mutable tags, URL forms, and credential-bearing references are +rejected. `--source` is never inferred. + +Before inventory, update activates the durable maintenance gate. Activation writes +`/data/settings/maintenance.json` in the mounted settings volume, closes admission, and waits for +all leases. A recreated candidate reads that marker at startup and therefore starts gated. The +loopback-only control endpoints cannot be reached through the frontend proxy and do not depend on +the configured authentication principal mode. Lost activation/deactivation responses are resolved +by querying gate status only when the original result is unknown. An explicit file or directory +durability failure is never converted to success by matching readback: the control API reports +`maintenance_durability_failed`, keeps or restores the safest durable marker state, and requires +recovery. + +Open, unarchived sessions stop an update. After an operator has completed or otherwise drained +their work, `--drain` makes the command poll the authenticated bare-array +`GET /sessions?scope=all` response until no active sessions remain. + +The configured `core.image` is never retagged or mutated. Each installation transaction creates +unique candidate and previous tags, including when two installations share a configured tag or +the configured image is digest-pinned. A temporary lifecycle-only Compose override selects those +tags for build, recreate, and rollback. Candidate build, pull, or candidate-tag failures happen +before `mutation_started` and therefore never recreate or roll back core. After verification, the +temporary candidate selector is atomically promoted to the durable current-image override. +Rollback atomically promotes the previous selector. Terminal cleanup removes only transaction +files and never deletes the durable selector. + +Only `core` is recreated, with `--no-deps --force-recreate`; `frontend` is not recreated and no +volume-replacement flags are used. Verification checks health; exact requested Pi version at all +three declared boundaries (the candidate executable, `PI_VERSION` environment, and +`io.thothii.pi.version` image label); the provider/model/settings smoke; unchanged +non-secret rendered configuration; and the complete persistence-mount fingerprint. + +## Recovery, rollback, and maintenance cleanup + +Recovery state and lock diagnostics live under: + +```text +/.tht//update-state.json +/.tht//restart-state.json +/.tht//*.lock.owner.json +``` + +Each recovery file is mode `0600`. Update state records transaction-scoped image identities, mount +fingerprints, target version/source, configuration digest, phase, and timestamp; restart state +records the retained image and its verification inputs. Neither file contains credentials, endpoint +values, secret paths, Compose output, or logs. A cross-platform OS advisory file lock serializes +both lifecycle operations; a crashed owner releases the lock automatically. Owner metadata is +diagnostic only and cannot wedge acquisition if empty, partial, or stale. + +Any post-candidate failure explicitly confirms or reactivates maintenance and rescans sessions +before compensation. Automatic rollback selects the transaction's previous image through the +lifecycle override and clears maintenance only after the previous image, configuration, mounts, +health, Pi smoke, and terminal recovery write are verified. Ambiguous compensation remains gated. +If the candidate core is stopped and cannot serve the maintenance endpoint, rollback proves that +state with Compose and writes the marker through a one-off previous-image `core` container sharing +the settings volume. It does not require the failed candidate, a host Node runtime, or the Docker +socket inside a container. The restored core is then recreated, verified, and rescanned before the +gate can open. + +For a failed update with `update-state.json`, first run: + +```text +tht pi rollback --yes +``` + +Rollback restores the image recorded in update state, but it checks restart state before making any +change. A failed, pending, or malformed restart state rejects rollback. A failed restart retains its +captured image and has no candidate image to roll back; first inspect status and logs, repair the +reported problem, then use maintenance recovery. + +Inspect and clean a stale durable gate with: + +```text +tht pi maintenance status +tht pi maintenance recover --yes +``` + +`maintenance recover` restores the captured restart image pin and lifecycle override when needed, +verifies and removes interrupted restart recovery material, and only then processes update state. +It completes an interrupted verified-image promotion, safely finalizes a preparation interrupted +before core mutation, and refuses other pending mutations. For terminal or absent recovery state, +it removes only a stale transaction override, verifies the running installation when the gate is +active, and only then removes the durable marker and reopens admission. It never removes +`current-image.yaml`. If rollback or recovery fails, leave the marker in place, preserve the +relevant recovery state and override, repair the reported Docker/configuration issue, and rerun +rollback or maintenance recovery. + +Missing confirmation, invalid arguments, active sessions, and an interrupted transaction exit +`2`. Docker and verification failures exit nonzero with concise, redacted guidance. Direct +read-only/log commands preserve the original Docker child exit code. + +## Go dependency security boundary + +The supported toolchain is Go `1.26.5`, released 2026-07-07, with module language version +`1.26.0`. The Docker builder is pinned by both patch tag and the multi-platform manifest-list +digest: + +```text +golang:1.26.5-bookworm@sha256:1ecb7edf62a0408027bd5729dfd6b1b8766e578e8df93995b225dfd0944eb651 +``` + +That manifest provides both `linux/amd64` and `linux/arm64/v8` builders. Go's official release +history is the authority for the patch level (`https://go.dev/doc/devel/release`); the Docker +Official Image is the authority for the builder (`https://hub.docker.com/_/golang`). +`golang.org/x/sys`, used by the Windows durable-replace implementation, is pinned to `v0.47.0`. +The directly used `github.com/sirupsen/logrus` is pinned to `v1.9.1`, which removes +GO-2025-4188 from the imported package set. The build contract verifies the exact toolchain, +dependencies, digest, and all five supported target builds (Windows amd64, Darwin amd64/arm64, and +Linux amd64/arm64). `go mod verify`, tests including the race detector, `go vet`, and +`govulncheck` are release gates. diff --git a/docs/contracts/workspace-evidence-v3.md b/docs/contracts/workspace-evidence-v3.md new file mode 100644 index 00000000..2461b4f0 --- /dev/null +++ b/docs/contracts/workspace-evidence-v3.md @@ -0,0 +1,188 @@ +# Workspace Evidence v3 contract + +This is the canonical public contract for the optional `evidence` object in a schema-v3 +workspace descriptor. Evidence is optional: a valid v3 descriptor without it remains operational. +When present, `evidence` is strict: it contains `source` and a defaulted strict `policy`; every +source variant and the policy reject unknown keys. + +## Filesystem source + +A filesystem source uses the exact URI `/evidence`. `patterns` is a nonempty list of +unique, normalized relative POSIX globs. Its defaults are `patterns: ["**/*.md"]` and +`max_bytes: 10485760`. + +### Example: filesystem + +```yaml +evidence: + source: + type: filesystem + uri: example/evidence + patterns: + - "**/*.md" + max_bytes: 10485760 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +Safe: `example/evidence`. Unsafe filesystem identities include `/srv/evidence`, +`another/evidence` and `example/evidence/../another` because cross-namespace and traversal +paths are not canonical. Legacy split-layout paths are rejected as noncanonical. + +## HTTP source + +`uris` is a nonempty, unique list of canonical HTTP or HTTPS provenance identities. Each URI must +have no whitespace, control character, backslash, userinfo, query, or fragment. Public mode uses +`authentication: none`; signed mode uses `authentication: signed_urls_file`. Defaults are +`authentication: none`, `connect_timeout_ms: 5000`, `read_timeout_ms: 30000`, +`max_bytes: 10485760`, `max_redirects: 5`, `allow_private_hosts: false`, and +`max_cache_bytes: 67108864`. Public HTTP (`authentication: none`) uses the declared query-free +URIs directly and requires no Evidence credential file. Signed HTTP uses the installation file +contract below and preserves a mandatory one-to-one provenance mapping in declared-URI order. + +### Example: http + +```yaml +evidence: + source: + type: http + uris: + - https://evidence.example.invalid/report.md + authentication: signed_urls_file + connect_timeout_ms: 5000 + read_timeout_ms: 30000 + max_bytes: 10485760 + max_redirects: 5 + allow_private_hosts: false + max_cache_bytes: 67108864 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +The shown provenance URI is safe and query-free. An HTTP fragment identity such as +`https://evidence.example.invalid/report.md#section` is unsafe. Query-bearing and userinfo-bearing +HTTP identities are rejected; public documentation must not spell or publish a signed transport +URL. + +## S3 source + +`uri` is a canonical `s3://` identity with a valid bucket and no port, userinfo, query, or fragment. +`endpoint_url`, when present, is an origin-only HTTP(S) URL; `region`, when present, is nonblank. +Defaults are `credentials: ambient`, `trusted_endpoint: false`, `allow_private_endpoint: false`, +`allow_insecure_endpoint: false`, `max_bytes: 10485760`, `max_objects: 10000`, `max_pages: 100`, +and `page_size: 1000` (and page size cannot exceed 1000). A custom endpoint requires +`trusted_endpoint: true`; an HTTP endpoint additionally requires `allow_insecure_endpoint: true`. +Endpoint-policy flags cannot be enabled without `endpoint_url`. Ambient S3 +(`credentials: ambient`) uses the runtime provider chain and requires no Evidence credential file. +Static mode uses `credentials: static_files`, requires access-key and secret-key files together, +and permits an optional session-token file. + +### Example: s3 + +```yaml +evidence: + source: + type: s3 + uri: s3://example-evidence/curated/ + credentials: static_files + trusted_endpoint: false + allow_private_endpoint: false + allow_insecure_endpoint: false + max_bytes: 10485760 + max_objects: 10000 + max_pages: 100 + page_size: 1000 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +Safe: `s3://example-evidence/curated/`. Unsafe identities include +`s3://Invalid_Bucket/evidence` and `s3://example-evidence/evidence#section`. S3 userinfo and query +identities are rejected in prose and implementation; no credential-bearing example is published. + +## Policy and numeric domains + +The strict policy defaults to `max_chunk_chars: 4000` and +`retain_published_generations: 3`. Filesystem `max_bytes`; HTTP `connect_timeout_ms`, +`read_timeout_ms`, `max_bytes`, and `max_cache_bytes`; S3 `max_bytes`, `max_objects`, `max_pages`, +and `page_size`; and both policy values must be positive safe integers from 1 through +9007199254740991. HTTP `max_redirects` must be a nonnegative safe integer from 0 through +9007199254740991. S3 `page_size` has the stricter maximum of 1000. + +## Installation files + +The namespace is the workspace ID uppercased with every `-` changed to `_`. All Evidence variables +hold file paths, never credential or signed-URL values. + +| Mode | Variable | File contract | +| --- | --- | --- | +| Signed HTTP | `THT_WS__EVIDENCE_SIGNED_URLS_FILE` | Required for `signed_urls_file`; at most 1048576 bytes; nonempty UTF-8 JSON string array in declared-URI order; query-stripped identities must match `uris` one-to-one. | +| Static S3 pair | `THT_WS__EVIDENCE_ACCESS_KEY_FILE` and `THT_WS__EVIDENCE_SECRET_KEY_FILE` | Required together for `static_files`; each file is at most 65536 bytes. | +| Static S3 session | `THT_WS__EVIDENCE_SESSION_TOKEN_FILE` | Optional, valid only with the required access/secret pair, and at most 65536 bytes. | + +At the connector boundary every variable is an absolute path to a readable regular file whose +resolved target is strictly below one of the roots configured by `THT_WORKSPACE_SECRET_ROOTS`. +Users enter the corresponding values through Workspace management; the backend stores them as +authenticated ciphertext and materializes these files only for a runtime lease. Scalar S3 files +are nonempty UTF-8 tokens without whitespace or NUL. Public docs, APIs, and rendered YAML never +expose file contents. `changeme`, `replace-me`, `YOUR_SECRET`, ``, access-key-looking strings, and any +credential-bearing or query-bearing URI are forbidden as public placeholder values. + +## One shared workspace repository + +All workspace namespaces live in one Git repository: + +```text +workspace-repository.git/ +├── thoth-workspaces.yaml +├── example/ +│ ├── workspace.yaml +│ └── evidence/... +└── another/ + └── workspace.yaml +``` + +The curator-owned root catalog `thoth-workspaces.yaml` uses the `schema_version` value `1` and the ordered +`workspaces` list of `{id, name, description?}` entries. It is authoritative for workspace ID, +name, description, and display order. The descriptor at `/workspace.yaml` must match the +catalog metadata exactly. Every catalog entry must have its descriptor at that same commit; +catalog-only entries are invalid and reject the complete candidate revision. + +Workspace source changes only through curator Git commit/push in a separate authoring clone, +followed by an installation pull. The API never writes `thoth-workspaces.yaml`, +`/workspace.yaml`, `/schema/**`, or `/evidence/**`. + +## Registry revision and phase ownership + +| Relationship | Contract | +| --- | --- | +| Revision identity | The catalog blob, descriptor blob, and filesystem Evidence root tree are checked at the same 40-hex Git commit. | +| Content-only revision | An Evidence-only commit changes authoritative `revision.commit` even when the catalog and descriptor blobs are unchanged. | +| Repository consumer | ThothII fetches and validates a complete candidate, atomically activates it only on success, and never edits, commits, or pushes repository content. | +| Runtime secrets | Workspace management returns configured/missing status only; decrypted values exist only for the lifetime of a diagnostic or runtime lease. | +| P1.1 | Validates the lexical URI `/evidence` and proves the declared filesystem root object is a Git tree at that same commit; it does not recursively inspect nested symlinks. Evidence materialization stays out of scope for P1.1. | +| P6 | Owns commit-addressed materialization, realpath and recursive containment, nested-symlink checks, and race checks. | + +P1.1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, +active-snapshot retention, or GC. + +## Operator validation + +After the runtime configuration is rendered or acquired, validate it with the exact per-command +option ordering: + +```sh +tht config check -c +``` + +Stop after validation. P2/P6 later owns preprocessing and materialization. + +## Acceptance states + +These gates are independent and are not implied by this documentation contract. + +automated integration: PENDING +manual acceptance: PENDING diff --git a/docs/contracts/workspace-preprocessing-cli.md b/docs/contracts/workspace-preprocessing-cli.md new file mode 100644 index 00000000..25986e24 --- /dev/null +++ b/docs/contracts/workspace-preprocessing-cli.md @@ -0,0 +1,161 @@ +# Workspace preprocessing CLI contract + +`tht` is the only supported host entrypoint for workspace preprocessing. + +## Invocation + +```text +tht --installation /thothii-installation.yaml workspace inspect + --workspace [--json] + +tht --installation /thothii-installation.yaml workspace preprocess dwh + --workspace [--resume <32hex>] [--json] + +tht --installation /thothii-installation.yaml workspace schema suggest-fks + --workspace + [--from-sql ]... [--assume ]... + [--output ] [--json] + +tht --installation /thothii-installation.yaml workspace schema check + --workspace + [--annotations --reviewed-candidates ] + [--json] + +tht --installation /thothii-installation.yaml workspace schema accept + --workspace --run <32hex> --yes [--json] + +tht --installation /thothii-installation.yaml workspace index-schema + --workspace [--json] + +tht --installation /thothii-installation.yaml workspace preprocess evidence + --workspace [--dry-run] [--resume <32hex>] [--json] + +tht --installation /thothii-installation.yaml workspace preprocess run + --workspace [--resume <32hex>] [--json] + +tht --installation /thothii-installation.yaml workspace vector inspect + --workspace [--json] + +tht --installation /thothii-installation.yaml workspace vector rebuild + --workspace --collection --confirm --destroy [--json] +``` + +## Qdrant collection lifecycle (P4) + +- `workspace vector inspect` reports the descriptor-owned Qdrant collection contract + (name, dimensions, distance, keyword indexes) **without mutation**. +- `workspace vector rebuild` deletes and recreates the descriptor-owned collection + with the exact contract (1024 dimensions, cosine distance, the 8 required keyword + payload indexes) under guards: + - `--collection ` must equal the descriptor's `semantic_index.vector_store.collection`; + - `--confirm ` must equal `--collection` (exact repetition); + - `--destroy` is required to confirm the destructive operation; + - the operator refuses any other combination with exit code 2 (usage). +- Self-heal at session admission: a missing collection is created and missing + keyword indexes are added by the shared collection manager; incompatible + dimensions/distance/index types are never mutated (`semantic_index_incompatible`). +- The operator path (`workspace-maintenance.js vector-inspect|vector-rebuild`) + performs the guarded rebuild; rebuild state is written before deletion and the + collection is verified after recreation. No prefix matching or global Qdrant + mutation is performed. +``` + +## Curated FK annotations (P5) + +- The canonical curated annotations file is `/schema/annotations.yaml`, a regular + Git blob at the same commit as the descriptor. Absence is compatible (empty canonical set + + warning); symlinks, trees/gitlinks, oversized (>16 MiB), non-UTF-8, and malformed objects are + refused at activation. +- Activation synchronizes the blob to the immutable revision-qualified root + `/data/sessions//revisions//artifacts/mschema/annotations.yaml` with a restrictive + mode and an adjacent ownership manifest `{ workspace, commit, blobId, contentDigest, + destination }`. Pinned runtimes resolve annotations from `paths.annotations_root`. +- `workspace schema accept --run --yes` is the only human FK review primitive: after + commit/push/pull, it reads the current synced blob, validates it with the harness parser against + the physical schema and the recorded candidate digest, and records + `{ reviewedCandidatesDigest, annotationsDigest, workspaceRevision, blobId }`. `--yes` is + required; an empty file, an unknown run, a malformed blob, or a non-matching candidate fails + closed without recording a review. `schema check` alone is not evidence of human review. +- `preprocess run` continues only with the exact accepted blob digest and a compatible reusable + DWH binding; otherwise it records a new review checkpoint. + +## Commit-addressed Evidence materialization (P6) + +- Filesystem Evidence `/evidence` is materialized from the exact pinned Git commit + into the immutable revision content root `/snapshots///evidence` at + activation, with a sibling bounded manifest `/evidence.manifest.json` whose digest is chained + into `snapshot.json`. +- Materialization uses fixed Git plumbing (`ls-tree -r -z` + `cat-file blob`) and refuses symlinks + and gitlinks at any depth, traversal/absolute/duplicate/cross-namespace paths, and non-regular + modes. Installation-local limits bound entry count (default 4096), total bytes (64 MiB), + per-file bytes (8 MiB), path bytes (4096), and manifest bytes (1 MiB); a size-sum preflight runs + before any bytes are written and no partial root is published. +- `preprocess evidence` and `preprocess run` operate directly on the materialized root; the + temporary `evidence_materialization_required` stop is retired (the code remains only for + pre-P6 compatibility). HTTP/S3 Evidence is unchanged. +- Materialized roots are retained with their commit-addressed snapshot directory and removed only + when the revision becomes unreferenced. + +## Validation + +- `--installation` is mandatory and absolute. +- `--workspace` is mandatory exactly once and must match `[a-z][a-z0-9-]{2,62}`. +- `--resume` values must be 32 lowercase hex characters. +- `--json` may be supplied once. +- `schema suggest-fks` + - allows at most 32 `--from-sql` files; + - each SQL file must be a canonical regular file, UTF-8, non-symlink, max 1 MiB; + - total SQL ingress must not exceed 16 MiB; + - allows at most 256 `--assume` values, each `column=table`, max 256 bytes; + - `--output` must name a new canonical path; existing targets are refused. +- `schema check` + - `--annotations` and `--reviewed-candidates` are all-or-nothing; + - annotations must be UTF-8, canonical, non-symlink, max 16 MiB; + - `--reviewed-candidates` must match `sha256:<64 lowercase hex>`. +- `schema accept` + - `--run` is mandatory and must be 32 lowercase hex characters; + - `--yes` is mandatory and may be supplied once; + - `--annotations`/`--reviewed-candidates`/`--from-sql`/`--assume` are not accepted. +- Unknown flags, passthrough separators, and shell fragments are rejected before Docker runs. + +## Container boundary + +`tht` resolves the selected `core` image from the rendered installation, converts it to an immutable local image ID, writes a one-shot final override that pins both `core` and `workspace-maintenance` to that ID with `pull_policy: never`, and runs only: + +```text +docker compose run --rm --no-deps --no-TTY --name workspace-maintenance +``` + +The request is streamed as one schema-versioned JSON document over stdin. Public stdout is always one schema-versioned JSON result; human mode is rendered from an allowlisted subset of that same result. + +## Public JSON result + +```json +{ + "schemaVersion": 1, + "status": "succeeded|unchanged|dry_run|blocked|failed", + "code": "ok|workspace_not_found|workspace_not_activatable|binding_missing|preprocessing_conflict|preprocessing_resume_mismatch|manual_review_required|evidence_materialization_required|effective_config_mismatch|semantic_index_incompatible|annotation_invalid|egress_policy_refused", + "workspaceId": "abc", + "workspaceRevision": "1234567890abcdef1234567890abcdef12345678", + "descriptorBlob": "sha256:<64 lowercase hex>", + "operation": "inspect|preprocess-dwh|schema-suggest-fks|schema-check|index-schema|preprocess-evidence|preprocess-run", + "runId": "", + "childRuns": {"stage": ""}, + "completedStages": ["stage"], + "counts": {"name": 1}, + "artifactIdentities": [{"kind": "fk_candidates", "digest": "sha256:<64 lowercase hex>"}], + "warnings": ["safe warning"] +} +``` + +`tht --json` parses the operator stdout strictly and re-encodes only the public fields above. + +`evidence_materialization_required` is retained for pre-P6 compatibility; since P6, filesystem +Evidence is materialized at activation and preprocesses directly. + +## Exit codes + +- `0`: `succeeded`, `unchanged`, or `dry_run` +- `3`: `blocked` +- `2`: host-side grammar or local file safety failure +- `1`: operational failure or operator-reported `failed` diff --git a/docs/general/pi-configuration.md b/docs/general/pi-configuration.md index ecc2606b..86a6d912 100644 --- a/docs/general/pi-configuration.md +++ b/docs/general/pi-configuration.md @@ -2,11 +2,19 @@ Pi (il coding agent che orchestra il workflow NL→SQL) può risolvere un `provider/model` in tre modi diversi. Non sono alternativi: coesistono, e la scelta di quale usare dipende da **quanto è standard l'endpoint** e da **quanto deve essere ampia la visibilità** del modello (tutti i progetti vs. un progetto solo). +> **ThothII operator note:** ThothII runs Pi only in Docker Compose. Paths under +> `~/.pi/agent/` in this document describe Pi's container-side behavior. Operators edit +> `deploy/pi/models.json` and `deploy/pi/settings.json` in the ThothII project root and use +> the protected host credential file selected by `PI_AUTH_FILE`; they do not edit files inside +> the running container. + ## Credenziali nel backend container -In produzione configurare una sola sorgente generica, `THT_MODEL_API_KEY_FILE`, come secret file -assoluto e non il valore della chiave. `PiProcessManager` rilegge e valida il file per ogni processo, -normalizza il provider selezionato e passa al solo child Pi la variabile nativa appropriata +In produzione configurare una sola sorgente generica: il bundle `THT_SECRETS_FILE` con la voce +`THT_MODEL_API_KEY` (predefinito della distribuzione unificata), oppure +`THT_MODEL_API_KEY_FILE` come secret file assoluto. Non mettere il valore della chiave in `.env`. +`PiProcessManager` rilegge e valida la sorgente per ogni processo, normalizza il provider +selezionato e passa al solo child Pi la variabile nativa appropriata (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `ZAI_API_KEY`, ecc.). Il percorso generico, le chiavi di provider non selezionati e il vecchio `PI_PROVIDER_API_KEY` vengono rimossi dall'ambiente del child. Provider locali come `ollama`, `lmstudio` e `aritmolab` continuano senza chiave; un provider @@ -37,6 +45,22 @@ Pi viene distribuito con un elenco di modelli già noti (`models.generated.js` d Per un endpoint **OpenAI-compatible** che non è tra i built-in — ma che non richiede nessuna logica di trasporto speciale — basta *dichiararlo*: baseUrl, apiKey, lista modelli. Questo file esiste **solo a livello utente**: non c'è un equivalente project-level (un `./.pi/models.json` non viene letto). +Nelle installazioni gestite da ThothII il file deve essere interamente dichiarativo. ThothII +rifiuta ricorsivamente qualsiasi valore JSON che inizi con `!`, anche dentro `headers`, `models`, +`modelOverrides`, `compat`, array o campi non ancora conosciuti. Pi 0.80.3 tratterebbe quel prefisso +come un comando shell al momento della richiesta; questa forma non è ammessa né dall'elenco gestito +dei modelli né dallo smoke isolato. L'errore restituito è fisso e non include comando, percorso o +secret. + +Per i secret usare un riferimento ambiente come `"$ZAI_API_KEY"` o `"${ZAI_API_KEY}"`. Il backend +può popolare la variabile nativa del solo provider selezionato leggendo +`THT_MODEL_API_KEY_FILE`, oppure dal bundle `THT_SECRETS_FILE` (`THT_MODEL_API_KEY`); in alternativa +le credenziali possono arrivare dal file protetto montato con `PI_AUTH_FILE`, omettendo `apiKey` da +`models.json`. Non esiste una sintassi di riferimento diretto a un secret file dentro +`models.json`: con le sorgenti `THT_MODEL_*` il file viene letto da ThothII e trasformato nella +variabile ambiente del child Pi; `PI_AUTH_FILE` viene invece montato come archivio credenziali Pi +protetto. Per un punto esclamativo letterale iniziale, la sintassi Pi dichiarativa è `$!`, non `!`. + Esempio reale in uso su questa macchina — GLM (provider `zai`): ```json diff --git a/docs/gestione-memory.md b/docs/gestione-memory.md index b718ce0c..e2e07d87 100644 --- a/docs/gestione-memory.md +++ b/docs/gestione-memory.md @@ -16,7 +16,7 @@ decisione concept_clarified nel ledger della sessione F8: il reviewer decide se promuoverla │ ├── registro globale registry.jsonl - └── indice semantico pgvector + └── indice semantico Qdrant │ ▼ F2 di una sessione futura @@ -33,7 +33,7 @@ Implementazione principale: [harness/tht/memory.py](../harness/tht/memory.py:14) | --- | --- | --- | | Ledger della sessione | `concept_clarified`, `memory_promoted`, `memory_promotion_declined` | Audit e stato della singola sessione | | Registro globale | Record `mem-XXXX` in `registry.jsonl` | Archivio canonico attuale delle memory | -| Indice pgvector | Embedding e metadati derivati dal registro | Ricerca semantica | +| Indice Qdrant | Embedding e metadati derivati dal registro | Ricerca semantica | Il ledger contiene la provenienza e le decisioni umane. Il record globale contiene il testo riutilizzabile. L'indice vettoriale è una proiezione per la ricerca, non il posto in cui il workflow registra direttamente le decisioni. @@ -114,9 +114,9 @@ Il registro attuale è: La scrittura viene fatta tramite file temporaneo e `os.replace`, quindi la sostituzione del registro è atomica. L'idempotenza della promozione è basata sulla coppia `session_id + decision_seq`: la stessa decisione della stessa sessione non genera due record globali. -### pgvector +### Qdrant -Dopo la promozione, `save-one` costruisce un solo `VectorRecord` e lo invia all'indice pgvector. Il testo indicizzato include: +Dopo la promozione, `save-one` costruisce un solo `VectorRecord` e lo invia all'indice Qdrant. Il testo indicizzato include: - tipo e soggetto; - dettaglio; @@ -124,13 +124,13 @@ Dopo la promozione, `save-one` costruisce un solo `VectorRecord` e lo invia all' - domanda di contesto; - eventuali concetti e mapping. -Il record vettoriale usa l'id `memory:mem-XXXX`, mentre i metadati conservano `subject`, `detail`, `rationale`, `tables` e `concepts`. L'hash SHA-256 del contenuto impedisce di ricalcolare embedding e upsert quando il testo non è cambiato. +Il record vettoriale usa l'id `memory:mem-XXXX`, mentre i metadati conservano `subject`, `detail`, `rationale`, `tables`, `concepts` e il discriminante `kind`. L'hash SHA-256 del contenuto impedisce di ricalcolare embedding e upsert quando il testo non è cambiato. Il comportamento è implementato in [harness/tht/memory.py](../harness/tht/memory.py:253) e [harness/tht/memory.py](../harness/tht/memory.py:305). ### Fonte canonica attuale -Oggi il registro JSONL è ancora la fonte canonica applicativa e pgvector è l'indice derivato. Il commento iniziale di [memory_cmd.py](../harness/tht/cli/memory_cmd.py:1) segnala un debito tecnico: l'architettura futura prevista sarebbe usare direttamente il vector DB come archivio unico, ma questa migrazione non è ancora completata. +Oggi il registro JSONL è ancora la fonte canonica applicativa e Qdrant resta un indice derivato ma persistente. Il workflow non registra direttamente le decisioni nel vector DB: usa Qdrant come proiezione interrogabile del registro e del ledger effettivo. ## Riutilizzo in F2 @@ -209,10 +209,10 @@ Questo evita che una singola modifica al prompt o a un solo componente reintrodu Il salvataggio segue sostanzialmente questa sequenza: ```text -registro JSONL → pgvector → marker memory_promoted nel ledger +registro JSONL → Qdrant → marker memory_promoted nel ledger ``` -Se pgvector non è disponibile, il registro può contenere una memory non ancora ricercabile; il comando segnala che sarà necessario reindicizzare. +Se Qdrant non è disponibile, il registro può contenere una memory non ancora ricercabile; il comando segnala che sarà necessario reindicizzare. Se il marker del ledger fallisce dopo il salvataggio nel vector DB, la memory può risultare globalmente presente ma senza audit completo nella sessione. Il gate restituisce un comando di recupero manuale. @@ -232,4 +232,4 @@ Vecchi record `table_promoted` o `table_excluded` possono ancora esistere in art La gestione attuale è coerente con il requisito funzionale: una memory è una conoscenza concettuale riutilizzabile, non una scelta di schema-linking. -La parte più solida è la difesa multilivello del tipo `concept_clarified`. Il principale debito tecnico riguarda invece la convivenza del registro JSONL con pgvector e l'assenza di una transazione unica tra archivio globale, indice semantico e ledger della sessione. +La parte più solida è la difesa multilivello del tipo `concept_clarified`. Il principale debito tecnico riguarda invece la convivenza del registro JSONL con Qdrant e l'assenza di una transazione unica tra archivio globale, indice semantico e ledger della sessione. diff --git a/docs/guida-utente.md b/docs/guida-utente.md new file mode 100644 index 00000000..06e05a83 --- /dev/null +++ b/docs/guida-utente.md @@ -0,0 +1,285 @@ +# ThothII — Guida utente + +Per login, **Remember me**, ruoli, invalidazione delle sessioni, gruppi OIDC e ripristino, vedere +la [guida autenticazione locale](install/authentication-local.md) e la [guida OIDC generica](install/authentication-oidc.md). + +Questa guida accompagna passo-passo chi deve **preparare** il repository dei workspace, **usare +gli strumenti** ThothII per quel repository e **usare l'applicazione** per fare domande in +linguaggio naturale e ottenere SQL validato. Usa parole semplici ed esempi; i dettagli tecnici +restano nei contratti citati in fondo. + +> **Che cos'è ThothII.** È un *datamart builder* con revisione umana: tu scrivi una domanda in +> linguaggio naturale, un modello propone via via i passaggi (chiarimenti, schema, CTE, SQL) e un +> **revisore umano decide** a ogni passaggio chiave. Il risultato finale è SQL validato pronto da +> eseguire sul data warehouse. + +--- + +## Parte 1 — Preparare il repository dei workspace su Git + +### 1.1 La struttura + +Il repository dei workspace è un **repository Git** che descrive *quali dati* sono disponibili e +*come raggiungerli*. Non contiene i dati e **non contiene segreti** (password, token, certificati). + +Un repository valido contiene: + +```text +thoth-workspaces.yaml ← catalogo: elenco dei workspace +/workspace.yaml ← descrittore del workspace (schema v3) +/evidence/ ← (facoltativo) documenti di contesto, es. *.md +/schema/annotations.yaml ← (facoltativo) join logici curati a mano (P5) +``` + +- Il **catalogo** `thoth-workspaces.yaml` è un semplice elenco: + +```yaml +schema_version: 1 +workspaces: + - id: psd-clinical + name: Policlinico San Donato + description: DWH clinico del Policlinico San Donato +``` + +- L'**id** deve essere minuscolo, senza spazi, es. `psd-clinical` (`[a-z][a-z0-9-]{2,62}`). +- Il **descrittore** `/workspace.yaml` è lo schema v3. È l'unica descrizione valida. + +### 1.2 Esempio di descrittore (Policlinico San Donato) + +```yaml +workspace: + schema_version: 3 + id: psd-clinical + name: Policlinico San Donato + description: DWH clinico — aritmologia + language: it # le descrizioni/evidence sono in italiano + +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [rest_api] # accesso tramite API REST (PostgREST) + +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 + +llm_policy: + allowed: [zai/glm-5.2] + +diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: x-api-key + response: { database: database, schema: schema } + +evidence: + source: + type: filesystem + uri: psd-clinical/evidence # percorso dentro il repository + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +Cosa cambia rispetto ai vecchi workspace (se ne avevi uno): + +- il database si raggiunge solo con **REST** o **Postgres diretto** (`rest_api` / + `postgres_direct`); il tunnel SSH resta disabilitato; +- l'indice semantico è **interno** (Qdrant + `qwen3-embedding:0.6b`, 1024 dimensioni, cosine); +- l'Evidence **filesystem** sta dentro il repository (`/evidence`) e viene materializzata dal + commit Git fissato (P6); è supportata anche l'Evidence HTTP. + +### 1.3 Regole da rispettare + +1. **Git è la fonte di verità.** Descriptor, catalogo ed Evidence si modificano solo con un + *commit* + *push* e poi un *pull* dell'installazione. +2. **Niente segreti nel repository.** Password, token, chiavi private e URL firmati vengono inseriti + a runtime nella gestione Workspace e conservati cifrati dal backend. +3. **Solo schema v3.** I descrittori v1/v2 vengono rifiutati prima dell'attivazione. +4. **L'applicazione non fa push di contenuti curati.** L'operatore che cura il repository lavora in + un clone autore separato. + +--- + +## Parte 2 — Usare gli strumenti ThothII per il repository + +Ci sono **due** strumenti: l'**applicazione web** (gestione workspace) e la **CLI `tht`** +(preprocessing/operator). L'installazione completa è descritta nei manuali +`docs/install/local-workspace-registry.md` (macOS/Windows/Linux) e +`docs/install/server-workspace-registry.md`. + +### 2.1 `tht` — comandi principali + +`tht` si invoca sempre con `--installation /thothii-installation.yaml`. I comandi +utili, nell'ordine tipico: + +```bash +# 1) vedere lo stato di un workspace (revisione e identità) +tht --installation workspace inspect --workspace --json + +# 2) introspezione del DWH (genera physical.yaml + LSH) +tht --installation workspace preprocess dwh --workspace --json + +# 3) suggerire le join (FK) da SQL già approvato +tht --installation workspace schema suggest-fks --workspace --from-sql .sql --output .yaml --json + +# 4) dopo la revisione: pubblicare gli FK curati in Git e accettarli +tht --installation workspace schema accept --workspace --run --yes --json + +# 5) indicizzare lo schema (Qdrant) +tht --installation workspace index-schema --workspace --json + +# 6) preprocessing dell'Evidence +tht --installation workspace preprocess evidence --workspace --json + +# 7) catena completa (DWH → FK → schema → Evidence) +tht --installation workspace preprocess run --workspace --json + +# 8) ispezione/ricostruzione della collection Qdrant (solo manutenzione) +tht --installation workspace vector inspect --workspace --json +tht --installation workspace vector rebuild --workspace --collection --confirm --destroy +``` + +Note importanti: + +- **`--json` produce solo JSON su stdout** (contratto macchina): usalo negli script. +- **`preprocess run` si ferma per la revisione umana** quando ci sono nuove join proposte: esce con + `manual_review_required`. Dopo la revisione si riparte con `schema accept ... --yes` e + `preprocess run --resume `. +- **Un file Evidence filesystem viene materializzato dal commit Git fissato** (niente checkout + mobile); symlink, percorsi pericolosi e alberi troppo grandi vengono rifiutati. +- **Il CLI non scrive mai nel repository** (nessun push di contenuti curati). + +### 2.2 Applicazione web — gestione workspace + +La gestione Workspace ha due livelli distinti. + +**Livello 1 — repository.** La parte iniziale spiega che il sorgente del workspace vive in una +directory separata, viene pubblicato dal curatore su un repository ospitato da un server Git come +GitHub, GitLab o Gitea, e viene letto da ThothII in sola lettura. Mostra host, repository, branch, +revisione attiva e stato dell'ultimo aggiornamento. + +- **Update workspace repository** non richiede la selezione di un workspace. Il backend esegue il + fetch/pull del branch configurato direttamente nel checkout gestito da ThothII, valida l'intera + revisione candidata e la attiva in modo atomico. Se la validazione fallisce, conserva la + revisione precedente. Non modifica il sorgente remoto e non salva contenuti nella GUI. +- Per creare un workspace locale, prepara una directory sorgente con catalogo, `workspace.yaml` e + le sottodirectory previste; quindi validala, esegui commit e push dal clone autore. ThothII non + offre comandi di creazione, modifica o pubblicazione del sorgente. + +**Livello 2 — workspace selezionato.** Questi comandi sono isolati perché richiedono prima la +selezione del workspace. + +- **Validate workspace** verifica nuovamente catalogo, descrittore, Evidence e invarianti della + revisione attiva selezionata. Non contatta il DWH e non modifica file. +- **Save runtime secrets** sostituisce alla cieca i valori compilati. I campi dipendono dal + trasporto DWH e dall'autenticazione Evidence dichiarati; il backend restituisce solo lo stato + configurato/mancante. +- **Forget** elimina dal vault cifrato il singolo secret indicato. Le sessioni o operazioni future + che lo richiedono restano bloccate finché non viene inserito di nuovo. +- **Test connections** materializza temporaneamente i secret necessari, contatta i servizi dati + configurati per quel workspace e rimuove i file temporanei alla fine. Non esporta né pubblica + nulla. + +Il repository Git remoto e le relative credenziali sono impostazioni di installazione. I secret +runtime DWH/Evidence sono invece persistenti nel vault cifrato del backend e non nel local storage +della GUI. La GUI è soltanto l'interfaccia: dopo l'invio cancella i valori dai campi e non può +rileggerli. + +--- + +## Parte 3 — Usare l'applicazione ThothII di base + +### 3.1 Nuova sessione + +Apri l'applicazione e usa il modulo **New session**: inserisci solo la **domanda** in linguaggio +naturale (workspace, modello e provider sono impostazioni globali già configurate). + +Esempio di domanda: + +> «Estrai i pazienti che hanno eseguito un'ablazione nell'ultimo anno, con nome, cognome e data +> dell'intervento.» + +### 3.2 Il workflow a 8 fasi e i gate + +La domanda attraversa **8 fasi**. Tu vedi i documenti intermedi e decidi nei punti chiave: + +1. **F1 chiarimento** — se serve, il modello chiede di togliere ambiguità; +2. **F2 memoria** — recupera le memory riutilizzabili; +3. **F3 riscrittura** — riscrive e approva la domanda; +4. **F4 schema-linking** — propone tabelle e colonne collegate; +5. **F5 sintesi** — riassume lo schema scelto; +6. **F6 CTE** — costruisce i CTE; +7. **F7 SQL finale** — produce `sql_final.sql`; +8. **F8 datamart** — esecuzione/export (dbt, CSV, Excel). + +I **gate di revisione** appaiono come widget: scegli un'opzione singola, seleziona più voci, o +conferma un artefatto/fase. Il modello *propone*, il revisore *decide*. Il lato destro mostra gli +artefatti (schema-linking, CTE, SQL); il pannello Model activity mostra domanda/ragionamento. + +### 3.3 Sessioni + +Le sessioni sono elencate nella barra laterale con id, domanda, data e autore. Una sessione +**riprende** dall'ultima fase incompleta ricostruendo lo stato dai documenti salvati su disco +(`session_manifest.yaml` + artefatti di fase + `review_decisions.jsonl`). Lo stato salvato **è** la +verità: ciò che non è registrato non è avvenuto. + +--- + +## Esempio pratico completo — Policlinico San Donato + +### Passo 0 — repository + +Crea il repository Git del workspace (es. `tht-workspace-psd`): + +```text +thoth-workspaces.yaml # catalogo con psd-clinical +psd-clinical/workspace.yaml # descrittore v3 (vedi §1.2) +psd-clinical/evidence/ # i documenti .md di contesto curati +psd-clinical/schema/annotations.yaml # (quando ci sono join curate) +``` + +Fai `commit` e `push`. Nell'installazione, l'applicazione fa `Pull` e **attiva** il workspace: +valida lo schema v3, materializza l'Evidence dal commit fissato e prepara la collection Qdrant +(1024/cosine + indici). + +### Passo 1 — preprocessing + +```bash +tht --installation ~/thothii-installation.yaml workspace preprocess dwh --workspace psd-clinical --json +tht --installation ~/thothii-installation.yaml workspace preprocess run --workspace psd-clinical --json +``` + +Se il run si ferma per le join (`manual_review_required`): + +```bash +# il curatore rivede i candidati e pubblica psd-clinical/schema/annotations.yaml, poi: +tht --installation ~/thothii-installation.yaml workspace schema accept --workspace psd-clinical --run --yes --json +tht --installation ~/thothii-installation.yaml workspace preprocess run --workspace psd-clinical --resume --json +``` + +### Passo 2 — la domanda + +Nell'applicazione seleziona il workspace `psd-clinical` e crea una sessione con la domanda. Segui +le fasi e conferma ai gate: il modello proporrà lo schema-linking (tabelle/colonne del DWH +`datawarehouse`), i CTE e infine l'SQL finale, che potrai copiare/visualizzare ed eseguire. + +--- + +## Dove trovare i dettagli tecnici + +- Contratto CLI: `docs/contracts/workspace-preprocessing-cli.md` +- Contratto `.tht-dwh`: `docs/contracts/tht-dwh.md` +- Evidence v3: `docs/contracts/workspace-evidence-v3.md` +- Installazione locale: `docs/install/local-workspace-registry.md` +- Installazione server: `docs/install/server-workspace-registry.md` +- Verifica manuale P2–P6: `docs/testing/p2-p6-manual-verification.md` diff --git a/docs/index.md b/docs/index.md index fa2a74c7..89682d8b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,8 +8,10 @@ La documentazione è divisa in due aree: Come funziona il sistema: architettura, specifiche di design delle singole funzionalità, piani di implementazione, report di test. Parte da qui: [Panoramica dell'architettura](architecture/overview.md). -Per installare l'applicazione in Docker nei quattro contesti operativi, partendo dal comando -predefinito `docker compose up --build -d` e dal bundle unico dei secret: +Per autenticazione locale, OIDC generico, Authentik e accettazione PSD: [documentazione autenticazione](architecture/authentication.md). + +Per installare l'applicazione in Docker nei quattro contesti operativi, usando il file env, +`compose.yaml`, l'overlay locale/server e il bundle di secret montato: [Installazione Docker nei quattro contesti](installazione-docker-4-contesti.md). ## Considerazioni Generali diff --git a/docs/install/authentication-local.md b/docs/install/authentication-local.md new file mode 100644 index 00000000..8e95f1df --- /dev/null +++ b/docs/install/authentication-local.md @@ -0,0 +1,70 @@ +# Local authentication + +Use local mode for a standalone PC or Mac. Configure it through `tht`; passwords are entered at an +echo-free prompt or read from a protected `--password-file`, never from a command argument. + +## Bootstrap + +After the installation descriptor and protected secret bundle exist, configure the first enabled +administrator: + +```sh +tht --installation /absolute/path/thothii-installation.yaml auth configure \ + --mode local --public-url http://127.0.0.1:8080 \ + --admin-user --admin-display-name \ + --password-file /absolute/path/protected-password-file +``` + +The password file is temporary operator input: keep it private and remove it after configuration. +The resulting `users.yaml` contains Argon2id hashes, never plaintext passwords. To use prompts, +omit the admin and password options in an interactive terminal. `tht setup` performs the same +bootstrap before it starts the stack. + +The non-secret local `auth.yaml` has this exact shape: + +~~~yaml +version: 1 +mode: local +publicUrl: http://127.0.0.1:8080 +session: + regularTtlSeconds: 43200 + regularIdleSeconds: 7200 + rememberTtlSeconds: 2592000 + rememberIdleSeconds: 604800 + oidcTtlSeconds: 28800 +local: + usersFile: users.yaml +~~~ + +## User administration + +```sh +tht auth user list [--json] +tht auth user add --role user|admin [--display-name ] [--password-file ] +tht auth user set-password [--password-file ] +tht auth user enable +tht auth user disable +tht auth user grant --role user|admin +tht auth user revoke --role user|admin +tht auth user logout-all --yes +``` + +User commands are unavailable in OIDC mode. The last enabled administrator cannot be disabled or +demoted. Every password, role, enabled-state, and `logout-all` change increments the user’s +`authRevision`, invalidating its sessions. `tht auth status --json` is redacted and suitable for +machine use; JSON output is pristine on stdout. + +## Session behavior and recovery + +An ordinary login expires after 2 hours idle or 12 hours absolute. Selecting **Remember me** makes +the cookie persistent and changes the limits to 7 days idle or 30 days absolute. Remembered +sessions survive a browser and backend restart, but not a user revision change, configuration +revision change, logout, or restore. Restore does not include sessions or OIDC state and requires +every user to authenticate again. + +If access is lost, use `tht auth user set-password`, `enable`, role changes, or `logout-all` as +appropriate, then log in again. Do not copy passwords, hashes, cookies, CSRF values, or secret +values into tickets, logs, or evidence. + +Check readiness with `tht auth check`; add `--json` for the machine contract. Use +`tht doctor --json` for the aggregate installation report. diff --git a/docs/install/authentication-oidc.md b/docs/install/authentication-oidc.md new file mode 100644 index 00000000..5f364336 --- /dev/null +++ b/docs/install/authentication-oidc.md @@ -0,0 +1,94 @@ +# Generic OIDC authentication + +OIDC mode supports a standards-based provider. The browser flow is generic: Authorization Code, +PKCE S256, state, nonce, issuer/signature/audience/expiry validation, and the fixed callback +`/api/auth/oidc/callback`. The browser and API must use the same origin; configure the +reverse proxy to preserve that public origin and callback path. + +Configure the installation with `tht`: + +```sh +tht auth configure --mode oidc --public-url \ + --issuer --client-id \ + --authentik-base-url \ + --user-group 'TOT Users' --admin-group 'TOT Admin' +``` + +The OIDC client secret is supplied through the protected secret bundle under the exact key +`THT_OIDC_CLIENT_SECRET`; it is never written into `auth.yaml`. The default scopes are exactly +`openid`, `profile`, and `email`. + +The non-secret OIDC configuration has this exact shape (replace angle-bracket placeholders with +operator values): + +~~~yaml +version: 1 +mode: oidc +publicUrl: +session: + regularTtlSeconds: 43200 + regularIdleSeconds: 7200 + rememberTtlSeconds: 2592000 + rememberIdleSeconds: 604800 + oidcTtlSeconds: 28800 +oidc: + issuer: + clientId: + clientSecretRef: THT_OIDC_CLIENT_SECRET + scopes: [openid, profile, email] + groupsClaim: groups +groupCatalog: + driver: authentik + baseUrl: + apiTokenRef: THT_AUTHENTIK_API_TOKEN +authorization: + groupRoles: + TOT Users: [user] + TOT Admin: [admin] +~~~ + +## The groups claim is mandatory + +The ID token must contain a direct, non-empty `groups` array of strings. ThothII does not follow +distributed claims, overage links, or indirect provider expansion. Missing or malformed claims fail +closed. A browser callback exposes only HTTP 401 `oidc_callback_failed`; it never reveals whether +the claim was missing, malformed, indirect, or overage-style. The redacted diagnostic and +interactive device-flow contract uses `oidc_groups_claim_invalid` for invalid group-claim or +device-flow identity results. + +Configured group names are exact and case-sensitive. The union of matched mappings determines the +Thoth roles. A token with no mapped group is authenticated but has no role and cannot use protected +routes. Additional provider groups are ignored silently, without an error or warning. Group +existence is separately proven by the configured catalog adapter; this is why a generic OIDC +provider may authenticate while still failing installation readiness. + +## Checks and diagnostics + +The surfaces have distinct semantics and this order is recommended: + +1. Workspace Validate performs static authentication validation without contacting the provider. +2. `tht auth check` performs live, non-interactive authentication diagnosis, including discovery, + issuer/JWKS, catalog credentials, and all configured mapped groups. +3. `tht auth check --interactive` repeats live diagnosis and additionally validates a device-flow + identity and its direct `groups` claim when Device Authorization is available. +4. Workspace Test performs aggregate live workspace and authentication validation. + +The live CLI forms are: + +```sh +tht auth check +tht auth check --json +``` + +For a provider that advertises Device Authorization, `tht auth check --interactive` presents a +verification URI and one-time user code on the terminal, waits for completion, and validates a +real ID token including `groups`. It is an operator check, not a replacement for browser login. + +`tht doctor` emits this exact ordered report: `descriptor`, `files`, `docker`, `compose`, +`configuration`, `authentication`, `services`, `core-http`, `frontend-http`, +`workspace-registry`, `workflow`, `pi`. Its authentication entry is live and non-interactive. +Any authentication failure makes Workspace Validate or Workspace Test non-activatable according +to that surface's static or live scope. + +The complete closed diagnostic-code union and exact role-to-permission expansion are in the +[authentication architecture](../architecture/authentication.md). diff --git a/docs/install/authentik.md b/docs/install/authentik.md new file mode 100644 index 00000000..196e467d --- /dev/null +++ b/docs/install/authentik.md @@ -0,0 +1,37 @@ +# Authentik provider setup + +Authentik is the first certified provider for PSD acceptance. The ThothII browser protocol remains +generic OIDC; these steps configure the provider-specific group catalog only. + +1. Create an OAuth2/OIDC application and provider in Authentik. Register exactly + `/api/auth/oidc/callback` as the callback and enable `openid`, `profile`, and `email`. +2. Configure the provider so the ID token contains a direct `groups` array of strings. Verify the + claim with a disposable test identity before running acceptance. +3. Create a dedicated API service account for the group catalog. Grant group-view-only privilege; + do not grant write, user-management, or directory-administration privilege. Put its bearer value + in the protected bundle under `THT_AUTHENTIK_API_TOKEN`. +4. Create or confirm the exact groups `TOT Users` and `TOT Admin`. Map them explicitly in + `auth.yaml` to `user` and `admin`, respectively. Keep other upstream groups out of the mapping. +5. Run Workspace Validate for static authentication validation. Then run live non-interactive + diagnosis, followed by the optional device-flow identity check: + + ```sh + tht auth check + tht auth check --interactive + tht doctor --json + ``` + +6. Run Workspace Test for aggregate live workspace and authentication validation. It must prove + discovery/JWKS, catalog access, and every configured group. The diagnostic result must contain + no secret values. `tht doctor --json` reports `authentication` after `configuration` and before + `services` in its exact ordered checklist. + +Only configured exact group names are queried. Additional Authentik or directory groups are ignored +silently, without a warning. A mapped group absent from Authentik fails closed with +`oidc_mapped_group_missing`; an ambiguous exact-name result uses +`oidc_mapped_group_ambiguous`. A group visible only in an upstream directory but not represented +in Authentik is missing from ThothII’s catalog and must not be treated as present. + +Rotate the two credentials independently through the protected secret-file procedure, then repeat +`tht auth check` and workspace Test. Never put either value in this guide, YAML, shell history, +diagnostic output, or acceptance evidence. diff --git a/docs/install/examples/thothii-installation.local.yaml b/docs/install/examples/thothii-installation.local.yaml new file mode 100644 index 00000000..193746f7 --- /dev/null +++ b/docs/install/examples/thothii-installation.local.yaml @@ -0,0 +1,13 @@ +# Copy this file to an operator-controlled path named exactly thothii-installation.yaml. +# Replace every absolute placeholder. Select exactly one Git transport override. +profile: local +projectDirectory: "/absolute/path/to/ThothII" +envFile: "/absolute/path/to/ThothII/deploy/env/local.env" +workspaceRepository: + remote: git@git.example.com:organization/workspaces.git + branch: main + access: ssh +authentication: + configDirectory: "/absolute/path/to/thothii-auth" +overrides: + - "/absolute/path/to/ThothII/deploy/compose.git-ssh.yaml" diff --git a/docs/install/examples/thothii-installation.server.yaml b/docs/install/examples/thothii-installation.server.yaml new file mode 100644 index 00000000..ab3e2912 --- /dev/null +++ b/docs/install/examples/thothii-installation.server.yaml @@ -0,0 +1,14 @@ +# Copy this file to a protected operator path named exactly thothii-installation.yaml. +# Replace every absolute placeholder. Select exactly one Git transport override. +profile: server +projectDirectory: "/absolute/path/to/ThothII" +envFile: "/absolute/path/to/thothii-server-operator/server.env" +workspaceRepository: + remote: git@git.example.com:organization/workspaces.git + branch: main + access: ssh +authentication: + configDirectory: "/absolute/path/to/thothii-auth" +overrides: + - "/absolute/path/to/ThothII/deploy/compose.session-server.yaml.example" + - "/absolute/path/to/ThothII/deploy/compose.git-ssh.yaml" diff --git a/docs/install/examples/workspace-bindings.env.example b/docs/install/examples/workspace-bindings.env.example new file mode 100644 index 00000000..09b0bfbd --- /dev/null +++ b/docs/install/examples/workspace-bindings.env.example @@ -0,0 +1,14 @@ +# Copy to an untracked operator file. This file contains only non-secret THT_WS_* bindings. +# Every *_FILE value is a container path supplied by the generated local connector override. +THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct +THT_WS_NORTH_STAR_RESEARCH_DWH_HOST=dwh.internal.example +THT_WS_NORTH_STAR_RESEARCH_DWH_PORT=5432 +THT_WS_NORTH_STAR_RESEARCH_DWH_USER=thoth_reader +THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password + +# Evidence examples use separate illustrative namespaces because one descriptor selects one mode. +# Values are container file paths only; signed URLs and credential contents stay in those files. +THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_FILE=/run/secrets/signed-http-evidence-urls.json +THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_FILE=/run/secrets/static-s3-evidence-access-key +THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_FILE=/run/secrets/static-s3-evidence-secret-key +THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_FILE=/run/secrets/static-s3-evidence-session-token diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md new file mode 100644 index 00000000..2f71f261 --- /dev/null +++ b/docs/install/local-workspace-registry.md @@ -0,0 +1,172 @@ +# Local workspace repository installation (macOS, Windows, and Linux) + +This manual connects a local ThothII installation to one remote Git repository hosted by a Git +server such as GitHub, GitLab, or Gitea. ThothII is a read-only consumer: it fetches, +validates, and activates workspace revisions, but never edits, commits, pushes, or publishes them. + +## Architecture ownership contract + +| Component | Ownership | Operator contract | +| --- | --- | --- | +| DWH | External | Configure the external endpoint and complete its runtime credentials in Workspace management. | +| LLM | External | Configure the external endpoint and model policy during installation. | +| Qdrant | Internal | Compose runs the internal service and persists `qdrant-data`. | +| Ollama embedding | Internal | Compose runs the internal `qwen3-embedding:0.6b` service and model-init job. | + +## Semantic index ownership contract + +| Scope | Ownership rule | Isolation rule | +| --- | --- | --- | +| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. | + +The mandatory semantic stack is CPU-first. Set `THOTH_ENABLE_EMBEDDING_GPU=1` only after the +documented GPU prerequisites are satisfied. The embedding contract is fixed at +`qwen3-embedding:0.6b`, 1024 dimensions, cosine distance. + +## Prerequisites + +- A working local installation described by [local.md](local.md). +- A remote Git repository and a read-only deploy credential for this ThothII installation. +- A separate authoring clone in which a workspace curator can edit and publish source revisions. +- `tht` built with `bash scripts/build-tht.sh`. + +## Prepare and publish a workspace source + +Create a local workspace in an ordinary source directory outside ThothII's data directories. The +canonical repository layout is: + +```text +thoth-workspaces.yaml +/workspace.yaml +/evidence/ # optional, repository-owned Evidence +/schema/annotations.yaml # optional curated annotations +``` + +The catalog lists `{id, name, description?}` and the descriptor at +`/workspace.yaml` must match that metadata. Use the examples in +`deploy/workspaces/` as authoring references. Do not store passwords, tokens, private keys, or +signed URLs in Git. + +Publishing is an author-side Git operation: validate the source, commit it, and push it from the +separate authoring clone to the configured branch. This is the only meaning of “publish” in the +workspace lifecycle. ThothII has no author identity and no Git write credential. + +## Use the workspace from the application + +After the installation is started, use Workspace management from the authenticated application: + +1. Run **Update workspace repository** to fetch and validate the configured Git branch into the + application-owned registry. The operation is all-or-nothing and does not modify the authoring + clone. +2. Confirm that the installation-owned `workspace-secrets` storage remains outside the source + repository and contains no credentials in the workspace descriptors. +3. Select the workspace and run **Validate workspace** to verify the active descriptor, catalog, + Evidence, annotations, and runtime bindings. +4. Run **Test connections** only with the approved read-only DWH/Evidence test configuration. + Results are redacted and the workspace source remains unchanged. + + +Schema v3 is the only accepted workspace descriptor. +Schema v1 and v2 workspace descriptors are rejected before activation. + + +## Configure the remote Git repository + +Copy `docs/install/examples/thothii-installation.local.yaml` to an operator-controlled absolute +path. Its `workspaceRepository` block records the remote, branch, and read-only access method. +Choose exactly one transport override: + +- SSH: `deploy/compose.git-ssh.yaml`, with a read-only deploy key and pinned `known_hosts` file. +- HTTPS: `deploy/compose.git-https.yaml`, with a read-only token in a Git credentials file and an + optional private CA file. + +The remote and branch are installation configuration. Git credentials remain protected +installation files and are never accepted by Workspace management or returned by its API. + +Example non-secret/operator paths: + +```dotenv +THT_WORKSPACE_GIT_REMOTE=git@git.example.com:organization/workspaces.git +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_INSTALLATION_ID=local +PI_AUTH_FILE=/absolute/path/to/operator/pi-auth.json +THT_SECRETS_FILE=/absolute/path/to/operator/thothii.secrets +THT_WORKSPACE_GIT_SSH_KEY_FILE=/absolute/path/to/operator/git-ssh-key +THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/absolute/path/to/operator/git-known-hosts +``` + +Keep these files outside both the ThothII checkout and the workspace source repository. Protect +them with mode `0600` on macOS/Linux or an equivalent single-user ACL on Windows. + +## Start and update the installation + +Use only the installation-aware lifecycle: + +```bash +export THT_SOURCE_ROOT=/absolute/path/to/ThothII +THT_BIN=tht +INSTALLATION=/absolute/path/to/operator/thothii-installation.yaml +"$THT_BIN" --installation "$INSTALLATION" start +"$THT_BIN" --installation "$INSTALLATION" doctor +``` + +At startup ThothII clones or fetches the configured repository into its application-managed +`workspace-registry` volume. Later, **Update workspace repository** performs a server-side fetch +and fast-forward candidate checkout. It does not copy anything to the user's computer. + +## Complete runtime secrets in Workspace management + +Open Workspace management after the first successful repository update. + +1. At the repository level, review the configured host, repository, branch, and current revision. +2. Select a workspace. Repository update does not require a selection; validation and connection + tests do. +3. Review the runtime fields derived from the selected DWH transport and Evidence authentication + mechanism. +4. Enter or rotate the required values and choose **Save runtime secrets**. +5. Run **Validate workspace** and then **Test connections**. + +Secret fields are write-only. The GUI receives only configured/missing status. Values are +encrypted by the backend in the platform-neutral `workspace-secrets` volume. ThothII temporarily +materializes a restrictive file only while an existing file-oriented connector needs it, then +removes that file when the runtime lease ends. **Forget** deletes the selected encrypted value. + +The workspace YAML stays environment-independent: it declares connector mechanisms, not host +paths or credentials. Installation trust material such as a Git CA or `known_hosts` remains an +operator concern; DWH and Evidence credentials are completed in the GUI. + +## Validation and activation behavior + +An update follows this sequence: + +1. Fetch the configured branch into a candidate checkout managed by ThothII. +2. Validate the catalog, every descriptor, repository-relative Evidence, and cross-workspace + invariants at the same Git commit. +3. If every workspace is valid, atomically mark that complete commit as active. +4. If any validation fails, report sanitized diagnostics and keep the previous active revision. + +The active checkout is read-only application state. Never edit files under +`/data/workspace-registry`. A source correction must be committed and pushed from the authoring +clone, then fetched again with **Update workspace repository**. + +## Backup, rotation, and recovery + +Back up the `workspace-registry`, `workspace-secrets`, `sessions`, `qdrant-data`, +`embedding-models`, `settings`, and `pi-state` volumes together. The encrypted vault is useless +without its generated master key, so preserve the entire `workspace-secrets` volume and protect +the backup as secret material. + +Rotate a runtime credential by saving its replacement in Workspace management and rerunning its +connection test. Rotate Git credentials in the installation files and restart `core`. To recover +from a bad remote revision, correct or revert it in the authoring repository and run the update; +until validation succeeds, the previous active snapshot remains available. + +## Troubleshooting + +| Symptom | Meaning and action | +| --- | --- | +| Repository unavailable | Check remote host, branch, read-only deploy credential, CA, and `known_hosts`. | +| Candidate rejected | Fix the reported source error in the authoring clone, commit, push, and update again. | +| Runtime configuration required | Select the workspace and complete each required secret field. | +| Connection test fails | Rotate the relevant secret or correct the non-secret endpoint in the source/installation as appropriate. | +| Active revision did not change | The candidate was invalid or was already active; inspect the repository status. | diff --git a/docs/install/local.md b/docs/install/local.md new file mode 100644 index 00000000..3dd98563 --- /dev/null +++ b/docs/install/local.md @@ -0,0 +1,499 @@ +# Install ThothII on a local PC or Mac + +This guide installs one loopback-only ThothII on the same Windows, macOS, or Linux computer that +runs Docker. The supported application is one Docker Compose distribution containing exactly +`frontend` and `core`; Pi is pinned inside `core`. DWH, vector database, embedding, and LLM remain +external configurable services even when they run on this computer. + +No host Pi, Node.js, Python, Go toolchain, Docker socket in core, or browser shell is required. +Commands that contain example paths must be changed to absolute paths on your computer. + +## Choose your platform + +- **macOS:** use Terminal and Docker Desktop. Apple Silicon and Intel are supported by the local + image build. +- **Windows PowerShell:** use Docker Desktop with its WSL2 engine, Git for Windows, the Windows + build launcher, and `tht-windows-amd64.exe`. +- **Windows WSL2 (recommended):** enable Docker Desktop integration for your Linux distribution, + clone under `/home/` rather than `/mnt/c`, and follow the Linux shell commands. +- **Linux PC:** use Docker Engine plus the Compose v2 plugin and the Linux `tht` binary. + +Windows users must also read [Windows and WSL2 line endings](windows-line-endings.md) before the +first build. + +## Prerequisites + +Install only: + +1. Git 2.39 or newer. +2. Docker Desktop on macOS/Windows, or Docker Engine on Linux. +3. Docker Compose v2 (`docker compose`, not legacy `docker-compose`). +4. About 10 GB of free disk for source, images, build cache, and initial volumes. +5. Network access to the workspace Git remote and configured DWH/vector/embedding/LLM endpoints. + +Verify the tools: + +```sh +git --version +docker version +docker compose version +docker run --rm hello-world +``` + +On Linux, add the operator to the Docker group only if local policy permits it; sign out and back +in afterward. A local installation needs no inbound firewall rule because ports bind only to +`127.0.0.1`. + +## Clone and verify LF + +Use a `git clone` command that disables automatic CRLF conversion for this checkout. + +macOS and Linux: + +```sh +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +cd ThothII +git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +``` + +Windows PowerShell: + +```powershell +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +Set-Location ThothII +git config --local core.autocrlf false +& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh +``` + +Windows WSL2: + +```sh +mkdir -p "$HOME/src" && cd "$HOME/src" +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +cd ThothII +git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +``` + +Stop if the verifier names any path. Do not build from a CRLF checkout. + +## Create the local operator files + +Copy the non-secret template. This untracked `.env` contains addresses and absolute source paths, +never secret values: + +```sh +cp deploy/env/local.env.example deploy/env/local.env +mkdir -p /absolute/path/to/thothii-operator/secrets +chmod 0700 /absolute/path/to/thothii-operator/secrets +``` + +Native Windows PowerShell performs the same setup without POSIX utilities. The ACL commands remove +inherited access from the new operator directory and grant full control only to the current Windows +identity. Stop if either `icacls.exe` command returns a nonzero exit code: + +```powershell +$OperatorDir = Join-Path $env:USERPROFILE 'thothii-operator' +$SecretsDir = Join-Path $OperatorDir 'secrets' +$CurrentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name +if (Test-Path $OperatorDir) { throw 'Use a new operator directory or review its ACLs manually.' } +New-Item -ItemType Directory -Force -Path $OperatorDir, $SecretsDir | Out-Null +icacls.exe $OperatorDir /inheritance:r +if ($LASTEXITCODE -ne 0) { throw 'Could not remove inherited operator-directory ACLs.' } +icacls.exe $OperatorDir /grant:r "${CurrentUser}:(OI)(CI)F" +if ($LASTEXITCODE -ne 0) { throw 'Could not grant the current user the operator-directory ACL.' } +Copy-Item deploy/env/local.env.example deploy/env/local.env +Copy-Item docs/install/examples/thothii-installation.local.yaml ` + (Join-Path $OperatorDir 'thothii-installation.yaml') +``` + +Edit `deploy/env/local.env`. At minimum set the workspace Git remote, `PI_AUTH_FILE`, +`THT_SECRETS_FILE`, and external service endpoints. Create the Pi/application and Git transport +files under the protected operator directory and set mode `0600`. On Windows use a user-only ACL +instead. DWH and Evidence credentials are entered later through Workspace management and stored +in the backend's encrypted `workspace-secrets` volume. + +Do not paste credentials into this guide's commands, `.env`, workspace YAML, Git, URLs, image build +arguments, or the installation descriptor. Secret contents are mounted read-only under +`/run/secrets` (Pi's auth store has its own protected read-only mount) and must never be committed, +embedded, rendered, or logged. + +Follow [the local workspace repository guide](local-workspace-registry.md) to choose exactly one +read-only Git SSH/HTTPS override. A fresh install requires a valid private workspace repository; +the remote Git repository remains the source of truth. + +Copy the installation example to an operator-controlled file named exactly +`thothii-installation.yaml`, then replace all placeholders with absolute paths: + +```sh +cp docs/install/examples/thothii-installation.local.yaml \ + /absolute/path/to/thothii-operator/thothii-installation.yaml +``` + +For HTTPS, replace the SSH override in that file with `deploy/compose.git-https.yaml`. Add only +reviewed local overrides. Paths may contain spaces when correctly represented as YAML strings. + +Native Windows uses the same four fields. Use single-quoted absolute Windows paths so backslashes +remain literal YAML characters: + +```yaml +profile: local +projectDirectory: 'C:\Users\operator\src\ThothII' +envFile: 'C:\Users\operator\src\ThothII\deploy\env\local.env' +overrides: + - 'C:\Users\operator\src\ThothII\deploy\compose.git-ssh.yaml' +``` + +## Address external services + +An address is interpreted inside `core`. Therefore container 127.0.0.1 means the container itself, +not the Docker host. Keep external DWH and LLM addresses configurable in the installation; Qdrant +and embedding are internal services in the standard stack. + +- **Docker Desktop (macOS and Windows):** use `host.docker.internal`, for example + `http://host.docker.internal:11434`. +- **Linux:** if a service runs on the host, create an untracked override and include its absolute + path in `thothii-installation.yaml`: + +```yaml +services: + core: + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +Then use `host.docker.internal` in the endpoint. `extra_hosts: host.docker.internal:host-gateway` +is a host routing aid, not a bundled service. Prefer a real DNS name for independently operated +services; retain TLS and authentication even when co-located. + +## Build ThothII and tht + +The canonical local Compose smoke uses the base file plus the local profile. Keep this exact +base+profile command available for install verification: + +~~~sh +docker compose --env-file deploy/env/local.env -f compose.yaml -f deploy/compose.local.yaml up --build -d +~~~ + +After the stack is ready, configure and check authentication with the single host CLI tht; see +the [local authentication guide](authentication-local.md). Authentication configuration is +installation-global and is checked before workspace tests. + +From the repository root, macOS/Linux/WSL2 users run: + +```sh +bash scripts/build-local.sh +bash scripts/build-tht.sh +``` + +Native PowerShell users run: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/build-local.ps1 +& "C:\Program Files\Git\bin\bash.exe" scripts/build-tht.sh +``` + +The second command uses Docker to create native operator binaries under `dist/tht`; users do +not need to know or install Go. Select `tht-darwin-arm64` or `-amd64` on macOS, +`tht-linux-amd64` or `-arm64` on Linux/WSL2, and `tht-windows-amd64.exe` on Windows. +Copy the selected file to the protected operator directory and, on macOS/Linux, run `chmod 0755` +on it. + +## Start and verify + +Set convenient variables (PowerShell users use `$THT_BIN` and `$INSTALLATION` with `& $THT_BIN`): +Every operator call has the form `tht --installation `. + +```sh +THT_BIN=/absolute/path/to/thothii-operator/tht +INSTALLATION=/absolute/path/to/thothii-operator/thothii-installation.yaml +"$THT_BIN" --installation "$INSTALLATION" update --check-only +"$THT_BIN" --installation "$INSTALLATION" start +"$THT_BIN" --installation "$INSTALLATION" status +"$THT_BIN" --installation "$INSTALLATION" doctor +``` + +Native PowerShell uses the same order: + +```powershell +$THT_BIN = 'C:\Users\operator\thothii-operator\tht.exe' +$INSTALLATION = 'C:\Users\operator\thothii-operator\thothii-installation.yaml' +& $THT_BIN --installation $INSTALLATION update --check-only +& $THT_BIN --installation $INSTALLATION start +& $THT_BIN --installation $INSTALLATION status +& $THT_BIN --installation $INSTALLATION doctor +``` + +Wait for both services, then check the same-origin frontend and direct loopback core: + +```sh +curl --fail http://127.0.0.1:8080/health +curl --fail http://127.0.0.1:8787/health +"$THT_BIN" --installation "$INSTALLATION" pi doctor +"$THT_BIN" --installation "$INSTALLATION" pi test +``` + +Native PowerShell must call `curl.exe` explicitly; Windows PowerShell may otherwise resolve `curl` +to `Invoke-WebRequest`: + +```powershell +curl.exe --fail --silent --show-error http://127.0.0.1:8080/health +curl.exe --fail --silent --show-error http://127.0.0.1:8787/health +& $THT_BIN --installation $INSTALLATION pi doctor +& $THT_BIN --installation $INSTALLATION pi test +``` + +Open . If a check fails, run `tht ... logs` or `pi logs`; these are +bounded and sanitize declared secrets. Do not publish either loopback port. + +## Update an installation + +Commit or back up local operator changes first and finish active sessions. A promoted Pi image is +selected by the durable, installation-specific `current-image.yaml` after every base/profile file. +Therefore rebuilding `thothii-core:local` followed by `update --check-only` does not reconcile a +previous `pi update`: the old promoted core would remain selected. + +Do not delete or edit the selector. `tht status` is the installation-aware selector test. If +the running core image is the base `thothii-core:local` image, no Pi update has promoted a durable +lifecycle image and an ordinary same-Pi-version source rebuild/start is supported. If status shows +a lifecycle image and the pulled Pi pin is unchanged, `pi update` would be a no-op and the procedure +must stop. A changed Pi pin uses transactional `pi update --source build` in either case. + +macOS, Linux, and WSL2: + +```sh +set -euo pipefail + +abort_update() { printf 'Source update stopped: %s\n' "$1" >&2; exit 1; } +require_clean_source() { + local source_state + if ! source_state="$(git status --porcelain --untracked-files=all)"; then + abort_update "git status failed" + fi + [[ -z "$source_state" ]] || abort_update "commit, remove, or back up every tracked/untracked source change" +} + +require_clean_source +if ! git pull --ff-only; then abort_update "git pull --ff-only failed"; fi +require_clean_source +if ! git config --local core.autocrlf false; then abort_update "could not set repository LF policy"; fi +if ! bash scripts/verify-line-endings.sh; then abort_update "the pulled checkout contains CRLF files"; fi +if ! SOURCE_REVISION="$(git rev-parse HEAD)"; then abort_update "could not record the pulled revision"; fi +if ! NEXT_PI_VERSION="$(sed -n 's/^ARG PI_VERSION=//p' docker/core.Dockerfile)"; then + abort_update "could not read the pulled Pi pin" +fi +[[ -n "$NEXT_PI_VERSION" && "$NEXT_PI_VERSION" != *$'\n'* ]] || abort_update "expected one pinned default PI_VERSION" +if ! INSTALLATION_STATUS="$("$THT_BIN" --installation "$INSTALLATION" status)"; then + abort_update "tht status failed" +fi +if ! RUNNING_PI_VERSION="$("$THT_BIN" --installation "$INSTALLATION" pi status)"; then + abort_update "tht pi status failed" +fi +RUNNING_PI_VERSION="${RUNNING_PI_VERSION#Pi version: }" +[[ -n "$RUNNING_PI_VERSION" ]] || abort_update "tht pi status returned no version" + +COMPACT_STATUS="${INSTALLATION_STATUS//[[:space:]]/}" +USES_BASE_CORE=false +if [[ "$COMPACT_STATUS" == *'"Image":"thothii-core:local"'* ]]; then + USES_BASE_CORE=true +fi +TRANSACTIONAL_PI_UPDATE=true +if [[ "$NEXT_PI_VERSION" == "$RUNNING_PI_VERSION" ]]; then + [[ "$USES_BASE_CORE" == true ]] || abort_update "same Pi version is selected by a durable lifecycle image" + TRANSACTIONAL_PI_UPDATE=false +fi + +if ! bash scripts/build-local.sh; then abort_update "the local image build failed"; fi +if ! bash scripts/build-tht.sh; then abort_update "the tht build failed"; fi +if ! "$THT_BIN" --installation "$INSTALLATION" update --check-only; then + abort_update "the installation render check failed" +fi +if [[ "$TRANSACTIONAL_PI_UPDATE" == true ]]; then + if ! "$THT_BIN" --installation "$INSTALLATION" pi update \ + --version "$NEXT_PI_VERSION" --source build --yes --drain; then + abort_update "the transactional core update failed" + fi +fi +if ! "$THT_BIN" --installation "$INSTALLATION" start; then abort_update "installation start failed"; fi +if ! curl --fail http://127.0.0.1:8080/health; then abort_update "frontend health check failed"; fi +if ! curl --fail http://127.0.0.1:8787/health; then abort_update "core health check failed"; fi +if ! FINAL_STATUS="$("$THT_BIN" --installation "$INSTALLATION" status)"; then abort_update "final status failed"; fi +if ! FINAL_PI_STATUS="$("$THT_BIN" --installation "$INSTALLATION" pi status)"; then abort_update "final pi status failed"; fi +[[ "${FINAL_PI_STATUS#Pi version: }" == "$NEXT_PI_VERSION" ]] || abort_update "running Pi version does not match the pulled pin" +if ! "$THT_BIN" --installation "$INSTALLATION" doctor; then abort_update "final doctor failed"; fi +require_clean_source +printf 'Built source revision: %s\n%s\n%s\n' "$SOURCE_REVISION" "$FINAL_STATUS" "$FINAL_PI_STATUS" +``` + +Native Windows PowerShell uses the same fail-closed version comparison and transactional promotion: + +```powershell +$ErrorActionPreference = 'Stop' +function Assert-NativeSuccess([string]$Step) { + if ($LASTEXITCODE -ne 0) { throw "$Step failed with exit code $LASTEXITCODE." } +} +function Assert-CleanSource { + $SourceState = @(git status --porcelain --untracked-files=all) + Assert-NativeSuccess 'git status' + if ($SourceState.Count -ne 0) { + throw 'Commit, remove, or back up every tracked/untracked source change.' + } +} + +Assert-CleanSource +git pull --ff-only +Assert-NativeSuccess 'source pull' +Assert-CleanSource +git config --local core.autocrlf false +Assert-NativeSuccess 'repository LF policy' +& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh +Assert-NativeSuccess 'pulled checkout LF verification' +$SourceRevision = git rev-parse HEAD +Assert-NativeSuccess 'source revision read' +$VersionLine = @(Select-String -Path docker/core.Dockerfile -Pattern '^ARG PI_VERSION=(.+)$') +if ($VersionLine.Count -ne 1) { throw 'Expected exactly one pinned default PI_VERSION.' } +$NextPiVersion = $VersionLine.Matches[0].Groups[1].Value +$InstallationStatus = @(& $THT_BIN --installation $INSTALLATION status) +Assert-NativeSuccess 'installation status' +$RunningPiStatus = (& $THT_BIN --installation $INSTALLATION pi status) +Assert-NativeSuccess 'Pi status' +$RunningPiVersion = $RunningPiStatus -replace '^Pi version:\s*', '' +if ([string]::IsNullOrWhiteSpace($RunningPiVersion)) { throw 'Pi status returned no version.' } +$Services = $InstallationStatus | ConvertFrom-Json +$CoreServices = @($Services | Where-Object { $_.Service -eq 'core' }) +if ($CoreServices.Count -ne 1) { throw 'Installation status did not identify exactly one core service.' } +$UsesBaseCore = $CoreServices[0].Image -eq 'thothii-core:local' +$TransactionalPiUpdate = $true +if ($NextPiVersion -eq $RunningPiVersion) { + if (-not $UsesBaseCore) { throw 'Same Pi version is selected by a durable lifecycle image.' } + $TransactionalPiUpdate = $false +} +powershell -ExecutionPolicy Bypass -File scripts/build-local.ps1 +Assert-NativeSuccess 'local image build' +& "C:\Program Files\Git\bin\bash.exe" scripts/build-tht.sh +Assert-NativeSuccess 'tht build' +& $THT_BIN --installation $INSTALLATION update --check-only +Assert-NativeSuccess 'installation render check' +if ($TransactionalPiUpdate) { + & $THT_BIN --installation $INSTALLATION pi update ` + --version $NextPiVersion --source build --yes --drain + Assert-NativeSuccess 'transactional core update' +} +& $THT_BIN --installation $INSTALLATION start +Assert-NativeSuccess 'installation start' +curl.exe --fail --silent --show-error http://127.0.0.1:8080/health +Assert-NativeSuccess 'frontend health check' +curl.exe --fail --silent --show-error http://127.0.0.1:8787/health +Assert-NativeSuccess 'core health check' +$FinalStatus = @(& $THT_BIN --installation $INSTALLATION status) +Assert-NativeSuccess 'final installation status' +$FinalPiStatus = (& $THT_BIN --installation $INSTALLATION pi status) +Assert-NativeSuccess 'final Pi status' +if (($FinalPiStatus -replace '^Pi version:\s*', '') -ne $NextPiVersion) { + throw 'Running Pi version does not match the pulled pin.' +} +& $THT_BIN --installation $INSTALLATION doctor +Assert-NativeSuccess 'final doctor' +Assert-CleanSource +Write-Output "Built source revision: $SourceRevision" +Write-Output $FinalStatus +Write-Output $FinalPiStatus +``` + +The revision is printed only after every source/build/start/health/installation-aware check passes +and a final porcelain check still reports no tracked or untracked source changes. For a changed Pi +pin, status reports the promoted lifecycle candidate; for a same-version installation with no +selector, status reports the rebuilt base core. `update --check-only` alone proves only that Compose +renders. +Review release notes before updating. See [Pi management](pi-management.md) for rollback; never +install a package in the running container. + +## Back up and restore + +Back up before source/Pi updates and test restoration periodically. First stop cleanly: + +```sh +"$THT_BIN" --installation "$INSTALLATION" stop +docker volume ls --format '{{.Name}}' | grep '^thothii-' +``` + +Identify the four exact volumes belonging to this installation: `settings`, `pi-state`, +`workspace-registry`, and `sessions`. Confirm their Compose project label with `docker volume +inspect`. For each exact volume, archive it to a protected backup directory: + +```sh +BACKUP_DIR=/absolute/path/to/backups/2026-08-05 +VOLUME=exact-installation-volume-name +mkdir -p "$BACKUP_DIR" +docker run --rm -v "$VOLUME:/source:ro" -v "$BACKUP_DIR:/backup" \ + alpine:3.22 tar -C /source -czf "/backup/$VOLUME.tgz" . +``` + +Native PowerShell can run the same read-only archive container: + +```powershell +$BackupDir = 'C:\Users\operator\thothii-backups\2026-08-05' +$Volume = 'exact-installation-volume-name' +New-Item -ItemType Directory -Force $BackupDir | Out-Null +docker run --rm -v "${Volume}:/source:ro" -v "${BackupDir}:/backup" ` + alpine:3.22 tar -C /source -czf "/backup/${Volume}.tgz" . +``` + +Also back up the installation descriptor, operator environment, generated overrides, and secret +files to separate encrypted/protected storage. Never commit them. Record image digests and the Git +revision. Do not back up while containers are running. + +Restore only while stopped and only into a new, verified-empty exact target volume. Test the +archive in a disposable installation first: + +```sh +TARGET_VOLUME=exact-empty-target-volume-name +ARCHIVE=/absolute/path/to/backups/2026-08-05/exact-volume-name.tgz +docker run --rm -v "$TARGET_VOLUME:/target" alpine:3.22 \ + sh -c 'test -z "$(ls -A /target)"' +docker run --rm -v "$TARGET_VOLUME:/target" -v "$(dirname "$ARCHIVE"):/backup:ro" \ + alpine:3.22 tar -C /target -xzf "/backup/$(basename "$ARCHIVE")" +``` + +Native PowerShell uses `Split-Path` to produce the read-only archive mount and archive name: + +```powershell +$TargetVolume = 'exact-empty-target-volume-name' +$Archive = 'C:\Users\operator\thothii-backups\2026-08-05\exact-volume-name.tgz' +$ArchiveDir = Split-Path -Parent $Archive +$ArchiveName = Split-Path -Leaf $Archive +docker run --rm -v "${TargetVolume}:/target" alpine:3.22 ` + sh -ceu 'test -z "$(ls -A /target)"' +if ($LASTEXITCODE -ne 0) { throw 'The restore target volume is not empty.' } +docker run --rm -v "${TargetVolume}:/target" -v "${ArchiveDir}:/backup:ro" ` + alpine:3.22 tar -C /target -xzf "/backup/${ArchiveName}" +if ($LASTEXITCODE -ne 0) { throw 'The volume restore failed.' } +``` + +Restore all four volumes from the same backup set, restore protected operator files separately, +then run `update --check-only`, `start`, `doctor`, registry status/diagnostics, and a known session +before normal use. Never merge an archive into a non-empty volume. + +## Data-preserving uninstall + +Run `tht stop`, retain the installation descriptor at the same absolute path, and make one +verified backup set. In Docker Desktop, remove only this installation's stopped `core` and +`frontend` containers and optional local images; leave its four named volumes. On Linux, use the +containers' exact Compose project labels to remove only those stopped containers. Do not prune +global Docker data. + +Do **not** run `docker compose down --volumes`: it deletes the application data this procedure is +meant to preserve. Keep the operator directory and protected secrets if you intend to reinstall. +Using the same descriptor path preserves the `tht` project identity and reconnects the same +named volumes after rebuilding the source checkout. + +## Next: workspaces and Pi + +Complete [local workspace-registry installation](local-workspace-registry.md), including Git trust, +bindings, pull, validation, diagnostics, and registry recovery. Then use [Pi management](pi-management.md) +for provider/model configuration, smoke testing, transactional update, and rollback. + +The Git-backed workspace registry is always the workspace source of truth. Local DWH, vector, +embedding, or LLM processes remain independent services and are never added to the mandatory +ThothII core. diff --git a/docs/install/pi-management.md b/docs/install/pi-management.md new file mode 100644 index 00000000..84ae9c44 --- /dev/null +++ b/docs/install/pi-management.md @@ -0,0 +1,162 @@ +# Pi management + +ThothII bundles Pi in the `core` image. Operators use the Pi Management page for safe application +defaults and the host-side `tht` CLI for lifecycle work. A local Pi installation is not +required. + +Run these commands from the root of the current ThothII checkout or worktree. `tht` discovers +the valid installation descriptor in that project tree, so it uses the `deploy/` files belonging to +the checkout from which you run it. Do not use `~/bin`: `~` is the user home directory, not the +project root. + +```sh +THT_BIN=tht +tht version --json +``` + +If `tht` is not on `PATH`, install the native host CLI using the installation procedure in +`local.md` or `server.md`, then set `THT_BIN` to that installed binary. For an installation +stored elsewhere, set `THOTHII_INSTALLATION` or pass +`--installation /thothii-installation.yaml` explicitly. + +## Choose application defaults + +Use the **Pi Management** page to select the supported provider, model, and reasoning default, then +choose **Save defaults**. The page shows credentials only as present or missing and can run bounded +diagnostics; it never accepts or displays a credential, opens a terminal, or updates an image. + +Alternatively, use the CLI from an administrator terminal: + +```sh +"$THT_BIN" pi configure +"$THT_BIN" pi configure --provider zai --model glm-5.2 --thinking medium +``` + +Use GUI Save defaults or CLI `pi configure`, not both for the same change. The CLI's interactive +choices are restricted to supported models; non-interactive use must supply all three values. Both +methods store application defaults in backend installation settings, not in the project policy file. + +Useful read-only checks are: + +```sh +"$THT_BIN" pi status +"$THT_BIN" pi doctor +"$THT_BIN" pi test +"$THT_BIN" pi check +"$THT_BIN" pi logs +``` + +`pi check` is an alias for `pi test`; logs are a sanitized, bounded snapshot with no follow mode. + +## Edit the provider catalog and enabled-model policy + +Edit these project-root files in source control, then review and deploy the change through the +normal project process: + +- `deploy/pi/models.json` is the provider catalog: provider endpoints and the models each provider + offers. +- `deploy/pi/settings.json` is the enabled-model policy only; it lists the models available to the + application and does not store application defaults. + +These files contain configuration, not credentials. Keep provider configuration declarative: Pi +management rejects executable `!command` values. Docker Compose mounts the selected configuration +and credential files read-only. + +## Store provider credentials + +`PI_AUTH_FILE` is a setting in the installation environment file (for example, +`deploy/env/local.env`). Its value is the absolute path of the protected host credential file that +this installation selects. Docker Compose mounts that selected file read-only for Pi. +Other declared protected material is likewise mounted read-only under `/run/secrets`. + +Set restrictive permissions on the host file (`0600` on macOS/Linux or a user-only ACL on Windows). +Never put its contents in installation YAML, Git, command arguments, the browser, screenshots, +tickets, rendered Compose output, or logs. Do not print the file while troubleshooting. + +## Reload changed configuration + +After changing the provider catalog, enabled-model policy, or selected credential file, reload the +running application with one confirmed restart: + +```sh +"$THT_BIN" pi restart --yes --drain +``` + +`--yes` confirms that core will be recreated. Without `--drain`, restart refuses active sessions; +with it, ThothII closes admission and waits for active sessions to finish without terminating them. +The wait is bounded. Restart retains the exact captured running image: it does not build, pull, or +upgrade an image. Before recreating core, it pins that image through transaction-scoped Compose +override material so a configured tag moving during the operation cannot change the selected +image, and Compose is explicitly told never to build or pull. It will restart only core, then +verifies health, Pi version, settings/model smoke, non-secret rendered configuration, and +persistence mounts before reopening admission. + +Use `pi restart --yes` when there are already no active sessions. Do not substitute `tht stop` +and `tht start` or raw Compose commands for this reload workflow. + +## Update the bundled Pi version + +`pi update` is for a new bundled Pi version; it is not a configuration reload. The simple command +uses the single `ARG PI_VERSION=...` pin in `docker/core.Dockerfile`, builds that version, waits for +active sessions to finish, and recreates only `core`: + +```sh +"$THT_BIN" pi update +``` + +To build a specific version, pass `--version`; source, confirmation, and drain are automatic for +this normal build path: + +```sh +"$THT_BIN" pi update --version 0.81.0 +``` + +A registry update must use an immutable digest, never a mutable tag: + +```sh +"$THT_BIN" pi update \ + --version 0.81.0 --source pull \ + --image registry.example.invalid/thothii-core@sha256:<64-lowercase-hex-digits> \ + --yes --drain +``` + +Update keeps new-session admission gated while it builds or pulls a candidate, recreates only +`core`, verifies it, and promotes the image only after success. It preserves the frontend and named +volumes. + +## Recover a failed lifecycle operation + +If a restart or update fails after core recreation, leave maintenance enabled and preserve the +reported recovery state and transaction override. Do not delete `.tht`, state files, +containers, or volumes. Inspect status and sanitized logs: + +```sh +"$THT_BIN" pi maintenance status +"$THT_BIN" pi status +"$THT_BIN" pi logs +``` + +For a failed update, restore its prior image: + +```sh +"$THT_BIN" pi rollback --yes +``` + +For a failed restart, use maintenance recovery instead of rollback. After repairing the reported +Docker, disk, or configuration problem, use the same command to complete either safe recovery path: + +```sh +"$THT_BIN" pi maintenance recover --yes +"$THT_BIN" pi doctor +"$THT_BIN" pi test +``` + +`pi rollback --yes` restores the prior update image. `pi maintenance recover --yes` checks both +restart and update recovery state before it can reopen admission. If either command fails, keep the +installation gated and collect only the sanitized diagnostics. + +## Direct support access + +Raw Compose access is unsupported because it can bypass the installation-specific environment and +durable image selector. For support, use the installation-aware `tht pi status`, +`tht pi doctor`, `tht pi test`, and `tht pi logs` commands. diff --git a/docs/install/psd-workspace-setup.md b/docs/install/psd-workspace-setup.md new file mode 100644 index 00000000..abba0596 --- /dev/null +++ b/docs/install/psd-workspace-setup.md @@ -0,0 +1,61 @@ +# Policlinico San Donato — setup workspace (nuova gestione) + +Authentication acceptance is documented in the [manual authentication matrix](../testing/authentication-manual-acceptance.md). +Use generic OIDC with Authentik as the certified group catalog, map only the exact TOT Users and +TOT Admin groups, then run Workspace Validate, `tht auth check`, `tht auth check --interactive`, +and Workspace Test in that order. Browser callback E2E, native Windows execution, approved PSD +manual identities, external L2, and the two parked restore-lock preconditions remain pending the +Task 15/release gates. + +Guida operativa per collegare ThothII al DWH di PSD con il nuovo sistema (registry Git + descriptor +v3 + `tht`). + +## Stato attuale (2026-08-13) + +- **Repository PSD pubblicato:** `https://github.com/mptyl/tht-workspace-psd` (privato), branch + `main`, commit `d4f9185`. Layout P1.1 già migrato e validato. +- **Deploy key SSH** (sola lettura, senza passphrase) in + `deploy/psd/secrets/git-ssh-key` e registrata sul repo come deploy key `thothii-psd`; il remote + Git usato dall'installazione è `git@github.com:mptyl/tht-workspace-psd.git`. +- **Config operatore pronta** (file reali gitignored in `deploy/psd/`): `operator.env`, + `thothii-installation.yaml` e i secret d'installazione in `secrets/` (pi-auth, secret bundle, + chiave SSH, known_hosts). L'API key DWH va completata nella gestione Workspace ed è conservata + nel vault cifrato del backend. Nessuna CA: il DWH REST usa HTTPS pubblico. +- **Stack avviato** (progetto `thothii-70417a3e30ea`, via `tht start`): `qdrant`, `embedding` + (con `qwen3-embedding:0.6b`), `core`, `frontend` sani. Il registry ha **clonato e attivato** + `psd-clinical` (stato `ready`). +- **`tht workspace inspect --workspace psd-clinical` = OK** (identità descrittore/catalogo + risolte); la configurazione runtime va completata e testata dalla GUI. +- **Bloccante residuo: VPN.** `supabase-aritmolab.policlinicosandonato.it` non risolve + (`NXDOMAIN`) → il preprocessing DWH e le sessioni live non possono ancora partire. + +## Avvio/arresto (canonico) + +Usare `tht` (stesso project name, quindi stessi volumi named): + +```bash +tht=dist/tht/tht-darwin-arm64 +"$tht" --installation "$(pwd)/deploy/psd/thothii-installation.yaml" start +"$tht" --installation "$(pwd)/deploy/psd/thothii-installation.yaml" workspace inspect --workspace psd-clinical --json +"$tht" --installation "$(pwd)/deploy/psd/thothii-installation.yaml" stop +``` + +> **Nota project name:** `tht` calcola un project name stabile dall'installation descriptor +> (`thothii-`); `docker compose` "a mano" usa invece `name: thothii` dal `compose.yaml`, quindi +> i volumi named non coinciderebbero. Perciò per lo stack si usa `tht start` (non +> `compose-with-preflight.sh up`). + +## Rimane: smoke live di una domanda (P8 L2) + +Il preprocessing è già completato. Resta solo: + +1. Aprire `http://localhost:8080` e selezionare `psd-clinical`. +2. Creare una sessione con una domanda reale in linguaggio naturale. +3. Seguire le 8 fasi fino al primo gate di revisione. + +## Cosa è già stato fatto + +- Ristrutturazione del repo PSD nel layout P1.1 + validazione locale. +- Pubblicazione GitHub + deploy key read-only + configurazione Git d'installazione. +- Avvio stack + attivazione registry + `tht inspect` verde. +- **Preprocessing live completato** su PSD: DWH → FK → schema → Evidence, idempotente. diff --git a/docs/install/reverse-proxy-caddy.md b/docs/install/reverse-proxy-caddy.md new file mode 100644 index 00000000..ed675fa8 --- /dev/null +++ b/docs/install/reverse-proxy-caddy.md @@ -0,0 +1,98 @@ +# Put ThothII behind Caddy + +Choose exactly one authentication mode. Direct ThothII-managed OIDC and deprecated upstream +authentication are mutually exclusive proxy contracts; never combine their directives. + +## Direct ThothII-managed OIDC + +Use this mode when `auth.yaml` has `mode: oidc`. Caddy terminates TLS and proxies every request to +`frontend`; ThothII performs login, callback validation, session creation, and authorization. +Caddy must not apply `forward_auth` or another external authentication gateway. + +The public `/api/auth/oidc/login` and `/api/auth/oidc/callback` paths pass unchanged through the +same proxy as the rest of `/api`. The configured `publicUrl` must match the browser origin. + +```caddyfile +thoth.example.invalid { + reverse_proxy 127.0.0.1:8080 { + # No URI rewrite: OIDC login and callback paths reach frontend unchanged. + flush_interval -1 + header_up Host {host} + header_up X-Forwarded-Proto https + header_up X-Forwarded-Host {host} + } + + log { + output file /var/log/caddy/thoth-access.log + format json + } +} +``` + +After reload, run Workspace Validate for static validation, `tht auth check` for live, +non-interactive authentication diagnosis, and then Workspace Test for aggregate live validation. + +## Deprecated upstream migration mode + +Use this section only while the installation explicitly uses deprecated `upstream` mode. Do not +use it with `mode: oidc` or `mode: local`. Here an external authentication gateway owns login and +Caddy applies `forward_auth` before forwarding normalized private identity headers to `frontend`. + +Forwarding identity headers alone does not authenticate a user. The authentication gateway returns +2xx only after validating its own credential or session. Clear browser-supplied public and trusted +headers before the subrequest, and map identity only from the successful auth response. + +```caddyfile +thoth.example.invalid { + route { + request_header -X-Authenticated-User + request_header -X-Thoth-Principal-Issuer + request_header -X-Thoth-Principal-Subject + request_header -X-Thoth-Principal-Display-Name + request_header -X-Thoth-Is-Admin + request_header -X-Thoth-Trusted-Principal-Issuer + request_header -X-Thoth-Trusted-Principal-Subject + request_header -X-Thoth-Trusted-Principal-Display-Name + request_header -X-Thoth-Trusted-Is-Admin + + forward_auth auth-gateway:4180 { + uri /verify + copy_headers { + X-Thoth-Principal-Issuer>X-Thoth-Trusted-Principal-Issuer + X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject + X-Thoth-Principal-Display-Name>X-Thoth-Trusted-Principal-Display-Name + X-Thoth-Is-Admin>X-Thoth-Trusted-Is-Admin + } + } + + reverse_proxy 127.0.0.1:8080 { + flush_interval -1 + header_up Host {host} + header_up X-Forwarded-Proto https + } + } +} +``` + +## Trust boundary + +Caddy is the only public listener and proxies only to loopback `frontend`, never directly to +`core`. Configure access logs to omit cookies, authorization data, query strings, and identity +headers. Keep Caddy keys and state outside ThothII source and operator directories. + +## Validate and reload + +Keep the public firewall closed while validating: + +```sh +curl --fail http://127.0.0.1:8080/health +caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile +sudo systemctl reload caddy +``` + +## Test authentication and SSE + +For direct OIDC, verify the login path redirects to the configured provider, the callback reaches +ThothII unchanged, forged identity headers grant nothing, and SSE is unbuffered. For deprecated +upstream mode, additionally verify the external gateway rejects unauthenticated traffic and only +its 2xx response can create trusted identity headers. diff --git a/docs/install/reverse-proxy-nginx.md b/docs/install/reverse-proxy-nginx.md new file mode 100644 index 00000000..2277112a --- /dev/null +++ b/docs/install/reverse-proxy-nginx.md @@ -0,0 +1,143 @@ +# Put ThothII behind Nginx + +Choose exactly one authentication mode. Direct ThothII-managed OIDC and deprecated upstream +authentication are mutually exclusive proxy contracts; never combine their locations or headers. + +## Direct ThothII-managed OIDC + +Use this mode when `auth.yaml` has `mode: oidc`. Nginx terminates TLS and proxies every request +to loopback `frontend`. ThothII owns OIDC login, callback validation, browser sessions, and +authorization. No external `auth_request` or authentication gateway belongs in this server. + +The `location /` block below has a `proxy_pass` without a replacement URI, so public +`/api/auth/oidc/login` and `/api/auth/oidc/callback` are forwarded unchanged. The configured +`publicUrl` must match the browser origin. + +```nginx +server { + listen 80; + server_name thoth.example.invalid; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name thoth.example.invalid; + + ssl_certificate /etc/nginx/tls/thoth/fullchain.pem; + ssl_certificate_key /etc/nginx/tls/thoth/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location / { + # No auth_request and no URI rewrite: ThothII receives OIDC paths unchanged. + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Connection ""; + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + add_header X-Accel-Buffering no always; + } +} +``` + +After reload, run Workspace Validate for static validation, `tht auth check` for live, +non-interactive authentication diagnosis, and then Workspace Test for aggregate live validation. + +## Deprecated upstream migration mode + +Use this section only while the installation explicitly uses deprecated `upstream` mode. Do not +use it with `mode: oidc` or `mode: local`. In this mode an external authentication gateway owns +login, and Nginx applies `auth_request` before forwarding normalized private identity headers. + +Forwarding identity headers alone does not authenticate a user. The authentication gateway returns +2xx only after validating its own credential or session. + +```nginx +server { + listen 443 ssl; + server_name thoth.example.invalid; + + ssl_certificate /etc/nginx/tls/thoth/fullchain.pem; + ssl_certificate_key /etc/nginx/tls/thoth/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location = /_authenticate { + internal; + proxy_pass http://auth-gateway:4180/verify; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Authenticated-User ""; + proxy_set_header X-Thoth-Principal-Issuer ""; + proxy_set_header X-Thoth-Principal-Subject ""; + proxy_set_header X-Thoth-Principal-Display-Name ""; + proxy_set_header X-Thoth-Is-Admin ""; + proxy_set_header X-Thoth-Trusted-Principal-Issuer ""; + proxy_set_header X-Thoth-Trusted-Principal-Subject ""; + proxy_set_header X-Thoth-Trusted-Principal-Display-Name ""; + proxy_set_header X-Thoth-Trusted-Is-Admin ""; + } + + location / { + auth_request /_authenticate; + auth_request_set $thoth_principal_issuer + $upstream_http_x_thoth_principal_issuer; + auth_request_set $thoth_principal_subject + $upstream_http_x_thoth_principal_subject; + auth_request_set $thoth_principal_display_name + $upstream_http_x_thoth_principal_display_name; + auth_request_set $thoth_is_admin + $upstream_http_x_thoth_is_admin; + + proxy_set_header X-Authenticated-User ""; + proxy_set_header X-Thoth-Principal-Issuer ""; + proxy_set_header X-Thoth-Principal-Subject ""; + proxy_set_header X-Thoth-Principal-Display-Name ""; + proxy_set_header X-Thoth-Is-Admin ""; + proxy_set_header X-Thoth-Trusted-Principal-Issuer $thoth_principal_issuer; + proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject; + proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name; + proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Connection ""; + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + add_header X-Accel-Buffering no always; + } +} +``` + +## Trust boundary + +Nginx is the only public listener and proxies only to loopback `frontend`, never directly to +`core`. Keep private keys outside the ThothII tree. Do not log cookies, authorization headers, +OIDC callback query values, authentication bodies, or trusted identity headers. + +## Validate and reload + +Keep the public firewall closed while validating: + +```sh +curl --fail http://127.0.0.1:8080/health +sudo nginx -t +sudo systemctl reload nginx +``` + +## Test authentication and SSE + +For direct OIDC, verify login redirects to the configured provider, callback traffic reaches +ThothII unchanged, forged identity headers grant nothing, and SSE is unbuffered. For deprecated +upstream mode, additionally verify the external gateway rejects unauthenticated traffic and only +its 2xx response can create trusted identity headers. diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md new file mode 100644 index 00000000..dccab696 --- /dev/null +++ b/docs/install/server-workspace-registry.md @@ -0,0 +1,128 @@ +# Server workspace repository installation + +This manual supplements [server.md](server.md). A server installation reads one remote Git +repository hosted by GitHub, GitLab, Gitea, Bitbucket, or another Git server. ThothII fetches and +validates complete revisions but never edits, commits, pushes, or publishes workspace source. + +## Architecture ownership contract + +| Component | Ownership | Operator contract | +| --- | --- | --- | +| DWH | External | Configure the external endpoint and complete runtime credentials through the authenticated GUI. | +| LLM | External | Configure the external endpoint and model policy under installation control. | +| Qdrant | Internal | Compose runs private Qdrant and persists `qdrant-data`; include it in Qdrant backup/restore. | +| Ollama embedding | Internal | Compose runs private Ollama with `qwen3-embedding:0.6b`. | + +## Semantic index ownership contract + +| Scope | Ownership rule | Isolation rule | +| --- | --- | --- | +| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. | + +The fixed semantic contract is 1024 dimensions and cosine distance. DWH and LLM remain external; +Qdrant, Ollama, and `embedding-model-init` remain private internal services. + +## Service account, storage, and firewall + +Run the application as the documented unprivileged service account. Keep the source checkout, +operator files, application data, and workspace authoring clone separate: + +```text +/srv/thothii/app/ # ThothII source release +/srv/thothii/operator/ # installation descriptor and protected Git files +/srv/thothii/data/ # application data, encrypted workspace vault, sessions +/srv/workspace-authoring/ # optional curator clone; never mounted into ThothII +``` + +Expose only the authenticated same-origin reverse proxy. Keep `core`, Qdrant, and Ollama private. + +## Prepare and publish a workspace source + +Create a local workspace in the external authoring repository, which contains +`thoth-workspaces.yaml`, one +`/workspace.yaml` per catalog entry, optional repository-owned Evidence, and optional +curated schema annotations. It contains no credentials. + +Publishing belongs to the curator workflow outside ThothII: validate, review, commit, and push the +source revision to the configured protected branch. Grant the ThothII service only read access. + + +Schema v3 is the only accepted workspace descriptor. +Schema v1 and v2 workspace descriptors are rejected before activation. + + +## Configure the remote Git repository + +Copy `docs/install/examples/thothii-installation.server.yaml` to +`/srv/thothii/operator/thothii-installation.yaml`. Set `workspaceRepository.remote`, `.branch`, and +`.access`, then select exactly one Git transport override. The remote and credential are normally +repository-scoped read-only deploy credentials. + +For SSH, mount a private key and pinned known-hosts file. For HTTPS, mount a Git credentials file +and the required CA chain. These installation credentials are not editable in Workspace +management and are never exposed by the API. + +## Start and update the installation + +Use the installation-aware controller described by `server.md`: + +```bash +THT_BIN=/srv/thothii/operator/tht +INSTALLATION=/srv/thothii/operator/thothii-installation.yaml +"$THT_BIN" --installation "$INSTALLATION" start +"$THT_BIN" --installation "$INSTALLATION" doctor +``` + +The descriptor composes `compose.yaml`, `deploy/compose.server.yaml`, the server session-storage +override, and one read-only Git transport override. **Update workspace repository** fetches a +candidate on the server; it does not transfer workspace files to the operator workstation. + +## Complete runtime secrets in Workspace management + +After repository activation, an authenticated user can: + +1. Review the configured repository identity and update it without selecting a workspace. +2. Select a workspace to see the DWH/Evidence credential fields required by its connector modes. +3. Blind-save or rotate values; returned responses contain status only. +4. Run **Validate workspace source** and then test its configured connections. +5. Forget an obsolete value after dependent sessions and jobs have ended. + +The backend encrypts values in `/data/workspace-secrets`, including the installation-specific +master key. The server profile persists that directory inside `THT_DATA_ROOT`; no workspace YAML +path depends on Linux, macOS, or Windows. Plaintext exists only in a restrictive temporary file +for the duration of a diagnostic, session, or maintenance lease. + +Authorization is intentionally the current installation-wide authenticated-user policy. A future +role model or external secret manager can replace that policy without changing workspace source. + +## Validation and activation behavior + +Repository update is all-or-nothing: ThothII fetches the configured branch, validates catalog, +descriptors, Evidence paths, and cross-workspace invariants at one commit, then atomically activates +the complete candidate. A rejected candidate never replaces the previous active snapshot. The +application-owned checkout and snapshots are read-only runtime state. + +Validation proves descriptor and repository structure. **Test connections** additionally +materializes the current runtime secrets and contacts only the selected workspace's configured +DWH/Evidence endpoints. Failure does not modify or publish workspace source. + +## Backup, rotation, and recovery + +Back up application data and Qdrant consistently. Qdrant backup/restore must cover `qdrant-data`; +application recovery must cover repository snapshots/state, sessions, settings, Pi state, and the +entire encrypted `/data/workspace-secrets` directory. Store backup encryption keys separately and +test restore procedures without production traffic. + +Rotate DWH/Evidence credentials through Workspace management. Rotate Git access by atomically +replacing its protected installation file and restarting `core`. Recover a bad source revision by +reverting or correcting it in the external authoring repository and updating again. + +## Troubleshooting + +| Symptom | Meaning and action | +| --- | --- | +| Git authentication failed | Verify repository-scoped read permission, branch, key/token, CA, and host-key pinning. | +| Candidate validation failed | Correct the source repository; the prior active commit remains in service. | +| Runtime configuration required | Select the workspace and complete all required write-only fields. | +| Secret store unavailable | Stop writes, preserve `/data/workspace-secrets`, and restore vault plus master key together. | +| Connection test failed | Rotate the indicated runtime credential or correct the relevant non-secret endpoint. | diff --git a/docs/install/server.md b/docs/install/server.md new file mode 100644 index 00000000..d0a371ad --- /dev/null +++ b/docs/install/server.md @@ -0,0 +1,501 @@ +# Install ThothII on a Linux server + +Server authentication uses generic OIDC with the reverse proxy preserving the configured public +origin and callback path. Follow the [OIDC guide](authentication-oidc.md), [Authentik guide](authentik.md) +when applicable, and the [authentication acceptance matrix](../testing/authentication-manual-acceptance.md). +The host authentication CLI is `tht`. Use `tht --installation workspace inspect +--workspace --json` for the active workspace snapshot, `tht --installation +auth check` for live non-interactive authentication diagnosis, `auth check --interactive` for +device-flow identity validation, and `tht ... doctor --json` for the aggregate installation gate. + +This guide is for an installer with basic Linux administration and very basic Docker knowledge. +It deploys the same Compose distribution used on a local PC: the mandatory application is exactly +`frontend` plus `core`, and pinned Pi is inside `core`. The server does not need host Pi, Node.js, +Python, Go, a browser shell, or a Docker socket inside either container. + +Examples use `/srv/thothii` as an **example operator root**, `thoth.example.com` as a replaceable +DNS name, and systemd-based command names. Adapt them to local policy. Complete the server session +storage overlay and migration procedure before exposing a production installation. + +## Deployment contract + +- The generic Linux host and Docker Compose v2 are the deployment platform. No other + application's Compose project, network, path, or runtime is required. +- `frontend` is the only published service and defaults to `127.0.0.1:8080`; `core` has no host + port. A host Nginx or Caddy listener terminates TLS and sends all application traffic to + `frontend`, never directly to `core`. +- DWH, vector database, embedding service, and LLM are external configurable endpoints. This + remains true when they happen to run on the same physical server. +- Application, Git, connector, and session credentials are protected host files mounted read-only + under `/run/secrets`. Pi's protected auth JSON uses its dedicated read-only Pi mount. No secret + value belongs in Git, images, browser storage, environment values, rendered Compose, or logs. +- The Git-backed workspace registry is the source of truth. Installation-local bindings identify + endpoints and secret-file paths; they do not replace the reviewed Git workspace descriptors. +- `tht` is the operator CLI for start, stop, status, health, logs, Pi lifecycle, drain, and + rollback. Raw Compose lifecycle commands bypass installation state and are unsupported. + +Read [server workspace-registry installation](server-workspace-registry.md), +[Pi management](pi-management.md), and the session-server comments in +`deploy/compose.session-server.yaml.example` before the first public start. + +## Service account and directories + +The container runtime identity is fixed at UID/GID 10001. Reserve the same host ID for a dedicated +non-login `thothii` account so bind-mounted ownership is obvious. Stop if either ID is already used +by another account; choose a reviewed host mapping instead of changing the image identity. + +```sh +getent passwd 10001 +getent group 10001 +sudo useradd --system --uid 10001 --user-group --home-dir /srv/thothii \ + --create-home --shell /usr/sbin/nologin thothii +``` + +Use a dedicated `thothii-ops` group for the small set of human operators. A human who runs +`tht` must be in both `thothii-ops` (to traverse operator paths and read declared secret files +for output redaction) and the host `docker` group (to invoke Docker). Docker-group membership is +effectively host-root access; grant both memberships only to reviewed administrators. The +non-login `thothii` account owns files and writable data but does not need Docker access. + +```sh +sudo groupadd --system thothii-ops +sudo usermod --append --groups thothii-ops,docker "$USER" +``` + +Log out and back in before continuing; `id` must show both groups. Do not run `tht` through +`sudo -u thothii`: that account deliberately lacks Docker access. Do not grant the human direct +write access to runtime bind trees. + +Create explicit directories. `source` contains the clone; `operator` contains untracked path-only +configuration; the three writable trees are bind-mounted into `core`; `secrets` contains regular +files only. Backups are separate from live data. Reset the account home explicitly because +distribution `useradd` defaults may otherwise leave `/srv/thothii` non-traversable by +`thothii-ops`. + +```sh +sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii +sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/source +sudo install -d -o 10001 -g thothii-ops -m 2770 /srv/thothii/operator +sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/secrets +sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data +sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/pi-state +sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/workspace-registry +sudo install -d -o root -g root -m 0700 /srv/thothii-backups +``` + +Verify `/srv/thothii` is owned by `10001:thothii-ops` with mode `2750`. The human operator can +traverse the parent but can write only `operator`; setgid keeps generated files in `thothii-ops`. +`source`, `secrets`, and all runtime bind trees remain non-group-writable. Do not make +`/srv/thothii` a shared application directory. + +## Firewall and network boundaries + +Set `THOTH_SERVER_BIND=127.0.0.1`. Permit inbound TCP 80/443 only to the TLS proxy; port 80 should +redirect to HTTPS. Do not open 8080 externally, and do not add a core port. If a separate proxy +host is used, replace loopback with a private, firewalled address and allow only that proxy source. + +Allow outbound DNS and HTTPS to the source/Git registries, plus only the configured ports for the +Git remote, DWH, vector database, embedding service, LLM, session PostgreSQL, and any approved +bastion. Docker's private `thothii` network carries only `frontend`↔`core` traffic. Do not attach +the mandatory stack to another application's network. + +After start, confirm the host listens as intended: + +```sh +sudo ss -lntp +``` + +Expected public listeners are the proxy on 80/443 and the frontend on loopback 8080. There must be +no host listener for core port 8787. + +## Address co-resident external services + +Endpoint values are resolved inside `core`. Therefore container 127.0.0.1 means the container +itself, not the Linux host. Prefer real DNS names with TLS, authentication, and firewall policy, +even for services on this physical server. + +When DNS is unavailable for a host-published service, create an untracked override such as +`/srv/thothii/operator/host-gateway.yaml` and add it to the installation descriptor: + +```yaml +services: + core: + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +Use `host.docker.internal` in the endpoint binding. The `host-gateway` mapping supplies routing; +it does not bundle or trust the target service. A host service listening only on host +`127.0.0.1` is **not reachable** through this mapping. Bind that service to the ThothII Docker +bridge gateway address or to a dedicated private host interface—never to `0.0.0.0` merely to make +the check pass. A stable internal DNS record routed through an authenticated private listener is +the preferred alternative. + +After the first bounded start attempt, copy the exact core container name from `tht status` +into `CORE_NAME`, then derive—not guess—the network ID, Linux bridge interface, gateway, and +subnet. Compose networks normally use `br-`; an explicit +`com.docker.network.bridge.name` option takes precedence: + +```sh +CORE_NAME=replace-with-exact-core-container-name +NETWORK_ID=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.NetworkID}}{{end}}' "$CORE_NAME") +NETWORK_NAME=$(docker network inspect --format '{{.Name}}' "$NETWORK_ID") +BRIDGE=$(docker network inspect --format '{{index .Options "com.docker.network.bridge.name"}}' "$NETWORK_ID") +test -n "$BRIDGE" || BRIDGE="br-${NETWORK_ID%${NETWORK_ID#????????????}}" +GATEWAY=$(docker network inspect --format '{{(index .IPAM.Config 0).Gateway}}' "$NETWORK_ID") +SUBNET=$(docker network inspect --format '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK_ID") +printf 'network=%s bridge=%s gateway=%s subnet=%s\n' "$NETWORK_NAME" "$BRIDGE" "$GATEWAY" "$SUBNET" +ip address show dev "$BRIDGE" +``` + +Bind the co-resident service to `$GATEWAY`. In the host firewall `INPUT` chain, allow its exact +TCP port only when source is `$SUBNET`, input interface is `$BRIDGE`, and destination is +`$GATEWAY`; reject other sources to that listener and persist the rules using the distribution's +firewall manager. Docker's `DOCKER-USER` chain governs forwarded/published traffic and does not +replace this host-input rule. Ask the firewall administrator to implement the equivalent policy +with nftables when iptables is not the site's source of truth. + +For an iptables-managed host, replace the port before applying these reviewed rules; the second +rule prevents any other interface/source from reaching that gateway listener: + +```sh +EXTERNAL_PORT=replace-with-exact-service-port +sudo iptables -I INPUT 1 -i "$BRIDGE" -s "$SUBNET" -d "$GATEWAY" -p tcp --dport "$EXTERNAL_PORT" -j ACCEPT +sudo iptables -I INPUT 2 -d "$GATEWAY" -p tcp --dport "$EXTERNAL_PORT" -j REJECT +``` + +Confirm reachability with `tht pi test` for the configured LLM/Pi path and with the +authenticated Workspace Diagnostics action for DWH, vector collection/embedding pairing, and +embedding endpoints. A timeout paired with `ss -lntp`, `ip address show dev "$BRIDGE"`, and the +firewall counters distinguishes a loopback bind from a subnet/interface rule failure. Do not add +a shell to the browser or mount the Docker socket into core for this diagnostic. + +Configure each boundary independently: + +- DWH: read-only runtime identity, database/schema, verified TLS, and direct or REST endpoint. +- Vector database: endpoint plus exact database/schema, collection, distance metric, and writer + policy declared by the reviewed workspace. +- Embedding service: endpoint and the collection/embedding pairing—model and dimensions must match + the existing collection. Co-residence does not permit silently changing that pairing. +- LLM: authenticated endpoint selected through deployment and Pi configuration. + +Never add those services to ThothII's mandatory Compose files. Follow +[the diagnostic protocol](../workspace-diagnostic-protocol.md) before enabling a workspace. + +## Prepare operator files and secrets + +Clone with LF line endings, then verify before every build: + +```sh +sudo -u thothii git -c core.autocrlf=false clone \ + https://github.example.invalid/your-org/ThothII.git /srv/thothii/source/ThothII +cd /srv/thothii/source/ThothII +sudo -u thothii git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +sudo /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh \ + /srv/thothii/pi-state 10001 10001 +``` + +The last command is a mandatory clean-install and restore preflight. The server profile bind-mounts +the writable Pi-state root and then overlays protected `auth.json` plus tracked `models.json` and +`settings.json` read-only below it. Docker requires those three hidden target files to exist under +the host parent bind before startup. The initializer creates them atomically with UID/GID 10001, +mode `0600`, rejects symlink roots or targets, and never overwrites existing contents. It is safe to rerun +after restoring `pi-state`; run it before any `tht start`, Compose render/start, or Pi update. + +Copy the path-only server environment and installation descriptor: + +```sh +sudo -u thothii cp deploy/env/server.env.example /srv/thothii/operator/server.env +sudo -u thothii cp docs/install/examples/thothii-installation.server.yaml \ + /srv/thothii/operator/thothii-installation.yaml +sudo chown 10001:thothii-ops /srv/thothii/operator/server.env \ + /srv/thothii/operator/thothii-installation.yaml +sudo chmod 0660 /srv/thothii/operator/server.env \ + /srv/thothii/operator/thothii-installation.yaml +``` + +The named human operator can now edit both placeholder files without `sudo`; use an editor that +preserves the group, or create replacements under `umask 0007` in the setgid operator directory. +Replace every placeholder with an absolute path. Use exactly one Git transport override. For +HTTPS, replace `deploy/compose.git-ssh.yaml` with `deploy/compose.git-https.yaml`. Keep the required +session-server overlay. Optional host-gateway or pinned +image overrides go after them. + +Create each installation credential (Pi/application, Git, and session storage) as an independent +regular file in `/srv/thothii/secrets`, owned by +UID 10001, group `thothii-ops`, and mode `0640`. Owner access lets the UID 10001 container read a +file mounted under `/run/secrets`; group access lets the reviewed human run `tht`. The +operator environment records only absolute `*_FILE` or `*_SOURCE` paths for those installation +credentials. DWH and Evidence values are entered later through Workspace management and persist +as ciphertext under `/data/workspace-secrets`; the frontend receives no secret values. Do not +print file contents while testing permissions. + +```sh +sudo find /srv/thothii/secrets -type f -exec chown 10001:thothii-ops {} + +sudo find /srv/thothii/secrets -type f -exec chmod 0640 {} + +sudo find /srv/thothii/secrets -type f \( ! -user thothii -o ! -group thothii-ops -o ! -perm 0640 \) -print +``` + +Configure the remote repository and exactly one read-only Git transport as described in +[server workspace repository installation](server-workspace-registry.md). After startup, complete +the selected workspace's DWH and Evidence credentials through Workspace management. Secret values +must never be pasted into `server.env`, the installation YAML, a URL, or a shell argument. + +## Build locally or select pinned images + +Choose one image source. For a source build, the repository's reproducible launcher builds the +same `core` and `frontend` images used by the local profile. It requires only Git, Docker, and +Compose; copy the reviewed path-only server environment to the launcher's untracked input first: + +```sh +cd /srv/thothii/source/ThothII +sudo -u thothii cp /srv/thothii/operator/server.env deploy/env/local.env +bash scripts/build-local.sh +``` + +The printed local-profile start command is not the server start command; use `tht` below. + +Alternatively, create a reviewed untracked override with release images pinned by immutable +digest. Mutable tags are not a production pin: + +```yaml +services: + core: + build: !reset null + image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits> + session-migrate: + build: !reset null + image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits> + frontend: + build: !reset null + image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits> +``` + +Add that absolute file last in `overrides`. `core` and `session-migrate` must use the exact same +core digest; neither may retain a local build or `:local` image. Frontend uses its own exact digest. +Both images must come from one compatible release; the core image must retain the declared Pi +version labels checked by `tht pi doctor`. Pull access +belongs in the host Docker credential store, not in Compose or the installation descriptor. + +## Install tht + +Build the operator binaries with Docker. No Go installation or Go knowledge is required: + +```sh +cd /srv/thothii/source/ThothII +THT_THT_OUTPUT_DIRECTORY=/srv/thothii/operator/build-output \ + bash scripts/build-tht.sh +sudo install -o root -g thothii-ops -m 0750 \ + /srv/thothii/operator/build-output/tht-linux-amd64 \ + /srv/thothii/operator/tht +``` + +The source checkout stays read-only to the human. The explicit output directory is the only build +write boundary; the build script rejects relative or non-canonical output paths. After installation, +remove or retain `build-output` according to the site's reviewed artifact policy. + +Use `tht-linux-arm64` on an ARM64 server. Set these variables in the maintenance shell; do +not source `server.env` as shell code: + +```sh +THT_BIN=/srv/thothii/operator/tht +INSTALLATION=/srv/thothii/operator/thothii-installation.yaml +"$THT_BIN" --help +"$THT_BIN" --installation "$INSTALLATION" update --check-only +``` + +Every operator command includes the descriptor explicitly. This preserves the installation's +profile, overrides, project identity, and durable current-image selector. The general form is +`tht --installation /absolute/path/thothii-installation.yaml `. + +## Start and verify readiness + +Keep the TLS proxy stopped or firewalled during bootstrap. First stop the app, run the +installation-aware session migration, and inspect its pristine JSON. The command activates only +the `session-migrate` profile/service with `--no-deps --no-TTY`; it derives the migrator image from the +selected core image after all installation overrides, so this procedure is identical for source +and pinned modes. It exits nonzero unless both arrays are empty: + +```sh +"$THT_BIN" --installation "$INSTALLATION" stop +"$THT_BIN" --installation "$INSTALLATION" sessions migrate --yes +``` + +Successful output has this shape (the `applied` list may contain versions on first use): + +```json +{"applied":[],"drifted":[],"pending":[]} +``` + +Only after seeing `"pending":[]` and `"drifted":[]`, start and verify: + +```sh +"$THT_BIN" --installation "$INSTALLATION" start +"$THT_BIN" --installation "$INSTALLATION" status +"$THT_BIN" --installation "$INSTALLATION" doctor +curl --fail http://127.0.0.1:8080/health +"$THT_BIN" --installation "$INSTALLATION" pi doctor +"$THT_BIN" --installation "$INSTALLATION" pi test +``` + +`/health` proves process liveness. Readiness additionally requires both healthy services, a valid +Pi provider/model smoke, a successful Git registry pull with an active validated snapshot, valid +workspace diagnostics, and ready session PostgreSQL. Use the authenticated Workspace Management +page to pull and diagnose the reviewed workspace. A liveness response alone is not release +approval. + +After configuring the proxy, open in a browser. Verify an unauthenticated +request is denied or redirected by the real identity provider, an authorized user can load the +same-origin UI and `/api`, an unauthorized user is denied, and an administrator alone can open Pi +Management. Keep port 8080 inaccessible from other hosts. + +## Configure TLS and upstream authentication + +Choose [Nginx](reverse-proxy-nginx.md) or [Caddy](reverse-proxy-caddy.md). Both examples terminate +TLS and proxy only to loopback `frontend`. They preserve SSE and clear client-supplied identity +headers before authentication. + +The authentication gateway must validate a real login/session and return normalized issuer, +subject, display-name, and admin claims only after success. Merely forwarding those headers does +not authenticate anyone. Do not enable `AUTH_MODE=upstream` on a listener reachable around the +trusted proxy, and never expose `core`. + +## Operate Pi, drain, and roll back + +Configure only closed provider/model/reasoning choices. Credentials remain protected files: + +```sh +"$THT_BIN" --installation "$INSTALLATION" pi status +"$THT_BIN" --installation "$INSTALLATION" pi configure +"$THT_BIN" --installation "$INSTALLATION" pi doctor +"$THT_BIN" --installation "$INSTALLATION" pi logs +``` + +Before an update, announce maintenance and ask users to finish active work. `--drain` closes new +admission and waits until no active sessions remain; it does not discard sessions. Build-source +and registry-source examples are: + +```sh +"$THT_BIN" --installation "$INSTALLATION" pi update \ + --version 0.81.0 --source build --yes --drain +"$THT_BIN" --installation "$INSTALLATION" pi update \ + --version 0.81.0 --source pull \ + --image registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits> \ + --yes --drain +``` + +The transaction recreates only `core`, preserves volumes, verifies health/configuration/Pi, and +automatically attempts rollback after a post-mutation failure. For interrupted or ambiguous state: + +```sh +"$THT_BIN" --installation "$INSTALLATION" pi maintenance status +"$THT_BIN" --installation "$INSTALLATION" pi rollback --yes +"$THT_BIN" --installation "$INSTALLATION" pi maintenance recover --yes +``` + +Leave maintenance active if rollback cannot be verified. Preserve `.tht//` +recovery state, repair the reported host/configuration issue, and rerun rollback or maintenance +recovery. Never delete or edit `current-image.yaml` or `update-state.json` to force progress. + +## Back up and restore + +Back up before source, workspace, session-schema, or Pi changes. Drain work, stop the installation, +record `git rev-parse HEAD`, image digests, and `tht status`, then archive the three bind trees +with numeric ownership. Do not include live secrets in this ordinary archive. + +```sh +"$THT_BIN" --installation "$INSTALLATION" stop +BACKUP=/srv/thothii-backups/2026-08-05 +sudo install -d -o root -g root -m 0700 "$BACKUP" +sudo tar --numeric-owner --xattrs --acls -C /srv/thothii -czf "$BACKUP/runtime-data.tgz" \ + data pi-state workspace-registry +sudo sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$BACKUP" +``` + +Back up the installation descriptor, path-only environment, generated overrides, source revision, +and secret files to separate encrypted access-controlled storage. Database-backed production +sessions require their own PostgreSQL-native consistent backup; the local bind tree is not a +substitute. Test both restore paths periodically. + +Restore only while stopped. Verify the checksum, extract first into a new empty root, inspect +ownership and expected registry layout, then retain the old trees by renaming them before placing +the restored set. This keeps the previous state recoverable: + +```sh +RESTORE=/srv/thothii-restore-2026-08-05 +sudo install -d -o root -g root -m 0700 "$RESTORE" +sudo sh -ceu 'cd "$1"; sha256sum --check SHA256SUMS' sh /srv/thothii-backups/2026-08-05 +sudo tar --numeric-owner --xattrs --acls -C "$RESTORE" \ + -xzf /srv/thothii-backups/2026-08-05/runtime-data.tgz +sudo test -d "$RESTORE/workspace-registry/repo" +sudo test -d "$RESTORE/workspace-registry/snapshots" +``` + +After placing the restored `pi-state` tree and before the first start, rerun +`sudo /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001`. +It validates or recreates only the hidden regular mount targets; it does not alter restored Pi +state or any protected configuration source. + +During the reviewed restore window, move each old tree to a timestamped sibling, move the matching +restored tree into `/srv/thothii`, restore the PostgreSQL session backup from the same recovery +point, and keep the proxy closed. Run `update --check-only`, `start`, `doctor`, `pi test`, registry +status, workspace diagnostics, and a known historical session before reopening traffic. Never +merge an archive into a non-empty tree. + +## Diagnostics + +Begin with bounded, sanitized installation-aware commands: + +```sh +"$THT_BIN" --installation "$INSTALLATION" status +"$THT_BIN" --installation "$INSTALLATION" doctor +"$THT_BIN" --installation "$INSTALLATION" logs +"$THT_BIN" --installation "$INSTALLATION" pi status +"$THT_BIN" --installation "$INSTALLATION" pi doctor +"$THT_BIN" --installation "$INSTALLATION" pi test +"$THT_BIN" --installation "$INSTALLATION" pi logs +"$THT_BIN" --installation "$INSTALLATION" pi maintenance status +``` + +Use the authenticated Workspace Management status and diagnostic actions for Git revision, +degraded snapshot, bindings, DWH, vector, and embedding checks. Review proxy logs separately, but +configure both proxy and log shipping to exclude cookies, authorization data, identity payloads, +query strings, and secret values. Do not render Compose or print an environment as a diagnostic. + +Typical boundaries are: `doctor` for Docker/Compose/LF/volume/service health; `pi doctor` for image +and provider/model integrity; registry status for Git/snapshot health; workspace diagnostics for +external service identity; and the proxy/identity provider for login failures. + +## Data-preserving uninstall + +Drain and stop through `tht`, take and verify one final backup, and disable the TLS proxy +route. Set `THT_BACKUP_ROOT=/srv/thothii-backups` in `server.env`; the removal command verifies the +filesystem identity of that backup root, all three bind trees, and every declared secret before +and after removing anything. + +First run without confirmation. It displays the exact installation project, service, container +name, container ID, and stopped state, then exits without mutation. Check every target: + +```sh +"$THT_BIN" --installation "$INSTALLATION" stop +"$THT_BIN" --installation "$INSTALLATION" remove +``` + +If and only if both targets are the expected stopped `frontend` and `core` containers, confirm: + +```sh +"$THT_BIN" --installation "$INSTALLATION" remove --yes exact-core-id exact-frontend-id +``` + +Replace both example IDs with the values from the immediately preceding dry-run. The command +refuses confirmation if the current target set differs. The confirmed operation passes only those +previously displayed immutable container IDs to Docker, +uses no force or volume option, rejects running/replaced containers, and proves the preservation +paths still identify the same filesystem objects. Keep `/srv/thothii/data`, `pi-state`, +`workspace-registry`, `operator`, protected secrets, database backups, and the installation +descriptor if reinstallation is possible. Do not prune global Docker data. + +Do **not** run `docker compose down --volumes`; it deletes persistent application data. Reusing the +same protected descriptor path preserves the `tht` installation identity and allows a later +compatible source checkout to reconnect the retained state. diff --git a/docs/install/windows-line-endings.md b/docs/install/windows-line-endings.md new file mode 100644 index 00000000..5be4a31f --- /dev/null +++ b/docs/install/windows-line-endings.md @@ -0,0 +1,250 @@ +# Windows and WSL2 line endings + +ThothII's containers execute shell scripts from the source checkout. Those files must stay LF, +even when the PC normally uses CRLF. The repository's `.gitattributes` is authoritative, but a +Windows Git setting or an old checkout can still leave incorrect bytes. Check line endings after +every clone and pull, before building an image. + +## Recommended WSL2 clone + +Use Docker Desktop with WSL2 integration. Clone inside the Linux filesystem, for example under +`/home//src`, rather than under `/mnt/c`. This avoids slow cross-filesystem builds, +permission surprises, and Windows tools rewriting files behind WSL. + +```sh +mkdir -p "$HOME/src" +cd "$HOME/src" +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +cd ThothII +git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +``` + +Keep Docker Desktop's integration enabled for that WSL distribution. Run the Linux build scripts +and the Linux `tht` binary from the same WSL shell. + +## Repository-local LF policy + +Set the option in this repository only. Do not change a company-wide or personal Git policy just +for ThothII. + +```sh +git config --local core.autocrlf false +git config --local --get core.autocrlf +``` + +The second command must print `false`. `.gitattributes` keeps shell, YAML, Dockerfile, JSON, +TypeScript, Python, and Markdown files at LF; PowerShell files remain CRLF. + +For a native PowerShell clone, disable conversion during the first checkout and then store the +repository-local setting: + +```powershell +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +Set-Location ThothII +git config --local core.autocrlf false +& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh +``` + +## Verify after clone or pull + +From WSL2, Git Bash, macOS, or Linux run: + +```sh +bash scripts/verify-line-endings.sh +``` + +Success exits with code 0 and prints no offending path. If it lists a file, do not build or start +ThothII. Correct the checkout first. Native PowerShell users can invoke the same script through +Git for Windows as shown above. + +## Recover an existing CRLF clone + +The safest recovery is to reclone into a new directory. First commit wanted work or copy it to a +backup outside both clones. Then clone with conversion disabled, run the verifier, and copy back +only reviewed changes. + +If a reviewed working tree must be repaired in place, Git must first normalize the index, export +that exact index to a separate repair directory, verify the exported bytes, and only then copy the +verified tracked files over the worktree. `git add --renormalize .` alone does not change existing +worktree bytes. + +> **WARNING — destructive worktree rewrite.** Make a backup outside the clone or commit every +> wanted tracked change before continuing. The copy step below overwrites tracked worktree bytes +> from the staged index export. Stop if the staged diff does not contain exactly the wanted content; +> untracked files are neither exported nor repaired. + +From WSL2, Git Bash, macOS, or Linux: + +```sh +set -euo pipefail + +abort_repair() { printf 'CRLF repair stopped: %s\n' "$1" >&2; exit 1; } +validate_index_export() { + git ls-files -s -z | while IFS= read -r -d '' entry; do + metadata="${entry%%$'\t'*}" + path="${entry#*$'\t'}" + mode="${metadata%% *}" + [[ "$path" != "$entry" ]] || exit 1 + case "$mode" in + 100644|100755) [[ -f "$REPAIR_DIR/$path" && ! -L "$REPAIR_DIR/$path" ]] || exit 1 ;; + 120000) [[ -L "$REPAIR_DIR/$path" ]] && readlink "$REPAIR_DIR/$path" >/dev/null || exit 1 ;; + *) printf 'Unsupported Git mode %s: %s\n' "$mode" "$path" >&2; exit 1 ;; + esac + done +} +validate_worktree_modes() { + git ls-files -s -z | while IFS= read -r -d '' entry; do + metadata="${entry%%$'\t'*}" + path="${entry#*$'\t'}" + mode="${metadata%% *}" + case "$mode" in + 100644|100755) [[ -f "$path" && ! -L "$path" ]] || exit 1 ;; + 120000) [[ -L "$path" ]] && readlink "$path" >/dev/null || exit 1 ;; + *) exit 1 ;; + esac + done +} +rewrite_index_entry() { + local mode="$1" path="$2" target temporary_link + case "$mode" in + 100644) + cp "$REPAIR_DIR/$path" "$path" && chmod a-x "$path" + ;; + 100755) + cp "$REPAIR_DIR/$path" "$path" && chmod a+x "$path" + ;; + 120000) + target="$(readlink "$REPAIR_DIR/$path")" || return 1 + temporary_link="${path}.thoth-lf-repair-link" + [[ ! -e "$temporary_link" && ! -L "$temporary_link" ]] || return 1 + ln -s "$target" "$temporary_link" || return 1 + rm -f "$path" || { rm -f "$temporary_link"; return 1; } + mv "$temporary_link" "$path" + ;; + *) return 1 ;; + esac +} + +if ! git status --short; then abort_repair "git status failed"; fi +if ! git config --local core.autocrlf false; then abort_repair "could not set repository LF policy"; fi +if ! git add --renormalize .; then abort_repair "index renormalization failed"; fi +if ! git diff --cached --check; then abort_repair "normalized index check failed"; fi +if ! git diff --cached; then abort_repair "normalized index review failed"; fi +REPAIR_DIR="$(cd .. && pwd -P)/ThothII-lf-repair" +if [[ -e "$REPAIR_DIR" ]]; then + abort_repair "choose a new empty LF repair directory: $REPAIR_DIR" +fi +if ! mkdir -p "$REPAIR_DIR"; then abort_repair "could not create LF repair directory"; fi +REPAIR_PREFIX="$REPAIR_DIR/" +if ! git checkout-index --all --force --prefix="$REPAIR_PREFIX"; then abort_repair "index export failed"; fi +if ! validate_index_export; then abort_repair "index export is missing entries or Git modes"; fi +if ! bash scripts/verify-line-endings.sh "$REPAIR_DIR"; then abort_repair "exported bytes failed LF verification"; fi +# WARNING: destructive copy; make a backup or commit wanted changes before this command. +if ! git ls-files -s -z | while IFS= read -r -d '' entry; do + metadata="${entry%%$'\t'*}" + path="${entry#*$'\t'}" + mode="${metadata%% *}" + rewrite_index_entry "$mode" "$path" || exit 1 +done; then + abort_repair "tracked-file rewrite failed; do not build from this worktree" +fi +if ! validate_worktree_modes; then abort_repair "repaired worktree does not match Git index modes"; fi +if ! bash scripts/verify-line-endings.sh; then abort_repair "repaired worktree failed LF verification"; fi +if ! git diff --cached --check; then abort_repair "repaired index check failed"; fi +``` + +Native Windows PowerShell runs the same Git operations and invokes the byte verifier through Git +for Windows: + +```powershell +$ErrorActionPreference = 'Stop' +function Assert-NativeSuccess([string]$Step) { + if ($LASTEXITCODE -ne 0) { throw "$Step failed with exit code $LASTEXITCODE." } +} +function ConvertFrom-IndexEntry([string]$Entry) { + if ($Entry -notmatch '^([0-9]{6}) [0-9a-f]+ [0-3]\t(.+)$') { + throw "Invalid Git index entry: $Entry" + } + [pscustomobject]@{ Mode = $Matches[1]; Path = $Matches[2] } +} + +git status --short +Assert-NativeSuccess 'git status' +git config --local core.autocrlf false +Assert-NativeSuccess 'repository LF policy' +git add --renormalize . +Assert-NativeSuccess 'index renormalization' +git diff --cached --check +Assert-NativeSuccess 'normalized index check' +git diff --cached +Assert-NativeSuccess 'normalized index review' +$RepairDir = Join-Path (Split-Path -Parent (Get-Location).Path) 'ThothII-lf-repair' +if (Test-Path $RepairDir) { throw 'Choose a new empty LF repair directory.' } +New-Item -ItemType Directory -Path $RepairDir | Out-Null +$RepairPrefix = $RepairDir.Replace('\', '/') + '/' +git -c core.symlinks=true checkout-index --all --force --prefix=$RepairPrefix +Assert-NativeSuccess 'index export' +$RawIndexEntries = @(git ls-files -s) +Assert-NativeSuccess 'index inventory' +$IndexEntries = @($RawIndexEntries | ForEach-Object { ConvertFrom-IndexEntry $_ }) +foreach ($Entry in $IndexEntries) { + $ExportPath = Join-Path $RepairDir $Entry.Path + $ExportItem = Get-Item -LiteralPath $ExportPath -Force -ErrorAction Stop + switch ($Entry.Mode) { + { $_ -in '100644', '100755' } { + if ($ExportItem.LinkType -eq 'SymbolicLink') { throw "Regular export became a symlink: $($Entry.Path)" } + } + '120000' { + if ($ExportItem.LinkType -ne 'SymbolicLink') { throw "Symlink export is not mode 120000: $($Entry.Path)" } + if ([string]::IsNullOrWhiteSpace([string]$ExportItem.Target)) { throw "Symlink target is empty: $($Entry.Path)" } + } + default { throw "Unsupported Git mode $($Entry.Mode): $($Entry.Path)" } + } +} +& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh $RepairDir +Assert-NativeSuccess 'exported byte LF verification' +# WARNING: destructive copy; make a backup or commit wanted changes before this command. +foreach ($Entry in $IndexEntries) { + $ExportPath = Join-Path $RepairDir $Entry.Path + switch ($Entry.Mode) { + { $_ -in '100644', '100755' } { + Copy-Item -LiteralPath $ExportPath -Destination $Entry.Path -Force -ErrorAction Stop + } + '120000' { + $LinkTarget = [string](Get-Item -LiteralPath $ExportPath -Force -ErrorAction Stop).Target + $TemporaryLink = "$($Entry.Path).thoth-lf-repair-link" + if (Test-Path -LiteralPath $TemporaryLink) { throw "Temporary symlink path exists: $TemporaryLink" } + New-Item -ItemType SymbolicLink -Path $TemporaryLink -Target $LinkTarget -ErrorAction Stop | Out-Null + Remove-Item -LiteralPath $Entry.Path -Force -ErrorAction Stop + Move-Item -LiteralPath $TemporaryLink -Destination $Entry.Path -ErrorAction Stop + } + default { throw "Unsupported Git mode $($Entry.Mode): $($Entry.Path)" } + } +} +foreach ($Entry in $IndexEntries) { + $WorktreeItem = Get-Item -LiteralPath $Entry.Path -Force -ErrorAction Stop + switch ($Entry.Mode) { + { $_ -in '100644', '100755' } { + if ($WorktreeItem.LinkType -eq 'SymbolicLink') { throw "Regular worktree entry became a symlink: $($Entry.Path)" } + } + '120000' { + if ($WorktreeItem.LinkType -ne 'SymbolicLink') { throw "Repaired worktree symlink is not mode 120000: $($Entry.Path)" } + if ([string]::IsNullOrWhiteSpace([string]$WorktreeItem.Target)) { throw "Repaired symlink target is empty: $($Entry.Path)" } + } + default { throw "Unsupported Git mode $($Entry.Mode): $($Entry.Path)" } + } +} +& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh +Assert-NativeSuccess 'repaired worktree LF verification' +git diff --cached --check +Assert-NativeSuccess 'repaired index check' +``` + +The export inventory must contain every regular mode (`100644`/`100755`) and recreate every tracked +workspace compatibility symlink (`120000`). The first verifier proves the complete +export before any overwrite; every copy/link operation is fail-closed; the final verifier examines +the repaired worktree bytes. On native Windows, creating symlinks requires Developer Mode or an +elevated account; failure stops the rewrite. Review the staged diff again before committing, then +remove the separate repair directory only after inspecting it. The procedure intentionally avoids +`git reset --hard`; replacing the clone is easier to audit and safer for uncommitted work. diff --git a/docs/installazione-docker-4-contesti.md b/docs/installazione-docker-4-contesti.md index b268bb4f..08cd5db5 100644 --- a/docs/installazione-docker-4-contesti.md +++ b/docs/installazione-docker-4-contesti.md @@ -1,234 +1,87 @@ -# Installazione Docker nei quattro contesti operativi +# Installazione Docker nei contesti operativi correnti -ThothII viene distribuito con due immagini applicative: +ThothII usa una topologia Compose unica: -- `thothii-core`: backend Fastify, harness `tht` e Pi; -- `thothii-frontend`: frontend React servito da nginx. +- `frontend` +- `core` +- `qdrant` +- `embedding` +- `embedding-model-init` -PostgreSQL/pgvector, DWH ed Evidence restano esterni nel profilo predefinito. Il profilo opzionale `local-vector` avvia PostgreSQL/pgvector nel progetto Compose. +Qdrant e Ollama embedding sono servizi interni obbligatori del progetto Compose. Restano esterni solo DWH e LLM. Il modello fissato è `qwen3-embedding:0.6b` con 1024 dimensioni e distanza +coseno; `embedding-model-init` lo prepara prima dell'avvio di `core`. -## Installazione comune (il comando standard) +## Contratto sintetico di ownership -Servono Docker Engine/Compose v2 su Linux oppure Docker Desktop su macOS/Windows. Dalla directory in cui si vuole conservare il clone: +| Componente | Ownership | Contratto operativo | +| --- | --- | --- | +| DWH | Esterno | Endpoint esterno configurato dall'installazione. | +| LLM | Esterno | Endpoint o policy esterna all'infrastruttura semantica interna. | +| Qdrant | Interno | Servizio Compose interno obbligatorio con volume persistente `qdrant-data`. | +| Ollama embedding | Interno | Servizio Compose interno obbligatorio per `qwen3-embedding:0.6b`. | + +## Comando standard locale ```sh -git clone ThothII -cd ThothII -cp .env.example .env -mkdir -p deploy/secrets deploy/workspaces +cp deploy/env/local.env.example deploy/env/local.env cp deploy/secrets/thothii.secrets.example deploy/secrets/thothii.secrets chmod 600 deploy/secrets/thothii.secrets + +docker compose --env-file deploy/env/local.env \ + -f compose.yaml -f deploy/compose.local.yaml up --build -d ``` -Modificare **solo** questi file interni al clone: +Compilare `deploy/env/local.env` con: -| File | Cosa contiene | -|---|---| -| `.env` | endpoint, database, provider, `COMPOSE_FILE` e `COMPOSE_PROFILES`; mai password/token | -| `deploy/secrets/thothii.secrets` | un bundle `NOME=VALORE`, mode host `0600` o `0400` | -| `deploy/workspaces/.yaml` | adapter, endpoint non riservati, `roots` ed Evidence | +- `PI_AUTH_FILE` +- `THT_SECRETS_FILE` +- `THT_WORKSPACE_GIT_REMOTE` +- endpoint DWH +- endpoint LLM -Il file `.env` viene caricato automaticamente da Docker Compose perché è nella radice del progetto. Il valore predefinito è `COMPOSE_FILE=compose.yaml`, con profili vuoti e `THT_SECRETS_FILE=deploy/secrets/thothii.secrets`. Perciò, dopo aver compilato `.env`, il bundle e almeno il workspace, l'avvio normale è sempre: +Non inserire secret nel file `.env`. I secret runtime stanno nel bundle +`deploy/secrets/thothii.secrets`. -```sh -docker compose up --build -d -``` +## Bundle dei secret -Non occorre usare `--env-file`, `-f` o `--profile` per questa installazione. Verificare lo stato con `docker compose ps` e aprire . `docker compose down` conserva il volume `thoth_data`; usare `down --volumes` solo per un ambiente effimero. - -### Formato del bundle unico - -`deploy/secrets/thothii.secrets` è un file di testo locale, non uno script shell. Sono ammessi commenti e righe vuote; ogni altra riga deve essere una sola assegnazione senza spazi: +Le chiavi documentate e supportate nel bundle sono: ```dotenv THT_MODEL_API_KEY=... THT_DWH_API_KEY=... -THT_VEC_API_KEY=... -THT_VEC_WRITE_API_KEY=... -THT_VECTOR_BOOTSTRAP_PASSWORD=... -THT_VECTOR_MIGRATOR_PASSWORD=... -THT_VECTOR_READER_PASSWORD=... -THT_VECTOR_WRITER_PASSWORD=... ``` -Inserire solo le chiavi necessarie al profilo scelto. Il bundle viene montato in sola lettura nel container come `/run/secrets/thothii.secrets`; il parser rifiuta duplicati, chiavi sconosciute, valori vuoti, symlink e permessi host troppo aperti. Non inserire secret in `.env`, nei workspace, negli URL o nell'output di `docker compose config`. +Una CA privata PEM resta esterna al bundle e va montata con un override Compose revisionato. -Una catena CA PEM **non può essere inserita nel bundle**: contiene whitespace e viene rifiutata dal parser. Se un endpoint usa una CA privata, conservarla nel secret manager/host e aggiungere un override Compose revisionato che monti il file in `/run/secrets/ca-chain.pem` e imposti `THT_SSL_CA` (o il parametro dell'adapter). Il clone base non crea quel mount: questa è una limitazione intenzionale da considerare in fase di deployment. +## Preprocessing -### Overlay opzionali tramite `.env` - -Gli overlay non cambiano il comando operativo. Impostare in `.env`: - -```dotenv -# DWH/vector/embedding remoti (server applicativo o server con i DB): -COMPOSE_FILE=compose.yaml:deploy/compose.production.yaml -COMPOSE_PROFILES= - -# pgvector locale (Mac, Windows o server autonomo): -COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml -COMPOSE_PROFILES=local-vector -``` - -Su Windows usare `;` come separatore di `COMPOSE_FILE`. Per il preprocessing locale aggiungere `deploy/compose.preprocess.yaml:deploy/compose.preprocess-local-vector.yaml` e impostare `COMPOSE_PROFILES=local-vector,preprocess`; poi usare `docker compose run --rm preprocess-evidence` oppure `docker compose run --rm preprocess-dwh`. - -## Workspace, adapter e Evidence - -Il workspace YAML seleziona il trasporto disponibile. Esempio DWH REST e vector DB HTTP: - -```yaml -language: en -dwh: - type: thoth_rest - database: {database: warehouse, schema: datawarehouse} - endpoint: {base_url: https://dwh.example.test} -vectors: - type: thoth_vector_http - reader: {base_url: https://vectors.example.test} - writer: {base_url: https://vectors.example.test} -roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} -evidence: {source_root: /data/source, evidence_dir: evidence} -embeddings: {base_url: https://embeddings.example.test, model: nomodel, dim: 768} -``` - -Esempio con accesso diretto a PostgreSQL e pgvector: - -```yaml -language: en -dwh: - type: postgres_direct - connection: {host: dwh.internal, database: warehouse, schema: public, - user: thoth_reader, password_file: /run/secrets/dwh_password} -vectors: - type: pgvector_direct - reader: {host: vector.internal, database: thoth, schema: vectors, - user: thoth_vector_reader, password_file: /run/secrets/vector_reader_password} - writer: {host: vector.internal, database: thoth, schema: vectors, - user: thoth_vector_writer, password_file: /run/secrets/vector_writer_password} -roots: {artifacts: artifacts, indexes: indexes, sessions: sessions} -``` - -Questo esempio mostra il contratto dell'adapter: i file indicati da `password_file` devono -essere montati da un override Compose approvato. Il profilo base monta soltanto il bundle unico; -per un DWH diretto occorre quindi materializzare il file password dal secret manager e aggiungere -il bind mount/runtime adapter corrispondente. Non inserire la password nel workspace o nell'URL. - -`roots` sono relativi e vengono risolti sotto `/data/workspaces/` nel volume Docker; non inserire path host come `/Users/...` o `C:\\...`. Per Evidence usare una radice filesystem montata in sola lettura oppure l'adapter HTTP/S3 previsto dal workspace. Per HTTP/S3 definire allowlist, limiti di dimensione/paginazione e una politica egress; non mettere token nelle URI. - -## 1. Server remoto insieme ai database e al vector DB - -Usare quando il server Docker è nella stessa rete del DWH e del vector DB (containerizzati o meno). Il file `.env` può restare sul default, senza profili, impostando gli endpoint raggiungibili localmente: - -```dotenv -COMPOSE_FILE=compose.yaml -COMPOSE_PROFILES= -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.internal.example -THT_VEC_REST_URL=https://vectors.internal.example -THT_OLLAMA_URL=https://embeddings.internal.example -AUTH_MODE=none -THOTH_PUBLIC_EXPOSURE=false -``` - -Riempire nel bundle le chiavi DWH/vector/model necessarie e avviare: +I job di preprocessing usano gli stessi servizi interni Qdrant/Ollama: ```sh -docker compose up --build -d -docker compose exec core /opt/venv/bin/tht doctor --json +docker compose --env-file deploy/env/local.env \ + -f compose.yaml -f deploy/compose.local.yaml \ + -f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-evidence ``` -Se si abilita l'overlay production, il proxy autenticato TLS deve essere l'unico listener pubblico -e deve iniettare `X-Authenticated-User`; non esporre direttamente la porta pubblicata da nginx. - -Se il server deve essere raggiungibile da altri host, sostituire `COMPOSE_FILE` con -`compose.yaml:deploy/compose.production.yaml`, configurare il proxy autenticato e impostare -`AUTH_MODE=upstream`/`THOTH_PUBLIC_EXPOSURE=true` come descritto nella sezione di trust boundary. - -## 2. Mac locale - -Installare Docker Desktop e, se usato, Ollama sul Mac. Nel `.env` selezionare il profilo locale: - -```dotenv -COMPOSE_FILE=compose.yaml:deploy/compose.local-vector.yaml -COMPOSE_PROFILES=local-vector -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.example.test -THT_OLLAMA_URL=http://host.docker.internal:11434 -THT_DOCS_ROOT=/data/source/evidence -``` - -Nel bundle aggiungere quattro password generate localmente: - -```dotenv -THT_VECTOR_BOOTSTRAP_PASSWORD= -THT_VECTOR_MIGRATOR_PASSWORD= -THT_VECTOR_READER_PASSWORD= -THT_VECTOR_WRITER_PASSWORD= -``` - -Poi eseguire il comando standard `docker compose up --build -d`. Il primo avvio esegue reconciliation dei ruoli e migrazione pgvector. Per preprocessing, impostare il preset indicato sopra e usare `docker compose run --rm preprocess-evidence`/`preprocess-dwh`. - -## 3. PC Windows locale - -Usare Docker Desktop con backend WSL2 e abilitare la condivisione della directory del clone. Modificare `.env` con il separatore Windows: - -```dotenv -COMPOSE_FILE=compose.yaml;deploy/compose.local-vector.yaml -COMPOSE_PROFILES=local-vector -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.example.test -THT_OLLAMA_URL=http://host.docker.internal:11434 -THT_DOCS_ROOT=/data/source/evidence -``` - -Creare `deploy/secrets/thothii.secrets` con un editor locale protetto (ACL leggibile solo dall'utente Docker) e le stesse quattro chiavi pgvector del profilo Mac. Non usare `ConvertFrom-SecureString`: il bundle deve contenere il valore in chiaro per il servizio, con accesso limitato al file. Da PowerShell, dalla radice del clone, eseguire: - -```powershell -docker compose up --build -d -docker compose ps -``` - -Se un bind mount viene rifiutato, aggiungere la cartella del repository a Docker Desktop → Settings → Resources → File Sharing. Per Ollama eseguito in WSL2 usare l'indirizzo raggiungibile dalla rete Docker invece di assumere `localhost`. - -## 4. Server applicativo distinto da DB ed Evidence - -Usare il profilo production e consentire dal firewall solo le destinazioni necessarie: - -```dotenv -COMPOSE_FILE=compose.yaml:deploy/compose.production.yaml -COMPOSE_PROFILES= -THT_DB_NAME=warehouse -THT_DWH_REST_URL=https://dwh.example.test -THT_VEC_REST_URL=https://vectors.example.test -THT_OLLAMA_URL=https://embeddings.example.test -``` - -Il DWH e il vector DB possono essere REST/HTTP oppure adapter diretti (`postgres_direct`, `pgvector_direct`) se il server ha connettività TCP. Le Evidence possono essere: - -- filesystem NFS/SMB montato sul server e presentato come root read-only; -- endpoint HTTPS, con allowlist e limiti SSRF; -- bucket S3 con secret references e endpoint custom esplicitamente autorizzati. - -Il preprocessing può girare sul server applicativo usando il volume `/data`; mantenere separati workspace, lock e artefatti dei job. Avviare con il comando standard e verificare `tht doctor`. - -## Migrazione da installazioni con secret separati - -Le variabili `THT_*_SECRET_FILE` e i file `dwh-api-key`, `vector-reader-api-key`, `vector-writer-api-key`, `model-api-key` e `vector_*_password` appartengono al layout precedente. Non vengono importati automaticamente dal bundle. Per migrare: - -1. creare `deploy/secrets/thothii.secrets` mode `0600`; -2. copiare ogni valore nel nome chiave corrispondente (`THT_DWH_API_KEY`, `THT_VEC_API_KEY`, `THT_VEC_WRITE_API_KEY`, `THT_MODEL_API_KEY` o `THT_VECTOR_*_PASSWORD`), senza virgolette né newline; -3. rimuovere dal `.env` le variabili `_SECRET_FILE` e impostare `THT_SECRETS_FILE` al percorso del bundle (il default relativo è già corretto); -4. eseguire `docker compose config --quiet` e poi `docker compose up --build -d`; -5. solo dopo la verifica, cancellare i vecchi file separati. - -Una CA PEM resta un'eccezione esterna come descritto sopra. Provider Pi con credenziali composte (Bedrock, Azure OpenAI Responses, Cloudflare Workers AI/Gateway) restano rifiutati finché non viene implementato un adapter dedicato. - -## Controlli post-installazione +Per introspezione DWH: ```sh -docker compose config --quiet -docker compose ps -docker compose exec core /opt/venv/bin/tht doctor --json -./scripts/docker-smoke.sh +docker compose --env-file deploy/env/local.env \ + -f compose.yaml -f deploy/compose.local.yaml \ + -f deploy/compose.preprocess.yaml --profile preprocess run --rm preprocess-dwh ``` -Per il profilo locale usare anche `./scripts/local-vector-smoke.sh`; per il preprocessing `./scripts/preprocess-smoke.sh`. Non pubblicare `.env` o `deploy/secrets/thothii.secrets` nei log, nei backup Git o nei ticket. +## Server + +Per installazioni server usare il profilo server con overlay sessioni: + +```sh +docker compose --env-file deploy/env/server.env \ + -f compose.yaml -f deploy/compose.server.yaml \ + -f deploy/compose.session-server.yaml.example up --build -d +``` + +Consultare anche: + +- `docs/install/local-workspace-registry.md` +- `docs/install/server-workspace-registry.md` diff --git a/docs/migrations/p1-to-p1-1-registry-layout.md b/docs/migrations/p1-to-p1-1-registry-layout.md new file mode 100644 index 00000000..75a272c3 --- /dev/null +++ b/docs/migrations/p1-to-p1-1-registry-layout.md @@ -0,0 +1,38 @@ +# P1 to P1.1 registry layout migration + +P1.1 is a repository-contract cutover. New ThothII builds reject the old flat layout and a +repository without `thoth-workspaces.yaml`, so migrate the registry in Git first and upgrade the +application only after that reviewed migration commit is pushed. + +## One reviewed migration commit + +Perform the layout move in a clean review clone and keep it in one reviewed Git commit: + +```sh +git mv workspaces/.yaml /workspace.yaml +git mv workspace-content//evidence /evidence +# create and review thoth-workspaces.yaml from descriptor metadata +``` + +For every workspace directory, preserve the existing descriptor bytes, move only the embedded +filesystem Evidence tree, and create `thoth-workspaces.yaml` with: + +- `schema_version: 1` +- the ordered `workspaces` list +- curator-owned `id`, `name`, and optional `description` copied from the reviewed descriptors + +Generated docs remain under `workspace-docs//`. Do not add an auto-migrator and do not let the +API rewrite the catalog or Evidence tree. + +## Cutover order + +1. Review the migration commit, including the new `thoth-workspaces.yaml` metadata. +2. Push that commit to the authoritative registry branch. +3. Upgrade ThothII only after that migration commit is pushed. +4. Pull the migrated registry into each installation before using workspace management. + +## Rollback + +Roll back the application revision and registry commit together. Do not point a P1.1 binary at the +old flat layout, and do not keep a migrated registry commit active while rolling the application +back to pre-P1.1 code. diff --git a/docs/plans/2026-08-08-internal-qdrant-ollama-design.md b/docs/plans/2026-08-08-internal-qdrant-ollama-design.md new file mode 100644 index 00000000..b6965b36 --- /dev/null +++ b/docs/plans/2026-08-08-internal-qdrant-ollama-design.md @@ -0,0 +1,210 @@ +# Internal Qdrant and Ollama Architecture Design + +**Status:** approved on 2026-08-08 + +## Objective + +ThothII owns its semantic infrastructure. Every supported deployment includes a private Qdrant +service and a private Ollama embedding service. The analytical DWH remains external and read-only; +each workspace descriptor associates that DWH with one Qdrant collection used for database schema, +Evidence, and approved Memory records. + +## Decisions + +- Qdrant replaces pgvector as the only operational vector store. +- Ollama replaces workspace-selected external embedding endpoints. +- The default and required model is `qwen3-embedding:0.6b` with 1024-dimensional normalized dense + embeddings and cosine distance. +- One Qdrant collection belongs to one workspace. Schema, Evidence, and Memory points share that + collection and are separated by indexed payload field `kind`. +- Qdrant and Ollama are mandatory base-Compose services. They are not published on host ports and + are reachable only from the private Compose network. +- Existing schema-v1 and schema-v2 descriptors remain readable for migration, but they are not + activatable. The new operational contract is workspace schema v3. + +The model choice is based on the published Qwen model card: the 0.6B model supports more than 100 +languages, a 32K context window, Matryoshka dimensions up to 1024, and instruction-aware retrieval. +Ollama distributes a CPU-viable quantized build and can use an exposed GPU without changing the +application protocol. + +References: + +- +- +- +- +- + +## Target topology + +```text +browser -> frontend -> core -> external DWH + -> private Qdrant + -> private Ollama embedding +``` + +The base Compose project contains: + +- `frontend`: static React application and same-origin API proxy. +- `core`: Fastify, Pi, and the Python `tht` harness. +- `qdrant`: pinned Qdrant server with persistent `qdrant-data` volume. +- `embedding`: pinned Ollama server with persistent `embedding-models` volume. +- `embedding-model-init`: bounded one-shot service that pulls and verifies + `qwen3-embedding:0.6b`; `core` starts only after it succeeds. + +`qdrant` and `embedding` use `expose`, not `ports`. The core receives installation-owned internal +URLs: + +```text +THT_INTERNAL_QDRANT_URL=http://qdrant:6333 +THT_INTERNAL_EMBEDDING_URL=http://embedding:11434 +THT_INTERNAL_EMBEDDING_MODEL=qwen3-embedding:0.6b +THT_INTERNAL_EMBEDDING_DIMENSIONS=1024 +``` + +These are deployment facts, not workspace connector bindings. The runtime rejects non-loopback or +non-Compose-service hosts when these variables are overridden for development. + +An optional Linux GPU override exposes an available NVIDIA/AMD device to Ollama. The base profile +must remain CPU-safe. macOS Docker remains CPU-only because Docker Desktop cannot expose the Apple +GPU to an Ollama container. + +## Workspace schema v3 + +The workspace itself is the association between the external database and the internal collection: + +```yaml +workspace: + schema_version: 3 + id: psd-clinical + name: PSD Clinical + language: it + +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: [postgres_direct] + +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 + +llm_policy: + allowed: [zai/glm-5.2] +``` + +Invariants: + +- the collection name is an explicit portable identifier; +- active workspaces cannot share a collection; +- vector and embedding dimensions are both 1024; +- distance is `cosine`; +- provider and model are exactly the supported internal values; +- no vector transport, vector credential, embedding URL, or embedding credential may appear in a + schema-v3 descriptor or installation contract; +- DWH connectors remain installation-local and can still use the supported external DWH transports. + +Schema-v1/v2 pgvector descriptors are listed as `migration_required`. Migration creates a reviewed +schema-v3 document; it does not copy vector data implicitly. Existing semantic data is rebuilt from +the canonical schema documents, Evidence corpus, and Memory registry. + +## Qdrant data model + +Each point has a deterministic UUIDv5 derived from: + +```text +workspace_id + kind + record_key +``` + +The vector is the 1024-dimensional Ollama result. The payload is: + +```json +{ + "workspace_id": "psd-clinical", + "kind": "schema", + "source_id": "datawarehouse.patients", + "record_key": "schema:table:datawarehouse.patients", + "content_hash": "sha256:...", + "workspace_revision": "", + "generation": "", + "language": "it", + "text": "...", + "metadata": {} +} +``` + +`kind`, `source_id`, `content_hash`, `workspace_revision`, and `generation` receive keyword payload +indexes. Queries always filter by `workspace_id` and an explicit allowed `kind` set. Upsert is +idempotent. Evidence generation deletion is an exact filtered delete. Collection creation is also +idempotent and fails closed if an existing collection has incompatible dimensions or distance. + +## Harness integration + +The existing `VectorStore` port remains the workflow boundary. A `QdrantVectorStore` adapter maps +its operations to Qdrant REST endpoints while preserving current schema/Evidence/Memory call sites. +The existing Ollama embedding client is narrowed to the internal `/api/embed` contract and verifies: + +- configured model exists; +- output count matches input count; +- every vector has 1024 finite numeric values; +- no remote URL or API key is accepted. + +The JSONL Memory registry and persisted phase documents remain canonical. Qdrant remains a derived, +rebuildable semantic index. Schema, Evidence, and Memory ingestion all use the same point builder, +content hashing, and retry policy. + +## Readiness and failure behavior + +Readiness is layered: + +1. Compose waits for Qdrant health. +2. Compose waits for Ollama health and successful model initialization. +3. Workspace activation validates the schema-v3 contract. +4. Harness readiness ensures the Qdrant collection and checks its vector configuration. +5. Harness embeds a bounded probe and verifies 1024 dimensions. + +Failures are sanitized and fail closed: + +- unavailable Qdrant -> `workspace_not_activatable` before session persistence; +- unavailable or missing Ollama model -> `model_unavailable` before session persistence; +- collection mismatch -> `semantic_index_incompatible` without recreating or deleting data; +- embedding dimension mismatch -> no point write; +- partial batch failure -> operation reports failure and remains safe to retry. + +No health response, API response, or diagnostic log exposes DWH credentials or indexed text. + +## Deployment and migration + +The pgvector deployment path is retired: + +- remove local-vector Compose overlays and pgvector bootstrap/migration services; +- remove vector PostgreSQL role and password contracts; +- remove runtime support for vector REST/SSH and external embedding URLs; +- keep only the descriptor parser and migration code needed to recognize legacy workspaces; +- update local/server manuals, examples, smoke tests, CI coupling scans, backup instructions, and + release gates for four persistent stores plus Qdrant and Ollama volumes. + +Qdrant backup/restore uses collection snapshots or the persistent volume according to the operator +manual. Ollama model storage is a cache: it may be backed up for offline recovery but is not an +application source of truth. + +## Acceptance criteria + +- Base local and server Compose renders include healthy private `qdrant` and `embedding` services. +- A clean CPU-only installation downloads the model, creates a workspace collection, and embeds a + probe without external vector or embedding configuration. +- GPU override uses the same API and persistent model volume. +- Schema-v3 workspaces activate; schema-v1/v2 workspaces report `migration_required`. +- Two workspaces cannot claim the same Qdrant collection. +- Schema, Evidence, and Memory records coexist in one collection and remain filter-isolated. +- Existing workflow behavior and persisted session contracts remain unchanged. +- Tests reject all active pgvector deployment, external vector binding, and external embedding + configuration paths. diff --git a/docs/plans/2026-08-08-internal-qdrant-ollama.md b/docs/plans/2026-08-08-internal-qdrant-ollama.md new file mode 100644 index 00000000..075c2fa6 --- /dev/null +++ b/docs/plans/2026-08-08-internal-qdrant-ollama.md @@ -0,0 +1,773 @@ +# Internal Qdrant and Ollama Implementation Plan + +> **Historical nomenclature:** this plan predates the native host CLI convergence. The current +> operator command is `tht`; any older `thothctl` smoke-script or rollback wording below is retained +> only as historical evidence. + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Make Qdrant and Ollama mandatory internal ThothII services while keeping the analytical +DWH external and associating each workspace with one Qdrant collection for schema, Evidence, and +Memory embeddings. + +**Architecture:** Introduce workspace schema v3, preserve v1/v2 only as migration inputs, and keep +the existing harness `VectorStore` port behind a new Qdrant REST adapter. Base Compose owns Qdrant, +Ollama, their persistent volumes, and model initialization; workspace descriptors contain semantic +identity but no vector/embedding endpoints or credentials. + +**Tech Stack:** TypeScript/Fastify/Zod, Python 3.12/Pydantic/requests, React 18, Docker Compose, +Qdrant REST API, Ollama `/api/embed`, Vitest, pytest. + +--- + +## Guardrails + +- Apply `@superpowers:test-driven-development` to every behavior change: add one focused failing + test, observe the expected failure, implement the minimum, and rerun the focused test. +- Do not run broad suites until the corresponding code/config changes exist; this preserves the + requested ordering while still using TDD. +- Preserve the external DWH connector contract and session persistence model. +- Do not retain an operational fallback to pgvector or an external embedding endpoint. +- Do not delete or rewrite user workspace repositories or Qdrant data. Migration is descriptor-only; + semantic data is rebuilt explicitly. +- Commit after each task only when focused tests are green. + +### Task 1: Define workspace schema v3 + +**Files:** + +- Modify: `backend/src/workspaces/schema.ts` +- Modify: `backend/src/workspaces/types.ts` +- Modify: `backend/test/workspaces-schema.test.ts` +- Modify: `backend/test/workspaces-migrate-legacy.test.ts` +- Create: `backend/src/workspaces/migrate-v2-qdrant.ts` +- Create: `backend/test/workspaces-migrate-v2-qdrant.test.ts` + +**Step 1: Write the failing schema tests** + +Add tests proving that schema v3 accepts only this semantic shape: + +```ts +const semantic_index = { + vector_store: { + engine: "qdrant", + collection: "psd-clinical", + dimensions: 1024, + distance: "cosine", + }, + embedding: { + provider: "ollama_internal", + model: "qwen3-embedding:0.6b", + dimensions: 1024, + }, +}; +``` + +Add separate rejection cases for `pgvector`, `supported_transports`, external embedding providers, +non-1024 dimensions, non-cosine distance, and unknown fields. Assert v1/v2 remain parseable as +legacy descriptors but `isOperationalWorkspace()` returns false. + +**Step 2: Run the tests and verify RED** + +Run: + +```bash +cd backend +npx vitest run test/workspaces-schema.test.ts test/workspaces-migrate-v2-qdrant.test.ts +``` + +Expected: failure because schema version 3 and `migrateWorkspaceV2ToV3` do not exist. + +**Step 3: Implement the minimum schema and migration** + +Add `QdrantVectorStore`, `InternalEmbedding`, and `WorkspaceV3` types. Replace the operational type +guard with schema-v3-only semantics. Implement: + +```ts +export function migrateWorkspaceV2ToV3( + legacy: WorkspaceV2, + collection: string, +): WorkspaceV3 { + return validateOperationalWorkspace({ + workspace: { ...legacy.workspace, schema_version: 3 }, + dwh: legacy.dwh, + semantic_index: { + vector_store: { + engine: "qdrant", + collection, + dimensions: 1024, + distance: "cosine", + }, + embedding: { + provider: "ollama_internal", + model: "qwen3-embedding:0.6b", + dimensions: 1024, + }, + }, + llm_policy: legacy.llm_policy, + ...(legacy.diagnostics?.dwh_rest + ? { diagnostics: { dwh_rest: legacy.diagnostics.dwh_rest } } + : {}), + }); +} +``` + +Do not copy vector/embedding diagnostics or transports. + +**Step 4: Verify GREEN** + +Run the command from Step 2. Expected: all selected tests pass. + +**Step 5: Commit** + +```bash +git add backend/src/workspaces/schema.ts backend/src/workspaces/types.ts \ + backend/src/workspaces/migrate-v2-qdrant.ts backend/test/workspaces-schema.test.ts \ + backend/test/workspaces-migrate-legacy.test.ts backend/test/workspaces-migrate-v2-qdrant.test.ts +git commit -m "feat: define internal semantic workspace schema" +``` + +### Task 2: Make collection ownership unique in the Git registry + +**Files:** + +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/src/workspaces/migrate-legacy.ts` +- Modify: `backend/test/workspace-registry.test.ts` +- Modify: `backend/test/workspaces-migrate-legacy.test.ts` + +**Step 1: Write failing registry tests** + +Add fixtures with two schema-v3 workspaces claiming `collection: shared`. Assert snapshot activation +fails with `workspace_invalid` and retains the previous active snapshot. Assert v1/v2 entries are +listed as `migration_required` and cannot be acquired with `acquireSessionRevision()`. + +**Step 2: Verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \ + -t "collection|migration_required" +``` + +Expected: duplicate collections are currently accepted and v2 is currently operational. + +**Step 3: Implement uniqueness and migration state** + +During snapshot validation, build `Map` for operational descriptors and +raise a sanitized `workspace_invalid` error on a duplicate. Update migration output and CLI wording +to require an explicit target collection and schema v3. + +**Step 4: Verify GREEN and commit** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts test/workspaces-migrate-legacy.test.ts \ + -t "collection|migration_required" +cd .. +git add backend/src/workspaces/registry.ts backend/src/workspaces/migrate-legacy.ts \ + backend/test/workspace-registry.test.ts backend/test/workspaces-migrate-legacy.test.ts +git commit -m "feat: reserve one qdrant collection per workspace" +``` + +### Task 3: Remove external semantic bindings and render internal endpoints + +**Files:** + +- Modify: `backend/src/workspaces/contracts.ts` +- Modify: `backend/src/workspaces/bindings.ts` +- Modify: `backend/src/workspaces/runtime-renderer.ts` +- Modify: `backend/src/config.ts` +- Modify: `backend/test/workspaces-contracts.test.ts` +- Modify: `backend/test/workspaces-bindings.test.ts` +- Modify: `backend/test/workspace-runtime-renderer.test.ts` +- Modify: `backend/test/config.test.ts` + +**Step 1: Write failing contract tests** + +Assert schema-v3 installation contracts contain DWH variables only. Assert environment variables +matching `*_VECTOR_*`, `*_EMBEDDING_BASE_URL`, or semantic API-key suffixes are ignored/rejected. +Assert the rendered harness config always contains: + +```yaml +resources: + vector: + engine: qdrant + base_url: http://qdrant:6333 + collection: psd-clinical + embeddings: + provider: ollama_internal + base_url: http://embedding:11434 + model: qwen3-embedding:0.6b + dimensions: 1024 +``` + +**Step 2: Verify RED** + +```bash +cd backend +npx vitest run test/workspaces-contracts.test.ts test/workspaces-bindings.test.ts \ + test/workspace-runtime-renderer.test.ts test/config.test.ts +``` + +Expected: current contracts require external vector and embedding bindings. + +**Step 3: Implement internal runtime configuration** + +Add typed backend config fields with Compose defaults: + +```ts +internalQdrantUrl: "http://qdrant:6333" +internalEmbeddingUrl: "http://embedding:11434" +internalEmbeddingModel: "qwen3-embedding:0.6b" +internalEmbeddingDimensions: 1024 +``` + +Accept only `qdrant`, `embedding`, `localhost`, or loopback hosts. Keep these values out of Git +workspace descriptors, API payloads, and generated installation docs. Render them into the +ephemeral backend-owned harness config after descriptor validation. + +**Step 4: Verify GREEN and commit** + +Run Step 2, then: + +```bash +git add backend/src/config.ts backend/src/workspaces/contracts.ts backend/src/workspaces/bindings.ts \ + backend/src/workspaces/runtime-renderer.ts backend/test/config.test.ts \ + backend/test/workspaces-contracts.test.ts backend/test/workspaces-bindings.test.ts \ + backend/test/workspace-runtime-renderer.test.ts +git commit -m "feat: render private semantic service endpoints" +``` + +### Task 4: Narrow harness embedding configuration to internal Ollama + +**Files:** + +- Modify: `harness/tht/config.py` +- Modify: `harness/tht/config_compat.py` +- Modify: `harness/tht/vectorstore/embeddings.py` +- Modify: `harness/tht/cli/ollama_cmd.py` +- Modify: `harness/tests/test_config_resources.py` +- Create: `harness/tests/test_internal_embeddings.py` + +**Step 1: Write failing embedding tests** + +Use a fake `requests.Session` to prove `OllamaInternalEmbeddings.embed()` calls `/api/embed` with +model and batch input, returns 1024-dimensional finite vectors, and rejects count/dimension/NaN +mismatches. Add config tests rejecting external providers, API keys, and non-private base URLs. + +**Step 2: Verify RED** + +```bash +cd harness +.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py -q +``` + +Expected: `OllamaInternalEmbeddings` and internal-only config do not exist. + +**Step 3: Implement the client** + +Implement one bounded `/api/embed` request per batch: + +```python +response = self._session.post( + f"{self.base_url}/api/embed", + json={"model": self.model, "input": texts}, + timeout=self.timeout, +) +``` + +Validate response shape before returning any vector. Keep retry behavior bounded and sanitize URLs +and response bodies from raised errors. + +**Step 4: Verify GREEN and commit** + +```bash +cd harness +.venv/bin/pytest tests/test_internal_embeddings.py tests/test_config_resources.py -q +cd .. +git add harness/tht/config.py harness/tht/config_compat.py harness/tht/vectorstore/embeddings.py \ + harness/tht/cli/ollama_cmd.py harness/tests/test_config_resources.py \ + harness/tests/test_internal_embeddings.py +git commit -m "feat: use internal ollama embeddings" +``` + +### Task 5: Implement the Qdrant VectorStore adapter + +**Files:** + +- Create: `harness/tht/adapters/vector/qdrant.py` +- Modify: `harness/tht/adapters/vector/__init__.py` +- Modify: `harness/tht/ports/vector.py` +- Modify: `harness/tht/vectorstore/records.py` +- Modify: `harness/tht/vectorstore/store.py` +- Create: `harness/tests/test_qdrant_vector_store.py` +- Modify: `harness/tests/test_vector_port_contract.py` + +**Step 1: Write failing adapter tests** + +Test a real adapter against a deterministic fake HTTP server. Cover: + +- idempotent collection create with 1024/Cosine; +- mismatch fails without delete/recreate; +- keyword payload-index creation; +- deterministic UUIDv5 point IDs; +- upsert payload for `schema`, `evidence`, and `memory`; +- query filtered by workspace and allowed kinds; +- `existing_hashes`, exact Evidence generation list/delete, and health; +- sanitized timeouts and malformed responses. + +The point ID helper must satisfy: + +```python +def point_id(workspace_id: str, kind: str, record_key: str) -> str: + return str(uuid5(NAMESPACE_URL, f"thothii:{workspace_id}:{kind}:{record_key}")) +``` + +**Step 2: Verify RED** + +```bash +cd harness +.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q +``` + +Expected: import failure for the Qdrant adapter. + +**Step 3: Implement minimal REST mappings** + +Use existing `requests` dependency and these endpoints: + +```text +GET /collections/{collection} +PUT /collections/{collection} +PUT /collections/{collection}/index +PUT /collections/{collection}/points?wait=true +POST /collections/{collection}/points/query +POST /collections/{collection}/points/scroll +POST /collections/{collection}/points/delete?wait=true +``` + +Every operation must include the workspace filter even though the collection is workspace-owned. +Map Qdrant scores and payloads back into existing `VectorHit` objects. + +**Step 4: Verify GREEN and commit** + +```bash +cd harness +.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py -q +cd .. +git add harness/tht/adapters/vector/qdrant.py harness/tht/adapters/vector/__init__.py \ + harness/tht/ports/vector.py harness/tht/vectorstore/records.py \ + harness/tht/vectorstore/store.py harness/tests/test_qdrant_vector_store.py \ + harness/tests/test_vector_port_contract.py +git commit -m "feat: add qdrant vector adapter" +``` + +### Task 6: Wire schema, Evidence, and Memory through Qdrant + +**Files:** + +- Modify: `harness/tht/vectorstore/reader.py` +- Modify: `harness/tht/cli/vector_cmd.py` +- Modify: `harness/tht/cli/memory_cmd.py` +- Modify: `harness/tht/corpus/pipeline.py` +- Modify: `harness/tht/search/evidence.py` +- Modify: `harness/tht/cli/schema_cmd.py` +- Modify: `harness/tests/test_memory_save_one.py` +- Modify: `harness/tests/test_search_pack.py` +- Create: `harness/tests/test_semantic_kind_isolation.py` + +**Step 1: Write failing integration tests** + +Use an in-memory fake implementing the `VectorStore` port. Assert: + +- schema records use `kind=schema`; +- corpus records use `kind=evidence` and exact generation; +- approved memories use `kind=memory`; +- search pack requests only its allowed kind set; +- all three paths share `workspace_id`, `workspace_revision`, hashing, and point-key construction; +- retries do not duplicate points. + +**Step 2: Verify RED** + +```bash +cd harness +.venv/bin/pytest tests/test_semantic_kind_isolation.py tests/test_memory_save_one.py \ + tests/test_search_pack.py -q +``` + +Expected: current factories select pgvector/HTTP adapters and payloads lack the v3 identity fields. + +**Step 3: Wire the adapter** + +Make schema-v3 `qdrant` the only operational vector factory branch. Reuse the current canonical +record builders; add only missing identity fields. Keep the JSONL Memory registry and filesystem +Evidence corpus as sources of truth. + +**Step 4: Verify GREEN and commit** + +Run Step 2, then commit the listed files with: + +```bash +git commit -m "feat: index semantic records in qdrant" +``` + +### Task 7: Add mandatory Qdrant and Ollama Compose services + +**Files:** + +- Modify: `compose.yaml` +- Create: `deploy/compose.embedding-gpu.yaml` +- Create: `docker/embedding-model-init.sh` +- Modify: `docker/core.Dockerfile` +- Modify: `deploy/env/local.env.example` +- Modify: `deploy/env/server.env.example` +- Modify: `scripts/run-stack.sh` +- Modify: `scripts/test-default-compose.sh` +- Modify: `scripts/test-unified-compose.sh` +- Create: `scripts/test-internal-semantic-compose.sh` + +**Step 1: Write failing Compose contract tests** + +Assert the rendered base profile has `core`, `frontend`, `qdrant`, `embedding`, and +`embedding-model-init`; private services have no published ports; persistent volumes exist; core +depends on Qdrant health and successful model init; no external vector/embedding binding is required. + +Also assert all service images use version plus immutable digest. Resolve and record supported +multi-architecture digests for Qdrant v1.18.x and Ollama v0.32.x during implementation: + +```bash +docker buildx imagetools inspect qdrant/qdrant:v1.18.2 +docker buildx imagetools inspect ollama/ollama:0.32.0 +``` + +**Step 2: Verify RED** + +```bash +./scripts/test-default-compose.sh +./scripts/test-unified-compose.sh +./scripts/test-internal-semantic-compose.sh +``` + +Expected: required services and volumes are absent. + +**Step 3: Implement the services** + +`embedding-model-init.sh` must wait with a bounded deadline, call `ollama pull` for the exact model, +and verify it appears in `/api/tags`. The Qdrant healthcheck uses its HTTP health endpoint. The CPU +base has no device reservation; the GPU override adds only the supported device stanza. + +**Step 4: Verify GREEN and commit** + +Run Step 2, then: + +```bash +git add compose.yaml deploy/compose.embedding-gpu.yaml docker/embedding-model-init.sh \ + docker/core.Dockerfile deploy/env/local.env.example deploy/env/server.env.example \ + scripts/run-stack.sh scripts/test-default-compose.sh scripts/test-unified-compose.sh \ + scripts/test-internal-semantic-compose.sh +git commit -m "feat: run qdrant and ollama inside thothii" +``` + +### Task 8: Retire pgvector deployment and external semantic connectors + +**Files:** + +- Delete: `deploy/compose.local-vector.yaml` +- Delete: `deploy/compose.preprocess-local-vector.yaml` +- Delete: `deploy/sql/20-vector-roles.sql` +- Delete: `deploy/vector/reconcile-roles.sh` +- Delete: `deploy/vector/rotate-bootstrap-password.py` +- Delete: `deploy/vector/secret-policy.sh` +- Delete: `deploy/vector/vector-db-entrypoint.sh` +- Delete: `scripts/local-vector-smoke.sh` +- Delete: `scripts/test-local-vector-smoke-safety.sh` +- Delete: `scripts/test-local-vector-smoke-live-collision.sh` +- Delete: `scripts/test-vector-bootstrap-rotation.sh` +- Delete: `scripts/test-vector-migration-image.sh` +- Delete: `scripts/test-vector-secret-policy.sh` +- Modify: `scripts/test-no-deployment-coupling.sh` +- Modify: `scripts/test-no-deployment-coupling-scope.sh` +- Modify: `scripts/test-compose-secret-policy.sh` +- Modify: `.github/workflows/deployment.yml` + +**Step 1: Write the failing coupling test** + +Teach the coupling gate to reject active `pgvector`, `local-vector`, `THT_VECTOR_*`, workspace +embedding URLs/API keys, and external vector transports while allowing historical specs and the +explicit descriptor migration module. + +**Step 2: Verify RED** + +```bash +./scripts/test-no-deployment-coupling-scope.sh +./scripts/test-no-deployment-coupling.sh +./scripts/test-compose-secret-policy.sh +``` + +Expected: active pgvector deployment paths are reported. + +**Step 3: Remove the retired paths and update CI** + +Remove only repository deployment machinery. Retain harness pgvector code temporarily only if it +is needed to read/export legacy data during migration; it must not be reachable from schema v3 or +Compose. Remove it in a follow-up task once migration fixtures no longer import it. + +**Step 4: Verify GREEN and commit** + +Run Step 2 and the workflow fixture tests, then commit all deletions and modifications: + +```bash +git add -A deploy scripts .github/workflows/deployment.yml +git commit -m "refactor: retire external vector deployment" +``` + +### Task 9: Update frontend workspace editing and examples + +**Files:** + +- Modify: `frontend/src/api/workspaces.ts` +- Modify: `frontend/src/shell/WorkspaceEditor.tsx` +- Modify: `frontend/src/shell/WorkspaceEditor.test.tsx` +- Modify: `frontend/src/shell/WorkspaceManager.test.tsx` +- Modify: `frontend/src/api/workspaces.test.ts` +- Modify: `frontend/src/workspaces/drafts.test.ts` +- Modify: `deploy/workspaces/example.yaml` +- Modify: `deploy/workspaces/psd.yaml.example` + +**Step 1: Write failing UI tests** + +Assert editor/preview show Qdrant collection and fixed internal embedding model, expose no vector +endpoint/credential fields, and publish schema v3. Assert legacy descriptors display a migration +banner and cannot be selected for a new session. + +**Step 2: Verify RED** + +```bash +cd frontend +npx vitest run src/shell/WorkspaceEditor.test.tsx src/shell/WorkspaceManager.test.tsx \ + src/api/workspaces.test.ts src/workspaces/drafts.test.ts +``` + +Expected: fixtures and controls still use pgvector/external embedding. + +**Step 3: Implement fixed semantic controls** + +Collection remains editable and validated. Engine, provider, model, dimensions, and distance render +as fixed architecture values. Remove external semantic diagnostics from drafts and publish payloads. + +**Step 4: Verify GREEN and commit** + +Run Step 2, then commit the listed files with: + +```bash +git commit -m "feat: edit qdrant workspace collections" +``` + +### Task 10: Add a real internal semantic smoke + +**Files:** + +- Create: `scripts/internal-semantic-smoke.sh` +- Modify: `scripts/unified-deployment-smoke.sh` +- Modify: `scripts/server-deployment-smoke.sh` +- Modify: `scripts/task13-runtime-fixture-check.ts` +- Modify: `scripts/test-task13-runtime-fixtures.sh` + +**Step 1: Write failing smoke fixture assertions** + +The fixture must require private Qdrant/Ollama services, model volume, Qdrant volume, fixed internal +URLs, and no host ports. It must reject wrong service names, external URLs, collection reuse, and +dimension changes. + +**Step 2: Verify RED** + +```bash +./scripts/test-task13-runtime-fixtures.sh local +./scripts/test-task13-runtime-fixtures.sh server +``` + +Expected: current fixture expects the two-service topology. + +**Step 3: Implement the live smoke** + +Using disposable volumes and a fixture workspace, start the stack on CPU, wait for the model, ensure +the collection, embed one record of each kind, query each kind with filters, restart offline, and +prove all points and the model remain available. Cleanup must remain exact and must not prune global +Docker resources. + +**Step 4: Verify GREEN and commit** + +```bash +./scripts/test-task13-runtime-fixtures.sh local +./scripts/test-task13-runtime-fixtures.sh server +./scripts/internal-semantic-smoke.sh +git add scripts/internal-semantic-smoke.sh scripts/unified-deployment-smoke.sh \ + scripts/server-deployment-smoke.sh scripts/task13-runtime-fixture-check.ts \ + scripts/test-task13-runtime-fixtures.sh +git commit -m "test: cover internal semantic services" +``` + +### Task 11: Update operator documentation and state + +**Files:** + +- Modify: `README.md` +- Modify: `AGENTS.md` +- Modify: `PROJECT_STATE.md` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `docs/installazione-docker-4-contesti.md` +- Modify: `docs/workspace-diagnostic-protocol.md` +- Modify: `docs/gestione-memory.md` +- Modify: `deploy/secrets/README.md` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/test-verify-workspace-install-docs.sh` + +**Step 1: Write failing documentation contract assertions** + +Require the four-service topology, CPU/GPU behavior, volume backup/restore, schema-v3 migration, +Qdrant collection ownership, and removal of external vector/embedding variables from active manuals. + +**Step 2: Verify RED** + +```bash +./scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +``` + +Expected: manuals still describe external pgvector/embedding and a two-service mandatory stack. + +**Step 3: Update documentation** + +Document Qdrant as a derived but persistent index, Ollama model cache behavior, CPU-first startup, +optional GPU override, snapshot/restore, explicit legacy migration, and the fact that only the DWH +and LLM remain external application endpoints. + +**Step 4: Verify GREEN and commit** + +Run Step 2, then: + +```bash +git add README.md AGENTS.md PROJECT_STATE.md docs deploy/secrets/README.md \ + scripts/verify-workspace-install-docs.sh scripts/test-verify-workspace-install-docs.sh +git commit -m "docs: document internal semantic infrastructure" +``` + +### Task 12: Remove unreachable pgvector runtime code + +**Files:** + +- Delete: `harness/tht/adapters/vector/pgvector.py` +- Delete: `harness/tht/adapters/vector/legacy_direct.py` +- Delete: `harness/tht/adapters/vector/thoth_http.py` +- Delete: `harness/tht/vectorstore/rest_client.py` +- Delete: `harness/tht/vectorstore/rest_writer.py` +- Delete: `harness/tht/migrations/vector/001_extensions.sql` +- Delete: `harness/tht/migrations/vector/002_schema_tables.sql` +- Delete: `harness/tht/migrations/vector/003_roles.sql` +- Delete: `harness/tht/migrations/vector/004_evidence_generation_gc.sql` +- Modify: `harness/pyproject.toml` +- Modify/Delete: affected pgvector and migration tests under `harness/tests/l0/` + +**Step 1: Prove the code is unreachable** + +```bash +rg -n "PgVectorStore|ThothHttpVectorStore|LegacyDirectVectorStore|migrations/vector" \ + harness backend frontend compose.yaml deploy scripts docker docs \ + --glob '!docs/plans/**' --glob '!docs/superpowers/**' +``` + +Expected before cleanup: matches only in the files scheduled for deletion and legacy tests. If an +operational call site remains, stop and migrate it before deleting anything. + +**Step 2: Delete obsolete runtime and tests** + +Retain descriptor migration tests, but remove PostgreSQL vector runtime/migration packaging tests. +Remove `psycopg2-binary` only if the DWH/session PostgreSQL paths do not need it; otherwise keep it. + +**Step 3: Verify focused imports and packaging** + +```bash +cd harness +.venv/bin/pytest tests/test_qdrant_vector_store.py tests/test_vector_port_contract.py \ + tests/test_semantic_kind_isolation.py tests/test_vector_migration_packaging.py -q +python -m build +``` + +Expected: Qdrant tests pass and the wheel contains no pgvector migrations. Adjust the packaging test +to assert Qdrant has no SQL migration payload. + +**Step 4: Commit** + +```bash +git add -A harness +git commit -m "refactor: remove pgvector runtime" +``` + +### Task 13: Run complete verification + +**Files:** + +- Modify only if a genuine regression is discovered. + +**Step 1: Deterministic layer gates** + +```bash +cd harness && .venv/bin/pytest -q && .venv/bin/ruff check . +cd ../backend && npx vitest run && npx tsc --noEmit -p . && npm run build +cd ../frontend && npx vitest run && npx tsc -b && npm run build +cd .. && git diff --check +``` + +Expected: all gates pass. Existing unrelated Ruff debt must be reported separately if it remains; +new/modified files must be Ruff-clean. + +**Step 2: Deployment contracts** + +```bash +./scripts/test-default-compose.sh +./scripts/test-unified-compose.sh +./scripts/test-internal-semantic-compose.sh +./scripts/test-no-deployment-coupling.sh +./scripts/test-compose-secret-policy.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +``` + +Expected: all pass without external vector/embedding settings. + +**Step 3: Docker smokes** + +```bash +./scripts/internal-semantic-smoke.sh +./scripts/workspace-registry-smoke.sh +./scripts/unified-deployment-smoke.sh +./scripts/thothctl-update-smoke.sh +./scripts/server-deployment-smoke.sh +``` + +Expected: CPU semantic smoke passes, persistence survives offline restart, and every script proves +exact cleanup. Investigate the previously observed `thothctl` rollback failure independently if it +recurs; do not weaken the new semantic gate to hide it. + +**Step 4: Final audit** + +```bash +rg -n "pgvector|local-vector|THT_VECTOR_|EMBEDDING_BASE_URL|openai_compatible|ollama_compatible" \ + . --glob '!docs/plans/**' --glob '!docs/superpowers/**' --glob '!**/node_modules/**' \ + --glob '!**/.venv/**' --glob '!**/.git/**' +git status --short +``` + +Expected: no active operational references; only explicit legacy descriptor migration fixtures may +remain. Worktree contains only intentional changes. + +**Step 5: Commit verification metadata** + +Update `PROJECT_STATE.md` with exact counts, image digests, smoke durations, CPU hardware, and any +manual GPU/Windows gates. Commit only verified claims: + +```bash +git add PROJECT_STATE.md +git commit -m "docs: record qdrant ollama verification" +``` diff --git a/docs/plans/2026-08-10-rimozione-schema-v1-v2.md b/docs/plans/2026-08-10-rimozione-schema-v1-v2.md new file mode 100644 index 00000000..81ffa54d --- /dev/null +++ b/docs/plans/2026-08-10-rimozione-schema-v1-v2.md @@ -0,0 +1,447 @@ +# Piano di implementazione: workspace descriptor esclusivamente schema v3 + +> **Per gli agenti esecutori:** SUB-SKILL OBBLIGATORIA: usare `superpowers:subagent-driven-development` (raccomandata) oppure `superpowers:executing-plans`, procedendo task per task con TDD e review tra i task. + +**Obiettivo:** rimuovere dal prodotto ogni capacità di leggere, migrare, rendere operativo o presentare workspace descriptor schema v1/v2. Il solo descriptor accettato diventa schema v3. Restano intatti i formati versionati non correlati e gli state file del registry già prodotti da versioni recenti con revisioni v3. + +**Architettura:** parser, registry, renderer, diagnostica, route e frontend convergono su un solo tipo `WorkspaceV3`. Il campo pubblico `WorkspaceRevision.state` scompare. Un decoder privato normalizza in memoria gli state file già scritti con `state: "operational"`, elimina quel campo prima di qualsiasi uso/API e rifiuta ogni combinazione non-v3 o incoerente. I build backend diventano clean-first, così la cancellazione dei migratori sorgente implica anche la loro assenza da `dist` e dall'immagine core. + +**Tech stack:** TypeScript 5, Zod 4, Fastify 5, React 18, Vitest, Node.js 22, Bash/PowerShell, Git e Docker Compose. + +**Stato:** piano revisionato dopo review indipendente. La sua approvazione non autorizza l'implementazione; attendere un esplicito ordine separato. + +--- + +## Decisioni confermate + +1. Nessun workspace v1/v2 reale deve essere preservato o migrato. +2. Eliminare `migrate-legacy.ts`, `migrate-v2-qdrant.ts` e le relative interfacce CLI. +3. Eliminare il campo `state` dal tipo/API `WorkspaceRevision` e da tutti i nuovi state/manifest del registry. +4. Descriptor v1/v2 presenti in Git o negli snapshot vengono rifiutati, senza conversione automatica. +5. Non toccare i documenti storici sotto `docs/superpowers/` e i vecchi piani; possono descrivere decisioni passate. +6. Non iniziare P2 finché P1 non dispone di nuova evidenza automatica e di una nuova decisione manuale esplicita. + +## Confini da non oltrepassare + +Questa rimozione riguarda soltanto il **workspace descriptor**. Non eliminare o rinominare: + +- `schemaVersion`/`schema_version` di bundle ZIP, report, job, ledger, manifest di sessione o artifact di fase; +- `RevisionLeaseRecord.state` (`creating`/`persisted`), maintenance state, process state o UI state non collegati a `WorkspaceRevision`; +- `migration_required` usato nei futuri piani P3–P6 per ownership DWH, punti semantici revisionless o altre migrazioni non-descriptor; +- `allowLegacy` del frontend sessioni, che significa “sessione senza revisione workspace” e non descriptor v1/v2; +- documenti storici o report conservati. + +L'unica compatibilità legacy mantenuta nel codice è il decoder privato degli state file già scritti con il campo revisionale `state: "operational"`. Non costituisce supporto a descriptor v1/v2. + +## Contratto v3-only + +- `WorkspaceDescriptor`, `CanonicalWorkspace` e `WorkspaceV3` rappresentano la stessa forma v3; mantenere gli alias soltanto quando migliorano la semantica dei confini. +- `parseWorkspaceYaml` e `validateWorkspaceDescriptor` accettano esclusivamente `workspace.schema_version === 3`. +- v1/v2 generano l'errore pubblico già sanitizzato `workspace_invalid`; non usare più il messaggio o lo stato `migration_required` per i descriptor. +- Un'attivazione Git contenente anche un solo descriptor non-v3 fallisce interamente e conserva il precedente active state. +- Le revisioni restituite dalle API contengono esattamente `id`, `commit`, `blob`, `snapshotPath`, senza `state`. +- Nuovi `active.json` e `snapshot.json` non contengono `state` nelle revisioni. + +## Compatibilità degli state file esistenti + +Definire due decoder stretti e distinti: + +```ts +interface StoredWorkspaceRevision { + id: string; + commit: string; + blob: string; + snapshotPath: string; + state?: "operational"; // solo input compatibile; mai restituito +} + +interface WorkspaceRevision { + id: string; + commit: string; + blob: string; + snapshotPath: string; +} +``` + +Regole: + +1. `active.json` accetta soltanto `{head,revisions}`; `snapshot.json` soltanto `{head,revisions,files}`. +2. Ogni revision object accetta soltanto i quattro campi correnti più l'opzionale vecchio `state: "operational"`. +3. `state: "migration_required"`, qualsiasi altro valore o campo sconosciuto è rifiutato. +4. Il decoder ricostruisce un nuovo oggetto `WorkspaceRevision`; non restituisce mai l'oggetto JSON originale. +5. Active state e snapshot manifest vengono confrontati dopo la normalizzazione. +6. L'integrità continua a validare path, commit, blob, digest, descriptor v3 e Evidence context. +7. La lettura non modifica snapshot storici. La successiva attivazione riscrive `active.json` nel formato corrente; tutti i nuovi snapshot sono state-free. +8. Un vecchio file già privo di `state` è naturalmente il formato corrente, ma il relativo descriptor deve comunque essere v3. + +## Mappa completa dei file + +### Backend produttivo + +- `backend/src/workspaces/schema.ts` +- `backend/src/workspaces/types.ts` +- `backend/src/workspaces/runtime-renderer.ts` +- `backend/src/workspaces/contracts.ts` +- `backend/src/workspaces/diagnostics.ts` +- `backend/src/workspaces/bindings.ts` +- `backend/src/workspaces/registry.ts` +- `backend/src/routes/workspaces.ts` +- `backend/src/routes/sessions.ts` +- `backend/src/routes/sql.ts` +- Eliminare `backend/src/workspaces/migrate-legacy.ts` +- Eliminare `backend/src/workspaces/migrate-v2-qdrant.ts` + +### Build e tooling P1 + +- `backend/package.json` +- Creare `backend/scripts/clean-dist.mjs` +- Creare un test Node per il clean build +- `backend/scripts/p1-manual-acceptance.mjs` +- `backend/scripts/p1-manual-acceptance.test.mjs` +- `backend/scripts/p1-render-snapshot.test.mjs` + +### Frontend + +- `frontend/src/api/workspaces.ts` +- `frontend/src/api/sessions.ts` +- `frontend/src/shell/SteerInput.tsx` +- `frontend/src/shell/WorkspaceManager.tsx` +- Test/fixture in `api`, `SteerInput`, `WorkspaceManager`, `NewSessionDialog`, `WorkspacePublishDialog` e `drafts`. + +### Deploy, fixture e verificatori + +- `scripts/workspace-registry-smoke.sh` +- Creare `scripts/fixtures/workspace-registry-smoke.yaml` +- `scripts/test-no-deployment-coupling-scope.sh` +- `scripts/test-windows-clone-contract.ps1` +- `scripts/verify-workspace-install-docs.sh` +- `scripts/test-verify-workspace-install-docs.sh` + +### Documentazione corrente + +- `README.md` +- sezione corrente di `PROJECT_STATE.md`, prima di `## Historical snapshots` +- `docs/workspace-diagnostic-protocol.md` +- `docs/install/local-workspace-registry.md` +- `docs/install/server-workspace-registry.md` + +--- + +### Task 0: Congelare scope e baseline prima delle modifiche + +**File:** nessuna modifica produttiva. + +- [ ] Registrare `BASE_SHA=$(git rev-parse HEAD)` e verificare che gli altri piani non vengano inclusi nei commit di implementazione. +- [ ] Salvare l'inventario iniziale dei simboli descriptor-legacy: + +```bash +git grep -nE 'WorkspaceV1|WorkspaceV2|LegacyWorkspace|migration_required|migrate-legacy|migrateWorkspaceV1ToV2|migrateWorkspaceV2ToV3' -- \ + backend/src backend/test backend/scripts frontend/src scripts README.md PROJECT_STATE.md docs/install docs/workspace-diagnostic-protocol.md +``` + +- [ ] Classificare ogni risultato come descriptor legacy, compatibility decoder previsto, contratto diverso o documento storico. +- [ ] Verificare nei registry/installazioni disponibili che i descriptor attivi siano v3; questa è una precondizione di deploy, non un migratore. +- [ ] Non procedere se il worktree contiene modifiche applicative non attribuibili a questo piano. + +### Task 1: Scrivere i test RED del contratto v3-only + +**File:** +- `backend/test/workspaces-schema.test.ts` +- `backend/test/workspace-registry.test.ts` +- `backend/test/routes-workspaces.test.ts` + +- [ ] Aggiungere test che `parseWorkspaceYaml`, `validateWorkspaceDescriptor` e le route validate/publish rifiutino esplicitamente v1 e v2. +- [ ] Aggiungere test registry per: + - bootstrap pulito con solo v1/v2: fallimento, nessun `active.json` pubblicato; + - repository misto v3+v2: attivazione atomica rifiutata; + - pull che introduce v1/v2: precedente active state ancora leggibile; + - retained snapshot contenente descriptor non-v3: rifiuto fail-closed; + - risposta API state-free. +- [ ] Eseguire: + +```bash +cd backend +npx vitest run test/workspaces-schema.test.ts test/workspace-registry.test.ts test/routes-workspaces.test.ts +``` + +Atteso: RED per i nuovi requisiti, non errori di fixture casuali. + +### Task 2: Rendere lo schema backend esclusivamente v3 + +**File:** +- `backend/src/workspaces/schema.ts` +- `backend/src/workspaces/types.ts` +- test del Task 1 + +- [ ] Eliminare `WorkspaceV1`, `WorkspaceV2`, `LegacyWorkspace`, relativi Zod schema e `migrateWorkspaceV1ToV2`. +- [ ] Rendere `WorkspaceDescriptorSchema = WorkspaceV3Schema`. +- [ ] Eliminare `validateCanonicalWorkspace`, aggiornando **tutti** i chiamanti in `routes/workspaces.ts`, incluso il chiamante attualmente oltre quelli elencati nel vecchio piano. +- [ ] Eliminare `isCanonicalWorkspace`/`isOperationalWorkspace` dopo aver sostituito i rami condizionali con validazione v3 diretta. +- [ ] Conservare test negativi v1/v2; non cancellare le sole prove che impediscono una regressione futura. +- [ ] Eseguire test focalizzati e typecheck. +- [ ] Commit: `refactor: make workspace descriptors schema v3 only`. + +### Task 3: Normalizzare in sicurezza active state e snapshot manifest + +**File:** +- `backend/src/workspaces/registry.ts` +- `backend/test/workspace-registry.test.ts` + +- [ ] Scrivere RED per state/manifest con: + - campo assente; + - vecchio `state: "operational"`; + - `state: "migration_required"`; + - valore sconosciuto; + - campo extra; + - active state e manifest con formati misti; + - snapshot attivo, storico e fallback offline. +- [ ] Rimuovere `state` da `WorkspaceRevision` e da tutti i nuovi writer. +- [ ] Sostituire cast e vecchie migrazioni con decoder stretti che restituiscono oggetti normalizzati state-free. +- [ ] Rimuovere `LegacyWorkspaceRevision`, `LegacyActiveState`, `LegacySnapshotManifest`, `deriveStateFromLegacyRevisions`, `migrateLegacyActiveState`, `migrateLegacySnapshotManifest`, `sameLegacyRevisions` e le condizioni operative basate su `state`. +- [ ] Mantenere tutti i controlli di integrità e far validare ogni YAML come v3. +- [ ] Provare che list/read/API non riemettono il vecchio campo anche immediatamente dopo un restart, prima di una nuova attivazione. +- [ ] Commit: `refactor: remove workspace revision state`. + +### Task 4: Eliminare i rami v1/v2 da renderer, contracts, bindings e diagnostica + +**File:** +- `backend/src/workspaces/runtime-renderer.ts` +- `backend/src/workspaces/contracts.ts` +- `backend/src/workspaces/diagnostics.ts` +- `backend/src/workspaces/bindings.ts` +- relativi test + +- [ ] Scrivere/aggiornare test RED che accettano v3 e rifiutano input non-v3 al confine, senza renderer/diagnoser legacy. +- [ ] Eliminare il renderer v2/pgvector e i rami v1. +- [ ] Eliminare variabili contract e diagnostica solamente v2. +- [ ] Semplificare bindings dopo la validazione v3, senza indebolire validazione secrets/trasporti. +- [ ] Eseguire i test focalizzati: + +```bash +cd backend +npx vitest run \ + test/workspace-runtime-renderer.test.ts \ + test/workspaces-contracts.test.ts \ + test/workspaces-diagnostics.test.ts \ + test/workspaces-bindings.test.ts \ + test/workspace-runtime-handoff.test.ts +``` + +- [ ] Commit: `refactor: remove legacy workspace runtime branches`. + +### Task 5: Rimuovere migratori senza perdere test di deployment non correlati + +**File:** +- Eliminare i due migratori e i test esclusivamente di migrazione. +- Creare/spostare in un test dedicato le prove deployment presenti in `workspaces-migrate-legacy.test.ts:81-114`. + +- [ ] Prima di eliminare `workspaces-migrate-legacy.test.ts`, spostare in un file con nome coerente: + - volume registry durevole e mount Git read-only; + - contratto Dockerfile; + - fallback offline smoke; + - self-test di cleanup dell'immagine per-run. +- [ ] Eliminare `migrate-legacy.ts`, `migrate-v2-qdrant.ts` e i test di trasformazione. +- [ ] Conservare un fixture v2 soltanto nei test negativi di rifiuto. +- [ ] Eseguire i nuovi test deployment e il typecheck. +- [ ] Commit: `refactor: remove workspace migration utilities`. + +### Task 6: Aggiornare tutte le route backend e il tooling P1 + +**File:** +- `backend/src/routes/workspaces.ts` +- `backend/src/routes/sessions.ts` +- `backend/src/routes/sql.ts` +- test route inclusi `routes-sql-meta.test.ts` +- `backend/scripts/p1-manual-acceptance.mjs` +- test manual/render P1 + +- [ ] Rimuovere filtri/gate `revision.state` da tutte le route. La garanzia deriva dal registry v3-only. +- [ ] Aggiornare mock/fixture `WorkspaceRevision` in tutti i test backend. +- [ ] Aggiornare il validatore del manifest P1 manuale affinché richieda esattamente la revisione state-free. +- [ ] Aggiornare i fixture `p1-manual-acceptance.test.mjs` e `p1-render-snapshot.test.mjs`. +- [ ] Aggiungere un test JS specifico che rifiuti manifest con revisioni malformate senza reintrodurre `migration_required`. +- [ ] Eseguire: + +```bash +cd backend +npx vitest run test/routes-workspaces.test.ts test/routes-sessions.test.ts test/routes-sql-meta.test.ts +cd .. +node --test --test-concurrency=1 \ + backend/scripts/p1-manual-acceptance.test.mjs \ + backend/scripts/p1-render-snapshot.test.mjs +``` + +- [ ] Commit: `refactor: remove workspace revision state consumers`. + +### Task 7: Rendere il build backend clean-first + +**File:** +- `backend/package.json` +- Creare `backend/scripts/clean-dist.mjs` +- Creare test Node del clean build + +- [ ] Scrivere RED: creare un file sentinella in `backend/dist/workspaces/`, eseguire il clean/build e verificare che non sopravviva. +- [ ] Implementare la pulizia con API Node multipiattaforma, non con `rm -rf` nella npm script. +- [ ] Fare eseguire il clean prima di `tsc` da `npm run build`. +- [ ] Verificare dopo il build: + +```bash +test ! -e backend/dist/workspaces/migrate-legacy.js +test ! -e backend/dist/workspaces/migrate-v2-qdrant.js +``` + +- [ ] Costruire l'immagine core in un contesto pulito e verificare che i due moduli non esistano nell'immagine. +- [ ] Verificare che i manifest di integrità P1 continuino a legare l'intero nuovo `dist`. +- [ ] Commit: `build: remove stale backend distribution files`. + +### Task 8: Aggiornare frontend e contratto API state-free + +**File:** +- `frontend/src/api/workspaces.ts` +- `frontend/src/api/sessions.ts` +- `frontend/src/shell/SteerInput.tsx` +- `frontend/src/shell/WorkspaceManager.tsx` +- test/fixture frontend correlati + +- [ ] Scrivere/aggiornare test per revisioni senza `state` e risposta non-v3 rifiutata al confine workspace. +- [ ] Eliminare `state` dal tipo e dal parser revisionale. +- [ ] Rimuovere gate/banner/filtro `migration_required` e anche la visualizzazione `record.revision.state`. +- [ ] Mantenere `allowLegacy` per sessioni senza revisione. +- [ ] Aggiornare fixture in: + - `api/workspaces.test.ts`, `api/sessions.test.ts`; + - `SteerInput.test.tsx`, `WorkspaceManager.test.tsx`; + - `NewSessionDialog.test.tsx`, `WorkspacePublishDialog.test.tsx`; + - `drafts.test.ts`, mantenendo il test negativo di schema non-3. +- [ ] Documentare che core e frontend devono essere aggiornati insieme; il parser nuovo non usa più `state`. +- [ ] Eseguire typecheck e suite frontend. +- [ ] Commit: `refactor: remove legacy workspace UI state`. + +### Task 9: Sostituire fixture e smoke con descriptor v3 completi + +**File:** +- `scripts/workspace-registry-smoke.sh` +- Creare `scripts/fixtures/workspace-registry-smoke.yaml` +- `scripts/test-no-deployment-coupling-scope.sh` +- `scripts/test-windows-clone-contract.ps1` +- test deployment spostati nel Task 5 + +- [ ] Creare un descriptor v3 completo `id: local`, collection `local`, embedding interno 1024/cosine, LLM policy e diagnostica DWH; omettere Evidence per non richiedere un tree Git nello smoke registry. +- [ ] Validare il fixture con il parser produttivo in un test backend. +- [ ] Copiare il fixture nello seed repository e rimuovere sia l'invocazione del migratore sia il build backend ormai inutile allo smoke. +- [ ] Nel test Windows non cambiare soltanto il numero di versione: fornire il contratto v3 completo mantenendo lo scopo path-with-spaces/clone. +- [ ] Aggiornare il fixture dello scope coupling senza indebolire l'assenza-gate. +- [ ] Eseguire test shell focalizzati e, con Docker disponibile, lo smoke reale senza retry. +- [ ] Commit: `test: replace legacy workspace deployment fixtures`. + +### Task 10: Aggiornare documentazione corrente e relativi verifier + +**File:** +- documenti/verifier indicati nella mappa + +- [ ] Aggiornare README e soltanto la sezione corrente di `PROJECT_STATE.md`; non riscrivere gli snapshot storici. +- [ ] Eliminare procedure di migrazione v1/v2 dai manuali local/server e dal protocollo diagnostico. +- [ ] Modificare `verify-workspace-install-docs.sh` perché richieda “schema v3 only” e l'assenza di `migration_required` nella documentazione corrente. +- [ ] Aggiornare i fixture negativi del test del verifier. +- [ ] Non cambiare gli usi di `migration_required` nei piani P3–P6 relativi a ownership/artifact diversi. +- [ ] Eseguire: + +```bash +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/verify-workspace-install-docs.sh --fixtures-only +``` + +- [ ] Commit: `docs: make schema v3 the only workspace contract`. + +### Task 11: Eseguire absence gate e suite complete + +- [ ] Eseguire backend clean build, typecheck e test: + +```bash +cd backend +npm run build +npx tsc --noEmit -p . +npx vitest run +``` + +- [ ] Eseguire frontend: + +```bash +cd frontend +npx tsc -b +npx vitest run +npm run build +``` + +- [ ] Eseguire script/verifier interessati, incluso lo smoke Docker obbligatorio se l'ambiente dispone di Docker. Non lasciarlo “opzionale” in una consegna che modifica lo smoke. +- [ ] Eseguire `git diff --check`. +- [ ] Eseguire l'absence gate ristretto: + +```bash +git grep -nE 'WorkspaceV1|WorkspaceV2|LegacyWorkspace|migrateWorkspaceV1ToV2|migrateWorkspaceV2ToV3' -- \ + backend/src frontend/src scripts && exit 1 || true + +git grep -nE 'migration_required|migrate-legacy|migrate-v2-qdrant' -- \ + backend/src backend/scripts frontend/src scripts README.md docs/install docs/workspace-diagnostic-protocol.md && exit 1 || true + +test ! -e backend/dist/workspaces/migrate-legacy.js +test ! -e backend/dist/workspaces/migrate-v2-qdrant.js +``` + +Nota: trasformare questi esempi in uno script con allowlist esplicita; non affidarsi a `&& exit 1 || true`, che può mascherare errori di esecuzione. Lo script deve distinguere “nessun match” da errore Git/I/O. + +- [ ] Ispezionare il diff per assicurarsi che nessun formato non-descriptor sia stato modificato. + +### Task 12: Rigenerare l'evidenza automatica P1 + +- [ ] Partire dal commit sorgente finale pulito. +- [ ] Eseguire una sola integrazione completa, senza retry automatico: + +```bash +./scripts/p1-acceptance.sh integration --keep +``` + +- [ ] Verificare report JSON/Markdown, hash dichiarati, manifest sorgente/dist, secret scan, ownership cleanup e porte chiuse. +- [ ] Aggiornare `PROJECT_STATE.md` con il nuovo commit/tree/report e con stati distinti: + +```text +automated integration: PASS +manual acceptance: PENDING +``` + +- [ ] Committare soltanto lo stato tracciato, mai `.artifacts`. +- [ ] Non riusare l'evidenza precedente legata a `c733896`. + +### Task 13: Riaprire e chiudere il gate manuale P1 + +- [ ] Preparare un ambiente manuale nuovo: + +```bash +./scripts/p1-manual-acceptance.sh prepare +./scripts/p1-manual-acceptance.sh serve +``` + +- [ ] Il reviewer segue integralmente il nuovo `GUIDE.md`, verificando anche che revisioni/API/manifest siano state-free e che v1/v2 siano rifiutati senza mutazione. +- [ ] Arrestare il server e verificare porte/processi: + +```bash +./scripts/p1-manual-acceptance.sh stop +``` + +- [ ] Solo il reviewer crea `VERDICT.md` e decide PASS/FAIL. +- [ ] Se PASS, aggiornare `PROJECT_STATE.md` e committare `docs: record schema-v3-only P1 acceptance`. +- [ ] Pulire il lab soltanto dopo conferma del reviewer. +- [ ] **STOP:** non iniziare P2 finché il reviewer non approva esplicitamente il nuovo P1. + +--- + +## Criteri finali di accettazione + +1. Nessun descriptor v1/v2 viene parsato, pubblicato, attivato, renderizzato, diagnosticato o mostrato. +2. I vecchi state file di revisioni v3 con `state: "operational"` continuano a caricarsi, ma API e nuovi file sono state-free. +3. Descriptor non-v3 o state incoerenti falliscono senza sostituire il precedente active state. +4. Nessun migratore sopravvive in sorgenti, `dist`, immagine core, script o documentazione corrente. +5. I formati versionati non collegati ai workspace descriptor sono invariati. +6. Backend, frontend, verifier, smoke e build interessati sono verdi. +7. Una nuova integrazione P1 è PASS al commit finale. +8. La nuova acceptance manuale P1 è decisa esplicitamente dal reviewer. +9. P2 resta non iniziato fino a ulteriore autorizzazione. diff --git a/docs/plans/2026-08-14-read-only-workspace-runtime-secrets-design.md b/docs/plans/2026-08-14-read-only-workspace-runtime-secrets-design.md new file mode 100644 index 00000000..1e7083bd --- /dev/null +++ b/docs/plans/2026-08-14-read-only-workspace-runtime-secrets-design.md @@ -0,0 +1,189 @@ +# Read-only Workspace Repository and Runtime Secrets Design + +**Date:** 2026-08-14 +**Status:** Approved + +## Purpose + +ThothII consumes workspaces from one administrator-configured Git repository. Workspace authors +prepare and publish source outside ThothII. The application fetches, validates, and activates +repository revisions, but never edits, commits, pushes, imports, or exports workspace source. + +Runtime credentials are intentionally absent from Git. After a workspace has been read, ThothII +derives the required credentials from its connector and authentication choices and lets an +authorized user complete them in the web application. The values are encrypted and persisted by +the backend; the browser retains neither workspace content nor secrets. + +## Ownership boundaries + +### Workspace source + +The workspace source is an ordinary directory maintained outside the ThothII runtime. It contains +the catalog, each `workspace.yaml`, curated evidence, annotations, and other repository-owned +content. Authors validate it using source-side tooling and publish it through their normal Git +workflow to GitHub, GitLab, Gitea, or another standards-compatible server. + +### ThothII installation + +The installation descriptor selects the Git remote, branch, and one read-only authentication +transport. SSH uses a read-only deploy key plus pinned known hosts. HTTPS uses a read-only deploy +token and may provide a private CA. Secret values remain outside versioned configuration. + +The installer performs a sanitized `git ls-remote` preflight. Credentials embedded in a remote URL +are rejected. The API exposes only a normalized repository identity: host, repository path, branch, +transport, active commit, and synchronization state. + +### ThothII runtime + +The local Git checkout, candidate validation area, immutable snapshots, and active state are +application-owned. They are read-only from the workspace-management API. A pull fetches a candidate +revision, validates the complete repository, and atomically activates it only if valid. A failed +candidate never replaces the last valid active revision. + +ThothII never generates or reconciles files back into the checkout and never invokes Git commit or +push. Generated operational artifacts live under application data, not in the source repository. + +## Repository synchronization states + +A repository refresh has these states: + +- `syncing`: fetching and validating a candidate revision; +- `active`: the candidate passed validation and became the active immutable revision; +- `invalid_candidate`: Git succeeded but repository validation failed; the previous revision stays active; +- `unavailable`: Git or authentication failed; the previous revision stays active; +- `empty`: no valid revision has ever been activated. + +Validation is atomic at repository-commit level. A malformed catalog, descriptor, evidence tree, or +cross-file reference rejects the complete candidate revision. + +## Runtime secret model + +### Requirement discovery + +The workspace descriptor contains connector type, authentication method, and non-secret logical +configuration. It never contains secret values or host filesystem paths. Connector adapters define +the secret fields required by each supported authentication method. For example: + +- PostgreSQL `username_password` requires `username` and `password`; +- REST `bearer` requires `api_key`; +- SSH tunnel authentication requires the connector password and SSH private key; +- Evidence HTTP signed URLs and static S3 credentials contribute their own secret requirements. + +Requirements have stable identifiers scoped by workspace and connector. Labels, descriptions, +input kinds, and required/optional status come from trusted application code rather than repository +HTML or executable metadata. + +### Persistent encrypted store + +The backend owns a `WorkspaceSecretStore` abstraction. The first implementation is a local encrypted +vault in application-managed persistent storage. Each secret is encrypted with authenticated +encryption and bound to its installation, workspace, connector, and field identifier as associated +data. Plaintext values never appear in Git, API responses, logs, error messages, diagnostics, or +browser storage. + +The installation bootstraps one vault key independently from workspace content. Deployment tooling +owns its platform-specific provisioning; the workspace schema and GUI never contain filesystem +paths. The storage interface allows a future Vault, cloud secret manager, or OS keychain provider +without changing workspace descriptors or API consumers. + +When an existing file-oriented harness connector needs a credential, the backend materializes it as +a restrictive temporary file in an application-owned runtime directory. Its lifetime is tied to the +diagnostic or runtime lease and it is removed on release. Persistent storage contains ciphertext +only. + +### Secret API + +For a selected workspace the API returns requirement metadata and status only: + +```json +{ + "workspaceId": "psd-clinical", + "state": "configuration_required", + "requirements": [ + { + "id": "dwh.password", + "connector": "dwh", + "label": "Database password", + "input": "password", + "required": true, + "configured": false + } + ] +} +``` + +A write request contains values only for the selected requirement identifiers. The response returns +status, never values. A delete operation forgets a configured value. Authorization is deliberately +deferred; the current authenticated application user may manage runtime workspace secrets. + +Workspace readiness is derived as follows: + +- `invalid`: repository structure or descriptor is invalid; +- `configuration_required`: structurally valid but required runtime values are missing; +- `ready`: required values exist but connectivity has not yet passed or is stale; +- `verified`: the most recent connector diagnostic passed for the active revision and current secret generation. + +Changing or deleting a secret invalidates the previous diagnostic result. + +## Browser behavior + +Workspace management is a two-level read-only interface occupying at least 60 percent of viewport +width and height. + +Level 1 explains the source/runtime separation and displays: + +- normalized repository host and path; +- configured branch and read-only transport; +- active revision and last synchronization result; +- `Update workspace repository`, which fetches, validates, and conditionally activates a revision; +- the workspace list, with selection required for workspace-specific actions. + +There is no Import bundle, Export bundle, Create, Edit, Delete, Publish, or conflict-resolution +operation. There are no browser-persisted workspace drafts or preferences. + +Level 2 for the selected workspace explains and displays: + +- immutable source identity and validation result; +- required runtime configuration grouped by connector; +- secret-entry controls whose values are write-only; +- `Save secrets`, `Forget` per configured value, and `Test workspace connection`; +- clear consequences for each button and a reminder that source changes must be committed and pushed + by an author outside ThothII before repository update. + +The browser keeps form values only in component memory and clears them after submission or dialog +close. It never receives saved secret values. + +## Compatibility and migration + +Existing Git author settings, publish endpoints, bundle endpoints, generated-document +reconciliation, bootstrap catalog slots, and browser draft storage are removed. Existing environment +bindings may be read during a bounded migration period only to seed non-secret connector values; +secret file paths are not part of the new public workspace contract. + +Session manifests continue to pin an immutable validated workspace revision. An already running +session keeps its acquired runtime lease; new or resumed work resolves the current encrypted secret +generation and fails closed when required credentials are unavailable. + +## Failure handling and security + +- Repository and vault errors use stable sanitized codes and never echo remotes with user info, + credential paths, secret identifiers that are not safe to disclose, or secret values. +- Vault writes are atomic and authenticated; corrupted ciphertext fails closed. +- Secret comparison uses no read API. Updating a secret is always a blind replacement. +- The backend applies request-size and field-count limits and rejects unknown requirement IDs. +- Temporary plaintext files use restrictive permissions, trusted directories, no-follow opens, and + deterministic cleanup. +- Git credentials are installation-only, read-only, and never sent to the frontend. + +## Verification + +Backend tests cover repository read-only behavior, atomic candidate activation, remote sanitization, +vault encryption and corruption, requirement discovery, blind secret writes/deletes, materialization +cleanup, readiness transitions, and absence of publish/bundle routes. + +Frontend tests cover the two-level explanation, viewport dimensions, repository identity, selection +gating, dynamic secret forms, write-only behavior, status changes, and absence of local-storage, +import, export, editing, and publishing controls. + +Deployment and CLI tests cover required remote/branch configuration, one read-only Git transport, +sanitized remote preflight, vault-key provisioning, and removal of Git author/write configuration. diff --git a/docs/plans/2026-08-14-read-only-workspace-runtime-secrets.md b/docs/plans/2026-08-14-read-only-workspace-runtime-secrets.md new file mode 100644 index 00000000..d1709de6 --- /dev/null +++ b/docs/plans/2026-08-14-read-only-workspace-runtime-secrets.md @@ -0,0 +1,401 @@ +# Read-only Workspace Runtime Secrets Implementation Plan + +> **Historical nomenclature:** this plan predates the native host CLI convergence. References to +> `thothctl` and `tools/thothctl` describe the implementation snapshot from which this plan was +> written; current operator commands and paths use native `tht` and `tools/tht`. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Make workspace consumption strictly read-only while adding installation-scoped Git identity and persistent GUI-managed runtime secrets. + +**Architecture:** Git remains the source of truth and is fetched into an application-owned checkout; complete candidate commits are validated before atomic activation and the backend has no Git write path. Runtime connector credentials are discovered from trusted connector contracts, stored as authenticated ciphertext by a backend vault, and materialized only for the lifetime of diagnostics or runtime leases. The browser exposes repository/readiness status and write-only secret forms without workspace persistence. + +**Tech Stack:** Fastify, TypeScript, Node.js crypto/filesystem, React 18, TanStack Query, Vitest, Go `thothctl`, Docker Compose. + +--- + +### Task 1: Freeze the Git repository boundary to read-only + +**Files:** +- Modify: `backend/src/workspaces/types.ts` +- Modify: `backend/src/workspaces/git-repository.ts` +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/test/workspaces-git-repository.test.ts` +- Modify: `backend/test/workspace-registry.test.ts` +- Modify: `backend/test/workspace-registry-deployment.test.ts` + +**Step 1: Write failing tests** + +Add tests proving that pull never configures a Git author, writes generated files, commits, or pushes; that a malformed candidate leaves the prior active snapshot intact; and that a missing catalog descriptor rejects the whole candidate instead of producing a bootstrap slot. + +**Step 2: Run the focused tests** + +Run: `cd backend && npx vitest run test/workspaces-git-repository.test.ts test/workspace-registry.test.ts test/workspace-registry-deployment.test.ts` + +Expected: FAIL on write/publish behavior and missing-descriptor semantics. + +**Step 3: Implement the read-only boundary** + +Remove `gitAuthorName`, `gitAuthorEmail`, mutation helpers, generated-document reconciliation, publish/conflict types, and bootstrap-slot activation. `pull()` must fetch, validate the complete commit in a candidate snapshot, and replace active state only after validation succeeds. + +**Step 4: Run focused tests** + +Run the command from Step 2. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend/src/workspaces backend/test/workspaces-git-repository.test.ts backend/test/workspace-registry.test.ts backend/test/workspace-registry-deployment.test.ts +git commit -m "refactor: make workspace repository strictly read only" +``` + +### Task 2: Remove publishing and bundle HTTP contracts + +**Files:** +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/test/routes-workspaces.test.ts` +- Modify: `backend/test/workspaces-runtime-v3-boundaries.test.ts` +- Modify: `backend/src/config.ts` +- Modify: `backend/test/workspaces-config.test.ts` + +**Step 1: Write failing route tests** + +Assert `POST /workspaces/publish`, `GET /workspaces/:id/export`, and `POST /workspaces/import` return 404 and that the backend no longer registers multipart or ZIP handling. Assert configuration no longer accepts Git author or bundle-limit settings as workspace-registry fields. + +**Step 2: Run tests and observe failure** + +Run: `cd backend && npx vitest run test/routes-workspaces.test.ts test/workspaces-config.test.ts test/workspaces-runtime-v3-boundaries.test.ts` + +Expected: FAIL because mutation and bundle routes still exist. + +**Step 3: Remove the mutation surface** + +Delete publish/import/export schemas and helpers, remove `multipart`, `yauzl`, and `yazl` usage from the route, and simplify safe workspace errors to read/validate/sync errors. + +**Step 4: Run tests** + +Run the command from Step 2 plus `cd backend && npx tsc --noEmit -p .`. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend/src backend/test package.json package-lock.json +git commit -m "refactor: remove workspace publishing and bundles" +``` + +### Task 3: Expose a sanitized installation repository identity + +**Files:** +- Modify: `backend/src/workspaces/git-repository.ts` +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/test/workspaces-git-repository.test.ts` +- Modify: `backend/test/routes-workspaces.test.ts` +- Modify: `tools/thothctl/internal/config/installation.go` +- Modify: `tools/thothctl/internal/config/installation_test.go` +- Modify: `deploy/psd/thothii-installation.yaml.example` +- Modify: `docs/install/examples/thothii-installation.local.yaml` +- Modify: `docs/install/examples/thothii-installation.server.yaml` + +**Step 1: Write failing parser and status tests** + +Cover HTTPS, SSH URL, and SCP-style remotes; reject embedded user-info for HTTPS; return only `host`, `repository`, `branch`, and `transport`; never return a token, key path, or raw credential-bearing URL. Add installation-descriptor tests for a required `workspaceRepository` block and exactly one read-only transport. + +**Step 2: Run focused tests** + +Run: `cd backend && npx vitest run test/workspaces-git-repository.test.ts test/routes-workspaces.test.ts && cd ../tools/thothctl && go test ./internal/config` + +Expected: FAIL because repository identity and typed installation configuration do not exist. + +**Step 3: Implement safe normalization and installation validation** + +Add the normalized identity to registry status. Extend `thothii-installation.yaml` with remote, branch, and SSH/HTTPS access metadata, validate it against the selected Compose override and environment without reading or returning secret values, and retain the existing environment rendering boundary. + +**Step 4: Run focused tests** + +Run the command from Step 2. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend tools/thothctl deploy docs/install/examples +git commit -m "feat: declare workspace repository in installation config" +``` + +### Task 4: Add the persistent encrypted workspace secret store + +**Files:** +- Create: `backend/src/workspaces/secret-store.ts` +- Create: `backend/test/workspace-secret-store.test.ts` +- Modify: `backend/src/config.ts` +- Modify: `backend/src/app.ts` +- Modify: `compose.yaml` +- Modify: `deploy/compose.local.yaml` +- Modify: `deploy/compose.server.yaml` + +**Step 1: Write failing vault tests** + +Test first-start initialization, atomic blind replacement, deletion, enumeration by configured ID only, AES-256-GCM ciphertext with installation/workspace/field associated data, corruption failure, restrictive files/directories, size limits, and absence of plaintext in persistent bytes. + +**Step 2: Run the vault test** + +Run: `cd backend && npx vitest run test/workspace-secret-store.test.ts` + +Expected: FAIL because `WorkspaceSecretStore` does not exist. + +**Step 3: Implement the vault** + +Create an injectable `WorkspaceSecretStore` backed by an application-managed data root. Persist a versioned encrypted document atomically, generate or load the installation vault key in the private control area, expose only `has`, `put`, `delete`, and scoped materialization operations, and never add a plaintext read API. + +**Step 4: Run tests and typecheck** + +Run: `cd backend && npx vitest run test/workspace-secret-store.test.ts && npx tsc --noEmit -p .` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend compose.yaml deploy +git commit -m "feat: persist encrypted workspace runtime secrets" +``` + +### Task 5: Derive connector requirements and integrate temporary materialization + +**Files:** +- Create: `backend/src/workspaces/secret-requirements.ts` +- Create: `backend/test/workspace-secret-requirements.test.ts` +- Modify: `backend/src/workspaces/bindings.ts` +- Modify: `backend/src/workspaces/runtime-config-lease.ts` +- Modify: `backend/src/tht/tht-runner.ts` +- Modify: `backend/src/app.ts` +- Modify: `backend/test/workspace-runtime-config-lease.test.ts` +- Modify: `backend/test/workspace-runtime-handoff.test.ts` +- Modify: `backend/test/workspaces-bindings.test.ts` + +**Step 1: Write failing requirement and lifecycle tests** + +Cover PostgreSQL password, REST bearer API key, unauthenticated REST, SSH private key/password, signed HTTP Evidence, and static S3 credentials. Assert temporary files are restrictive, live for exactly one diagnostic/runtime lease, disappear on release and error, and are never persisted in the encrypted vault document. + +**Step 2: Run focused tests** + +Run: `cd backend && npx vitest run test/workspace-secret-requirements.test.ts test/workspaces-bindings.test.ts test/workspace-runtime-config-lease.test.ts test/workspace-runtime-handoff.test.ts` + +Expected: FAIL because requirements still come from installation secret-file paths. + +**Step 3: Implement dynamic requirement resolution** + +Use the selected DWH transport and Evidence authentication contract to map trusted installation-contract suffixes to stable GUI requirement IDs. Overlay materialized temporary file paths only while resolving existing file-oriented connectors, and attach cleanup to every runtime lease. + +**Step 4: Run tests and typecheck** + +Run the command from Step 2 plus `cd backend && npx tsc --noEmit -p .`. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend/src backend/test +git commit -m "feat: resolve workspace secrets from connector requirements" +``` + +### Task 6: Add write-only workspace secret and readiness APIs + +**Files:** +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/src/app.ts` +- Modify: `backend/src/workspaces/types.ts` +- Modify: `backend/test/routes-workspaces.test.ts` + +**Step 1: Write failing API tests** + +Test `GET /workspaces/:id/runtime-configuration`, blind `PUT /workspaces/:id/secrets`, and `DELETE /workspaces/:id/secrets/:requirementId`. Assert strict bodies, limits, unknown-ID rejection, status-only responses, diagnostic invalidation, and `configuration_required`/`ready` state transitions. + +**Step 2: Run tests** + +Run: `cd backend && npx vitest run test/routes-workspaces.test.ts` + +Expected: FAIL because the routes do not exist. + +**Step 3: Implement the routes and readiness projection** + +Inject the secret store into workspace routes and runtime support. Compute per-workspace readiness from active descriptor, current requirement set, configured IDs, and diagnostic generation. Materialize values only inside the diagnostic request and always clean up. + +**Step 4: Run backend gates** + +Run: `cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build`. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend +git commit -m "feat: manage runtime workspace secrets through the API" +``` + +### Task 7: Replace workspace management with the two-level read-only UI + +**Files:** +- Modify: `frontend/src/api/workspaces.ts` +- Modify: `frontend/src/api/workspaces.test.ts` +- Modify: `frontend/src/shell/WorkspaceManager.tsx` +- Modify: `frontend/src/shell/WorkspaceManager.test.tsx` +- Delete: `frontend/src/shell/WorkspacePublishDialog.tsx` +- Delete: corresponding publish-dialog tests +- Modify/Delete: `frontend/src/shell/WorkspaceEditor.tsx` and bootstrap-only tests as references permit +- Modify: `frontend/src/workspaces/drafts.ts` +- Modify: `frontend/src/workspaces/drafts.test.ts` + +**Step 1: Write failing UI/API tests** + +Assert the dialog uses at least 60% viewport width and height, shows general repository concepts and exact button consequences at level 1, gates workspace-specific controls on selection, renders requirement explanations and write-only fields at level 2, and has no create/edit/publish/import/export/bundle controls. + +**Step 2: Run focused tests** + +Run: `cd frontend && npx vitest run src/api/workspaces.test.ts src/shell/WorkspaceManager.test.tsx src/workspaces/drafts.test.ts` + +Expected: FAIL on the old draft/publish interface. + +**Step 3: Implement the read-only interface** + +Replace bootstrap editor state with repository status, selection, validation/readiness details, dynamic secret fields, blind save/forget actions, and connection test. Remove workspace draft persistence and clear secret field component state after submit/close. + +**Step 4: Run focused tests and typecheck** + +Run the command from Step 2 plus `cd frontend && npx tsc -b`. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add frontend +git commit -m "feat: add read-only workspace and secret management UI" +``` + +### Task 8: Remove browser-persisted workspace preferences + +**Files:** +- Modify: `frontend/src/workspaces/preferences.ts` +- Modify: `frontend/src/workspaces/preferences.test.ts` +- Modify: `frontend/src/api/sessions.ts` +- Modify: `frontend/src/api/sessions.test.ts` +- Modify: `frontend/src/shell/SteerInput.tsx` +- Modify: `frontend/src/shell/SteerInput.test.tsx` + +**Step 1: Write failing persistence-boundary tests** + +Assert workspace/model/thinking choices are kept only in current application memory or saved through the existing backend settings API, and that no workspace code calls `localStorage`. + +**Step 2: Run focused tests** + +Run: `cd frontend && npx vitest run src/workspaces/preferences.test.ts src/api/sessions.test.ts src/shell/SteerInput.test.tsx` + +Expected: FAIL because preferences still use browser storage. + +**Step 3: Implement ephemeral preferences** + +Replace the storage adapter with an in-memory external store seeded from backend settings. Preserve concurrent workspace-policy gates and session request determinism without persisting selections in the browser. + +**Step 4: Run frontend gates** + +Run: `cd frontend && npx vitest run && npx tsc -b && npm run build`. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add frontend +git commit -m "refactor: stop persisting workspace state in the browser" +``` + +### Task 9: Update deployment contracts and documentation + +**Files:** +- Modify: `compose.yaml` +- Modify: `deploy/compose.git-ssh.yaml` +- Modify: `deploy/compose.git-https.yaml` +- Modify: `deploy/workspace-registry.env.example` +- Modify: `deploy/psd/operator.env.example` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `docs/guida-utente.md` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/workspace-registry-smoke.sh` + +**Step 1: Update executable contract tests first** + +Require read-only Git wording and configuration, repository identity visibility, vault persistence, +and absence of author/push/bundle/browser-secret instructions. + +**Step 2: Run contract tests and observe failure** + +Run: `bash scripts/verify-workspace-install-docs.sh` + +Expected: FAIL against the old manuals and examples. + +**Step 3: Update deployment and manuals** + +Remove Git author settings and write-oriented documentation. Document installation Git bootstrap, +GUI runtime-secret completion, platform-neutral application storage, rotation/forget flows, and +candidate validation semantics. + +**Step 4: Run contract and Go gates** + +Run: `bash scripts/verify-workspace-install-docs.sh && cd tools/thothctl && go test ./...` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add compose.yaml deploy docs scripts tools/thothctl +git commit -m "docs: describe read-only workspace runtime configuration" +``` + +### Task 10: Full verification and deployed-container refresh + +**Files:** +- Modify only files needed to fix failures found by verification. + +**Step 1: Run static and unit gates** + +```bash +cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build +cd ../frontend && npx vitest run && npx tsc -b && npm run build +cd ../harness && .venv/bin/pytest -q +cd ../tools/thothctl && go test ./... +``` + +Expected: all gates PASS. + +**Step 2: Run deployment contract gates** + +Run: `bash scripts/verify-workspace-install-docs.sh` and the focused workspace registry smoke appropriate to the configured installation. + +Expected: PASS without Git writes or secret disclosure. + +**Step 3: Inspect the final diff and secret scan** + +Run: `git diff --check`, inspect `git status --short`, and search active code/config for removed publish, bundle, Git author, and workspace-localStorage contracts. + +Expected: no whitespace errors, no accidental secrets, and only intended changes. + +**Step 4: Rebuild and restart affected services** + +Use the installation-aware `thothctl` lifecycle for the configured installation to rebuild/restart `core` and `frontend`, then verify health and repository status. Do not restart if no valid local installation descriptor is available; report that external gate explicitly. + +**Step 5: Commit verification fixes** + +```bash +git add +git commit -m "test: verify read-only workspace secret flow" +``` diff --git a/docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md b/docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md new file mode 100644 index 00000000..930194a5 --- /dev/null +++ b/docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md @@ -0,0 +1,408 @@ +# ThothII Authentication Acceptance and PSD Deployment Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to execute this plan task-by-task. + +**Goal:** Validate local and OIDC authentication on macOS, deploy the exact feat/thoth-auth candidate to the Aritmolab/PSD server before merging it into main, and complete end-to-end acceptance with remote Authentik. + +**Architecture:** Test the candidate first as a standalone local installation. Then install the same immutable Git revision on the existing PSD installation with the installation-aware tht lifecycle, leaving main untouched. Authentik provides OIDC login and a mandatory direct groups claim; ThothII maps exact external groups to roles and validates mapped groups through the Authentik catalog API. + +**Tech Stack:** macOS, Docker Desktop, Docker Compose, native host `tht` plus Python workflow `tht`, local Argon2id authentication, generic OIDC Authorization Code + PKCE, Authentik, PSD workspace registry, reverse proxy/TLS. + +--- + +## Scope and release rules + +Do not merge feat/thoth-auth into main until every mandatory gate in Task 9 is PASS and the PSD owner accepts the evidence. + +Capture one candidate revision and reuse it everywhere: + +```bash +export CANDIDATE_SHA="$(git rev-parse HEAD)" +git fetch origin feat/thoth-auth +test "$CANDIDATE_SHA" = "$(git rev-parse origin/feat/thoth-auth)" +git show -s --format='%H%n%P%n%s' "$CANDIDATE_SHA" +git status --short --untracked-files=all +``` + +Never deploy a moving branch name without checking its resolved SHA. Never put passwords, OIDC client secrets, Authentik API tokens, cookies, authorization headers, raw ID tokens, or password hashes in Git, shell history, screenshots, logs, or evidence. + +Use protected operator values for , , , , , , , and . + +The Authentik contract is mandatory: a direct non-empty JSON array claim named groups; exact groups TOT Users and TOT Admin; mappings TOT Users -> user and TOT Admin -> admin; and a separate group-view-only API service account exposed only as THT_AUTHENTIK_API_TOKEN. Extra upstream groups are valid and silently ignored. + +## Task 0: Freeze the candidate and collect approvals + +**Files:** None; record results in the acceptance report in Task 9. + +Run from the candidate worktree: + +```bash +git diff --check +go test ./... -count=1 +go test -race ./... +go vet ./... +go build ./... +``` + +Expected: all commands pass, the candidate is pushed, and existing evidence is bound to the same SHA. A historical result from another revision is not evidence for this run. + +Before touching PSD, obtain the maintenance window, server access, public URL, Authentik provider details, protected secret locations, test identities for ordinary/admin/unmapped users, and permission to test PSD DWH/Evidence connections. + +## Task 1: Prepare and start the local macOS installation + +**Files:** + +- Read: docs/install/local.md +- Read: docs/install/authentication-local.md +- Read: docs/testing/authentication-manual-acceptance.md +- Use: an untracked local installation descriptor and protected secret/password files + +**Step 1: Verify prerequisites** + +```bash +docker version +docker compose version +bash scripts/verify-line-endings.sh +``` + +Expected: Docker Desktop and Compose are available and line-ending validation passes. + +**Step 2: Build and configure** + +```bash +bash scripts/build-local.sh +bash scripts/build-tht.sh +tht setup --profile local +``` + +For an existing installation, do not overwrite data; run tht --installation update --check-only instead of setup. + +**Step 3: Start and inspect** + +```bash +tht --installation start --build +tht --installation status +tht --installation doctor --json +curl --fail http://127.0.0.1:8080/health +curl --fail http://127.0.0.1:8787/health +``` + +Expected: core, frontend, qdrant, embedding, and the completed model initializer are healthy; doctor includes authentication after configuration and before services. + +## Task 2: Configure and test local login + +**Files:** + +- Read: docs/install/authentication-local.md +- Modify only protected installation state through tht auth configure and tht auth user + +**Step 1: Bootstrap the administrator** + +```bash +tht --installation auth configure \ + --mode local --public-url http://127.0.0.1:8080 \ + --admin-user --admin-display-name \ + --password-file +``` + +Remove the temporary password file immediately. Expected: non-secret auth.yaml is created and the user store contains Argon2id hashes, never plaintext passwords. + +**Step 2: Add and inspect a normal user** + +```bash +tht --installation auth user add --role user --display-name --password-file +tht --installation auth status --json +tht --installation auth check --json +``` + +Expected: pristine redacted JSON and no credential, hash, or session secret in output. + +**Step 3: Test browser authorization** + +At http://127.0.0.1:8080, in a private browser profile: + +1. Verify unauthenticated access reaches login and protected routes are denied. +2. Log in as the normal user and verify application/session routes work. +3. Verify Pi Management and other admin-only operations return HTTP 403 or are not exposed. +4. Log out and verify the session is invalidated. +5. Log in as the administrator and verify admin-only routes work. + +Expected: ordinary users authenticate without receiving admin permissions; administrators receive the configured admin permission set. + +**Step 4: Test account failure paths** + +Use auth user disable, enable, set-password, and logout-all user --yes on the test user. Test a wrong password and refresh the old browser session after logout-all. + +Expected: generic safe failures, disabled login rejection, re-enabled login success, and forced reauthentication. The last enabled administrator cannot be disabled or demoted. + +## Task 3: Test remembered sessions and local recovery + +**Files:** + +- Read: docs/architecture/authentication.md, Browser sessions +- Read: docs/install/authentication-local.md, Session behavior and recovery + +**Step 1: Test browser restart** + +Log in as the normal user with Remember me, close the browser completely, reopen it, and revisit the application. + +Expected: the session survives within the 7-day idle / 30-day absolute limits. Do not record the cookie. + +**Step 2: Test ThothII restart** + +```bash +tht --installation stop +tht --installation start +``` + +Expected: the remembered session remains valid after backend restart. + +**Step 3: Test invalidation** + +Change the test user password or role, and separately run auth user logout-all user --yes. Refresh after each operation. + +Expected: affected sessions are rejected and reauthentication is required; configuration revision changes invalidate all sessions. + +Go/no-go: do not proceed to PSD if local login, role separation, logout, or remembered-session behavior fails. + +## Task 4: Snapshot the current PSD installation + +**Files:** + +- Read: docs/install/server.md +- Read: docs/install/server-workspace-registry.md +- Use: protected server operator and backup locations + +**Step 1: Capture live state** + +```bash +THT_BIN= +INSTALLATION= +"$THT_BIN" --installation "$INSTALLATION" status +"$THT_BIN" --installation "$INSTALLATION" doctor +"$THT_BIN" --installation "$INSTALLATION" pi status +"$THT_BIN" --installation "$INSTALLATION" pi doctor +git -C /srv/thothii/source/ThothII status --short --untracked-files=all +git -C /srv/thothii/source/ThothII rev-parse HEAD +``` + +Save the live SHA as and capture image identities, workspace registry status, and maintenance/recovery state. Stop if the checkout is dirty or recovery is pending. + +**Step 2: Drain and back up** + +Announce maintenance, close the reverse proxy or show its maintenance page, drain active work, and stop through tht. Create the protected, checksummed backup specified in docs/install/server.md, including runtime trees and PSD PostgreSQL/session data where applicable. Back up credentials separately. Never run docker compose down --volumes. + +**Step 3: Check preconditions** + +```bash +git -C /srv/thothii/source/ThothII config --local core.autocrlf false +bash /srv/thothii/source/ThothII/scripts/verify-line-endings.sh +"$THT_BIN" --installation "$INSTALLATION" update --check-only +``` + +Expected: descriptor, protected secrets, Pi-state mount, workspace repository binding, and Compose render remain valid before source changes. + +## Task 5: Deploy the feature revision to PSD without merging main + +**Files:** + +- Server source checkout: /srv/thothii/source/ThothII +- Server operator binary: protected THT_BIN path +- Server installation descriptor and secret files: unchanged paths unless a reviewed auth update is required + +**Step 1: Select the exact candidate** + +```bash +git -C /srv/thothii/source/ThothII fetch origin feat/thoth-auth +git -C /srv/thothii/source/ThothII switch --detach +test "$(git -C /srv/thothii/source/ThothII rev-parse HEAD)" = "" +git -C /srv/thothii/source/ThothII status --short --untracked-files=all +``` + +Do not merge or rebase main. The running installation is intentionally based on the detached feature revision until acceptance completes. + +**Step 2: Build candidate artifacts** + +```bash +cd /srv/thothii/source/ThothII +bash scripts/build-local.sh +THT_THT_OUTPUT_DIRECTORY=/srv/thothii/operator/build-output bash scripts/build-tht.sh +``` + +Install the architecture-appropriate candidate tht only after its build succeeds. Keep the old operator binary recoverable. + +**Step 3: Start and verify the candidate** + +```bash +"$THT_BIN" --installation "$INSTALLATION" update --check-only +"$THT_BIN" --installation "$INSTALLATION" start --build +"$THT_BIN" --installation "$INSTALLATION" status +"$THT_BIN" --installation "$INSTALLATION" doctor --json +curl --fail http://127.0.0.1:8080/health +"$THT_BIN" --installation "$INSTALLATION" pi doctor +"$THT_BIN" --installation "$INSTALLATION" pi test +``` + +Expected: candidate frontend/core and internal services are healthy, no data volume was replaced, and the candidate SHA is recorded. Liveness alone is not release approval. + +## Task 6: Configure and validate remote Authentik + +**Files:** + +- Modify protected server authentication state through tht auth configure +- Modify protected secret entries THT_OIDC_CLIENT_SECRET and THT_AUTHENTIK_API_TOKEN +- Read: docs/install/authentik.md and docs/install/authentication-oidc.md + +**Step 1: Verify Authentik** + +Verify the OAuth2/OIDC callback exactly /api/auth/oidc/callback, scopes openid/profile/email, direct groups array mapping, exact groups TOT Users and TOT Admin, and a separate group-view-only catalog service account. Inspect a disposable identity without copying its token. + +**Step 2: Configure the ThothII mapping** + +```bash +tht --installation "$INSTALLATION" auth configure \ + --mode oidc --public-url \ + --issuer --client-id \ + --authentik-base-url \ + --user-group 'TOT Users' --admin-group 'TOT Admin' +``` + +Secrets are read from protected files, never command-line arguments. Confirm the non-secret mapping is: + +```yaml +authorization: + groupRoles: + TOT Users: [user] + TOT Admin: [admin] +``` + +**Step 3: Run static and live checks** + +```bash +tht --installation "$INSTALLATION" auth status --json +tht --installation "$INSTALLATION" auth check --json +tht --installation "$INSTALLATION" auth check --interactive +tht --installation "$INSTALLATION" doctor --json +``` + +Expected: configuration, discovery, issuer, JWKS, client-secret access, catalog access, and exact existence of every mapped group pass. Doctor lists authentication after configuration and before services. No output contains credentials or bearer tokens. + +A missing mapped group must fail with redacted oidc_mapped_group_missing. Unmapped groups produce neither error nor warning. Missing, indirect, malformed, or overage-style groups claims fail closed. + +**Step 4: Reload if required** + +If configuration requires process reload: + +```bash +"$THT_BIN" --installation "$INSTALLATION" pi restart --yes --drain +``` + +Repeat authentication, doctor, and health checks. Do not substitute raw Compose commands. + +## Task 7: Test PSD browser login and authorization + +**Files:** + +- Read: docs/testing/authentication-manual-acceptance.md +- Evidence: redacted report from Task 9 + +**Step 1: Ordinary user** + +In a private profile, authenticate with an identity in TOT Users but not TOT Admin. Verify callback success, application/session routes, denial of Pi Management/admin operations, opaque HttpOnly ThothII cookie, no bearer token in Web Storage, and logout invalidation. + +**Step 2: Administrator** + +Authenticate with TOT Admin. Verify Pi Management and allowed workspace-management operations. Access must derive from the exact mapped group, not a client-supplied header or browser-local flag. + +**Step 3: Unmapped and malformed groups** + +Authenticate with a valid token containing no mapped group. Expected: login may complete, but protected operations return 403 with no warning. Use a disposable provider mapping that omits or corrupts groups; expected: generic HTTP 401 oidc_callback_failed, with no internal claim details exposed. + +**Step 4: Provider outage/group drift** + +During a controlled window, make discovery/JWKS unavailable or rename a mapped group, run the CLI check, and restore it immediately. Expected: redacted fail-closed diagnostics followed by a successful check after restoration. Do not leave production broken. + +## Task 8: Test PSD workspace validation and real connections + +**Files:** + +- Read: docs/install/server-workspace-registry.md +- Use: authenticated PSD browser sessions + +**Step 1: Validate the workspace and authentication from the host CLI** + +```bash +"$THT_BIN" --installation "$INSTALLATION" \ + workspace inspect --workspace "$WORKSPACE_ID" --json +"$THT_BIN" --installation "$INSTALLATION" auth check --json +``` + +Expected: the workspace registry is ready, authentication readiness passes, and output is redacted while identifying the active workspace revision. + +**Step 2: Verify the application boundary** + +Open the configured public URL, authenticate with the approved identity, and verify that the application reaches the selected workspace without unexpected `401`/`403` responses. Keep DWH/Evidence connection tests read-only and use only the existing approved smoke question. + +**Step 3: Test ordinary-user authorization** + +Log in as TOT Users. Confirm inspection follows ordinary permissions while validation, secret mutation, and connection tests remain unavailable unless explicitly granted. + +**Step 4: Run a harmless end-to-end smoke** + +As an authorized PSD user, create or resume one harmless known-good session: + +```text +browser login -> same-origin API -> workspace readiness -> Pi/core -> result -> logout +``` + +Do not run mutating production queries. Preserve only a session ID and redacted outcome if approved. + +## Task 9: Close acceptance, rollback if needed, and decide merge readiness + +**Files:** + +- Create: docs/testing/evidence/2026-08-18-thothii-authentication-psd-acceptance.md or the approved external evidence location +- Read: docs/install/server.md and docs/contracts/tht-pi.md + +**Step 1: Mandatory gates** + +| Gate | Required evidence | +|---|---| +| Candidate identity | Local and server SHA exactly match pushed feat/thoth-auth | +| Local startup | macOS Compose, doctor, health, and Pi smoke pass | +| Local auth | Bootstrap, ordinary/admin roles, logout, bad password, disable/enable, logout-all pass | +| Local session | Remembered session survives browser and ThothII restart; revisions invalidate it | +| Server safety | Old SHA/image/status captured; backup checksummed; maintenance/drain completed | +| Candidate deploy | Server candidate status/doctor/health pass | +| Authentik | Direct groups claim, issuer/JWKS, secrets, catalog, and mapped groups pass | +| OIDC authorization | ordinary, admin, unmapped, malformed, logout, and outage cases pass | +| Workspace integration | `tht workspace inspect` and `tht auth check` pass; the authenticated application reaches the selected workspace | +| PSD smoke | One harmless known-good session completes | +| Hygiene | No secrets, tokens, cookies, hashes, or raw claims in evidence | + +**Step 2: Write the redacted report** + +Include candidate SHA, old SHA, timestamps, commands, browser cases, redacted diagnostic/HTTP codes, Authentik issuer/client/group names, workspace ID/revision, backup/checksum location, rollback decision, and unrelated CI failures. Never include secret values, raw tokens, cookies, or hashes. + +**Step 3: Roll back a failed candidate** + +1. Keep the proxy closed and preserve .tht// recovery state. +2. Do not use tht pi rollback as the whole-application rollback; it addresses only Pi lifecycle images. +3. Stop with tht. +4. Return the source checkout to , rebuild old application/operator artifacts, and start through the same descriptor. +5. Run update --check-only, status, doctor, health, Pi smoke, workspace diagnostics, and one harmless session. +6. For ambiguous recovery, leave maintenance active and follow pi maintenance status / pi maintenance recover --yes. Never delete volumes, selectors, or recovery files to force progress. + +Expected: the previous application serves again with prior data and workspace state intact. Record the failure and do not merge. + +**Step 4: Reopen traffic** + +After every gate passes, restore the reverse proxy, repeat one unauthenticated redirect and one authorized public login, and confirm only the proxy is externally reachable. + +**Step 5: Merge decision** + +Merge only after PSD owner acceptance, exact-SHA evidence, no unresolved auth/workspace/provider/deployment gate, and an accepted rollback path. If the merge creates a new commit, repeat Tasks 0, 5, 6, and 7 against the merge SHA. + +## Handoff checklist + +Deliver the redacted report, local result/SHA, PSD candidate SHA/images, Authentik provider and group mapping confirmation, workspace validation/connection results, backup/rollback status, and an explicit READY TO MERGE or NOT READY TO MERGE decision. diff --git a/docs/plans/2026-08-19-tht-documentation-convergence.md b/docs/plans/2026-08-19-tht-documentation-convergence.md new file mode 100644 index 00000000..7bbe506d --- /dev/null +++ b/docs/plans/2026-08-19-tht-documentation-convergence.md @@ -0,0 +1,92 @@ +# Tht Documentation Convergence Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Align current documentation and documentation smoke checks with the converged native host CLI `tht`, while preserving historical references only where they describe past decisions or evidence. + +**Architecture:** Treat `tools/tht/cmd/tht/main.go` as the canonical host CLI surface for installation, authentication, diagnostics, lifecycle, and workspace operations. Keep the Python `harness/.venv/bin/tht` distinction explicit for the workflow runtime, and update current operator/test instructions to invoke the native `tht` with `--installation`. + +**Tech Stack:** Markdown documentation, shell smoke tests, Go CLI command surface, repository search-based verification. + +--- + +### Task 1: Classify current and historical legacy CLI references + +**Files:** +- Inspect: `README.md`, `PROJECT_STATE.md`, `AGENTS.md`, `docs/**`, `scripts/**` +- Reference: `tools/tht/cmd/tht/main.go` + +**Step 1:** Build a complete occurrence inventory with a case-insensitive search for the former host CLI name and classify every match. + +**Step 2:** Classify each occurrence as current operator documentation, documentation smoke expectation, executable/script contract, or historical design/evidence. + +**Step 3:** Record the classification in the implementation notes before editing. + +### Task 2: Update canonical operator and installation documentation + +**Files:** +- Modify: `README.md` +- Modify: `AGENTS.md` +- Modify: `PROJECT_STATE.md` +- Modify: `docs/guida-utente.md` +- Modify: `docs/contracts/workspace-preprocessing-cli.md` +- Rename/update: `docs/contracts/tht-pi.md` as the current `tht` Pi contract +- Modify: relevant installation and architecture pages that expose operator commands + +**Step 1:** Replace current host/operator invocations with `tht --installation ...`. + +**Step 2:** Document the distinction between the native host CLI `tht` and the Python harness CLI invoked by the backend/runtime. + +**Step 3:** Update command examples for `start`, `status`, `doctor`, `auth`, `workspace`, and `pi`. + +**Step 4:** Add a short historical note only where a document must explain the former name. + +### Task 3: Rewrite authentication acceptance and manual test instructions + +**Files:** +- Modify: `docs/testing/authentication-manual-acceptance.md` +- Modify: `docs/plans/2026-08-18-thothii-authentication-acceptance-and-psd-deployment.md` +- Modify: `docs/install/authentication-local.md` +- Modify: `docs/install/authentication-oidc.md` +- Modify: `docs/install/authentik.md` + +**Step 1:** Make `tht auth status`, `tht auth check`, `tht auth check --interactive`, and `tht doctor --json` the canonical terminal preflight. + +**Step 2:** Use `tht status`, `tht start`, and `tht workspace inspect --workspace psd-clinical --json` for PSD deployment checks. + +**Step 3:** Clarify that the P8 L2 gate is authentication-to-application integration through the first reviewer gate. + +**Step 4:** Retain the prior functional test suite as a baseline and add only the authentication boundary smoke required for this acceptance. + +### Task 4: Align documentation smoke tests + +**Files:** +- Modify: `scripts/auth-docs-smoke.sh` +- Modify: `scripts/test-auth-docs-smoke.sh` +- Inspect/update: any current smoke script whose user-facing command examples still require the legacy CLI name + +**Step 1:** Replace forbidden/current command assertions with `tht` equivalents. + +**Step 2:** Preserve negative checks for obsolete authentication CLI wording. + +**Step 3:** Run the positive and negative documentation fixtures. + +### Task 5: Preserve or annotate historical material + +**Files:** +- Inspect the historical discovery specification for context, without treating it as current operator documentation. +- Inspect: dated reports and archived acceptance scripts + +**Step 1:** Do not rewrite historical titles, commit evidence, or old implementation names solely to erase history. + +**Step 2:** Add a concise “historical nomenclature” note where an archived document could otherwise be mistaken for current instructions. + +### Task 6: Verify the convergence + +**Step 1:** Run `scripts/auth-docs-smoke.sh` and `scripts/test-auth-docs-smoke.sh`. + +**Step 2:** Search active documentation for remaining legacy CLI references. + +**Step 3:** Confirm every remaining match is either an explicit historical note, an ignored runtime directory name, or a non-document executable compatibility artifact. + +**Step 4:** Run `git diff --check` and report the exact files changed plus any intentionally retained historical references. diff --git a/docs/prd/2026-08-09-workspace-preprocessing-prd.md b/docs/prd/2026-08-09-workspace-preprocessing-prd.md new file mode 100644 index 00000000..60fc54db --- /dev/null +++ b/docs/prd/2026-08-09-workspace-preprocessing-prd.md @@ -0,0 +1,543 @@ +# PRD — Preprocessing per-workspace su ThothII (Qdrant + Git workspace registry) + +**Status:** PRD in revisione — decisioni D1–D9 chiuse il 2026-08-09; i piani P1–P10 partiranno solo dopo +revisione e conferma del proprietario +**Data:** 2026-08-09 +**Autore:** analisi dello stato attuale (branch `codex/git-workspace-registry`) + decisioni con il proprietario +**Uso:** riferimento stabile di requisiti e decisioni; da ogni punto nascerà un piano separato in +`docs/superpowers/plans/` (sez. 11) — questo documento non è un piano di lavoro + +--- + +> **Aggiornamento P1.1 (2026-08-11):** il layout del repository registry descritto nelle sezioni +> attive di questo PRD segue il contratto P1.1 accettato: catalogo di root `thoth-workspaces.yaml`, +> descriptor `/workspace.yaml`, evidence embedded `/evidence/`, annotazioni FK curate +> `/schema/annotations.yaml` (P5), docs generate `workspace-docs//`. I vecchi percorsi +> piatti (`workspaces/.yaml`, `workspace-content//evidence/`) sono superseded; le uniche +> occorrenze rimaste sono storiche (changelog/revisioni). Vedi +> `docs/superpowers/plans/2026-08-11-p2-p6-adaptation-to-p1-1-registry.md`. + +--- + +## 1. Contesto + +ThothII è passato da un indice semantico **pgvector sul server PSD** (descrizioni di tabelle/colonne, +evidence e memory embeddate nella stessa istanza Postgres del DWH, lettura via RPC `search_similar`, +scrittura via REST dedicato o loading diretto) a un'architettura con: + +- **infrastruttura semantica interna obbligatoria**: Qdrant + Ollama (`qwen3-embedding:0.6b`, 1024 dim, + cosine) come servizi Compose privati; una **collection Qdrant per workspace**, con schema/evidence/memory + separati dal payload `kind`; +- **workspace definito da un descriptor schema-v3 in un repo Git esterno** (id, DWH, collection, LLM + policy, diagnostics), con binding DWH locali all'installazione; +- **config harness renderizzata dal backend** a runtime (`runtime_identity` + `resources.vector` + + `resources.embeddings` + `roots` sotto `/sessions//`). + +La **macchina di preprocessing** (comandi, job a generazioni con publish atomico, adapter Qdrant, corpus +evidence, FK, memory) **esiste già ed è testata**: `tht preprocess evidence|dwh`, `tht vector init|index-schema`, +`tht schema introspect|suggest-fks|check`, `tht evidence extract|index`, `tht lsh build`, memory/solved. +Esistono job Compose fixture (`deploy/compose.preprocess.yaml` + `deploy/workspaces/preprocess-{dwh,evidence}.yaml`) +e uno smoke (`scripts/preprocess-smoke.sh`). + +### Il problema + +Il preprocessing **non è collegato al workspace reale del registry**: + +1. i job fixture usano una collection fissa (`preprocess-evidence`), un workspace_id derivato dal nome + file (`preprocess-evidence`) e roots sotto `/data/workspaces/preprocess-*`, che **non coincidono** con + quelli del runtime (`/sessions//`); +2. il backend **non espone alcun modo** di eseguire `tht preprocess` contro la config renderizzata di un + workspace (nessun endpoint, nessuno script, nessun comando documentato); +3. il **descriptor v3 e la config renderizzata non hanno la sezione `evidence`**: non c'è un posto canonico + dove dichiarare da dove arrivano le evidence di un workspace; +4. l'**ammissione sessione** (`ThtRunner.qdrantEnsure`) richiede la collection già esistente con 1024/cosine + e 8 payload keyword-index, ma **nessuno la crea esplicitamente** (`tht vector init` fallisce se manca); +5. le **generazioni `.tht-dwh` sono legate a un fingerprint della config completa** (`OWNER.json`: + workspace_id + config_fingerprint + input_fingerprint): se il preprocessing non usa la config identica a + quella renderizzata dal runtime, a runtime la generazione viene **rifiutata**; +6. la **cura FK** (`annotations.yaml`) è manuale e vive nel runtime artifacts; il registry **non sincronizza** + file dal repo ai roots runtime; +7. i **vecchi embedding pgvector (nomic 768d) non sono riusabili** (modello e dimensioni cambiati): serve + re-indicizzare i contenuti PSD. + +**Sintesi:** la parte "motore" è pronta; manca il **collegamento per-workspace** (config, esecuzione, +bootstrap, sorgente evidence) e la **documentazione operator**. + +--- + +## 2. Obiettivo + +Rendere l'attuale versione di ThothII (Qdrant + workspace su repo esterno) **configurabile e utilizzabile** +per un workspace reale, inclusa l'intera catena di preprocessing: **tabelle/colonne (catalogo + embedding), +FK (cura), evidence (sorgente → corpus → embedding), memory/solved**, con un flusso operator riproducibile, +documentato e verificato da smoke end-to-end. + +### Obiettivi secondari + +- O1. Un solo modo canonico di eseguire il preprocessing per un workspace (niente più fixture "speciali"). +- O2. Il preprocessing è **idempotente e ripristinabile**: rerun senza duplicati, publish atomico, GC. +- O3. Nessun segreto/endpoint entra nel repository registry né nei descriptor (invariante attuale preservato). +- O4. Il flusso è **documentato nei manuali operator** (`local/server-workspace-registry.md`) e coperto da + smoke automatici. +- O5. La **migrazione PSD** è definita (cosa si riusa, cosa si rigenera, cosa si esporta dal pgvector). +- O6. Ogni piano tecnico definisce, dove applicabile, un **goal automatico di processo completo**: da stato + pulito costruisce un ambiente isolato, simula il flusso end-to-end entro lo scope del piano e lo porta a + successo con un integration test riproducibile. +- O7. Dopo il successo automatico, un **percorso manuale separato** permette al reviewer di ripetere il + processo attraverso le interfacce reali, comprenderne l'architettura e approvare gli artefatti. + +## 3. Non-obiettivi (fuori scope di questo PRD) + +- Riscrivere il workflow NL→SQL o i gate (F1..F8) — restano invariati. +- Cambiare modello/architettura semantica (Qdrant/Ollama/1024/cosine) — già deciso e verificato. +- Rifare la UI di gestione workspace oltre a quanto già esiste. +- Il **deploy reale sul server PSD** (VPN, credenziali, portale, auth upstream): è un progetto operativo + separato che userà questo PRD come prerequisito tecnico. +- Supportare di nuovo pgvector o endpoint embedding esterni come percorso operativo. +- Il **comando di preprocessing avviabile dalla GUI**: è una release futura (fuori scope della release 0, + che è CLI sul host — vedi D2). + +--- + +## 4. Utenti + +| Utente | Esigenza | +| --- | --- | +| **Operatore/amministratore** (chi installa e cura un workspace) | Configurare DWH+evidence, eseguire il preprocessing, curare le FK, verificare lo stato, fare backup/restore. | +| **Autore ETL / curatore dominio** (es. il cliente PSD) | Mantenere evidence e annotazioni FK nel namespace del workspace nel repository registry con un flusso semplice. | +| **Reviewer umano** (usa l'app) | Vede search pack con tabelle/evidence/solved corretti: la qualità del retrieval dipende dal preprocessing. | +| **Sviluppatore ThothII** | Comandi/endpoint deterministici, testabili, senza sorprese di configurazione. | + +--- + +## 5. Scenario target (end-to-end) + +1. **Setup repo**: l'operatore usa un **unico repository Git registry** per tutti i workspace e crea + `/workspace.yaml` (schema-v3: DWH, collection, LLM policy) insieme al tree curato + `/evidence/`; descriptor e contenuti sono pubblicati nello stesso commit. +2. **Installazione**: `.env` + bindings `THT_WS_*` + secrets; `up` dello stack (frontend/core/qdrant/embedding). +3. **Registry**: pull → validazione → snapshot attivo; diagnostic DWH verdi. +4. **Preprocessing DWH**: introspezione (physical.yaml: tabelle/colonne/descrizioni/esempi/eligibility) + + LSH; generazione pubblicata sotto `.tht-dwh` del workspace. +5. **Cura FK**: `tht schema suggest-fks` → revisione umana → `annotations.yaml` (check senza orfani); + versionata dove deciso (vedi D5). +6. **Indice schema**: `tht vector index-schema` → record schema nella collection del workspace; la + collection, se inesistente, viene creata all'ammissione (self-heal) o dal primo write (vedi D4). +7. **Preprocessing evidence**: `tht preprocess evidence` → corpus generation (chunk+embed) nella collection + (kind `evidence`) + manifest ACTIVE nel corpus root del workspace. +8. **Memory/solved**: promozioni F8 (`memory promote/save-one`) e finalize (`solved-index`) scrivono nella + collection (kind `memory`). +9. **Uso**: nuova sessione → admission verde (collection+Ollama) → F1 `search pack` con tabelle, evidence e + solved del workspace; F4/F6 con FK curate. +10. **Operatività**: backup/restore volumi (Qdrant, corpus, `.tht-dwh`, registry), update, ripristino da + outage. + +--- + +## 6. Requisiti funzionali + +### RF1 — Configurazione per-workspace +- RF1.1 Un workspace del registry deve poter dichiarare **tutto ciò che serve al preprocessing** in un unico + posto canonico: DWH (già nel descriptor), **sorgente evidence completa** (protocollo/tipo, URI, parametri + non-secret), eventuali policy di chunk/retention. +- RF1.2 I segreti (password, API key, CA) restano fuori dal repo e dal descriptor (invariante attuale). +- RF1.3 La config harness usata dal preprocessing deve essere **derivata dalla stessa renderizzazione del + runtime** (stesso workspace_id, stessi roots, stessa configurazione effettiva). +- RF1.4 La configurazione (descriptor + bindings + config renderizzata) deve **prevedere i tre trasporti + DWH**: `postgres_direct`, `rest_api`, `ssh_tunnel`. PSD usa `rest_api`; altri database potranno usare + direct o tunnel. +- RF1.5 Il registry usa **un unico repository Git** per più workspace. Per una sorgente evidence + `filesystem`, l'URI è relativa alla root del repository ed è confinata lessicalmente a + `/`; path assoluti, traversal (`..`) e riferimenti al namespace di un + altro workspace sono invalidi. Descriptor e sorgente devono essere risolti dalla **stessa revisione Git**. + +### RF2 — Preprocessing DWH (tabelle/colonne) +- RF2.1 Comando/azione per eseguire `introspect` + `lsh` per un workspace del registry, contro la sua config + effettiva, con output JSON e resume. +- RF2.2 La CLI di preprocessing raggiunge il DWH **con il trasporto dichiarato dal workspace** (direct, + REST o tunnel SSH), come il runtime. +- RF2.3 Il catalogo risultante (`physical.yaml`) alimenta: cache `tht schema introspect`, render mschema + (F1/F4), record schema per l'embedding (RF4). +- RF2.4 Refreshing esplicito quando il DWH cambia (`--refresh`/nuova generazione), senza invalidare le + sessioni esistenti (generazioni + ACTIVE pointer, già implementato). + +### RF3 — FK +- RF3.1 Flusso curato per-workspace: `tht schema suggest-fks` (+ `--from-sql`, `--assume`, `--write`), + revisione umana, `tht schema check` (zero orfani). +- RF3.2 Le FK curate devono essere **disponibili a runtime** (sezione `【Foreign keys】` del render mschema, + usata da F4/F6) e **versionate nel repository registry** (D5). +- RF3.3 Nessuna FK derivata dal modello: il modello usa solo la lista curata (contratto SKILL invariato). + +### RF4 — Indice semantico schema + bootstrap collection +- RF4.1 `tht vector index-schema` embedda i record schema (tabella+colonna, con descrizioni/esempi/sinonimi) + nella collection del workspace (kind `schema`), idempotente (hash → upsert solo del cambiato). +- RF4.2 **Bootstrap della collection**: se inesistente all'ammissione sessione, il runtime la crea + (self-heal) con 1024/cosine + i payload keyword-index richiesti (`content_hash, document_id, kind, + record_key, record_kind, vector_generation, workspace_id, workspace_revision`). +- RF4.3 La **CLI deve poter cancellare e ricreare** la collection di un workspace (rebuild esplicito con + guardie di sicurezza e conferma). +- RF4.4 Prima di una sessione, l'ammissione resta invariata (collection compatibile + Ollama). + + +### RF5 — Evidence +- RF5.1 Sorgente evidence dichiarabile per-workspace nel descriptor (protocollo/tipo + URI). Per PSD è un + **tree di file `.md` versionato nell'unico repository registry**, sotto + `psd/evidence/`; in generale ogni workspace usa + `/evidence/`. HTTP manifest e S3 restano opzioni del motore per sorgenti + esterne. +- RF5.2 `tht preprocess evidence` per-workspace: discover → acquire → normalize/chunk → embed → upsert + (kind `evidence`, payload `document_id`/`vector_generation`) → publish ACTIVE nel corpus root del workspace, + con resume e dry-run (già implementato nel motore). +- RF5.3 GC/retention delle generazioni evidence (filesystem + punti Qdrant) con le policy esistenti. +- RF5.4 A runtime la ricerca evidence è filtrata dalla generazione ACTIVE e dal workspace_id (già + implementato: `ActiveEvidenceSearcher`); il flusso RF5 deve garantire che il corpus ACTIVE appartenga al + workspace giusto. + +### RF6 — Memory e domande risolte +- RF6.1 `memory promote/save-one` (F8) e `memory solved-index` (finalize) scrivono nella collection del + workspace (kind `memory`/`solved_question`) — verificare end-to-end con Qdrant e risolvere i TODO residui + in `memory_cmd.py`. +- RF6.2 Il registro JSONL resta la fonte canonica; Qdrant è proiezione di ricerca (invariante attuale). + +### RF7 — Migrazione PSD +- RF7.1 Definire cosa si **riusa** (physical.yaml, annotations.yaml con le ~228 FK curate, le 895 evidence + `.md`), cosa si **rigenera** (tutti gli embedding, modello diverso) e cosa si **esporta** dal pgvector del + server prima della dismissione. +- RF7.2 La migrazione è un'operazione documentata e rieseguibile, non un one-shot nel codice. + +### RF8 — Operatività e documentazione +- RF8.1 Manuali operator aggiornati con la sequenza completa per-workspace (config → preprocess → cura → + verifica → uso → backup/restore). +- RF8.2 La documentazione di progetto spiega **cos'è `.tht-dwh`** (generazioni, `OWNER.json`, `ACTIVE`, + vincolo di fingerprint) in modo comprensibile per l'operatore (D3). +- RF8.3 Smoke end-to-end automatico (workspace nuovo → tutto il ciclo → sessione reale → cleanup) che + sostituisce/completa `preprocess-smoke.sh` (oggi solo fixture). +- RF8.4 Backup/restore coprono Qdrant (già `vector-backup.sh`/`vector-restore.sh`), corpus, `.tht-dwh` e + registry. +- RF8.5 Ogni piano successivo traduce il proprio risultato operativo in un **process goal** verificabile da + un integration test completo per quello scope; eventuali interventi umani iniziali o intermedi sono + ammessi solo se inevitabili, espliciti, documentati e riprendibili. +- RF8.6 Ogni process goal automatico riuscito è seguito, quando utile, da un walkthrough manuale su un + ambiente nuovo e separato; automazione e accettazione umana producono evidenze distinte. + +--- + +## 7. Requisiti non funzionali + +- **RNF1 Sicurezza**: nessun segreto in repo/descriptor/config renderizzata/log; la CLI di preprocessing + (D2) non espone credenziali, non le logga e non le scrive negli artefatti. +- **RNF2 Determinismo/idempotenza**: rerun del preprocessing = zero duplicati (hash content), publish + atomico, generazioni immutabili (già nel motore). +- **RNF3 Robustezza**: degradazione controllata (workspace senza evidence o senza collection funziona, con + warning); errori sanitizzati; nessun fallimento che corrompa la generazione attiva. +- **RNF4 Isolamento per-workspace**: ogni filtro Qdrant legato a workspace_id; rifiuto di namespace + conflittuali (già implementato nell'adapter). +- **RNF5 Compatibilità**: il preprocessing deve funzionare con la config renderizzata dal backend + (fingerprint `OWNER.json` compatibile) — è il vincolo chiave di design (vedi D3). +- **RNF6 Performance**: introspezione ~minuti (non nel path di sessione), embedding batch, LSH boundato; + il retrieval a runtime non cambia i costi attuali. +- **RNF7 Manutenibilità**: nessun fork dei fixture; un solo percorso canonico (O1). +- **RNF8 Integration-first**: il successo di un piano tecnico richiede un'esecuzione completa da ambiente + pulito, senza retry automatici che mascherino errori; ogni fallimento viene diagnosticato, corretto alla + radice e seguito da una nuova esecuzione completa. +- **RNF9 Evidenza e cleanup**: ogni ambiente simulato ha identità/ownership esplicita, risorse univoche, + segreti fittizi, report machine-readable e leggibile, scansione anti-secret e cleanup confinato alle sole + risorse possedute dal run. + +--- + +## 8. Standard di esecuzione e verifica — integration-first + +Questo standard si applica a P1 e, **ovunque sia tecnicamente significativo**, a tutti i piani successivi. +Un piano che non possa applicarlo deve motivare esplicitamente l'eccezione e definire il verifier più vicino +possibile al processo reale. + +### S1 — Goal automatico di processo + +- Ogni piano definisce il **processo completo entro il proprio scope**, con punto iniziale pulito, input, + componenti attraversati, risultato osservabile e criteri di successo. +- Il goal non è "far passare alcuni test", ma **simulare con successo il processo operativo** che la feature + deve rendere possibile. Per P1 il confine completo è Git → registry → API → snapshot/docs → render → + `tht config check`; estrazione evidence e Qdrant appartengono ai piani successivi. +- Durante l'esecuzione il goal resta aperto fino a una prova integrale verde. Se l'ambiente agentico supporta + goal persistenti, l'esecutore lo registra all'inizio e lo completa soltanto dopo l'evidenza finale. + +### S2 — Ambiente di integrazione isolato + +- Il test costruisce dipendenze controllate sotto `.artifacts///`: repository Git simulati, + checkout, roots runtime, secret fixture, richieste/risposte, log e output. +- Ogni run usa identità e nomi univoci e un manifest di ownership; non usa credenziali, repository o dati + reali salvo quando il piano dichiara esplicitamente un gate L2. +- I servizi reali appartenenti allo scope vengono attraversati tramite le loro interfacce normali; quelli + esterni o non ancora nello scope sono sostituiti da fixture fedeli e deterministiche. + +### S3 — Contratto di successo + +- Il test parte da stato pulito, esegue il processo una volta senza retry automatici, termina con exit code + zero e produce `report.json` più un report leggibile. +- Un fallimento richiede diagnosi della causa, test regressivo/correzione e una nuova esecuzione completa da + stato pulito; ripetere alla cieca non costituisce progresso verso il goal. +- Il gate finale comprende determinismo/idempotenza pertinenti, scansione anti-secret, verifica degli + artefatti e prova del cleanup confinato. Gli artefatti possono essere conservati con `--keep` per review. + +### S4 — Interventi umani inevitabili + +- Passi umani iniziali o intermedi sono ammessi solo quando non simulabili in modo affidabile (per esempio + accesso approvato a un sistema reale o review di contenuto curato). +- Ogni passo umano dichiara precondizioni, istruzioni, evidenza richiesta, criterio di decisione e checkpoint + di ripresa; l'automazione copre e verifica tutto ciò che precede e segue il checkpoint. +- Un intervento umano non può essere sostituito da un'assunzione silenziosa né rendere non riproducibile il + resto del processo. + +### S5 — Walkthrough manuale successivo + +- Dopo il goal automatico verde, il reviewer ripete il processo in un **ambiente nuovo e separato**, usando + le interfacce reali e una guida passo-passo che spiega componente, stato letto, artefatto prodotto e + invariante verificata. +- Il walkthrough serve a comprensione architetturale e accettazione; non sostituisce l'integration test e non + ne riusa lo stato già mutato. +- Lo stato di consegna distingue almeno `automated integration: PASS` e `manual acceptance: PENDING/PASS`. + Un piano non è pienamente accettato finché l'eventuale gate manuale richiesto non è stato deciso dal reviewer. + +### S6 — Contenuto obbligatorio dei piani + +Ogni piano tecnico riporta, adattandoli al proprio scope: + +1. **Automated process goal** e comando unico di esecuzione; +2. topologia dell'ambiente simulato e confini delle dipendenze; +3. asserzioni del full integration test e contratto del report; +4. checkpoint umani inevitabili, oppure dichiarazione esplicita che non ve ne sono; +5. walkthrough/gate manuale successivo, quando utile; +6. evidenze di completamento, retention degli artefatti e cleanup esatto. + +--- + +## 9. Criteri di accettazione (bozza) + +1. Da un repository registry vuoto si arriva a una sessione funzionante seguendo **solo i manuali aggiornati**, + senza toccare file fixture. +2. La CLI di preprocessing funziona **sia sul PC/Mac dell'utente sia sul server che ospita il DWH** + (stesso comando, config derivata dal workspace). +3. La configurazione di un workspace dichiara e usa uno dei **tre trasporti DWH** (`postgres_direct`, + `rest_api`, `ssh_tunnel`); PSD usa `rest_api`. +4. Il goal automatico P1 costruisce da zero repository Git simulati e ambiente isolato, attraversa con + HTTP reale il processo Git → registry → validate/publish/read/export → snapshot/docs → render → + `tht config check`, supera casi positivi e negativi senza retry e produce report/artefatti secret-free. +5. Solo dopo il punto 4, un ambiente manuale nuovo avvia il backend su `127.0.0.1:8791` e permette al + reviewer di ripetere ogni chiamata e ispezionare commit, snapshot, ZIP e config renderizzate seguendo una + guida; il gate resta `PENDING` finché il reviewer non lo approva. +6. `search pack` di una domanda reale restituisce tabelle (con descrizioni), evidence della generazione + ACTIVE e solved dello stesso workspace; F4/F6 mostrano le FK curate. +7. `qdrantEnsure`/`ollamaEnsure` verdi all'ammissione; la collection ha esattamente 1024/cosine + gli 8 + keyword-index. +8. Rerun del preprocessing: `unchanged` (nessun duplicato); modifica di un'evidence → nuova generazione, + ACTIVE aggiornato, vecchie generazioni in GC. +9. Smoke end-to-end automatico verde in CI con cleanup esatto (stile `preprocess-smoke.sh`). +10. Migrazione PSD documentata e provata almeno in dry-run (re-introspection o riuso catalogo + re-embedding). +11. Ogni piano tecnico successivo include un process goal automatico completo per il proprio scope e un + walkthrough manuale quando utile, oppure documenta l'inevitabile eccezione umana secondo S4. + +--- + +## 10. Decisioni chiuse (2026-08-09) + +> La sezione nasceva come "punti di discussione"; le decisioni sono state prese con il proprietario del +> prodotto il 2026-08-09. Ogni punto resta il riferimento del proprio piano (sez. 11). Le opzioni scartate +> sono omesse; la motivazione della scelta è inclusa in ogni punto. + +### D1 — Config per-workspace: **c) misto, con sorgente completa nel descriptor** +- Il descriptor v3 guadagna una sezione `evidence` che configura **tutta la lettura della sorgente**: + protocollo/tipo, URI/sorgente, eventuali parametri non-secret. +- Si usa **un unico repository Git registry** per tutti i workspace. Ogni workspace possiede il proprio tree + versionato sotto `/evidence/`; per PSD il path canonico è + `psd/evidence/`. +- Per `filesystem`, l'URI del descriptor è repo-relative, confinata al namespace dello stesso workspace e + risolta dalla stessa revisione Git del descriptor. Sono vietati path assoluti, traversal e riferimenti al + contenuto di un altro workspace; il controllo reale di symlink/containment durante la materializzazione + appartiene a P6. +- Eventuali segreti (HTTP autenticato, S3) restano in overlay d'installazione — invariante: nessun segreto + nel repo/descriptor. +- P1 applica lo standard integration-first: prima persegue un goal automatico Git→registry→HTTP→render→ + harness sotto `.artifacts/p1-integration//`, poi offre un walkthrough manuale separato sotto + `.artifacts/manual-acceptance/p1/`. Non include ancora estrazione, embedding o verifica degli artefatti + `artifacts/evidence` (P2+P6). + +### D2 — Esecuzione: **CLI sul host in release 0; GUI in release futura** +- **Release 0**: una **CLI installata con ThothII sul host** — sia il PC/Mac dell'utente sia il server che + ospita il DWH — che esegue **tutta la catena di preprocessing** (DWH introspect+LSH, FK, index-schema, + evidence e quanto serve) per un workspace del registry. +- La CLI deriva la config dal descriptor+bindings con la stessa identità del runtime → soddisfa il vincolo + D3 senza dipendere dal backend. +- **Release futura (fuori scope)**: comando avviabile dalla GUI (endpoint backend da progettare poi). + +### D3 — Fingerprint `.tht-dwh`: **accettare il vincolo + documentarlo** +- Le generazioni DWH restano legate alla config effettiva (workspace_id + config_fingerprint + + input_fingerprint in `OWNER.json`). +- La CLI (D2) gira con la config derivata dal descriptor+bindings, quindi identica alla runtime. +- **La documentazione di progetto deve spiegare chiaramente cos'è `.tht-dwh`** (directory delle generazioni + catalogo/LSH, `OWNER.json`, `ACTIVE` pointer, perché il fingerprint protegge da artefatti di un'altra + config) — oggi non è chiaro. + +### D4 — Bootstrap collection: **self-heal all'ammissione + CLI delete/recreate** +- Se la collection non esiste all'ammissione sessione, il runtime la crea (1024/cosine + payload + keyword-index) — self-heal. +- La **CLI deve poter cancellare e ricreare le collection** (rebuild esplicito, con guardie di sicurezza). + +### D5 — Versioning artifacts curati: **c) misto** +- `annotations.yaml` (cura FK, cura umana) **versionata nel repository registry** e sincronizzata ai roots + runtime (il registry copia il file negli snapshots → sync). +- `physical.yaml` (derivato dall'introspezione) rigenerato localmente, non versionato. + +### D6 — Evidence: **a) nell'unico repository registry, con namespace per-workspace** +- Ogni workspace contiene il proprio tree versionato sotto + `/evidence/`; le dimensioni non sono un vincolo. +- P6 materializza il tree dalla **stessa revisione Git** del descriptor, verifica il containment reale + (inclusi i symlink) e lo rende disponibile al preprocessing senza usare un checkout mobile. +- HTTP/S3 restano opzioni future per sorgenti esterne (il motore le supporta già). + +### D7 — Migrazione PSD: **inclusa, con accesso al server** +- Il piano P7 copre: riuso di physical.yaml + annotations.yaml + evidence `.md` dal repository registry; export + dal pgvector del server PSD (accesso disponibile); re-embedding con `qwen3-embedding:0.6b`; dry-run + documentato. + +### D8 — Verifica end-to-end: **integration-first + walkthrough manuale; remote Git libero; DWH multi-trasporto** +- Ogni fase adotta lo standard della sez. 8: prima un process goal automatico da ambiente pulito, poi — + quando utile o richiesto — un walkthrough manuale su stato separato. P1 è il primo riferimento concreto. +- Il livello finale del PRD resta: smoke automatico su workspace sintetico (CI) + gate manuale L2 su PSD. +- Il namespace PSD sarà **prima alimentato nel repository registry** (descriptor + evidence + annotations + nello stesso flusso Git), **poi** usato da ThothII. Accesso al server PSD disponibile. +- **Remote Git**: lo creiamo noi, nessun vincolo tecnico (consigliato GitHub via HTTPS; SSH resta + possibile se servirà). +- **Trasporto DWH**: PSD via **REST** (come oggi); **la configurazione deve prevedere le tre modalità** — + `rest_api`, `postgres_direct`, `ssh_tunnel` — perché altri database potrebbero richiedere accesso TCP + diretto o via tunnel. Oggi `ssh_tunnel` è solo diagnostico a runtime: va reso operativo dove serve + (vedi P10). + +### D9 — Retention/GC: **confermata** +- Default invariati (`retain_published_generations: 3`, chunk 4000 char), configurabili per-workspace via + la sezione `evidence`/policy del descriptor (D1). + +## 11. Mappa dei piani (uno per punto del PRD) + +Questo PRD non diventa un unico piano: **ogni decisione/requisito produce un piano separato (P1–P10)** in +`docs/superpowers/plans/`, eseguibile in sequenza o come workstream indipendenti. Il PRD resta il +riferimento stabile (requisiti + decisioni); ogni piano cita il punto di origine e i criteri di +accettazione applicabili (sez. 9) e adotta lo standard integration-first (sez. 8). + +| Piano | Punto PRD | Contenuto sintetico | Dipende da | +| --- | --- | --- | --- | +| P1 | D1 | Descriptor v3: sezione `evidence` (protocollo/tipo, URI repo-relative sotto `/evidence/`) + policy e isolamento namespace; goal automatico Git→registry→HTTP→render→harness, seguito da walkthrough manuale | — | +| P2 | D2 | **CLI di preprocessing sul host (release 0)**: comando per-workspace che esegue l'intera catena (DWH, FK, index-schema, evidence) con la config derivata da descriptor+bindings; funziona su PC/Mac utente e server DWH | P1 | +| P3 | D3 | Vincolo fingerprint `.tht-dwh` (test: preprocess con config identica alla runtime) + **documentazione di progetto su cos'è `.tht-dwh`** | P2 | +| P4 | D4 | Bootstrap collection: **self-heal all'ammissione** (creazione 1024/cosine + keyword-index) + **comandi CLI delete/recreate** con guardie | — | +| P5 | D5 | `annotations.yaml` versionata nel repository registry + sync registry → roots runtime | P1 | +| P6 | D6 | Materializzazione del tree `/evidence/` dalla revisione Git fissata → preprocess; containment reale e protezione da symlink escape | P1 | +| P7 | D7 | Migrazione PSD: riuso catalogo/annotations/evidence, **export pgvector (accesso server)**, re-embedding, dry-run | P1–P6 | +| P8 | D8 | Verifica end-to-end: smoke CI + gate L2 su PSD (**namespace PSD nel repository registry alimentato prima dell'uso**; remote Git a scelta) | P1–P7 | +| P9 | D9 | GC/retention per-workspace: policy configurabili, default invariati | P1 | +| P10 | D8/RF1.4 | Trasporti DWH operativi: rendere `ssh_tunnel` utilizzabile a runtime e nella CLI di preprocessing (oggi solo diagnostico); verifica dei tre trasporti (direct, REST, tunnel) | P1, P2 | + +### Standard di verifica obbligatorio del futuro piano P1 + +P1 è il primo piano che applica integralmente la sez. 8 e deve contenere due task/gate distinti e ordinati. + +#### 1. Automated integration goal — complete P1 configuration process + +Un comando unico (nome definitivo nel piano, interfaccia indicativa +`./scripts/p1-acceptance.sh integration --keep`) costruisce da zero: + +```text +.artifacts/p1-integration// +├── ownership.json +├── remote.git/ # remote bare locale +├── author/ # clone curatore + tree evidence +├── installation/ # checkout, snapshot e stato registry +├── runtime-data/ +├── fixture-secrets/ +├── requests/ # payload HTTP positivi/negativi +├── responses/ +├── rendered/ +├── exports/ +├── logs/ +├── report.json +└── report.md +``` + +Il test attraversa le interfacce reali appartenenti a P1: Git reale locale, backend Fastify su una porta +loopback temporanea, route HTTP validate/publish/pull/read/export, snapshot/docs/contract, renderer di +produzione e `tht config check`. Verifica anche stessa revisione Git per descriptor/tree, determinismo, +path/protocolli/secret fields invalidi, assenza di leak e cleanup confinato. Non usa frontend, Docker, DWH, +Qdrant o Ollama perché non appartengono allo scope P1. + +L'esecuzione non applica retry automatici. In caso di errore l'esecutore diagnostica, aggiunge la copertura +regressiva necessaria, corregge e rilancia l'intero scenario da una nuova root pulita. Il goal è raggiunto +solo con exit code zero e report integralmente verde; con `--keep` le evidenze restano disponibili. + +#### 2. Manual acceptance gate — descriptor and rendered configuration artifacts + +Dopo il goal automatico verde, il piano prepara uno stato nuovo e indipendente sotto: + +```text +.artifacts/manual-acceptance/p1/ +├── remote.git/ +├── author/ +├── installation/ +├── runtime-data/ +├── requests/ +├── responses/ +├── output/ +├── logs/ +└── GUIDE.md +``` + +Un helper esegue soltanto `prepare/serve/stop/cleanup`; `serve` avvia il backend reale sull'host, senza +Docker e senza frontend, vincolato a `127.0.0.1:8791`. Il reviewer segue `GUIDE.md` ed esegue personalmente +le chiamate HTTP, i comandi Git, l'export ZIP, il doppio rendering, il confronto e `tht config check`, poi +prova i casi invalidi e decide il gate. + +Il gate verifica manualmente: sezione `evidence`; pubblicazione/rilettura; commit e snapshot immutabile; +workspace docs/contract; config harness; assenza di segreti; sicurezza protocollo/path e isolamento +cross-workspace; output deterministico. Gli artefatti restano fino alla decisione e il cleanup rimuove solo +la root posseduta dal test. + +P1 **non** dichiara di aver generato o validato `artifacts/evidence`: estrazione e mirroring richiedono +P2+P6; record Qdrant, embedding, generazioni ACTIVE e retention appartengono ai piani successivi. Dopo il +goal automatico lo stato è `automated integration: PASS / manual acceptance: PENDING`; P1 diventa pienamente +accettato soltanto dopo la decisione del reviewer. + +Ordine consigliato: **P1 → P2 → P3** (catena config/esecuzione), **P4** e **P5/P6** in parallelo dopo P1, +poi **P7 → P8**; P9 può essere assorbito in P1 o restare autonomo; **P10** dopo P1+P2 (necessario solo se un +workspace target richiede davvero il tunnel — per PSD non serve, usa REST). + +Ogni piano segue la prassi del repo: TDD, commit scoping, verifica layer (pytest/vitest/tsc/build) e lo +standard della sez. 8; gate deployment e smoke Docker si aggiungono quando appartengono allo scope. Lo stato +traccia separatamente implementazione, automated integration e manual acceptance. + +--- + +## 12. Storico revisioni + +| Versione | Data | Contenuto | +| --- | --- | --- | +| v0.1 | 2026-08-09 | Bozza da analisi dello stato attuale (gap preprocessing per-workspace) | +| v0.2 | 2026-08-09 | Decisioni D1–D9 chiuse con il proprietario; mappa piani P1–P10; requisiti RF1–RF8 aggiornati (evidence nel descriptor, CLI sul host, self-heal collection, multi-trasporto DWH) | +| v0.3 | 2026-08-09 | Revisione di coerenza (numerazioni, riferimenti incrociati, header di stato) — pronto per revisione del proprietario | +| v0.4 | 2026-08-09 | D1/D6: repository registry unico, namespace `workspace-content//evidence/` *(percorso storico P1, superseded da P1.1)*, pin alla stessa revisione Git e gate manuale P1 con remote locale usa-e-getta sotto `.artifacts/` | +| v0.5 | 2026-08-09 | Standard integration-first per P1–P10: process goal automatico completo da ambiente simulato e pulito, gestione esplicita degli interventi umani inevitabili e walkthrough manuale successivo su stato separato | + +--- + +## 13. Riferimenti + +- Stato attuale: `PROJECT_STATE.md` (sezioni "Internal Qdrant + Ollama semantic infrastructure", snapshot + registry) e `AGENTS.md`. +- Design architettura semantica: `docs/plans/2026-08-08-internal-qdrant-ollama-design.md` e relativo piano. +- Registry: `docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md`, manuali + `docs/install/local-workspace-registry.md` / `server-workspace-registry.md`. +- Motore preprocessing: `harness/tht/cli/preprocess_cmd.py`, `harness/tht/corpus/pipeline.py`, + `harness/tht/jobs/dwh_pipeline.py`, `harness/tht/adapters/vector/qdrant.py`, + `harness/tht/vectorstore/records.py`, `harness/tht/cli/{vector,schema,evidence,memory}_cmd.py`. +- Fixture attuali: `deploy/compose.preprocess.yaml`, `deploy/workspaces/preprocess-{dwh,evidence}.yaml`, + `scripts/preprocess-smoke.sh`. +- Ammissione runtime: `backend/src/tht/tht-runner.ts` (`qdrantEnsure`/`ollamaEnsure`), + `backend/src/workspaces/runtime-renderer.ts`. diff --git a/docs/reports/2026-08-15-tht-command-audit.md b/docs/reports/2026-08-15-tht-command-audit.md new file mode 100644 index 00000000..d2cc3a00 --- /dev/null +++ b/docs/reports/2026-08-15-tht-command-audit.md @@ -0,0 +1,171 @@ +# Audit critico dei comandi `tht` + +Data: 2026-08-15 + +## Scopo + +Questo audit valuta tutti i comandi terminali registrati dall'attuale CLI Python `tht` prima di +unificare la CLI di ThothII sotto un solo eseguibile pubblico. La valutazione incrocia: + +- il contratto del workflow Pi in `harness/.pi/skills/tht-sessione/SKILL.md`; +- le invocazioni reali del gate in `harness/.pi/extensions/tht-gate.js`; +- le invocazioni del backend in `backend/src/tht/tht-runner.ts`; +- i job operatore in `backend/src/workspaces/preprocessing-service.ts`; +- il migratore in `docker/session-migrate.sh`; +- test e documentazione esistenti. + +L'inventario autorevole contiene 76 comandi Typer più il comando callback `doctor`: 77 comandi +terminali complessivi. + +## Legenda + +- **WF — intoccabile workflow**: chiamato da Pi, dal gate o dal contratto delle otto fasi. Va + conservato con semantica, output JSON ed exit code compatibili. Non deve necessariamente apparire + nell'help ordinario dell'utente. +- **PL — intoccabile piattaforma**: chiamato dal backend, dai job workspace o dal deployment. Anche + questo è un contratto interno, non necessariamente un comando da mostrare all'utente. +- **ADV — mantenere avanzato**: non è nel flusso automatico, ma offre una capacità amministrativa o + di recupero che sarebbe imprudente perdere. Va nascosto dall'help base. +- **ACCORPA**: la capacità serve, ma non merita un comando autonomo. +- **RIMUOVI**: il comando non ha chiamanti reali ed è duplicato, superato, pericoloso o incompleto. + L'eventuale logica riutilizzata da altri flussi resta una libreria interna. + +## Risultato sintetico + +| Esito | Numero | Conseguenza | +|---|---:|---| +| WF o PL, intoccabili | 55 | Conservare il contratto; nascondere i primitivi tecnici dall'help base | +| ADV o ACCORPA | 8 | Conservare la capacità riducendo la superficie UX | +| RIMUOVI | 14 | Eliminare il comando dalla nuova CLI | +| **Totale** | **77** | Una sola CLI pubblica molto più semplice, senza riscrivere il workflow vivo | + +## Matrice completa + +### Diagnostica, configurazione e dipendenze + +| Comando | Valutazione | +|---|---| +| `config check` | **ACCORPA** in `tht doctor`: la validazione della configurazione serve, ma due preflight distinti confondono l'utente. | +| `doctor` | **ACCORPA/MANTIENI pubblico** come unico `tht doctor`, includendo controlli host, Compose, storage e configurazione runtime. | +| `db ping` | **PL**: il backend lo usa per rifiutare correttamente una nuova sessione quando il DWH non è raggiungibile o non è read-only. Interno. | +| `db fetch-ca` | **ACCORPA** in `tht setup` o nella configurazione workspace: utile per TLS, ma non giustifica un comando isolato. | +| `ollama ensure` | **PL**: preflight automatico dell'embedder usato dal backend. Interno. | + +### Fasi e decision ledger + +| Comando | Valutazione | +|---|---| +| `phase advance` | **WF**: il gate lo usa per avanzare solo dopo la decisione umana. Primitivo anti-bypass, quindi interno. | +| `phase meta` | **WF**: fornisce al gate la definizione data-driven delle fasi e dei tipi di decisione. Interno. | +| `phase reopen` | **WF**: è il percorso canonico per tornare a una fase precedente e invalidare deterministicamente gli artefatti successivi. | +| `phase show` | **WF**: il gate lo usa per calcolare la fase corrente. Interno. | +| `decision add` | **WF**: persistenza fondamentale delle decisioni del reviewer. Solo gate, non shell utente. | +| `decision add-batch` | **WF**: scrittura atomica delle decisioni multiple. Evita ledger parziali. | +| `decision add-join-set` | **WF**: sostituzione atomica dell'intero insieme di join. | +| `decision list` | **RIMUOVI**: nessun chiamante; `session show --json` contiene già il ledger necessario. | +| `decision retract` | **RIMUOVI** dalla CLI: nessun flusso vivo lo invoca e `phase reopen` è il percorso di correzione supportato. La semantica tombstone può restare nel dominio finché utile. | + +### Sessioni + +| Comando | Valutazione | +|---|---| +| `session archive` | **PL**: usato dalla gestione sessioni del backend. | +| `session check` | **WF**: gate oggettivo della fase 5; verifica decisioni e schema linking. | +| `session close` | **PL**: usato dal backend. | +| `session delete` | **PL**: usato dal backend con i relativi controlli applicativi. | +| `session documents` | **WF/PL**: ricostruisce il contesto persistito e alimenta sia Pi sia la GUI. | +| `session fail` | **PL**: usato dal backend per rappresentare il fallimento terminale. | +| `session finalize` | **WF**: chiusura deterministica della fase finale e indicizzazione della domanda risolta. | +| `session list` | **PL**: alimenta la lista sessioni della GUI. | +| `session migrate` | **PL**: eseguito dal servizio one-shot di migrazione server; resta interno dietro `tht sessions migrate`. | +| `session new` | **WF/PL**: crea la persistenza iniziale della domanda; il backend dipende dal JSON restituito. | +| `session preferences get` | **PL**: lettura delle preferenze applicative. Interno. | +| `session preferences set` | **PL**: scrittura delle preferenze applicative. Interno. | +| `session reopen` | **PL**: riapertura dello stato terminale esposta dalla gestione sessioni. | +| `session retrieval-pack` | **WF**: legge il retrieval pack già persistito per il kickoff di Pi. Distinto da `search pack`, che lo costruisce. | +| `session set-group` | **PL**: rinomina il raggruppamento dalla GUI. | +| `session set-name` | **PL**: rinomina la sessione dalla GUI. | +| `session set-question` | **WF**: persiste deterministicamente domanda riscritta e assunzioni. Solo gate. | +| `session set-schema-linking` | **WF**: valida e scrive `schema_linking.json`. Solo gate. | +| `session show` | **WF/PL**: fonte compatta dello stato persistito per resume, gate e backend. | +| `session sync-schema-linking` | **WF**: riproietta deterministicamente il ledger nello schema linking. | +| `session unarchive` | **PL**: usato dalla gestione sessioni del backend. | + +### Schema e retrieval + +| Comando | Valutazione | +|---|---| +| `schema check` | **PL**: validazione delle annotazioni curate nel workflow workspace. | +| `schema columns` | **WF**: il gate usa il catalogo colonne per validare e correggere il linking. | +| `schema introspect` | **WF**: fallback previsto dal contratto quando manca lo schema fisico; la modalità refresh resta manutenzione. | +| `schema render` | **WF**: produce il contesto mschema usato dal modello. | +| `schema suggest-fks` | **PL**: comando del flusso operatore per le annotazioni FK curate. | +| `search find` | **WF**: ricerca mirata di evidence, valori e formule durante le fasi. | +| `search pack` | **WF/PL**: costruisce e persiste il contesto iniziale F1; usato anche dal backend. | + +### CTE, SQL e datamart + +| Comando | Valutazione | +|---|---| +| `cte info` | **WF**: restituisce SQL persistito, posizione nel piano e ultimo test. | +| `cte list` | **RIMUOVI**: nessun chiamante o test; `cte plan`, `cte info` e `session documents` coprono il bisogno. | +| `cte next` | **WF**: il gate determina il prossimo CTE da revisionare. | +| `cte plan` | **WF**: persiste l'ordine completo dei CTE. | +| `cte save` | **WF**: tool deterministico di scrittura usato dal gate. | +| `cte test` | **WF**: verifica read-only dei CTE prevista esplicitamente dal contratto. | +| `sql validate` | **WF**: validazione strutturale e read-only prima dell'esecuzione. | +| `sql preview` | **WF/PL**: preview controllata usata dal modello e dalla GUI. | +| `sql set-final` | **WF**: unica scrittura canonica di `sql_final.sql` attraverso il repository di sessione. | +| `sql export` | **PL**: esportazione richiesta dalla GUI. | +| `sql explain` | **RIMUOVI**: nessun chiamante, test o requisito nel workflow corrente. Si reintroduce solo con un vero passo di analisi del piano. | +| `sql save` | **RIMUOVI**: duplica `set-final` ed `export` e permette un percorso di scrittura non usato. | +| `datamart generate` | **WF**: fase 8 del workflow. | + +### Memory + +| Comando | Valutazione | +|---|---| +| `memory promote` | **WF**: preview dei candidati di promozione usata dal gate. | +| `memory save-one` | **WF**: persistenza atomica della singola memory approvata. | +| `memory search` | **WF**: recupero delle memory riutilizzabili nella fase 2. | +| `memory solved-index` | **WF**: recupero manuale previsto se l'indicizzazione al finalize fallisce. | +| `memory solved-search` | **WF**: recupero di domande risolte simili nelle fasi successive. | +| `memory list` | **ADV**: mantenere per amministrare record errati, ma fuori dall'help base. | +| `memory show` | **ADV**: mantenere insieme a `list` per ispezione puntuale. | +| `memory update` | **ADV**: mantenere per correggere il merito di una memory senza alterarne la provenienza. | +| `memory delete` | **ADV**: mantenere come rimedio selettivo; richiede conferma esplicita nella nuova CLI. | +| `memory index` | **ADV**: utile come riparazione/full-resync, ma va presentato come manutenzione e non come uso normale. | +| `memory clear` | **RIMUOVI**: distruzione globale non usata; confligge con una UX sicura di backup/ripristino. | +| `memory migrate` | **RIMUOVI**: migrazione legacy una tantum senza dati di produzione da preservare. | + +### Preprocessing, evidence e indici + +| Comando | Valutazione | +|---|---| +| `preprocess dwh` | **PL**: pipeline canonica usata da `tht workspace preprocess dwh/run`. | +| `preprocess evidence` | **PL**: pipeline canonica usata da `tht workspace preprocess evidence/run`. | +| `vector index-schema` | **PL**: indicizzazione schema usata dal workflow workspace. | +| `evidence extract` | **RIMUOVI**: primitivo superato dalla pipeline versionata `preprocess evidence`. Conservare soltanto la logica riusata. | +| `evidence index` | **RIMUOVI**: primitivo superato dalla stessa pipeline versionata. | +| `lsh build` | **RIMUOVI** come comando: è già uno step di `preprocess dwh`; il builder resta interno. | +| `lsh query` | **RIMUOVI**: probe visuale senza chiamanti, test o documentazione operativa. La ricerca applicativa passa da `search find`. | +| `vector init` | **RIMUOVI**: il controllo di Qdrant/embedder è ormai coperto dal reconciler di collezione, da `ollama ensure` e dal nuovo `tht doctor`. | + +### Formule di concetto + +| Comando | Valutazione | +|---|---| +| `formula save` | **RIMUOVI** dalla CLI corrente: nessun chiamante, test o flusso di approvazione lo usa. Conservare il formato/store e la lettura tramite `search find --kind formula`. | +| `formula list` | **RIMUOVI**: stesso sottosistema incompleto. Un futuro flusso di curation dovrà progettare insieme creazione, approvazione, elenco e modifica. | + +## Conseguenza per la nuova CLI unica + +La semplificazione migliore non consiste nel rinominare tutti i 55 contratti vivi o nel mostrarli +all'utente. Consiste nel mantenere un unico eseguibile `tht` con due livelli di visibilità: + +1. l'help ordinario mostra soltanto setup, lifecycle, backup/restore, Pi e workspace; +2. i contratti WF/PL restano invocabili dallo stesso eseguibile, ma sono interni/nascosti e usati da + backend, gate e job one-shot. + +In questo modo l'utente vede una CLI piccola, mentre il workflow non subisce una riscrittura inutile +e rischiosa. Non serve un secondo eseguibile né un alias `thothctl`. diff --git a/docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md b/docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md new file mode 100644 index 00000000..9f2095d9 --- /dev/null +++ b/docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md @@ -0,0 +1,148 @@ +# Proposta maintain-erase-enhance per i comandi `tht` + +Data: 2026-08-15 + +## Criterio + +- **MAINTAIN**: il comando resta disponibile senza modifiche sostanziali. Come richiesto, non viene + aggiunta una motivazione. +- **ERASE**: il comando viene eliminato dalla nuova CLI; la motivazione indica la duplicazione, il + superamento o l'assenza di un utilizzo reale. +- **ENHANCE**: la capacità viene mantenuta, ma il comando viene migliorato, accorpato o reso più + sicuro. La proposta indica l'intervento. + +La proposta copre tutti i 77 comandi terminali dell'attuale CLI Python. + +## Sintesi + +| Proposta | Numero | +|---|---:| +| MAINTAIN | 55 | +| ENHANCE | 8 | +| ERASE | 14 | +| **Totale** | **77** | + +## Lista completa + +### Diagnostica, configurazione e dipendenze + +| Comando | Proposta | +|---|---| +| `config check` | **ENHANCE** — incorporare la validazione nel comando pubblico `tht doctor`, mantenendo una funzione interna riutilizzabile e l'output strutturato. Evita due preflight sovrapposti. | +| `doctor` | **ENHANCE** — farne l'unica diagnostica multilivello: installazione, descriptor, Compose, storage, configurazione runtime, DWH, Pi, Qdrant ed embedder. Deve offrire output umano e `--json`, senza mutare lo stato. | +| `db ping` | **MAINTAIN** | +| `db fetch-ca` | **ENHANCE** — integrarlo nel setup guidato del workspace, mostrando endpoint e fingerprint prima della conferma. Può restare disponibile come operazione TLS avanzata, ma non come passaggio manuale obbligatorio. | +| `ollama ensure` | **MAINTAIN** | + +### Fasi e decision ledger + +| Comando | Proposta | +|---|---| +| `phase advance` | **MAINTAIN** | +| `phase meta` | **MAINTAIN** | +| `phase reopen` | **MAINTAIN** | +| `phase show` | **MAINTAIN** | +| `decision add` | **MAINTAIN** | +| `decision add-batch` | **MAINTAIN** | +| `decision add-join-set` | **MAINTAIN** | +| `decision list` | **ERASE** — non ha chiamanti reali e duplica il ledger già restituito da `session show --json`. | +| `decision retract` | **ERASE** — non è invocato dal workflow corrente; `phase reopen` è il percorso supportato per correggere e invalidare deterministicamente le decisioni. La semantica tombstone può restare nel dominio. | + +### Sessioni + +| Comando | Proposta | +|---|---| +| `session archive` | **MAINTAIN** | +| `session check` | **MAINTAIN** | +| `session close` | **MAINTAIN** | +| `session delete` | **MAINTAIN** | +| `session documents` | **MAINTAIN** | +| `session fail` | **MAINTAIN** | +| `session finalize` | **MAINTAIN** | +| `session list` | **MAINTAIN** | +| `session migrate` | **MAINTAIN** | +| `session new` | **MAINTAIN** | +| `session preferences get` | **MAINTAIN** | +| `session preferences set` | **MAINTAIN** | +| `session reopen` | **MAINTAIN** | +| `session retrieval-pack` | **MAINTAIN** | +| `session set-group` | **MAINTAIN** | +| `session set-name` | **MAINTAIN** | +| `session set-question` | **MAINTAIN** | +| `session set-schema-linking` | **MAINTAIN** | +| `session show` | **MAINTAIN** | +| `session sync-schema-linking` | **MAINTAIN** | +| `session unarchive` | **MAINTAIN** | + +### Schema e retrieval + +| Comando | Proposta | +|---|---| +| `schema check` | **MAINTAIN** | +| `schema columns` | **MAINTAIN** | +| `schema introspect` | **MAINTAIN** | +| `schema render` | **MAINTAIN** | +| `schema suggest-fks` | **MAINTAIN** | +| `search find` | **MAINTAIN** | +| `search pack` | **MAINTAIN** | + +### CTE, SQL e datamart + +| Comando | Proposta | +|---|---| +| `cte info` | **MAINTAIN** | +| `cte list` | **ERASE** — non ha chiamanti o test e sovrappone informazioni già disponibili con `cte plan`, `cte info` e `session documents`. | +| `cte next` | **MAINTAIN** | +| `cte plan` | **MAINTAIN** | +| `cte save` | **MAINTAIN** | +| `cte test` | **MAINTAIN** | +| `sql validate` | **MAINTAIN** | +| `sql preview` | **MAINTAIN** | +| `sql set-final` | **MAINTAIN** | +| `sql export` | **MAINTAIN** | +| `sql explain` | **ERASE** — non è usato né testato dal workflow attuale. Va reintrodotto soltanto se l'analisi del piano diventa un passo esplicito del processo. | +| `sql save` | **ERASE** — duplica `sql set-final` e `sql export` e introduce un percorso di scrittura non utilizzato. | +| `datamart generate` | **MAINTAIN** | + +### Memory + +| Comando | Proposta | +|---|---| +| `memory promote` | **MAINTAIN** | +| `memory save-one` | **MAINTAIN** | +| `memory search` | **MAINTAIN** | +| `memory solved-index` | **MAINTAIN** | +| `memory solved-search` | **MAINTAIN** | +| `memory list` | **ENHANCE** — trasformarlo in una vista amministrativa paginata, con filtri, provenienza, stato e output `--json`; non mostrarlo nell'help base. | +| `memory show` | **ENHANCE** — mostrare provenienza immutabile, decisione sorgente, stato dell'indice e riferimenti necessari a una correzione consapevole. | +| `memory update` | **ENHANCE** — limitare l'aggiornamento ai campi modificabili, mostrare un diff prima della conferma e impedire modifiche alla provenienza. | +| `memory delete` | **ENHANCE** — richiedere identificatore esatto e conferma esplicita, mostrare l'impatto e verificare la rimozione coerente da registro e indice. | +| `memory index` | **ENHANCE** — riposizionarlo come comando di repair: prima rileva il drift, poi ricostruisce soltanto con conferma e verifica finale. Non deve sembrare un'operazione ordinaria. | +| `memory clear` | **ERASE** — cancellazione globale non usata e troppo facile da eseguire per errore; backup/ripristino e cancellazione selettiva sono percorsi più sicuri. | +| `memory migrate` | **ERASE** — migrazione legacy una tantum; non esistono dati di produzione da preservare e la nuova architettura può partire direttamente dal formato corrente. | + +### Preprocessing, evidence e indici + +| Comando | Proposta | +|---|---| +| `preprocess dwh` | **MAINTAIN** | +| `preprocess evidence` | **MAINTAIN** | +| `vector index-schema` | **MAINTAIN** | +| `evidence extract` | **ERASE** — è un primitivo superato dalla pipeline versionata `preprocess evidence`; l'eventuale logica condivisa resta interna. | +| `evidence index` | **ERASE** — è un secondo primitivo superato dalla stessa pipeline, che già gestisce materializzazione, indicizzazione, versionamento e resume. | +| `lsh build` | **ERASE** — la costruzione LSH è già uno step di `preprocess dwh`; mantenere due ingressi permette esecuzioni parziali incoerenti. | +| `lsh query` | **ERASE** — probe visuale senza chiamanti, test o documentazione operativa; il workflow usa `search find`. | +| `vector init` | **ERASE** — il controllo di Qdrant ed embedder è già coperto dal reconciler della collezione, da `ollama ensure` e dal nuovo `tht doctor`. | + +### Formule di concetto + +| Comando | Proposta | +|---|---| +| `formula save` | **ERASE** — non ha chiamanti, test o un flusso di approvazione completo. Il formato e lo store possono restare disponibili alla ricerca finché non viene progettata una vera curation. | +| `formula list` | **ERASE** — appartiene allo stesso sottosistema incompleto; un futuro flusso deve progettare insieme creazione, approvazione, elenco, modifica e cancellazione. | + +## Impatto sulla UX + +I 55 comandi `MAINTAIN` comprendono molti contratti macchina intoccabili. Mantenerli non implica +mostrarli tutti nell'help principale. La futura CLI unica può conservare gli stessi percorsi per +backend, gate e job, mostrando all'utente soltanto i gruppi operativi di primo livello. diff --git a/docs/superpowers/plans/2026-08-03-diagnostic-contract-extension.md b/docs/superpowers/plans/2026-08-03-diagnostic-contract-extension.md new file mode 100644 index 00000000..56c8b003 --- /dev/null +++ b/docs/superpowers/plans/2026-08-03-diagnostic-contract-extension.md @@ -0,0 +1,59 @@ +# Workspace Diagnostic Contract Extension Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:subagent-driven-development` to implement this plan task-by-task. + +**Goal:** Make workspace connector diagnostics executable without weakening least privilege or storing secrets in Git. + +**Architecture:** Extend the canonical descriptor with vector database/schema identity, optional writer-only bindings, and declared REST/embedding diagnostic contracts. The backend resolves only local `*_FILE` values, uses bounded transport adapters, and runs a vector write probe only when a reversible writer RPC and a distinct local writer binding both exist. + +**Tech Stack:** TypeScript, Zod, YAML, Fastify, native fetch, OpenSSH, Vitest. + +## Global Constraints + +- Descriptor Git files never contain secrets; all secrets are deterministic local variables ending `_FILE`. +- Writer credentials are optional and distinct from reader credentials; never substitute a reader key. +- A write probe requires a declared reversible RPC, bounded cleanup in `finally`, and must never call an upsert-only endpoint. +- REST diagnostics use only descriptor-declared method, path, auth mode and response fields. +- SSH uses `StrictHostKeyChecking=yes`, a short-lived local forward, and cleanup in `finally`. +- Every diagnostic uses `workspaceDiagnosticTimeoutMs`, emits only stable redacted errors, and is tested with fakes or loopback only. + +## File Structure + +| Path | Responsibility | +| --- | --- | +| `backend/src/workspaces/schema.ts` | Canonical vector identity and diagnostics contracts. | +| `backend/src/workspaces/contracts.ts` | Reader/writer variable names and `.env.example` documentation. | +| `backend/src/workspaces/runtime-renderer.ts` | Renders vector `database` and `schema`, not DWH identity. | +| `backend/src/workspaces/diagnostics.ts` | Bounded direct/REST/SSH/vector/embedding adapters. | +| `backend/test/workspaces-{schema,contracts,diagnostics}.test.ts` | TDD coverage for validation, protocol and cleanup. | +| `docs/workspace-diagnostic-protocol.md` | Service-operator protocol and local variable contract. | + +### Task 1: Define canonical diagnostic contracts + +**Files:** modify `backend/src/workspaces/schema.ts`, `backend/src/workspaces/contracts.ts`, `backend/src/workspaces/runtime-renderer.ts`; test `backend/test/workspaces-schema.test.ts`, `backend/test/workspaces-contracts.test.ts`. + +- [ ] Write failing tests that reject blank `vector_store.database`/`schema`, render their distinct values, and generate `VECTOR_WRITER_*_FILE` only for an optional `vector_writer` role. +- [ ] Run `npx vitest run test/workspaces-schema.test.ts test/workspaces-contracts.test.ts` and observe failure. +- [ ] Implement `vector_store.database`, `vector_store.schema`, optional `vector_writer`, `diagnostics.dwh_rest`, `diagnostics.vector_rest` (metadata plus optional reversible probe), and `diagnostics.embedding`, all strictly validated. Keep reader-only workspace valid. +- [ ] Re-run focused tests and `npx tsc --noEmit -p .`. +- [ ] Commit `feat: define workspace diagnostic contracts`. + +### Task 2: Implement declared bounded diagnostics + +**Files:** modify `backend/src/workspaces/diagnostics.ts`; test `backend/test/workspaces-diagnostics.test.ts`. + +- [ ] Write failing faked-transport tests for `POST /rpc/ping`, vector metadata dimensions/metric/collection, reader-only non-write activation, writer probe cleanup after timeout, direct/SSH declared vector database/schema, strict SSH known-host arguments, and redacted malformed response/timeout errors. +- [ ] Run `npx vitest run test/workspaces-diagnostics.test.ts` and observe failure. +- [ ] Implement production adapters using descriptor-declared contracts only, `AbortController` timeouts, injected direct-protocol and SSH process factories, local secret files, and bounded `finally` cleanup after a successful writer probe. +- [ ] Run focused diagnostics, `npx vitest run`, and `npx tsc --noEmit -p .`. +- [ ] Commit `feat: run bounded workspace connector diagnostics`. + +### Task 3: Specify installation and server protocols + +**Files:** create `docs/workspace-diagnostic-protocol.md`; modify `docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md`; test `backend/test/workspaces-contracts.test.ts`. + +- [ ] Write a failing documentation-contract test that writer workspaces render the writer `_FILE` variables. +- [ ] Run `npx vitest run test/workspaces-contracts.test.ts` and observe failure. +- [ ] Document request/response requirements for DWH ping, vector metadata, reversible writer probe, embedding dimensions, SSH known-hosts, reader-only fallback, and every local variable name. +- [ ] Run `npx vitest run && npx tsc --noEmit -p . && git diff --check`. +- [ ] Commit `docs: specify workspace diagnostic protocols`. diff --git a/docs/superpowers/plans/2026-08-03-git-workspace-registry.md b/docs/superpowers/plans/2026-08-03-git-workspace-registry.md new file mode 100644 index 00000000..d1413a21 --- /dev/null +++ b/docs/superpowers/plans/2026-08-03-git-workspace-registry.md @@ -0,0 +1,906 @@ +# Git-backed Workspace Registry Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a Git-backed, portable Workspace Registry with a right-sidebar CRUD experience, deterministic installation bindings, revision-pinned sessions, and detailed tested installation manuals for local and server Docker deployments. + +**Architecture:** A versioned canonical YAML repository is the shared source of truth. The Fastify backend owns schema validation, a persistent Git checkout, immutable runtime snapshots, binding resolution, diagnostics, and publish conflict handling; it renders compatible harness runtime YAML from the validated snapshot. Browser-local storage owns anonymous preferences and drafts, while sessions record the resolved workspace revision and model configuration. + +**Tech Stack:** Node.js 22, TypeScript 5.6, Fastify 5, React 18, Vite, TanStack Query, Vitest, MSW, Python 3.12/Pydantic harness, Git CLI, Docker Compose. + +## Global Constraints + +- The remote Git repository is the sole shared source of truth; GitHub, Gitea, GitLab, and generic SSH/HTTPS remotes are supported through standard Git commands only. +- Never persist secret values in Git, API responses, logs, browser storage, generated documentation, or export bundles. Secret inputs use fixed `THT_WS__*_FILE` names. +- Workspace IDs match `^[a-z][a-z0-9-]{2,62}$`, are immutable, and generate a stable uppercase underscore namespace. +- Keep DWH, vector collection, embedding model, dimensions, and distance metric in shared workspace configuration. Keep active workspace, LLM choice, reasoning level, and drafts in browser-local storage until identity support exists. +- A vector collection and its embedding contract are atomic: dimensions and metric must agree; a user may not switch embeddings for the same collection. +- Support `postgres_direct`, `rest_api`, and `ssh_tunnel` for DWH; support direct, REST, and SSH-tunnel bindings for the vector store. +- A portable workspace may be valid but not activatable on an installation missing its local bindings. Only session start requires operational validation. +- Publish uses a short repository lock, optimistic base-commit/blob checks, field-level HTTP 409 conflicts, and no automatic YAML merge. +- The server and local Docker profiles use a persistent `/data/workspace-registry` volume; the container image and Git repository checkout are separate. +- Snapshot activation is atomic. New sessions record workspace ID and immutable Git revision; resume uses that revision. +- UI labels remain English. Generated workspace documentation remains in the workspace language. +- Preserve the existing `tht` contract: `-c` is a per-command option appended after the subcommand. `--json` stdout remains pristine JSON. +- Use test-first development for every behavioral change. Run `backend` Vitest and `tsc --noEmit`, `frontend` Vitest and `tsc -b`, and relevant harness `pytest` gates before each task commit. + +--- + +## File Structure + +| Path | Responsibility | +|---|---| +| `backend/src/workspaces/types.ts` | Shared registry DTOs, canonical workspace types, stable error codes, API request/response types. | +| `backend/src/workspaces/schema.ts` | YAML parse/serialize, schema/version validation, static semantic validation, canonical renderer, generated docs/contract. | +| `backend/src/workspaces/bindings.ts` | Deterministic variable naming and sanitized local binding resolution. | +| `backend/src/workspaces/git-repository.ts` | Safe Git CLI wrapper, checkout bootstrap, fetch/pull/commit/push, lock, status and blob inspection. | +| `backend/src/workspaces/registry.ts` | CRUD, optimistic publish, snapshots, legacy migration and export/import orchestration. | +| `backend/src/workspaces/diagnostics.ts` | Direct/REST/SSH connector diagnostics and semantic-index probes. | +| `backend/src/workspaces/runtime-renderer.ts` | Renders a validated canonical workspace + bindings to the harness-compatible runtime YAML. | +| `backend/src/routes/workspaces.ts` | Registry HTTP API and strict request validation. | +| `backend/src/config.ts`, `backend/src/app.ts` | Registry configuration and dependency injection. | +| `backend/src/routes/sessions.ts`, `backend/src/tht/tht-runner.ts` | Revision-pinned session start/resume and snapshot config resolution. | +| `harness/tht/session/models.py`, `harness/tht/session/store.py` | Persist and surface `workspace_id` and `workspace_revision`. | +| `frontend/src/api/workspaces.ts` | Typed registry client, multipart import/download helpers. | +| `frontend/src/workspaces/drafts.ts` | Browser-local draft/preference persistence and stale-draft detection. | +| `frontend/src/shell/WorkspaceManager.tsx` | Right-sidebar entry point and page shell. | +| `frontend/src/shell/WorkspaceEditor.tsx` | Sectioned CRUD form, closed choices, field errors, validation and diagnostics view. | +| `frontend/src/shell/WorkspacePublishDialog.tsx` | Diff, pull/publish, conflicts, delete confirmation, import/export controls. | +| `compose.yaml`, `docker-compose.dev.yml`, `docker/core.Dockerfile` | Persistent registry volume, Git/SSH runtime tools, mounted Git trust and secrets. | +| `docs/install/local-workspace-registry.md` | Detailed PC/Mac local Docker installation manual. | +| `docs/install/server-workspace-registry.md` | Detailed server installation manual. | + +## Task 1: Establish backend dependencies and registry configuration + +**Files:** +- Modify: `backend/package.json` +- Modify: `backend/package-lock.json` +- Modify: `backend/src/config.ts` +- Modify: `backend/test/config.test.ts` +- Create: `backend/src/workspaces/types.ts` +- Test: `backend/test/workspaces-config.test.ts` + +**Interfaces:** +- Produces `WorkspaceRegistryConfig`: + +```ts +export interface WorkspaceRegistryConfig { + root: string; + remoteUrl?: string; + branch: string; + gitAuthorName: string; + gitAuthorEmail: string; + installationId: string; + secretRoots: readonly string[]; + maxImportBytes: number; + maxImportEntries: number; +} +``` + +- Produces shared error shape: + +```ts +export type WorkspaceErrorCode = + | "workspace_invalid" | "binding_missing" | "workspace_not_activatable" + | "workspace_stale" | "workspace_conflict" | "git_unavailable" + | "git_auth_failed" | "git_non_fast_forward" | "git_push_rejected" + | "connector_unavailable" | "semantic_index_incompatible"; +``` + +- Consumed by Tasks 2–11. + +- [ ] **Step 1: Write failing configuration tests** + +```ts +test("loads a safe Git workspace registry configuration", () => { + const cfg = loadConfig({ + THT_WORKSPACE_REGISTRY_ROOT: "/data/workspace-registry", + THT_WORKSPACE_GIT_REMOTE: "ssh://git@gitea.example/thoth/workspaces.git", + THT_WORKSPACE_GIT_BRANCH: "main", + THT_WORKSPACE_INSTALLATION_ID: "server-psd-1", + THT_WORKSPACE_SECRET_ROOTS: "/run/secrets,/data/secrets", + }); + expect(cfg.workspaceRegistry).toMatchObject({ root: "/data/workspace-registry", branch: "main" }); +}); + +test("rejects a relative registry root and invalid import limits", () => { + expect(() => loadConfig({ THT_WORKSPACE_REGISTRY_ROOT: "registry" })).toThrow(/registry/i); + expect(() => loadConfig({ THT_WORKSPACE_REGISTRY_ROOT: "/data/registry", THT_WORKSPACE_MAX_IMPORT_BYTES: "0" })).toThrow(/import/i); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/workspaces-config.test.ts` in `backend/` +Expected: FAIL because `workspaceRegistry` does not exist on `AppConfig`. + +- [ ] **Step 3: Add minimal dependencies and configuration** + +Add runtime dependencies `yaml`, `zod`, `yauzl`, and `yazl`; add `@types/yauzl` as a development dependency. Extend `AppConfig` and `loadConfig` with absolute-root, branch, installation-ID, positive-limit, and absolute-secret-root validation. Default the root to `/data/workspace-registry`, branch to `main`, and import limits to 10 MiB/32 entries. Define the DTO and error-code module exactly as above. + +- [ ] **Step 4: Run the focused test and typecheck** + +Run: `npx vitest run test/workspaces-config.test.ts && npx tsc --noEmit -p .` in `backend/` +Expected: PASS with zero TypeScript errors. + +- [ ] **Step 5: Commit** + +```bash +git add backend/package.json backend/package-lock.json backend/src/config.ts backend/src/workspaces/types.ts backend/test/workspaces-config.test.ts +git commit -m "feat: configure Git workspace registry" +``` + +## Task 2: Implement canonical workspace schema, contracts, and generated documentation + +**Files:** +- Create: `backend/src/workspaces/schema.ts` +- Create: `backend/src/workspaces/contracts.ts` +- Create: `backend/test/workspaces-schema.test.ts` +- Create: `backend/test/workspaces-contracts.test.ts` + +**Interfaces:** +- Produces: + +```ts +export interface CanonicalWorkspace { + workspace: { schema_version: 1; id: string; name: string; description?: string; language: "en" | "it" }; + dwh: { engine: "postgres"; database: string; schema: string; supported_transports: DwhTransport[] }; + semantic_index: { + vector_store: { engine: "pgvector"; collection: string; dimensions: number; distance: "cosine" | "l2" | "inner_product"; supported_transports: VectorTransport[] }; + embedding: { provider: "ollama_compatible" | "openai_compatible"; model: string; dimensions: number }; + }; + llm_policy: { default?: `${string}/${string}`; allowed: `${string}/${string}`[] }; +} +export function parseWorkspaceYaml(source: string): CanonicalWorkspace; +export function serializeWorkspaceYaml(workspace: CanonicalWorkspace): string; +export function buildInstallationContract(workspace: CanonicalWorkspace): InstallationContract; +export function renderWorkspaceDocs(workspace: CanonicalWorkspace): { envExample: string; markdown: string }; +``` + +- Consumed by Tasks 3–10. + +- [ ] **Step 1: Write failing schema and contract tests** + +```ts +test("rejects a workspace whose embedding dimensions differ from its collection", () => { + expect(() => parseWorkspaceYaml(validYaml.replace("dimensions: 768", "dimensions: 1536"))).toThrow(/dimensions/i); +}); + +test("rejects an LLM default outside its allowlist", () => { + expect(() => parseWorkspaceYaml(validYaml.replace("- zai/glm-5.2", "- openai/gpt-5"))).toThrow(/allowlist/i); +}); + +test("generates stable FILE-based secret requirements from an immutable ID", () => { + const contract = buildInstallationContract(validWorkspace); + expect(contract.variables.map((v) => v.name)).toContain("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"); + expect(renderWorkspaceDocs(validWorkspace).envExample).not.toContain("secret-value"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/workspaces-schema.test.ts test/workspaces-contracts.test.ts` in `backend/` +Expected: FAIL because schema and contract modules do not exist. + +- [ ] **Step 3: Implement the canonical schema** + +Use Zod strict objects to reject unknown keys. Enforce workspace ID syntax, positive dimensions/ports/timeouts, allowed enum values, matching vector/embedding dimensions, and default-in-allowlist. Use `yaml` with sorted canonical keys for serialization. Generate a contract with role/suffix metadata, not arbitrary variable names. Generate English UI-oriented documentation and workspace-language prose; render all secret requirements as `*_FILE` variables. + +- [ ] **Step 4: Run focused tests and backend typecheck** + +Run: `npx vitest run test/workspaces-schema.test.ts test/workspaces-contracts.test.ts && npx tsc --noEmit -p .` in `backend/` +Expected: PASS; repeated serialize/parse returns the same canonical object. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/workspaces/schema.ts backend/src/workspaces/contracts.ts backend/test/workspaces-schema.test.ts backend/test/workspaces-contracts.test.ts +git commit -m "feat: add canonical workspace schema" +``` + +## Task 3: Resolve local bindings and render harness-compatible runtime configuration + +**Files:** +- Create: `backend/src/workspaces/bindings.ts` +- Create: `backend/src/workspaces/runtime-renderer.ts` +- Create: `backend/test/workspaces-bindings.test.ts` +- Create: `backend/test/workspace-runtime-renderer.test.ts` +- Modify: `backend/src/tht/tht-runner.ts` + +**Interfaces:** +- Consumes `CanonicalWorkspace` and `InstallationContract` from Task 2. +- Produces: + +```ts +export interface ResolvedBinding { transport: DwhTransport | VectorTransport; values: Record; missing: string[]; } +export function resolveBinding(workspace: CanonicalWorkspace, role: "DWH" | "VECTOR" | "EMBEDDING", env: NodeJS.ProcessEnv, secretRoots: readonly string[]): ResolvedBinding; +export function renderRuntimeConfig(workspace: CanonicalWorkspace, bindings: RuntimeBindings, paths: RuntimePaths): string; +``` + +- Extends `ThtRunner.buildArgv(args, workspaceConfigPath?)` so it accepts an absolute immutable snapshot file and still appends `-c ` after the `tht` subcommand. + +- [ ] **Step 1: Write failing binding and renderer tests** + +```ts +test("marks a portable workspace non-activatable when its local REST key file is absent", () => { + const result = resolveBinding(workspace, "DWH", { THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api" }, ["/run/secrets"]); + expect(result.missing).toContain("THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE"); +}); + +test("renders a direct PostgreSQL binding to the legacy harness shape", () => { + const yaml = renderRuntimeConfig(workspace, directBindings, runtimePaths); + expect(yaml).toContain("type: postgres_direct"); + expect(yaml).toContain("schema: datawarehouse"); +}); + +test("passes an absolute snapshot config after the tht subcommand", () => { + expect(runner.buildArgv(["session", "new"], "/data/workspace-registry/snapshots/a/psd-clinical.yaml")).toEqual([ + "session", "new", "-c", "/data/workspace-registry/snapshots/a/psd-clinical.yaml", + ]); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts` in `backend/` +Expected: FAIL because binding resolution and runtime rendering do not exist. + +- [ ] **Step 3: Implement bindings and renderer** + +Normalize ID namespaces by uppercasing and replacing `-` with `_`. Require `*_FILE` paths to be absolute, regular/readable, and within configured secret roots; return names only in diagnostics. Render legacy `database`, `rest`, `vector_db`, `embeddings`, and `paths` fields required by the current harness from canonical schema plus local binding values. Implement direct and REST first; implement SSH as a temporary local port created by Task 5 diagnostics. Do not alter existing `-c` ordering. + +- [ ] **Step 4: Run focused tests, current ThtRunner tests, and typecheck** + +Run: `npx vitest run test/workspaces-bindings.test.ts test/workspace-runtime-renderer.test.ts test/tht-runner.test.ts && npx tsc --noEmit -p .` in `backend/` +Expected: PASS with no secret value present in assertions or output. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/workspaces/bindings.ts backend/src/workspaces/runtime-renderer.ts backend/src/tht/tht-runner.ts backend/test/workspaces-bindings.test.ts backend/test/workspace-runtime-renderer.test.ts +git commit -m "feat: resolve workspace bindings into runtime configs" +``` + +## Task 4: Implement a safe persistent Git repository and immutable snapshots + +**Files:** +- Create: `backend/src/workspaces/git-repository.ts` +- Create: `backend/src/workspaces/registry.ts` +- Create: `backend/test/workspaces-git-repository.test.ts` +- Create: `backend/test/workspace-registry.test.ts` +- Modify: `backend/src/app.ts` + +**Interfaces:** +- Produces: + +```ts +export interface GitStatus { branch: string; head?: string; ahead: number; behind: number; degraded: boolean; lastError?: WorkspaceErrorCode; } +export interface WorkspaceRevision { id: string; commit: string; blob: string; snapshotPath: string; } +export class WorkspaceRegistry { + bootstrap(): Promise; + list(): Promise; + read(id: string): Promise<{ workspace: CanonicalWorkspace; revision: WorkspaceRevision }>; + pull(): Promise; + publish(request: PublishWorkspaceRequest): Promise; +} +``` + +- `buildApp` receives an injected registry in tests and creates the configured registry in production. + +- [ ] **Step 1: Write failing Git lifecycle tests using a temporary bare remote** + +```ts +test("bootstraps a checkout and activates a validated immutable snapshot", async () => { + const registry = await registryFor(tempBareRemote); + const status = await registry.bootstrap(); + expect(status.head).toMatch(/[0-9a-f]{40}/); + expect(await exists(registry.snapshotPath(status.head!, "psd-clinical"))).toBe(true); +}); + +test("keeps the last valid snapshot when a pulled commit has invalid YAML", async () => { + await pushInvalidWorkspace(tempBareRemote); + await expect(registry.pull()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(await registry.read("psd-clinical")).toMatchObject({ revision: { commit: initialCommit } }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/workspaces-git-repository.test.ts test/workspace-registry.test.ts` in `backend/` +Expected: FAIL because `GitWorkspaceRepository` and `WorkspaceRegistry` do not exist. + +- [ ] **Step 3: Implement Git operations and snapshots** + +Use `spawn`/`execFile` with fixed argument arrays and `cwd` pinned under the registry root. Create `repo`, `snapshots`, `state`, and `locks` at bootstrap. Clone only when checkout is absent; otherwise fetch and fast-forward. Validate every workspace and generated artifact before atomically writing `state/active.json` and snapshot directories. Use a promise-based in-process lock plus an advisory lock file for publish/pull. Classify Git stderr into stable sanitized error codes. Keep the last active state when clone/fetch/pull fails. + +- [ ] **Step 4: Run focused tests and full backend test suite** + +Run: `npx vitest run test/workspaces-git-repository.test.ts test/workspace-registry.test.ts && npx vitest run && npx tsc --noEmit -p .` in `backend/` +Expected: PASS; tests prove no shell interpolation and active snapshot fallback. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/workspaces/git-repository.ts backend/src/workspaces/registry.ts backend/src/app.ts backend/test/workspaces-git-repository.test.ts backend/test/workspace-registry.test.ts +git commit -m "feat: manage workspace Git checkout and snapshots" +``` + +## Task 5: Add diagnostics for direct, REST, SSH, vector, and embedding bindings + +**Files:** +- Create: `backend/src/workspaces/diagnostics.ts` +- Create: `backend/test/workspaces-diagnostics.test.ts` +- Modify: `docker/core.Dockerfile` +- Modify: `backend/src/config.ts` + +**Interfaces:** +- Produces: + +```ts +export interface Diagnostic { level: "error" | "warning" | "info"; code: WorkspaceErrorCode | "binding_ok"; field?: string; message: string; } +export interface WorkspaceDiagnostics { activatable: boolean; diagnostics: Diagnostic[]; } +export async function diagnoseWorkspace(workspace: CanonicalWorkspace, bindings: RuntimeBindings, options: { writeProbe: boolean }): Promise; +``` + +- Consumed by the registry API and frontend. + +- [ ] **Step 1: Write failing adapter tests** + +```ts +test("reports the missing vector collection dimensions as semantic-index incompatibility", async () => { + const result = await diagnoseWorkspace(workspace, fakeBindings({ vectorDimensions: 1536 }), { writeProbe: false }); + expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "semantic_index_incompatible" })); +}); + +test("refuses an SSH tunnel when known-hosts is missing", async () => { + const result = await diagnoseWorkspace(workspace, sshBindingsWithoutKnownHosts, { writeProbe: false }); + expect(result.activatable).toBe(false); + expect(result.diagnostics[0].field).toContain("SSH_KNOWN_HOSTS_FILE"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/workspaces-diagnostics.test.ts` in `backend/` +Expected: FAIL because diagnostics adapters do not exist. + +- [ ] **Step 3: Implement sanitized diagnostic adapters** + +Add `git` and `openssh-client` to the Debian runtime image. Implement injectable adapter interfaces so tests use fakes. Direct and REST diagnostics must test resolution, TLS, authentication, and logical resource metadata without returning response bodies. SSH diagnostics must require explicit known-hosts verification and create a temporary loopback tunnel only for the probe. Vector diagnostics must compare collection dimensions/metric; embedding diagnostics must check model availability and probe vector dimensions. Add an explicit write-probe path that writes and removes only a random diagnostic record; ordinary validation remains read-only. + +- [ ] **Step 4: Run tests, Docker build, and typecheck** + +Run: `npx vitest run test/workspaces-diagnostics.test.ts && npx tsc --noEmit -p .` in `backend/` +Run: `docker build -f docker/core.Dockerfile .` from repository root +Expected: PASS; the image contains `git` and `ssh` while still running as non-root. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/workspaces/diagnostics.ts backend/test/workspaces-diagnostics.test.ts backend/src/config.ts docker/core.Dockerfile +git commit -m "feat: diagnose workspace connector bindings" +``` + +## Task 6: Expose validated registry CRUD, pull/publish, conflict, and bundle APIs + +**Files:** +- Create: `backend/src/routes/workspaces.ts` +- Create: `backend/test/routes-workspaces.test.ts` +- Modify: `backend/src/routes/meta.ts` +- Modify: `backend/src/app.ts` +- Modify: `backend/package.json` only if a Fastify multipart plugin is required +- Modify: `backend/package-lock.json` only if dependencies change + +**Interfaces:** +- Replaces metadata-only workspace listing with: + +```ts +GET /workspace-registry/status +POST /workspace-registry/pull +GET /workspaces +GET /workspaces/:id +POST /workspaces/validate +POST /workspaces/:id/test +POST /workspaces/publish +GET /workspaces/:id/export +POST /workspaces/import +``` + +- `POST /workspaces/publish` accepts: + +```ts +type PublishWorkspaceRequest = + | { action: "create"; workspace: CanonicalWorkspace; baseCommit: string } + | { action: "update"; workspace: CanonicalWorkspace; baseCommit: string; baseBlob: string } + | { action: "delete"; id: string; baseCommit: string; baseBlob: string }; +``` + +- [ ] **Step 1: Write failing route tests** + +```ts +test("returns a 409 field conflict instead of overwriting a changed workspace", async () => { + const res = await app.inject({ method: "POST", url: "/workspaces/publish", payload: staleUpdate }); + expect(res.statusCode).toBe(409); + expect(res.json()).toMatchObject({ code: "workspace_conflict", fields: ["semantic_index.embedding.model"] }); +}); + +test("rejects a zip-slip import without writing a checkout file", async () => { + const res = await importBundle(app, zipWith("../escape.yaml", "bad")); + expect(res.statusCode).toBe(400); + expect(res.json()).toMatchObject({ code: "workspace_invalid" }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/routes-workspaces.test.ts` in `backend/` +Expected: FAIL because registry routes do not exist. + +- [ ] **Step 3: Implement routes and secure archive handling** + +Use schema-validated JSON payloads; never accept raw target paths. Register multipart parsing with a 10 MiB upload limit. Use `yauzl` lazy entry enumeration and reject absolute names, `..`, backslashes, symlinks, extra entries, and checksum/schema failures before creating a browser draft response. Use `yazl` to create `manifest.json`, `workspace.yaml`, `contract.env.example`, and `README.md`; set attachment headers. Publish generated docs with the YAML in one commit. Preserve `/models` and existing workspace selector compatibility by returning summary records from `GET /workspaces`. + +- [ ] **Step 4: Run route tests, full backend suite, and typecheck** + +Run: `npx vitest run test/routes-workspaces.test.ts && npx vitest run && npx tsc --noEmit -p .` in `backend/` +Expected: PASS; error bodies are sanitized and status/pull endpoints do not expose Git credentials. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/routes/workspaces.ts backend/src/routes/meta.ts backend/src/app.ts backend/test/routes-workspaces.test.ts backend/package.json backend/package-lock.json +git commit -m "feat: expose workspace registry API" +``` + +## Task 7: Pin sessions to canonical workspace snapshots and move preferences to the browser + +**Files:** +- Modify: `backend/src/routes/sessions.ts` +- Modify: `backend/src/tht/tht-runner.ts` +- Modify: `backend/src/settings/settings-store.ts` +- Modify: `backend/src/routes/settings.ts` +- Modify: `backend/test/routes-sessions.test.ts` +- Modify: `backend/test/routes-settings.test.ts` +- Modify: `harness/tht/session/models.py` +- Modify: `harness/tht/session/store.py` +- Modify: `harness/tests/test_session_documents.py` + +**Interfaces:** +- New session request becomes: + +```ts +interface CreateSessionRequest { + question: string; + name?: string; + workspaceId: string; + provider?: string; + model?: string; + thinking?: string; +} +``` + +- Session manifest adds optional legacy-compatible fields: + +```python +workspace_id: str | None = None +workspace_revision: str | None = None +``` + +- [ ] **Step 1: Write failing session and manifest tests** + +```ts +test("creates a session from the active immutable workspace revision", async () => { + await app.inject({ method: "POST", url: "/sessions", payload: { question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" } }); + expect(runner.sessionNew).toHaveBeenCalledWith(expect.objectContaining({ workspaceConfigPath: "/data/workspace-registry/snapshots/abc/psd-clinical.yaml" })); +}); +``` + +```python +def test_manifest_persists_workspace_revision(): + manifest = new_session_manifest("q", db, workspace_id="psd-clinical", workspace_revision="a" * 40) + assert manifest.workspace_id == "psd-clinical" + assert manifest.workspace_revision == "a" * 40 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/routes-sessions.test.ts test/routes-settings.test.ts` in `backend/` +Run: `.venv/bin/pytest tests/test_session_documents.py -q` in `harness/` +Expected: FAIL because session requests and manifests do not carry workspace revisions. + +- [ ] **Step 3: Implement revision pinning and browser preference contract** + +Resolve `workspaceId` from the active registry snapshot, perform operational validation before creating a session, validate the selected LLM against `llm_policy`, then pass the absolute snapshot config to `ThtRunner`. Persist ID/revision with provider/model/thinking. Resume uses the manifest revision and fails with a sanitized compatibility error only if the retained snapshot is unavailable. Remove server-global workspace/model/thinking persistence from the settings flow; retain only installation-wide defaults required for backward compatibility. Keep legacy session behavior when manifest revision is absent and emit a warning in its response. + +- [ ] **Step 4: Run backend and harness verification** + +Run: `npx vitest run test/routes-sessions.test.ts test/routes-settings.test.ts && npx tsc --noEmit -p .` in `backend/` +Run: `.venv/bin/pytest tests/test_session_documents.py tests/test_session_mutations.py -q` in `harness/` +Expected: PASS; manifest serialization remains backward compatible. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/routes/sessions.ts backend/src/tht/tht-runner.ts backend/src/settings/settings-store.ts backend/src/routes/settings.ts backend/test/routes-sessions.test.ts backend/test/routes-settings.test.ts harness/tht/session/models.py harness/tht/session/store.py harness/tests/test_session_documents.py +git commit -m "feat: pin sessions to workspace revisions" +``` + +## Task 8: Implement browser-local workspace preferences, drafts, and typed registry API client + +**Files:** +- Modify: `frontend/src/api/workspaces.ts` +- Modify: `frontend/src/api/client.ts` +- Create: `frontend/src/workspaces/drafts.ts` +- Create: `frontend/src/api/workspaces.test.ts` +- Create: `frontend/src/workspaces/drafts.test.ts` +- Modify: `frontend/src/api/sessions.ts` +- Modify: `frontend/src/shell/SteerInput.tsx` +- Modify: `frontend/src/shell/SteerInput.test.tsx` + +**Interfaces:** +- Produces: + +```ts +export interface WorkspacePreference { workspaceId?: string; provider?: string; model?: string; thinking?: string; } +export interface WorkspaceDraft { workspaceId: string; baseCommit: string; baseBlob?: string; workspace: CanonicalWorkspace; updatedAt: string; } +export const workspacePreferences = { load(): WorkspacePreference; save(value: WorkspacePreference): void; }; +export const workspaceDrafts = { load(id: string): WorkspaceDraft | undefined; save(draft: WorkspaceDraft): void; discard(id: string): void; }; +``` + +- [ ] **Step 1: Write failing API and browser-storage tests** + +```ts +test("keeps an anonymous user's model selection in browser storage", () => { + workspacePreferences.save({ workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "medium" }); + expect(workspacePreferences.load()).toMatchObject({ model: "glm-5.2" }); +}); + +test("uploads a workspace bundle without JSON content type", async () => { + await importWorkspace(new File(["zip"], "clinical.thoth-workspace.zip")); + expect(request.headers.get("content-type")).toMatch(/multipart\/form-data/); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/api/workspaces.test.ts src/workspaces/drafts.test.ts src/shell/SteerInput.test.tsx` in `frontend/` +Expected: FAIL because preference/draft modules and multipart client support do not exist. + +- [ ] **Step 3: Implement local preference and draft storage** + +Namespace LocalStorage keys by `thothii.workspace-registry.v1`. Store only canonical drafts, revision metadata, and non-secret display preferences. Add multipart-aware `apiFetch` behavior that does not override `FormData` content type. Update the composer footer to load workspace policy and model choices from the registry client, store its choice locally, and pass the explicit selection to session creation. Do not put secrets or diagnostics response bodies in LocalStorage. + +- [ ] **Step 4: Run focused tests, all frontend tests, and typecheck** + +Run: `npx vitest run src/api/workspaces.test.ts src/workspaces/drafts.test.ts src/shell/SteerInput.test.tsx && npx vitest run && npx tsc -b` in `frontend/` +Expected: PASS; existing session creation tests update their payload expectation to include `workspaceId`. + +- [ ] **Step 5: Commit** + +```bash +git add frontend/src/api/workspaces.ts frontend/src/api/client.ts frontend/src/workspaces/drafts.ts frontend/src/api/workspaces.test.ts frontend/src/workspaces/drafts.test.ts frontend/src/api/sessions.ts frontend/src/shell/SteerInput.tsx frontend/src/shell/SteerInput.test.tsx +git commit -m "feat: store workspace preferences and drafts locally" +``` + +## Task 9: Build the right-sidebar Workspace Management CRUD page + +**Files:** +- Create: `frontend/src/shell/WorkspaceManager.tsx` +- Create: `frontend/src/shell/WorkspaceEditor.tsx` +- Create: `frontend/src/shell/WorkspaceManager.test.tsx` +- Create: `frontend/src/shell/WorkspaceEditor.test.tsx` +- Modify: `frontend/src/shell/AppShell.tsx` +- Modify: `frontend/src/shell/ModelActivityPanel.tsx` + +**Interfaces:** +- `WorkspaceManager` receives `open: boolean`, `onClose(): void`, and uses registry React Query keys `workspace-registry-status`, `workspaces`, and `workspace:`. +- `WorkspaceEditor` receives `{ draft?: WorkspaceDraft; onSaveDraft(draft): void; onPublish(request): Promise }`. + +- [ ] **Step 1: Write failing interaction tests** + +```tsx +test("opens Workspace management from the right-side activity panel", async () => { + render(); + await user.click(screen.getByRole("button", { name: "Workspace management" })); + expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeVisible(); +}); + +test("uses closed choices for transport and rejects an invalid free-form port before save", async () => { + render(); + expect(screen.getByRole("combobox", { name: "DWH transport" })).toHaveTextContent("postgres_direct"); + await user.clear(screen.getByLabelText("DWH port")); + await user.type(screen.getByLabelText("DWH port"), "70000"); + await user.click(screen.getByRole("button", { name: "Save draft" })); + expect(screen.getByText("Port must be between 1 and 65535")).toBeVisible(); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/shell/WorkspaceManager.test.tsx src/shell/WorkspaceEditor.test.tsx` in `frontend/` +Expected: FAIL because the manager/editor components do not exist. + +- [ ] **Step 3: Implement the page and editor** + +Add a right-panel header button labelled `Workspace management` to `ModelActivityPanel`; `AppShell` opens the dedicated manager without interrupting active session streams. Implement a list/detail page with General, DWH, Semantic index, LLM policy, Installation requirements, and Git status/history sections. Use native/select component controls for every enum; use typed numeric/URL/text fields for free values. Show client validation immediately, server validation after `validate`, and diagnostics only as sanitized codes/messages. `New` generates an ID proposal, `Duplicate` requires a new immutable ID, `Delete` creates a deletion draft, and `Save draft` only writes browser storage. + +- [ ] **Step 4: Run focused tests and full frontend gates** + +Run: `npx vitest run src/shell/WorkspaceManager.test.tsx src/shell/WorkspaceEditor.test.tsx && npx vitest run && npx tsc -b` in `frontend/` +Expected: PASS; the active session and right-panel resize controls retain existing behavior. + +- [ ] **Step 5: Commit** + +```bash +git add frontend/src/shell/WorkspaceManager.tsx frontend/src/shell/WorkspaceEditor.tsx frontend/src/shell/WorkspaceManager.test.tsx frontend/src/shell/WorkspaceEditor.test.tsx frontend/src/shell/AppShell.tsx frontend/src/shell/ModelActivityPanel.tsx +git commit -m "feat: add workspace management editor" +``` + +## Task 10: Add publish, pull, conflict, import/export, and diagnostics user flows + +**Files:** +- Create: `frontend/src/shell/WorkspacePublishDialog.tsx` +- Create: `frontend/src/shell/WorkspacePublishDialog.test.tsx` +- Modify: `frontend/src/shell/WorkspaceManager.tsx` +- Modify: `frontend/src/shell/WorkspaceEditor.tsx` + +**Interfaces:** +- `WorkspacePublishDialog` consumes: + +```ts +interface WorkspaceConflict { + code: "workspace_conflict"; + base: CanonicalWorkspace; + local: CanonicalWorkspace; + remote: CanonicalWorkspace; + fields: string[]; +} +``` + +- Produces a publish request only after explicit confirmation. + +- [ ] **Step 1: Write failing publish-flow tests** + +```tsx +test("shows a field-level conflict and does not overwrite the remote workspace", async () => { + server.use(http.post("*/workspaces/publish", () => HttpResponse.json(conflict, { status: 409 }))); + render(); + await user.click(screen.getByRole("button", { name: "Publish" })); + expect(await screen.findByText("semantic_index.embedding.model")).toBeVisible(); + expect(screen.queryByText("Published")).not.toBeInTheDocument(); +}); + +test("imports a bundle as a local draft and never publishes it automatically", async () => { + render(); + await user.upload(screen.getByLabelText("Import workspace bundle"), bundleFile); + expect(await screen.findByText("Imported draft" )).toBeVisible(); + expect(publishSpy).not.toHaveBeenCalled(); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/shell/WorkspacePublishDialog.test.tsx` in `frontend/` +Expected: FAIL because the publish dialog and flows do not exist. + +- [ ] **Step 3: Implement collaboration and fallback controls** + +Display status (active commit, ahead/behind, degraded) and provide Pull before Publish. Render canonical field-level diffs and HTTP 409 base/local/remote comparisons; allow the user to choose remote or local value per conflicting field, then save a revised browser draft. Download export bundles through a Blob URL and revoke it. Upload imports as `FormData`, save the returned draft locally, and require normal validation/publish. Offer `Test on this installation` and render activatable/degraded diagnostics without secret details. + +- [ ] **Step 4: Run focused tests and full frontend gates** + +Run: `npx vitest run src/shell/WorkspacePublishDialog.test.tsx && npx vitest run && npx tsc -b` in `frontend/` +Expected: PASS; no automatic publish occurs on import or stale-draft detection. + +- [ ] **Step 5: Commit** + +```bash +git add frontend/src/shell/WorkspacePublishDialog.tsx frontend/src/shell/WorkspacePublishDialog.test.tsx frontend/src/shell/WorkspaceManager.tsx frontend/src/shell/WorkspaceEditor.tsx +git commit -m "feat: publish and synchronize workspace drafts" +``` + +## Task 11: Migrate existing workspace descriptors and deploy persistent registry storage + +**Files:** +- Create: `backend/src/workspaces/migrate-legacy.ts` +- Create: `backend/test/workspaces-migrate-legacy.test.ts` +- Modify: `compose.yaml` +- Modify: `docker-compose.dev.yml` +- Modify: `.env.example` +- Modify: `deploy/thothii.env.example` +- Create: `deploy/workspace-registry.env.example` +- Create: `scripts/workspace-registry-smoke.sh` + +**Interfaces:** +- Produces CLI entry point: + +```text +node dist/workspaces/migrate-legacy.js --input --output +``` + +- The smoke script accepts `WORKSPACE_GIT_REMOTE`, initializes an isolated Compose project, proves persistence, Git pull, and last-valid-snapshot fallback. + +- [ ] **Step 1: Write failing migration and Compose-contract tests** + +```ts +test("migrates the current local PSD descriptor without copying secret values", () => { + const result = migrateLegacyWorkspace(readFixture("local.yaml")); + expect(result.workspace.workspace.id).toBe("local"); + expect(JSON.stringify(result)).not.toMatch(/password:|api_key:/i); +}); +``` + +```sh +./scripts/workspace-registry-smoke.sh +# Expected before implementation: fail because no workspace registry volume/configuration exists. +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/workspaces-migrate-legacy.test.ts` in `backend/` +Run: `./scripts/workspace-registry-smoke.sh` from repository root +Expected: FAIL because the migration CLI and registry deployment contract do not exist. + +- [ ] **Step 3: Implement migration and container configuration** + +Translate current `harness/workspaces/*.yaml` and `deploy/workspaces/*.yaml` into canonical documents while replacing runtime secrets with binding requirements. Add `THT_WORKSPACE_REGISTRY_ROOT=/data/workspace-registry`, remote/branch/installation-ID variables, and an explicit persistent mount to server and local Compose files. Mount Git credentials, CA, SSH key, and known-hosts files read-only from installation secrets. Do not mount the canonical repository into the image. Ensure Compose examples distinguish server external networks from local loopback deployment. + +- [ ] **Step 4: Run migration, smoke, Docker build, and test gates** + +Run: `npx vitest run test/workspaces-migrate-legacy.test.ts && npx tsc --noEmit -p .` in `backend/` +Run: `./scripts/workspace-registry-smoke.sh` from repository root +Run: `docker compose config && docker compose -f docker-compose.dev.yml config` from repository root +Expected: PASS; a replaced core container retains its checkout and last valid snapshot. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/workspaces/migrate-legacy.ts backend/test/workspaces-migrate-legacy.test.ts compose.yaml docker-compose.dev.yml .env.example deploy/thothii.env.example deploy/workspace-registry.env.example scripts/workspace-registry-smoke.sh +git commit -m "feat: deploy portable workspace registry" +``` + +## Task 12: Produce detailed local and server installation manuals and verify them + +**Files:** +- Create: `docs/install/local-workspace-registry.md` +- Create: `docs/install/server-workspace-registry.md` +- Create: `docs/install/examples/local-compose.workspace-registry.yaml` +- Create: `docs/install/examples/server-compose.workspace-registry.yaml` +- Create: `scripts/verify-workspace-install-docs.sh` +- Modify: `README.md` +- Test: `scripts/workspace-registry-smoke.sh` + +**Interfaces:** +- The manual verifier accepts: + +```text +./scripts/verify-workspace-install-docs.sh --profile local +./scripts/verify-workspace-install-docs.sh --profile server +``` + +- It extracts only marked fenced commands from the corresponding manual, validates Compose, and runs bootstrap/recovery smoke fixtures without contacting production services. + +- [ ] **Step 1: Write failing documentation-verification tests** + +```sh +./scripts/verify-workspace-install-docs.sh --profile local +# Expected before implementation: fail because the local manual and runnable example do not exist. + +./scripts/verify-workspace-install-docs.sh --profile server +# Expected before implementation: fail because the server manual and runnable example do not exist. +``` + +- [ ] **Step 2: Run commands to verify they fail** + +Run: `./scripts/verify-workspace-install-docs.sh --profile local` from repository root +Run: `./scripts/verify-workspace-install-docs.sh --profile server` from repository root +Expected: both FAIL with a missing-manual error. + +- [ ] **Step 3: Write complete manuals and verifier** + +Write the local PC/Mac manual with Docker Desktop/local-engine prerequisites, clone or remote bootstrap, Git SSH/HTTPS setup, persistent volume, local binding file, secret file permissions, direct/REST/SSH examples, startup, first pull, diagnostics, publish, update, backup, remote-outage recovery, and rollback. Write the server manual with service account ownership, persistent bind/volume layout, Gitea/remote setup, outbound firewall requirements, CA/known-hosts/secret mounts, same-origin reverse proxy, startup, health/status, pull/publish, upgrade, backup, degraded recovery, and snapshot rollback. In both manuals explicitly separate Git-shared values from installation-local variables and secret files, document all stable error codes, and include runnable marked Compose examples. Implement a shell verifier that checks required headings/commands, runs Compose config, runs the isolated smoke script, and rejects examples containing secret literals. + +- [ ] **Step 4: Run manual verification and all final gates** + +Run: `./scripts/verify-workspace-install-docs.sh --profile local && ./scripts/verify-workspace-install-docs.sh --profile server` from repository root +Run: `npx vitest run && npx tsc --noEmit -p .` in `backend/` +Run: `npx vitest run && npx tsc -b` in `frontend/` +Run: `.venv/bin/pytest -q` in `harness/` +Expected: PASS; manuals are complete, runnable against fixtures, and contain no credential values. + +- [ ] **Step 5: Commit** + +```bash +git add docs/install/local-workspace-registry.md docs/install/server-workspace-registry.md docs/install/examples/local-compose.workspace-registry.yaml docs/install/examples/server-compose.workspace-registry.yaml scripts/verify-workspace-install-docs.sh README.md +git commit -m "docs: add workspace registry installation manuals" +``` + +## Task 13: Final migration rehearsal, end-to-end regression, and release verification + +**Files:** +- Modify: `PROJECT_STATE.md` +- Modify: `README.md` +- Test: `backend/test/routes-workspaces.test.ts` +- Test: `backend/test/routes-sessions.test.ts` +- Test: `frontend/src/shell/WorkspaceManager.test.tsx` +- Test: `harness/tests/test_session_documents.py` + +**Interfaces:** +- Verifies the public contract produced by Tasks 1–12; no new production interface is introduced. + +- [ ] **Step 1: Write failing cross-layer regression tests** + +```ts +test("a session created before a workspace pull resumes from its original snapshot", async () => { + const created = await createSessionAtRevision("a".repeat(40)); + await publishWorkspaceRevision("b".repeat(40)); + await resumeSession(created.id); + expect(runner.reopenSession).toHaveBeenCalledWith(created.id, expect.stringContaining(`/snapshots/${"a".repeat(40)}/`)); +}); +``` + +```tsx +test("a local installation can pull a Git workspace, configure bindings, validate it, and create a revision-pinned session", async () => { + let created: unknown; + server.use( + http.post("*/workspace-registry/pull", () => HttpResponse.json({ head: "a".repeat(40), degraded: false })), + http.post("*/workspaces/psd-clinical/test", () => HttpResponse.json({ activatable: true, diagnostics: [] })), + http.post("*/sessions", async ({ request }) => { + created = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + render(); + await user.click(await screen.findByRole("button", { name: "Pull" })); + await user.click(screen.getByRole("button", { name: "Test on this installation" })); + expect(await screen.findByText("This installation can activate this workspace")).toBeVisible(); + await createSession({ question: "count patients", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" }); + expect(created).toMatchObject({ workspaceId: "psd-clinical", model: "glm-5.2" }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run test/routes-sessions.test.ts test/routes-workspaces.test.ts` in `backend/` +Run: `npx vitest run src/shell/WorkspaceManager.test.tsx` in `frontend/` +Expected: FAIL until snapshot retention and the full UI/API flow are connected. + +- [ ] **Step 3: Implement retention, regression fixes, and operator state** + +Add snapshot-retention logic that preserves revisions referenced by resumable manifests. Repair only issues revealed by the cross-layer tests. Record active remote/branch, migration state, manual locations, and tested commands in `PROJECT_STATE.md`; add README links to the two manuals and the workspace registry operator workflow. + +- [ ] **Step 4: Run complete verification** + +Run: `git diff --check` from repository root +Run: `npx vitest run && npx tsc --noEmit -p .` in `backend/` +Run: `npx vitest run && npx tsc -b` in `frontend/` +Run: `.venv/bin/pytest -q` in `harness/` +Run: `./scripts/workspace-registry-smoke.sh && ./scripts/verify-workspace-install-docs.sh --profile local && ./scripts/verify-workspace-install-docs.sh --profile server` from repository root +Expected: every command exits 0; server/local deployment examples, Git fallback, workspace conflict handling, semantic-index validation, and session revision pinning are covered. + +- [ ] **Step 5: Commit** + +```bash +git add PROJECT_STATE.md README.md backend/test/routes-sessions.test.ts backend/test/routes-workspaces.test.ts frontend/src/shell/WorkspaceManager.test.tsx harness/tests/test_session_documents.py +git commit -m "test: verify portable workspace registry end to end" +``` + +## Plan Self-Review + +### Spec coverage + +- Git source of truth, generic remote support, server/local persistent checkout, snapshots, offline bundles, conflicts, and Git failure behavior are covered by Tasks 1, 4, 6, 10, and 11. +- Canonical workspace YAML, deterministic secret-variable contracts, generic DWH/vector/embedding/LLM model, and semantic-index invariants are covered by Tasks 2 and 3. +- Direct, REST, and SSH connectivity checks are covered by Task 5. +- Browser-local preferences/drafts and no-auth behavior are covered by Task 8. +- The right-sidebar CRUD, closed lists, free fields, three validation levels, diagnostics, and delete behavior are covered by Tasks 6, 9, and 10. +- Session revision pinning and legacy compatibility are covered by Task 7 and verified by Task 13. +- Docker server/local wiring, migration, detailed manuals, runnable examples, and documentation verification are covered by Tasks 11 and 12. + +### Placeholder scan + +The scan found no placeholder markers or vague test instructions. + +### Type consistency + +`CanonicalWorkspace`, `InstallationContract`, `WorkspaceRevision`, `WorkspaceDraft`, `WorkspaceErrorCode`, and `PublishWorkspaceRequest` are introduced before later tasks consume them. Snapshot paths are provided by `WorkspaceRegistry`, and `ThtRunner` only receives an absolute rendered snapshot config path. diff --git a/docs/superpowers/plans/2026-08-04-unified-compose-deployment.md b/docs/superpowers/plans/2026-08-04-unified-compose-deployment.md new file mode 100644 index 00000000..06c87147 --- /dev/null +++ b/docs/superpowers/plans/2026-08-04-unified-compose-deployment.md @@ -0,0 +1,687 @@ +# Unified Docker Compose Deployment Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Convert ThothII into one autonomous Docker Compose distribution for Windows, macOS, and Linux servers, with external configurable data/AI services, embedded Pi, guided Pi management, reproducible local builds, and deterministic line endings. + +**Architecture:** `compose.yaml` is the sole complete stack definition; local and server files are small overrides. The mandatory stack contains only `frontend` and `core`; DWH, VectorDB, embedding, and LLM remain independently operated endpoints even when co-resident. Pi is pinned inside `core`; a Go `thothctl` executable wraps host-side Compose operations, while a web Pi Management page handles safe configuration and diagnostics. + +**Tech Stack:** Docker BuildKit/Compose v2, Node.js 22, TypeScript/Fastify, React/Vite, nginx-unprivileged, Python 3.12 harness, Go 1.24 for `thothctl`, Vitest, Playwright, shell/PowerShell verification. + +**Design:** `docs/superpowers/specs/2026-08-04-unified-compose-deployment-design.md` + +## Global Constraints + +- ThothII has no active build/runtime dependency on PSD, Chirone, or `omics_portal`. +- The mandatory stack contains only `frontend` and `core`; it never starts DWH, VectorDB, embedding, LLM, or a reverse proxy. +- External services use installation/workspace addresses even when they run on the Docker host. +- Pi is pinned inside `core`; no host Pi or host Node/Python/Go runtime is required. +- `core` never mounts a Docker daemon endpoint. +- Secrets remain file-mounted under `/run/secrets` and never enter Git, images, browser storage, rendered Compose, or logs. +- Local ports bind to `127.0.0.1`; server deployment publishes only the frontend port. +- Shell, YAML, Dockerfile, JSON, TypeScript, Python, and Markdown use LF; PowerShell uses CRLF. +- Every task follows red-green TDD and ends with a reviewable commit. + +## Target file map + +- `.gitattributes`, `.editorconfig`, `scripts/verify-line-endings.sh`: line-ending contract. +- `compose.yaml`, `deploy/compose.local.yaml`, `deploy/compose.server.yaml`: portable stack and profiles. +- `deploy/compose.git-*.yaml`, `deploy/compose.connector-secrets.yaml`: optional secret mounts. +- `.env.example`, `deploy/env/*.env.example`: non-secret operator contracts. +- `docker/core.Dockerfile`, `docker/frontend.Dockerfile`, `docker/nginx.conf.template`: reproducible images and same-origin routing. +- `tools/thothctl/`: cross-platform host control executable. +- `backend/src/pi/management.ts`, `backend/src/routes/pi-management.ts`: sanitized Pi APIs. +- `frontend/src/api/pi-management.ts`, `frontend/src/shell/PiManagement.tsx`: Pi operator UI. +- `docs/install/local.md`, `docs/install/server.md`, `docs/install/pi-management.md`: installation manuals. + +--- + +### Task 1: Enforce deterministic line endings + +**Files:** +- Create: `.gitattributes` +- Create: `.editorconfig` +- Create: `scripts/verify-line-endings.sh` +- Create: `scripts/test-verify-line-endings.sh` +- Modify: `docker/core.Dockerfile` +- Test: `scripts/test-verify-line-endings.sh` + +**Interfaces:** +- Produces: `scripts/verify-line-endings.sh [root]`, exit `0` when compliant and `1` with offending paths for CRLF. +- Consumes: tracked files at repository root or an explicit fixture root; never scans volumes or secrets. + +- [ ] **Step 1: Write the failing verifier test** + +Create a temporary fixture with LF `ok.sh` and CRLF `bad.sh`, `compose.yaml`, and `Dockerfile`. Assert all bad paths are reported, `ok.sh` is absent, and LF-only input exits `0`. + +```sh +fixture_root="$(mktemp -d)" +trap 'rm -rf "$fixture_root"' EXIT +printf '#!/bin/sh\r\nexit 0\r\n' > "$fixture_root/bad.sh" +if ./scripts/verify-line-endings.sh "$fixture_root"; then + echo "expected CRLF rejection" >&2 + exit 1 +fi +``` + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/test-verify-line-endings.sh` + +Expected: failure because the verifier does not exist. + +- [ ] **Step 3: Add attributes, editor settings, and verifier** + +Use this `.gitattributes` contract: + +```gitattributes +* text=auto +*.sh text eol=lf +Dockerfile* text eol=lf +*.Dockerfile text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.py text eol=lf +*.md text eol=lf +*.ps1 text eol=crlf +``` + +Use `git ls-files` for the repository and `find` only for fixture mode. Detect carriage returns with `LC_ALL=C grep -Il $'\r'`. Add an image-build check before `chmod` of Docker scripts. + +- [ ] **Step 4: Renormalize and verify** + +Run: + +```sh +git add --renormalize . +bash scripts/test-verify-line-endings.sh +bash scripts/verify-line-endings.sh +git diff --check +``` + +Expected: both scripts pass; review renormalized files to confirm line-ending-only changes. + +- [ ] **Step 5: Commit** + +```sh +git add .gitattributes .editorconfig scripts/verify-line-endings.sh scripts/test-verify-line-endings.sh docker/core.Dockerfile +git commit -m "build: enforce portable line endings" +``` + +### Task 2: Define the portable Compose contract + +**Files:** +- Modify: `compose.yaml` +- Modify: `deploy/compose.local.yaml` +- Create: `deploy/compose.server.yaml` +- Create: `deploy/env/local.env.example` +- Create: `deploy/env/server.env.example` +- Create: `.env.example` +- Modify: `scripts/test-default-compose.sh` +- Create: `scripts/test-unified-compose.sh` + +**Interfaces:** +- Produces: base services `core` and `frontend`, network `thothii`, and volumes `settings`, `pi-state`, `workspace-registry`, `sessions`. +- Produces: supported pairs base+local and base+server. + +- [ ] **Step 1: Write failing structural assertions** + +Render both profiles as JSON and assert: + +```js +const services = Object.keys(config.services).sort(); +if (services.join(",") !== "core,frontend") throw new Error("mandatory stack must be core,frontend"); +if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) { + throw new Error("forbidden application coupling"); +} +``` + +Assert local publishes loopback frontend and optional loopback core; server publishes frontend only. + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/test-unified-compose.sh` + +Expected: failure on current portal networks and host paths. + +- [ ] **Step 3: Replace root Compose with the portable base** + +Define only `core`, `frontend`, private network, health checks, and named volumes. `depends_on` may connect frontend to healthy core but never external services. Keep endpoint variables generic and secret-free. + +- [ ] **Step 4: Add local/server overrides** + +Local: `AUTH_MODE=none`, loopback ports, named volumes, installation ID `local`. Server: configurable frontend bind, `AUTH_MODE=upstream`, no core host port, `THT_DATA_ROOT` mounts, installation ID `server`. + +- [ ] **Step 5: Add environment examples** + +Use documentation domains such as `https://dwh.example.invalid`. Assert absent `THT_WORKSPACE_GIT_REMOTE` fails rendering with that exact variable name. + +- [ ] **Step 6: Verify and commit** + +```sh +bash scripts/test-default-compose.sh +bash scripts/test-unified-compose.sh +docker compose -f compose.yaml -f deploy/compose.local.yaml config --quiet +docker compose -f compose.yaml -f deploy/compose.server.yaml config --quiet +git add compose.yaml deploy/compose.local.yaml deploy/compose.server.yaml deploy/env .env.example scripts/test-default-compose.sh scripts/test-unified-compose.sh +git commit -m "deploy: unify local and server compose stack" +``` + +### Task 3: Isolate Git and connector secrets + +**Files:** +- Create: `deploy/compose.git-ssh.yaml` +- Create: `deploy/compose.git-https.yaml` +- Create: `deploy/compose.connector-secrets.yaml` +- Modify: `deploy/workspace-registry.env.example` +- Create: `scripts/test-compose-secret-policy.sh` +- Modify: `scripts/verify-workspace-install-docs.sh` + +**Interfaces:** +- Produces: mutually exclusive Git overrides and explicit connector targets under `/run/secrets`. +- Consumes: `THT_WS_*_FILE` and host-only `*_SOURCE` variables. + +- [ ] **Step 1: Write failing rendered-secret tests** + +Assert base has no `/dev/null` mounts; SSH mounts only key/known-hosts; HTTPS mounts only credentials/CA; connector mounts match declared `_FILE` targets; rendered output never contains fixture secret values. + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/test-compose-secret-policy.sh` + +Expected: failure because secret mounts currently live in the portal-oriented base. + +- [ ] **Step 3: Implement overrides** + +Use read-only mounts and `${VAR:?message}` only in selected overrides. Keep strict SSH host checking and HTTPS CA verification. Reject relative/non-normalized `_SOURCE` paths. + +- [ ] **Step 4: Verify and commit** + +```sh +bash scripts/test-compose-secret-policy.sh +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/workspace-registry-smoke.sh +git add deploy/compose.git-ssh.yaml deploy/compose.git-https.yaml deploy/compose.connector-secrets.yaml deploy/workspace-registry.env.example scripts/test-compose-secret-policy.sh scripts/verify-workspace-install-docs.sh +git commit -m "deploy: isolate git and connector secrets" +``` + +### Task 4: Make frontend-to-core routing same-origin + +**Files:** +- Modify: `docker/nginx.conf.template` +- Modify: `docker/frontend-entrypoint.sh` +- Modify: `docker/frontend.Dockerfile` +- Modify: `frontend/src/api/runtime-config.ts` +- Test: `frontend/src/api/runtime-config.test.ts` +- Modify: `docker/smoke/frontend-policy-smoke.sh` +- Modify: `scripts/test-backend-url-policy.sh` + +**Interfaces:** +- Produces: browser base `/api`; nginx proxies to `http://core:8787` privately. +- Consumes: optional internal `THT_FRONTEND_API_UPSTREAM` only. + +- [ ] **Step 1: Write failing routing tests** + +Assert `/api` default, rejection of browser-facing absolute production URLs, and nginx SSE settings: + +```nginx +proxy_http_version 1.1; +proxy_buffering off; +proxy_read_timeout 3600s; +``` + +- [ ] **Step 2: Confirm red** + +```sh +npm --prefix frontend test -- --run src/api/runtime-config.test.ts +bash scripts/test-backend-url-policy.sh +``` + +Expected: failure because deployment-specific build arguments remain. + +- [ ] **Step 3: Implement runtime routing and blocking frontend build** + +Build once with `/` assets and `/api`. Render private upstream at startup, strip `/api`, preserve SSE. Replace non-blocking typecheck with `RUN npm run build`. Remove `/datamart-builder` assumptions. + +- [ ] **Step 4: Verify and commit** + +```sh +npm --prefix frontend test -- --run src/api/runtime-config.test.ts +npm --prefix frontend run build +bash scripts/test-backend-url-policy.sh +bash docker/smoke/frontend-policy-smoke.sh +git add docker/nginx.conf.template docker/frontend-entrypoint.sh docker/frontend.Dockerfile docker/smoke/frontend-policy-smoke.sh frontend/src/api/runtime-config.ts frontend/src/api/runtime-config.test.ts scripts/test-backend-url-policy.sh +git commit -m "deploy: route frontend and core through one origin" +``` + +### Task 5: Harden local image builds and embedded Pi + +**Files:** +- Modify: `docker/core.Dockerfile` +- Modify: `docker/pi-runtime/package.json` +- Modify: `docker/pi-runtime/package-lock.json` +- Create: `.dockerignore` +- Modify: `docker/smoke/core-smoke.sh` +- Modify: `scripts/verify-container-images.sh` +- Create: `scripts/build-local.sh` +- Create: `scripts/build-local.ps1` +- Test: `scripts/test-container-deployment.sh` + +**Interfaces:** +- Produces: `core` containing the pinned `/usr/local/bin/pi` and a standalone `frontend` image. +- Produces: local build launchers requiring only Docker, Compose, and Git. + +- [ ] **Step 1: Write failing image-contract assertions** + +Inside `core`, assert `pi --version` matches `PI_VERSION`, UID is `10001`, Docker socket is absent, `/data` is writable, and no portal/Chirone path exists. Assert frontend health and `/api` routing. + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/test-container-deployment.sh` + +Expected: failure on at least one old deployment contract. + +- [ ] **Step 3: Pin Pi from its lock input** + +Make `docker/pi-runtime/package-lock.json` the sole Pi dependency lock. Install with `npm ci --omit=dev` in a build stage, copy into `core`, and fail build when `pi --version` differs from `PI_VERSION`. + +- [ ] **Step 4: Add exclusions and platform launchers** + +Exclude `.git`, `.worktrees`, `.env`, secrets, dependencies, virtual environments, coverage, and runtime data. Both launchers run: + +```text +docker compose -f compose.yaml -f deploy/compose.local.yaml build --pull +``` + +They print the same next command and preserve Docker's exit code. + +- [ ] **Step 5: Verify and commit** + +```sh +bash scripts/build-local.sh +bash scripts/test-container-deployment.sh +bash scripts/verify-container-images.sh +git add .dockerignore docker/core.Dockerfile docker/pi-runtime docker/smoke/core-smoke.sh scripts/build-local.sh scripts/build-local.ps1 scripts/test-container-deployment.sh scripts/verify-container-images.sh +git commit -m "build: make embedded pi images reproducible" +``` + +Windows gate: `powershell -ExecutionPolicy Bypass -File scripts/build-local.ps1`. + +### Task 6: Build the cross-platform `thothctl` foundation + +**Files:** +- Create: `tools/thothctl/go.mod` +- Create: `tools/thothctl/cmd/thothctl/main.go` +- Create: `tools/thothctl/internal/compose/runner.go` +- Create: `tools/thothctl/internal/config/installation.go` +- Create: `tools/thothctl/internal/output/sanitize.go` +- Test: `tools/thothctl/internal/compose/runner_test.go` +- Test: `tools/thothctl/internal/config/installation_test.go` +- Test: `tools/thothctl/internal/output/sanitize_test.go` +- Create: `docker/thothctl.Dockerfile` +- Create: `scripts/build-thothctl.sh` + +**Interfaces:** +- Produces: `thothctl --installation ` binaries for Windows amd64, macOS amd64/arm64, Linux amd64/arm64. +- Produces: `Runner.Run(ctx, args, stdin) (Result, error)` using argument arrays, never shell concatenation. + +- [ ] **Step 1: Write failing tests** + +Cover local/server Compose selection, paths containing spaces, missing Docker, propagated exit codes, and replacement of values matching password/token/key fields or secret-file contents with `[REDACTED]`. + +- [ ] **Step 2: Confirm red** + +Run: `docker run --rm -v "$PWD:/src" -w /src/tools/thothctl golang:1.24 go test ./...` + +Expected: failure because packages do not exist. + +- [ ] **Step 3: Implement installation discovery and safe runner** + +Read `thothii-installation.yaml` fields `profile`, `projectDirectory`, `envFile`, and `overrides`. Resolve and validate absolute paths. Invoke `docker compose` with `exec.CommandContext` argument slices. + +- [ ] **Step 4: Implement base commands** + +Add `status`, `doctor`, `logs`, `start`, `stop`, and `update --check-only`. `doctor` validates Docker/Compose versions, rendered config, LF, volumes, and frontend/core health without printing environment values. + +- [ ] **Step 5: Cross-compile with Docker and verify** + +Produce `dist/thothctl/thothctl-windows-amd64.exe`, Darwin amd64/arm64, and Linux amd64/arm64 from `docker/thothctl.Dockerfile`. + +```sh +docker run --rm -v "$PWD:/src" -w /src/tools/thothctl golang:1.24 go test ./... +bash scripts/build-thothctl.sh +``` + +Run the host-matching binary with `--help`, then commit: + +```sh +git add tools/thothctl docker/thothctl.Dockerfile scripts/build-thothctl.sh +git commit -m "feat: add cross-platform thothctl" +``` + +### Task 7: Implement safe Pi lifecycle commands in `thothctl` + +**Files:** +- Create: `tools/thothctl/internal/pi/commands.go` +- Create: `tools/thothctl/internal/pi/update.go` +- Create: `tools/thothctl/internal/pi/state.go` +- Test: `tools/thothctl/internal/pi/commands_test.go` +- Test: `tools/thothctl/internal/pi/update_test.go` +- Modify: `tools/thothctl/cmd/thothctl/main.go` +- Create: `docs/contracts/tht-pi.md` + +**Interfaces:** +- Produces: `pi status|doctor|configure|test|update|logs`. +- Produces: `.thothctl/update-state.json` with previous/new image references and phase, never credentials. +- Consumes: existing `/health`, `/models`, and `/settings` APIs plus `docker compose exec core pi --version`; Task 8 replaces the temporary composite checks with the dedicated Pi Management API. + +- [ ] **Step 1: Write failing update-state tests** + +With a fake Compose runner cover success and failure during build/pull, recreate, health, version, and smoke. Assert every post-recreate failure restores the prior image and leaves volume names unchanged. + +- [ ] **Step 2: Confirm red** + +Run: `docker run --rm -v "$PWD:/src" -w /src/tools/thothctl golang:1.24 go test ./internal/pi -v` + +Expected: failure because Pi commands are absent. + +- [ ] **Step 3: Implement read-only commands** + +`status` runs `core pi --version`; `doctor` verifies image version, writable Pi volume, provider configuration presence, and `/health`; `test` combines `/models`, `/settings`, and a fixed `core pi --version` probe until Task 8 supplies the dedicated smoke endpoint; `logs` uses the shared sanitizer. + +- [ ] **Step 4: Implement guided configuration** + +Offer provider/model/reasoning choices returned by the backend. Atomically write non-secret defaults. For credentials print the expected secret filename and permissions; never accept secret text as an argument. + +- [ ] **Step 5: Implement transactional update** + +Record current image ID, pull/build requested pinned version, recreate only `core`, verify health/version/test, and roll back on failure. Refuse update while sessions are active unless `--drain` completes. + +- [ ] **Step 6: Verify and commit** + +```sh +docker run --rm -v "$PWD:/src" -w /src/tools/thothctl golang:1.24 go test ./internal/pi -v +docker run --rm -v "$PWD:/src" -w /src/tools/thothctl golang:1.24 go test ./... +git add tools/thothctl docs/contracts/tht-pi.md +git commit -m "feat: manage embedded pi with thothctl" +``` + +### Task 8: Add sanitized Pi Management backend APIs + +**Files:** +- Create: `backend/src/pi/management.ts` +- Create: `backend/src/routes/pi-management.ts` +- Modify: `backend/src/app.ts` +- Modify: `backend/src/config.ts` +- Modify: `tools/thothctl/internal/pi/commands.go` +- Test: `backend/test/pi-management.test.ts` +- Test: `backend/test/routes-pi-management.test.ts` +- Test: `tools/thothctl/internal/pi/commands_test.go` + +**Interfaces:** +- Produces: `GET /pi-management/status`, `GET /pi-management/options`, `PUT /pi-management/config`, `POST /pi-management/test`, `GET /pi-management/logs`. +- Produces: sanitized `PiStatus`, `PiOptions`, `PiInstallationConfig`, and stable errors. + +- [ ] **Step 1: Write failing service/route tests** + +Mock process execution and settings. Assert version parsing, closed choices, free-field validation, atomic writes, smoke timeout, 200-line log limit, redaction, and `403 pi_management_forbidden` in exposed server mode without trusted admin identity. + +- [ ] **Step 2: Confirm red** + +Run: `cd backend && npx vitest run test/pi-management.test.ts test/routes-pi-management.test.ts` + +Expected: module-not-found failure. + +- [ ] **Step 3: Implement service and authorization** + +Use `execFile` with fixed argument arrays. Return only version, readiness, provider names, model IDs, reasoning choices, timestamps, and sanitized messages. Allow `AUTH_MODE=none` only when public exposure is false; upstream mode requires the documented admin claim. Expose no image-update API. + +- [ ] **Step 4: Switch `thothctl` to the dedicated API** + +Replace the Task 7 composite `/health`/`/models`/`/settings` test with `POST /pi-management/test`; load closed configuration choices from `GET /pi-management/options`. Preserve the direct in-container `pi --version` check as an independent image-integrity signal. + +- [ ] **Step 5: Verify and commit** + +```sh +cd backend +npx vitest run test/pi-management.test.ts test/routes-pi-management.test.ts +npx vitest run +npx tsc --noEmit -p . +cd .. +docker run --rm -v "$PWD:/src" -w /src/tools/thothctl golang:1.24 go test ./internal/pi -v +git add backend/src/pi/management.ts backend/src/routes/pi-management.ts backend/src/app.ts backend/src/config.ts backend/test/pi-management.test.ts backend/test/routes-pi-management.test.ts tools/thothctl/internal/pi/commands.go tools/thothctl/internal/pi/commands_test.go +git commit -m "feat: expose safe pi management api" +``` + +### Task 9: Add the Pi Management interface + +**Files:** +- Create: `frontend/src/api/pi-management.ts` +- Create: `frontend/src/api/pi-management.test.ts` +- Create: `frontend/src/shell/PiManagement.tsx` +- Create: `frontend/src/shell/PiManagement.test.tsx` +- Modify: `frontend/src/shell/AppShell.tsx` +- Modify: `frontend/src/shell/AppShell.session-mgmt.test.tsx` + +**Interfaces:** +- Consumes: Task 8 Pi APIs. +- Produces: right-sidebar Pi Management panel without image-update execution or browser terminal. + +- [ ] **Step 1: Write failing UI tests** + +Cover loading/version, closed selects, validation, save, smoke test, sanitized logs, forbidden state, and copyable `thothctl pi update` instruction. Assert no secret input or browser terminal. + +- [ ] **Step 2: Confirm red** + +Run: `cd frontend && npx vitest run src/shell/PiManagement.test.tsx src/api/pi-management.test.ts` + +Expected: module-not-found failure. + +- [ ] **Step 3: Implement API and panel** + +Use query keys `['pi-management','status']` and `['pi-management','options']`. Render provider/model/reasoning as closed choices, validate numeric fields before PUT, and show credentials only as present/missing. + +- [ ] **Step 4: Attach to the right sidebar** + +Add Pi next to Workspace Management, preserve current session/activity panels and accessibility, and remove AppShell comments/layout assumptions about Omics Portal chrome. + +- [ ] **Step 5: Verify and commit** + +```sh +cd frontend +npx vitest run src/shell/PiManagement.test.tsx src/api/pi-management.test.ts src/shell/AppShell.session-mgmt.test.tsx +npx vitest run +npx tsc -b +git add src/api/pi-management.ts src/api/pi-management.test.ts src/shell/PiManagement.tsx src/shell/PiManagement.test.tsx src/shell/AppShell.tsx src/shell/AppShell.session-mgmt.test.tsx +git commit -m "feat: add pi management interface" +``` + +### Task 10: Remove active PSD and portal deployment coupling + +**Files:** +- Delete: `deploy/compose.production.yaml` +- Delete: `deploy/compose.psd-local.yaml.example` +- Modify: `README.md` +- Modify: `PROJECT_STATE.md` +- Modify: `scripts/run-stack.sh` +- Create: `scripts/test-no-deployment-coupling.sh` +- Modify: `scripts/test-external-compose-lifecycle.sh` + +**Interfaces:** +- Produces: active deployment files free of PSD/Chirone/portal networks, paths, and prefixes. +- Preserves: generic external endpoint support and historical design documents. + +- [ ] **Step 1: Write the failing coupling scan** + +Scan active Compose, Docker, root README, install guides, env examples, and run scripts for `omics_portal`, `/home/chirone`, `localllm_default`, `datamart-builder`, and PSD deployment filenames. Exclude historical specs/plans and canonical workspace content. + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/test-no-deployment-coupling.sh` + +Expected: failure listing current portal-oriented files. + +- [ ] **Step 3: Remove superseded files and genericize launch behavior** + +Delete only deployment files replaced by Tasks 2–5. Make `run-stack.sh` call base+local or direct users to `thothctl start`. Preserve data migration utilities that do not affect runtime coupling. + +- [ ] **Step 4: Update root documentation and verify** + +State that co-location never puts DWH/vector/embedding inside ThothII. + +```sh +bash scripts/test-no-deployment-coupling.sh +bash scripts/test-unified-compose.sh +bash scripts/test-external-compose-lifecycle.sh +git add -A deploy/compose.production.yaml deploy/compose.psd-local.yaml.example README.md PROJECT_STATE.md scripts/run-stack.sh scripts/test-no-deployment-coupling.sh scripts/test-external-compose-lifecycle.sh +git commit -m "refactor: remove portal deployment coupling" +``` + +### Task 11: Write and verify the PC/Mac installation manual + +**Files:** +- Create: `docs/install/local.md` +- Create: `docs/install/windows-line-endings.md` +- Create: `docs/install/pi-management.md` +- Create: `docs/install/examples/thothii-installation.local.yaml` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/test-verify-workspace-install-docs.sh` + +**Interfaces:** +- Produces: complete clean-install, build, update, backup, restore, Pi, and CRLF instructions for Windows/WSL2, macOS, and Linux PC. + +- [ ] **Step 1: Extend the verifier first** + +Require prerequisites, Git clone, LF verification, `.env`, external-service addressing, build, start, health, browser URL, `thothctl`, Pi update/rollback, backup/restore, pull update, and data-preserving uninstall. Require `pi-management.md` to cover UI permissions, every `thothctl pi` command, secret handling, direct support access, and failed-update recovery. Copy examples to a temporary path containing spaces and render there. + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/test-verify-workspace-install-docs.sh` + +Expected: failure naming missing local-guide sections. + +- [ ] **Step 3: Write platform-specific instructions** + +Document macOS, Windows PowerShell, Windows WSL2, and Linux separately. Recommend cloning inside the WSL filesystem. Include LF checks after clone/pull. For an existing CRLF clone, prefer recloning; describe repository-local `core.autocrlf=false` and renormalization. If mentioning `git reset --hard`, require explicit backup/commit and a destructive-action warning immediately before it. + +- [ ] **Step 4: Document external services on the host** + +Show `host.docker.internal` for Docker Desktop and `extra_hosts: host.docker.internal:host-gateway` for Linux. Explain that container `127.0.0.1` is not the host. All addresses remain configurable. + +- [ ] **Step 5: Verify and commit** + +```sh +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/verify-workspace-install-docs.sh --profile local +bash scripts/docker-smoke.sh +git add docs/install/local.md docs/install/windows-line-endings.md docs/install/pi-management.md docs/install/examples/thothii-installation.local.yaml docs/install/local-workspace-registry.md scripts/verify-workspace-install-docs.sh scripts/test-verify-workspace-install-docs.sh +git commit -m "docs: add autonomous local installation guide" +``` + +### Task 12: Write and verify the server installation manual + +**Files:** +- Create: `docs/install/server.md` +- Create: `docs/install/reverse-proxy-nginx.md` +- Create: `docs/install/reverse-proxy-caddy.md` +- Create: `docs/install/examples/thothii-installation.server.yaml` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Test: `scripts/test-verify-workspace-install-docs.sh` + +**Interfaces:** +- Produces: generic Linux server deployment independent of another application's network. +- Consumes: server profile, secret overrides, `thothctl`, and same-origin frontend. + +- [ ] **Step 1: Add failing server-guide assertions** + +Require service account/UID, directories, firewall, co-resident external endpoints, DNS/host-gateway choices, generic TLS proxy, upstream auth, local build or pinned images, startup, readiness, Pi management, drain, rollback, backup, restore, and diagnostics. + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/test-verify-workspace-install-docs.sh` + +Expected: failure naming missing server sections. + +- [ ] **Step 3: Write the server and proxy guides** + +Use `/srv/thothii` only as an example operator root. State that DWH/vector/embedding on the same physical machine remain independently addressed services. Nginx/Caddy proxy only to frontend, preserve SSE, terminate TLS, and forward identity only after authentication. + +- [ ] **Step 4: Verify and commit** + +```sh +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/verify-workspace-install-docs.sh --profile server +bash scripts/test-unified-compose.sh +git add docs/install/server.md docs/install/reverse-proxy-nginx.md docs/install/reverse-proxy-caddy.md docs/install/examples/thothii-installation.server.yaml docs/install/server-workspace-registry.md scripts/verify-workspace-install-docs.sh scripts/test-verify-workspace-install-docs.sh +git commit -m "docs: add autonomous server installation guide" +``` + +### Task 13: Add end-to-end deployment and update gates + +**Files:** +- Create: `scripts/unified-deployment-smoke.sh` +- Create: `scripts/thothctl-update-smoke.sh` +- Create: `scripts/test-windows-clone-contract.ps1` +- Create: `.github/workflows/deployment.yml` +- Modify: `PROJECT_STATE.md` +- Modify: `README.md` + +**Interfaces:** +- Produces: release gate for rendering, image build, embedded Pi, registry persistence, offline recovery, and update rollback. + +- [ ] **Step 1: Write failing orchestration smoke** + +Create an isolated Compose project and bare Git registry, build, start local, verify frontend/core/Pi/registry, recreate offline, perform a valid Git update, and prove volumes survive. Inject a bad Pi image/version and prove rollback. + +- [ ] **Step 2: Confirm red** + +Run: `bash scripts/unified-deployment-smoke.sh` + +Expected: failure because the script is absent. + +- [ ] **Step 3: Implement exact-resource cleanup** + +Generate a unique project name and temporary directory, label resources, and remove only those exact resources on exit. Never prune global Docker state. Sanitize captured logs. + +- [ ] **Step 4: Add Windows and CI gates** + +PowerShell scans tracked shell/YAML/Docker files for byte `0x0D`, renders Compose, and invokes Windows `thothctl`. CI runs LF and TypeScript gates everywhere, Docker smoke on Linux, and clone/Compose contract on Windows. + +- [ ] **Step 5: Run final verification** + +```sh +bash scripts/verify-line-endings.sh +bash scripts/test-unified-compose.sh +bash scripts/test-compose-secret-policy.sh +bash scripts/unified-deployment-smoke.sh +bash scripts/thothctl-update-smoke.sh +bash scripts/test-verify-workspace-install-docs.sh +cd backend && npx vitest run && npx tsc --noEmit -p . +cd ../frontend && npx vitest run && npx tsc -b +cd ../harness && .venv/bin/pytest -q +``` + +Expected: all non-L2 tests pass; Docker-dependent harness tests run on a Docker-capable host. + +- [ ] **Step 6: Commit** + +```sh +git add scripts/unified-deployment-smoke.sh scripts/thothctl-update-smoke.sh scripts/test-windows-clone-contract.ps1 .github/workflows/deployment.yml PROJECT_STATE.md README.md +git commit -m "test: gate unified compose deployment" +``` + +## Completion criteria + +- Clean GitHub clones build/run on Windows Docker Desktop/WSL2 and macOS using only Docker, Compose, and Git. +- The same source/images deploy on Linux through the server override. +- Active deployment files contain no PSD, Chirone, or `omics_portal` dependency. +- DWH, VectorDB, embedding, and LLM are always configurable external endpoints. +- Pi exists in `core`, is configurable through Pi Management, and is safely updated by `thothctl`. +- Git attributes prevent CRLF and verification catches corruption before image startup. +- Local/server guides pass executable documentation checks. +- Failed updates restore the previous image while preserving registry, sessions, settings, and Pi state. diff --git a/docs/superpowers/plans/2026-08-09-prd-p1-descriptor-evidence.md b/docs/superpowers/plans/2026-08-09-prd-p1-descriptor-evidence.md new file mode 100644 index 00000000..d377668d --- /dev/null +++ b/docs/superpowers/plans/2026-08-09-prd-p1-descriptor-evidence.md @@ -0,0 +1,1697 @@ +# P1 Descriptor Evidence Configuration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement P1/D1 so a schema-v3 workspace can declare a complete, non-secret Evidence source and policy, bind any source credentials from installation-local files, preserve descriptor/source identity at one Git revision, and render a harness configuration that passes `tht config check`. + +**Architecture:** The canonical descriptor stays in `workspaces/.yaml` inside the shared registry repository. Filesystem Evidence is named by a lexical repo-relative URI under `workspace-content//evidence`; the registry verifies that the tree and descriptor exist in the same commit, while P6—not P1—will materialize and realpath-check that tree. The backend derives source-specific installation bindings, immutable revision identity, `evidence.sources`, and vector policy into the same runtime YAML already used by sessions; the harness parses file-backed HTTP/S3 credentials without acquiring content. P1 is proven first by a clean-state, real-Git/real-HTTP automated process and then by an independent manual artifact walkthrough. + +**Tech Stack:** TypeScript 5, Zod 4, Fastify 5, Git CLI, YAML, React 18, Python 3.12, Pydantic 2, pytest, Vitest, Node.js 22, Bash. + +**Source PRD:** `docs/prd/2026-08-09-workspace-preprocessing-prd.md` v0.5 (`208b299`), especially RF1, RNF1–RNF8, D1, D6, D8, D9, P1, and “Standard di verifica obbligatorio del futuro piano P1”. + +## P1 completion contract + +P1 implements D1 and the P1-owned prerequisites of RF1; it does not claim all of RF1 complete. In particular, P2 owns the backend-independent host renderer/preprocessing consumer needed to finish RF1.3, and P10 owns operational `ssh_tunnel` support for RF1.4. + +P1 is complete only when all of the following are true: + +1. Schema v3 accepts an optional strict `evidence` section. Absence remains operational for existing workspaces (RNF3); P2 will warn when preprocessing is requested without Evidence. +2. `filesystem`, `http`, and `s3` are supported descriptor source types. The descriptor contains source identity and non-secret behavior only. +3. Filesystem URI is exactly the workspace Evidence root, `workspace-content//evidence`, using normalized POSIX segments. It is checked lexically in schema validation and checked as a Git tree at the same immutable commit during publication/activation. +4. HTTP query-bearing signed URLs and S3 static credentials enter only through installation-local `*_FILE` bindings. Public descriptor HTTP URIs never contain userinfo, query, or fragment. No secret file content is emitted into Git, registry docs, exports, API errors, reports, or rendered YAML. +5. The existing backend production renderer emits a harness-compatible `evidence.sources` entry plus `vector.max_chunk_chars` and `vector.retain_published_generations`, under the same `runtime_identity.workspace_revision` used by sessions. P1 proves that backend-runtime half of RF1.3; P2 must provide and prove a backend-independent host-CLI render path before claiming preprocessing uses the same effective config. +6. Existing frontend workspace flows parse, preserve, conflict-resolve, and re-publish Evidence. P1 adds only a read-only summary, not a new preprocessing or Evidence-authoring UI. +7. The automated gate passes from clean state through local Git → registry → real HTTP → snapshot/docs/export → production render → real `tht config check`, with deterministic outputs, negative cases, secret scanning, inspectable reports, and ownership-confined cleanup. +8. The manual gate uses entirely new state and remains `PENDING` until a human reviewer records approval. + +## Canonical descriptor contract + +Use this exact filesystem form in the generic examples and acceptance fixture: + +```yaml +evidence: + source: + type: filesystem + uri: workspace-content/example/evidence + patterns: + - "**/*.md" + max_bytes: 10485760 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +HTTP is an explicit, non-secret manifest. `authentication: signed_urls_file` requires one installation file containing a JSON array of signed transport URLs in the same order as `uris`; the harness must verify that stripping each transport URL's query produces the declared URI before it accepts the config. + +```yaml +evidence: + source: + type: http + uris: + - https://evidence.example.test/guide.md + authentication: signed_urls_file # or none + connect_timeout_ms: 5000 + read_timeout_ms: 30000 + max_bytes: 10485760 + max_redirects: 5 + allow_private_hosts: false + max_cache_bytes: 67108864 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +S3 uses one canonical `s3://` URI. `credentials: static_files` requires access-key and secret-key files and permits an optional session-token file; `ambient` delegates to the installation's AWS-compatible provider chain. + +```yaml +evidence: + source: + type: s3 + uri: s3://evidence-bucket/example/ + region: eu-west-1 + credentials: static_files # or ambient + trusted_endpoint: false + allow_private_endpoint: false + allow_insecure_endpoint: false + max_bytes: 10485760 + max_objects: 10000 + max_pages: 100 + page_size: 1000 + policy: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +The implementation must preserve these existing engine defaults exactly: + +| Field | Default | +|---|---:| +| filesystem `patterns` | `["**/*.md"]` | +| per-object `max_bytes` | `10 * 1024 * 1024` | +| HTTP connect/read timeout | `5000 ms` / `30000 ms` | +| HTTP redirects/cache | `5` / `64 * 1024 * 1024` bytes | +| S3 objects/pages/page size | `10000` / `100` / `1000` | +| `max_chunk_chars` | `4000` | +| `retain_published_generations` | `3` | + +P1 deliberately covers the **configuration-only** path for all three already-supported engine source types: strict declaration, file-path binding, render, and parse. This is required by RF1.1/D1's “sorgente completa” and the agreed three-source P1 scope. D6's “HTTP/S3 future” boundary still applies to network acquisition, materialization, preprocessing, and operational acceptance: P1 never calls either adapter. The signed-URL loader is the narrow bridge needed to keep authenticated transport values out of Git; it is not a new HTTP auth/header protocol. + +Installation variables are derived only when the selected source mode needs them: + +```text +THT_WS__EVIDENCE_SIGNED_URLS_FILE +THT_WS__EVIDENCE_ACCESS_KEY_FILE +THT_WS__EVIDENCE_SECRET_KEY_FILE +THT_WS__EVIDENCE_SESSION_TOKEN_FILE # optional +``` + +`` uses the existing `contractNamespace` normalization. These values are absolute file paths below configured secret roots, never secret values. + +## Hard scope boundaries + +- Do **not** run `tht preprocess evidence`, `tht evidence extract`, adapter discovery/acquisition, embeddings, Qdrant writes, ACTIVE publication, corpus GC, or retention execution. +- Do **not** create or validate `artifacts/evidence`, `corpus/ACTIVE`, evidence points, or embedding vectors. Those belong to P2/P4/P6/P8/P9. +- Do **not** copy the Evidence tree into current immutable descriptor snapshots. P6 owns commit-addressed materialization, realpath checks, nested symlink escape rejection, and race-safe consumption. +- Do **not** resolve a filesystem source against the mobile registry checkout. P1 renders the reserved future path `/snapshots//workspace-content//evidence`; `tht config check` validates structure without requiring that path to exist. +- Do **not** broaden `writeRegistryFile`/`commitAndPush` to arbitrary `workspace-content` writes. Curators change Evidence content through a normal Git clone; the API publishes descriptors and generated docs only. +- Do **not** add a browser preprocessing endpoint or host render/preprocessing CLI. The backend-independent host CLI and its equivalence proof are P2; full Evidence editor UX is future work. +- Do **not** make `ssh_tunnel` operational. Preserve the renderer's fail-closed behavior until P10. +- Do **not** claim same-revision identity from descriptor blob equality. A content-only Evidence commit has the same descriptor blob but a different authoritative commit. + +## Target file map + +**Backend contract and registry** + +- `backend/src/workspaces/schema.ts`: authoritative Evidence interfaces, Zod schemas, and cross-field invariants. +- `backend/src/workspaces/types.ts`: remove or synchronize the duplicate exported v3 shape so it cannot contradict the authoritative contract. +- `backend/src/workspaces/git-repository.ts`: fixed-argv read-only tree-at-revision assertion. +- `backend/src/workspaces/registry.ts`: contextual filesystem tree checks at publish/activate and content-only revision behavior. +- `backend/src/workspaces/contracts.ts`: Evidence installation variables and generated public README. +- `backend/src/workspaces/bindings.ts`: source-specific Evidence file binding resolution. +- `backend/src/workspaces/diagnostics.ts`: report installation-local `binding_missing` when required Evidence files are missing/unsafe. +- `backend/src/workspaces/runtime-renderer.ts`: runtime `evidence.sources` and vector policy mapping. +- `backend/src/tht/tht-runner.ts`: commit-addressed render context. +- `backend/src/routes/workspaces.ts`: validation/read/publish/import/export round-trip and safe errors. + +**Harness compatibility** + +- `harness/tht/config.py`: strict Evidence runtime config, signed-URL file resolution, provenance validation. +- `harness/tht/adapters/factory.py`: consume validated HTTP transport URLs without exposing them. +- `harness/tests/test_config_resources.py`: source/policy parsing and secret masking. +- `harness/tests/test_registry_evidence_config.py`: renderer-facing runtime contract and config-check behavior. + +**Frontend compatibility** + +- `frontend/src/api/workspaces.ts`: canonical Evidence types and conflict field allowlist. +- `frontend/src/workspaces/drafts.ts`: strict sanitizer/copy functions that preserve Evidence. +- `frontend/src/shell/WorkspaceEditor.tsx`: read-only Evidence summary; existing edits preserve the object. + +**Examples, docs, and gates** + +- `deploy/workspaces/example.yaml`, `deploy/workspaces/psd.yaml.example`: generic descriptor examples. +- `docs/contracts/workspace-evidence-v3.md`: human-readable canonical contract. +- `docs/install/local-workspace-registry.md`, `docs/install/server-workspace-registry.md`: operator registry layout and secret boundary. +- `docs/install/examples/workspace-bindings.env.example`: file-path binding examples only. +- `scripts/verify-workspace-install-docs.sh`, `scripts/test-verify-workspace-install-docs.sh`: executable doc contract. +- `scripts/p1-acceptance.sh`, `backend/scripts/p1-acceptance.mjs`, `scripts/test-p1-acceptance.sh`: automated process goal. +- `scripts/p1-manual-acceptance.sh`, `backend/scripts/p1-manual-acceptance.mjs`, `backend/scripts/p1-render-snapshot.mjs`, `docs/testing/p1-manual-acceptance.md`: manual walkthrough and explicit production-render tooling. +- `PROJECT_STATE.md`: separate automated/manual status and retained artifact path. + +--- + +### Task 1: Define the optional, strict schema-v3 Evidence contract + +**Files:** +- Modify: `backend/src/workspaces/schema.ts` +- Modify: `backend/src/workspaces/types.ts` +- Modify: `backend/test/workspaces-schema.test.ts` +- Modify: `backend/test/workspaces-migrate-v2-qdrant.test.ts` +- Modify: `backend/test/workspaces-migrate-legacy.test.ts` + +**Interfaces:** + +```ts +export interface EvidencePolicy { + max_chunk_chars: number; + retain_published_generations: number; +} + +export type EvidenceSource = + | { + type: "filesystem"; + uri: string; + patterns: string[]; + max_bytes: number; + } + | { + type: "http"; + uris: string[]; + authentication: "none" | "signed_urls_file"; + connect_timeout_ms: number; + read_timeout_ms: number; + max_bytes: number; + max_redirects: number; + allow_private_hosts: boolean; + max_cache_bytes: number; + } + | { + type: "s3"; + uri: string; + endpoint_url?: string; + region?: string; + credentials: "ambient" | "static_files"; + trusted_endpoint: boolean; + allow_private_endpoint: boolean; + allow_insecure_endpoint: boolean; + max_bytes: number; + max_objects: number; + max_pages: number; + page_size: number; + }; + +export interface WorkspaceEvidence { + source: EvidenceSource; + policy: EvidencePolicy; +} +``` + +`WorkspaceV3` gains `evidence?: WorkspaceEvidence`; v1/v2 remain strict and unchanged. Prefer deleting the unused duplicate `WorkspaceV2`/`WorkspaceV3` declarations from `backend/src/workspaces/types.ts` and importing the authoritative schema types wherever needed. If a public compatibility reason prevents deletion, re-export the schema types instead of maintaining a second handwritten structure. + +- [ ] **Step 1: Write failing schema tests** + +Add table-driven positive tests for: + +- filesystem with explicit values; +- filesystem with all defaults applied; +- HTTP `none` and `signed_urls_file` modes; +- S3 `ambient` and `static_files` modes; +- `evidence` absent on a valid v3 workspace; +- parse → canonical object → `serializeWorkspaceYaml` → parse equality. + +Add table-driven negative tests for: + +- absolute filesystem paths, `..`, `.`, empty/doubled segments, trailing traversal, backslashes, NUL/control characters, and another workspace's namespace; +- a filesystem URI above or below the canonical root (patterns select descendants; URI itself is the exact root); +- unsupported source discriminators and unknown keys; +- credential-shaped Git fields such as `password`, `api_key`, `access_key`, `secret_key`, `session_token`, `signed_url`, `headers`, and `ca_contents`; +- HTTP userinfo, query, fragment, non-HTTP schemes, duplicates after canonicalization, empty manifests, and invalid bounds; +- invalid S3 schemes, empty bucket, userinfo/query/fragment, unsafe endpoint syntax, inconsistent endpoint opt-ins, and invalid limits; +- chunk/retention values outside explicit bounds; +- `evidence` on schema v1/v2. + +Use explicit canaries in tests and assert the resulting public error message contains a safe field path but not the canary value. + +- [ ] **Step 2: Run the focused test and verify RED** + +Run: + +```bash +cd backend +npx vitest run test/workspaces-schema.test.ts +``` + +Expected: FAIL because `WorkspaceV3Schema` rejects `evidence` and the types do not exist. + +- [ ] **Step 3: Implement strict source schemas and defaults** + +In `schema.ts`, create `.strict()` Zod objects and one discriminated union. Use safe-integer validation while preserving the engine's existing lower-bound semantics: + +```ts +const EvidencePolicySchema = z.object({ + max_chunk_chars: z.number().int().safe().positive().default(4_000), + retain_published_generations: z.number().int().safe().min(1).default(3), +}).strict(); +``` + +Define source defaults exactly as listed in the completion contract. Default the whole `policy` object to `{ max_chunk_chars: 4000, retain_published_generations: 3 }`, so the canonical parsed object and serialized YAML are explicit even when the author omitted policy fields. Convert descriptor milliseconds to harness seconds only in the renderer; keep integer milliseconds in Git. Validate URLs with `URL`, but return sanitized Zod issues that identify the field/index rather than interpolating the rejected URL. + +Add small, named helpers used by `workspaceInvariants`, for example: + +```ts +function expectedEvidenceRoot(workspaceId: string): string { + return `workspace-content/${workspaceId}/evidence`; +} + +function isNormalizedRepoRelativePath(value: string): boolean { + const parts = value.split("/"); + return value.length > 0 + && !value.startsWith("/") + && !value.includes("\\") + && !/[\u0000-\u001f\u007f]/u.test(value) + && parts.every((part) => part !== "" && part !== "." && part !== ".."); +} +``` + +The cross-field issue must be attached to `evidence.source.uri` and require exact equality with `expectedEvidenceRoot(workspace.workspace.id)`. Do not call `resolve`, `realpath`, or inspect the filesystem here. Require a nonempty, stably deduplicated `patterns` list; each glob must be relative, slash-normalized, free of empty/`.`/`..` segments, backslashes, and control characters, so a glob cannot escape the declared root. + +For HTTP, reject userinfo/query/fragment in descriptor URIs and reject duplicates after a stable canonical form. For S3, parse the `s3://` URI into a nonempty bucket and optional prefix, and keep `endpoint_url`, region, trust flags, and limits non-secret. + +- [ ] **Step 4: Preserve migration behavior** + +Legacy and v2→v3 migration output must omit `evidence`, not invent a filesystem tree. Add assertions to both migration test files. This makes migrated workspaces operational for existing sessions but not yet configured for preprocessing. + +- [ ] **Step 5: Run focused tests and typecheck** + +Run: + +```bash +cd backend +npx vitest run \ + test/workspaces-schema.test.ts \ + test/workspaces-migrate-v2-qdrant.test.ts \ + test/workspaces-migrate-legacy.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add backend/src/workspaces/schema.ts backend/src/workspaces/types.ts \ + backend/test/workspaces-schema.test.ts \ + backend/test/workspaces-migrate-v2-qdrant.test.ts \ + backend/test/workspaces-migrate-legacy.test.ts +git commit -m "feat: define workspace evidence descriptor contract" +``` + +--- + +### Task 2: Bind filesystem declarations to a Git tree at the same revision + +**Files:** +- Modify: `backend/src/workspaces/git-repository.ts` +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/test/workspaces-git-repository.test.ts` +- Modify: `backend/test/workspace-registry.test.ts` + +**Interfaces:** + +```ts +// Read-only. It never stages, checks out, or follows worktree symlinks. +GitWorkspaceRepository.assertTreeAtRevision( + revision: string, + repoRelativePath: string, +): Promise +``` + +The helper invokes Git with a fixed argv, equivalent to: + +```text +git cat-file -t <40-hex-commit>:workspace-content//evidence +``` + +and accepts only output `tree`. A missing path, blob/symlink at the declared root, malformed revision, or Git failure becomes a sanitized `workspace_invalid`/`git_unavailable` registry error without command stderr or source content. + +- [ ] **Step 1: Write failing Git repository tests** + +Extend the existing `temporaryRemote()`-based suite. Seed one commit with: + +```text +workspaces/research.yaml +workspace-content/research/evidence/guide.md +workspace-content/other/evidence/other.md +``` + +Test that the helper: + +- accepts the `research` Evidence tree at that commit; +- rejects a missing path; +- rejects a blob and a Git symlink at the declared root; +- distinguishes old/new commits after a content-only Evidence change; +- uses an argv array and never passes the path through a shell. + +Do not recursively reject a symlink nested inside the tree. Add an explicit test/comment that nested containment is intentionally deferred to P6. + +- [ ] **Step 2: Run the Git test and verify RED** + +```bash +cd backend +npx vitest run test/workspaces-git-repository.test.ts +``` + +Expected: FAIL because `assertTreeAtRevision` is missing. + +- [ ] **Step 3: Implement the fixed-argv read-only helper** + +Reuse the repository's existing Git runner and revision/path safety helpers. Do not add `workspace-content` to `isRegistryArtifactPath`, `writeRegistryFile`, the publish staging allowlist, or generated API artifacts. + +- [ ] **Step 4: Write failing registry tests for contextual validation** + +Add real-bare-repository cases proving: + +1. publication succeeds only when the descriptor's filesystem tree already exists in the pulled base commit; +2. the resulting publication commit contains both the descriptor and the unchanged Evidence tree; +3. activation validates the tree against the exact `safeHead` used for the descriptor; +4. a remote descriptor with a missing/non-tree source fails pull and retains the previously active snapshot; +5. a content-only remote commit creates a new `WorkspaceRevision.commit` and immutable descriptor snapshot even when the descriptor blob is unchanged; +6. a stale API update based on the pre-content commit receives `workspace_stale`/409 semantics rather than overwriting the curator's content commit; +7. retained and pinned historical revisions remain distinguishable by commit. + +Assertions must compare commit IDs and Git object existence, not mutable checkout paths. + +- [ ] **Step 5: Run the registry test and verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts +``` + +Expected: at least the missing-tree and content-only revision cases FAIL. + +- [ ] **Step 6: Add one contextual validation function in the registry** + +Use a private helper such as: + +```ts +private async assertEvidenceContext( + workspace: WorkspaceDescriptor, + revision: string, +): Promise { + if (workspace.workspace.schema_version !== 3) return; + if (workspace.evidence?.source.type !== "filesystem") return; + await this.repository.assertTreeAtRevision(revision, workspace.evidence.source.uri); +} +``` + +Call it: + +- after pull/read of the base and before descriptor publication; +- during activation against `safeHead`, before replacing the active revision; +- during integrity repair/reload wherever an existing snapshot is re-associated with a Git commit. + +Publication still stages only `workspaces/.yaml` and generated `workspace-docs//...`. Activation still snapshots only descriptor/contract/README/manifest. Document the deliberate P6 boundary adjacent to the code. + +- [ ] **Step 7: Run focused tests** + +```bash +cd backend +npx vitest run \ + test/workspaces-git-repository.test.ts \ + test/workspace-registry.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS, including the content-only revision regression. + +- [ ] **Step 8: Commit** + +```bash +git add backend/src/workspaces/git-repository.ts backend/src/workspaces/registry.ts \ + backend/test/workspaces-git-repository.test.ts backend/test/workspace-registry.test.ts +git commit -m "feat: bind evidence trees to registry revisions" +``` + +--- + +### Task 3: Resolve HTTP/S3 credentials from installation-local files only + +**Files:** +- Modify: `backend/src/workspaces/contracts.ts` +- Modify: `backend/src/workspaces/bindings.ts` +- Modify: `backend/src/workspaces/diagnostics.ts` +- Read/verify wiring: `backend/src/app.ts` +- Modify: `backend/test/workspaces-contracts.test.ts` +- Modify: `backend/test/workspaces-bindings.test.ts` +- Modify: `backend/test/workspaces-diagnostics.test.ts` +- Modify: `backend/test/routes-sessions.test.ts` +- Modify: `harness/tht/config.py` +- Modify: `harness/tht/adapters/factory.py` +- Modify: `harness/tests/test_config_resources.py` +- Create: `harness/tests/test_registry_evidence_config.py` + +**Backend interfaces:** + +```ts +export type InstallationRole = /* existing roles */ | "EVIDENCE"; +export type InstallationSuffix = + | /* existing suffixes */ + | "SIGNED_URLS_FILE" + | "ACCESS_KEY_FILE" + | "SECRET_KEY_FILE" + | "SESSION_TOKEN_FILE"; + +export interface ResolvedEvidenceBinding { + values: Record; // safe file paths only + missing: string[]; +} + +export interface RuntimeBindings { + // existing roles... + evidence: ResolvedEvidenceBinding; +} +``` + +`resolveEvidenceBinding(workspace, env, secretRoots)` returns no variables/missing values for filesystem, HTTP `none`, or S3 `ambient`. It requires the signed-URL file for HTTP `signed_urls_file`; it requires access-key and secret-key files for S3 `static_files`, and accepts the session-token file only when present and safe. + +**Harness runtime shape for signed HTTP:** + +```yaml +evidence: + sources: + - type: http + provenance_urls: + - https://evidence.example.test/guide.md + signed_urls_file: /run/secrets/example-evidence-signed-urls + # limits follow +``` + +The file is UTF-8 JSON with a nonempty array of strings, maximum 1 MiB. `load_config` replaces the file reference in memory with `urls: list[SecretStr]`; it verifies one-to-one order and `canonical_provenance_uri(signed_url) == provenance_urls[index]`. It never stores the file contents in the Pydantic repr, validation message, CLI output, or returned public metadata. + +- [ ] **Step 1: Write failing backend contract/binding tests** + +Cover: + +- stable namespace and exact variable names; +- no Evidence variables for modes without file credentials; +- source-specific variables only (HTTP never gets S3 files and vice versa); +- HTTP signed file required; +- S3 access/secret required together; session token optional; +- absolute readable regular files under a configured realpath secret root accepted; +- relative paths, missing files, directories, unreadable files, and symlink escapes rejected as `missing`; +- binding results contain file paths, never file contents; +- generated contract/README and diagnostic errors do not contain secret canaries; +- `/workspaces/:id/test` reports local `binding_missing` without changing the canonical registry revision; +- real session admission through `buildApp`/`routes-sessions` refuses before Pi spawn when a declared required Evidence file is missing/unsafe, accepts a safe binding far enough to reach the existing next admission boundary, and preserves no-Evidence compatibility; +- schema-v3 DWH/vector/embedding behavior remains unchanged. + +- [ ] **Step 2: Run backend tests and verify RED** + +```bash +cd backend +npx vitest run \ + test/workspaces-contracts.test.ts \ + test/workspaces-bindings.test.ts \ + test/workspaces-diagnostics.test.ts \ + test/routes-sessions.test.ts +``` + +Expected: FAIL because Evidence is not an installation role and `RuntimeBindings` has no Evidence binding. + +- [ ] **Step 3: Implement conditional contract generation and binding resolution** + +Keep the existing principle from `bindings.ts`: validate paths and pass them through, but never read their contents. Add `resolveEvidenceBinding` rather than overloading the DWH/vector transport resolver with a source type that is not one of those transports. + +Update all `RuntimeBindings` construction sites/tests. `supportsSessionRuntime` must return false when a descriptor-selected Evidence credential file is missing/unsafe, while an absent Evidence section yields an empty successful binding and preserves RNF3 session compatibility. Prove the existing `app.ts` admission closure actually applies that result in `routes-sessions.test.ts`; registry publication/activation remains installation-independent and must not read local bindings. In V3 `/workspaces/:id/test` diagnostic preflight, convert missing required Evidence bindings to existing sanitized `binding_missing` diagnostics with the descriptor field (`evidence.source.authentication` or `evidence.source.credentials`) and variable name; never include an environment value. + +- [ ] **Step 4: Run backend tests and typecheck** + +```bash +cd backend +npx vitest run \ + test/workspaces-contracts.test.ts \ + test/workspaces-bindings.test.ts \ + test/workspaces-diagnostics.test.ts \ + test/routes-sessions.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS. + +- [ ] **Step 5: Write failing harness config tests** + +In `test_registry_evidence_config.py`, build raw runtime YAML for: + +- filesystem with a deliberately nonexistent absolute root (parse/check succeeds; no adapter construction); +- public HTTP URLs; +- signed HTTP with a valid JSON secret file and matching provenance; +- signed HTTP with missing/oversized/malformed/non-list files; +- signed HTTP with reordered, extra, query-free mismatch, userinfo, or duplicate canonical provenance; +- S3 ambient credentials; +- S3 `access_key_file`, `secret_key_file`, and optional `session_token_file` resolved into `SecretStr`; +- all source/policy defaults and non-default values; +- unknown Evidence keys rejected. + +Capture config model repr, `tht config check` stdout/stderr, and exception text; assert no signed query/access key/secret key/session token canary occurs. + +- [ ] **Step 6: Run harness tests and verify RED** + +```bash +cd harness +.venv/bin/pytest -q \ + tests/test_config_resources.py \ + tests/test_registry_evidence_config.py +``` + +Expected: signed URL file cases FAIL because the loader only knows scalar password/access/secret/session `*_file` fields. + +- [ ] **Step 7: Implement bounded signed-URL file loading and strict Evidence models** + +Add a dedicated loader before Pydantic validation; do not teach the generic scalar secret resolver to parse arbitrary JSON. The outline is: + +```py +def _resolve_http_signed_url_files(value: Any) -> Any: + # Recurse only through mappings/lists. + # For {type: "http", signed_urls_file: ...}: + # lstat/stat/read at most 1 MiB as UTF-8 + # parse JSON array[str] + # set urls to the array and remove signed_urls_file + # Never interpolate array values in ConfigError. + ... +``` + +`HttpEvidenceSourceConfig` accepts `provenance_urls` for the file-backed form, holds actual transport `urls` as `SecretStr`, validates the one-to-one canonical mapping, and exposes a method returning secret values only to the adapter factory. Add `model_config = {"extra": "forbid"}` to the three modern source models, `EvidenceSourcesConfig`, and the policy model involved in this contract so renderer typos fail loudly. + +Do not continue formatting raw `ValidationError` with `f"{e}"` after secret files have been resolved: Pydantic may include rejected input. Add a safe formatter based on `e.errors(include_input=False, include_url=False)` that retains only location, stable error type, and a custom message that never interpolates transport URLs/credential values. Apply it to `load_config` and prove existing non-secret diagnostics remain useful. + +The factory may unwrap transport URLs only at the last moment when constructing `HttpManifestEvidenceSource`; `config check` must not construct any Evidence adapter or touch network/source roots. + +- [ ] **Step 8: Run focused harness tests and lint** + +```bash +cd harness +.venv/bin/pytest -q \ + tests/test_config_resources.py \ + tests/test_registry_evidence_config.py +.venv/bin/ruff check \ + tht/config.py \ + tht/adapters/factory.py \ + tests/test_config_resources.py \ + tests/test_registry_evidence_config.py +``` + +Expected: PASS. + +- [ ] **Step 9: Commit** + +```bash +git add \ + backend/src/workspaces/contracts.ts \ + backend/src/workspaces/bindings.ts \ + backend/src/workspaces/diagnostics.ts \ + backend/test/workspaces-contracts.test.ts \ + backend/test/workspaces-bindings.test.ts \ + backend/test/workspaces-diagnostics.test.ts \ + backend/test/routes-sessions.test.ts \ + harness/tht/config.py \ + harness/tht/adapters/factory.py \ + harness/tests/test_config_resources.py \ + harness/tests/test_registry_evidence_config.py +git commit -m "feat: bind evidence credentials through local files" +``` + +--- + +### Task 4: Render Evidence through the production runtime handoff + +**Files:** +- Modify: `backend/src/workspaces/runtime-renderer.ts` +- Modify: `backend/src/tht/tht-runner.ts` +- Modify: `backend/test/workspace-runtime-renderer.test.ts` +- Modify: `backend/test/workspace-runtime-handoff.test.ts` + +**Renderer context:** + +Extend the current explicit context, rather than reading the registry checkout or ambient cwd: + +```ts +export interface RuntimeRenderContext { + // existing identity, roots, semantic resources... + revisionContentRoot: string; // /snapshots/ +} +``` + +For a filesystem URI, render: + +```yaml +runtime_identity: + workspace_id: example + workspace_revision: <40-hex-commit> +evidence: + sources: + - type: filesystem + root: /snapshots//workspace-content/example/evidence + patterns: ["**/*.md"] + max_bytes: 10485760 +vector: + max_chunk_chars: 4000 + retain_published_generations: 3 +``` + +HTTP mapping: + +- descriptor `authentication: none` → harness `urls` containing the public descriptor `uris`; +- descriptor `authentication: signed_urls_file` → `provenance_urls` plus `signed_urls_file` from `RuntimeBindings.evidence`; +- convert `_ms` descriptor timeouts to exact seconds without lossy rounding; +- map all limits/SSRF policy fields. + +S3 mapping: + +- split canonical `s3://bucket/prefix` into `bucket` and `prefix`; +- map endpoint/region/trust/limits; +- ambient mode emits no credential keys; +- static mode emits only `access_key_file`, `secret_key_file`, and an optional `session_token_file` path. + +If `evidence` is absent, omit `evidence` and its policy override. Preserve all existing roots, DWH/Qdrant/Ollama behavior and the intentional `ssh_tunnel` error. + +- [ ] **Step 1: Write failing renderer tests** + +Add exact parsed-YAML assertions for: + +- filesystem root under the commit directory, never under `/repo`; +- public/signed HTTP; +- ambient/static S3; +- default and non-default policy; +- no-Evidence omission; +- missing required Evidence bindings rejected before rendering; +- `runtime_identity.workspace_revision` equal to the directory commit; +- no secret file contents in YAML; +- two renders with identical inputs are byte-identical; +- a content-only commit changes identity/root even when descriptor YAML is unchanged; +- existing `ssh_tunnel` fail-closed test remains unchanged. + +- [ ] **Step 2: Run renderer test and verify RED** + +```bash +cd backend +npx vitest run test/workspace-runtime-renderer.test.ts +``` + +Expected: FAIL because no Evidence/runtime content root is rendered. + +- [ ] **Step 3: Implement pure renderer mapping** + +Keep path derivation lexical and deterministic: + +```ts +const filesystemRoot = join( + context.revisionContentRoot, + workspace.evidence.source.uri, +); +``` + +This is safe only because Task 1 canonicalized the URI and Task 2 checked it as a tree at the same commit. Do not `realpath` it or require existence in P1. + +Do not access `process.env` inside the renderer. Receive already validated binding file paths through `RuntimeBindings` so the backend has one deterministic runtime path. Treat the exact descriptor→rendered-YAML mapping and golden handoff tests as P2's compatibility contract; do not claim that the future host CLI can import this backend TypeScript module or that RF1.3 is complete before P2 supplies its backend-independent caller. + +- [ ] **Step 4: Write failing production handoff tests** + +Extend `workspace-runtime-handoff.test.ts` using its real local bare Git fixture and harness invocation. Test: + +1. a registry revision with descriptor plus Evidence tree is activated; +2. `ThtRunner.acquireWorkspaceRuntime(snapshotPath)` reads canonical descriptor/identity and passes `` as `revisionContentRoot`; +3. its leased runtime YAML has the exact Evidence source/policy mapping; +4. `harness/.venv/bin/tht config check -c ` exits zero; +5. acquire/check/release twice yields identical copied YAML while lease file names may differ; +6. release removes only the owned lease file; +7. signed HTTP/S3 file paths resolve from configured secret roots and no canary reaches captured output. + +Use the exact CLI ordering: + +```bash +harness/.venv/bin/tht config check -c /absolute/path/to/rendered.yaml +``` + +Never place `-c` before `config check`. + +- [ ] **Step 5: Run handoff test and verify RED** + +```bash +cd backend +npx vitest run test/workspace-runtime-handoff.test.ts +``` + +Expected: new Evidence assertions FAIL. + +- [ ] **Step 6: Pass the immutable render context from `ThtRunner`** + +`readCanonicalWorkspaceSnapshot` already proves that the descriptor path is under `//` and matches `runtime_identity`. Derive `revisionContentRoot` from that validated path/commit and pass it to the renderer. Never consult the live checkout after the snapshot is acquired. + +- [ ] **Step 7: Run focused cross-layer tests and checks** + +```bash +cd backend +npx vitest run \ + test/workspace-runtime-renderer.test.ts \ + test/workspace-runtime-handoff.test.ts +npx tsc --noEmit -p . +npm run build +``` + +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add \ + backend/src/workspaces/runtime-renderer.ts \ + backend/src/tht/tht-runner.ts \ + backend/test/workspace-runtime-renderer.test.ts \ + backend/test/workspace-runtime-handoff.test.ts +git commit -m "feat: render revision-bound evidence configuration" +``` + +--- + +### Task 5: Preserve Evidence across registry docs, HTTP routes, conflicts, and exports + +**Files:** +- Modify: `backend/src/workspaces/contracts.ts` +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/test/workspaces-contracts.test.ts` +- Modify: `backend/test/workspace-registry.test.ts` +- Modify: `backend/test/routes-workspaces.test.ts` + +**Public artifact rule:** Generated docs describe the source type, canonical non-secret URI(s), limits/policy, same-revision rule, and required installation file variable names. They never include secret contents. Export remains exactly: + +```text +manifest.json +workspace.yaml +contract.env.example +README.md +``` + +P1 does not include `workspace-content` bytes in the browser/API ZIP. + +- [ ] **Step 1: Write failing contract and registry artifact tests** + +Assert for all three sources: + +- contract ordering and generated README are deterministic; +- only applicable variables are present; +- filesystem docs explain same-revision Git ownership and P6 materialization boundary; +- HTTP/S3 docs explain file/ambient credential modes without sample secrets; +- snapshot descriptor/contract/README/manifest hashes match the active commit; +- the snapshot directory contains no copied Evidence tree; +- a secret canary present only in a fixture file never appears in Git blobs, generated docs, snapshot metadata, or error messages. + +Run and confirm RED where the generated docs omit Evidence: + +```bash +cd backend +npx vitest run \ + test/workspaces-contracts.test.ts \ + test/workspace-registry.test.ts +``` + +- [ ] **Step 2: Extend generated public documentation** + +Render a compact `Evidence source` section from canonical descriptor fields. Never read binding files while generating it. Preserve current stable ordering so re-publication of the same descriptor/base remains idempotent. + +If registry snapshot integrity lists expected files, keep the current allowlist deliberately descriptor-only and add a comment pointing to P6 rather than adding `workspace-content` now. + +- [ ] **Step 3: Write failing real-route tests** + +Using `app.inject()` route tests plus the real registry fixture, cover: + +- `/workspaces/validate` returns the canonical Evidence defaults and conditional contract; +- publish create/update, pull, list, and read preserve the whole descriptor; +- an Evidence-only concurrent edit reports a safe conflict field such as `evidence.source.uri` or `evidence.policy.max_chunk_chars`; +- invalid absolute/traversal/cross-workspace/protocol/credential payloads return safe 400 `workspace_invalid`, do not mutate HEAD, and do not echo canaries; +- contextual missing Git tree fails publish/pull safely; +- export, safe extraction, and import preserve canonical descriptor/docs; +- extracted file bytes/hashes are stable across two exports; +- ZIP contains no Evidence bytes and no secrets. + +Do not require raw ZIP byte equality unless production ZIP metadata is explicitly fixed; the P1 determinism contract is stable extracted files and manifest hashes. + +- [ ] **Step 4: Run route test and verify RED** + +```bash +cd backend +npx vitest run test/routes-workspaces.test.ts +``` + +Expected: Evidence route/export expectations FAIL until all payload/artifact paths use the new canonical schema and docs. + +- [ ] **Step 5: Make the smallest route/artifact changes** + +Prefer existing `validateCanonicalWorkspace`, `serializeWorkspaceYaml`, recursive conflict folding, and `exportBundle` paths. Do not create a parallel Evidence DTO and do not add an Evidence upload/preprocess route. Keep public errors on the current sanitized `WorkspaceRegistryError` path. + +- [ ] **Step 6: Run focused backend tests and typecheck** + +```bash +cd backend +npx vitest run \ + test/workspaces-contracts.test.ts \ + test/workspace-registry.test.ts \ + test/routes-workspaces.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add \ + backend/src/workspaces/contracts.ts \ + backend/src/workspaces/registry.ts \ + backend/src/routes/workspaces.ts \ + backend/test/workspaces-contracts.test.ts \ + backend/test/workspace-registry.test.ts \ + backend/test/routes-workspaces.test.ts +git commit -m "feat: preserve evidence in workspace artifacts" +``` + +--- + +### Task 6: Keep existing browser workspace flows lossless without adding authoring UX + +**Files:** +- Modify: `frontend/src/api/workspaces.ts` +- Modify: `frontend/src/api/workspaces.test.ts` +- Modify: `frontend/src/workspaces/drafts.ts` +- Modify: `frontend/src/workspaces/drafts.test.ts` +- Modify: `frontend/src/shell/WorkspaceEditor.tsx` +- Modify: `frontend/src/shell/WorkspaceEditor.test.tsx` + +**Scope:** The browser must accept and preserve canonical Evidence returned by the backend. P1 does not add source-edit controls or launch preprocessing. A small read-only summary prevents the field from being invisible while the registry descriptor remains its authoring surface. + +- [ ] **Step 1: Write failing frontend contract tests** + +Add one fixture for each source type and assert: + +- `sanitizeCanonicalWorkspace` accepts the new top-level key and returns a deep sanitized copy; +- all unknown keys, secret-shaped keys, unsafe URIs, invalid policy values, and malformed unions are rejected rather than passed into browser state; +- draft save/load preserves Evidence; +- API validate/read/publish/conflict parsing preserves Evidence; +- conflict fields under `evidence.source.*` and `evidence.policy.*` are accepted by the sanitized allowlist; +- changing an existing DWH/LLM editor field and publishing does not drop or mutate Evidence; +- no-Evidence workspaces continue to work. + +- [ ] **Step 2: Run focused tests and verify RED** + +```bash +cd frontend +npx vitest run \ + src/workspaces/drafts.test.ts \ + src/api/workspaces.test.ts \ + src/shell/WorkspaceEditor.test.tsx +``` + +Expected: FAIL because `exactRecord` currently rejects the `evidence` top-level key. + +- [ ] **Step 3: Add explicit frontend source types and strict copy helpers** + +Mirror the backend wire contract in `CanonicalWorkspace` without importing server code into the frontend build. Add focused helpers such as `copyEvidencePolicy`, `copyFilesystemEvidence`, `copyHttpEvidence`, and `copyS3Evidence`; use the same bounds and lexical checks as defense-in-depth. + +Update the top-level sanitizer allowlist: + +```ts +const source = exactRecord(value, [ + "workspace", "dwh", "semantic_index", "llm_policy", "diagnostics", "evidence", +]); +``` + +Return `...(evidence ? { evidence } : {})` in the sanitized object. Add the complete stable Evidence field paths to `conflictFields`; do not accept arbitrary server-provided conflict paths. + +- [ ] **Step 4: Add a read-only editor summary** + +When Evidence exists, show source type, safe canonical URI/count, chunk size, and retention with copy such as “Evidence is managed by the registry descriptor in P1.” Never render a signed URL or secret-file binding—those are not descriptor fields. Existing immutable updates already spread the workspace; add the regression test before relying on that behavior. + +- [ ] **Step 5: Run tests and typecheck** + +```bash +cd frontend +npx vitest run \ + src/workspaces/drafts.test.ts \ + src/api/workspaces.test.ts \ + src/shell/WorkspaceEditor.test.tsx +npx tsc -b +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add \ + frontend/src/api/workspaces.ts \ + frontend/src/api/workspaces.test.ts \ + frontend/src/workspaces/drafts.ts \ + frontend/src/workspaces/drafts.test.ts \ + frontend/src/shell/WorkspaceEditor.tsx \ + frontend/src/shell/WorkspaceEditor.test.tsx +git commit -m "fix: preserve workspace evidence in browser drafts" +``` + +--- + +### Task 7: Document and mechanically verify the shared-registry Evidence contract + +**Files:** +- Modify: `deploy/workspaces/example.yaml` +- Modify: `deploy/workspaces/psd.yaml.example` +- Create: `docs/contracts/workspace-evidence-v3.md` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `docs/install/examples/workspace-bindings.env.example` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/test-verify-workspace-install-docs.sh` + +**Required documented repository layout:** + +```text +registry.git/ +├── workspaces/ +│ ├── example.yaml +│ └── another.yaml +├── workspace-content/ +│ ├── example/evidence/... +│ └── another/evidence/... +└── workspace-docs/ + ├── example/{contract.env.example,README.md} + └── another/{contract.env.example,README.md} +``` + +Correct any current prose that claims generated `.env.example`/`.md` files live directly under `workspaces/`; production writes them under `workspace-docs//`. + +- [ ] **Step 1: Add failing verifier self-tests** + +Create mutated fixture copies that must fail when they: + +- omit the `workspace-content//evidence` layout or same-commit rule; +- show an absolute/cross-workspace Evidence path; +- place generated docs in the wrong registry directory; +- omit HTTP/S3 file credential boundaries; +- contain credential literals, signed query examples, or unsafe placeholder values; +- claim P1 materializes/extracts/indexes Evidence; +- omit the exact `tht config check -c ` ordering; +- omit separate automated/manual acceptance states. + +Also retain all existing adversarial doc-verifier cases. + +- [ ] **Step 2: Run self-test and verify RED** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +``` + +Expected: new mutation cases are not detected yet. + +- [ ] **Step 3: Update generic descriptors and the canonical contract document** + +Add the explicit filesystem section to: + +- `deploy/workspaces/example.yaml` with `workspace-content/example/evidence`; +- `deploy/workspaces/psd.yaml.example` using its generic fixture ID and matching namespace. + +Do not add real PSD/client content or secrets to ThothII. + +`docs/contracts/workspace-evidence-v3.md` must include: + +- exact strict shapes/defaults for filesystem, public/signed HTTP, and ambient/static S3; +- safe/unsafe URI examples; +- installation file formats and variable naming; +- one Git repo for all workspace namespaces; +- descriptor/tree commit identity and content-only revision semantics; +- browser/export behavior; +- optional Evidence/no-Evidence compatibility; +- P1 lexical/tree checks versus P6 materialization/symlink checks; +- exact `tht config check -c` command; +- explicit statement that P1 does no acquisition, extraction, embeddings, Qdrant writes, ACTIVE publication, or GC. + +- [ ] **Step 4: Update local/server operator guides and binding example** + +Describe curator flow in the right order: + +1. clone/pull shared registry; +2. place source content under the workspace namespace and commit/push it; +3. validate/publish descriptor against that base commit; +4. inspect generated public docs; +5. provision any `*_FILE` paths outside Git below allowed secret roots; +6. render/check config; +7. stop—preprocessing/materialization is later P2/P6. + +The env example contains only non-secret values and file paths. It may use obvious non-working paths such as `/run/secrets/...`; never include a credential/signed URL. + +- [ ] **Step 5: Strengthen the verifier and run both directions** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/verify-workspace-install-docs.sh --fixtures-only +``` + +Expected: both PASS; every adversarial mutation fails inside the self-test for the intended reason. + +- [ ] **Step 6: Commit** + +```bash +git add \ + deploy/workspaces/example.yaml \ + deploy/workspaces/psd.yaml.example \ + docs/contracts/workspace-evidence-v3.md \ + docs/install/local-workspace-registry.md \ + docs/install/server-workspace-registry.md \ + docs/install/examples/workspace-bindings.env.example \ + scripts/verify-workspace-install-docs.sh \ + scripts/test-verify-workspace-install-docs.sh +git commit -m "docs: define workspace evidence registry contract" +``` + +--- + +### Task 8: Automated integration goal — complete P1 configuration process + +**Files:** +- Create: `scripts/p1-acceptance.sh` +- Create: `backend/scripts/p1-acceptance.mjs` +- Create: `backend/scripts/p1-acceptance.test.mjs` +- Create: `scripts/test-p1-acceptance.sh` +- Modify: `.gitignore` only if `.artifacts/` is not already ignored (it currently is; normally no edit) +- Modify: `PROJECT_STATE.md` + +**Public command:** + +```bash +./scripts/p1-acceptance.sh integration --keep +``` + +It uses Node stdlib plus the built backend's normal dependencies. It must not require Docker, frontend, DWH, Qdrant, Ollama, external network, or a real remote. It starts a real Fastify listener on `127.0.0.1` with OS-assigned port `0` and makes actual HTTP requests with `fetch`; `app.inject()` does not satisfy this gate. + +**Exact run topology:** + +```text +.artifacts/p1-integration// +├── ownership.json +├── remote.git/ +├── author/ +├── installation/ +│ ├── registry/ +│ ├── data/ +│ ├── runtime/ +│ └── bindings.env +├── fixture-secrets/ # sole secret-scan exclusion +├── fixtures/ +│ ├── descriptors/ +│ └── requests/ +├── requests/ +├── responses/ +├── exports/ +│ ├── raw/ +│ └── extracted/ +├── rendered/ +├── logs/ +├── report.json +└── report.md +``` + +`ownership.json` is written before creating child resources and includes schema version, run ID, random nonce, absolute root, repository root, start time, current PID, owned listener identity, and the exact resources the run may delete/stop. Never store tokens, secret contents, or full signed URLs in ownership/report files. + +- [ ] **Step 1: Write failing acceptance-runner unit tests** + +Use `node:test` for library-level guards. Test: + +- run ID/root validation accepts only a direct child of the repository's canonical `.artifacts/p1-integration`; +- cleanup refuses a missing/malformed/mismatched ownership file, wrong nonce, symlink root, parent root, manual-acceptance root, and foreign sibling; +- cleanup removes one correctly owned synthetic run and nothing else; +- report schema requires a single result per check, no duplicate/retry attempt field, safe relative artifact paths, hashes, timestamps, command names, and `overall` derived from checks; +- injected failure records exactly one failed scenario, retains its run for diagnosis, and exits nonzero; +- secret scanner skips only `fixture-secrets` and detects canaries everywhere else, including JSON/Markdown/logs/responses/rendered/export files; +- successful non-`--keep` cleanup and successful `--keep` retention; +- no command helper accepts shell strings; Git/tht/backend commands use argv arrays. + +Provide a test-only `P1_ACCEPTANCE_FAIL_AT=` hook. It is not a retry mechanism; it deterministically proves failure reporting. + +- [ ] **Step 2: Run the runner tests and verify RED** + +```bash +bash scripts/test-p1-acceptance.sh +``` + +Expected: FAIL because the acceptance runner does not exist. + +- [ ] **Step 3: Implement preflight and owned lab lifecycle** + +`scripts/p1-acceptance.sh` must: + +1. resolve repository root from the script location; +2. require `node`, `npm`, `git`, and an executable `harness/.venv/bin/tht` (or an explicit `THT_BIN` override); +3. run `npm --prefix backend run build` once; +4. invoke `node backend/scripts/p1-acceptance.mjs integration [--keep]`; +5. preserve the Node exit code. + +The Node runner must: + +- create a cryptographically random run ID/nonce; +- use `mkdir`-exclusive semantics and refuse reuse; +- write files atomically where they are process evidence; +- use `spawn`/`execFile` with argv and bounded timeouts; +- execute each scenario exactly once; +- always close its owned Fastify instance in `finally`; +- retain a failed run unconditionally; +- delete a successful run only when `--keep` is absent and ownership validation passes. + +Do not poll/restart a failed scenario. Awaiting `app.listen()` or a bounded `/health` readiness probe is startup synchronization, not a scenario retry; record it separately. + +- [ ] **Step 4: Create the local Git registry from zero** + +Inside the run: + +```bash +git init --bare --initial-branch=main /remote.git +git clone /remote.git /author +``` + +Configure fixture-only author identity. In the author clone, create non-secret curated trees for at least the filesystem workspace and push the bootstrap commit: + +```text +workspace-content/p1-filesystem/evidence/guide.md +workspace-content/p1-filesystem/evidence/domain/table.md +``` + +The API, not the fixture writer, publishes `workspaces/*.yaml` and `workspace-docs/*`. Additional HTTP/S3 descriptor fixtures may share the same repository but do not pretend to be acquired. + +Create file bindings under `fixture-secrets/` for: + +- the normal DWH config required by the renderer/config loader; +- one signed-HTTP JSON list containing a unique query canary; +- S3 access/secret/session values containing distinct canaries. + +Set only file-path environment bindings and configure `THT_WORKSPACE_SECRET_ROOTS` to the fixture secret directory. Provision the required DWH transport/host/port/user/password-file variables separately for the `P1_FILESYSTEM`, `P1_HTTP`, and `P1_S3` contract namespaces (they may reference one shared fixture password file); then add the source-specific Evidence variables. Capture a redacted `bindings.env` containing paths and non-secret endpoints, not values. + +- [ ] **Step 5: Start production backend boundaries and use real HTTP** + +Import `loadConfig`, `buildApp`, `WorkspaceRegistry`, and `ThtRunner` from `backend/dist`. Build them with the run's remote/root/data/runtime settings and pass those production instances to `buildApp`. Listen on `127.0.0.1:0`; save only the loopback base URL. + +Perform and persist each request/response once: + +1. `GET /workspace-registry/status`; +2. positive `POST /workspaces/validate` for filesystem, signed HTTP, and static-file S3 descriptors; +3. `POST /workspaces/publish` create for each descriptor, based on the current commit returned by the preceding step/read; +4. `POST /workspace-registry/pull`; +5. `GET /workspaces/:id` and list; +6. `GET /workspaces/:id/export` for each workspace; +7. safe extraction with the production ZIP dependency and manifest/file hash verification. + +Use a sequential current-base workflow; do not blindly replay a stale base after each publication. Every recorded response must be parsed/sanitized before entering `report.json`. + +- [ ] **Step 6: Prove one immutable Git identity** + +For the filesystem workspace, assert and report hashes for: + +```text +API revision.commit +installation registry checkout HEAD +snapshot manifest commit +runtime_identity.workspace_revision +``` + +All four must be identical. Then use fixed-argv Git object checks: + +```text +git cat-file -e :workspaces/p1-filesystem.yaml +git cat-file -e :workspace-content/p1-filesystem/evidence/guide.md +git cat-file -t :workspace-content/p1-filesystem/evidence +``` + +The last output must be `tree`. Also assert the immutable snapshot contains descriptor/docs/manifest but **not** a materialized `workspace-content` tree. + +Fast-forward the curator clone to the API publication HEAD, then create and push a content-only update to `guide.md`; invoke the registry pull once and prove: + +- active revision changes to the new commit; +- descriptor blob stays equal; +- runtime identity/root changes to the new commit; +- old retained snapshot remains immutable. + +This is the regression that prevents blob identity from masquerading as revision identity. + +- [ ] **Step 7: Exercise production rendering and the real harness check** + +For each workspace snapshot: + +1. call the production `ThtRunner.acquireWorkspaceRuntime`; +2. copy the lease YAML into `rendered/-1.yaml`; +3. execute `harness/.venv/bin/tht config check -c ` exactly once; +4. release the lease; +5. repeat the acquire/check as an explicit determinism/idempotence check, not a retry; +6. copy `-2.yaml` and compare bytes; +7. assert identity/source/policy fields and that all leased files were released. + +The HTTP/S3 commands parse file credentials but never construct adapters or touch network. The filesystem root is allowed not to exist because P6 has not materialized it. Any Evidence acquisition call is a gate failure. + +- [ ] **Step 8: Execute negative scenarios with no mutation/no leak** + +Send separate validate requests for: + +- absolute, traversal, backslash, and cross-workspace filesystem URIs; +- unsupported source type/protocol; +- descriptor credential field containing a canary; +- HTTP userinfo/query canary; +- malformed policy/limits. + +For contextual validation, use a separate invalid workspace/branch state whose canonical filesystem path is absent at the referenced commit; pull/publish must fail and preserve the last valid active snapshot. Do not damage and repair the primary scenario as a hidden retry. + +For every negative case assert: + +- expected safe status/code/field; +- Git HEAD/snapshot state unchanged where applicable; +- response/log/report contains no rejected canary or Git stderr. + +- [ ] **Step 9: Verify export/docs/determinism/secrets/cleanup and write reports** + +The final report has stable check IDs including at least: + +```text +preflight +clean_state +ownership +local_git_bootstrap +http_validate_publish_pull_read_export +same_revision_git_objects +content_only_revision +snapshot_and_docs +runtime_render_determinism +tht_config_check +negative_schema_cases +negative_context_case +no_p1_scope_artifacts +secret_scan +cleanup_confinement +``` + +`no_p1_scope_artifacts` asserts that the run contains no `artifacts/evidence`, `corpus/ACTIVE`, embedding output, Qdrant records, or preprocessing invocation. + +Scan every regular file below the run except `fixture-secrets/` for all fixture canaries and known credential values. Scan Git blobs reachable from the remote, extracted ZIPs, requests/responses, logs, YAML, JSON, and Markdown. File paths and safe variable names are allowed; values are not. + +Write `report.json` atomically, then derive `report.md` from it. End with exactly: + +```text +automated integration: PASS +manual acceptance: PENDING +``` + +when all checks pass. With `--keep`, print the absolute retained run path. Without `--keep`, validate ownership and clean only that run after printing/writing the successful result. + +- [ ] **Step 10: Run acceptance-runner tests** + +```bash +bash -n scripts/p1-acceptance.sh scripts/test-p1-acceptance.sh +bash scripts/test-p1-acceptance.sh +``` + +Expected: PASS. + +- [ ] **Step 11: Run the complete automated process once from clean state** + +Before running, verify no previous command is active and do not reuse a run root: + +```bash +./scripts/p1-acceptance.sh integration --keep +``` + +Expected: exit `0`; output names one new retained run; its `report.json` has `overall: "PASS"`; `report.md` says automated PASS/manual PENDING; no scenario has a retry/attempt count greater than one. + +If it fails: stop. Diagnose from the retained run, add/fix a regression test and implementation, then invoke a **new** full run with a new ID. Do not rerun the same failed scenario blindly and do not overwrite the old report. + +- [ ] **Step 12: Inspect retained evidence and update project state** + +Manually inspect the report plus a sample descriptor, Git object proof, snapshot manifest, generated README, extracted export, and the two rendered configs. Record the retained relative run path and: + +```text +automated integration: PASS +manual acceptance: PENDING +``` + +in `PROJECT_STATE.md`. Do not mark manual acceptance complete. + +- [ ] **Step 13: Commit** + +```bash +git add \ + scripts/p1-acceptance.sh \ + backend/scripts/p1-acceptance.mjs \ + backend/scripts/p1-acceptance.test.mjs \ + scripts/test-p1-acceptance.sh \ + PROJECT_STATE.md +git commit -m "test: prove P1 configuration process end to end" +``` + +--- + +### Task 9: Build the independent manual-acceptance tooling + +**Files:** +- Create: `scripts/p1-manual-acceptance.sh` +- Create: `backend/scripts/p1-manual-acceptance.mjs` +- Create: `backend/scripts/p1-manual-acceptance.test.mjs` +- Create: `backend/scripts/p1-render-snapshot.mjs` +- Create: `backend/scripts/p1-render-snapshot.test.mjs` +- Create: `scripts/test-p1-manual-acceptance.sh` +- Create: `docs/testing/p1-manual-acceptance.md` +- Modify after human approval only in Task 11: `PROJECT_STATE.md` + +**Public lifecycle:** + +```bash +./scripts/p1-manual-acceptance.sh prepare +./scripts/p1-manual-acceptance.sh serve +./scripts/p1-manual-acceptance.sh stop +./scripts/p1-manual-acceptance.sh cleanup +``` + +The helper supports only those four lifecycle actions. It uses the fixed, independent root `.artifacts/manual-acceptance/p1/` and backend address `http://127.0.0.1:8791`. It never reads or copies an automated integration run. + +**Manual topology:** + +```text +.artifacts/manual-acceptance/p1/ +├── ownership.json +├── backend.pid # only while served +├── remote.git/ +├── author/ +├── installation/ +├── fixture-secrets/ +├── fixtures/ +├── requests/ +├── responses/ +├── exports/ +├── rendered/ +├── logs/ +├── commands/ # generated concrete reviewer commands +├── GUIDE.md +└── VERDICT.md # created by reviewer, never by automation +``` + +The generated render commands use this tracked, acceptance-only interface (not an HTTP route and not the future P2 host renderer): + +```bash +node backend/scripts/p1-render-snapshot.mjs \ + --ownership .artifacts/manual-acceptance/p1/ownership.json \ + --snapshot \ + --output .artifacts/manual-acceptance/p1/rendered/runtime-1.yaml +``` + +The script imports the built production `ThtRunner`, reconstructs its non-secret settings from the owned manual installation, resolves descriptor bindings from the command environment, acquires one runtime lease, copies it atomically with mode `0600`, and releases the lease in `finally`. It accepts only owned snapshot/output paths under the fixed manual root; it never starts a backend, calls a render HTTP route, or reads secret contents itself. + +- [ ] **Step 1: Write failing lifecycle guard tests** + +Test without approving the gate: + +- `prepare` refuses a pre-existing root, a symlink root, automated-run input, or missing prerequisites; +- `prepare` creates fresh ownership, bare remote, author clone/content commit, installation directories, descriptor/request fixtures, secret files, output directories, commands, and guide; +- `serve` refuses unowned state, an occupied `127.0.0.1:8791`, an existing live PID, a stale/mismatched PID, or any non-loopback bind; +- `stop` signals only the PID whose ownership nonce, executable, cwd/root, and recorded start identity match; +- `cleanup` refuses while the owned server is live and removes only the exact owned fixed root after stop; +- foreign siblings and `.artifacts/p1-integration` are never removed; +- no lifecycle action writes `VERDICT.md` or changes manual status to PASS; +- the generated render commands fail safely before rendering when the saved read response is missing/malformed, its snapshot path escapes the owned installation, or its revision differs from the published Git commit; +- `p1-render-snapshot.mjs` rejects unowned/symlink/out-of-root snapshot or output paths, copies one production lease, always releases it on success/failure, writes mode `0600`, and produces byte-identical outputs for two identical invocations without leaving runtime lease files. + +- [ ] **Step 2: Run lifecycle tests and verify RED** + +```bash +bash scripts/test-p1-manual-acceptance.sh +``` + +Expected: FAIL because the helper does not exist. + +- [ ] **Step 3: Implement guarded preparation and backend-only serving** + +`prepare` must: + +- require that Task 8 has been implemented, but not consume its state; +- build backend once; +- create the fixed root exclusively and write ownership first; +- initialize a new bare remote and curator clone; +- seed a fresh filesystem Evidence tree and secret-file fixtures; +- create concrete positive/negative JSON request files; +- generate `commands/render-1.sh` and `render-2.sh` that, after the reviewer has saved the successful read response, extract `revision.snapshotPath` with a bounded Node JSON parser, verify its commit equals the saved API/Git commit and that it lies below the owned installation snapshot root, then invoke `backend/scripts/p1-render-snapshot.mjs` with concrete owned output paths/environment; also generate safe scripts for Git inspection, `diff`, config checks, ZIP extraction/manifest verification, and secret scanning; +- generate `GUIDE.md` with absolute/concrete paths and expected safe outcomes; +- leave the server stopped and status `PENDING`. + +`serve` must start only `node backend/dist/server.js` with the lab's environment, bind exactly `127.0.0.1:8791`, redirect stdout/stderr to owned logs, atomically persist PID/start identity, and perform one bounded health readiness wait. It must not start Docker or frontend. + +`stop` must validate ownership/process identity before sending TERM, wait a bounded interval, and report if the operator must intervene; it must never fall back to a broad `pkill`. `cleanup` must apply the same root/nonce/symlink checks as Task 8. + +- [ ] **Step 4: Write the permanent manual guide** + +`docs/testing/p1-manual-acceptance.md` explains prerequisites, four lifecycle commands, separation from automated state, expected outputs, how to preserve a failed lab, and the verdict format. It must state that the reviewer—not the helper—performs and judges the walkthrough. + +The generated `GUIDE.md` must contain this ordered checklist: + +1. inspect `ownership.json`, the pre-publication Evidence tree, fixture descriptor, and binding paths; +2. run `serve` and verify only `127.0.0.1:8791` listens; +3. personally execute real `curl` status → validate → publish → pull → read → export calls, saving each response; +4. only after publish, use `git log`, `git ls-tree`, and `git show :workspaces/.yaml` plus `git show :workspace-content//evidence/...` to inspect descriptor/content identity at that one commit; +5. inspect generated `workspace-docs`, immutable descriptor snapshot, and snapshot manifest; +6. safely extract ZIP and verify manifest hashes and absence of Evidence bytes/secrets; +7. run the generated production render command twice and `diff` the YAML; +8. inspect runtime identity, absolute reserved filesystem root, Evidence limits, and policy; +9. personally execute `harness/.venv/bin/tht config check -c ` and the second config; +10. submit invalid absolute/traversal/cross-workspace/protocol/credential requests and verify safe rejection/no mutation/no canary; +11. run the generated secret scan outside `fixture-secrets`; +12. confirm no preprocessing, Evidence materialization, embedding, Qdrant, ACTIVE, or retention artifact exists; +13. run `stop` and confirm the PID/port are gone; +14. record `VERDICT.md` with reviewer, UTC time, every checklist result, observations, and either `manual acceptance: PASS` or `manual acceptance: FAIL`. + +The guide must not tell the reviewer to inspect raw secret file contents. It may verify file ownership/mode and canary absence outside the excluded directory. + +- [ ] **Step 5: Run lifecycle tests and syntax checks** + +```bash +bash -n \ + scripts/p1-manual-acceptance.sh \ + scripts/test-p1-manual-acceptance.sh +bash scripts/test-p1-manual-acceptance.sh +``` + +Expected: PASS. This proves tooling only; it does **not** approve manual acceptance. + +- [ ] **Step 6: Commit the manual gate tooling** + +```bash +git add \ + scripts/p1-manual-acceptance.sh \ + backend/scripts/p1-manual-acceptance.mjs \ + backend/scripts/p1-manual-acceptance.test.mjs \ + backend/scripts/p1-render-snapshot.mjs \ + backend/scripts/p1-render-snapshot.test.mjs \ + scripts/test-p1-manual-acceptance.sh \ + docs/testing/p1-manual-acceptance.md +git commit -m "test: add P1 manual configuration walkthrough" +``` + +--- + +### Task 10: Re-run branch-wide verification and hand off explicit gate status + +**Files:** +- Modify only if status/path changes: `PROJECT_STATE.md` + +This task runs after all implementation/tooling commits and immediately before the human walkthrough. It must leave manual acceptance PENDING. + +- [ ] **Step 1: Run the complete backend gate** + +```bash +cd backend +npx vitest run +npx tsc --noEmit -p . +npm run build +``` + +Expected: all tests, typecheck, and build PASS. + +- [ ] **Step 2: Run the complete harness gate** + +```bash +cd harness +.venv/bin/pytest -q +.venv/bin/ruff check . +``` + +Expected: PASS under the repository's default marker selection. Do not enable L2 or external services for P1. + +- [ ] **Step 3: Run the complete frontend gate** + +```bash +cd frontend +npx vitest run +npx tsc -b +npm run build +``` + +Expected: PASS. + +- [ ] **Step 4: Run every root verifier/tooling test** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/verify-workspace-install-docs.sh --fixtures-only +bash scripts/test-p1-acceptance.sh +bash scripts/test-p1-manual-acceptance.sh +bash -n \ + scripts/p1-acceptance.sh \ + scripts/p1-manual-acceptance.sh \ + scripts/test-p1-acceptance.sh \ + scripts/test-p1-manual-acceptance.sh +``` + +Expected: PASS. + +- [ ] **Step 5: Execute one final clean automated run at branch HEAD** + +```bash +./scripts/p1-acceptance.sh integration --keep +``` + +Expected: a new run ID, exit `0`, all report checks PASS, secret scan PASS, cleanup confinement PASS, and `manual acceptance: PENDING`. No human walkthrough has occurred in this plan sequence yet. + +Do not reuse Task 8's run as the final evidence after later commits. If this command fails, follow the diagnose/fix/new-clean-run rule; never loop it automatically. + +- [ ] **Step 6: Inspect status and diff integrity** + +```bash +git diff --check +git status --short --branch +git log --oneline --decorate -12 +``` + +Expected: + +- `git diff --check` exits zero; +- no `.artifacts` file is tracked; +- no secret/example canary is present in tracked files; +- implementation commits are small and correspond to plan tasks; +- `PROJECT_STATE.md` names the latest retained automated run and reports manual status truthfully. + +If only the retained run path/status changed, commit that state: + +```bash +git add PROJECT_STATE.md +git commit -m "docs: finalize P1 verification status" +``` + +- [ ] **Step 7: Report completion with two independent gates** + +The pre-walkthrough handoff must state, on separate lines: + +```text +automated integration: PASS — +manual acceptance: PENDING — awaiting the independent reviewer walkthrough +``` + +Also state explicitly: + +- P1 proves configuration, same-revision Git identity, rendering, and harness parsing; +- P1 does not prove acquisition/materialization/preprocessing/indexing/ACTIVE/GC; +- P6 is the next required step before filesystem Evidence can be consumed; +- P2/P4/P9/P10 retain their documented responsibilities. + +Do not summarize P1 as fully accepted when manual status is PENDING or FAIL. + +--- + +### Task 11: Manual acceptance gate — descriptor and rendered configuration artifacts + +**Files:** +- Read: `.artifacts/manual-acceptance/p1/GUIDE.md` +- Create by reviewer only: `.artifacts/manual-acceptance/p1/VERDICT.md` +- Modify after explicit human approval only: `PROJECT_STATE.md` + +- [ ] **Step 1: Stop and request the human checkpoint** + +Only after Task 10 reports automated PASS at branch HEAD, ask the reviewer to execute: + +```bash +./scripts/p1-manual-acceptance.sh prepare +./scripts/p1-manual-acceptance.sh serve +# follow .artifacts/manual-acceptance/p1/GUIDE.md in full +./scripts/p1-manual-acceptance.sh stop +``` + +This is a controlled, resumable pause. If access/session is interrupted, leave the owned root intact, rerun only `serve` if the guide says no stateful scenario has begun, or use `cleanup` then `prepare` for a genuinely fresh walkthrough. Never infer approval from automated outputs. + +- [ ] **Step 2: Record the human result without hiding failures** + +If `VERDICT.md` says FAIL or is absent, keep: + +```text +automated integration: PASS +manual acceptance: PENDING (or FAIL with observation) +``` + +in `PROJECT_STATE.md` and preserve the manual root for diagnosis. + +Only if the reviewer explicitly records PASS, update `PROJECT_STATE.md` with reviewer/date and: + +```text +automated integration: PASS +manual acceptance: PASS +``` + +Then optionally clean the lab after the reviewer confirms artifacts are no longer needed: + +```bash +./scripts/p1-manual-acceptance.sh cleanup +``` + +Commit only the project-state update, never `.artifacts`: + +```bash +git add PROJECT_STATE.md +git commit -m "docs: record P1 manual acceptance" +``` + +--- + +## Requirement traceability + +| Requirement/decision | Implemented/proven by | +|---|---| +| RF1.1 / D1 complete source + policy | Tasks 1, 3, 4, 5, 7 | +| RF1.2 / RNF1 no secrets in Git | Tasks 1, 3, 5, 7, 8 | +| RF1.3 same runtime rendering | P1 prerequisite only: Task 4 and Task 8 prove backend session render → harness parse; P2 must prove its backend-independent host render is equivalent before RF1.3 is complete | +| RF1.4 three DWH transports represented | Existing descriptor/contract retained; SSH remains fail-closed for P10; regression tests Tasks 3–4 | +| RF1.5 one registry, namespace isolation, same revision | Tasks 1–2 and Git-object proof Task 8 | +| RNF2 deterministic/idempotent relevant outputs | Tasks 4–5, Task 8 repeated read/render/config-check/content-only revision | +| RNF3 workspace without Evidence still works | Task 1 optional field and Tasks 4/6 regressions | +| RNF4 workspace isolation | Lexical namespace + exact Git tree checks Tasks 1–2; runtime point filtering remains outside P1 | +| RNF5 renderer compatibility | Task 4 production handoff and harness check | +| RNF7 one canonical path | Schema/renderer only; no parallel fixture contract | +| RNF8 integration-first | Automated Task 8 and branch-wide Task 10 before human Task 11 | +| D6/P6 boundary | No materialization/realpath/symlink claim; reserved commit root only | +| D9 policy defaults | Tasks 1 and 4; execution/GC remains P9 | +| Automated acceptance standard | Task 8 clean state, no retry, reports, secret scan, cleanup | +| Manual acceptance standard | Task 9 independent tooling plus Task 11 explicit resumable human checkpoint | + +## Execution notes + +- Every task is red → green → focused verification → commit. Do not batch several red tasks into one implementation change. +- Use existing local bare-Git fixtures and production interfaces rather than mocks where a boundary already exists. +- A test-only fake is acceptable only for an external dependency that P1 explicitly excludes; the core P1 path itself must remain real Git, real Fastify HTTP, production registry/renderer, and real harness CLI. +- Keep failed acceptance artifacts. Fix the cause with a regression test, then start a new clean run. “Try it again” is not a diagnostic step. +- Human approval is a durable decision, not a command exit code. The manual helper must never create a PASS verdict. diff --git a/docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md b/docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md new file mode 100644 index 00000000..a4f872df --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md @@ -0,0 +1,593 @@ +# P2 Host Workspace Preprocessing CLI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement P2/D2 as a native `thothctl workspace` interface that runs the existing preprocessing engine in a hardened one-shot container, derives its configuration from one active schema-v3 Git workspace revision plus installation-local bindings, and requires no Python, Node, Pi, or running backend HTTP service on the host. + +**Architecture:** `thothctl` validates a closed command grammar, reconstructs the exact installation Compose project, resolves the selected core image to an immutable Docker image ID, and starts only the profile-gated `workspace-maintenance` service with `--no-deps`. A compiled Node entrypoint reads an already-active immutable registry snapshot, uses the same binding resolver and runtime renderer as sessions, writes a deterministic revision-owned protected harness config, and invokes fixed existing `tht` commands. A versioned coordinator state and one kernel-released writer lock serialize mutation, preserve outer/child resume identity, and stop at a digest-bound FK review checkpoint. + +**Tech Stack:** Go 1.24 (`thothctl`), Docker Compose v2, Node.js 22, TypeScript 5, Python 3.12, Typer, Pydantic 2, Qdrant 1.18.2, the internal Ollama-compatible embedding interface, Vitest, pytest, Bash/Node acceptance tooling. + +**Source PRD and design:** `docs/prd/2026-08-09-workspace-preprocessing-prd.md` D2/P2, RF1.2–RF1.4, RF2, RF3.1, RF4.1, RF5.2, RF8.5–RF8.6, RNF1–RNF9; `docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md` §§1–4, 9–10; `docs/testing/p2-p6-manual-verification.md` P2. + +**Planning status:** DESIGN/PLAN ONLY. Do not change production code, start P2 implementation, or create a persistent implementation goal until the reviewer asks for plan validation and then gives explicit implementation approval. + +--- + +## P2 completion contract + +P2 is complete only when all of the following are true: + +1. The only public host interface is the installed native `thothctl` binary. Docker/Compose is required, but host Python, Node, Pi, `tht`, and a running Fastify backend are not. +2. Every command consumes an already-active, validated P1.1 registry snapshot and binds the exact workspace ID, 40-hex commit, catalog blob (`thoth-workspaces.yaml`), descriptor blob/digest, installation bindings, runtime roots, and selected internal semantic contract before mutation. A docs-only or content-only commit is still a distinct revision even when the descriptor blob is unchanged, because the commit is authoritative. +3. Operator and session configuration use the same `resolveRuntimeBindings` and `renderRuntimeConfig` implementation. P2 uses one deterministic same-revision config-source path so current schema-v1 DWH/Evidence resume works; P3 later introduces cross-revision canonical effective identity and explicit migrations. +4. DWH introspection+LSH, FK suggestion/check, schema indexing, and HTTP Evidence preprocessing invoke the existing harness engine through fixed argv and pristine JSON machine interfaces. No second preprocessing engine is added. +5. A full run with new FK candidates stops before schema/Evidence writes. Continuation requires a reviewer-supplied annotations file and an explicit acknowledgement of the exact candidate digest; `schema check` alone is not treated as human approval. +6. Mutating P2 operations are safe while roots and semantic point IDs are still workspace-global: under the workspace writer lock they refuse if any resumable session is pinned to a different workspace revision. P3 removes this temporary restriction by introducing revision-scoped curated/semantic state. +7. P2 never creates, repairs, deletes, or rebuilds a Qdrant collection. Schema/Evidence writes require an already-existing, exactly compatible collection and a harness `require_existing` mode that cannot race into auto-create. P4 owns lifecycle reconciliation. +8. HTTP Evidence is operational only under installation-local egress policy. Private hosts require an exact installation allowlist; redirects are rechecked; metadata/link-local targets are always refused. S3 custom/private/insecure endpoints and ambient credentials remain fail-closed in P2 unless a later separately reviewed plan expands policy. +9. Filesystem Evidence is rendered but execution stops before discovery with `evidence_materialization_required` and no partial corpus/vector publication. P6 owns materialization and symlink/containment checks. +10. `postgres_direct` and `rest_api` routing remain supported and are regression-tested; the clean P2 process goal uses controlled REST. `ssh_tunnel` returns a stable fail-closed result until P10. +11. One clean-state product-path integration command passes without retry, produces retained machine/human reports and a secret scan, and proves exact cleanup. Manual P2 acceptance remains independent and PENDING. +12. Work stops after the P2 handoff. No P3 work begins without a new explicit user authorization. + +## Truthful command status at the P2 checkpoint + +| Command | P2 status | Deliberate boundary | +|---|---|---| +| `workspace inspect` | Operational | Reads active snapshot only; does not pull/activate Git | +| `workspace preprocess dwh` | Operational for REST/direct | Same-revision config identity; cross-revision reuse is P3 | +| `workspace schema suggest-fks` | Operational, machine-safe | Candidate export only; no automatic human acceptance | +| `workspace schema check` | Operational | Imports reviewed annotations and records digest-bound local P2 acknowledgement | +| `workspace index-schema` | Operational with compatible pre-existing collection | Collection create/repair/rebuild is P4 | +| `workspace preprocess evidence` | Operational for policy-allowed HTTP; filesystem deferred | Filesystem materialization is P6; S3 expansion needs separate policy review | +| `workspace preprocess run` | Operational with FK checkpoint | Git-canonical annotations are P5; revision-global writes use the P2 session-inventory guard | + +P2 is therefore the host CLI/orchestration checkpoint, not final acceptance of the PSD filesystem path or the complete PRD chain. + +## Frozen host command grammar + +```text +thothctl --installation /thothii-installation.yaml workspace inspect + --workspace [--json] + +thothctl ... workspace preprocess dwh + --workspace [--resume ] [--json] + +thothctl ... workspace schema suggest-fks + --workspace + [--from-sql ]... [--assume ]... + [--output ] [--json] + +thothctl ... workspace schema check + --workspace + [--annotations --reviewed-candidates ] + [--json] + +thothctl ... workspace index-schema + --workspace [--json] + +thothctl ... workspace preprocess evidence + --workspace [--dry-run] [--resume ] [--json] + +thothctl ... workspace preprocess run + --workspace [--resume ] [--json] +``` + +Rules: + +- `--workspace` occurs exactly once and matches `[a-z][a-z0-9-]{2,62}`. +- All run IDs are 32 lowercase hex characters and identify outer P2 state, never a path or raw child checkpoint. +- At most 32 `--from-sql` files, 1 MiB each and 16 MiB total. `thothctl` opens each as a canonical regular non-symlink/reparse-point file, rechecks identity after reading, and streams a schema-versioned request over stdin. No host directory is mounted. +- `--assume` occurs at most 256 times; each value is at most 256 bytes and is validated before Compose. +- `--annotations` is a single UTF-8 YAML file, at most 16 MiB. `--reviewed-candidates` is mandatory with it and must equal the persisted candidate artifact digest. The pair is invalid without both flags. +- `--output` is created exclusively with restrictive permissions after the returned workspace/run/digest identity has been verified. Existing files, symlinks, hardlinks, and Windows reparse targets are refused. +- Existing harness `suggest-fks --write` is intentionally not exposed: an automatic merge is not a human review decision. +- No unknown flag, passthrough separator, environment-selected command, shell fragment, or arbitrary container entrypoint is accepted. + +## Public result and exit contract + +The one-shot entrypoint always emits exactly one bounded schema-versioned JSON object. `thothctl --json` parses it strictly and re-encodes it, so Compose progress cannot contaminate stdout. Human mode renders only allowlisted fields. + +```ts +interface WorkspaceOperationResult { + schemaVersion: 1; + status: "succeeded" | "unchanged" | "dry_run" | "blocked" | "failed"; + code: + | "ok" | "workspace_not_found" | "workspace_not_activatable" + | "binding_missing" | "preprocessing_conflict" + | "preprocessing_resume_mismatch" | "manual_review_required" + | "evidence_materialization_required" | "effective_config_mismatch" + | "semantic_index_incompatible" | "annotation_invalid" + | "egress_policy_refused"; + workspaceId: string; + workspaceRevision: string; + descriptorBlob: string; + operation: string; + runId?: string; + childRuns?: Record; + completedStages: string[]; + counts?: Record; + artifactIdentities?: Array<{ kind: string; digest: string }>; + warnings?: string[]; +} +``` + +- Exit `0`: `succeeded`, `unchanged`, or `dry_run`. +- Exit `3`: expected operator checkpoint/block (`manual_review_required`, `evidence_materialization_required`, lock/revision conflict). +- Exit `2`: host grammar or unsafe local file error. +- Exit `1`: operational failure. +- Stdout JSON maximum: 1 MiB. Sanitized stderr maximum: 64 KiB. Child stdout/stderr and every stage have explicit limits/timeouts. +- Never return descriptor endpoints with credentials/query strings, secret contents or paths, signed URLs, raw SQL, rendered configuration, raw child stderr, arbitrary exception text, Qdrant payload contents, or host/container environment dumps. + +## P2 state and identity layout + +```text +/data/sessions//preprocessing/ +├── writer.lock +├── runtime-config/ +│ └── <40-hex-revision>.yaml +├── runtime-config-manifests/ +│ └── <40-hex-revision>.json +├── jobs/ +│ └── <32-hex-outer-run-id>.json +├── fk-candidates/ +│ └── <32-hex-outer-run-id>.yaml +└── fk-reviews/ + └── <32-hex-outer-run-id>.json +``` + +- `writer.lock` is a regular `0600` file held by a Linux kernel advisory lock for the entire outer operation. The file may persist; the kernel lock is released on crash/container death. `inspect` never takes it. +- Lock order is always P2 workspace writer lock → existing harness stage lock. Harness code never acquires the P2 lock, preventing inversion/deadlock. +- Every directory component is opened/validated without following symlinks. State files are `0600`, written to an exclusive sibling, fsynced, renamed, and parent-fsynced. Hardlink count must be one. +- The deterministic config path fixes P2 same-revision `config_source` identity. Its manifest binds workspace, revision, descriptor blob, config SHA-256, file identity, and the current existing harness ownership binding. Same path + different bytes returns `effective_config_mismatch`; P3 introduces semantic cross-revision equivalence. +- Job state binds operation, revision, catalog blob, descriptor blob, config digest, non-secret binding identity, completed stage records, child run IDs, candidate/review digests, and terminal status. Resume revalidates all fields and reconciles a child publication that completed immediately before an outer-state crash. +- Before any schema/Evidence mutation, enumerate resumable session manifests for the workspace. A different pinned revision returns `preprocessing_conflict`; no write begins. This is the explicit P2 bridge until P3 revision isolation. + +## One-shot service security contract + +`workspace-maintenance` is a dedicated profile service, not `compose run core`: + +- same exact selected core image, resolved to its immutable local image ID; generated final override uses that ID and `pull_policy: never`; +- `docker compose run --rm --no-deps --no-TTY --name workspace-maintenance ...`; +- no `build`, frontend, published port, Pi auth, Pi state, Pi trust initialization, Docker socket, home credential directory, or arbitrary command; +- non-root `10001`, `read_only: true`, `cap_drop: [ALL]`, `no-new-privileges:true`, restrictive tmpfs, registry active snapshots read-only, sessions root writable; +- only operation-required connector/Evidence secret files are mounted; AWS ambient environment is cleared; +- semantic commands require already-running healthy Qdrant/embedding services and do not start/stop them; DWH/inspect commands do not start dependencies; +- exact operation labels and container identity are recorded. Cancellation terminates the process group, verifies the owned container labels/image, removes only that container, and preserves all pre-existing services, volumes, and networks; +- fully rendered Compose is validated before launch, and post-run container/image identity is checked before accepting output. + +--- + +## Target file map + +**Native host CLI** + +- `tools/thothctl/cmd/thothctl/main.go`, `main_test.go`: public grammar/help, dispatch, exit codes. +- Create `tools/thothctl/internal/workspaceops/operations.go`, `operations_test.go`: immutable image resolution, generated override, Compose run, cancellation cleanup, JSON validation. +- `tools/thothctl/internal/config/installation.go`, `installation_test.go`: maintenance service override and operation-specific binding discovery. +- `tools/thothctl/internal/safeio/files.go`, platform files/tests: bounded no-follow input and exclusive output. +- `tools/thothctl/internal/output/sanitize.go`, tests: bounded redaction. +- `tools/thothctl/internal/pi/update.go`, tests: selected image override must pin both `core` and `workspace-maintenance`. + +**Compose/image boundary** + +- `compose.yaml`: dedicated profile-gated service with shared image identity and least privilege. +- `deploy/compose.local.yaml`, `deploy/compose.server.yaml`: correct registry/session storage semantics. +- `deploy/compose.git-https.yaml`, `deploy/compose.git-ssh.yaml`: do not attach Git credentials to P2 active-snapshot operations. +- `scripts/generate-connector-secrets-override.sh`: operation-specific maintenance secrets. +- Create `docker/workspace-maintenance-entrypoint.sh`; modify `docker/core.Dockerfile`. +- Retire/redirect fixture-only `deploy/compose.preprocess.yaml` as a non-public compatibility test path; do not leave two operator commands. + +**Shared Node operator** + +- Create `backend/src/workspaces/runtime-config-lease.ts`: shared snapshot read/render and deterministic protected config lease. +- Modify `backend/src/tht/tht-runner.ts` to delegate session/operator rendering to the shared component without changing route behavior. +- Create `backend/src/workspaces/preprocessing-state.ts`: state schema, durable writes, locks, resume reconciliation. +- Create `backend/src/workspaces/preprocessing-service.ts`: closed stage coordinator and security preflights. +- Create `backend/src/workspace-maintenance.ts`: compiled stdin/argv entrypoint and pristine result encoder. +- Add tests: `backend/test/workspace-runtime-config-lease.test.ts`, `workspace-preprocessing-state.test.ts`, `workspace-preprocessing-service.test.ts`, `workspace-maintenance.test.ts`. + +**Harness machine contracts** + +- `harness/tht/cli/preprocess_cmd.py`: authoritative runtime workspace identity; require-existing collection mode. +- `harness/tht/cli/schema_cmd.py`: extracted deterministic helpers, JSON suggest/check, safe SQL staging and annotation validation. +- `harness/tht/cli/vector_cmd.py`: JSON schema-index result. +- `harness/tht/adapters/vector/qdrant.py`: explicit non-creating strict mode for P2. +- Tests: `harness/tests/test_preprocess_cli.py`, `test_schema_fk_annotations.py`, `test_qdrant_cli_commands.py`, `test_registry_evidence_config.py`, `test_http_evidence_source.py`, plus new focused security cases. + +**Docs and gates** + +- Create `docs/contracts/workspace-preprocessing-cli.md`. +- Update local/server installation manuals and `docs/testing/p2-p6-manual-verification.md` P2 only. +- Create `scripts/p2-acceptance.sh`, `backend/scripts/p2-acceptance.mjs`, `backend/scripts/p2-acceptance.test.mjs`. +- Create `scripts/p2-manual-acceptance.sh`, `backend/scripts/p2-manual-acceptance.mjs` only if needed to generate the isolated walkthrough lab; automation must never create PASS. +- Update `PROJECT_STATE.md` only after implementation evidence exists. + +--- + +### Task 1: Freeze the native CLI, file-ingress, and result contracts + +**Files:** +- Modify: `tools/thothctl/cmd/thothctl/main.go` +- Modify: `tools/thothctl/cmd/thothctl/main_test.go` +- Modify: `tools/thothctl/internal/safeio/files.go` +- Modify platform-specific safe-I/O tests +- Create: `tools/thothctl/internal/workspaceops/operations.go` +- Create: `tools/thothctl/internal/workspaceops/operations_test.go` +- Create: `docs/contracts/workspace-preprocessing-cli.md` + +- [ ] **Step 1: Write RED parser-table tests** for every valid command above and for duplicate/missing/unknown flags, invalid IDs, incompatible annotation flags, option-count/size limits, and passthrough/shell attempts. +- [ ] **Step 2: Run** `cd tools/thothctl && go test ./cmd/thothctl ./internal/workspaceops -run 'Workspace|workspace' -v` and verify the new tests fail because `workspace` is unknown. +- [ ] **Step 3: Add closed request types** (`InspectRequest`, `DwhRequest`, `SuggestFksRequest`, `CheckSchemaRequest`, `IndexSchemaRequest`, `EvidenceRequest`, `RunRequest`) and a parser that cannot represent arbitrary argv. +- [ ] **Step 4: Write RED safe-I/O tests** for symlinks, hardlinks, directory input, replacement during read, Windows reparse points, existing output, >1 MiB SQL, >16 MiB total, and non-UTF-8 annotation input. +- [ ] **Step 5: Implement bounded reads and exclusive restrictive output** using existing platform seams; return only generic file errors. +- [ ] **Step 6: Add schema-v1 stdin request/result validation** with exact field allowlists and output bounds. +- [ ] **Step 7: Run focused Go tests and `gofmt -w`**, then `go test ./...`. +- [ ] **Step 8: Commit:** `feat: define P2 host workspace command contract`. + +### Task 2: Add pristine harness JSON interfaces without changing the engine + +**Files:** +- Modify: `harness/tht/cli/schema_cmd.py` +- Modify: `harness/tht/cli/vector_cmd.py` +- Modify: `harness/tht/cli/preprocess_cmd.py` +- Modify: `harness/tests/test_schema_fk_annotations.py` +- Modify: `harness/tests/test_qdrant_cli_commands.py` +- Modify: `harness/tests/test_preprocess_cli.py` + +- [ ] **Step 1: Write RED tests** requiring `schema suggest-fks --json`, `schema check --json`, and `vector index-schema --json` to emit exactly one JSON object on stdout for success and failure, with no color/prose contamination. +- [ ] **Step 2: Write RED deterministic FK tests** for bounded staged SQL files, stable candidate ordering, candidate SHA-256, annotation import, orphan counts, and no implicit review/write. +- [ ] **Step 3: Write RED Evidence identity test** showing a config named `/dev/fd/3` still uses `runtime_identity.workspace_id`, never the config basename. +- [ ] **Step 4: Run:** + +```bash +cd harness +.venv/bin/pytest -q \ + tests/test_schema_fk_annotations.py \ + tests/test_qdrant_cli_commands.py \ + tests/test_preprocess_cli.py +``` + +Expected: FAIL only on the new machine-contract assertions. + +- [ ] **Step 5: Extract pure helpers** returning typed dictionaries/models; keep existing human commands as renderers over the same helpers. +- [ ] **Step 6: Implement the JSON flags and authoritative workspace identity**. Catch expected exceptions and emit stable safe codes; never serialize arbitrary exception text. +- [ ] **Step 7: Run the three focused files and touched Ruff**: + +```bash +.venv/bin/ruff check \ + tht/cli/schema_cmd.py tht/cli/vector_cmd.py tht/cli/preprocess_cmd.py \ + tests/test_schema_fk_annotations.py tests/test_qdrant_cli_commands.py tests/test_preprocess_cli.py +``` + +- [ ] **Step 8: Commit:** `feat: add P2 harness machine contracts`. + +### Task 3: Add a non-creating semantic writer mode + +**Files:** +- Modify: `harness/tht/config.py` +- Modify: `harness/tht/adapters/factory.py` +- Modify: `harness/tht/adapters/vector/qdrant.py` +- Modify: `harness/tests/test_qdrant_vector_store.py` +- Modify: `harness/tests/test_qdrant_cli_commands.py` +- Modify: `harness/tests/test_registry_evidence_config.py` + +- [ ] **Step 1: Write RED tests** proving operator mode refuses a missing collection without issuing create/index mutations, refuses wrong dimensions/distance/index type, and still writes to an existing compatible collection. +- [ ] **Step 2: Run the focused tests** and confirm current `_ensure_collection(strict=True)` incorrectly creates the collection. +- [ ] **Step 3: Add an internal rendered field** such as `vectors.collection_lifecycle: require_existing`; it is not a descriptor option and defaults to legacy behavior for non-operator configs. +- [ ] **Step 4: Thread the mode through the factory/store** and perform a read-only exact collection/index preflight before any upsert. +- [ ] **Step 5: Add a race regression**: delete the collection after preflight and prove the write fails rather than recreates it. +- [ ] **Step 6: Run focused pytest and touched Ruff.** +- [ ] **Step 7: Commit:** `fix: prevent P2 from owning Qdrant lifecycle`. + +### Task 4: Extract the shared runtime configuration lease + +**Files:** +- Create: `backend/src/workspaces/runtime-config-lease.ts` +- Create: `backend/test/workspace-runtime-config-lease.test.ts` +- Modify: `backend/src/tht/tht-runner.ts` +- Modify: `backend/test/tht-runner.test.ts` +- Modify: `backend/test/workspace-runtime-handoff.test.ts` + +- [ ] **Step 1: Write RED equivalence tests** feeding the same immutable snapshot, env, roots, installation overlay, and semantic contract to the session and operator callers and requiring byte-identical YAML. +- [ ] **Step 2: Write RED identity/safety tests** for snapshot replacement, wrong commit/path, symlink/hardlink, wrong workspace ID, unstable config destination, same-revision changed bytes, mode, fsync/rename failure, and cleanup. +- [ ] **Step 3: Run:** + +```bash +cd backend +npx vitest run \ + test/workspace-runtime-config-lease.test.ts \ + test/tht-runner.test.ts \ + test/workspace-runtime-handoff.test.ts +``` + +- [ ] **Step 4: Move snapshot validation, runtime roots, installation-overlay parsing, binding resolution, and rendering** out of `ThtRunner` into one explicit-input component. +- [ ] **Step 5: Preserve session behavior**: `ThtRunner.acquireWorkspaceRuntime` delegates to the component and retains its current opaque FD-backed temporary lease. +- [ ] **Step 6: Add operator mode**: deterministically publish `/data/sessions//preprocessing/runtime-config/.yaml` plus a manifest, mode `0400/0600`, and set `collection_lifecycle: require_existing`. +- [ ] **Step 7: Prove same-revision rerun path identity** and changed config/binding refusal. Do not implement P3 semantic cross-revision canonicalization. +- [ ] **Step 8: Run focused tests, `npx tsc --noEmit -p .`, and `npm run build`.** +- [ ] **Step 9: Commit:** `refactor: share registry runtime configuration leases`. + +### Task 5: Build durable outer state, locking, and revision guard + +**Files:** +- Create: `backend/src/workspaces/preprocessing-state.ts` +- Create: `backend/test/workspace-preprocessing-state.test.ts` +- Modify: `docker/core.Dockerfile` (install/pin the kernel lock utility only when the implementation proves it is absent) + +- [ ] **Step 1: Write RED state-schema tests** for valid state, same-operation resume, cross-workspace/revision/operation/config mismatch, tampering, run-ID traversal, restrictive modes, atomic failure, and bounded fields. +- [ ] **Step 2: Write RED cross-process lock tests** with two processes/containers: one wins, one receives `preprocessing_conflict`, and SIGKILL releases the kernel lock without deleting unrelated state. +- [ ] **Step 3: Write RED session-inventory tests**: no sessions/current-only sessions permit mutation; a resumable different-revision manifest blocks; finalized/archived sessions follow existing resume policy. +- [ ] **Step 4: Implement the exact state layout and durable write protocol** described above. +- [ ] **Step 5: Implement lock acquisition ordering and safe conflict mapping.** Do not invent stale-PID deletion; the kernel owns lock lifetime. +- [ ] **Step 6: Implement active-snapshot revalidation immediately before each mutating child stage** and the different-revision resumable-session guard. +- [ ] **Step 7: Add crash reconciliation tests** where a child publishes DWH/corpus state but outer state has not yet advanced. +- [ ] **Step 8: Run focused Vitest, typecheck, and build.** +- [ ] **Step 9: Commit:** `feat: add P2 preprocessing operation state`. + +### Task 6: Build the compiled inspect/operator boundary + +**Files:** +- Create: `backend/src/workspace-maintenance.ts` +- Create: `backend/src/workspaces/preprocessing-service.ts` +- Create: `backend/test/workspace-maintenance.test.ts` +- Create: `backend/test/workspace-preprocessing-service.test.ts` +- Modify: `backend/src/workspaces/types.ts` only if a separate operator-code union cannot stay private + +- [ ] **Step 1: Write RED entrypoint process tests** for exact JSON, malformed/extra stdin, unknown command/field, stdout/stderr bounds, timeout, signal, raw exception/stderr redaction, and no Fastify listener. +- [ ] **Step 2: Write RED `inspect` tests** for absent/corrupt/stale active state, migration-required descriptor, missing bindings, exact commit/blob/config identities, safe capability warnings, and no URL/secret output. +- [ ] **Step 3: Run focused Vitest** and verify no operator exists. +- [ ] **Step 4: Implement a closed `WorkspacePreprocessingService` dependency interface**: active registry reader, shared config lease, fixed child runner, state store, session inventory, semantic preflight, egress policy. +- [ ] **Step 5: Implement active-snapshot-only acquisition.** P2 does not pull or activate Git; clean installations receive `workspace_not_activatable` with safe instructions. +- [ ] **Step 6: Implement bounded child execution** with fixed executable/argv, `-c` after the subcommand, FD-backed config/input, process-group cancellation, per-stage timeout, and strict one-document child JSON parsing. +- [ ] **Step 7: Implement `inspect` and result encoding.** +- [ ] **Step 8: Run tests, typecheck, build, and verify `dist/workspace-maintenance.js` exists.** +- [ ] **Step 9: Commit:** `feat: add P2 workspace maintenance operator`. + +### Task 7: Implement DWH preprocessing and outer resume + +**Files:** +- Modify: `backend/src/workspaces/preprocessing-service.ts` +- Modify: `backend/test/workspace-preprocessing-service.test.ts` +- Modify: `harness/tests/test_dwh_preprocess_job.py` +- Modify: `harness/tests/test_lsh_job_resume.py` + +- [ ] **Step 1: Write RED service tests** requiring fixed `preprocess dwh --steps introspect,lsh --json -c /dev/fd/N`, child result validation, outer/child run IDs, completed stages, safe artifact digests, and failure mapping. +- [ ] **Step 2: Add real harness regressions** for deterministic same-revision config-source path, second clean-process rerun, resume after introspection, changed binding/config refusal, and ACTIVE preservation on failure. +- [ ] **Step 3: Run focused backend and harness tests.** +- [ ] **Step 4: Implement `preprocess dwh`** under the outer writer lock and persist state before/after every child transition. +- [ ] **Step 5: Reconcile a published child run after an injected outer crash** without rerunning or corrupting ACTIVE. +- [ ] **Step 6: Verify REST and direct rendered routing.** SSH returns `workspace_not_activatable` with a P10 warning. +- [ ] **Step 7: Run focused gates and commit:** `feat: run workspace DWH preprocessing from thothctl operator`. + +### Task 8: Implement FK candidate export and digest-bound review + +**Files:** +- Modify: `backend/src/workspaces/preprocessing-service.ts` +- Modify: `backend/src/workspaces/preprocessing-state.ts` +- Modify: relevant backend tests +- Modify: `tools/thothctl/internal/workspaceops/operations.go` +- Modify: Go tests + +- [ ] **Step 1: Write RED end-to-end unit/process tests**: new candidates create a bounded artifact and return `manual_review_required`; schema/Evidence child calls are absent. +- [ ] **Step 2: Add safe host ingress tests** proving SQL and annotations travel only over stdin, are absent from Compose argv/state/logs, and staging files are removed. +- [ ] **Step 3: Add safe host egress tests** for candidate export identity/digest, existing destination refusal, and sanitized JSON mode. +- [ ] **Step 4: Implement `schema suggest-fks`** with candidate count/digest and optional exclusive output. +- [ ] **Step 5: Implement `schema check`** in two modes: read-only orphan validation; or reviewed annotation import requiring the exact candidate digest. Persist review digest + annotation digest + workspace/revision. +- [ ] **Step 6: Require the review record on full-run resume.** A mere zero-orphan result without reviewer digest is insufficient. +- [ ] **Step 7: Preserve the boundary:** P2 updates runtime-local annotations only; it never writes Git. Output warns that P5 will supersede this local acknowledgement. +- [ ] **Step 8: Run Go/backend/harness focused gates and commit:** `feat: add P2 FK review checkpoint`. + +### Task 9: Implement schema indexing and Evidence policy boundaries + +**Files:** +- Modify: `backend/src/workspaces/preprocessing-service.ts` +- Modify: backend service tests +- Modify: `harness/tests/test_http_evidence_source.py` +- Modify: `harness/tests/test_registry_evidence_config.py` +- Modify: `harness/tests/test_semantic_kind_isolation.py` + +- [ ] **Step 1: Write RED schema-index tests** for compatible pre-existing collection, deterministic JSON counts, idempotent repeat, missing/incompatible refusal, and no collection-create request. +- [ ] **Step 2: Write RED Evidence tests** for no-Evidence warning/skip, HTTP dry-run/run/resume/unchanged/mutation, authoritative workspace identity, ACTIVE preservation, and filesystem early stop before adapter/Qdrant calls. +- [ ] **Step 3: Write RED egress tests** for exact private-host allowlist, DNS re-resolution, redirect to private/link-local/metadata, signed URL query redaction, and refusal of S3 ambient/custom/private/insecure modes. +- [ ] **Step 4: Implement installation-local egress-policy parsing** with exact bounded hostnames and no wildcard. Descriptor flags alone never grant network access. +- [ ] **Step 5: Implement `index-schema` and `preprocess evidence`** with semantic preflight and stable result mapping. +- [ ] **Step 6: Revalidate active revision and session inventory immediately before each write.** +- [ ] **Step 7: Run focused tests/touched Ruff/backend typecheck/build and commit:** `feat: add guarded P2 semantic preprocessing`. + +### Task 10: Implement the ordered full-run coordinator + +**Files:** +- Modify: `backend/src/workspaces/preprocessing-service.ts` +- Modify: `backend/test/workspace-preprocessing-service.test.ts` +- Modify: `backend/test/workspace-maintenance.test.ts` + +- [ ] **Step 1: Write a RED stage-table test** for exact order `dwh → fk_suggest → fk_review/check → schema_index → evidence` and for no hidden/skipped mutation. +- [ ] **Step 2: Add scenarios**: pre-curated/no-new-candidate completion; new-candidate block; digest-reviewed resume; no-Evidence warning; filesystem deferred block; each child failure; resume mismatch; outer crash reconciliation. +- [ ] **Step 3: Implement the coordinator as an explicit state machine**, not recursive command dispatch. +- [ ] **Step 4: Persist completion after each verified child artifact** and never mark a stage based only on exit code. +- [ ] **Step 5: Prove unchanged rerun creates no duplicate schema/Evidence points or generation.** +- [ ] **Step 6: Run focused Vitest, typecheck, build, and commit:** `feat: orchestrate the P2 preprocessing chain`. + +### Task 11: Add the hardened maintenance service and selected-image handoff + +**Files:** +- Modify: `compose.yaml` +- Modify: `deploy/compose.local.yaml` +- Modify: `deploy/compose.server.yaml` +- Modify: connector/Git override files and generators as required +- Create: `docker/workspace-maintenance-entrypoint.sh` +- Modify: `docker/core.Dockerfile` +- Modify: `tools/thothctl/internal/workspaceops/operations.go` +- Modify: `tools/thothctl/internal/pi/update.go` +- Modify relevant Go/Bash/Compose tests + +- [ ] **Step 1: Write RED Compose contract tests** for the exact security contract, profile, mounts, no Pi/no port/no build, local/server storage, and absence of Git credentials on active-snapshot operations. +- [ ] **Step 2: Write RED image-precedence tests** across base/profile/operator overrides/current-image override; `core` and maintenance must resolve to the same immutable ID. +- [ ] **Step 3: Write RED lifecycle tests** for `--no-deps`, pre-existing service preservation, interruption cleanup, hostile container name/label collision, tag replacement, and output rejection on post-run image mismatch. +- [ ] **Step 4: Add the dedicated service and entrypoint**; the entrypoint executes only the compiled operator and never calls Pi trust setup. +- [ ] **Step 5: Generate a per-operation final override** that pins immutable image ID, exact secrets, egress policy, and owned labels. Validate `docker compose config` structurally before run. +- [ ] **Step 6: Extend Pi update/rollback override generation** so future selected images cannot split core and maintenance. +- [ ] **Step 7: Run:** + +```bash +bash scripts/test-preprocess-compose-config.sh +bash scripts/test-compose-secret-policy.sh +bash scripts/test-default-compose.sh +bash scripts/test-unified-compose.sh +bash scripts/test-no-deployment-coupling.sh +cd tools/thothctl && go test ./... +``` + +- [ ] **Step 8: Build the core image and invoke operator `--help` through the exact service** without starting Pi/backend/dependencies. +- [ ] **Step 9: Commit:** `feat: package the P2 maintenance service`. + +### Task 12: Complete host dispatch and supported-platform build contract + +**Files:** +- Modify: `tools/thothctl/cmd/thothctl/main.go`, tests +- Modify: `tools/thothctl/internal/workspaceops/operations.go`, tests +- Modify: `scripts/build-thothctl.sh` +- Modify: `scripts/test-thothctl-build-contract.sh` +- Update operator docs + +- [ ] **Step 1: Add RED command-to-request-to-Compose tests** for all seven public commands, JSON/human output, exit mapping, secret redaction, and exact stdin. +- [ ] **Step 2: Implement dispatcher integration** using only typed requests. +- [ ] **Step 3: Cross-build the existing release matrix** and verify Windows input/output safety compiles. Do not claim Windows Docker behavior without a Windows Docker run. +- [ ] **Step 4: Run `go test ./...`, build contract, `go vet ./...`, and `gofmt` check.** +- [ ] **Step 5: Commit:** `feat: expose P2 workspace commands in thothctl`. + +### Task 13: Build the clean-state automated P2 process goal + +**Files:** +- Create: `scripts/p2-acceptance.sh` +- Create: `backend/scripts/p2-acceptance.mjs` +- Create: `backend/scripts/p2-acceptance.test.mjs` +- Update: `.gitignore` only if the existing `.artifacts/` rule is insufficient + +The public command is: + +```bash +./scripts/p2-acceptance.sh integration --keep +``` + +- [ ] **Step 1: Write RED acceptance-runner tests** for ownership-first state, unique run/project/container/image names, exact cleanup, `--keep`, injected failure, signal cleanup, report bounds, and no automatic retry. +- [ ] **Step 2: Build a clean owned topology** under `.artifacts/p2-integration/p2-/`: local bare Git + author clone, active P1.1 snapshot (root catalog + `/workspace.yaml` + `/evidence`), installation descriptor/env, fixture-only secrets, controlled REST DWH, controlled HTTP Evidence, real compatible Qdrant, deterministic Ollama-compatible embedding fixture, selected core image, and no backend/Pi/frontend. +- [ ] **Step 3: Pre-provision the exact compatible Qdrant collection** outside the product operation and record that setup as a P4-deferred fixture step. +- [ ] **Step 4: Exercise only built `thothctl` product commands** and assert: + 1. exact inspect revision/config identity; + 2. DWH introspection+LSH, clean-process rerun/resume, physical/LSH artifacts; + 3. FK pristine JSON, pause-before-index, candidate export, explicit digest review, resume; + 4. pre-curated full-run completion; + 5. schema index counts and unchanged rerun; + 6. HTTP Evidence dry-run, publish, unchanged rerun, input mutation/new generation/ACTIVE; + 7. no-Evidence warning/skip; + 8. filesystem `evidence_materialization_required` with no partial output; + 9. direct renderer regression and SSH fail-closed result; + 10. missing workspace/binding, resume mismatch, different-revision resumable session, concurrent writer, annotation invalid, egress refusal, and semantic incompatibility; + 11. no collection creation, no backend listener, no Pi init, no arbitrary mount; + 12. exact cleanup preserving all foreign/pre-existing resources. +- [ ] **Step 5: Produce bounded `report.json` and `report.md`**, declare hashes for every retained owned artifact, and scan raw Git objects, names, state, reports, logs, configs, candidates, and Qdrant payloads for fixture canaries/signed queries/raw SQL. +- [ ] **Step 6: Run runner unit tests, then one clean integration run without retry.** On failure, diagnose/fix/regress and start one new clean run; never loop blindly. +- [ ] **Step 7: Commit tooling:** `test: add P2 host preprocessing acceptance`. + +### Task 14: Finalize P2 documentation and independent manual walkthrough + +**Files:** +- Update: `docs/testing/p2-p6-manual-verification.md` P2 section only +- Update: local/server installation manuals +- Update: `docs/contracts/workspace-preprocessing-cli.md` +- Optionally create manual lab helper files if concrete setup cannot remain concise + +- [ ] **Step 1: Document prerequisites and boundaries**: Docker/Compose, active registry snapshot, existing compatible collection, running semantic services for semantic commands, no host language runtimes, no backend/Pi. +- [ ] **Step 2: Fill exact P2 commands** for inspect, DWH/resume, FK export/review digest, check/import, index, HTTP dry/run, full run, unchanged rerun, filesystem deferred result, secret scan, and cleanup. +- [ ] **Step 3: Explain each observed component/artifact** without exposing config or secret contents. +- [ ] **Step 4: Require a new manual root and `VERDICT.md`** with reviewer, UTC time, explicit result for every P2 check, observations, and exactly `P2 manual acceptance: PASS|FAIL`. Automation never writes it. +- [ ] **Step 5: Add mechanical docs tests** for all released commands and stable codes. +- [ ] **Step 6: Commit:** `docs: add P2 preprocessing operator walkthrough`. + +### Task 15: Run affected-layer verification and hand off the hard checkpoint + +**Files:** +- Modify after evidence exists: `PROJECT_STATE.md` + +- [ ] **Step 1: Run complete affected Go gates:** `cd tools/thothctl && go test ./... && go vet ./...`, plus the release build contract. +- [ ] **Step 2: Run complete backend gates:** `cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build`. +- [ ] **Step 3: Run focused harness tests listed in the target map, then full `.venv/bin/pytest -q` if feasible.** Any baseline failure must be reported exactly; touched Python files must be Ruff-clean. +- [ ] **Step 4: Run all affected Compose/security contracts** from Task 11 and `git diff --check`. +- [ ] **Step 5: Run exactly one final clean P2 integration at the final source commit:** + +```bash +./scripts/p2-acceptance.sh integration --keep +``` + +Expected final lines: + +```text +P2 automated integration: PASS +P2 manual acceptance: PENDING +``` + +- [ ] **Step 6: Verify report hashes, declared artifacts, secret scan, closed listeners, no maintenance container, and exact cleanup/retention.** +- [ ] **Step 7: Update and commit only tracked project state** with retained report path and truthful scope: + +```bash +git add PROJECT_STATE.md +git commit -m "docs: record P2 automated acceptance" +``` + +- [ ] **Step 8: Report separate statuses and STOP:** + +```text +P2 automated integration: PASS — +P2 manual acceptance: PENDING — docs/testing/p2-p6-manual-verification.md#p2 +P3 authorization: PENDING — awaiting explicit user decision +``` + +Do not begin P3, mark manual PASS, or infer implementation approval from plan approval or automated evidence. + +--- + +## Requirement traceability + +| Requirement/decision | P2 implementation/proof | Deferred truth | +|---|---|---| +| D2 / P2 / RNF7 | Native `thothctl`, dedicated one-shot service, existing engine | GUI/backend endpoint excluded | +| RF1.2 / RNF1 | File-only bindings, operation-specific mounts, redaction/scan | No secret in Git/rendered output | +| RF1.3 / RNF5 | Same binding+renderer code and byte-equivalence test | P3 canonical cross-revision identity/migration | +| RF1.4 / RF2.2 | REST process goal; direct regression | SSH operational support P10 | +| RF2.1 | DWH command, JSON, outer+child resume | — | +| RF2.3 | Physical/LSH production then schema-index consumption | — | +| RF2.4 / RNF2 | Immutable engine generations, unchanged rerun, ACTIVE preservation | Cross-revision DWH reuse P3 | +| RF3.1 | JSON candidates/check, bounded SQL ingress, explicit digest review | Git-canonical review/sync P5 | +| RF3.2 / D5 | Runtime-local P2 annotations only | Repository annotations and pinned sync P5 | +| RF3.3 | No model-derived FK path added | Existing workflow invariant preserved | +| RF4.1 | Existing schema hash/upsert + JSON counts | Collection lifecycle P4 | +| RF5.2 | Policy-allowed HTTP dry/run/resume/publish | Filesystem P6; broader S3 policy separately reviewed | +| RF5.3 / D9 | Existing per-run behavior only | Long-term GC/retention P9 | +| RNF3 | Safe errors, prior ACTIVE preserved, no-Evidence warning | — | +| RNF4 | Workspace binding + P2 different-revision session guard | Revision-scoped points/roots P3 | +| RF8.5–8.6 / RNF8–9 | Clean process goal + separate walkthrough | Aggregate P2–P6 verification after P6 | +| PRD AC2 | Native release binary on local/server installation profiles | Windows Docker is a separate manual claim | +| PRD AC8 | HTTP unchanged/mutation cases | Filesystem change/GC P6/P9 | + +## Explicit exclusions + +- No frontend/GUI or backend HTTP preprocessing endpoint. +- No host Python, Node, Pi, `tht`, arbitrary shell, arbitrary entrypoint, or arbitrary host mount. +- No Git pull/publish/push, active-revision transition, or authoring API in P2. +- No P3 canonical effective fingerprint, ownership migration, revision-scoped roots/points, or `.tht-dwh` operator chapter. +- No P4 collection creation/index repair/rebuild/maintenance drain. +- No P5 Git-canonical FK annotation sync/acceptance. +- No P6 filesystem Evidence materialization, realpath/symlink containment, or pinned-tree retention. +- No PSD migration/re-embedding (P7), final search/session/L2 gate (P8), policy-driven long-term GC (P9), or SSH runtime (P10). +- No changed embedding model/dimensions/distance, external vector service, pgvector compatibility path, or NL→SQL workflow change. + +## Execution notes + +- Every implementation task is RED → minimal GREEN → focused verification → commit. +- Never weaken an existing P1 security invariant to simplify P2. +- P2's session-inventory guard and deterministic same-revision config path are deliberate temporary safety mechanisms, not substitutes for P3. +- Keep automated FK mechanics distinct from human review and from the later P5 Git decision. +- A passed plan review authorizes only plan acceptance. Implementation starts only after the user's separate explicit approval. diff --git a/docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md b/docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md new file mode 100644 index 00000000..da66b5a3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md @@ -0,0 +1,1250 @@ +# P1.1 Workspace-Directory Git Registry Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Use superpowers:test-driven-development for every behavior change and superpowers:verification-before-completion before any completion claim. + +**Goal:** Replace P1's split/flat Git source layout with an authoritative root catalog and one self-contained directory per workspace, while making descriptor publication bootstrap-only and all later descriptor changes curator-owned through Git. + +**Architecture:** `thoth-workspaces.yaml` becomes the strict curator-owned catalog; descriptors move to `/workspace.yaml`, embedded Evidence moves to `/evidence`, and generated docs remain under `workspace-docs/`. The API may create a descriptor only when its catalog slot exists and the descriptor Git object is absent at the exact base commit; existing descriptors and curated content are read-only to the API. Internal immutable snapshot paths stay flat to preserve ThtRunner/session compatibility. P1.1 gets new automated and manual acceptance evidence; accepted P1 evidence remains historical and untouched. + +**Tech Stack:** TypeScript 5, Zod 4, YAML, Fastify 5, Git CLI with fixed argv, React 18, Vitest, Node.js 22, Bash, Python harness `tht config check`. + +**Companion design draft:** `docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md` + +--- + +## P1.1 completion contract + +P1.1 is complete only when all of the following are true: + +1. The only accepted source-repository layout is: + + ```text + thoth-workspaces.yaml + /workspace.yaml + /evidence/** # optional; required only for filesystem Evidence + workspace-docs//{contract.env.example,README.md} + ``` + +2. The strict root catalog is curator-owned and authoritative for ID, name, description, and display order. Catalog-only entries are valid bootstrap slots with `configuration_required`; orphan descriptors/directories and catalog/descriptor metadata mismatches invalidate the candidate atomically. +3. Schema v3 remains the only descriptor schema. For filesystem Evidence the only URI is `/evidence`; HTTP/S3 and absent-Evidence behavior stay as delivered by P1. +4. The API creates `/workspace.yaml` only when no Git object exists there at the exact base commit. A present empty, malformed, symlink, submodule, tree, or valid descriptor is never replaced or deleted. Existing update/delete payloads fail with safe `workspace_curator_owned` semantics. +5. Curator-pushed descriptor/catalog/Evidence changes become active only after strict pull validation. The API never stages, writes, cleans, or pushes catalog or curated content. +6. Generated docs remain API-owned under `workspace-docs/`. Explicit registry synchronization may produce one deterministic docs-only commit; it must preserve catalog, descriptor, and Evidence object IDs and activate only the final validated commit. +7. Internal snapshots remain `//.yaml`; session revision pins, retention leases, runtime acquisition, export, and resume retain their current contract. +8. Existing workspace UI is read-only for repository-backed descriptors. Only a catalog slot with no descriptor offers an editable bootstrap draft; stale old drafts cannot update/delete. Pull/sync, validate, installation test, export, and Evidence summary remain available. +9. A new clean-state P1.1 automated run and a separate P1.1 manual walkthrough prove the complete process. Accepted retained P1 artifacts remain immutable historical evidence and are not relabelled as P1.1. Old P1 process commands are not required to execute successfully against the superseding P1.1 repository contract. +10. No preprocessing, materialization, FK synchronization, Qdrant write, embedding, ACTIVE publication, GC, or P2–P6 plan edit is performed. + +## Explicit decisions frozen by this plan + +- Root catalog name: `thoth-workspaces.yaml`. +- Workspace source directory: `/`. +- Descriptor filename: `workspace.yaml`. +- Catalog schema: `schema_version: 1`, ordered `workspaces` list with strict `{id,name,description?}` entries. +- Catalog-only entries are allowed and listable as `configuration_required`. +- Descriptor metadata remains present for self-contained snapshots/exports and must exactly equal catalog metadata. +- "Empty" means **absent Git object**, not zero bytes. +- Old repository layouts are rejected; there is no dual reader or automatic remote migration. +- Internal snapshot filenames do not change. +- P2–P6 documents are inventoried but not edited until after owner manual acceptance of P1.1. + +--- + +### Task 1: Freeze the P1.1 design, catalog schema, and strict parser + +**Files:** +- Add: `docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md` +- Add: `backend/src/workspaces/catalog.ts` +- Add: `backend/test/workspaces-catalog.test.ts` +- Modify: `backend/src/workspaces/types.ts` + +**Step 1: Write failing catalog parser tests** + +Define the exact public shape: + +```ts +export interface WorkspaceCatalogEntry { + id: string; + name: string; + description?: string; +} + +export interface WorkspaceCatalog { + schema_version: 1; + workspaces: WorkspaceCatalogEntry[]; +} +``` + +Test: + +- one and many ordered entries parse without reordering; +- optional description presence is preserved (missing is not silently converted to an empty string); +- Unicode names/descriptions and the descriptor's existing trim/nonblank semantics are preserved without adding a new length limit; +- duplicate IDs, invalid/reserved IDs (including `workspace-docs`), blank names, unknown keys, duplicate YAML keys, aliases, tags, multiple documents, non-mappings, and malformed YAML are rejected with `workspace_invalid`; +- safe errors include a stable catalog field location but never rejected canary text; +- `assertCatalogMatchesDescriptor(entry, descriptor)` accepts exact ID/name/optional-description equality and rejects every mismatch safely; +- serialization, if exposed, is deterministic and does not become an API write path. + +**Step 2: Run the focused test and verify RED** + +```bash +cd backend +npx vitest run test/workspaces-catalog.test.ts +``` + +Expected: FAIL because `catalog.ts` does not exist. + +**Step 3: Implement the strict parser** + +Use `yaml.parseAllDocuments` with the same safe-document rules as `parseWorkspaceYaml` and a strict Zod schema. Keep catalog parsing separate from descriptor parsing. Export only: + +```ts +export const CATALOG_PATH = "thoth-workspaces.yaml"; +export function parseWorkspaceCatalogYaml(source: string): WorkspaceCatalog; +export function assertCatalogMatchesDescriptor( + entry: WorkspaceCatalogEntry, + workspace: WorkspaceDescriptor, +): void; +``` + +Do not project catalog metadata into a descriptor and do not read the filesystem from this module. + +Add any new stable error code only when needed by later tasks; catalog syntax/matching errors remain `workspace_invalid`. + +**Step 4: Run tests and typecheck** + +```bash +cd backend +npx vitest run test/workspaces-catalog.test.ts test/workspaces-schema.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md \ + backend/src/workspaces/catalog.ts \ + backend/src/workspaces/types.ts \ + backend/test/workspaces-catalog.test.ts +git commit -m "feat: define P1.1 workspace catalog contract" +``` + +--- + +### Task 2: Enforce the nested repository paths and curator/API ownership boundary + +**Files:** +- Modify: `backend/src/workspaces/git-repository.ts` +- Modify: `backend/test/workspaces-git-repository.test.ts` + +**Step 1: Write failing low-level Git tests** + +Create a real bare-repository fixture with: + +```text +thoth-workspaces.yaml +research/workspace.yaml +research/evidence/guide.md +workspace-docs/research/README.md +workspace-docs/research/contract.env.example +``` + +Test fixed-argv helpers that: + +- read exactly `thoth-workspaces.yaml` as a regular Git blob at HEAD/revision; +- discover only `/workspace.yaml` descriptors, without treating nested Evidence files as descriptors; +- reject flat `workspaces/.yaml`, `workspace-content/**`, the reserved `workspace-docs` ID, unlisted top-level workspace directories, traversal, alternate descriptor names, symlink descriptor, tree-at-descriptor, submodule/gitlink, and malformed IDs; +- read/resolve the exact descriptor blob at `/workspace.yaml`; +- accept only `/evidence` as the filesystem Evidence root and require a Git tree at the exact revision; +- keep nested symlink checks deferred to P6 while still rejecting a symlink at the declared root; +- prove catalog and `/evidence/**` are not API-writable/stageable paths; +- allow only a create-only descriptor path and generated docs in API publication helpers; +- refuse an exclusive descriptor create if any filesystem/Git object already occupies the path; +- journal each exact API-owned path before mutation (object type/mode/blob/bytes or explicit absence); +- restore overwritten/deleted generated docs to their exact pre-operation objects and remove only absent-before files on failure, never by running a directory-wide `git clean`; +- cover failed bootstrap with pre-existing stale docs plus failed docs update/deletion, and preserve all curator object IDs across cleanup; +- use argv arrays only and do not invoke Git filters, hooks, shell interpolation, or helper-bearing repository state. + +**Step 2: Run the focused test and verify RED** + +```bash +cd backend +npx vitest run test/workspaces-git-repository.test.ts +``` + +Expected: old flat-path assertions fail and required helpers are missing. + +**Step 3: Implement narrow path helpers** + +Introduce named guards such as: + +```ts +function workspaceDescriptorPath(id: string): string { + return `${safeWorkspaceId(id)}/workspace.yaml`; +} + +function evidenceRootPath(id: string): string { + return `${safeWorkspaceId(id)}/evidence`; +} +``` + +Add read-only helpers for catalog/descriptor object type and descriptor discovery. Replace broad `writeRegistryFile` use for descriptors with an explicit exclusive creation primitive. Keep generated-doc writes separate and exact. + +`restoreFailedPublication` must consume the bounded per-path journal from the current operation. It restores tracked generated docs from the prior blob/mode and deletes only paths proven absent before the operation. Remove any directory-wide `git clean` that can descend into curated workspace content. + +**Step 4: Run tests and typecheck** + +```bash +cd backend +npx vitest run test/workspaces-git-repository.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend/src/workspaces/git-repository.ts backend/test/workspaces-git-repository.test.ts +git commit -m "refactor: enforce P1.1 registry path ownership" +``` + +--- + +### Task 3: Make activation catalog-driven while preserving internal snapshots + +**Files:** +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/test/workspace-registry.test.ts` +- Modify: `backend/src/workspaces/types.ts` + +**Step 1: Write failing activation/state tests** + +Cover real Git candidates for: + +- valid catalog plus one/many matching descriptors activates in catalog order; +- catalog-only entry activates as `configuration_required` without a `WorkspaceRevision` and without allowing session/read/test/export; +- missing catalog, malformed catalog, duplicate catalog ID, orphan descriptor/directory, metadata mismatch, wrong descriptor path, duplicate Qdrant collection, invalid descriptor, or unsafe Evidence root leaves the prior active snapshot unchanged; +- a present empty/comments-only descriptor fails activation and is never converted into a bootstrap slot; +- removing a descriptor while leaving its catalog entry produces `configuration_required`, while retained historical revisions/session pins remain readable; +- removing catalog entry and its workspace directory together removes it from the active catalog but retains historical pinned snapshots; +- removing a catalog entry while leaving its workspace directory/descriptor is invalid; +- catalog order controls summaries independently from Git path order; +- offline fallback restores the last complete catalog and ready revisions; +- internal snapshot paths remain exactly `//.yaml`; +- the immutable snapshot binds a canonical catalog copy/digest plus canonical descriptor/docs, but contains no Evidence bytes; +- pre-P1.1 historical internal snapshots needed by already retained session pins remain readable, without accepting old source-repository layout for new activation. + +**Step 2: Run the registry suite and verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts +``` + +Expected: nested repository and catalog-only cases fail. + +**Step 3: Refactor candidate parsing from activation** + +Introduce an internal candidate model, for example: + +```ts +interface WorkspaceCatalogRecord { + entry: WorkspaceCatalogEntry; + state: "ready" | "configuration_required"; + revision?: WorkspaceRevision; +} + +interface RegistryCandidate { + commit: string; + catalog: WorkspaceCatalog; + ready: Array<{ + entry: WorkspaceCatalogEntry; + workspace: WorkspaceDescriptor; + descriptorPath: string; + blob: string; + }>; + missing: WorkspaceCatalogEntry[]; +} +``` + +Separate: + +1. `readCandidate(commit)` — read/validate Git objects without mutating active state; +2. `stageSnapshot(candidate)` — write immutable local snapshot bytes; +3. `activateCandidate(candidate)` — atomically publish active state only after all checks/synchronization succeed. + +Keep `WorkspaceRevision` and internal flat snapshot filenames unchanged. Persist enough canonical catalog data in the immutable snapshot to list catalog-only entries during offline fallback. Do not force `WorkspaceRevision` to represent a missing descriptor. + +Expose a catalog-aware method for routes, while preserving `list()`/retained-revision methods used by sessions: + +```ts +listCatalog(): Promise; +``` + +**Step 4: Run focused cross-boundary tests** + +```bash +cd backend +npx vitest run \ + test/workspace-registry.test.ts \ + test/routes-sessions.test.ts \ + test/workspace-runtime-handoff.test.ts +npx tsc --noEmit -p . +``` + +Expected: PASS; ready workspaces remain session-activatable and missing descriptors do not. + +**Step 5: Commit** + +```bash +git add backend/src/workspaces/registry.ts backend/src/workspaces/types.ts backend/test/workspace-registry.test.ts +git commit -m "feat: activate workspaces from the root catalog" +``` + +--- + +### Task 4: Implement bootstrap-only descriptor publication and deterministic docs synchronization + +**Files:** +- Modify: `backend/src/workspaces/registry.ts` +- Modify: `backend/src/workspaces/git-repository.ts` +- Modify: `backend/src/workspaces/types.ts` +- Modify: `backend/test/workspace-registry.test.ts` + +**Step 1: Write failing create-only publication tests** + +Prove: + +- catalog entry exists + descriptor absent + exact base commit + matching metadata + valid Evidence context → API creates descriptor and generated docs once; +- catalog bytes/blob and every Evidence tree/blob are unchanged by bootstrap; +- a descriptor path containing zero bytes, invalid YAML, a symlink/blob/tree/gitlink, or valid YAML is considered present and is not overwritten; +- update and delete requests return `workspace_curator_owned`, perform no write/stage/commit, and preserve HEAD/object IDs; +- create for an unknown catalog ID or mismatched name/description fails without mutation; +- stale base and a race in which a curator creates the descriptor first fail safely; +- failed commit/push replays the per-path journal, including stale pre-existing generated docs, and leaves curator paths untouched; +- a curator modifies catalog metadata and the descriptor together, pushes, and pull activates the exact curator bytes without reserializing the descriptor in Git; +- a content-only Evidence commit changes the active workspace commit even when descriptor/catalog blobs are unchanged; +- a curator descriptor-only commit changes the descriptor blob and active revision without any API descriptor write; +- explicit pull computes generated docs and, when stale, produces at most one docs-only follow-up commit; +- that docs-only commit changes only `workspace-docs/**`, preserves catalog/descriptor/Evidence object IDs, and becomes the active revision; +- startup/status activation never pushes; before explicit sync, local snapshots/exports contain freshly derived docs even if committed `workspace-docs` are stale; +- no-op synchronization makes no commit; +- a docs push race/rejection keeps the prior active snapshot and restores a clean checkout; +- generated docs are removed only when the catalog/descriptor state no longer owns them, never by directory-wide cleanup. + +**Step 2: Run the focused tests and verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts -t "bootstrap|curator|generated docs|content-only" +``` + +Expected: FAIL against create/update/delete publication. + +**Step 3: Narrow the public mutation contract** + +Add: + +```ts +export type BootstrapWorkspaceRequest = { + action: "create"; + workspace: CanonicalWorkspace; + baseCommit: string; +}; +``` + +Keep legacy request parsing only long enough to return the stable refusal; do not keep update/delete implementation branches. Add `workspace_curator_owned` to `WorkspaceErrorCode` and map it to HTTP 409. + +`publishBootstrap` must: + +1. pull and read a candidate without activating it; +2. compare the exact requested base; +3. locate the authoritative catalog slot; +4. verify descriptor absence and metadata equality; +5. verify contextual filesystem Evidence at that base; +6. exclusively create the descriptor plus deterministic docs; +7. commit/push fixed paths with fixed argv; +8. read/validate the resulting candidate; +9. activate only after complete success. + +Refactor explicit pull to reconcile docs as defined in the design. GET/status/bootstrap paths remain read-only with respect to the remote. + +**Step 4: Run focused tests, typecheck, and build** + +```bash +cd backend +npx vitest run test/workspace-registry.test.ts test/workspaces-git-repository.test.ts +npx tsc --noEmit -p . +npm run build +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + backend/src/workspaces/registry.ts \ + backend/src/workspaces/git-repository.ts \ + backend/src/workspaces/types.ts \ + backend/test/workspace-registry.test.ts +git commit -m "feat: make workspace publication bootstrap-only" +``` + +--- + +### Task 5: Update workspace routes and machine contracts + +**Files:** +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/test/routes-workspaces.test.ts` +- Modify: `backend/test/routes-sessions.test.ts` + +**Step 1: Write failing real-route tests** + +Using the real local bare-repository fixture, assert: + +- `GET /workspaces` returns root-catalog order/metadata and explicit `ready` vs `configuration_required` state; +- summary `file` is exactly `/workspace.yaml`; summary omits `language` because a catalog-only slot has none, while ready detail/bootstrap drafts retain descriptor language; +- catalog-only entries have no revision and no descriptor body; +- `GET /workspaces/:id`, diagnostic, export, and session creation for a catalog-only entry return safe `workspace_not_activatable` and never start Pi; +- `POST /workspaces/validate` remains context-free and says nothing about catalog publication eligibility; +- create for one matching catalog slot succeeds once; +- second create, update, and delete produce HTTP 409 `workspace_curator_owned` with no conflict field/value payload and no mutation; +- unknown slot, catalog metadata mismatch, stale base, invalid Evidence tree, and present-empty descriptor produce safe errors without canary/Git stderr; +- curator-pushed descriptor/catalog/Evidence changes are visible after pull and API bytes remain unchanged; +- import remains an untrusted draft and cannot update an existing workspace; +- export remains descriptor/docs only and never includes Evidence bytes or secrets. + +**Step 2: Run the focused routes and verify RED** + +```bash +cd backend +npx vitest run test/routes-workspaces.test.ts test/routes-sessions.test.ts +``` + +Expected: FAIL because routes expose full CRUD and cannot list missing descriptors. + +**Step 3: Implement the new DTOs and route semantics** + +Create a stable summary shape with catalog authority and explicit state. Route `POST /workspaces/publish` to bootstrap only. Recognize legacy update/delete payload discriminators before rejecting them as `workspace_curator_owned`; never pass them to a file mutation method. + +Remove field-level `WorkspaceConflictError` serialization if it has no remaining production caller. Preserve generic stale-commit 409 behavior for bootstrap races. + +**Step 4: Run tests and checks** + +```bash +cd backend +npx vitest run \ + test/routes-workspaces.test.ts \ + test/routes-sessions.test.ts \ + test/workspace-registry.test.ts +npx tsc --noEmit -p . +npm run build +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add backend/src/routes/workspaces.ts backend/test/routes-workspaces.test.ts backend/test/routes-sessions.test.ts +git commit -m "feat: expose catalog-driven bootstrap workspace API" +``` + +--- + +### Task 6: Change the filesystem Evidence root without changing P1 source semantics + +**Files:** +- Modify: `backend/src/workspaces/schema.ts` +- Modify: `backend/test/workspaces-schema.test.ts` +- Modify: `backend/test/workspaces-contracts.test.ts` +- Modify: `backend/test/workspaces-bindings.test.ts` +- Modify: `backend/test/workspace-runtime-renderer.test.ts` +- Modify: `backend/test/workspace-runtime-handoff.test.ts` +- Modify: `deploy/workspaces/example.yaml` +- Modify: `deploy/workspaces/psd.yaml.example` + +**Step 1: Change tests first** + +Replace every positive filesystem URI with: + +```text +/evidence +``` + +Negative coverage must reject: + +- old `workspace-content//evidence`; +- flat/cross-workspace paths; +- absolute paths, `.`/`..`, doubled segments, backslashes, controls, query/fragment-like content; +- roots above or below the exact canonical Evidence root. + +Renderer/handoff tests must expect: + +```text +/snapshots///evidence +``` + +while allowing that root not to exist until P6 materializes it. Keep HTTP/S3, local secret files, policy defaults, deterministic bytes, `runtime_identity.workspace_revision`, and `ssh_tunnel` fail-closed behavior unchanged. + +**Step 2: Run focused tests and verify RED** + +```bash +cd backend +npx vitest run \ + test/workspaces-schema.test.ts \ + test/workspaces-contracts.test.ts \ + test/workspaces-bindings.test.ts \ + test/workspace-runtime-renderer.test.ts \ + test/workspace-runtime-handoff.test.ts +``` + +Expected: FAIL on the old hardcoded invariant/fixtures. + +**Step 3: Implement the smallest production change** + +Change the cross-field invariant to: + +```ts +const expected = `${workspace.workspace.id}/evidence`; +``` + +The runtime renderer already joins a validated repo-relative URI to `revisionContentRoot`; do not add a second path mapping or materialization branch. + +**Step 4: Run cross-layer verification** + +```bash +cd backend +npx vitest run \ + test/workspaces-schema.test.ts \ + test/workspaces-contracts.test.ts \ + test/workspaces-bindings.test.ts \ + test/workspace-runtime-renderer.test.ts \ + test/workspace-runtime-handoff.test.ts +npx tsc --noEmit -p . +npm run build +``` + +Then: + +```bash +cd harness +.venv/bin/pytest -q tests/test_config_resources.py tests/test_registry_evidence_config.py +``` + +Expected: PASS. No harness production change should be necessary. + +**Step 5: Commit** + +```bash +git add \ + backend/src/workspaces/schema.ts \ + backend/test/workspaces-schema.test.ts \ + backend/test/workspaces-contracts.test.ts \ + backend/test/workspaces-bindings.test.ts \ + backend/test/workspace-runtime-renderer.test.ts \ + backend/test/workspace-runtime-handoff.test.ts \ + deploy/workspaces/example.yaml \ + deploy/workspaces/psd.yaml.example +git commit -m "refactor: colocate filesystem Evidence with its workspace" +``` + +--- + +### Task 7: Narrow frontend API and draft persistence to bootstrap-only authoring + +**Files:** +- Modify: `frontend/src/api/workspaces.ts` +- Modify: `frontend/src/api/workspaces.test.ts` +- Modify: `frontend/src/workspaces/drafts.ts` +- Modify: `frontend/src/workspaces/drafts.test.ts` +- Modify: `frontend/src/test/workspace-fixtures.ts` +- Review/test: `frontend/src/api/sessions.test.ts` +- Review/test: `frontend/src/shell/SteerInput.test.tsx` +- Review/test: `frontend/src/shell/NewSessionDialog.test.tsx` + +**Step 1: Write failing frontend contract tests** + +Test: + +- summary parsing accepts exact catalog metadata, direct-root descriptor path, explicit state, no summary `language`, and optional revision only for `ready`; +- malformed or contradictory summary state/revision combinations are rejected; +- canonical workspace sanitization accepts only `/evidence` for filesystem sources; +- publish request type and client emit create only; +- backend `workspace_curator_owned` is decoded safely without conflict fields; +- removed field-level conflict/update/delete payloads are rejected rather than stored; +- imported bundle becomes a bootstrap candidate only; no existing-workspace update request can be constructed; +- v1 update/deletion localStorage records are purged/ignored and never returned as actionable drafts; +- new versioned bootstrap drafts contain a catalog slot identity, base commit, and workspace body but no `baseBlob` or delete intent; +- session/new-session consumers still require a ready workspace revision and ignore `configuration_required` entries. + +**Step 2: Run focused tests and verify RED** + +```bash +cd frontend +npx vitest run \ + src/api/workspaces.test.ts \ + src/workspaces/drafts.test.ts \ + src/api/sessions.test.ts \ + src/shell/SteerInput.test.tsx \ + src/shell/NewSessionDialog.test.tsx +``` + +Expected: FAIL against full CRUD DTOs and old URI sanitizer. + +**Step 3: Implement strict client contracts** + +Replace `PublishWorkspaceRequest` with the bootstrap-only request. Add `configurationState` to `WorkspaceSummary`. Remove `WorkspaceConflict`, conflict-field allowlists, deletion-draft types/storage, and any serializer that can produce update/delete. + +Version browser storage keys so old drafts cannot be interpreted under P1.1. On initialization, remove old known draft/delete keys only; never clear unrelated localStorage. + +**Step 4: Run tests and typecheck** + +```bash +cd frontend +npx vitest run \ + src/api/workspaces.test.ts \ + src/workspaces/drafts.test.ts \ + src/api/sessions.test.ts \ + src/shell/SteerInput.test.tsx \ + src/shell/NewSessionDialog.test.tsx +npx tsc -b +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + frontend/src/api/workspaces.ts \ + frontend/src/api/workspaces.test.ts \ + frontend/src/workspaces/drafts.ts \ + frontend/src/workspaces/drafts.test.ts \ + frontend/src/test/workspace-fixtures.ts \ + frontend/src/api/sessions.test.ts \ + frontend/src/shell/SteerInput.test.tsx \ + frontend/src/shell/NewSessionDialog.test.tsx +git commit -m "refactor: make browser workspace writes bootstrap-only" +``` + +--- + +### Task 8: Make existing workspaces read-only in Workspace Management + +**Files:** +- Modify: `frontend/src/shell/WorkspaceManager.tsx` +- Modify: `frontend/src/shell/WorkspaceManager.test.tsx` +- Modify: `frontend/src/shell/WorkspaceEditor.tsx` +- Modify: `frontend/src/shell/WorkspaceEditor.test.tsx` +- Modify or remove: `frontend/src/shell/WorkspacePublishDialog.tsx` +- Modify or remove: `frontend/src/shell/WorkspacePublishDialog.test.tsx` +- Review/test: `frontend/src/shell/AppShell.new-session.test.tsx` +- Review/test: `frontend/src/shell/AppShell.session-mgmt.test.tsx` + +**Step 1: Write failing behavior tests** + +Prove: + +- catalog order/name/description render even when no descriptor exists; +- `configuration_required` entry offers a prefilled editable bootstrap form with ID/name/description locked to catalog values; +- Save Draft is browser-local, Validate is explicit, and Create requires a separate confirmation; +- successful create discards the bootstrap draft and reloads as read-only; +- a ready workspace renders all descriptor fields read-only, plus Evidence summary and Git edit guidance; +- ready workspace has no Save, Publish update, Delete, Duplicate, conflict merge, or imported-update action; +- Pull/Sync, Export, Validate, and installation Test remain available where meaningful; +- stale update/delete localStorage fixtures do not make controls appear or send a request; +- import can populate only a matching unconfigured catalog slot; mismatch/existing target is refused safely; +- a curator-pushed change appears after Pull and is not written back by the browser; +- configured-only workspace selection remains enforced for new sessions. + +**Step 2: Run focused tests and verify RED** + +```bash +cd frontend +npx vitest run \ + src/shell/WorkspaceManager.test.tsx \ + src/shell/WorkspaceEditor.test.tsx \ + src/shell/WorkspacePublishDialog.test.tsx \ + src/shell/AppShell.new-session.test.tsx \ + src/shell/AppShell.session-mgmt.test.tsx +``` + +Expected: FAIL because existing workspaces expose full CRUD. + +**Step 3: Implement two explicit UI modes** + +Use a discriminated prop rather than inferring editability from `baseBlob`: + +```ts +type WorkspaceEditorMode = + | { kind: "bootstrap"; catalog: WorkspaceSummary; draft: WorkspaceBootstrapDraft } + | { kind: "read_only"; catalog: WorkspaceSummary; record: WorkspaceRecord }; +``` + +Do not rely only on disabled controls; remove mutation handlers and mutation buttons entirely in read-only mode. Keep descriptive text telling curators to edit `/workspace.yaml`, commit/push, then use Pull/Sync. + +Reduce `WorkspacePublishDialog` to one bootstrap confirmation or fold it into the manager and delete the obsolete conflict UI/tests. + +**Step 4: Run frontend verification** + +```bash +cd frontend +npx vitest run \ + src/shell/WorkspaceManager.test.tsx \ + src/shell/WorkspaceEditor.test.tsx \ + src/shell/WorkspacePublishDialog.test.tsx \ + src/shell/AppShell.new-session.test.tsx \ + src/shell/AppShell.session-mgmt.test.tsx +npx tsc -b +npm run build +``` + +If `WorkspacePublishDialog` is removed, omit its test from the command and prove no imports remain with `git grep`. + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add -A \ + frontend/src/shell/WorkspaceManager.tsx \ + frontend/src/shell/WorkspaceManager.test.tsx \ + frontend/src/shell/WorkspaceEditor.tsx \ + frontend/src/shell/WorkspaceEditor.test.tsx \ + frontend/src/shell/WorkspacePublishDialog.tsx \ + frontend/src/shell/WorkspacePublishDialog.test.tsx \ + frontend/src/shell/AppShell.new-session.test.tsx \ + frontend/src/shell/AppShell.session-mgmt.test.tsx +git commit -m "feat: make curator-owned workspaces read-only in the browser" +``` + +--- + +### Task 9: Update active contracts, examples, operator guides, and executable doc gates + +**Files:** +- Modify: `docs/contracts/workspace-evidence-v3.md` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `README.md` +- Add: `docs/migrations/p1-to-p1-1-registry-layout.md` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/test-verify-workspace-install-docs.sh` +- Review/modify if required: `scripts/workspace_descriptor_doc_contract.py` +- Review/modify if required: `scripts/test-workspace-descriptor-doc-contract.sh` +- Modify: `scripts/verify-schema-v3-only.sh` +- Modify: `scripts/test-verify-schema-v3-only.sh` + +**Step 1: Add failing verifier mutations** + +The self-tests must reject docs/examples that: + +- omit `thoth-workspaces.yaml` or put it below a workspace; +- use flat `workspaces/.yaml` or old `workspace-content//evidence`; +- omit exact `/workspace.yaml` or `/evidence` paths; +- claim catalog metadata comes from the descriptor; +- claim the API updates/deletes existing descriptors or writes catalog/Evidence; +- treat zero-byte descriptors as API-writable; +- omit catalog-only bootstrap and curator commit/push/pull flow; +- place generated docs inside a workspace directory; +- claim P1.1 materializes/preprocesses/indexes Evidence; +- omit migration ordering and explicit rejection of old layout; +- silently edit or claim completion of P2–P6. + +Retain secret/path/protocol/adversarial verifier coverage from P1. + +**Step 2: Run self-tests and verify RED** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/test-verify-schema-v3-only.sh +``` + +Expected: new mutations are not detected yet. + +**Step 3: Rewrite the active contract and manuals** + +Document the exact layout, root catalog schema, metadata equality, missing-descriptor bootstrap, create-once rule, curator ownership, docs-only API ownership, embedded/external Evidence, same-commit identity, migration cutover, and manual Git workflow. + +The migration guide must require one reviewed commit that: + +```bash +git mv workspaces/.yaml /workspace.yaml +git mv workspace-content//evidence /evidence +# create/review thoth-workspaces.yaml from descriptor metadata +``` + +It must say to upgrade ThothII only after that commit is pushed and to roll back application and repository revision together. Do not add an executable auto-migrator. + +Add an explicit P1.1 note to historical P1 design/plan references only if needed for navigation; do not rewrite accepted P1 history. + +**Step 4: Strengthen and run verifiers** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +bash scripts/test-verify-schema-v3-only.sh +./scripts/verify-schema-v3-only.sh +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add \ + docs/contracts/workspace-evidence-v3.md \ + docs/install/local-workspace-registry.md \ + docs/install/server-workspace-registry.md \ + docs/migrations/p1-to-p1-1-registry-layout.md \ + README.md \ + scripts/verify-workspace-install-docs.sh \ + scripts/test-verify-workspace-install-docs.sh \ + scripts/workspace_descriptor_doc_contract.py \ + scripts/test-workspace-descriptor-doc-contract.sh \ + scripts/verify-schema-v3-only.sh \ + scripts/test-verify-schema-v3-only.sh +git commit -m "docs: define the P1.1 registry layout and curator flow" +``` + +--- + +### Task 10: Update deployment fixtures and cross-platform registry smokes + +**Files:** +- Modify: `scripts/workspace-registry-smoke.sh` +- Modify: `backend/test/workspace-registry-deployment.test.ts` +- Modify: `scripts/unified-deployment-smoke.sh` +- Modify: `scripts/test-windows-clone-contract.ps1` +- Modify as needed: `scripts/fixtures/workspace-registry-smoke.yaml` +- Modify as needed: `scripts/fixtures/workspace-registry-task13.yaml` +- Modify as needed: `scripts/fixtures/workspace-registry-windows.yaml` +- Review: `.github/workflows/deployment.yml` + +**Step 1: Write failing deterministic fixture/smoke tests** + +Make every seed create a root catalog and nested descriptor path. Add mutations proving: + +- valid catalog + descriptor + Evidence starts; +- catalog/descriptor display metadata must change together in curator commits; +- orphan descriptor/mismatch/old layout is rejected while prior active snapshot stays usable; +- content-only Evidence update changes revision; +- API/bootstrap and curator paths remain separate; +- Windows paths with spaces preserve nested layout and LF/YAML contracts. + +**Step 2: Run focused deployment-contract tests and verify RED** + +```bash +cd backend +npx vitest run test/workspace-registry-deployment.test.ts +``` + +```bash +bash -n scripts/workspace-registry-smoke.sh scripts/unified-deployment-smoke.sh +``` + +Expected: old flat fixture assertions fail. + +**Step 3: Update seed/update/corruption helpers** + +Scripts copy standalone schema-v3 descriptor fixtures into `/workspace.yaml` and write a matching `thoth-workspaces.yaml`. Evidence goes below `/evidence` only for filesystem fixtures. Keep generated docs API-owned. + +Do not edit P2–P6 preprocessing fixtures in this task unless they are directly used by the generic registry deployment smoke; record deferred preprocessing paths for the later adaptation plan. + +**Step 4: Run deterministic gates** + +```bash +cd backend +npx vitest run test/workspace-registry-deployment.test.ts +``` + +Run non-Docker contract modes provided by the scripts and the Windows PowerShell contract on its supported CI/host. Run Docker smokes only at the final verification task so each is executed once from clean state. + +**Step 5: Commit** + +```bash +git add \ + scripts/workspace-registry-smoke.sh \ + backend/test/workspace-registry-deployment.test.ts \ + scripts/unified-deployment-smoke.sh \ + scripts/test-windows-clone-contract.ps1 \ + scripts/fixtures/workspace-registry-smoke.yaml \ + scripts/fixtures/workspace-registry-task13.yaml \ + scripts/fixtures/workspace-registry-windows.yaml \ + .github/workflows/deployment.yml +git commit -m "test: migrate registry deployment fixtures to P1.1" +``` + +--- + +### Task 11: Build independent automated P1.1 process acceptance + +**Files:** +- Add: `scripts/p11-acceptance.sh` +- Add: `scripts/test-p11-acceptance.sh` +- Add: `backend/scripts/p11-acceptance.mjs` +- Add: `backend/scripts/p11-acceptance.test.mjs` +- Add: `backend/scripts/acceptance-support.mjs` +- Add: `backend/scripts/acceptance-support.test.mjs` +- Modify only to import proven-equivalent generic guards: `backend/scripts/p1-acceptance.mjs` +- Modify/test: `backend/scripts/p1-acceptance.test.mjs` + +**Public command:** + +```bash +./scripts/p11-acceptance.sh integration --keep +``` + +**Artifact root:** + +```text +.artifacts/p11-integration// +``` + +Do not relabel, overwrite, or consume `.artifacts/p1-integration/**`. + +**Step 1: Write failing runner/lifecycle tests** + +Preserve P1's ownership-first, no-retry, fixed-argv, listener, secret-scan, report-hash, and confined-cleanup guards under the new P1.1 namespace. Test that P11 cleanup refuses P1/manual/sibling roots and vice versa. + +Extract only genuinely namespace-agnostic ownership, report, fixed-argv, secret-scan, and cleanup guards into `acceptance-support.mjs`. Keep P1/P11 roots, kinds, check IDs, reports, and process semantics in their versioned runners. Run both support and P1 unit suites to prove the extraction does not weaken P1 safety; do not claim the old P1 full integration scenario remains compatible with the new application contract. + +**Step 2: Run the runner test and verify RED** + +```bash +node --test backend/scripts/acceptance-support.test.mjs backend/scripts/p1-acceptance.test.mjs +bash scripts/test-p11-acceptance.sh +``` + +Expected: P1/support regression tests stay PASS; P11 test fails because P11 tooling does not exist. + +**Step 3: Implement the clean-state process** + +The retained run must: + +1. create a bare remote and curator clone from zero; +2. curator-push `thoth-workspaces.yaml` with filesystem/HTTP/S3 catalog slots, plus nested filesystem Evidence, but no descriptors; +3. start the production backend on loopback and list all slots as `configuration_required`; +4. validate and bootstrap-create all three descriptors through real HTTP, sequentially using the current base commit; +5. prove API writes only nested descriptor + `workspace-docs`, never catalog/Evidence; +6. prove second create, update, delete, catalog mismatch, present-empty descriptor, orphan descriptor, old layout, invalid path/protocol/secret field, and missing Git tree fail without mutation/leak; +7. curator-modify an existing descriptor and matching catalog metadata, push, then pull/sync and prove exact curator bytes are activated without descriptor rewrite; +8. push a content-only Evidence change and prove new commit identity with unchanged descriptor blob; +9. prove any docs-only follow-up commit changes only `workspace-docs/**`; +10. inspect exact catalog/descriptor/Evidence Git objects and immutable local snapshots; +11. acquire/release two production runtime configs for filesystem/HTTP/S3, compare bytes, and run real `tht config check -c ` in correct option order; +12. prove no P2 artifacts/commands, scan every non-secret-fixture byte and reachable Git blob for canaries, close listeners, and clean only owned resources. + +Required stable check IDs include at least: + +```text +preflight +clean_state +ownership +catalog_bootstrap +catalog_only_listing +bootstrap_create_once +api_curator_boundary +curator_descriptor_update +content_only_revision +docs_only_reconciliation +same_revision_git_objects +snapshot_and_export +runtime_render_determinism +tht_config_check +negative_catalog_layout_cases +negative_schema_context_cases +no_p2_scope_artifacts +secret_scan +cleanup_confinement +``` + +`report.md` must end with: + +```text +P1.1 automated integration: PASS +P1.1 manual acceptance: PENDING +``` + +**Step 4: Run runner tests** + +```bash +bash -n scripts/p11-acceptance.sh scripts/test-p11-acceptance.sh +bash scripts/test-p11-acceptance.sh +``` + +Expected: PASS. + +**Step 5: Run one fresh complete retained process** + +First verify no P11 runner/listener is active, then: + +```bash +./scripts/p11-acceptance.sh integration --keep +``` + +Expected: exit 0, one new root, all checks PASS, no retry/attempt loop, reports hash-bound to the exact clean source/runtime graph. + +On failure: retain the run, diagnose, add a regression test/fix, and execute a new full run with a new ID. Never overwrite or retry a failed run in place. + +**Step 6: Commit the tested P1.1 acceptance tooling** + +```bash +git add \ + scripts/p11-acceptance.sh \ + scripts/test-p11-acceptance.sh \ + backend/scripts/p11-acceptance.mjs \ + backend/scripts/p11-acceptance.test.mjs \ + backend/scripts/acceptance-support.mjs \ + backend/scripts/acceptance-support.test.mjs \ + backend/scripts/p1-acceptance.mjs \ + backend/scripts/p1-acceptance.test.mjs +git commit -m "test: prove the P1.1 registry process end to end" +``` + +--- + +### Task 12: Build the separate P1.1 manual acceptance environment + +**Files:** +- Add: `scripts/p11-manual-acceptance.sh` +- Add: `scripts/test-p11-manual-acceptance.sh` +- Add: `backend/scripts/p11-manual-acceptance.mjs` +- Add: `backend/scripts/p11-manual-acceptance.test.mjs` +- Add: `backend/scripts/p11-render-snapshot.mjs` +- Add: `backend/scripts/p11-render-snapshot.test.mjs` +- Add: `docs/testing/p11-manual-acceptance.md` + +**Public lifecycle:** + +```bash +./scripts/p11-manual-acceptance.sh prepare +./scripts/p11-manual-acceptance.sh serve +./scripts/p11-manual-acceptance.sh stop +./scripts/p11-manual-acceptance.sh cleanup +``` + +**Fixed independent root:** + +```text +.artifacts/manual-acceptance/p11/ +``` + +`serve` owns two loopback-only processes so the reviewer can exercise both real surfaces without Docker: the production Fastify backend on `127.0.0.1:8791` and a production-built frontend preview on a second fixed loopback port recorded in ownership. The lifecycle manifest binds both executable/start identities and listeners; `stop` and `cleanup` refuse partial or foreign ownership. It must never read/copy P1 or P11 automated run state. + +**Step 1: Write failing lifecycle/ownership tests** + +Port P1's hardened manual safeguards to the distinct P11 namespace while preserving P1 tests unchanged: + +- prepare refuses existing/symlink/unowned roots and creates ownership before child resources; +- serve binds only the fixed backend and frontend-preview loopback ports with exact PID/start/executable/build identities; +- stop signals only the two owned process groups/listeners and fails closed on a partial identity mismatch; +- cleanup refuses live/foreign state and removes only P11 root; +- no helper writes `VERDICT.md` or marks manual PASS; +- renderer accepts only owned immutable snapshots/output, writes 0600 atomically, always releases leases, and leaves deterministic bytes; +- fixture/command generation cannot accept path escapes, wrong catalog/commit, old layout, or P1 roots. + +**Step 2: Run lifecycle tests and verify RED** + +```bash +bash scripts/test-p11-manual-acceptance.sh +``` + +Expected: FAIL because tooling does not exist. + +**Step 3: Generate a reviewer-owned walkthrough** + +`prepare` creates a new bare remote/clone with catalog slots and nested filesystem Evidence but no descriptors, fixture secrets, requests, command scripts, and `GUIDE.md`. It does not call any positive API operation for the reviewer. + +The guide requires the reviewer personally to: + +1. inspect catalog, nested workspace dirs, Evidence, ownership, and secret path bindings; +2. serve the production backend plus production-built frontend preview and inspect every owned loopback listener; +3. list `configuration_required` slots; +4. validate and bootstrap-create descriptors once; +5. inspect exact Git objects and separate generated docs; +6. retry create/update/delete and verify refusal plus unchanged object IDs; +7. edit existing descriptor and matching catalog metadata in the curator clone, commit/push/pull, and verify API did not rewrite curator bytes; +8. make an Evidence-only commit and inspect revision identity; +9. inspect live UI read-only existing workspace and editable missing-slot bootstrap behavior; +10. export/import under bootstrap-only rules; +11. render twice, diff, and run `tht config check`; +12. run negative catalog/path/secret cases and a bounded secret scan; +13. stop, inspect listener/PID cleanup, record `VERDICT.md`, and only then cleanup when desired. + +**Step 4: Run tooling tests** + +```bash +bash -n scripts/p11-manual-acceptance.sh scripts/test-p11-manual-acceptance.sh +bash scripts/test-p11-manual-acceptance.sh +``` + +Expected: PASS. + +**Step 5: Commit tooling and guide** + +```bash +git add \ + scripts/p11-manual-acceptance.sh \ + scripts/test-p11-manual-acceptance.sh \ + backend/scripts/p11-manual-acceptance.mjs \ + backend/scripts/p11-manual-acceptance.test.mjs \ + backend/scripts/p11-render-snapshot.mjs \ + backend/scripts/p11-render-snapshot.test.mjs \ + docs/testing/p11-manual-acceptance.md +git commit -m "test: add independent P1.1 manual acceptance" +``` + +--- + +### Task 13: Final verification, retained evidence, and handoff to owner review + +**Files:** +- Modify only after successful verification: `PROJECT_STATE.md` +- Do not modify: P2–P6 plans/designs in this task + +**Step 1: Run deterministic source gates** + +```bash +git diff --check +bash scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +bash scripts/test-verify-schema-v3-only.sh +./scripts/verify-schema-v3-only.sh +bash scripts/test-p11-acceptance.sh +bash scripts/test-p11-manual-acceptance.sh +``` + +Expected: PASS. + +**Step 2: Run complete backend verification** + +```bash +cd backend +npx vitest run +npx tsc --noEmit -p . +npm run build +``` + +Expected: PASS. Record exact test counts. + +**Step 3: Run complete frontend verification** + +```bash +cd frontend +npx vitest run +npx tsc -b +npm run build +``` + +Expected: PASS. Record exact test counts. + +**Step 4: Run harness regression verification** + +```bash +cd harness +.venv/bin/pytest -q +.venv/bin/ruff check \ + tht/config.py \ + tht/adapters/factory.py \ + tests/test_config_resources.py \ + tests/test_registry_evidence_config.py +``` + +Expected: pytest PASS and touched/relevant Python files Ruff-clean. Do not claim broad pre-existing Ruff debt is fixed unless `ruff check .` is also green. + +**Step 5: Run deployment/registry smokes once from clean state** + +Run the repository's normal deterministic deployment gates first, then each Docker smoke exactly once with its built-in timeout/ownership cleanup: + +```bash +./scripts/workspace-registry-smoke.sh +./scripts/unified-deployment-smoke.sh +``` + +Run the Windows native/clone contract in CI or an available supported Windows environment. If no Windows Docker runner is available, record the deterministic contract result and leave the manual Windows Docker gate explicitly unclaimed. + +Expected: PASS with exact cleanup and no global prune. + +**Step 6: Run one final P1.1 automated acceptance from clean state** + +```bash +./scripts/p11-acceptance.sh integration --keep +``` + +Expected: PASS, new unique retained path, reports bound to the clean implementation commit/tree and compiled graph immediately before the evidence-only PROJECT_STATE update. + +**Step 7: Audit forbidden scope and deferred plans** + +Use `git diff --name-only` plus targeted scans to prove: + +- no P2–P6 PRD/plan/design/manual-verification content was changed; +- no preprocessing/materialization/Qdrant/embedding implementation was added; +- no active runtime/doc/example still relies on flat `workspaces/.yaml` or `workspace-content//evidence`; +- any remaining old-path references are only historical P1 evidence/documents or the deliberately deferred P2–P6 sources inventoried for the later adjustment plan. + +**Step 8: Update project state to automated PASS/manual PENDING** + +Record exact source commit/tree, report paths/hashes, suite counts, smoke results, known limitations, and: + +```text +P1.1 automated integration: PASS +P1.1 manual acceptance: PENDING +``` + +Do not mark manual PASS. + +**Step 9: Prepare the independent manual environment and stop** + +```bash +./scripts/p11-manual-acceptance.sh prepare +``` + +Return the generated `GUIDE.md` path and lifecycle commands to the owner. Stop implementation work. Do not begin the P2–P6 adaptation plan before the owner completes and approves P1.1 manual acceptance. + +**Step 10: Commit final evidence metadata** + +```bash +git add PROJECT_STATE.md +git commit -m "docs: record P1.1 automated acceptance" +``` + +--- + +## Owner checkpoint after implementation + +The implementation session ends with: + +```text +P1.1 implementation: COMPLETE +P1.1 automated integration: PASS +P1.1 manual acceptance: PENDING +P2–P6 plans: UNCHANGED / ADAPTATION DEFERRED +``` + +The owner then executes `docs/testing/p11-manual-acceptance.md`. Only after an explicit manual PASS may a new planning-only task create the P2–P6 adaptation plan requested in steps 5–7 of the owner sequence. + +## Deferred P2–P6 impact inventory (do not edit during P1.1) + +The later adaptation-planning step must revisit at least: + +- `docs/prd/2026-08-09-workspace-preprocessing-prd.md` — old descriptor/Evidence layout and P1/P6 rows; +- `docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md` — P5 annotations path and P6 Evidence materialization path; +- `docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md` — exact descriptor/catalog identity, active snapshot fixture, and P1 dependency assumptions; +- `docs/testing/p2-p6-manual-verification.md` — old-path examples and future manual commands. + +Expected future canonical paths, subject to the separately approved adaptation plan: + +```text +/schema/annotations.yaml +/evidence +``` + +No P3/P4/P5/P6 implementation plan files currently exist separately; their present contract lives in the combined design/PRD and must be split or revised only in the later authorized phase. diff --git a/docs/superpowers/plans/2026-08-11-p2-p6-adaptation-to-p1-1-registry.md b/docs/superpowers/plans/2026-08-11-p2-p6-adaptation-to-p1-1-registry.md new file mode 100644 index 00000000..bcb06908 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-p2-p6-adaptation-to-p1-1-registry.md @@ -0,0 +1,228 @@ +# P2–P6 Adaptation to the P1.1 Workspace-Directory Registry — Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to apply this plan task-by-task. This plan only edits documentation (PRD, design, plans, manual verification); it changes no application code. + +**Goal:** Bring every P2–P6 planning artifact in line with the P1.1 repository contract (root catalog `thoth-workspaces.yaml`, `/workspace.yaml`, `/evidence`, `/schema/annotations.yaml`, generated docs `workspace-docs/`), so the future P2–P10 implementation starts from the correct layout and identity semantics. + +**Architecture:** The P1.1 contract is the single source of truth for registry layout and ownership. P2–P6 documents must stop referencing the retired flat layout (`workspaces/.yaml`, `workspace-content//evidence`) and must bind preprocessing identity to the catalog+descriptor+evidence objects at one immutable commit. + +**Tech Stack:** Markdown (PRD, specs, plans, manual verification). Verification is grep-based plus the existing schema/doc gates. + +**Approved design / source of truth:** `docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md` and the P1.1 implementation. + +--- + +## Completion contract + +The adaptation is complete when: + +1. No active P2–P10 planning artifact (PRD, P2–P6 design, P2 plan, P2–P6 manual verification, and any later P3–P6 plan files created after this adaptation) references `workspace-content//evidence`, `workspaces/.yaml`, or `workspaces//schema/annotations.yaml` as a canonical path. +2. Every reference to the canonical Evidence root uses `/evidence`; every reference to curated FK annotations uses `/schema/annotations.yaml`; every descriptor reference uses `/workspace.yaml`; the catalog is named `thoth-workspaces.yaml`. +3. Identity semantics state that a preprocessing run binds the exact workspace ID, the exact 40-hex commit, the catalog blob, the descriptor blob/digest, and (for filesystem Evidence/annotations) the Git objects at that same commit; a docs-only follow-up commit is a valid new revision even when descriptor/catalog blobs are unchanged. +4. The P1.1 supersession is recorded: P2 depends on P1.1, not on the P1 flat layout; the "active P1 snapshot" language becomes "active P1.1 snapshot" or "active registry snapshot". +5. Historical changelog/revision-history entries that describe the P1-era layout are preserved as history (they are not active contract); a superseded-note is added where a reader could mistake them for current contract. +6. P1/P1.1 docs, design, plans, and accepted evidence are not rewritten by this plan. +7. Grep gates and the existing verifier suites (`bash scripts/test-verify-workspace-install-docs.sh`, `./scripts/verify-workspace-install-docs.sh --fixtures-only`, schema-v3 gates) pass unchanged. + +## Explicit decisions frozen by this plan + +- Canonical paths after adaptation: + + ```text + catalog thoth-workspaces.yaml + workspace descriptor /workspace.yaml + embedded filesystem Evidence /evidence + curated FK annotations (P5) /schema/annotations.yaml + generated docs workspace-docs//{contract.env.example,README.md} + materialized Evidence (P6) /workspace-registry/snapshots///evidence + runtime annotation sync (P5) /sessions//revisions//artifacts/mschema/annotations.yaml + ``` + +- "Descriptor identity" in preprocessing language means the catalog entry plus the descriptor blob at one exact commit; the descriptor blob may stay byte-identical across a content-only or docs-only revision, so identity must bind the commit, not the blob alone. +- No application code changes are made by this plan. Any fixture/script path inside `docs/` prose that names old registry files is corrected only in prose. +- Historical P1 evidence (`.artifacts/p1-integration/**`, `.artifacts/manual-acceptance/p1/**`) and P1 acceptance documents remain untouched and are treated as immutable history. + +--- + +### Task 1: Record P1.1 supersession in the PRD + +**Files:** +- Modify: `docs/prd/2026-08-09-workspace-preprocessing-prd.md` + +**Step 1: Add a supersession notice** + +Insert a short status block near the top (after the header/status paragraph) stating that the P1-era layout is superseded by P1.1 (accepted 2026-08-11) and that all canonical paths in this PRD follow the P1.1 contract: `thoth-workspaces.yaml`, `/workspace.yaml`, `/evidence`, `/schema/annotations.yaml`, `workspace-docs/`. + +**Step 2: Replace the canonical path references in active requirement/decision/roadmap sections** + +Apply the path mapping: + +- line ~106-107 (`workspaces/.yaml` + `workspace-content//evidence/`): `workspaces/.yaml` → `/workspace.yaml`; `workspace-content//evidence/` → `/evidence/`. +- line ~141 (namespace confinement `workspace-content//`): → `/` (a workspace owns its top-level directory). +- line ~175-176 (RF5.1 PSD evidence tree): `workspace-content/psd/evidence/` → `psd/evidence/`; `workspace-content//evidence/` → `/evidence/`. +- line ~342-343 (D1): same mapping. +- line ~383 (D6): same mapping. +- roadmap rows P1 (~419) and P6 (~424): update the evidence-path phrases; P1 row may gain a note "P1.1" where it is referenced as the executed predecessor. +- line ~515 (v0.4 changelog): keep as history, add "(historical P1-era path; superseded by P1.1)" inline or leave and rely on the top supersession note — choose the inline parenthetical only if it does not rewrite the revision history content. + +Do not touch the preprocessing engine requirements (RF2–RF8) beyond path references. + +**Step 3: Verify with grep** + +```bash +git grep -n 'workspace-content/' -- docs/prd/2026-08-09-workspace-preprocessing-prd.md +``` + +Expected: only the historical changelog line(s) (if any kept as history) remain; all active contract lines use the P1.1 paths. + +**Step 4: Commit** + +```bash +git add docs/prd/2026-08-09-workspace-preprocessing-prd.md +git commit -m "docs: align PRD preprocessing contract with the P1.1 registry" +``` + +--- + +### Task 2: Align the P2–P6 design document + +**Files:** +- Modify: `docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md` + +**Step 1: Replace the P5 annotation source path** + +Line ~172: `workspace-content//schema/annotations.yaml` → `/schema/annotations.yaml`. Verify the surrounding prose still says the registry validates the blob at the same commit and rejects symlinks/trees/submodules; keep the runtime sync target at `/data/sessions//revisions//artifacts/mschema/annotations.yaml` unchanged. + +**Step 2: Replace the P6 materialization source path** + +Line ~209: `workspace-content//evidence` → `/evidence` (from the pinned commit into an immutable revision content root). Keep the materialized target under the snapshot content root and the containment/symlink rules unchanged. + +**Step 3: Tighten identity wording** + +Wherever the design binds "descriptor snapshot" or "exact descriptor snapshot", add the catalog: preprocessing binds workspace ID, exact 40-hex commit, catalog blob, descriptor blob/digest, and any same-commit Evidence/annotation objects. Add one sentence that a docs-only or content-only commit is still a distinct revision even when the descriptor blob is unchanged (the commit is authoritative). + +**Step 4: Verify with grep** + +```bash +git grep -n 'workspace-content/' -- docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md +``` + +Expected: zero matches. + +**Step 5: Commit** + +```bash +git add docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md +git commit -m "docs: align P2-P6 design with the P1.1 registry layout" +``` + +--- + +### Task 3: Align the P2 host-CLI plan + +**Files:** +- Modify: `docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md` + +**Step 1: Update dependency and identity language** + +- Completion contract (~line 22): "descriptor blob/digest" → "catalog blob and descriptor blob/digest"; "active, validated registry snapshot" stays, but ensure it means a P1.1 snapshot. +- State manifest (~lines 143-144): bind "revision, descriptor blob, config SHA-256" → "revision, catalog blob, descriptor blob, config SHA-256". +- Same-revision resume (~line 145): unchanged, but confirm the wording uses commit identity. +- Fixture topology (~line 476): "active P1 snapshot" → "active P1.1 snapshot (root catalog + `/workspace.yaml` + `/evidence`)". + +**Step 2: Verify** + +```bash +git grep -n 'active P1 snapshot\|workspace-content/\|workspaces/.yaml' -- docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md +``` + +Expected: zero matches. + +**Step 3: Commit** + +```bash +git add docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md +git commit -m "docs: align the P2 host-CLI plan with the P1.1 registry" +``` + +--- + +### Task 4: Align the P2–P6 manual verification document + +**Files:** +- Modify: `docs/testing/p2-p6-manual-verification.md` + +**Step 1: Update the annotation curation path** + +Line ~82: `workspace-content//schema/annotations.yaml` → `/schema/annotations.yaml`. Scan the rest of the file for any other old-layout examples (evidence paths, flat descriptor names, "P1 snapshot" phrasing) and apply the mapping. + +**Step 2: Verify** + +```bash +git grep -n 'workspace-content/\|workspaces/.yaml\|active P1 snapshot' -- docs/testing/p2-p6-manual-verification.md +``` + +Expected: zero matches. + +**Step 3: Commit** + +```bash +git add docs/testing/p2-p6-manual-verification.md +git commit -m "docs: align P2-P6 manual verification with the P1.1 registry" +``` + +--- + +### Task 5: Final grep gate and consistency sweep + +**Files:** +- None modified (verification only), or minimal edits if the sweep finds a straggler in the four P2–P6 artifacts. + +**Step 1: Grep the whole P2–P6 surface for retired paths** + +```bash +git grep -n 'workspace-content/' -- docs/prd docs/superpowers/specs docs/superpowers/plans docs/testing +git grep -n 'workspaces/.yaml' -- docs/prd docs/superpowers/specs docs/superpowers/plans docs/testing +``` + +Expected: no matches in active P2–P6 contract text. Allowed exceptions: the PRD historical changelog line (if deliberately retained with a historical note) and any P1-era historical plans that are explicitly marked superseded. + +**Step 2: Confirm new canonical paths appear where expected** + +```bash +git grep -n '/evidence\|thoth-workspaces.yaml' -- docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md docs/superpowers/plans/2026-08-10-p2-host-workspace-preprocessing-cli.md docs/testing/p2-p6-manual-verification.md +``` + +Expected: matches in each file where the layout is described. + +**Step 3: Run the existing gates** + +```bash +bash scripts/test-verify-workspace-install-docs.sh +./scripts/verify-workspace-install-docs.sh --fixtures-only +bash scripts/test-verify-schema-v3-only.sh +./scripts/verify-schema-v3-only.sh +``` + +Expected: PASS (this plan touches docs only; the gates must not regress). + +**Step 4: Commit any stragglers** + +```bash +git add docs +git commit -m "docs: finish P2-P6 path adaptation sweep" +``` + +(Only if Task 5 changed files; otherwise skip.) + +--- + +## Owner checkpoint + +After Task 5 the adaptation is complete and the implementation session stops for the final recap (step 8 of the owner sequence). The next owner action is to authorize the P2 implementation against the updated P1.1-based documents, then proceed with P2 (and later P3–P10) using the new canonical paths. + +## Non-goals + +- No change to preprocessing engine code, fixtures under `deploy/`, scripts, or the registry implementation. +- No change to P1/P1.1 design, plans, PROJECT_STATE acceptance blocks, or retained evidence. +- No migration of real repository content (that remains a curator operation documented in `docs/migrations/p1-to-p1-1-registry-layout.md`). diff --git a/docs/superpowers/plans/2026-08-11-p3-effective-config-and-tht-dwh.md b/docs/superpowers/plans/2026-08-11-p3-effective-config-and-tht-dwh.md new file mode 100644 index 00000000..2ef120da --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-p3-effective-config-and-tht-dwh.md @@ -0,0 +1,150 @@ +# P3 Effective Configuration and `.tht-dwh` Ownership — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Use superpowers:test-driven-development for every behavior change and superpowers:verification-before-completion before any completion claim. + +**Goal:** Make the effective DWH/preprocessing configuration reproducible and versioned across the operator and session paths, key reusable DWH generations by a stable logical identity instead of random temporary config files, scope schema/Evidence state to the pinned revision, add an explicit workspace-global memory root with a safe migration, and document `.tht-dwh` for operators. + +**Architecture:** A versioned shared canonicalizer (TS, shared by the backend session renderer and the compiled operator entrypoint) produces the non-secret effective DWH/preprocessing configuration and its stable logical config-source identity; the harness consumes the same canonical form when writing `OWNER.json`. DWH cache roots are keyed by the versioned effective DWH binding; revision-scoped runtime roots receive verified physical/LSH snapshots from that cache. An explicit `paths.memory` root becomes workspace-global; schema/Evidence Qdrant records and queries carry `workspace_revision` while memory/solved stay workspace-wide. Existing `OWNER.json` schema-v1 roots and legacy memory JSONL are read-compatible and migrated explicitly under the workspace lock; no in-place reinterpretation. + +**Tech Stack:** TypeScript 5, Node 22, Python 3.12 (harness), Pydantic 2, Qdrant, Vitest, pytest, Bash. Host interface remains `thothctl` (Go) unchanged in grammar; only its effective-config identity benefits. + +**Source contract:** `docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md` §5 (P3), PRD D3, and the P1.1 registry contract. + +--- + +## P3 completion contract + +P3 is complete only when all of the following are true: + +1. A **versioned shared canonicalizer** (`effective-config` TS module, plus the matching harness canonical form) derives the non-secret effective DWH/preprocessing configuration deterministically. Both the session runtime renderer (`ThtRunner`) and the operator entrypoint (`workspace-maintenance`) consume the same canonicalizer and produce byte-identical effective DWH bindings for the same workspace revision. +2. **Stable logical config-source identity** replaces dependence on random temporary config filenames: the operator config lease path and the `OWNER.json` `input_fingerprint` derive from the same canonical logical identity, so two runs over the same revision reuse the same DWH generation. +3. **Content-only revisions never invalidate DWH generations**: `session_storage` and `runtime_identity` remain excluded from the effective-config fingerprint; a Git content-only Evidence/annotation commit does not force introspection. A semantically identical revision (same effective DWH binding) reuses the existing generation. +4. **Effective-config changes fail closed**: a changed endpoint/transport/database/schema/root-affecting policy produces a different binding identity; the harness refuses to reuse a generation owned by a different effective configuration (`effective_config_mismatch` semantics) and the operator surfaces a stable code. +5. **`OWNER.json` schema-v1 compatibility**: existing schema-v1 roots remain readable; an explicit, reviewed migration upgrades them to the versioned identity without in-place reinterpretation. Migration is documented and tested; conflicting legacy roots fail closed. +6. **Explicit workspace-global memory root**: the rendered harness config includes `paths.memory` = `/sessions//memory`. All memory commands, locks, registry JSONL, and Qdrant projection rebuilds use that root. A migration copies and verifies one legacy canonical JSONL under the workspace lock before rebuilding the projection; conflicting legacy registries fail closed. +7. **Revision-scoped schema/Evidence state**: Qdrant schema and Evidence point IDs and queries include `workspace_revision`; `annotations`, corpus `ACTIVE`, and schema/Evidence Qdrant records are revision-scoped. Memory/solved records and queries remain workspace-wide. +8. **Reusable DWH cache layout**: a workspace cache keyed by the versioned effective DWH binding holds verified `physical.yaml`/LSH generations; each pinned revision's runtime root receives verified snapshots from that cache. No pinned runtime consumes the mutable cache root directly. +9. **Documentation**: `.tht-dwh`, immutable generations, `OWNER.json`, `ACTIVE`, input vs config fingerprints, safe migration, regeneration, and recovery are explained in the operator manuals and a dedicated `docs/contracts/tht-dwh.md`. +10. A clean-state P3 automated process goal passes without retry, retains machine/human reports and a secret scan, and proves exact cleanup; the P3 manual walkthrough remains PENDING until the owner decides. +11. No P4 work (collection lifecycle/rebuild), P5 (Git annotations), or P6 (Evidence materialization) is performed. + +## Frozen decisions + +- Canonicalizer version starts at `1`; the canonical serialization is a deterministic JSON document of the non-secret effective DWH/preprocessing configuration (DWH resource, roots, vector/embedding contract, evidence policy) with a fixed key order. +- Logical config-source identity = `@` (versioned), replacing the P2 `_config_source` fallback that used a temporary file path. +- `input_fingerprint` = sha256 of the logical config-source identity; `config_fingerprint` = sha256 of the canonical effective-config document (versioned). +- `paths.memory` is rendered by the backend for both session and operator; legacy configs without it continue to resolve memory beneath `artifacts/memory` only through the explicit migration path. +- Qdrant schema/Evidence record key and query filter include `workspace_revision`; memory/solved keep `workspace_id` only. +- No automatic remote migration or in-place reinterpretation of legacy roots; operators run the documented migration under the workspace lock. + +## Target file map + +**Shared canonicalizer (TS)** +- Create `backend/src/workspaces/effective-config.ts`, `effective-config.test.ts`: versioned canonical serialization, logical identity, fingerprint helpers. +- Modify `backend/src/workspaces/runtime-config-lease.ts`: lease naming/identity from the logical identity (deterministic, no random names for the same revision); publish the canonical effective-config identity in the lease manifest. +- Modify `backend/src/workspace-maintenance.ts` and `backend/src/workspaces/preprocessing-service.ts`: consume the canonicalizer; stable `effective_config_mismatch` surfacing. +- Tests: `backend/test/workspace-runtime-config-lease.test.ts`, `workspace-preprocessing-service.test.ts`, `workspace-maintenance.test.ts`, plus new `effective-config.test.ts`. + +**Harness (Python)** +- Modify `harness/tht/config.py`: canonical effective-config document and logical identity; `paths.memory` explicit root. +- Modify `harness/tht/jobs/dwh_pipeline.py`: `config_dwh_binding` consumes the canonical identity; `OWNER.json` schema-v1 compatibility reader + migration guard. +- Modify `harness/tht/cli/memory_cmd.py` (and locks/registry): explicit `paths.memory` root; migration of one legacy canonical JSONL under the workspace lock. +- Modify `harness/tht/vectorstore/records.py` and `harness/tht/adapters/vector/qdrant.py`: `workspace_revision` in schema/Evidence point IDs and queries. +- Tests: `harness/tests/test_dwh_preprocess_job.py`, `test_lsh_job_resume.py`, `test_memory_*.py`, `test_qdrant_*.py`, `test_registry_evidence_config.py`. + +**Docs** +- Create `docs/contracts/tht-dwh.md`. +- Modify `docs/install/local-workspace-registry.md`, `docs/install/server-workspace-registry.md`, `docs/testing/p2-p6-manual-verification.md` (P3 section). +- Modify after evidence exists: `PROJECT_STATE.md`. + +**Acceptance** +- Create `scripts/p3-acceptance.sh`, `scripts/test-p3-acceptance.sh`, `backend/scripts/p3-acceptance.mjs`, `backend/scripts/p3-acceptance.test.mjs` (pattern: P2 acceptance runner). + +--- + +### Task 1: Versioned shared canonicalizer and logical identity (TS) + +**Files:** create `backend/src/workspaces/effective-config.ts` + test; modify `backend/src/workspaces/runtime-config-lease.ts`, `workspace-maintenance.ts`, `preprocessing-service.ts` + tests. + +1. Write failing tests: canonical document is deterministic (byte-identical for equal inputs, key-ordered, versioned); logical identity is `@sha256:<64hex>`; fingerprint helpers produce `sha256:` values; `session_storage`/`runtime_identity` are excluded; a content-only revision (descriptor change without DWH-affecting fields) yields the same identity; a DWH endpoint/transport/database/schema/root-affecting change yields a different identity; the operator lease path for the same revision is deterministic (no random component for the same logical identity). +2. Implement the canonicalizer: fixed key order, non-secret fields only (never binding file contents, endpoints are allowed as non-secret identity inputs), versioned envelope. +3. Wire the operator lease manifest to carry `effectiveConfigIdentity`; replace the random lease-name component for the same revision with the deterministic identity suffix (retaining uniqueness across revisions). +4. Commit: `feat: versioned effective-config canonicalizer (P3)`. + +### Task 2: Harness canonical form and `OWNER.json` versioned identity + +**Files:** modify `harness/tht/config.py`, `harness/tht/jobs/dwh_pipeline.py` + tests. + +1. Failing tests: `config_dwh_binding` derives `config_fingerprint`/`input_fingerprint` from the versioned canonical document and logical identity; the schema-v1 `OWNER.json` shape remains readable; a versioned root is written with the new fields; a semantically identical revision reuses the generation; a changed endpoint fails closed (never reuses the old generation); `session_storage`/`runtime_identity` exclusion is preserved (content-only commit does not invalidate). +2. Implement: canonical JSON document + logical identity in the harness (mirror of Task 1, versioned); `OWNER.json` compatibility reader accepting schema-v1 keys and the versioned shape; explicit `effective_config_mismatch` refusal when a root belongs to a different canonical identity. +3. Commit: `feat: versioned OWNER.json identity and compatibility (P3)`. + +### Task 3: DWH cache keyed by effective binding + revision-scoped runtime roots + +**Files:** modify `harness/tht/jobs/dwh_pipeline.py`, runtime root selection (`harness/tht/config.py` roots), `backend/src/workspaces/runtime-config-lease.ts` + tests. + +1. Failing tests: the workspace DWH cache is keyed by the versioned effective binding; each pinned revision's runtime root receives verified `physical.yaml`/LSH snapshots from the cache; no pinned runtime reads the mutable cache root directly; a content-only commit changes the revision runtime root but reuses the cache; a binding change creates a new cache root and fails closed on reuse. +2. Implement cache/root separation and verified snapshot handoff (hash-verified copies under the revision runtime root). +3. Commit: `feat: effective-binding DWH cache with revision-scoped runtime roots (P3)`. + +### Task 4: Explicit workspace-global memory root + legacy migration + +**Files:** modify `harness/tht/config.py`, `harness/tht/cli/memory_cmd.py`, memory registry/lock; `backend/src/workspaces/runtime-config-lease.ts` (render `paths.memory`); tests. + +1. Failing tests: rendered config includes `paths.memory` = `/sessions//memory`; memory commands/locks/registry JSONL use it; migration copies and verifies exactly one legacy canonical JSONL under the workspace lock and rebuilds the Qdrant projection; two conflicting legacy registries fail closed; no in-place reinterpretation. +2. Implement the memory root plumbing and the guarded migration. +3. Commit: `feat: explicit workspace memory root with guarded migration (P3)`. + +### Task 5: Revision-scoped Qdrant schema/Evidence records + +**Files:** modify `harness/tht/vectorstore/records.py`, `harness/tht/adapters/vector/qdrant.py`, schema-index and evidence writers + tests. + +1. Failing tests: schema and Evidence point IDs include `workspace_revision`; schema/Evidence queries filter by `workspace_revision`; memory/solved identities and queries remain workspace-wide; existing workspace-global points are not silently reinterpreted (explicit re-index after migration is required). +2. Implement the revision-scoped record keys/filters. +3. Commit: `feat: revision-scoped schema and Evidence vector records (P3)`. + +### Task 6: Proofs — operator/session identity, reuse, fail-closed + +**Files:** extend `backend/test/workspace-runtime-config-lease.test.ts`, `workspace-preprocessing-service.test.ts`, `harness/tests/test_dwh_preprocess_job.py`. + +1. Failing tests (cross-layer): for the same workspace revision, the operator-rendered effective DWH binding is byte-identical to the session-rendered one; a semantically identical revision reuses the DWH generation (rerun `unchanged`); a changed endpoint/transport/database/schema/root-affecting policy fails closed with `effective_config_mismatch` (never silently reusing artifacts). +2. Implement any gap the tests expose (expect the canonicalizer to be the single shared source). +3. Commit: `test: prove operator/session effective-config identity (P3)`. + +### Task 7: Documentation — `.tht-dwh`, generations, fingerprints, migration, recovery + +**Files:** create `docs/contracts/tht-dwh.md`; modify install manuals and `docs/testing/p2-p6-manual-verification.md` (P3 section). + +1. Write the contract doc explaining: what `.tht-dwh` is, immutable generations, `OWNER.json` (schema-v1 vs versioned), `ACTIVE`, input vs config fingerprints, why a fingerprint protects against artifacts of another configuration, the safe migration procedure, regeneration, and recovery. +2. Update the operator manuals with the P3 migration step and the P3 walkthrough section (decision PENDING). +3. Commit: `docs: explain .tht-dwh and P3 migration (P3)`. + +### Task 8: Clean-state P3 automated process goal + +**Files:** create `scripts/p3-acceptance.sh`, `scripts/test-p3-acceptance.sh`, `backend/scripts/p3-acceptance.mjs`, `backend/scripts/p3-acceptance.test.mjs` (pattern: P2 acceptance runner, owned root `.artifacts/p3-integration/p3-/`). + +1. Write RED runner tests (ownership, run-id, cleanup, --keep, injected failure, report bounds, no retry). +2. Implement the clean-state scenario: build fixtures (P1.1 registry + REST DWH + HTTP Evidence + pre-provisioned Qdrant + embedding stub); run `thothctl` product commands; prove: identical operator/session effective binding, content-only revision reuse (`unchanged`), DWH-affecting change fails closed, memory migration + rebuild, revision-scoped Qdrant records, no P4/P5/P6 scope, secret scan, exact cleanup. +3. Run the runner tests, then one clean integration run without retry; retain the report (`report.md` ends with `P3 automated integration: PASS` / `P3 manual acceptance: PENDING`). +4. Commit: `test: add P3 effective-config process acceptance`. + +### Task 9: Final verification and owner handoff + +1. Full backend Vitest + tsc + build; full frontend Vitest + tsc + build (unchanged expectations); harness pytest (excluding the documented pre-existing debt) + Ruff on touched files; Go build/tests (unchanged grammar); compose config checks; docs gates; one final clean P3 acceptance run. +2. Update `PROJECT_STATE.md` (P3 implementation complete, automated PASS, manual PENDING) only after evidence exists. +3. Stop. No P4 work begins without a new explicit authorization. + +--- + +## Owner checkpoint + +After Task 9 the implementation session stops. The owner executes the P3 walkthrough in +`docs/testing/p2-p6-manual-verification.md` and records the decision. P4 (Qdrant collection +lifecycle), P5 (Git FK annotations), and P6 (Evidence materialization) start only after explicit +authorization. + +## Explicit exclusions + +- No collection create/repair/rebuild (P4), no Git annotation synchronization (P5), no Evidence + materialization (P6), no changes to the host `thothctl` grammar, no migration of real PSD + content, no changes to accepted P1/P1.1/P2 evidence. diff --git a/docs/superpowers/plans/2026-08-11-p4-qdrant-collection-lifecycle.md b/docs/superpowers/plans/2026-08-11-p4-qdrant-collection-lifecycle.md new file mode 100644 index 00000000..e8d0f537 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-p4-qdrant-collection-lifecycle.md @@ -0,0 +1,61 @@ +# P4 Qdrant Collection Lifecycle — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development (recommended) or superpowers:executing-plans. Use TDD and verification-before-completion. + +**Goal:** Own the Qdrant collection lifecycle with one shared manager: self-heal a missing/incomplete collection at session admission and in the operator path, refuse incompatible collections, and provide a guarded host CLI for inspection and destructive rebuild under a durable maintenance/quiescence protocol. + +**Architecture:** A shared TypeScript collection manager (`qdrant-collection.ts`) reconciles collection + payload keyword indexes. Session admission (`qdrantEnsure`) uses it to self-heal (create missing, add missing indexes) but never mutates incompatible collections (`semantic_index_incompatible`). The operator path uses the same manager with `require_existing` (no auto-create outside admission). `thothctl workspace vector inspect|rebuild` drive the maintenance service; rebuild requires exact workspace id + collection name confirmation + explicit destructive flag, and runs under a backend-mediated maintenance marker with a loopback quiescence endpoint (admission leases and PiProcessManager count zero), stopping core before the destructive job. + +**Tech Stack:** TypeScript 5, Node 22, Qdrant HTTP API, Go (thothctl), Fastify, Vitest, Go tests, Bash. + +**Source contract:** `docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md` §6 (P4), PRD D4, and P3 effective-config/revision contracts. + +## P4 completion contract + +1. One shared TS manager owns collection create/index reconciliation and validation; both session admission and the operator use it (operator path stays `require_existing`). +2. Admission self-heal: a missing collection is created with exactly 1024 dimensions, cosine distance, and the 8 required keyword payload indexes; missing indexes are added; an already-compatible concurrent creator is tolerated (re-read final state). +3. Incompatible dimensions/distance/index types are never mutated: admission and operator both return `semantic_index_incompatible`. +4. `thothctl workspace vector inspect --workspace [--json]` reports the collection contract without mutation. +5. `thothctl workspace vector rebuild --workspace --collection --confirm --destroy` deletes only the descriptor-owned collection and recreates the complete contract, under: installation lifecycle lock; backend maintenance marker activated durably; session inventory all closed/finalized/archived; loopback quiescence endpoint with zero admission leases and zero PiProcessManager count; core stopped and rechecked; no preprocessing lock held; durable rebuild state written before deletion. No prefix matching or global Qdrant mutation. +6. Failure after deletion leaves maintenance active and provides an explicit recovery/recreate command (never claims rollback of lost data); success restarts core and clears maintenance only after health verification. +7. Memory/solved and schema/Evidence payload contracts (P3 revision scoping) are preserved by the recreated collection. +8. A clean-state P4 automated process goal passes; P4 manual walkthrough stays PENDING. No P5/P6 work. + +## Target file map + +**TS collection manager + admission** +- Create `backend/src/workspaces/qdrant-collection.ts`, `qdrant-collection.test.ts`. +- Modify `backend/src/tht/tht-runner.ts` (`qdrantEnsure`) to use the manager (self-heal). +- Modify `backend/src/workspaces/preprocessing-service.ts` (operator path uses manager with `require_existing`). +- Modify `backend/src/runtime/maintenance-gate.ts`/maintenance state and add a loopback internal quiescence endpoint + admission-lease drain, tests. + +**Go host CLI** +- Modify `tools/thothctl/internal/workspaceops/operations.go` (+tests): `workspace vector inspect`, `workspace vector rebuild` with exact grammar, confirmation, destructive flag, lifecycle lock, maintenance activation (calls backend loopback), quiescence polling, stop/start core, durable rebuild state, recovery command. + +**Docs** +- Modify `docs/contracts/workspace-preprocessing-cli.md`, install manuals, `docs/testing/p2-p6-manual-verification.md` (P4 section). +- Modify after evidence: `PROJECT_STATE.md`. + +**Acceptance** +- Create `scripts/p4-acceptance.sh`, `scripts/test-p4-acceptance.sh`, `backend/scripts/p4-acceptance.mjs`, `backend/scripts/p4-acceptance.test.mjs` (pattern: P3 acceptance runner). + +### Task 1 — Shared TS collection manager (create/reconcile/validate) +Failing tests: creates missing collection with 1024/cosine; adds missing keyword indexes; tolerates concurrent compatible creator (re-read); refuses incompatible dimensions/distance/index with `semantic_index_incompatible`; never mutates incompatible. Implement manager over the Qdrant HTTP API; wire into `qdrantEnsure` (self-heal) and the operator (`require_existing`). Commit. + +### Task 2 — Durable maintenance marker + loopback quiescence endpoint +Failing tests: backend can durably activate maintenance (marker survives restart); a new loopback-only internal endpoint reports admission leases and PiProcessManager count; drain waits until both are zero; a maintenance marker refuses new admission; health/restart behavior defined. Implement; keep the endpoint loopback-only and unauthenticated-but-internal. Commit. + +### Task 3 — thothctl vector inspect and guarded rebuild +Failing tests (Go): `workspace vector inspect` reads the collection contract and emits pristine JSON; `workspace vector rebuild` requires exact `--workspace`, `--collection`, `--confirm `, `--destroy`; refuses mismatched confirmation or missing `--destroy`; acquires the installation lifecycle lock; asks the backend to activate maintenance; verifies session inventory closed; polls quiescence; stops core, rechecks; verifies no preprocessing lock; deletes only the descriptor collection; recreates + verifies; writes durable rebuild state before deletion; restarts core and clears maintenance only after health; on failure after deletion keeps maintenance and prints the explicit recovery command. No prefix/global mutation. Commit. + +### Task 4 — Docs + P4 walkthrough +Update `workspace-preprocessing-cli.md`, install manuals, and the P4 section of `docs/testing/p2-p6-manual-verification.md` (decision PENDING). Commit. + +### Task 5 — Clean-state P4 automated process goal +P4 acceptance runner (pattern P3): bootstrap fixtures (P1.1 registry + REST DWH + HTTP evidence + embedding stub); pre-provision Qdrant; run thothctl product commands; prove: admission self-heal creates the collection on a fresh volume, incompatible collection refused, `vector inspect` contract, `vector rebuild` guarded flow with confirmation/destroy and exact cleanup, collection recreated with the 8 indexes + revision-scoped payload contract, no P5/P6 scope, secret scan, cleanup. Run runner tests, then one clean integration run without retry; report ends `P4 automated integration: PASS` / `P4 manual acceptance: PENDING`. Commit. + +### Task 6 — Final verification + owner handoff +Full backend/frontend/harness/Go gates; one final clean P4 acceptance run; update `PROJECT_STATE.md`; stop. No P5 work without a new authorization. + +## Exclusions +No Evidence materialization (P6), no Git FK annotations (P5), no changes to accepted P1.1/P2/P3 evidence, no migration of real PSD content. diff --git a/docs/superpowers/plans/2026-08-13-p5-curated-fk-annotations-in-git.md b/docs/superpowers/plans/2026-08-13-p5-curated-fk-annotations-in-git.md new file mode 100644 index 00000000..a4186310 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-p5-curated-fk-annotations-in-git.md @@ -0,0 +1,197 @@ +# P5 — Curated FK annotations in Git — Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to apply this plan task-by-task. + +**Goal:** Make the curated FK annotation file the canonical, revision-pinned human review input. The +registry validates `/schema/annotations.yaml` as a regular Git blob at the same commit +as the descriptor, synchronizes it to an immutable revision-qualified runtime root on activation, and +replaces the P2 host-file FK review with an explicit operator command +`workspace schema accept --run --yes`. A pinned historical runtime keeps reading its own +revision's annotations; a newer active revision writes a different directory. + +**Source of truth:** PRD D5 (`docs/prd/2026-08-09-workspace-preprocessing-prd.md`) and design §7 +(`docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md`). + +**Architecture:** The backend registry and the compiled operator entrypoint share the sync logic. Git +reads use fixed plumbing (`rev-parse`, `cat-file -t`, `show`) at an exact 40-hex commit — never a +mobile checkout and never author files. The harness keeps owning the annotation *parser* (Pydantic +`Annotations`) and the physical-schema orphan check. + +**Tech Stack:** TypeScript (backend registry/preprocessing/runtime rendering), Go (`thothctl`), +Python (`tht schema`), YAML. TDD throughout. + +--- + +## Current-state findings recorded by this plan + +- P3 implemented revision-scoped Qdrant schema/Evidence records and a binding-keyed DWH cache + (`.tht-dwh` at `paths.artifacts.parent`), but it did **not** repurpose `paths.artifacts`/`indexes` + into a revision root (they remain workspace-global under `/data/sessions//`). +- The harness resolves curated annotations at `paths.artifacts/mschema/annotations.yaml` and already + parses them with `Annotations.from_yaml`; `tht schema check` performs the physical orphan check. +- P2 already records FK candidates and an `FkReviewRecord` + (`{ reviewedCandidatesDigest, annotationsDigest, workspaceRevision }`) and writes host-file reviews + from `schema check --annotations --reviewed-candidates`. + +## Explicit decisions frozen by this plan + +1. **Canonical path** is fixed `/schema/annotations.yaml` (not descriptor-configurable). + Absence is compatible and yields an empty canonical annotation set plus a warning. Symlinks, + submodules/trees at the file path, cross-namespace paths, oversized, non-UTF-8, and malformed + annotations are rejected before activation. +2. **Revision-qualified annotations root** is rendered as a new explicit path + `paths.annotations_root = /data/sessions//revisions//artifacts`; the immutable synced + file is `/mschema/annotations.yaml`. `paths.artifacts`/`indexes`/`memory`/ + `sessions` remain exactly as accepted by P3 because the binding-keyed DWH cache lives at + `artifacts.parent` and must stay shared across content-only revisions. This is a deliberate, + surgical refinement of design §7's literal "artifacts and indexes select the revision root": the + revision-pinned *annotations* requirement is satisfied without destabilizing the accepted P3 cache + contract. The harness resolves annotations from `paths.annotations_root` when present and falls + back to the legacy `artifacts/mschema/annotations.yaml` for unmigrated workspaces. +3. **Bounds:** the annotation blob is ≤ 16 MiB, UTF-8, and structurally parsed (Pydantic `Annotations`) + before synchronization; the full physical orphan check still runs at review time. +4. **Sync trigger:** on registry activation (pull/validate) and before session admission or + preprocessing, each active revision's annotation blob is read with fixed Git argv, validated, and + atomically written no-follow to its revision root with restrictive mode, alongside an ownership + manifest `{ workspace, commit, blobId, contentDigest, destination }`. Re-sync is idempotent and + re-verifies the manifest. +5. **Human review primitive is `workspace schema accept --run --yes`.** After commit/push/pull, + the operator reviews the current Git blob against the recorded candidate, then runs the accept + command. It parses the current blob, validates it against the physical schema via the harness + parser, and records `{ reviewedCandidatesDigest, annotationsDigest, workspaceRevision }` plus the + current Git blob id and the new revision. `--yes` is required. An empty file or `schema check` + alone is **not** evidence of human review. +6. **Continuation gate:** `preprocess run` FK review now requires an accepted review whose + `annotationsDigest` equals the *current* revision's synced blob digest and a compatible reusable + DWH binding; otherwise the run starts a new review. The P2 host-file review path + (`schema check --annotations --reviewed-candidates` writing an `FkReviewRecord`) is superseded: + `schema check` remains available as read-only validation but no longer records a review. +7. **Error/output:** reuse `annotation_invalid`, `manual_review_required`, and + `preprocessing_resume_mismatch`; JSON results gain the accepted `blobId`/`annotationsDigest` + artifact identities. No new public error code is introduced unless a gap is proven by a test. +8. **No push/curation:** the preprocessing CLI never stages, commits, or pushes curated content. + Curators work in an ordinary author clone. + +## Completion contract + +The phase is complete when: + +1. A workspace whose Git tree contains a valid `/schema/annotations.yaml` blob activates and + syncs it to exactly `/data/sessions//revisions//artifacts/mschema/annotations.yaml` + with a verified ownership manifest; a workspace without the file activates with a warning and an + empty canonical set. +2. Symlink/tree-at-path, cross-namespace, oversized (>16 MiB), non-UTF-8, and malformed annotation + objects are refused without mutating the snapshot or runtime roots. +3. The harness resolves annotations from `paths.annotations_root` (legacy fallback preserved); a + session pinned to an older revision reads that revision's synced annotations, and a newer active + revision writes/reads a different directory. +4. `thothctl ... workspace schema accept --run --yes` records the accepted candidate/current-blob + digests and the new revision; `--yes` missing, an unknown run, an empty file, a malformed blob, or + a blob not matching the recorded candidate fails closed without recording a review. +5. `preprocess run` continuation succeeds only with the exact accepted blob digest and compatible DWH + binding; the superseded host-file `schema check` path no longer records a review. +6. `docs/contracts/workspace-preprocessing-cli.md` documents `schema accept` and the annotations + lifecycle; the P5 manual walkthrough section is runnable; PROJECT_STATE.md records the result. +7. The clean-state automated process goal passes 1/1 (no retry), and backend/Go/harness focused + suites plus the existing P2–P4 gates do not regress. + +--- + +### Task 1: Registry reads and validates the annotation blob at the exact commit + +**Files:** modify `backend/src/workspaces/git-repository.ts`, `backend/src/workspaces/registry.ts`; add +tests `backend/test/registry-annotations.test.ts`. + +1. Failing tests: `gitObjectType`-style read of `/schema/annotations.yaml` at an exact commit + returns `blob` or absent; a `tree`/`submodule`/other type is refused; the blob id (`rev-parse`) and + bytes (`show`) match; UTF-8 and 16 MiB bounds are enforced; path grammar rejects + `workspace-docs/...` and cross-namespace paths. +2. Implement `GitWorkspaceRepository.annotationsObject(revision, id)` returning + `{ blobId, type, contents } | undefined` with fixed Git argv and bounded sanitized errors. +3. In `WorkspaceRegistry.activate`, validate every active revision's annotation object; a present-but- + invalid object fails activation closed (`workspace_invalid`), absence is a safe warning. +4. Commit: `feat: read and validate curated FK annotations at the pinned commit (P5)`. + +### Task 2: Atomic revision-qualified annotations sync + ownership manifest + +**Files:** add `backend/src/workspaces/annotations-sync.ts`; wire into activation and +`renderActiveWorkspaceRuntime`; tests `backend/test/annotations-sync.test.ts`. + +1. Failing tests: sync writes `/sessions//revisions//artifacts/mschema/ + annotations.yaml` (mode restrictive, no-follow, exclusive staging + atomic rename + fsync) and an + adjacent ownership manifest `{ workspace, commit, blobId, contentDigest, destination }`; re-sync is + idempotent and re-verifies the manifest; a tampered destination or wrong manifest fails closed; + a different revision writes a different directory. +2. Implement the sync (shared by registry activation and the operator/session runtime render). +3. Commit: `feat: atomic revision-qualified annotations sync with ownership manifest (P5)`. + +### Task 3: Render `paths.annotations_root` and make the harness resolve it + +**Files:** modify `backend/src/workspaces/runtime-config-lease.ts`, `harness/tht/config.py`, +`harness/tht/cli/schema_cmd.py`; tests both layers. + +1. Failing tests: rendered config includes `paths.annotations_root = + /data/sessions//revisions//artifacts` while `paths.artifacts`/`indexes`/`memory`/ + `sessions` stay unchanged; `tht schema` `annotations_path` prefers `paths.annotations_root` and + falls back to the legacy `artifacts/mschema/annotations.yaml` when absent; a missing annotations + file yields an empty canonical set (not a crash). +2. Implement the render field and harness resolution with the legacy fallback. +3. Commit: `feat: revision-qualified annotations root for pinned runtimes (P5)`. + +### Task 4: Operator `workspace schema accept --run --yes` + +**Files:** modify `backend/src/workspaces/preprocessing-service.ts`, +`backend/src/workspace-maintenance.ts`, `tools/thothctl/internal/workspaceops/operations.go`, +`tools/thothctl/cmd/thothctl/main.go`; tests `workspace-preprocessing-service.test.ts` and +`operations_test.go`. + +1. Failing tests: the accept command reads the current synced Git blob, stages it, validates it with + the harness parser (structural + orphan check against the recorded candidate), and records + `{ reviewedCandidatesDigest, annotationsDigest, workspaceRevision }` plus `blobId`; missing + `--yes`, unknown run, empty/malformed blob, and non-matching candidate fail closed with no review; + the recorded review is keyed by the run id. +2. Implement `WorkspacePreprocessingService.acceptSchema`, `workspace-maintenance` dispatch + (`schema-accept`), and the `thothctl` grammar/validation/execute path. +3. Commit: `feat: operator schema accept command for curated FK review (P5)`. + +### Task 5: Continuation gate on the accepted blob; supersede host-file review + +**Files:** modify `backend/src/workspaces/preprocessing-service.ts` (+ tests). + +1. Failing tests: `preprocess run` FK review requires an accepted review whose `annotationsDigest` + equals the current revision's synced blob digest and a compatible DWH binding; a digest mismatch + starts a new review (`manual_review_required`); the host-file `schema check --annotations + --reviewed-candidates` path validates but does not record a review. +2. Implement the gate and the supersession. +3. Commit: `feat: gate FK review on the accepted revision blob (P5)`. + +### Task 6: Contract, manual walkthrough, and clean-state acceptance + +**Files:** modify `docs/contracts/workspace-preprocessing-cli.md`, +`docs/testing/p2-p6-manual-verification.md` (P5 section), `PROJECT_STATE.md`; add +`scripts/p5-acceptance.sh`, `scripts/test-p5-acceptance.sh`, `backend/scripts/p5-acceptance.mjs`, +`backend/scripts/p5-acceptance.test.mjs` (pattern: P4 acceptance, owned root +`.artifacts/p5-integration/p5-/`). + +1. Update the CLI contract (new command, annotations lifecycle, exit codes, JSON fields). +2. Implement the clean-state scenario: fixture P1.1 registry + curated annotations + REST DWH + + pre-provisioned Qdrant; run `thothctl` product commands; prove activation sync + ownership + manifest, revision isolation, accept happy path, `--yes`/empty/malformed/mismatch negatives, the + continuation gate, no push of curated content, secret scan, exact cleanup. +3. Finalize the P5 manual walkthrough section and record the phase in PROJECT_STATE.md. +4. Commit: `feat: P5 curated FK annotations in Git (acceptance + docs)`. + +--- + +## Owner checkpoint + +After Task 6 the implementation stops for recap. The owner records the P5 manual acceptance +(automated PASS is never recorded as manual PASS), then authorizes P6. + +## Non-goals + +- No GUI/backend preprocessing endpoint, no push/stage/commit of curated content. +- No P6 filesystem Evidence materialization (the `evidence_materialization_required` stop remains). +- No real PSD migration, no SSH runtime transport, no policy-driven GC. +- No change to the accepted P1/P1.1/P2/P3/P4 contracts or retained evidence beyond the documented + P5 supersession of the host-file FK review. diff --git a/docs/superpowers/plans/2026-08-13-p6-commit-addressed-evidence-materialization.md b/docs/superpowers/plans/2026-08-13-p6-commit-addressed-evidence-materialization.md new file mode 100644 index 00000000..5a15963a --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-p6-commit-addressed-evidence-materialization.md @@ -0,0 +1,179 @@ +# P6 — Commit-addressed Evidence materialization — Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to apply this plan task-by-task. + +**Goal:** Materialize the filesystem Evidence tree `/evidence` from the exact pinned Git +commit into an immutable revision content root, verify real containment (no symlink/gitlink/ +traversal/special-file escape), and make P2's filesystem Evidence path operational end-to-end by +removing the temporary `evidence_materialization_required` stop. + +**Source of truth:** PRD D6 (`docs/prd/2026-08-09-workspace-preprocessing-prd.md`) and design §8 +(`docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md`). + +**Architecture:** A new shared TypeScript materializer enumerates the tree with fixed Git plumbing +(`ls-tree -r -z` + `cat-file blob`) and writes regular files no-follow/exclusive beneath a fresh +owned staging directory, hashing every file into a bounded manifest. The registry runs it during +snapshot staging so the materialized root lands atomically inside the already-retained, commit- +addressed snapshot directory; a tampered or mismatched manifest fails closed. The runtime renderer +and the harness filesystem Evidence adapter are already rooted at that directory and need no change. + +**Tech Stack:** TypeScript (registry + materializer), Python (existing filesystem Evidence adapter), +YAML. TDD throughout. + +--- + +## Current-state findings recorded by this plan + +- The registry already validates the filesystem Evidence *root object* is a Git tree at the pinned + commit (`assertTreeAtRevision`) and freezes it as `revisionContentRoot = dirname(snapshotPath)`. +- `renderEvidence` already resolves filesystem Evidence to `join(revisionContentRoot, source.uri)` + (`///evidence`), and the harness `FilesystemEvidenceSource` reads exactly + that directory with no-follow opens and `**/*.md` discovery. +- `WorkspacePreprocessingService.evidencePolicy` currently returns + `evidence_materialization_required` for filesystem sources (the P2 temporary stop). +- `reconcileSnapshotRetention` removes whole commit-addressed `//` directories, + so materialized evidence beneath that directory is automatically retained while pinned and removed + only when the revision becomes unreferenced. +- The registry `activate()` staging already writes immutable `.yaml`/`.env.example`/`.md` + + `snapshot.json` and renames atomically; the comment at `expectedSnapshotFiles` marks P6 as the + owner of workspace-content materialization. + +## Explicit decisions frozen by this plan + +1. **Target layout.** Materialized filesystem Evidence lives at + `/snapshots///evidence/` with a sibling bounded manifest + `/snapshots///evidence.manifest.json`. The manifest records + `{ workspace, commit, tree, entryCount, totalBytes, files: { "": { mode, oid, digest, bytes } } }`. + The sibling manifest is outside the discovery root so the Evidence adapter never ingests it. +2. **Eager, fail-closed materialization at activation.** During `activate()` snapshot staging, every + filesystem-Evidence workspace is materialized before the staging directory is atomically renamed. + A missing Evidence root, an unsafe object, a bound violation, or a write failure aborts activation + (`workspace_invalid`); no partial root is published. An empty Evidence tree is valid (empty root + + zero-entry manifest). +3. **Fixed Git plumbing, no shell, no mobile checkout.** Enumeration is + `git ls-tree -r -z -- /evidence`; blob bytes come from `git cat-file blob ` + (buffer, per-object bound). No archive is extracted and no author files are consulted. +4. **Object safety.** Reject at any depth: symlink (`120000`), gitlink/submodule (`160000`), non- + regular modes other than `100644`/`100755`, non-`blob` type, absolute/`.`/`..`/NUL/newline/ + non-normalized paths, duplicate normalized paths, cross-workspace namespaces, and any object whose + id or bytes change between enumeration and read. +5. **Bounds.** Installation-local non-secret limits with conservative defaults: + `maxEvidenceEntries` (4096 files), `maxEvidenceBytes` (64 MiB total), `maxEvidenceFileBytes` + (8 MiB per file), `maxEvidencePathBytes` (4096 total, 255 per segment), `maxEvidenceManifestBytes` + (1 MiB). The materializer sums `cat-file -s` sizes before writing as a disk-space preflight and + streams blobs so per-file-valid adversarial trees cannot exhaust memory or inodes. +6. **Integrity chain.** The snapshot `snapshot.json` manifest gains an entry + `.evidence.manifest.json` (its sha256) for every filesystem-Evidence workspace; the existing + `assertManifestFiles` chain therefore verifies the evidence manifest before reuse. On re-activation + of a commit, an already-materialized root is reused only when its manifest digest matches the + snapshot manifest; a missing or mismatched manifest fails closed (never silently reuses). +7. **Stop removal.** `evidencePolicy` no longer blocks filesystem sources; `preprocess evidence` and + `preprocess run` proceed against the materialized root. HTTP/S3 evidence behavior is unchanged. +8. **No GC change.** Retention of materialized roots is inherited from the commit-addressed snapshot + directory; no separate cleanup owns Evidence files. + +## Completion contract + +The phase is complete when: + +1. A workspace whose pinned commit contains a valid `/evidence` tree activates and materializes + every regular blob to `///evidence/` with a verified sibling manifest whose + digest appears in `snapshot.json`; a filesystem-Evidence workspace preprocesses, indexes, and + re-runs idempotently through the existing engine (no `evidence_materialization_required`). +2. Symlink/gitlink at any depth, traversal/absolute/duplicate/cross-namespace paths, oversized + files, and total/entry/path/manifest bound violations are refused without publishing a partial + root; the previous valid snapshot remains active. +3. Re-activation of the same commit reuses a valid materialized root and fails closed on a tampered + evidence manifest or file digest mismatch. +4. A pinned historical revision retains its materialized root; an unreferenced revision's root is + removed together with its snapshot directory by the existing retention scan. +5. `docs/contracts/workspace-preprocessing-cli.md` (or a dedicated P6 contract section) and the P6 + manual walkthrough are runnable; PROJECT_STATE.md records the result. +6. The clean-state automated process goal passes 1/1 (no retry), and backend/Go/harness focused + suites plus the existing P1.1–P5 gates do not regress. + +--- + +### Task 1: Safe Git tree enumeration + bounded blob streaming + +**Files:** modify `backend/src/workspaces/git-repository.ts`; tests +`backend/test/workspaces-git-evidence.test.ts`. + +1. Failing tests: `evidenceTreeObjects(revision, id)` returns ordered regular-blob entries + (`mode`, `oid`, `posixPath`) for a valid `/evidence` tree, and refuses symlink/gitlink/ + non-regular modes, non-blob types, traversal/absolute/NUL/newline/duplicate/cross-namespace paths, + and malformed revisions; `gitBlobBuffer` returns bounded bytes and refuses oversized objects. +2. Implement enumeration (`ls-tree -r -z`, path grammar, mode/type checks, duplicate detection) and + bounded blob reads (`cat-file blob`, `maxBuffer` + size guard). +3. Commit: `feat: safe Evidence tree enumeration and bounded blob streaming (P6)`. + +### Task 2: Evidence materializer with manifest and atomic publication + +**Files:** add `backend/src/workspaces/evidence-materialization.ts`; tests +`backend/test/evidence-materialization.test.ts`. + +1. Failing tests: materialize a fixture tree into a fresh owned staging root with exclusive/no-follow + writes, per-file hashes, an ordered manifest, fsync + atomic rename; refuse symlink/gitlink/ + special-file/traversal entries; enforce entry/total/per-file/path/manifest bounds (including a + size-sum preflight); a tampered destination or manifest fails closed on reuse. +2. Implement `materializeEvidenceTree({ repository, revision, id, stagingParent, limits })` returning + `{ root, manifestPath, manifest, manifestDigest }`. +3. Commit: `feat: bounded Evidence materializer with manifest and atomic publication (P6)`. + +### Task 3: Registry activation integration + integrity chain + +**Files:** modify `backend/src/workspaces/registry.ts`, `backend/src/workspaces/types.ts`, +`backend/src/config.ts`; tests `backend/test/registry-evidence.test.ts`. + +1. Failing tests: activation with a filesystem-Evidence workspace materializes the tree inside the + staged snapshot directory, writes the sibling manifest, records its digest in `snapshot.json`, + and atomically renames; re-activation reuses a valid root and fails closed on a tampered manifest; + an unsafe tree leaves the previous valid snapshot active; the evidence limits are configurable + through `WorkspaceRegistryConfig`. +2. Implement the staging integration, manifest-digest recording, integrity verification, and the new + config limits with env defaults. +3. Commit: `feat: activate commit-addressed Evidence materialization with an integrity chain (P6)`. + +### Task 4: Remove the filesystem Evidence stop + +**Files:** modify `backend/src/workspaces/preprocessing-service.ts`; tests +`workspace-preprocessing-service.test.ts`. + +1. Failing tests: `preprocess evidence` and `preprocess run` on a filesystem-Evidence workspace no + longer return `evidence_materialization_required` and instead invoke the evidence stage; HTTP/S3 + policy guards still apply unchanged. +2. Implement the `evidencePolicy` change. +3. Commit: `feat: make filesystem Evidence operational after materialization (P6)`. + +### Task 5: Contract, manual walkthrough, and clean-state acceptance + +**Files:** modify `docs/contracts/workspace-preprocessing-cli.md`, +`docs/testing/p2-p6-manual-verification.md` (P6 section), `PROJECT_STATE.md`; add +`scripts/p6-acceptance.sh`, `scripts/test-p6-acceptance.sh`, `backend/scripts/p6-acceptance.mjs`, +`backend/scripts/p6-acceptance.test.mjs` (pattern: P5 acceptance, owned root +`.artifacts/p6-integration/p6-/`). + +1. Document the Evidence lifecycle, limits, and exit codes. +2. Implement the clean-state scenario: fixture P1.1 registry with a filesystem Evidence tree + REST + DWH + pre-provisioned Qdrant; run `thothctl` product commands; prove materialization + manifest, + preprocessing/idempotency, revision-filtered Qdrant retrieval and corpus ACTIVE, unsafe-tree and + bound negatives without partial publication, retention while pinned and cleanup after release, + secret scan, exact cleanup. +3. Finalize the P6 manual walkthrough section and record the phase in PROJECT_STATE.md. +4. Commit: `feat: P6 commit-addressed Evidence materialization (acceptance + docs)`. + +--- + +## Owner checkpoint + +After Task 5 the implementation stops for recap. The owner records the P6 manual acceptance +(automated PASS is never recorded as manual PASS), then authorizes the final aggregate P2–P6 +verification and the user-guide deliverable. + +## Non-goals + +- No HTTP/S3 Evidence changes (they remain supported as before). +- No real PSD migration, SSH runtime transport, or policy-driven GC beyond the existing snapshot + retention. +- No change to the accepted P1/P1.1/P2/P3/P4/P5 contracts or retained evidence beyond the documented + P6 removal of the temporary filesystem stop. diff --git a/docs/superpowers/plans/2026-08-13-p7-psd-migration.md b/docs/superpowers/plans/2026-08-13-p7-psd-migration.md new file mode 100644 index 00000000..55d1fae3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-13-p7-psd-migration.md @@ -0,0 +1,122 @@ +# P7 — PSD migration to the workspace registry — Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to apply this plan task-by-task. + +**Goal:** Move the existing Policlinico San Donato workspace from the legacy flat layout +(`tht-workspace-psd/psd.yaml` + root `evidence/` + `artifacts/mschema/annotations.yaml`) into the +P1.1 workspace registry repository, then re-embed/re-index it on the new internal semantic stack so +ThothII can run live against the real PSD DWH. + +**Source of truth:** PRD D7 (`docs/prd/2026-08-09-workspace-preprocessing-prd.md`), the P1.1 layout +(`docs/superpowers/plans/2026-08-11-p1-1-workspace-directory-registry.md`), and +`docs/workspace-diagnostic-protocol.md`. + +**Architecture:** The PSD content becomes an ordinary Git workspace repository consumed by ThothII +via `THT_WORKSPACE_GIT_REMOTE`. DWH access stays REST (PostgREST); Qdrant and Ollama embedding are +the internal Compose services. The legacy pgvector (768-dim, nomic) is superseded and must be +re-embedded with `qwen3-embedding:0.6b` (1024-dim). + +**Tech Stack:** Git + YAML (descriptor/catalog), the existing `thothctl`/registry for validation. +No new code is required unless a validator gap is proven by a test. + +--- + +## Current-state findings recorded by this plan + +- `/Users/mp/projects/tht-workspace-psd` is already a Git repo on `main` (38 tracked files: + `.gitignore`, legacy `psd.yaml`, and 36 curated `evidence/*.md`) with **no remote**. +- The legacy `psd.yaml` declares REST DWH (`database: postgres`, `schema: datawarehouse`), X-API-Key + auth, internal CA, and the old external pgvector + Ollama; it has **no** `llm_policy`. +- Curated FK annotations exist at `artifacts/mschema/annotations.yaml` (367 FK lines) in the new + canonical `Annotations` shape and can be copied directly to `/schema/annotations.yaml`. +- `physical.yaml` is absent, so DWH introspection must be re-run (requires VPN + DWH access). +- The legacy runtime dirs (`.tht-dwh/`, `.tht-jobs/`, `config/`, `corpus/`, `runtime-v2/`, + `.legacy-artifacts-backup-premerge/`) are untracked runtime state and must not enter the curated repo. + +## Explicit decisions frozen by this plan + +1. **Repository identity.** Reuse `/Users/mp/projects/tht-workspace-psd` as the author/curator clone; + publish it to a GitHub remote the owner creates. The ThothII installation clones that remote, not + any path inside the ThothII repo. +2. **Workspace id:** `psd-clinical` (catalog, descriptor, Qdrant collection, bindings namespace + `PSD_CLINICAL`). +3. **Descriptor (schema v3):** `dwh` = `postgres` / database `postgres` / schema `datawarehouse` / + `supported_transports: [rest_api]`; `semantic_index` = Qdrant `psd-clinical` 1024/cosine + + `qwen3-embedding:0.6b`; `language: it`; `llm_policy.allowed` starts from the historically active + PSD models (`zai/glm-5.2`, `deepseek/deepseek-v4-flash`, `deepseek/deepseek-v4-pro`, + `aritmolab/qwen3.6-35b-a3b`) and is owner-adjustable. +4. **DWH diagnostic:** `diagnostics.dwh_rest` = `POST /rpc/ping`, `auth: x-api-key`, response + `{ database: postgres, schema: datawarehouse }`. The exact ping RPC path/auth is verified by the + owner during the first live smoke and adjusted only in the Git descriptor. +5. **Curated content only:** the repo contains `thoth-workspaces.yaml`, `psd-clinical/workspace.yaml`, + `psd-clinical/evidence/`, `psd-clinical/schema/annotations.yaml`, plus API-generated + `workspace-docs/`. Legacy runtime dirs stay untracked (gitignore). +6. **Re-embedding:** the legacy pgvector is not reused. DWH introspection + schema/Evidence indexing + run afresh on the new stack (or, if the owner prefers, pgvector is exported and re-embedded); + this task is gated on VPN + DWH credentials + a running stack. +7. **Secrets stay out of Git:** DWH X-API-Key and CA are written as local secret files referenced by + `THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE` / `_TLS_CA_FILE`. + +## Completion contract + +1. `tht-workspace-psd` is restructured to the P1.1 layout and commits cleanly (no legacy runtime dirs). +2. A local registry bootstrap against that repository activates `psd-clinical` (schema-v3 valid, + Evidence materialized, annotations parsed and synced, docs generated). +3. `thothctl … workspace inspect --workspace psd-clinical --json` succeeds once the installation + bindings + VPN are present. +4. `thothctl … workspace preprocess run --workspace psd-clinical --json` completes DWH → FK → schema → + Evidence against the real DWH and indexes into the internal Qdrant. +5. A live session on `psd-clinical` reaches the first reviewer gate (P8 L2 smoke). + +--- + +### Task 1: Restructure the repository (autonomous) + +**Repo:** `/Users/mp/projects/tht-workspace-psd` (separate checkout). + +1. Write `thoth-workspaces.yaml` (catalog with `psd-clinical`). +2. Write `psd-clinical/workspace.yaml` (schema v3, decisions 2–4). +3. `git mv` the 36 `evidence/*.md` files to `psd-clinical/evidence/`. +4. Copy the curated FK to `psd-clinical/schema/annotations.yaml`. +5. Add `.gitignore` for legacy runtime dirs; remove/leave legacy `psd.yaml` as a non-contract + historical note (do not commit the old flat paths as canonical). +6. Commit on `main` (no remote yet). + +### Task 2: Local registry validation (autonomous, no DWH/secret) + +1. From a scratch bare remote of the restructured repo, bootstrap a `WorkspaceRegistry` and assert + `psd-clinical` activates: descriptor valid, catalog matches, Evidence materialized with manifest, + annotations parsed/synced, `workspace-docs/psd-clinical` generated. +2. Verify `thothctl workspace inspect` fails only on the missing DWH bindings (not on the descriptor). + +### Task 3: Installation bindings + secrets (owner) + +1. Owner creates the GitHub remote and provides its URL + push credentials. +2. Owner provides (or confirms reuse of) the DWH X-API-Key and CA; write them as local secret files. +3. Fill `THT_WS_PSD_CLINICAL_DWH_*` bindings + `THT_WORKSPACE_GIT_REMOTE` + LLM provider in the + installation env (VPN active). + +### Task 4: Re-embedding/indexing (owner + stack) + +1. Start the stack; `embedding-model-init` pulls `qwen3-embedding:0.6b`. +2. `thothctl … workspace preprocess run --workspace psd-clinical --json` (DWH → FK → schema → Evidence) + against the real DWH; verify the Qdrant collection is populated and revision-scoped. + +### Task 5: Live smoke + manual acceptance (owner, P8 L2) + +1. New session on `psd-clinical` reaches the first reviewer gate; finalize one real query. +2. Record the P7/P8 manual acceptance in `docs/testing/p2-p6-manual-verification.md` and + PROJECT_STATE.md. + +--- + +## Owner checkpoint + +Tasks 1–2 are executed now by the agent. Tasks 3–5 are blocked on owner-provided secrets/access +(GitHub remote, VPN, DWH key/CA, LLM provider) and on the live stack. + +## Non-goals + +- No P8/P9/P10 work (end-to-end CI, retention policy changes, ssh_tunnel runtime). +- No change to the accepted P1.1–P6 contracts. +- No secret value, certificate, or response body is committed or printed. diff --git a/docs/superpowers/plans/2026-08-14-pi-management-operator-workflow.md b/docs/superpowers/plans/2026-08-14-pi-management-operator-workflow.md new file mode 100644 index 00000000..836220c6 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-pi-management-operator-workflow.md @@ -0,0 +1,764 @@ +# Pi Management Operator Workflow Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a safe `thothctl pi restart` command and replace the Pi Management dialog's duplicated technical copy with a structured Docker-only operator workflow. + +**Architecture:** Keep image changes in the existing recoverable `pi update` transaction. Implement configuration reload as a separate restart transaction that retains the current image, shares the Pi lifecycle lock, and uses a separate recovery file so the latest update remains rollback-capable. The React dialog remains a settings/diagnostics surface and only explains platform-specific host actions. + +**Tech Stack:** Go 1.26, Docker Compose v2, React 18, TypeScript, TanStack Query, Tailwind CSS, Vitest, Testing Library, shell documentation gates. + +**Spec:** `docs/superpowers/specs/2026-08-14-pi-management-operator-workflow-design.md` + +## Global Constraints + +- Pi runs only inside the Docker Compose `core` service; add no host-Pi or raw-Compose operator guidance. +- `pi restart` requires `--yes`; without `--drain` it refuses active sessions, and with `--drain` it waits without terminating them. +- Restart retains the currently selected image and never builds, pulls, or promotes an image. +- Restart and update use separate recovery files but one installation-scoped lifecycle lock. +- The browser receives no Docker access, shell, credential values, or write access to `deploy/pi/*.json`. +- Use `~` for GUI home-directory examples. All GUI strings remain English. +- Platform tabs remain closed initially; dialog and instruction scrolling remain functional. +- Do not stage or modify the unrelated existing changes in `frontend/src/shell/WorkspaceManager.tsx` and `frontend/src/shell/WorkspaceManager.test.tsx`. +- `PiManagement.tsx` and its test contain approved uncommitted work from earlier revisions; edit and commit them only in the frontend task. + +## File map + +- Create `tools/thothctl/internal/pi/restart.go` and `restart_test.go` for the restart transaction. +- Modify `tools/thothctl/internal/pi/state.go` and `state_test.go` for one shared lifecycle lock. +- Modify `tools/thothctl/internal/config/installation.go` and its test for `restart-state.json`. +- Modify `tools/thothctl/internal/pi/update.go` and its test to share the active-session drain helper and compose restart recovery. +- Modify the `Target.Source` comment in `tools/thothctl/internal/pi/state.go` when `"restart"` becomes a valid non-image lifecycle source. +- Modify `tools/thothctl/cmd/thothctl/main.go` and its test for usage, parsing, dispatch, and recovery. +- Modify `frontend/src/shell/PiManagement.tsx` and its test for the structured workflow. +- Modify `docs/contracts/tht-pi.md`, `docs/install/pi-management.md`, `docs/general/pi-configuration.md`, and `scripts/verify-workspace-install-docs.sh`. +- Modify `README.md` only if it claims to list the complete Pi command surface. + +--- + +### Task 1: Separate restart state and share the lifecycle lock + +**Files:** +- Modify: `tools/thothctl/internal/config/installation.go:234-249` +- Test: `tools/thothctl/internal/config/installation_test.go` +- Modify: `tools/thothctl/internal/pi/state.go:171-224` +- Test: `tools/thothctl/internal/pi/state_test.go` + +**Interfaces:** +- Produces: `func (Installation) RestartStatePath() string` +- Produces: `func lifecycleLockPath(statePath string) string` +- Preserves: `func acquireLock(statePath string) (*updateLock, error)` + +- [ ] **Step 1: Write failing path and cross-operation lock tests** + +Add to `installation_test.go`: + +```go +if got, want := installation.RestartStatePath(), filepath.Join(installation.ControlDirectory(), "restart-state.json"); got != want { + t.Fatalf("RestartStatePath() = %q, want %q", got, want) +} +``` + +Add to `state_test.go`: + +```go +func TestUpdateAndRestartStatePathsShareOneLifecycleLock(t *testing.T) { + dir := t.TempDir() + first, err := acquireLock(filepath.Join(dir, "update-state.json")) + if err != nil { + t.Fatal(err) + } + defer first.Release() + + second, err := acquireLock(filepath.Join(dir, "restart-state.json")) + if !errors.Is(err, ErrLockHeld) || second != nil { + t.Fatalf("second lock = %#v, %v; want nil, ErrLockHeld", second, err) + } +} +``` + +- [ ] **Step 2: Run the focused tests and verify RED** + +```bash +cd tools/thothctl +go test ./internal/config ./internal/pi -run 'Test.*(RestartStatePath|ShareOneLifecycleLock)' -count=1 +``` + +Expected: compile failure for `RestartStatePath`, then lock-test failure until both files resolve to one lock. + +- [ ] **Step 3: Implement the installation path and common lock** + +Add to `installation.go`: + +```go +func (i Installation) RestartStatePath() string { + return filepath.Join(i.ControlDirectory(), "restart-state.json") +} +``` + +Change lock derivation in `state.go`: + +```go +func lifecycleLockPath(statePath string) string { + return filepath.Join(filepath.Dir(statePath), "pi-lifecycle.lock") +} + +var ErrLockHeld = errors.New("another Pi update, restart, or rollback is already in progress") +``` + +Keep `acquireLock(statePath)` but set `path := lifecycleLockPath(statePath)`. Preserve owner metadata and durable cleanup. + +- [ ] **Step 4: Run config, state, update, and rollback tests** + +```bash +cd tools/thothctl +go test ./internal/config ./internal/pi -count=1 +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add tools/thothctl/internal/config/installation.go tools/thothctl/internal/config/installation_test.go tools/thothctl/internal/pi/state.go tools/thothctl/internal/pi/state_test.go +git commit -m "refactor(thothctl): share Pi lifecycle lock" +``` + +--- + +### Task 2: Implement the core-only restart transaction + +**Files:** +- Create: `tools/thothctl/internal/pi/restart.go` +- Create: `tools/thothctl/internal/pi/restart_test.go` +- Modify: `tools/thothctl/internal/pi/update.go:108-145,802-849` +- Test: `tools/thothctl/internal/pi/update_test.go` + +**Interfaces:** +- Consumes existing `Runner`, `lifecycleHooks`, lock, maintenance, session inventory, `Doctor`, `Status`, `renderedCore`, `runningImage`, `recreateCore`, state, and recovery helpers. +- Produces: + +```go +type RestartRequest struct { + StatePath string + UpdateStatePath string + Confirm bool + Drain bool +} + +type RestartResult struct { + StatePath string + Version string +} + +func Restart(context.Context, Runner, RestartRequest) (RestartResult, error) +func RecoverLifecycleMaintenance(context.Context, Runner, string, string, bool) error +``` + +- [ ] **Step 1: Write failing restart transaction tests** + +Create `restart_test.go` in package `pi`, reusing `newFakeRunner`, `assertCalled`, and `assertNotCalled`: + +```go +func TestRestartRequiresConfirmationWithoutInvokingCompose(t *testing.T) { + dir := t.TempDir() + fake := newFakeRunner() + _, err := Restart(context.Background(), fake, RestartRequest{ + StatePath: filepath.Join(dir, "restart-state.json"), + UpdateStatePath: filepath.Join(dir, "update-state.json"), + }) + if !errors.Is(err, ErrConfirmationRequired) { + t.Fatalf("Restart() error = %v, want ErrConfirmationRequired", err) + } + assertNotCalled(t, fake.calls, "compose") +} + +func TestRestartDrainsRecreatesOnlyCoreAndRetainsImage(t *testing.T) { + fake := newFakeRunner() + fake.activeSessions = true + dir := t.TempDir() + hooks := defaultLifecycleHooks + hooks.sleep = func(time.Duration) { fake.activeSessions = false } + + result, err := restartWithHooks(context.Background(), fake, RestartRequest{ + StatePath: filepath.Join(dir, "restart-state.json"), + UpdateStatePath: filepath.Join(dir, "update-state.json"), + Confirm: true, + Drain: true, + }, hooks) + if err != nil { + t.Fatal(err) + } + if result.Version != fake.version { + t.Fatalf("version = %q, want %q", result.Version, fake.version) + } + assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core") + assertNotCalled(t, fake.calls, "build --pull") + assertNotCalled(t, fake.calls, "pull ") + assertNotCalled(t, fake.calls, "frontend") + if _, err := os.Stat(result.StatePath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("successful restart state still exists: %v", err) + } +} +``` + +Also add: + +- `TestRestartRefusesActiveSessionsWithoutDrain` +- `TestRestartRefusesInterruptedUpdateOrRestartState` +- `TestRestartPreflightFailureNeverRecreatesCoreAndClearsMaintenance` +- `TestRestartPostRecreateFailureKeepsMaintenanceAndRecoveryState` +- `TestRecoverLifecycleMaintenanceVerifiesAndClearsRestartState` +- `TestRestartRejectsImageConfigurationAndMountDrift` + +Extend the shared fake runner with a `recreated bool` field, set it when the force-recreate call is +observed, and make the existing post-candidate failure branches apply when `fake.built || +fake.recreated`. This lets restart failures occur after mutation without pretending an image build +happened. For post-recreate failure set `fake.fail = "health"`, then assert maintenance remains +true and `restart-state.json` remains. + +- [ ] **Step 2: Run restart tests and verify RED** + +```bash +cd tools/thothctl +go test ./internal/pi -run 'TestRestart|TestRecoverLifecycleMaintenance' -count=1 +``` + +Expected: compile failure for the missing restart interfaces. + +- [ ] **Step 3: Extract the existing active-session loop** + +Move the 30-second loop from `updateWithHooks` into `update.go`: + +```go +func waitForInactiveSessions(ctx context.Context, runner Runner, drain bool, sleep func(time.Duration)) error { + running, err := activeSessions(ctx, runner) + if err != nil { + return err + } + if !running { + return nil + } + if !drain { + return ErrActiveSessions + } + for attempts := 0; attempts < 30; attempts++ { + running, err = activeSessions(ctx, runner) + if err != nil { + return err + } + if !running { + return nil + } + sleep(time.Second) + } + return ErrActiveSessions +} +``` + +Replace the original update loop with `waitForInactiveSessions(...)`. Preserve the second inventory check immediately before mutation. + +- [ ] **Step 4: Implement `Restart` in `restart.go`** + +Implement `Restart` as a wrapper around `restartWithHooks`. The transaction must execute in this exact order: + +1. validate both state paths; +2. acquire the shared lock using `RestartStatePath`; +3. require confirmation; +4. reject recovery-required update or restart state; +5. activate maintenance and arrange cleanup for pre-mutation returns; +6. wait/refuse through `waitForInactiveSessions`; +7. run `Doctor` as preflight; +8. read current version, rendered core, running image, configuration SHA, and mount identity; +9. write restart state with `Target{Version: version, Source: "restart"}`; +10. recheck active sessions; +11. durably mark `MutationStarted`; +12. call `recreateCore(ctx, runner)` directly, without image override; +13. prove maintenance remains active; +14. record `PhaseRecreated`; +15. run `verifyRestart(ctx, runner, version, previous)`; +16. record `PhaseVerified`, remove only `restart-state.json`, and clear maintenance. + +Implement `verifyRestart` to run `Doctor`, reload rendered configuration and running image, require +the same image ID, require the same `ConfigurationSHA`, and require `sameMounts(previous.Mounts, +after.Mounts)`. Use stable errors for image, external-configuration, and persistence-mount drift. +Update the `Target.Source` comment in `state.go` to include the non-image `restart` operation. + +Use: + +```go +func Restart(ctx context.Context, runner Runner, request RestartRequest) (RestartResult, error) { + return restartWithHooks(ctx, runner, request, defaultLifecycleHooks) +} +``` + +Use `recoveryRequired("Pi restart ...", err)` for every post-mutation failure and set the deferred maintenance cleanup flag to false. Do not overwrite or remove a verified `update-state.json`; it remains the image rollback record. + +- [ ] **Step 5: Implement combined maintenance recovery** + +Add: + +```go +func RecoverLifecycleMaintenance( + ctx context.Context, + runner Runner, + updateStatePath string, + restartStatePath string, + confirm bool, +) error +``` + +When restart state requires recovery, run `verifyRestart` using the recorded target version and +previous image contract, and remove `restart-state.json` only after verification. Then call existing +update-state `RecoverMaintenance` without opening admission between the two checks. Missing restart +state is allowed; malformed restart state fails closed. + +- [ ] **Step 6: Run internal Pi tests and verify GREEN** + +```bash +cd tools/thothctl +go test ./internal/pi -count=1 +``` + +Expected: PASS, including existing update, rollback, mount identity, durability, and transport-loss tests. + +- [ ] **Step 7: Commit** + +```bash +git add tools/thothctl/internal/pi/restart.go tools/thothctl/internal/pi/restart_test.go tools/thothctl/internal/pi/update.go tools/thothctl/internal/pi/update_test.go tools/thothctl/internal/pi/state.go +git commit -m "feat(thothctl): add safe Pi core restart" +``` + +--- + +### Task 3: Expose `pi restart` through thothctl + +**Files:** +- Modify: `tools/thothctl/cmd/thothctl/main.go:25-53,263-363,470-520` +- Test: `tools/thothctl/cmd/thothctl/main_test.go:68-110,567-666` + +**Interfaces:** +- Consumes Task 2's restart and recovery interfaces and Task 1's state paths. +- Produces `func parsePiRestartArgs([]string, string, string) (pi.RestartRequest, error)`. +- Public syntax: `thothctl --installation pi restart --yes [--drain]`. + +- [ ] **Step 1: Write failing usage, parser, and dispatch tests** + +Extend the usage test: + +```go +for _, expected := range []string{ + "pi restart --yes [--drain]", + "Recreate only core with the currently selected Pi image", +} { + if !strings.Contains(usage, expected) { + t.Fatalf("usage missing %q", expected) + } +} +``` + +Add a table test for: + +```go +{name: "confirmed", args: []string{"--yes"}, want: pi.RestartRequest{StatePath: restartPath, UpdateStatePath: updatePath, Confirm: true}} +{name: "drain", args: []string{"--yes", "--drain"}, want: pi.RestartRequest{StatePath: restartPath, UpdateStatePath: updatePath, Confirm: true, Drain: true}} +{name: "duplicate yes", args: []string{"--yes", "--yes"}, wantErr: "--yes may be supplied once"} +{name: "duplicate drain", args: []string{"--drain", "--drain"}, wantErr: "--drain may be supplied once"} +{name: "unknown", args: []string{"--force"}, wantErr: "unknown pi restart option"} +``` + +Add command tests proving missing `--yes` exits 2 before mutation, success is sanitized, and maintenance recovery passes both state paths. + +- [ ] **Step 2: Run focused CLI tests and verify RED** + +```bash +cd tools/thothctl +go test ./cmd/thothctl -run 'Test.*(Restart|Usage|Maintenance)' -count=1 +``` + +Expected: failure because parsing and dispatch are absent. + +- [ ] **Step 3: Add usage, parser, dispatch, and recovery routing** + +Add: + +```text + pi restart --yes [--drain] + Recreate only core with the currently selected Pi image and verify readiness. +``` + +Dispatch before `case "update"`: + +```go +case "restart": + request, err := parsePiRestartArgs( + args[1:], + installation.RestartStatePath(), + installation.UpdateStatePath(), + ) + if err != nil { + return commandUsageError(stderr, err.Error()) + } + result, err := pi.Restart(ctx, controlled, request) + if err != nil { + return piFailure(stderr, err, secretValues) + } + fmt.Fprintf(stdout, "Pi core restarted with the existing image; version %s readiness and smoke checks passed.\n", result.Version) + return 0 +``` + +Implement exact duplicate and unknown-option errors from the tests. Route `pi maintenance recover --yes` through `RecoverLifecycleMaintenance` with both state paths. + +- [ ] **Step 4: Run CLI and full Go tests** + +```bash +cd tools/thothctl +go test ./cmd/thothctl -count=1 +go test ./... -count=1 +``` + +Expected: PASS with no secret leakage. + +- [ ] **Step 5: Build supported binaries** + +```bash +cd ../.. +./scripts/build-thothctl.sh +``` + +Expected: supported artifacts under `dist/thothctl/`, exit 0. + +- [ ] **Step 6: Commit** + +```bash +git add tools/thothctl/cmd/thothctl/main.go tools/thothctl/cmd/thothctl/main_test.go +git commit -m "feat(thothctl): expose Pi restart command" +``` + +--- + +### Task 4: Update contracts, operator docs, and documentation gates + +**Files:** +- Modify: `docs/contracts/tht-pi.md` +- Modify: `docs/install/pi-management.md` +- Modify: `docs/general/pi-configuration.md` +- Modify: `scripts/verify-workspace-install-docs.sh:1160-1188` +- Modify: `README.md` only if it claims command completeness. + +**Interfaces:** +- Consumes the exact Task 3 syntax and Task 2 recovery behavior. +- Produces one consistent distinction among GUI defaults, host files, credentials, reload, update, and recovery. + +- [ ] **Step 1: Strengthen the documentation gate first** + +Require: + +```bash +"pi restart --yes --drain" \ +"restart only core" \ +"deploy/pi/models.json" \ +"deploy/pi/settings.json" \ +"PI_AUTH_FILE" \ +"pi update" \ +"pi rollback --yes" \ +"pi maintenance recover --yes" +``` + +Add this negative gate: + +```bash +if grep -Fq '~/.pi/agent/' "$guide"; then + echo "Pi management guide must not direct ThothII operators to native Pi paths" >&2 + return 1 +fi +``` + +- [ ] **Step 2: Run the documentation verifier and verify RED** + +```bash +./scripts/verify-workspace-install-docs.sh +``` + +Expected: FAIL because restart and separated workflows are not documented. + +- [ ] **Step 3: Rewrite the two authoritative operator documents** + +In `docs/contracts/tht-pi.md`, document `pi restart --yes [--drain]`, confirmation, maintenance, bounded drain, current-image retention, core-only recreation, verification, separate restart state, shared lock, and recovery. + +In `docs/install/pi-management.md`, use these headings: + +- **Choose application defaults** — GUI Save defaults or CLI configure, not both. +- **Edit the provider catalog and enabled-model policy** — project-root `deploy/pi/` files. +- **Store provider credentials** — `PI_AUTH_FILE` from the installation environment file. +- **Reload changed configuration** — one restart command. +- **Update the bundled Pi version** — build command and digest-pinned pull alternative. +- **Recover a failed lifecycle operation** — status, logs, rollback, maintenance recovery. + +- [ ] **Step 4: Add the Docker-operator callout** + +Add below the introduction of `docs/general/pi-configuration.md`: + +```markdown +> **ThothII operator note:** ThothII runs Pi only in Docker Compose. Paths under +> `~/.pi/agent/` in this document describe Pi's container-side behavior. Operators edit +> `deploy/pi/models.json` and `deploy/pi/settings.json` in the ThothII project root and use +> the protected host credential file selected by `PI_AUTH_FILE`; they do not edit files inside +> the running container. +``` + +- [ ] **Step 5: Run documentation gates** + +```bash +./scripts/verify-workspace-install-docs.sh +./scripts/test-thothctl-build-contract.sh +``` + +Expected: both exit 0. + +- [ ] **Step 6: Commit** + +```bash +git add docs/contracts/tht-pi.md docs/install/pi-management.md docs/general/pi-configuration.md scripts/verify-workspace-install-docs.sh +git commit -m "docs: clarify Pi reload and update workflows" +``` + +Add `README.md` only if it changed. + +--- + +### Task 5: Restructure the Pi Management dialog + +**Files:** +- Modify: `frontend/src/shell/PiManagement.tsx:1-225,286-305,376-449` +- Test: `frontend/src/shell/PiManagement.test.tsx:170-230` + +**Interfaces:** +- Consumes the public Task 3 commands. +- Preserves API calls, readiness rail, defaults, test, logs, all-tabs-closed state, dialog height, and scrolling. +- Removes `UPDATE_COMMAND`, `copyUpdateCommand`, global clipboard assertions, and the bottom Host update section. + +- [ ] **Step 1: Rewrite the frontend test first** + +Add these assertions: + +```tsx +expect(screen.getByText("Using the host terminal:")).toBeVisible(); +expect(screen.queryByText(/not this browser page/i)).not.toBeInTheDocument(); +expect(screen.queryByRole("region", { name: "Host update" })).not.toBeInTheDocument(); +expect(screen.queryByRole("button", { name: "Copy update command" })).not.toBeInTheDocument(); +expect(screen.queryByText(/:5173/)).not.toBeInTheDocument(); + +await user.click(within(tablist).getByRole("tab", { name: "Linux" })); +const linux = screen.getByRole("tabpanel", { name: "Linux" }); +expect(within(linux).getAllByRole("listitem")).toHaveLength(7); +expect(linux).toHaveTextContent("The deploy directory is in the ThothII project root, beside compose.yaml"); +expect(linux).toHaveTextContent("deploy/pi/models.json"); +expect(linux).toHaveTextContent("deploy/pi/settings.json"); +expect(linux).toHaveTextContent("baseUrl is the provider API endpoint"); +expect(linux).toHaveTextContent("enabledModels uses provider/model identifiers"); +expect(linux).toHaveTextContent("PI_AUTH_FILE is a setting in the installation environment file"); +expect(linux).toHaveTextContent("~/bin/thothctl --installation ~/thothii-installation.yaml pi restart --yes --drain"); +expect(linux).toHaveTextContent("pi update --version --source build --yes --drain"); +expect(linux).toHaveTextContent("pi rollback --yes"); +expect(linux).not.toHaveTextContent("~/.pi/agent/"); +``` + +Add equivalent macOS and Windows path/command assertions. Preserve tests for tabs closed, dialog `max-h-[calc(100vh-6rem)]`, and `overflow-y-scroll`. + +Add: + +```tsx +expect(screen.getByText("Select the provider, model, and reasoning used for new Pi work. Credentials stay in protected host files.")).toBeVisible(); +expect(screen.getByText("Shows at most 200 recent lines with declared secret values removed.")).toBeVisible(); +``` + +- [ ] **Step 2: Run the focused test and verify RED** + +```bash +cd frontend +npx vitest run src/shell/PiManagement.test.tsx +``` + +Expected: FAIL on the new lead-in, ordered workflow, restart command, explanations, removed duplicate section, and revised descriptions. + +- [ ] **Step 3: Introduce shared structured platform data** + +Define: + +```tsx +type PiPlatformDetails = { + modelsPath: string; + settingsPath: string; + terminal: string; + credentialProtection: string; + restartCommand: string; + updateCommand: string; + pullCommand: string; + recoveryCommands: string; +}; +``` + +Render `PiInstructionSteps` as an ordered list with exactly these seven headings: + +1. Open the project root +2. Edit the provider catalog +3. Enable the model +4. Set the provider credential +5. Reload Pi configuration +6. Update the Pi version +7. Recover a failed update + +Use these normal commands: + +```text +Linux/macOS: +~/bin/thothctl --installation ~/thothii-installation.yaml pi restart --yes --drain +~/bin/thothctl --installation ~/thothii-installation.yaml pi update --version --source build --yes --drain + +Windows PowerShell: +& (Resolve-Path "~\bin\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\thothii-installation.yaml") pi restart --yes --drain +& (Resolve-Path "~\bin\thothctl-windows-amd64.exe") --installation (Resolve-Path "~\thothii-installation.yaml") pi update --version --source build --yes --drain +``` + +Explain `baseUrl`, `api`, `models`, `id`, `name`, `enabledModels`, and `PI_AUTH_FILE` in compact lists. The lead-in must be exactly: + +```tsx +

Using the host terminal:

+``` + +Keep digest-pinned pull and recovery commands visually subordinate. + +- [ ] **Step 4: Remove duplicate and developer-only content** + +Delete: + +- `UPDATE_COMMAND` +- `copyUpdateCommand` +- the bottom `Host update` section +- `Clipboard` import if unused +- Vite, `:5173`, frontend rebuild, native Pi, and container-edit text +- mandatory configure, stop/start, status/doctor/test sequences + +Keep the positive statement that Docker mounts the selected host credential file read-only for Pi. + +- [ ] **Step 5: Clarify defaults and diagnostics** + +Use exactly: + +```text +Select the provider, model, and reasoning used for new Pi work. Credentials stay in protected host files. +Shows at most 200 recent lines with declared secret values removed. +``` + +Do not change API behavior or readiness semantics. + +- [ ] **Step 6: Run focused and full frontend gates** + +```bash +cd frontend +npx vitest run src/shell/PiManagement.test.tsx +npx vitest run +npx tsc -b +npm run build +``` + +Expected: focused tests, full suite, typecheck, and production build pass. + +- [ ] **Step 7: Commit only Pi Management files** + +```bash +git add frontend/src/shell/PiManagement.tsx frontend/src/shell/PiManagement.test.tsx +git commit -m "feat(frontend): simplify Pi operator workflow" +``` + +Confirm Workspace Manager files remain unstaged. + +--- + +### Task 6: Final verification and local deployment + +**Files:** +- Verify only; change source only to correct a failing gate attributable to Tasks 1-5. + +**Interfaces:** +- Produces fresh evidence that CLI, docs, frontend, and port 8080 agree. + +- [ ] **Step 1: Run all relevant gates** + +```bash +cd tools/thothctl +go test ./... -count=1 +cd ../.. +./scripts/build-thothctl.sh +./scripts/test-thothctl-build-contract.sh +./scripts/verify-workspace-install-docs.sh +cd frontend +npx vitest run +npx tsc -b +npm run build +cd .. +git diff --check +``` + +Expected: every command exits 0. Existing Vite chunk-size warnings are acceptable. + +- [ ] **Step 2: Verify the built CLI** + +```bash +dist/thothctl/thothctl-darwin-arm64 --help +``` + +Expected: output contains `pi restart --yes [--drain]` plus update, rollback, and maintenance commands. + +- [ ] **Step 3: Rebuild only the active frontend service** + +Use project `thothii-9307255178c1`, the current installation environment values, and these Compose files: + +```bash +docker compose --project-name thothii-9307255178c1 \ + -f compose.yaml \ + -f deploy/compose.local.yaml \ + -f deploy/compose.git-ssh.yaml \ + -f deploy/psd/connector-secrets.yaml \ + up -d --build --no-deps frontend +``` + +Never print or inline credential contents. + +- [ ] **Step 4: Verify port 8080 and health** + +Fetch `http://127.0.0.1:8080/` and its JavaScript assets. Require: + +```text +Using the host terminal: +pi restart --yes --drain +PI_AUTH_FILE is a setting in the installation environment file +The deploy directory is in the ThothII project root +``` + +Reject: + +```text +not this browser page +For native Pi outside Compose +Copy update command +:5173 +``` + +Run: + +```bash +docker ps --format '{{.Names}}|{{.Status}}' +``` + +Expected: `thothii-9307255178c1-frontend-1` is healthy. + +- [ ] **Step 5: Inspect final scope** + +```bash +git status --short +git log --oneline -6 +``` + +Expected: lifecycle state, restart transaction, CLI, docs, and frontend commits are present. Only unrelated pre-existing Workspace Manager changes remain. diff --git a/docs/superpowers/plans/2026-08-14-thothctl-discovery-and-pi-update.md b/docs/superpowers/plans/2026-08-14-thothctl-discovery-and-pi-update.md new file mode 100644 index 00000000..9358f5c4 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-thothctl-discovery-and-pi-update.md @@ -0,0 +1,86 @@ +# Simplified `thothctl` installation selection and Pi update Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Allow all existing `thothctl` commands to discover the installation descriptor automatically and make `thothctl pi update` use the checkout's pinned Pi version by default. + +**Architecture:** Add a small config-level resolver that chooses one validated installation descriptor from an explicit flag, environment variable, or bounded upward search from the working directory. Keep the existing Pi lifecycle transaction intact; resolve only the requested version/source at the CLI boundary so the update engine retains its safety and recovery guarantees. + +**Tech Stack:** Go 1.26, Docker Compose v2, existing `tools/thothctl` config and Pi lifecycle packages, Go tests. + +**Spec:** `docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md` + +## Global Constraints + +- Preserve every existing `pi` subcommand, alias, safety check, and explicit invocation form. +- `--installation` remains an explicit override and accepts only an absolute descriptor path named `thothii-installation.yaml`. +- Automatic discovery must not recursively scan `.artifacts`, home directories, or unrelated descendants. +- The default Pi version is the single `ARG PI_VERSION=` in the selected project's `docker/core.Dockerfile`. +- The default Pi update uses the existing transactional build path and must not install an arbitrary network “latest”. +- All failures remain sanitized and must not reveal secret values. + +## File Map + +- Create `tools/thothctl/internal/config/discovery.go` and `discovery_test.go` for bounded descriptor resolution and safe diagnostics. +- Modify `tools/thothctl/cmd/thothctl/main.go` and `main_test.go` for optional global selection, `pi update` defaults, help text, and dispatch. +- Create `tools/thothctl/internal/pi/version.go` and `version_test.go` for reading the project Pi pin. +- Modify `tools/thothctl/internal/pi/update.go` and `update_test.go` only if the default request needs a typed source/confirmation adjustment; keep lifecycle internals unchanged otherwise. +- Modify `docs/contracts/tht-pi.md`, `docs/install/pi-management.md`, and relevant command-contract verification scripts. + +### Task 1: Add bounded installation descriptor discovery + +**Files:** +- Create: `tools/thothctl/internal/config/discovery.go` +- Test: `tools/thothctl/internal/config/discovery_test.go` + +**Interface:** `func Resolve(explicit string, environment func(string) string, workingDirectory string) (string, error)`. + +- [x] Write failing tests for explicit-path precedence, `THOTHII_INSTALLATION`, `deploy/*/thothii-installation.yaml` discovery, parent discovery, `.artifacts` exclusion, invalid candidates, ambiguity, and no-candidate errors. +- [x] Run `cd tools/thothctl && go test ./internal/config -run 'TestResolve' -count=1`; confirm RED because `Resolve` is absent. +- [x] Implement a bounded upward walk. At each level inspect only the exact descriptor and immediate `deploy/*/thothii-installation.yaml` entries; skip `.artifacts`; require regular files; validate candidates through `config.Load`; deduplicate canonical paths; fail clearly on zero or multiple valid candidates. +- [x] Re-run the focused tests and confirm GREEN. +- [x] Refactor only after green, keeping path collection separate from candidate validation. + +### Task 2: Make the global installation option optional + +**Files:** +- Modify: `tools/thothctl/cmd/thothctl/main.go` +- Test: `tools/thothctl/cmd/thothctl/main_test.go` + +- [x] Add failing CLI tests proving `thothctl pi status` works from a project tree, the environment variable is used, an explicit flag wins, ambiguity fails before Docker, and all existing commands retain their dispatch. +- [x] Run the focused CLI tests and confirm RED because the current parser requires `--installation`. +- [x] Parse optional `--installation`, call `config.Resolve` with `THOTHII_INSTALLATION` and the process working directory, and update help to `thothctl [--installation PATH] `. +- [x] Run `cd tools/thothctl && go test ./cmd/thothctl -count=1`; confirm GREEN. + +### Task 3: Default `pi update` to the repository Pi pin + +**Files:** +- Create: `tools/thothctl/internal/pi/version.go` +- Test: `tools/thothctl/internal/pi/version_test.go` +- Modify: `tools/thothctl/cmd/thothctl/main.go` +- Test: `tools/thothctl/cmd/thothctl/main_test.go` + +**Interface:** `func ReadPinnedVersion(projectDirectory string) (string, error)`. + +- [x] Add failing tests for one valid Dockerfile pin, missing Dockerfile, duplicate default pins, malformed versions, and `pi update` without `--version`; retain explicit version and advanced pull tests. +- [x] Run `cd tools/thothctl && go test ./internal/pi ./cmd/thothctl -run 'Test(ReadPinnedVersion|ParsePiUpdate|RunPiUpdate)' -count=1`; confirm RED. +- [x] Read only `docker/core.Dockerfile`, require one default `ARG PI_VERSION=...`, validate it with the existing version grammar, and make the short request select build mode while preserving the lifecycle transaction. +- [x] Re-run focused tests and confirm GREEN. + +### Task 4: Update contracts without removing commands + +**Files:** +- Modify: `docs/contracts/tht-pi.md` +- Modify: `docs/install/pi-management.md` +- Modify: the documentation verification script that asserts the old mandatory update invocation. + +- [x] Document automatic descriptor discovery, the explicit override, `thothctl pi update` as the normal path, `--version` as an explicit pin, and the advanced pull/digest form. +- [x] Keep status, doctor, test/check, configure, restart, rollback, maintenance, and logs documented. +- [x] Run the targeted documentation checks and `git diff --check`. + +### Task 5: Full verification + +- [x] Run `cd tools/thothctl && go test ./... -count=1`. +- [x] Run `cd tools/thothctl && go build ./cmd/thothctl`. +- [x] Run the relevant documentation contract script and inspect `git status --short`. +- [x] Verify the help text contains the optional form and all existing commands; do not mutate the live Docker installation unless separately requested. diff --git a/docs/superpowers/plans/2026-08-15-unified-tht-cli-product-step.md b/docs/superpowers/plans/2026-08-15-unified-tht-cli-product-step.md new file mode 100644 index 00000000..cc856a12 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-unified-tht-cli-product-step.md @@ -0,0 +1,1159 @@ +# Unified `tht` CLI Product Step Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Turn the repository's fragmented operator experience into one installable `tht` command that configures, builds, starts, updates, diagnoses, backs up, restores, and manages Pi, while preserving the indispensable NL-to-SQL workflow commands and simplifying the remaining command surface according to the approved maintain/erase/enhance audit. + +**Architecture:** The host-facing command is the existing native Go operator CLI, renamed from `thothctl` to `tht` and installed on the operating-system `PATH`. It discovers the current ThothII repository or Git worktree and its installation descriptor automatically. The Python workflow CLI remains named `tht` inside the `core` container and continues to own sessions, decisions, documents, SQL, and persistence. Host operations call Docker Compose directly or, when workflow checks are needed, invoke the container-local Python CLI. There is no compatibility alias, wrapper, second public command, host Python virtual environment, or requirement to build the CLI manually. + +**Tech Stack:** Go standard library, Docker Compose, Python/Typer, pytest, React 18, TypeScript, Vite, Vitest, Testing Library, Playwright, POSIX shell, PowerShell. + +**Spec:** `docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md` + +## Global Constraints + +- [ ] Expose exactly one public product command: `tht`. Remove `thothctl` rather than retaining an alias, wrapper, or deprecation period. +- [ ] Keep the Python workflow executable named `tht` inside `core`; do not introduce `tht-runtime` or a public `runtime` namespace. +- [ ] Preserve all 55 commands classified **MAINTAIN**, implement the 8 approved **ENHANCE** outcomes, and remove the 14 commands classified **ERASE**. The two audit reports are normative inputs. +- [ ] Make `--installation` optional on host commands. Explicit paths always win; otherwise discover the descriptor from the repository/worktree and then from the documented installation registry. +- [ ] Make `tht` callable from a project root or Git worktree root without `./`, `./bin/`, `~/bin/`, a shell wrapper, a Go build, or activation of a Python virtual environment. +- [ ] Make `tht setup` perform configuration, image build, container start, health verification, and diagnostics by default. `--configure-only` is the explicit opt-out before build/start. +- [ ] Make `tht pi update` resolve the latest stable Pi version when `--version` is omitted. A failed lookup must stop before any mutation; it must not silently reuse the Dockerfile pin. +- [ ] Preserve the user's current GLM 5.3 changes in `deploy/pi/models.json` and `deploy/pi/settings.json`; never replace those files with stale fixtures. +- [ ] Treat secrets as secret references by default. Do not print secret values, write them into tracked files, or include them in a backup unless the operator explicitly supplies both `--include-secrets` and `--yes`. +- [ ] Preserve unrelated dirty worktree changes and `.playwright-cli/`. Stage only files named by the current task. +- [ ] Keep frontend strings in English. Documentation may explain concepts in prose but all shown commands must be directly executable. +- [ ] Work test-first for every behavior change: add or tighten a failing test, run it to confirm the expected failure, implement the smallest complete change, rerun the focused test, then run the relevant suite. +- [ ] Do not deploy to the live Mac or restart port 8080 until all code, documentation, and automated gates pass. + +## Approved Command-Surface Baseline + +The implementation must end with this host-facing surface: + +```text +tht setup [--configure-only] [--installation PATH] +tht version +tht start [--build] [--installation PATH] +tht stop [--installation PATH] +tht status [--installation PATH] +tht doctor [--json] [--installation PATH] +tht logs [SERVICE] [--installation PATH] +tht update [--check-only] [--yes] [--drain] [--installation PATH] +tht backup [--output PATH] [--include-secrets --yes] [--drain] [--installation PATH] +tht restore ARCHIVE --yes [--drain] [--installation PATH] +tht sessions migrate --yes [--installation PATH] +tht remove [--yes ID...] [--installation PATH] +tht pi ... +tht workspace ... +``` + +`tht workspace ...` preserves every currently implemented native workspace operation, including +the existing inspect, preprocessing, schema, evidence, index, and vector operations. This plan does +not invent a second workspace-management surface merely to make the help tree look symmetrical. + +The Python workflow command surface inside `core` is governed by the approved audit: + +- **MAINTAIN:** 55 commands remain behaviorally and contractually available. +- **ENHANCE:** `config check`, `doctor`, `db fetch-ca`, `memory list`, `memory show`, `memory update`, `memory delete`, and `memory index` are improved or consolidated as described in Task 12. +- **ERASE:** `decision list`, `decision retract`, `cte list`, `sql explain`, `sql save`, `memory clear`, `memory migrate`, `evidence extract`, `evidence index`, `lsh build`, `lsh query`, `vector init`, `formula save`, and `formula list` are removed in Task 11. + +--- + +## Task 1: Rename the Native Operator CLI and Its Build Artifacts + +**Files:** + +- Rename: `tools/thothctl/` → `tools/tht/` +- Rename: `tools/tht/cmd/thothctl/` → `tools/tht/cmd/tht/` +- Rename: `docker/thothctl.Dockerfile` → `docker/tht.Dockerfile` +- Rename: `scripts/build-thothctl.sh` → `scripts/build-tht.sh` +- Rename/update: existing `scripts/test-thothctl-*.sh` files → corresponding `scripts/test-tht-*.sh` files +- Modify: Go module/import paths and package references under `tools/tht/` +- Modify: `.gitignore` + +### Steps + +- [ ] Add a command-identity test in `tools/tht/cmd/tht/main_test.go` that invokes the existing `run` entry point, requires the help banner and error prefix to use `tht`, exercises the `version` path, and proves `thothctl` is not an alias. +- [ ] Run the focused test before renaming and confirm it fails because the current executable and root command are still `thothctl`. + +```bash +cd tools/thothctl +go test ./cmd/thothctl -run 'TestRootCommandIdentity' -count=1 +``` + +- [ ] Rename the directory, command package, Dockerfile, build script, smoke scripts, binary outputs, archive names, and image labels. Update imports mechanically, including the Go module path if it contains `/tools/thothctl`. +- [ ] Make `scripts/build-tht.sh` emit only `tht` binaries and archives such as `tht-darwin-arm64`, never `thothctl-*`. +- [ ] Remove all executable aliases and wrapper generation. Historical design documents may retain the old name as history; active source, tests, packaging, and user documentation may not. +- [ ] Run the renamed test and all Go tests. + +```bash +cd tools/tht +go test ./cmd/tht -run 'TestRootCommandIdentity' -count=1 +go test ./... +``` + +- [ ] Run a scoped stale-name scan over the renamed implementation and build assets. Active documentation is intentionally updated in Task 14; do not partially rewrite it here. + +```bash +rg -n 'thothctl|THOTHCTL' tools/tht docker scripts \ + -g '!scripts/test-tht-command-docs.sh' +``` + +- [ ] Commit only the rename and mechanical identity changes. + +```bash +git add -A -- tools/thothctl tools/tht docker/thothctl.Dockerfile docker/tht.Dockerfile \ + scripts/build-thothctl.sh scripts/build-tht.sh \ + scripts/test-thothctl-build-contract.sh scripts/test-tht-build-contract.sh \ + scripts/thothctl-update-smoke.sh scripts/tht-update-smoke.sh .gitignore +git commit -m "refactor(cli): rename operator command to tht" +``` + +--- + +## Task 2: Add Cross-Platform `tht` Installers + +**Files:** + +- Create: `scripts/install-tht.sh` +- Create: `scripts/install-tht.ps1` +- Create: `scripts/test-install-tht.sh` +- Create: `scripts/test-install-tht.ps1` +- Modify: `scripts/build-tht.sh` + +### Installer contract + +- macOS/Linux: use the repository-pinned Docker builder to produce the native binary for the current OS/architecture, install it as `/usr/local/bin/tht` by default, use elevation only for the final atomic install when required, and verify that the installed command is resolvable on `PATH`. +- Windows: install `tht.exe` under `%LOCALAPPDATA%\ThothII\bin`, add that directory to the current user's `PATH` when absent, and explain when a new terminal is required. +- Tests may override the destination with `THT_INSTALL_DIRECTORY`; production users are not instructed to set this variable. +- Re-running the installer replaces only the installed `tht` binary atomically and leaves installation data untouched. + +### Steps + +- [ ] Write shell installer tests that use a temporary `THT_INSTALL_DIRECTORY`, a fake build artifact, and a controlled `PATH`. Assert executable permissions, atomic replacement, `tht version`, and idempotency. +- [ ] Write PowerShell tests for the equivalent Windows behavior, including paths containing spaces and a pre-existing user `PATH` entry. +- [ ] Run both tests and confirm failure because the installers do not exist. + +```bash +bash scripts/test-install-tht.sh +pwsh -NoProfile -File scripts/test-install-tht.ps1 +``` + +- [ ] Implement `scripts/install-tht.sh` with explicit OS/architecture detection, a temporary staging directory, checksum validation when using a packaged artifact, and an atomic final rename. +- [ ] Implement `scripts/install-tht.ps1` with the same contract and user-level `PATH` update. +- [ ] Make both installers invoke `scripts/build-tht.sh` internally, which uses the repository-pinned Docker builder. The user must never install Go, select an artifact, or invoke a Go compiler. +- [ ] Rerun focused tests and package builds for Darwin arm64/amd64, Linux arm64/amd64, and Windows amd64. + +```bash +bash scripts/test-install-tht.sh +pwsh -NoProfile -File scripts/test-install-tht.ps1 +bash scripts/build-tht.sh --all +``` + +- [ ] Commit the installer slice. + +```bash +git add scripts/install-tht.sh scripts/install-tht.ps1 scripts/test-install-tht.sh \ + scripts/test-install-tht.ps1 scripts/build-tht.sh +git commit -m "feat(cli): install tht as a system command" +``` + +--- + +## Task 3: Discover the Project, Worktree, and Installation Descriptor Automatically + +**Files:** + +- Create: `tools/tht/internal/project/discovery.go` +- Create: `tools/tht/internal/project/discovery_test.go` +- Modify: `tools/tht/internal/config/discovery.go` +- Modify: `tools/tht/internal/config/discovery_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package project + +type Root struct { + Path string + IsWorktree bool +} + +func Discover(start string) (Root, error) +``` + +The descriptor resolver keeps its existing explicit-path support and applies this precedence: + +1. `--installation PATH` supplied to the current command. +2. `THOTHII_INSTALLATION` when explicitly set by automation. +3. One valid `thothii-installation.yaml` in the current directory or its immediate `deploy/*` children. +4. The same bounded search while walking parent directories to the discovered repository/worktree root. +5. A concise actionable error suggesting `tht setup` when no descriptor exists, or listing bounded candidates and requiring optional `--installation` when more than one exists. + +### Steps + +- [ ] Add table-driven tests for invocation from the repository root, a nested directory, a linked Git worktree, one immediate `deploy//thothii-installation.yaml`, multiple immediate descriptors, a missing descriptor, `THOTHII_INSTALLATION`, and an explicit descriptor override. +- [ ] Add tests proving `tht`, `tht help`, `tht version`, and `tht setup` do not require a pre-existing descriptor. +- [ ] Run the focused tests and confirm the current resolver fails root/worktree and descriptor-free bootstrap cases. + +```bash +cd tools/tht +go test ./internal/project ./internal/config ./cmd/tht -run 'TestDiscover|TestResolve|TestBootstrapCommands' -count=1 +``` + +- [ ] Implement root detection using repository markers (`.git` file or directory, `compose.yaml`, `deploy/`, and the expected ThothII source layout). Resolve symlinks for identity while retaining the user's invocation path for messages. +- [ ] Extend descriptor discovery without changing explicit `--installation` semantics. Keep the search bounded to current/ancestor directories and immediate `deploy/*`; do not scan the home directory or fall back to `/Users/mp/thothii-installation.yaml`. +- [ ] Return ambiguity as an error listing installation IDs and descriptor paths without exposing secret values. +- [ ] Run all Go tests. Defer system-PATH smoke calls to Task 15 so the old Mac installation is not changed prematurely. + +```bash +cd tools/tht && go test ./... +``` + +- [ ] Commit discovery behavior. + +```bash +git add tools/tht/internal/project tools/tht/internal/config tools/tht/cmd/tht +git commit -m "feat(cli): discover ThothII projects and installations" +``` + +--- + +## Task 4: Generate Setup Files Safely + +**Files:** + +- Create: `tools/tht/internal/setup/request.go` +- Create: `tools/tht/internal/setup/files.go` +- Create: `tools/tht/internal/setup/files_test.go` +- Modify: `.gitignore` +- Modify: `deploy/env/local.env.example` +- Modify: `deploy/psd/thothii-installation.yaml.example` +- Modify: `deploy/psd/operator.env.example` +- Modify: `tools/tht/cmd/tht/main.go` + +### Interfaces + +```go +package setup + +type Request struct { + ProjectRoot string + InstallationID string + Profile string + ConfigureOnly bool + NonInteractive bool +} + +type FilesResult struct { + DescriptorPath string + EnvironmentPath string + Created []string +} + +func EnsureFiles(request Request, input io.Reader, output io.Writer) (FilesResult, error) +``` + +### Steps + +- [ ] Add tests for a fresh checkout, a linked worktree, existing compatible files, conflicting files, interrupted writes, paths with spaces, and secret prompts. Assert that tracked examples are never modified. +- [ ] Require generated files to live under `deploy//`, be ignored by Git except for tracked examples, and contain secret file references rather than secret values. +- [ ] Run the tests and confirm they fail because setup file generation is absent. + +```bash +cd tools/tht +go test ./internal/setup -run 'TestEnsureFiles' -count=1 +``` + +- [ ] Implement prompts for installation ID, deployment profile, externally reachable endpoints, workspace selection, and secret-file locations. Accept safe defaults in interactive mode and explicit flags/environment in non-interactive automation. +- [ ] Write `deploy//thothii-installation.yaml` and `deploy//operator.env` atomically with restrictive permissions where the platform supports them. +- [ ] Refuse to overwrite a conflicting descriptor or environment file. Report the exact file and corrective action. +- [ ] Create protected external secret-file templates only after explicit confirmation, never overwrite an existing secret file, and store only their paths in generated configuration. +- [ ] Rerun setup tests and verify the bounded discovery from Task 3 finds the generated descriptor and no generated file is tracked. + +```bash +cd tools/tht && go test ./internal/setup -count=1 +``` + +- [ ] Commit setup-file generation. + +```bash +git add tools/tht/internal/setup tools/tht/cmd/tht/main.go \ + deploy/env/local.env.example deploy/psd/thothii-installation.yaml.example \ + deploy/psd/operator.env.example .gitignore +git commit -m "feat(setup): generate local installation configuration" +``` + +--- + +## Task 5: Make `tht setup` Build, Start, and Verify the Product + +**Files:** + +- Create: `tools/tht/internal/setup/run.go` +- Create: `tools/tht/internal/setup/run_test.go` +- Modify: `tools/tht/internal/compose/runner.go` +- Modify: `tools/tht/internal/compose/runner_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package setup + +type Result struct { + DescriptorPath string + ProjectName string + Configured bool + Built bool + Started bool + Healthy bool +} + +func Run( + ctx context.Context, + runner compose.Runner, + request Request, + input io.Reader, + output io.Writer, +) (Result, error) +``` + +### Ordered setup workflow + +1. Discover and validate the repository/worktree. +2. Check Docker Engine, Docker Compose, supported architecture, and line-ending compatibility. +3. Generate or validate local configuration. +4. Run `docker compose config` against the resolved descriptor and profiles. +5. Stop successfully when `--configure-only` is set. +6. Build required images, including the `core` image containing Pi. +7. Start the stack with the installation-specific Compose project name. +8. Wait for frontend, core, qdrant, embedding, and one-shot model initialization health. +9. Run aggregate `tht doctor` and `tht pi doctor`. +10. Print the frontend URL and concise next actions. + +### Steps + +- [ ] Add runner-fake tests asserting the exact order above, immediate stop for `--configure-only`, failure propagation, retryable health polling, and cleanup messaging after partial startup. +- [ ] Add CLI tests proving `tht setup` defaults to build/start and that only `--configure-only` disables those phases. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/setup ./cmd/tht -run 'TestRun|TestSetupCommand' -count=1 +``` + +- [ ] Implement orchestration using the existing descriptor/Compose abstractions. Do not duplicate command execution logic in the top-level argument dispatcher. +- [ ] Make health waits bounded and identify the failing service, last health state, and useful `tht logs ` command. +- [ ] Ensure setup can be rerun idempotently to repair/start an already configured checkout. +- [ ] Rerun focused and full Go suites. + +```bash +cd tools/tht +go test ./internal/setup ./internal/compose ./cmd/tht -count=1 +go test ./... +``` + +- [ ] Commit the complete setup workflow. + +```bash +git add tools/tht/internal/setup tools/tht/internal/compose tools/tht/cmd/tht +git commit -m "feat(setup): build start and verify ThothII" +``` + +--- + +## Task 6: Add `version`, Aggregate `doctor`, and `start --build` + +**Files:** + +- Create: `tools/tht/internal/version/info.go` +- Create: `tools/tht/internal/version/info_test.go` +- Create: `tools/tht/internal/doctor/report.go` +- Create: `tools/tht/internal/doctor/report_test.go` +- Create: `tools/tht/internal/service/service.go` +- Create: `tools/tht/internal/service/service_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package doctor + +type Check struct { + Name string `json:"name"` + Status string `json:"status"` + Detail string `json:"detail"` +} + +type Report struct { + OK bool `json:"ok"` + Checks []Check `json:"checks"` +} + +func Run(ctx context.Context, installation config.Installation, runner Runner) (Report, error) +``` + +### Required behavior + +- `tht version` works without an installation and reports CLI semantic version, commit, build time, OS, and architecture. When an installation is discoverable, it may additionally report the deployed product and Pi versions. +- `tht doctor` aggregates descriptor validation, Compose availability/configuration, file permissions, required volume presence, service health, frontend/core reachability, workspace registry validity, container-local workflow diagnostics, and Pi diagnostics. +- `tht doctor --json` writes one pristine JSON document to stdout; all progress and warnings go to stderr. +- `tht start` starts without rebuilding. `tht start --build` runs the required build before Compose up and then performs bounded health checks. + +### Steps + +- [ ] Add tests for descriptor-free `version`, deterministic JSON, unavailable Docker, stopped/running core, a failed workflow check, and a redacted secret path. +- [ ] Add service tests proving `--build` changes the runner sequence from `up` to `build → up → health`, while normal `start` remains `up → health`. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/version ./internal/doctor ./internal/service ./cmd/tht \ + -run 'TestVersion|TestDoctor|TestStart' -count=1 +``` + +- [ ] Implement build metadata with linker defaults that remain useful in local source builds. +- [ ] Implement aggregate diagnostics as typed checks. Invoke the Python workflow `tht doctor --json` only through `docker compose exec -T core ...` when `core` is running; never require a host virtual environment. +- [ ] Implement `start --build` through the shared Compose runner. +- [ ] Verify JSON output and full tests. + +```bash +cd tools/tht && go test ./... +go run ./cmd/tht version +go run ./cmd/tht doctor --json | jq -e '.ok != null and (.checks | type == "array")' +``` + +- [ ] Commit this operator-observability slice. + +```bash +git add tools/tht/internal/version tools/tht/internal/doctor tools/tht/internal/service tools/tht/cmd/tht +git commit -m "feat(cli): add version diagnostics and build-aware start" +``` + +--- + +## Task 7: Make `tht pi update` Resolve the Latest Stable Pi Version + +**Files:** + +- Create: `tools/tht/internal/pi/latest.go` +- Create: `tools/tht/internal/pi/latest_test.go` +- Modify: `tools/tht/internal/pi/update.go` +- Modify: `tools/tht/internal/pi/update_test.go` +- Modify: `tools/tht/internal/pi/commands.go` +- Modify: `tools/tht/cmd/tht/main.go` + +### Interfaces + +```go +package pi + +type RegistryClient interface { + LatestStable(ctx context.Context, packageName string) (string, error) +} + +func ResolveRequestedVersion( + ctx context.Context, + requested string, + packageName string, + registry RegistryClient, +) (string, error) +``` + +### Required behavior + +- `tht pi update` means “install the latest stable Pi release available from the package registry.” +- `tht pi update --version X.Y.Z` installs that explicit stable version and skips latest-version discovery. +- Prerelease versions require an explicit full version; latest discovery ignores prereleases. +- Failure, malformed registry data, timeout, or an empty version stops before drain, Dockerfile mutation, image build, container replacement, or configuration changes. +- The command keeps the existing `--source build|pull`, immutable image digest, `--yes`, `--drain`, rollback, status, doctor, test, logs, and configure capabilities. `--installation` remains optional through Task 3 discovery. + +### Steps + +- [ ] Add registry-client tests for a stable release, prerelease-only data, malformed JSON, timeout, package-not-found, and explicit version bypass. +- [ ] Change the update transaction test so an omitted version expects the resolved latest stable release rather than the current `ARG PI_VERSION` value from `docker/core.Dockerfile`. +- [ ] Add a no-mutation assertion around every discovery failure. +- [ ] Run focused tests and confirm the old default-to-Dockerfile behavior fails them. + +```bash +cd tools/tht +go test ./internal/pi ./cmd/tht -run 'TestResolveRequestedVersion|TestPiUpdate' -count=1 +``` + +- [ ] Implement a bounded HTTPS client for the registry used by the Pi package declared in `docker/core.Dockerfile`. Parse and validate strict semantic versions; do not shell out to a globally installed npm executable. +- [ ] Resolve the final version before acquiring a drain or update lock. Feed the resolved version into the existing transactional build/pull and rollback path. +- [ ] Leave the project release pin in `docker/core.Dockerfile` unchanged as the reproducible clean-build default. Record the selected newer image/version in installation update state so ordinary restart does not revert it; do not use or rewrite the pin as the meaning of “latest.” +- [ ] Rerun all Pi and Go tests, including explicit build and immutable pull paths. + +```bash +cd tools/tht +go test ./internal/pi -count=1 +go test ./... +``` + +- [ ] Commit latest-version resolution without altering the user's GLM files. + +```bash +git add tools/tht/internal/pi tools/tht/cmd/tht/main.go +git commit -m "feat(pi): update to the latest stable release by default" +``` + +--- + +## Task 8: Implement Transactional Installation Backups + +**Files:** + +- Create: `tools/tht/internal/backup/manifest.go` +- Create: `tools/tht/internal/backup/manifest_test.go` +- Create: `tools/tht/internal/backup/create.go` +- Create: `tools/tht/internal/backup/create_test.go` +- Create: `tools/tht/internal/lifecycle/lock.go` +- Create: `tools/tht/internal/lifecycle/lock_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package backup + +type Manifest struct { + SchemaVersion int `json:"schema_version"` + InstallationID string `json:"installation_id"` + CreatedAt time.Time `json:"created_at"` + SourceRevision string `json:"source_revision"` + IncludesSecrets bool `json:"includes_secrets"` + Entries []Entry `json:"entries"` +} + +type CreateRequest struct { + Output string + IncludeSecrets bool + Confirm bool + Drain bool +} + +func Create(ctx context.Context, installation config.Installation, request CreateRequest) (Result, error) +``` + +### Backup contents + +- Effective installation descriptor and non-secret local environment/configuration files. +- Pi declarative host configuration, including `deploy/pi/models.json` and `deploy/pi/settings.json`. +- Named volumes: `settings`, `pi-state`, `workspace-registry`, `workspace-secrets`, `sessions`, `qdrant-data`, and `embedding-models`. +- Server preservation roots declared by the installation descriptor. +- A manifest containing checksums, logical ownership, source revision, image identities, Compose project name, volume metadata, and whether secret contents are present. + +The installation-owned `workspace-secrets` volume is part of the consistent volume snapshot. External secret-file contents referenced by the installation are excluded by default; their paths and digests are recorded so restore can verify that they still exist. Including those external files requires `--include-secrets --yes`, writes the archive with mode `0600` where supported, and emits a clear custody warning without printing values. + +### Steps + +- [ ] Add manifest tests for deterministic entry ordering, checksums, path normalization, secret markers, and schema-version validation. +- [ ] Add create tests for the seven required volumes, stopped and running installations, active sessions without `--drain`, explicit drain, custom output, default output, write failure, and cleanup of incomplete archives. +- [ ] Assert the default path is `~/.thothii/backups//` and the final archive name contains a UTC timestamp and source revision. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/backup ./internal/lifecycle ./cmd/tht \ + -run 'TestManifest|TestCreate|TestBackupCommand' -count=1 +``` + +- [ ] Implement a per-installation lifecycle lock shared by backup, restore, Pi update, and product update. +- [ ] Quiesce or stop mutable services before snapshotting. Refuse an unsafe live snapshot when active sessions exist and `--drain` was not supplied. +- [ ] Stream volume data into an archive through a minimal helper container; do not materialize secret data in the repository or process arguments. +- [ ] Write the manifest last, fsync the temporary archive, atomically rename it, and remove partial output on error. +- [ ] Run backup tests and inspect a fixture archive to verify that default backups contain no secret payloads. + +```bash +cd tools/tht && go test ./internal/backup ./internal/lifecycle -count=1 +go test ./... +``` + +- [ ] Commit backup support. + +```bash +git add tools/tht/internal/backup tools/tht/internal/lifecycle tools/tht/cmd/tht +git commit -m "feat(cli): add transactional installation backups" +``` + +--- + +## Task 9: Implement Validated Restore with a Recovery Checkpoint + +**Files:** + +- Create: `tools/tht/internal/backup/restore.go` +- Create: `tools/tht/internal/backup/restore_test.go` +- Create: `tools/tht/internal/backup/preflight.go` +- Create: `tools/tht/internal/backup/preflight_test.go` +- Modify: `tools/tht/internal/backup/create.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package backup + +type RestoreRequest struct { + Archive string + Confirm bool + Drain bool +} + +type RestoreResult struct { + Checkpoint string + Restarted bool + Verified bool +} + +func Restore( + ctx context.Context, + installation config.Installation, + request RestoreRequest, +) (RestoreResult, error) +``` + +### Restore contract + +Before modifying the installation, restore must validate the archive schema, every checksum, path traversal safety, installation identity, secret policy, required free disk space, target ownership/permissions, volume mapping, and image/config compatibility. It then creates a non-secret recovery checkpoint of the current installation, acquires the lifecycle lock, drains or refuses active sessions, restores into controlled targets, restarts the stack only when it was previously running, and runs health, aggregate doctor, Pi doctor, and workspace inspection. A failure after mutation leaves the target in a recoverable stopped state and prints the checkpoint path; it must not compound damage with an unrequested automatic restore. + +### Steps + +- [ ] Add adversarial archive tests for `../` traversal, absolute paths, symlink escapes, duplicate entries, checksum mismatch, unknown schema, wrong installation ID, secret-bearing archives without required protections, insufficient disk, and invalid volume ownership. +- [ ] Add transaction tests for success, failure before mutation, failure after one restored volume, failed restart, and failed health verification. Every post-mutation failure must stop the target, retain the checkpoint, and report a deterministic recovery command. +- [ ] Require positional archive plus `--yes`; do not allow an interactive typo to start restore without a complete preflight. +- [ ] Run focused tests and confirm failure. + +```bash +cd tools/tht +go test ./internal/backup ./cmd/tht -run 'TestPreflight|TestRestore' -count=1 +``` + +- [ ] Implement archive validation without extracting untrusted paths directly to final destinations. +- [ ] Create the rollback checkpoint through the same manifest/archive primitives as Task 8. +- [ ] Restore configuration and volumes in a deterministic order; retain the checkpoint path in both success and error messages. +- [ ] Verify with service health, `tht doctor`, `tht pi doctor`, and workspace inspection before declaring success. +- [ ] Rerun focused, package, and full Go tests. + +```bash +cd tools/tht +go test ./internal/backup -count=1 +go test ./... +``` + +- [ ] Commit restore support. + +```bash +git add tools/tht/internal/backup tools/tht/cmd/tht +git commit -m "feat(cli): add validated restore with rollback" +``` + +--- + +## Task 10: Turn `tht update` into a Full Product Update Transaction + +**Files:** + +- Create: `tools/tht/internal/productupdate/plan.go` +- Create: `tools/tht/internal/productupdate/plan_test.go` +- Create: `tools/tht/internal/productupdate/run.go` +- Create: `tools/tht/internal/productupdate/run_test.go` +- Modify: `tools/tht/internal/backup/create.go` +- Modify: `tools/tht/internal/lifecycle/lock.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +### Interfaces + +```go +package productupdate + +type Request struct { + CheckOnly bool + Confirm bool + Drain bool +} + +type Result struct { + PreviousImages map[string]string + CurrentImages map[string]string + Checkpoint string + RolledBack bool +} + +func Plan(ctx context.Context, installation config.Installation) (UpdatePlan, error) +func Run(ctx context.Context, installation config.Installation, request Request) (Result, error) +``` + +### Transaction contract + +- `tht update --check-only` validates compatibility and prints the exact image pull/build, migration, restart, and verification plan without mutating the Git checkout, containers, volumes, or configuration. +- `tht update --yes` updates the complete ThothII deployment according to the current checkout and installation descriptor: pull externally sourced images, build source-defined images, run required preflight/migrations, recreate changed services, and verify the complete product. +- Dirty source files are not reset, overwritten, committed, or pulled. Updating source from Git is deliberately outside this command; the operator chooses the checkout/revision and `tht update` deploys it. +- Before mutation, acquire the lifecycle lock, enforce active-session drain policy, and create a rollback checkpoint through Task 8. +- Record previous immutable image identities. A build/pull, migration, restart, health, aggregate-doctor, or Pi-doctor failure restores configuration/volumes as needed and recreates the previous images. + +### Steps + +- [ ] Add plan tests for a no-op installation, changed built image, changed pulled image, migration required, incompatible descriptor, dirty worktree, and unavailable registry. +- [ ] Add transaction tests for every failure boundary and assert rollback uses recorded image digests rather than mutable tags. +- [ ] Add CLI tests for `--check-only`, required `--yes` before mutation, optional `--drain`, and optional `--installation`. +- [ ] Run focused tests and confirm the current check-only implementation cannot execute the transaction. + +```bash +cd tools/tht +go test ./internal/productupdate ./cmd/tht \ + -run 'TestUpdatePlan|TestProductUpdate|TestUpdateCommand' -count=1 +``` + +- [ ] Implement pure planning first, then the transactional runner. Keep user confirmation outside the mutation core so tests can call it deterministically. +- [ ] Reuse Compose, lifecycle, backup, health, doctor, and Pi diagnostic components; do not introduce parallel shell orchestration. +- [ ] Make failure output state whether rollback completed and provide the retained checkpoint path. +- [ ] Rerun update, backup, Pi, and full Go suites. + +```bash +cd tools/tht +go test ./internal/productupdate ./internal/update ./internal/backup ./internal/pi -count=1 +go test ./... +``` + +- [ ] Commit the product-update transaction. + +```bash +git add tools/tht/internal/productupdate tools/tht/internal/backup \ + tools/tht/internal/lifecycle tools/tht/cmd/tht +git commit -m "feat(cli): update the full product transactionally" +``` + +--- + +## Task 11: Apply the 14 Approved **ERASE** Decisions to the Python Workflow CLI + +**Files:** + +- Create: `harness/tests/fixtures/approved_cli_surface.json` +- Create: `harness/tests/test_cli_surface.py` +- Modify: `harness/tht/cli/decision_cmd.py` +- Modify: `harness/tht/cli/cte_cmd.py` +- Modify: `harness/tht/cli/sql_cmd.py` +- Modify: `harness/tht/cli/memory_cmd.py` +- Modify: `harness/tht/cli/evidence_cmd.py` +- Modify: `harness/tht/cli/lsh_cmd.py` +- Modify: `harness/tht/cli/vector_cmd.py` +- Modify: `harness/tht/cli/formula_cmd.py` +- Modify: `harness/tests/integration/test_gate_cli_signatures.py` +- Delete or rewrite: tests dedicated exclusively to erased CLI entry points, including `harness/tests/test_decision_retract_cli.py` + +### Commands to erase + +```text +decision list +decision retract +cte list +sql explain +sql save +memory clear +memory migrate +evidence extract +evidence index +lsh build +lsh query +vector init +formula save +formula list +``` + +Removing a command means removing its Typer registration, help entry, CLI-only parsing code, CLI-only tests, and active documentation. Underlying domain functions may remain only when a maintained workflow, preprocessing pipeline, or test imports them directly. Delete dead implementation only after a repository-wide reachability check. + +### Steps + +- [ ] Build `approved_cli_surface.json` from the two approved audit reports. It must enumerate all 55 maintained paths and the 8 enhanced paths and explicitly blacklist the 14 erased paths. +- [ ] Add a recursive Typer help test that compares the actual command tree to this fixture. Add integration assertions that every command invoked by `harness/.pi/extensions/tht-gate.js`, `harness/.pi/skills/tht-sessione/SKILL.md`, backend `ThtRunner`, preprocessing jobs, and deployment smoke tests is still present. +- [ ] Run the command-surface tests before removal and confirm they fail because the 14 erased commands are still exposed. + +```bash +cd harness +.venv/bin/pytest -q tests/test_cli_surface.py tests/integration/test_gate_cli_signatures.py +``` + +- [ ] Remove the 14 registrations and CLI-only code. Preserve `phase reopen` as the supported correction/invalidation flow, `session show --json` as the ledger view, `sql set-final`/`sql export`, versioned `preprocess evidence`/`preprocess dwh`, collection reconciliation, `ollama ensure`, and `search find`. +- [ ] Delete or rewrite tests that assert the obsolete surface. Add negative tests requiring a nonzero exit and normal “no such command” message for every erased path. +- [ ] Use import and call-site scans before deleting any shared function. + +```bash +rg -n 'decision_retract|cte_list|sql_explain|sql_save|memory_clear|memory_migrate|evidence_(extract|index)|lsh_(build|query)|vector_init|formula_(save|list)' \ + harness backend frontend scripts docs +``` + +- [ ] Run the full non-L2 harness suite and the gate signature test. + +```bash +cd harness +.venv/bin/ruff check . +.venv/bin/pytest -q +``` + +- [ ] Commit the approved surface reduction. + +```bash +git add harness/tht/cli/decision_cmd.py harness/tht/cli/cte_cmd.py \ + harness/tht/cli/sql_cmd.py harness/tht/cli/memory_cmd.py \ + harness/tht/cli/evidence_cmd.py harness/tht/cli/lsh_cmd.py \ + harness/tht/cli/vector_cmd.py harness/tht/cli/formula_cmd.py \ + harness/tests/fixtures/approved_cli_surface.json harness/tests/test_cli_surface.py \ + harness/tests/integration/test_gate_cli_signatures.py +git add -u -- harness/tests/test_decision_retract_cli.py +git commit -m "refactor(cli): remove obsolete workflow commands" +``` + +--- + +## Task 12: Implement the 8 Approved **ENHANCE** Outcomes + +**Files:** + +- Modify: `harness/tht/cli/config_cmd.py` +- Modify: `harness/tht/cli/doctor_cmd.py` +- Modify: `harness/tht/cli/db_cmd.py` +- Modify: `harness/tht/cli/memory_cmd.py` +- Modify: `harness/tht/memory.py` +- Create: `harness/tests/test_cli_enhanced_surface.py` +- Modify: `harness/tests/test_doctor_cli.py` +- Modify: `harness/tests/test_cli_config_environment.py` +- Modify: `harness/tests/test_memory_metadata.py` +- Modify: `harness/tests/test_qdrant_cli_commands.py` +- Create: `tools/tht/internal/setup/tls.go` +- Create: `tools/tht/internal/setup/tls_test.go` +- Modify: `tools/tht/internal/setup/run.go` +- Modify: `tools/tht/internal/setup/run_test.go` + +### Exact enhanced outcomes + +1. `config check`: keep its validation as a reusable internal function and structured result, but make public `tht doctor` the normal single preflight. Avoid two competing operator diagnostics. +2. `doctor`: make it the non-mutating, multilayer diagnostic for installation, descriptor, Compose, storage, runtime configuration, DWH, Pi, Qdrant, and embedder, with readable output and pristine `--json`. +3. `db fetch-ca`: integrate CA retrieval into guided workspace setup. Show endpoint, certificate subject, validity, SHA-256 fingerprint, and destination before confirmation. Keep the standalone operation as an advanced TLS command, not a mandatory manual setup step. +4. `memory list`: make it an advanced paginated administrative view with filters for state, provenance, and indexing status plus `--json`; keep it out of concise/basic help. +5. `memory show`: display immutable provenance, source decision, mutable fields, index state, timestamps, and references needed for a safe correction. +6. `memory update`: permit only documented mutable fields, print a diff, require explicit confirmation, and reject attempts to alter identity or provenance. +7. `memory delete`: require an exact identifier and explicit confirmation, show registry/index impact, remove consistently from both, and verify absence afterward. +8. `memory index`: treat it as repair. Detect and report drift first; rebuild only with explicit confirmation; verify registry/index consistency afterward. + +### Steps + +- [ ] Add focused tests for every outcome above, including JSON purity, no mutation by doctor, TLS fingerprint confirmation, pagination bounds, provenance immutability, exact-ID deletion, drift-only repair, confirmation refusal, and partial index failure. +- [ ] Run the focused tests and confirm they fail against current behavior. + +```bash +cd harness +.venv/bin/pytest -q tests/test_cli_enhanced_surface.py tests/test_doctor_cli.py \ + tests/test_cli_config_environment.py tests/test_memory_metadata.py \ + tests/test_qdrant_cli_commands.py +``` + +- [ ] Extract configuration validation into a typed result consumed by both container-local doctor and the host aggregate doctor from Task 6. Keep `config check` callable for automation but mark it advanced in help. +- [ ] Extend doctor without adding repair side effects. A failing layer changes exit status and report data but never changes files, volumes, indexes, or containers. +- [ ] Integrate TLS CA fetch into host workspace create/update setup with a two-stage inspect/confirm flow. Reject hostname mismatch and invalid/expired certificates before writing. +- [ ] Implement memory list/show/update/delete/index with a shared repository/index transaction boundary and postcondition checks. +- [ ] Rerun focused tests, then the full harness suite and Go workspace tests. + +```bash +cd harness +.venv/bin/ruff check . +.venv/bin/pytest -q +cd ../tools/tht +go test ./internal/setup ./internal/doctor -count=1 +go test ./... +``` + +- [ ] Commit enhanced diagnostics, TLS setup, and memory administration. + +```bash +git add harness/tht/cli/config_cmd.py harness/tht/cli/doctor_cmd.py \ + harness/tht/cli/db_cmd.py harness/tht/cli/memory_cmd.py harness/tht/memory.py \ + harness/tests/test_cli_enhanced_surface.py harness/tests/test_doctor_cli.py \ + harness/tests/test_cli_config_environment.py harness/tests/test_memory_metadata.py \ + harness/tests/test_qdrant_cli_commands.py tools/tht/internal/setup/tls.go \ + tools/tht/internal/setup/tls_test.go tools/tht/internal/setup/run.go \ + tools/tht/internal/setup/run_test.go tools/tht/internal/doctor +git commit -m "feat(cli): harden diagnostics TLS and memory administration" +``` + +--- + +## Task 13: Rewrite and Verify the Pi Management Frontend Guidance + +**Files:** + +- Modify: `frontend/src/shell/PiManagement.tsx` +- Modify: `frontend/src/shell/PiManagement.test.tsx` +- Modify: `frontend/src/api/pi-management.ts` +- Modify: `frontend/src/api/pi-management.test.ts` +- Modify if required by the verified behavior: `backend/src/pi/management.ts` +- Modify if required by the verified behavior: `backend/src/routes/pi-management.ts` +- Modify if required by the verified behavior: `backend/test/pi-management.test.ts` +- Modify if required by the verified behavior: `backend/test/routes-pi-management.test.ts` + +### Information architecture and copy contract + +- The whole section starts collapsed. After the operator expands it, the section title remains “Update Pi configuration and relaunch its container.” +- Its introduction starts exactly with “Using the host terminal”. It must not say “not this browser page”. +- Linux, macOS, and Windows are three separate disclosure tabs/panels. All are closed on initial render; opening one does not require another to remain open. +- The containing window is shorter than the current one and has an always-available vertical scrollbar. Mouse wheel, trackpad, keyboard, and touch scrolling must not be trapped. +- Guidance covers only Pi managed by ThothII Docker Compose. Remove every native/local Pi branch. +- Explain in plain language that `deploy` is a directory in the root of the ThothII project, at the same level as `compose.yaml`. +- Explain that `deploy/pi/models.json` and `deploy/pi/settings.json` are host files mounted read-only into `core`: edit them from the host project/worktree, never from inside the container. +- Break every explanation longer than two rendered lines into short paragraphs, numbered steps, bullets, field tables, or command blocks. +- Explain configuration fields before naming them: + - `baseUrl`: the provider endpoint used by Pi. + - `api`: the Pi adapter/protocol expected by that provider. + - `models`: the provider's available model objects and identifiers. + - `enabledModels`: every `provider/model` pair that Pi may select. + - credential/auth file settings: paths to host-side files containing provider credentials; never paste a secret into the page, command, Compose file, or tracked JSON. +- Show direct commands only: `tht pi configure`, `tht pi update`, `tht pi status`, `tht pi doctor`, `tht pi test`, `tht pi logs`, and `tht pi rollback`. Do not show `thothctl`, `~/bin`, `./bin`, `./tht`, shell wrappers, Go builds, or mandatory `--installation`. +- State that commands work from a repository root or Git worktree root once `tht` has been installed. Explain the optional `--installation PATH` only as an ambiguity/automation override. +- State that omitting `--version` from `tht pi update` installs the latest stable Pi release. Do not conflate the Pi package version with the provider/model selected by `tht pi configure`. +- Preserve and display GLM 5.3 when it is present in the live Pi configuration. + +### Platform-specific command blocks + +Each panel uses the native terminal syntax but the same operation sequence: + +```text +1. Open Terminal/PowerShell in the ThothII repository or worktree root. +2. Edit deploy/pi/models.json and deploy/pi/settings.json on the host. +3. Run tht pi configure --provider --model --thinking . +4. Run tht pi update (or add --version X.Y.Z for an explicit Pi version). +5. Use tht pi restart when only configuration changed and no Pi package update is needed. +6. Run tht pi status, tht pi doctor, and tht pi test. +7. If needed, inspect tht pi logs or run tht pi rollback. +``` + +macOS and Windows explicitly require Docker Desktop to be running. Linux requires a running Docker Engine and permission to use Docker. PowerShell examples use PowerShell line continuation only when genuinely necessary; prefer one executable command per line. + +### “Show sanitized logs” behavior + +The button must request recent core/Pi diagnostic logs, display timestamps and severity where available, and redact credentials, authorization headers, API keys, tokens, cookies, connection strings, and secret file contents. It is diagnostic only: it must not change Pi or start/restart containers. Empty, unavailable, loading, success, and failure states must all be visible and understandable. If the current backend already satisfies this contract, change only tests/copy; otherwise make the smallest backend correction needed. + +### Steps + +- [ ] Extend component tests to require all platform panels closed initially, independent open/close state, a bounded scrollable region, exact introductory wording, structured copy, Docker-only guidance, root/worktree commands, field explanations, latest-Pi semantics, and absence of every obsolete command/path. +- [ ] Add accessibility tests for disclosure names, `aria-expanded`, focus order, keyboard activation, and scroll-region labeling. +- [ ] Extend API/backend tests for sanitized-log redaction and all UI states. Include adversarial fake logs containing every secret class listed above. +- [ ] Run focused tests and confirm current copy/layout fail the new contract. + +```bash +cd frontend +npx vitest run src/shell/PiManagement.test.tsx src/api/pi-management.test.ts +cd ../backend +npx vitest run test/pi-management.test.ts test/routes-pi-management.test.ts +``` + +- [ ] Refactor the long instruction blob into data-driven platform sections and small semantic components. Keep state local to Pi Management unless there is an existing shared disclosure component. +- [ ] Apply a bounded height plus `overflow-y: auto`/`scroll` to the actual element containing the full section; remove ancestor wheel/overflow rules that block movement. +- [ ] Implement or verify sanitized-log behavior end to end without exposing raw secrets to the browser. +- [ ] Rerun focused tests, type checks, builds, and complete frontend/backend suites. + +```bash +cd frontend +npx vitest run +npx tsc -b +npm run build +cd ../backend +npx vitest run +npx tsc --noEmit -p . +npm run build +``` + +- [ ] Commit the Pi Management UX slice without overwriting `deploy/pi/models.json` or `deploy/pi/settings.json`. + +```bash +git add frontend/src/shell/PiManagement.tsx frontend/src/shell/PiManagement.test.tsx \ + frontend/src/api/pi-management.ts frontend/src/api/pi-management.test.ts \ + backend/src/pi/management.ts backend/src/routes/pi-management.ts \ + backend/test/pi-management.test.ts backend/test/routes-pi-management.test.ts +git commit -m "feat(frontend): simplify Pi management guidance" +``` + +--- + +## Task 14: Replace Active Installation, CLI, and Pi Documentation + +**Files:** + +- Rename: `docs/contracts/tht-pi.md` → `docs/contracts/tht-pi.md` +- Modify: `README.md` +- Modify: `PROJECT_STATE.md` +- Modify: `AGENTS.md` +- Modify: `docs/architecture/overview.md` +- Modify: `docs/guida-utente.md` +- Modify: `docs/install/local.md` +- Modify: `docs/install/server.md` +- Modify: `docs/install/pi-management.md` +- Modify: `docs/install/local-workspace-registry.md` +- Modify: `docs/install/server-workspace-registry.md` +- Modify: `docs/install/psd-workspace-setup.md` +- Modify: `docs/install/windows-line-endings.md` +- Modify: `docs/contracts/tht-dwh.md` +- Modify: `docs/contracts/workspace-preprocessing-cli.md` +- Modify: `docs/testing/p2-p6-manual-verification.md` +- Create: `scripts/test-tht-command-docs.sh` +- Modify: `scripts/verify-workspace-install-docs.sh` +- Modify: `scripts/test-verify-workspace-install-docs.sh` + +### Required onboarding story + +After cloning ThothII, the normal path is exactly: + +```bash +# macOS or Linux, from the repository/worktree root +bash scripts/install-tht.sh +tht setup +``` + +```powershell +# Windows PowerShell, from the repository/worktree root +powershell -ExecutionPolicy Bypass -File scripts/install-tht.ps1 +tht setup +``` + +The documentation must say that `tht setup` validates Docker, creates local configuration, builds images, starts containers, waits for health, and runs diagnostics. It must present `--configure-only` as the explicit way to stop before build/start. `scripts/run-stack.sh` may remain documented as an advanced contributor shortcut, not the primary user onboarding path. + +### Steps + +- [ ] Create a documentation contract test that scans active docs and scripts for forbidden user instructions: `thothctl`, `~/bin`, `./bin/tht`, `./tht`, `tht.sh`, `go build`, a required `--installation`, native Pi setup, or editing files inside `core`. +- [ ] Exclude historical `docs/superpowers/specs/`, `docs/superpowers/plans/`, `docs/plans/`, and `docs/reports/` from stale-name failure; history remains immutable context. Active docs and code examples are not excluded. +- [ ] Add positive assertions for both installer commands, `tht setup`, `setup --configure-only`, `start --build`, full `update`, `backup`, `restore`, Pi latest-version behavior, worktree discovery, and the three Pi platforms. +- [ ] Run documentation tests and confirm they fail against current active guidance. + +```bash +bash scripts/test-tht-command-docs.sh +bash scripts/test-verify-workspace-install-docs.sh +``` + +- [ ] Rewrite active documents around one lifecycle: clone → install `tht` → `tht setup` → `tht doctor` → normal operation → `tht update`/`tht pi update` → `tht backup`/`tht restore`. +- [ ] Document the host/container command-name boundary once: users invoke the installed native `tht`; the backend and Pi gate invoke the Python `tht` inside `core`. Do not expose a second binary name. +- [ ] Document root/worktree discovery and optional `--installation` precedence with examples of ambiguity and automation, not as boilerplate on every command. +- [ ] Update the Pi contract and management guide with Docker-only host-file editing, declarative mounts, credential-file meaning, latest stable Pi default, rollback, and sanitized logs. +- [ ] Update command reference material from `approved_cli_surface.json`, explicitly omitting the 14 erased commands and marking enhanced administrative commands as advanced where applicable. +- [ ] Rerun documentation contracts and link checks. + +```bash +bash scripts/test-tht-command-docs.sh +bash scripts/test-verify-workspace-install-docs.sh +rg -n '\]\([^)]*\.md(#[^)]*)?\)' README.md PROJECT_STATE.md AGENTS.md docs/install docs/contracts docs/architecture docs/testing +``` + +- [ ] Commit active documentation and contracts. + +```bash +git add README.md PROJECT_STATE.md AGENTS.md docs/architecture docs/guida-utente.md \ + docs/install docs/contracts docs/testing scripts/test-tht-command-docs.sh \ + scripts/verify-workspace-install-docs.sh scripts/test-verify-workspace-install-docs.sh +git commit -m "docs: define the unified tht product workflow" +``` + +--- + +## Task 15: Run Cross-Platform Gates, Install on This Mac, and Update the Live Stack + +**Files:** + +- Create: `docs/reports/2026-08-15-unified-tht-cli-acceptance.md` +- Modify only if a gate finds a defect: files owned by Tasks 1–14, with a new failing regression test first + +### Automated acceptance matrix + +- Go: all native CLI unit, contract, transaction, race, and cross-platform compile tests. +- Python: Ruff and full non-L2 pytest; run opt-in L2 only when its external GLM/DWH prerequisites are available and record the result separately. +- Backend: Vitest, TypeScript no-emit typecheck, production build. +- Frontend: Vitest, TypeScript build, production build, Playwright. +- Deployment: Compose config for base+local, server, GPU, HTTPS/SSH workspace, preprocessing, and session-server variants. +- Installer: macOS/Linux shell tests and Windows PowerShell tests; cross-build native binaries. +- Documentation: active command/copy contracts and link/path checks. + +### Steps + +- [ ] Run the complete automated matrix from the repository/worktree root and save command, revision, result, and meaningful skips in the acceptance report. + +```bash +cd tools/tht && go test -race ./... && cd ../.. +cd harness && .venv/bin/ruff check . && .venv/bin/pytest -q && cd .. +cd backend && npx vitest run && npx tsc --noEmit -p . && npm run build && cd .. +cd frontend && npx vitest run && npx tsc -b && npm run build && npm run e2e && cd .. +bash scripts/test-install-tht.sh +pwsh -NoProfile -File scripts/test-install-tht.ps1 +bash scripts/test-tht-command-docs.sh +bash scripts/test-verify-workspace-install-docs.sh +bash scripts/unified-deployment-smoke.sh +``` + +- [ ] Cross-build and inspect all packaged binaries. Run Windows behavior tests in the existing Windows CI/VM path rather than claiming success from compilation alone. + +```bash +bash scripts/build-tht.sh --all +``` + +- [ ] Before touching the Mac installation, resolve the exact existing executables with `command -v`, `type -a`, file metadata, and hashes. Remove only the obsolete development `thothctl` binary or symlink that was positively identified; do not remove any directory or installation data. +- [ ] Install the newly tested Mac binary with `bash scripts/install-tht.sh`, start a fresh terminal lookup, and verify that `tht` resolves without a relative path while `thothctl` no longer resolves. + +```bash +command -v tht +type -a tht +tht version +tht help +``` + +- [ ] From `/Users/mp/projects/ThothII/.worktrees/p8-l2-live-session-smoke`, verify automatic worktree discovery and the optional installation override. Confirm no command searches for `/Users/mp/thothii-installation.yaml`. +- [ ] Inspect current sessions and container health. If no unsafe active work exists, update the live installation through the new transaction, using drain only as needed, then verify the stack. + +```bash +tht update --check-only +tht update --yes --drain +tht status +tht doctor +tht pi status +tht pi doctor +tht pi test +``` + +- [ ] Verify port 8080 in a real browser with Playwright: open Pi Management; require all three platform panels closed initially; open each independently; scroll the bounded panel using wheel and keyboard; verify structured Linux/macOS/Windows text; verify only direct `tht` commands; click “Show sanitized logs” and inspect loading/success/empty/error behavior; confirm GLM 5.3 remains available; require no console errors, failed API calls, or exposed secrets. +- [ ] Compare the live Pi configuration and provider/model list before and after deployment to prove the existing GLM 5.3 changes were preserved. +- [ ] Record exact versions, image digests, test totals, manual observations, live URL, rollback checkpoint, and any explicitly deferred L2/Windows gate in `docs/reports/2026-08-15-unified-tht-cli-acceptance.md`. +- [ ] Run `git status --short` and a final diff audit. Confirm no unrelated files, secrets, `.playwright-cli/`, or stale model fixtures were staged. +- [ ] Commit only the acceptance report and regression fixes, if any. + +```bash +git add docs/reports/2026-08-15-unified-tht-cli-acceptance.md +git commit -m "test: record unified tht acceptance" +``` + +--- + +## Approval Boundary + +Approval of this plan authorizes implementation of Tasks 1–15 in order, including: + +- replacing `thothctl` with the single installed command `tht` without compatibility aliases; +- installing the finished CLI on this Mac after automated gates pass; +- removing only the positively identified obsolete Mac development executable; +- updating/recreating the live Docker Compose services and verifying the GUI on port 8080; +- removing the 14 approved workflow CLI commands and implementing the 8 approved enhancements; +- changing active project documentation and Pi Management guidance as specified; +- creating backup/restore checkpoints needed to make updates recoverable. + +Approval does **not** authorize deleting user data, secrets, workspaces, sessions, unrelated worktree changes, or historical design/audit documents. It does not authorize overwriting the current GLM 5.3 configuration. Any newly discovered decision that materially changes this architecture, command surface, data-safety policy, or live-deployment scope must return to the user for approval before implementation continues. + +Implementation should stop for review after these four milestones: + +1. Tasks 1–5: one installable command and complete setup. +2. Tasks 6–10: diagnostics, Pi/product updates, backup, restore, and rollback. +3. Tasks 11–14: approved command-surface changes, frontend, and documentation. +4. Task 15: Mac installation and live acceptance on port 8080. diff --git a/docs/superpowers/plans/2026-08-16-thothii-authentication.md b/docs/superpowers/plans/2026-08-16-thothii-authentication.md new file mode 100644 index 00000000..2196b381 --- /dev/null +++ b/docs/superpowers/plans/2026-08-16-thothii-authentication.md @@ -0,0 +1,1456 @@ +# ThothII Authentication Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add secure local authentication with remembered sessions and generic OIDC authentication with certified Authentik group validation, all administered through `tht` and included in workspace diagnostics. + +**Architecture:** Fastify owns authentication, opaque file-backed browser sessions, CSRF protection, OIDC, and permission enforcement. The Go `tht` CLI owns protected local-user/configuration writes and delegates live provider checks to a container-local backend diagnostic command. React becomes a same-origin authenticated shell and never handles passwords beyond login submission or stores bearer/session tokens. + +**Tech Stack:** Node 24.16.0, Fastify 5, TypeScript, React 18, `openid-client` 6.8.5, `@fastify/cookie` 11.1.2, `@fastify/rate-limit` 11.2.0, Go 1.26, `golang.org/x/crypto/argon2` 0.55.0, `golang.org/x/term` 0.45.0, Vitest, Playwright, Docker Compose. + +**Spec:** `docs/superpowers/specs/2026-08-16-thothii-authentication-design.md` + +## Global Constraints + +- The only host CLI is `tht`; do not add another executable or revive `thothctl`. +- Production modes are `local` and `oidc`; `none` and `mock` are development/test only, while `upstream` remains a deprecated migration adapter. +- Authentik is the first-release certified OIDC provider; the browser OIDC protocol layer must not contain Authentik-specific login logic. +- The ID token must contain direct claim `groups: string[]`; absent, malformed, indirect, or overage claims fail closed. +- Only configured groups are checked and mapped. Unmapped provider groups are ignored silently and never produce a warning. +- Every configured group must be proven to exist through the configured group-catalog adapter; first release provides `authentik`. +- Local passwords use Argon2id v19 with `m=65536,t=3,p=1`, 16-byte random salt, and 32-byte output. +- A remembered local session has a seven-day idle timeout and thirty-day absolute timeout and survives browser/backend restarts. +- No raw session cookie, password, CSRF token, OIDC token, client secret, Authentik API token, or password hash may enter logs or diagnostic output. +- Browser authentication uses an opaque `HttpOnly`, `SameSite=Lax`, path-scoped cookie; `Secure` is conditional on an HTTPS public URL so loopback HTTP remains functional. +- Every cookie-authenticated state-changing route requires a CSRF token and same-origin browser checks. +- Authentication configuration is installation-global, but static checks appear in workspace validation and live checks appear in workspace connection tests. +- Workspace document content remains in its workspace language; application chrome and new authentication UI strings are English. +- JSON CLI stdout is pristine. Prompts, progress, and human guidance go to stderr. +- Node is exactly `24.16.0`; Docker uses `sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7`. +- Existing session artifacts, workflow persistence, workspace ownership, and Pi RPC behavior must not change. +- Preserve unrelated user changes and the existing untracked `.playwright-cli/` and `.thothctl/` paths. + +## Target file structure + +New backend files are split by responsibility: + +```text +backend/src/auth/ + types.ts roles, permissions, principal and diagnostic contracts + config.ts strict auth.yaml parser and canonical revision + authorization.ts permission expansion and route guards + local-registry.ts bounded/safe users.yaml reader and revision lookup + password.ts PHC parsing and Node Argon2id verification + session-store.ts opaque durable session and OIDC-state files + csrf.ts CSRF token and browser-origin enforcement + oidc-client.ts provider-neutral OIDC protocol adapter + group-catalog.ts catalog interface + authentik-group-catalog.ts Authentik read-only group existence adapter + diagnostics.ts shared static/live authentication diagnostics + routes.ts local/OIDC login, callback, logout, config and /me + diagnostic-command.ts machine/device-flow checker invoked by tht +``` + +New frontend files are: + +```text +frontend/src/auth/ + AuthGate.tsx + LoginPage.tsx + authState.ts +frontend/src/api/auth.ts +``` + +New Go files are: + +```text +tools/tht/internal/authconfig/ + types.go + password.go + store.go + users.go + commands.go +``` + +Focused tests use matching `*.test.ts`, `*.test.tsx`, and `*_test.go` files. Avoid adding auth +logic to `backend/src/app.ts`, `frontend/src/shell/AppShell.tsx`, or +`tools/tht/cmd/tht/main.go` beyond dependency wiring and command dispatch. + +--- + +### Task 1: Align the Runtime on Node 24 and Install Authentication Dependencies + +**Files:** +- Modify: `docker/core.Dockerfile` +- Modify: `docker/frontend.Dockerfile` +- Modify: `docker/smoke/core-smoke.sh` +- Modify: `backend/package.json` +- Modify: `backend/package-lock.json` +- Modify: `frontend/package-lock.json` only if `npm install` normalizes lock metadata under Node 24 +- Test: `backend/test/health.test.ts` + +**Interfaces:** +- Produces: Node `24.16.0` in backend build, frontend build, Pi build, and final core runtime. +- Produces: backend imports for `openid-client`, `@fastify/cookie`, and `@fastify/rate-limit`. +- Preserves: the Pi package engine floor `>=22.19.0` and all existing image/runtime contracts. + +- [ ] **Step 1: Pin the failing runtime expectation** + +Add an assertion to `backend/test/health.test.ts` that the build/runtime contract exposes major +version 24, and update `docker/smoke/core-smoke.sh` to reject Node 22/23. The accepted shell case is: + +```sh +case "$node_version" in + v24.16.*) ;; + *) echo "Node 24.16 required, found $node_version" >&2; exit 1 ;; +esac +``` + +- [ ] **Step 2: Run the focused checks and observe the old runtime failure** + +Run: + +```bash +cd backend && npx vitest run test/health.test.ts +docker build --target backend-build -f docker/core.Dockerfile . +``` + +Expected: the source/runtime assertion or smoke inspection still reports Node 22. + +- [ ] **Step 3: Update all official Node stages and package metadata** + +Replace every Node stage in `docker/core.Dockerfile` and `docker/frontend.Dockerfile` with: + +```dockerfile +FROM node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 +``` + +Update comments and smoke messages from Node 22 to Node 24.16. Then install exact backend versions: + +```bash +cd backend +npm install --save-exact openid-client@6.8.5 @fastify/cookie@11.1.2 @fastify/rate-limit@11.2.0 +npm install --save-dev --save-exact @types/node@24.13.3 +``` + +- [ ] **Step 4: Run the complete Node compatibility gate** + +Run: + +```bash +cd backend && npx tsc --noEmit -p . && npx vitest run && npm run build +cd ../frontend && npx tsc -b && npx vitest run && npm run build +cd .. && docker build -f docker/core.Dockerfile -t thothii-core:auth-node24 . +docker run --rm --entrypoint /bin/sh thothii-core:auth-node24 -c 'node --version && /app/docker/smoke/core-smoke.sh' +``` + +Expected: Node reports `v24.16.0`; backend/frontend gates and the core smoke pass. Any Pi or native +dependency regression blocks this task and is fixed before proceeding. + +- [ ] **Step 5: Commit the runtime baseline** + +```bash +git add docker/core.Dockerfile docker/frontend.Dockerfile docker/smoke/core-smoke.sh \ + backend/package.json backend/package-lock.json backend/test/health.test.ts frontend/package-lock.json +git commit -m "build: align authentication runtime on Node 24" +``` + +### Task 2: Define and Validate Authentication Configuration + +**Files:** +- Create: `backend/src/auth/types.ts` +- Create: `backend/src/auth/config.ts` +- Create: `backend/test/auth-config.test.ts` +- Modify: `backend/src/config.ts` +- Modify: `backend/test/config.test.ts` + +**Interfaces:** +- Produces: + +```ts +export type AuthMode = "local" | "oidc" | "upstream" | "none" | "mock"; +export type Role = "user" | "admin"; +export type Permission = + | "session.use" | "session.read_all" | "session.manage_all" + | "settings.manage" | "workspace.manage" | "workspace.secrets.manage" + | "pi.manage" | "auth.diagnostics.read"; + +export interface LoadedAuthConfig { + value: AuthenticationConfig; + revision: string; + sourcePath: string; +} + +export function loadAuthenticationConfig(path: string): LoadedAuthConfig; +export interface AuthenticationConfigProvider { current(): LoadedAuthConfig } +export function createAuthenticationConfigProvider(path: string): AuthenticationConfigProvider; +export function rolesToPermissions(roles: readonly Role[]): readonly Permission[]; +``` + +- Consumes: existing `yaml` and `zod` backend dependencies. + +- [ ] **Step 1: Write strict parser and permission tests first** + +Cover at least these fixtures in `backend/test/auth-config.test.ts`: + +```ts +test.each([ + ["unknown root key", "unexpected"], + ["relative users file", "../users.yaml"], + ["OIDC without groups claim", undefined], + ["OIDC without admin mapping", {}], + ["HTTP non-loopback public URL", "http://thoth.example"], +])("rejects %s", (_label, mutation) => { + expect(() => loadAuthenticationConfig(writeFixture(mutation))).toThrow(); +}); + +test("canonical group map order produces one stable revision", () => { + expect(loadAuthenticationConfig(first).revision).toBe(loadAuthenticationConfig(reordered).revision); +}); +``` + +Also assert `admin` expands to every admin permission plus `session.use`, duplicate roles collapse, +and unknown roles fail parsing. + +- [ ] **Step 2: Run tests and verify missing-module failures** + +Run: `cd backend && npx vitest run test/auth-config.test.ts test/config.test.ts` + +Expected: failure because the new parser/types and AppConfig fields do not exist. + +- [ ] **Step 3: Implement strict configuration types and canonical revision** + +Implement `loadAuthenticationConfig()` with bounded 1 MiB reads, `yaml.parseDocument`, explicit +duplicate-key rejection, `z.strictObject`, and a canonical SHA-256 revision over sorted JSON. +`createAuthenticationConfigProvider()` caches by inode/size/mtime and reloads after the CLI's atomic +replacement so mapping revisions invalidate sessions without a process restart. Add these fields +to `AppConfig`: + +```ts +authMode: AuthMode; +authConfigFile: string; +authStateRoot: string; +authentication?: AuthenticationConfigProvider; +``` + +`THT_AUTH_CONFIG_FILE` defaults to `/run/thothii-auth/auth.yaml` and +`THT_AUTH_STATE_ROOT` defaults to `/data/auth`. A present config file is the sole source for +`local` or `oidc`; reject a simultaneous `AUTH_MODE` to prevent split-brain configuration. When the +file is absent, accept `AUTH_MODE=none|mock|upstream` only for development or migration. Public +exposure accepts config-derived `oidc` or deprecated `upstream`, never `local`, `none`, or `mock`. + +- [ ] **Step 4: Add complete local and OIDC fixture coverage** + +Assert the exact default lifetimes from the spec, loopback HTTP exception, the fixed references +`THT_OIDC_CLIENT_SECRET` and `THT_AUTHENTIK_API_TOKEN`, exact group-name matching, one admin +mapping, and no secret values in thrown messages. + +- [ ] **Step 5: Run focused and compile gates** + +Run: + +```bash +cd backend +npx vitest run test/auth-config.test.ts test/config.test.ts +npx tsc --noEmit -p . +``` + +Expected: all tests pass and TypeScript is clean. + +- [ ] **Step 6: Commit the configuration contract** + +```bash +git add backend/src/auth/types.ts backend/src/auth/config.ts backend/src/config.ts \ + backend/test/auth-config.test.ts backend/test/config.test.ts +git commit -m "feat(auth): define strict installation authentication config" +``` + +### Task 3: Centralize Roles, Permissions, and Route Authorization + +**Files:** +- Create: `backend/src/auth/authorization.ts` +- Create: `backend/test/authorization.test.ts` +- Modify: `backend/src/auth/principal.ts` +- Modify: `backend/src/auth/auth.ts` +- Modify: `backend/src/app.ts` +- Modify: `backend/src/routes/pi-management.ts` +- Modify: `backend/src/routes/settings.ts` +- Modify: `backend/src/routes/sessions.ts` +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/src/tht/tht-runner.ts` +- Modify: relevant `backend/test/routes-*.test.ts`, `backend/test/auth.test.ts`, and `backend/test/tht-runner.test.ts` + +**Interfaces:** +- Produces: + +```ts +export interface PrincipalContext { + issuer: string; + subject: string; + displayName?: string; + roles: readonly Role[]; + permissions: readonly Permission[]; + isAdmin: boolean; +} + +export function hasPermission(principal: PrincipalContext, permission: Permission): boolean; +export function requirePermission( + request: FastifyRequest, + reply: FastifyReply, + permission: Permission, +): PrincipalContext | FastifyReply; +``` + +- Preserves: `issuer + subject` ownership and `THT_PRINCIPAL_IS_ADMIN` for the harness transition. + +- [ ] **Step 1: Write a route authorization matrix that fails under scattered `isAdmin` checks** + +In `backend/test/authorization.test.ts`, create principals for no role, `user`, and `admin`; assert +every permission in the spec. Add focused route cases: + +```ts +expect(await requestAs(user, "PUT", "/settings", body)).toHaveStatus(403); +expect(await requestAs(admin, "PUT", "/settings", body)).toHaveStatus(200); +expect(await requestAs(user, "POST", "/pi-management/test")).toHaveStatus(403); +expect(await requestAs(admin, "POST", "/pi-management/test")).toHaveStatus(200); +``` + +Add session cases proving a user can mutate an owned session, cannot request `scope=all`, and an +admin can do both. + +- [ ] **Step 2: Run focused route tests and record the expected failures** + +Run: + +```bash +cd backend +npx vitest run test/authorization.test.ts test/routes-settings.test.ts \ + test/routes-pi-management.test.ts test/routes-sessions.test.ts test/routes-workspaces.test.ts +``` + +Expected: failures expose missing permissions and existing mode-specific authorization branches. + +- [ ] **Step 3: Implement one authorization guard and migrate every route** + +Implement `requirePermission()` to return one stable response: + +```json +{"code":"auth_forbidden","error":"This operation is not permitted"} +``` + +Remove `managementAllowed()` and route-local admin booleans. Apply the route table in the spec. +Keep object ownership inside the session locator/authorizer, with admin bypass based on +`session.read_all` or `session.manage_all` rather than `isAdmin`. + +For compatibility adapters, derive roles as follows: + +```ts +none -> admin only when publicExposure is false +mock -> user, or admin when the explicit mock-admin test header is true +upstream -> user plus admin when X-Thoth-Is-Admin is true +``` + +- [ ] **Step 4: Preserve the harness principal environment contract** + +Continue exporting issuer, subject, display name, and derived admin status. Add a new trusted +comma-separated `THT_PRINCIPAL_PERMISSIONS` value containing only catalog values, and clear it in +`clearPrincipalEnvironment()`. + +- [ ] **Step 5: Run all backend authorization and type gates** + +Run: + +```bash +cd backend +npx vitest run test/auth.test.ts test/authorization.test.ts test/routes-pi-management.test.ts \ + test/routes-settings.test.ts test/routes-sessions.test.ts test/routes-workspaces.test.ts \ + test/tht-runner.test.ts +npx tsc --noEmit -p . +``` + +- [ ] **Step 6: Commit the permission boundary** + +```bash +git add backend/src/auth backend/src/app.ts backend/src/routes backend/src/tht/tht-runner.ts backend/test +git commit -m "feat(auth): centralize ThothII permission enforcement" +``` + +### Task 4: Build the Safe Go Authentication Store and Argon2id Registry + +**Files:** +- Create: `tools/tht/internal/authconfig/types.go` +- Create: `tools/tht/internal/authconfig/password.go` +- Create: `tools/tht/internal/authconfig/store.go` +- Create: `tools/tht/internal/authconfig/users.go` +- Create: `tools/tht/internal/authconfig/password_test.go` +- Create: `tools/tht/internal/authconfig/store_test.go` +- Create: `tools/tht/internal/authconfig/users_test.go` +- Modify: `tools/tht/internal/safeio/files.go` +- Create: `tools/tht/internal/safeio/replace_unix.go` +- Create: `tools/tht/internal/safeio/replace_windows.go` +- Modify: `tools/tht/internal/safeio/files_unix_test.go` +- Modify: `tools/tht/internal/safeio/files_windows_test.go` +- Modify: `tools/tht/go.mod` +- Modify: `tools/tht/go.sum` + +**Interfaces:** +- Produces: + +```go +type Role string +const (RoleUser Role = "user"; RoleAdmin Role = "admin") + +type User struct { + ID string `yaml:"id" json:"id"` + Username string `yaml:"username" json:"username"` + DisplayName string `yaml:"displayName,omitempty" json:"displayName,omitempty"` + PasswordHash string `yaml:"passwordHash" json:"-"` + Roles []Role `yaml:"roles" json:"roles"` + Enabled bool `yaml:"enabled" json:"enabled"` + AuthRevision uint64 `yaml:"authRevision" json:"authRevision"` +} + +func HashPassword(password []byte, random io.Reader) (string, error) +func VerifyPassword(password []byte, encoded string) bool +func Load(directory string) (Config, Registry, error) +func MutateUsers(directory string, mutate func(*Registry) error) error +func ReplaceCanonicalRegular(path string, contents []byte, mode os.FileMode) error +``` + +- Consumes: `golang.org/x/crypto/argon2` 0.55.0 and existing `gofrs/flock`. + +- [ ] **Step 1: Write fixed Argon2 and unsafe-filesystem tests** + +Use one fixed salt and password to assert an exact PHC string. Cover wrong password, malformed PHC, +oversized parameters, short/long passwords, symlinked directory/file, hard link, duplicate YAML +key, unknown field, concurrent mutation, and last-admin refusal. + +The shared vector file is `backend/test/fixtures/argon2id-vectors.json` with: + +```json +[{"password":"correct horse battery staple","saltHex":"000102030405060708090a0b0c0d0e0f","memoryKiB":65536,"passes":3,"parallelism":1,"keyLength":32,"phc":"$argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4"}] +``` + +The committed literal is the cross-language oracle; both Go and Node independently derive it from +the password and salt and compare it byte-for-byte. + +- [ ] **Step 2: Run Go tests and verify missing implementation failures** + +Run: `cd tools/tht && go test ./internal/authconfig ./internal/safeio` + +- [ ] **Step 3: Implement PHC parsing and bounded Argon2id hashing** + +Accept only the PHC grammar demonstrated by the committed vector, with decimal `m`, `t`, and `p` +fields and unpadded base64 salt/digest fields. Reject memory above 256 MiB, passes above ten, +parallelism above four, salt outside 16–64 bytes, and digest outside 16–64 bytes. +Use `subtle.ConstantTimeCompare` for verification. + +- [ ] **Step 4: Implement safe atomic replacement and locked mutations** + +Write a same-directory exclusive temporary file, set `0600`, fsync file and directory, then replace +the regular single-link target. Unix uses `rename`; Windows uses +`MoveFileEx(MOVEFILE_REPLACE_EXISTING|MOVEFILE_WRITE_THROUGH)`. Never follow a symlinked path +component. Hold `/.auth.lock` for the complete read-check-write transaction. + +- [ ] **Step 5: Implement registry invariants** + +Require the spec's ASCII username grammar and use ASCII lowercase for lookup while preserving +display spelling. Reject control characters in Unicode display names. Generate UUIDv4 IDs with +`crypto/rand`. Increment `authRevision` on every security mutation and reject any result with no +enabled `admin`. + +- [ ] **Step 6: Run Go race and platform compilation gates** + +Run: + +```bash +cd tools/tht +go test -race ./internal/authconfig ./internal/safeio +GOOS=windows GOARCH=amd64 go test -c ./internal/authconfig +GOOS=windows GOARCH=amd64 go test -c ./internal/safeio +``` + +Delete only the two generated test binaries after recording successful compilation. + +- [ ] **Step 7: Commit the safe local registry** + +```bash +git add tools/tht/internal/authconfig tools/tht/internal/safeio tools/tht/go.mod tools/tht/go.sum \ + backend/test/fixtures/argon2id-vectors.json +git commit -m "feat(auth): add safe Argon2id local user registry" +``` + +### Task 5: Add `tht auth configure`, User Management, and Status + +**Files:** +- Create: `tools/tht/internal/authconfig/commands.go` +- Create: `tools/tht/internal/authconfig/commands_test.go` +- Modify: `tools/tht/internal/config/installation.go` +- Modify: `tools/tht/internal/config/installation_test.go` +- Modify: `tools/tht/internal/setup/files.go` +- Modify: `tools/tht/internal/setup/files_test.go` +- Modify: `tools/tht/cmd/tht/main.go` +- Modify: `tools/tht/cmd/tht/main_test.go` +- Modify: `tools/tht/go.mod` +- Modify: `tools/tht/go.sum` + +**Interfaces:** +- Produces all `tht auth` commands in the spec except live `check`, which Task 12 wires to the + backend diagnostic command. +- Adds to `config.Installation`: + +```go +Authentication struct { ConfigDirectory string } +func (i Installation) AuthenticationDirectory() string +func Run(ctx context.Context, installation config.Installation, args []string, stdin io.Reader, stdout, stderr io.Writer) int +``` + +- Consumes: `golang.org/x/term` 0.45.0 for echo-free TTY password reads. + +- [ ] **Step 1: Write CLI grammar and pristine-JSON tests** + +Assert root help contains exactly one `auth` subtree and still rejects the retired CLI name. Test +local configure, OIDC configure, list JSON redaction, add/set-password/enable/disable/grant/revoke, +last-admin refusal, logout-all revision increment, non-TTY password refusal, and OIDC-mode refusal +for `auth user`. + +The local non-interactive configure grammar is: + +```text +tht auth configure --mode local --public-url URL --admin-user USER \ + [--admin-display-name NAME] --password-file FILE +``` + +TTY mode may omit the admin flags and prompts for them. Password-file reads are bounded to 1025 +bytes and remove one trailing CRLF/LF only. + +- [ ] **Step 2: Run CLI tests and observe unknown-command failures** + +Run: `cd tools/tht && go test ./cmd/tht ./internal/config ./internal/setup ./internal/authconfig` + +- [ ] **Step 3: Extend the strict installation descriptor** + +Add: + +```yaml +authentication: + configDirectory: /absolute/operator-controlled/thothii-auth +``` + +Require an absolute canonical path and verify `THT_AUTH_CONFIG_ROOT` in the env file equals the +descriptor. `setup` and `auth configure` may create a missing final directory with private +permissions; `start`, `doctor`, `status`, user commands, and Compose operations require it to exist, +be private, and contain no symlinked component. Known-fields decoding must reject misspellings. + +- [ ] **Step 4: Implement command dispatch without growing `main.go` business logic** + +`main.go` parses the first-level `auth` verb and delegates to the exact `authconfig.Run` signature +above. All mutation logic stays in `internal/authconfig`. `status --json` returns mode, public URL, +user counts by role, and config revision; it never returns password hashes, secret refs' values, or +session data. + +- [ ] **Step 5: Implement protected bootstrap behavior** + +`tht auth configure --mode local` creates `auth.yaml` and `users.yaml` atomically with an initial +enabled admin. OIDC configuration writes the two exact group mappings supplied by +`--user-group` and `--admin-group`; if both names are equal, the command refuses the ambiguous +configuration. + +- [ ] **Step 6: Run CLI, race, and root-identity gates** + +Run: + +```bash +cd tools/tht +go test -race ./internal/authconfig ./internal/config ./internal/setup ./cmd/tht +go build ./cmd/tht +./tht --help +``` + +Expected: only `tht` is named; JSON outputs parse with `jq`; no password/hash appears in captured +stdout/stderr. + +- [ ] **Step 7: Commit the host operator surface** + +```bash +git add tools/tht +git commit -m "feat(auth): add local and OIDC management to tht" +``` + +### Task 6: Read Local Users and Verify Go-Generated Argon2 Hashes in Node + +**Files:** +- Create: `backend/src/auth/password.ts` +- Create: `backend/src/auth/local-registry.ts` +- Create: `backend/test/auth-password.test.ts` +- Create: `backend/test/local-registry.test.ts` +- Consume: `backend/test/fixtures/argon2id-vectors.json` + +**Interfaces:** +- Produces: + +```ts +export interface LocalUserRecord { + id: string; + username: string; + normalizedUsername: string; + displayName?: string; + passwordHash: string; + roles: readonly Role[]; + enabled: boolean; + authRevision: number; +} + +export interface LocalUserRegistry { + findByUsername(username: string): Promise; + findBySubject(id: string): Promise; + verify(user: LocalUserRecord | undefined, password: string): Promise; +} + +export function createLocalUserRegistry(usersPath: string): LocalUserRegistry; +``` + +- Consumes: Node 24 native `crypto.argon2` and the exact Go PHC format from Task 4. + +- [ ] **Step 1: Write cross-language vector and safe-file tests** + +Assert Node accepts every committed Go vector, rejects a one-byte password change, and refuses +oversized PHC parameters before allocating Argon2 memory. Registry tests cover known fields, +duplicate names/IDs, symlinks, hard links, mode wider than `0600`, file larger than 1 MiB, and a +same-size atomic replacement with changed mtime. + +- [ ] **Step 2: Run focused tests and observe missing exports** + +Run: `cd backend && npx vitest run test/auth-password.test.ts test/local-registry.test.ts` + +- [ ] **Step 3: Implement Node PHC verification and dummy verification** + +Parse the same bounded PHC parameters as Go, derive exactly 32 bytes, and compare with +`timingSafeEqual`. Construct one process-local dummy hash at startup so unknown and disabled users +perform an indistinguishable Argon2 verification path. + +- [ ] **Step 4: Implement safe registry reload** + +Read only the configured `users.yaml`, reject unsafe metadata before/after read, parse strictly, +and cache by inode/size/mtime. Reload after an atomic host replacement. Error messages expose only +`local_user_registry_invalid`, not usernames, hashes, paths, or YAML content. + +- [ ] **Step 5: Run focused tests, typecheck, and Go/Node round trip** + +Run: + +```bash +cd backend +npx vitest run test/auth-password.test.ts test/local-registry.test.ts +npx tsc --noEmit -p . +cd ../tools/tht && go test ./internal/authconfig +``` + +- [ ] **Step 6: Commit the backend local identity reader** + +```bash +git add backend/src/auth/password.ts backend/src/auth/local-registry.ts \ + backend/test/auth-password.test.ts backend/test/local-registry.test.ts +git commit -m "feat(auth): verify local ThothII users in the backend" +``` + +### Task 7: Implement Durable Opaque Sessions and Revision Invalidation + +**Files:** +- Create: `backend/src/auth/session-store.ts` +- Create: `backend/test/auth-session-store.test.ts` +- Modify: `backend/src/auth/types.ts` + +**Interfaces:** +- Produces: + +```ts +export interface SessionCreateInput { + principal: PrincipalContext; + method: "local" | "oidc" | "upstream"; + remembered: boolean; + userAuthRevision?: number; + authConfigRevision: string; + idleTtlMs: number; + absoluteTtlMs: number; +} + +export interface CreatedAuthSession { + token: string; + csrfToken: string; + record: AuthSessionRecord; +} + +export interface AuthSessionStore { + create(input: SessionCreateInput, now?: Date): Promise; + resolve(token: string, now?: Date): Promise; + touch(token: string, now?: Date): Promise; + revoke(token: string): Promise; + prune(now?: Date): Promise; +} + +export function createFileAuthSessionStore(root: string): AuthSessionStore; +export function deriveCsrfToken(sessionToken: string): string; +``` + +- Produces: matching bounded OIDC-state create/consume methods with ten-minute expiry and + single-use semantics. + +- [ ] **Step 1: Write lifecycle, restart, and attack-path tests** + +Use two store instances against the same temporary directory to prove a remembered session +survives a backend restart. Cover 256-bit token entropy/format, digest-only filenames, no raw token +in file content, idle and absolute expiry, five-minute touch throttling, logout deletion, prune, +single-use OIDC state, symlink/hard-link refusal, malformed/oversized records, and concurrent +resolve/revoke. + +- [ ] **Step 2: Run the focused test and observe missing store failures** + +Run: `cd backend && npx vitest run test/auth-session-store.test.ts` + +- [ ] **Step 3: Implement one-file-per-session storage** + +Generate tokens with `randomBytes(32).toString("base64url")`; derive filenames with SHA-256 and +derive the frontend CSRF token with HKDF-SHA-256 using context `thothii-csrf-v1`. Persist neither +raw value. Create root/subdirectories as `0700` and files as `0600`. Use exclusive create for new +records and same-directory write/fsync/rename for touches. Validate filename and record schema +before use. + +- [ ] **Step 4: Implement session validity hooks** + +Add a resolver callback that compares `authConfigRevision` on every request and, for local +sessions, looks up `subject`, `enabled`, and `authRevision`. Revoke on any mismatch before returning +a principal. + +- [ ] **Step 5: Run focused tests and a restart simulation** + +Run: + +```bash +cd backend +npx vitest run test/auth-session-store.test.ts --repeat 3 +npx tsc --noEmit -p . +``` + +Expected: repeated runs pass without timing flakes; the second store instance resolves the first +instance's token. + +- [ ] **Step 6: Commit durable sessions** + +```bash +git add backend/src/auth/types.ts backend/src/auth/session-store.ts \ + backend/test/auth-session-store.test.ts +git commit -m "feat(auth): persist opaque remembered sessions" +``` + +### Task 8: Add Local Login, Cookies, Logout, Rate Limits, and CSRF + +**Files:** +- Create: `backend/src/auth/csrf.ts` +- Create: `backend/src/auth/routes.ts` +- Create: `backend/test/auth-csrf.test.ts` +- Create: `backend/test/auth-routes-local.test.ts` +- Modify: `backend/src/auth/auth.ts` +- Modify: `backend/src/app.ts` +- Modify: `backend/src/server.ts` +- Modify: `backend/test/auth.test.ts` +- Modify: every state-changing route test that now needs a CSRF header + +**Interfaces:** +- Produces public routes `GET /auth/config`, `POST /auth/local/login`, OIDC route placeholders, + authenticated `POST /auth/logout`, and authenticated `GET /me`. +- Produces: + +```ts +export function registerAuthRoutes(app: FastifyInstance, deps: AuthRouteDependencies): void; +export function authenticateSession(deps: AuthDependencies): preHandlerHookHandler; +export function requireCsrf(request: FastifyRequest, reply: FastifyReply): true | FastifyReply; +``` + +- Consumes: Task 6 local registry and Task 7 session store. + +- [ ] **Step 1: Write end-to-end Fastify injection tests before route code** + +Cover successful ordinary and remembered login, generic failure for unknown/disabled/wrong +password, missing/wrong Origin, cookie attributes under HTTP and HTTPS public URLs, `/me` response, +logout, restart persistence, rate-limit response, concurrent Argon2 cap, and CSRF rejection for +every non-GET API family. + +The remembered cookie must contain `Max-Age=2592000`; the ordinary cookie must not contain +`Max-Age`. Both contain `HttpOnly`, `SameSite=Lax`, and `Path=/`. + +- [ ] **Step 2: Run tests and observe missing route/plugin failures** + +Run: `cd backend && npx vitest run test/auth-csrf.test.ts test/auth-routes-local.test.ts` + +- [ ] **Step 3: Register cookie and bounded login protection** + +Register `@fastify/cookie` and `@fastify/rate-limit` before auth routes. Set conservative default +limits of ten failed login attempts per normalized username and twenty per source address per ten +minutes. Bound Argon2 verification to two concurrent jobs; excess requests return 429 without +queuing unbounded work. + +- [ ] **Step 4: Implement session authentication and CSRF once at the app boundary** + +Replace the current global `authPreHandler` with a hook that explicitly allows `/health` and +protocol endpoints, then resolves the opaque cookie for protected routes. For state-changing +methods compare `X-ThothII-CSRF` in constant time with `deriveCsrfToken(cookieToken)`, require +`Origin` to equal the configured `publicUrl` origin, and require `Sec-Fetch-Site: same-origin` when +that header exists. Keep test helpers that generate a session+CSRF pair so individual route tests +do not bypass production hooks. + +- [ ] **Step 5: Return a frontend-safe `/me` representation** + +Return only: + +```ts +{ + issuer, subject, displayName, roles, permissions, isAdmin, + csrfToken, + session: { method, remembered, idleExpiresAt, absoluteExpiresAt } +} +``` + +Never return cookie tokens, hashes, auth revisions, config revisions, or file paths. + +- [ ] **Step 6: Run backend auth, route, and build gates** + +Run: + +```bash +cd backend +npx vitest run test/auth*.test.ts test/routes-*.test.ts test/sse-route.test.ts +npx tsc --noEmit -p . +npm run build +``` + +- [ ] **Step 7: Commit the secure local web session** + +```bash +git add backend/src/auth backend/src/app.ts backend/src/server.ts backend/test +git commit -m "feat(auth): add local login and CSRF-protected sessions" +``` + +### Task 9: Gate the React Application and Expose Remember Me + +**Files:** +- Create: `frontend/src/api/auth.ts` +- Create: `frontend/src/auth/authState.ts` +- Create: `frontend/src/auth/AuthGate.tsx` +- Create: `frontend/src/auth/LoginPage.tsx` +- Create: `frontend/src/auth/AuthGate.test.tsx` +- Create: `frontend/src/auth/LoginPage.test.tsx` +- Modify: `frontend/src/api/client.ts` +- Modify: `frontend/src/api/sessions.ts` +- Modify: `frontend/src/api/types.ts` +- Modify: `frontend/src/App.tsx` +- Modify: `frontend/src/shell/AppShell.tsx` +- Modify: `frontend/src/stream/useSessionStream.ts` +- Modify: `frontend/src/api/client.test.ts` +- Modify: relevant `frontend/src/shell/*.test.tsx` + +**Interfaces:** +- Produces: + +```ts +export interface AuthenticatedUser { + issuer: string; + subject: string; + displayName?: string; + roles: readonly ("user" | "admin")[]; + permissions: readonly string[]; + isAdmin: boolean; + csrfToken: string; + session: { method: "local" | "oidc" | "upstream"; remembered: boolean; idleExpiresAt: string; absoluteExpiresAt: string }; +} + +export function getAuthConfig(): Promise; +export function loginLocal(username: string, password: string, remember: boolean): Promise; +export function logout(): Promise; +``` + +- Consumes: same-origin `/api` and the backend CSRF contract. + +- [ ] **Step 1: Write shell-state and network-boundary tests** + +Test loading, local login form, invalid credentials, remembered checkbox, OIDC button, authenticated +shell, logout, expired-session 401, 403 presentation, and admin-only visibility. Assert neither +password nor any token is written to `localStorage`/`sessionStorage`. + +- [ ] **Step 2: Run focused frontend tests and observe missing UI failures** + +Run: + +```bash +cd frontend +npx vitest run src/auth/AuthGate.test.tsx src/auth/LoginPage.test.tsx src/api/client.test.ts +``` + +- [ ] **Step 3: Add an in-memory auth state and automatic CSRF header** + +`apiFetch` keeps default same-origin credentials and, for `POST|PUT|PATCH|DELETE`, reads the current +in-memory CSRF token and adds `X-ThothII-CSRF`. It never sends credentials to a cross-origin base +URL; extend the runtime URL policy to reject such a production configuration. + +- [ ] **Step 4: Build the authentication gate and login page** + +`AuthGate` calls `/me`; 200 renders `AppShell`, 401 renders `LoginPage`, and transient 503 renders a +retryable provider-unavailable state. The password input is uncontrolled beyond submission and is +cleared after every attempt. **Remember me** is unchecked by default and is shown only in local +mode. + +- [ ] **Step 5: Apply permission-aware chrome without relying on it for security** + +Hide Pi management and workspace mutation/test controls unless the relevant permission exists. +Hide `scope=all` unless `session.read_all` exists. Keep backend 403 handling because frontend +visibility is not authorization. + +- [ ] **Step 6: Preserve cookie-authenticated SSE** + +Keep EventSource same-origin under `/api`. On an authentication-generation change, close the old +EventSource and reset live session state before reconnecting. Do not add query-string tokens. + +- [ ] **Step 7: Run frontend unit, type, and build gates** + +```bash +cd frontend +npx vitest run +npx tsc -b +npm run build +``` + +- [ ] **Step 8: Commit the authenticated frontend** + +```bash +git add frontend/src +git commit -m "feat(auth): add remembered local login to the frontend" +``` + +### Task 10: Implement Provider-Neutral OIDC Authorization Code Flow + +**Files:** +- Create: `backend/src/auth/oidc-client.ts` +- Create: `backend/test/oidc-client.test.ts` +- Create: `backend/test/auth-routes-oidc.test.ts` +- Modify: `backend/src/auth/routes.ts` +- Modify: `backend/src/auth/session-store.ts` +- Modify: `backend/src/app.ts` + +**Interfaces:** +- Produces: + +```ts +export interface OidcIdentity { + issuer: string; + subject: string; + displayName?: string; + groups: readonly string[]; + tokenExpiresAt: Date; +} + +export interface OidcProtocol { + authorizationUrl(input: { state: string; nonce: string; codeVerifier: string }): Promise; + callback(input: { currentUrl: URL; state: string; nonce: string; codeVerifier: string }): Promise; + diagnose(signal: AbortSignal): Promise; + verifyDeviceFlow?(signal: AbortSignal, present: (uri: string, code: string) => void): Promise; +} +``` + +- Consumes: `openid-client` 6.8.5 and Task 7 single-use OIDC state storage. + +- [ ] **Step 1: Write protocol-validation and callback tests** + +Cover discovery issuer mismatch, missing HTTPS, state mismatch, replay, nonce mismatch, wrong +audience, expired token, invalid signature, missing `sub`, absent groups, non-array groups, empty +group item, distributed/overage groups, extra groups, unmapped groups, and a successful callback. + +Tests inject a deterministic `OidcProtocol`; one concrete-adapter test supplies a local in-process +discovery/JWKS/token fixture through the library's custom fetch hook and never contacts the network. + +- [ ] **Step 2: Run focused tests and observe missing OIDC adapter failures** + +Run: `cd backend && npx vitest run test/oidc-client.test.ts test/auth-routes-oidc.test.ts` + +- [ ] **Step 3: Implement discovery and Authorization Code + PKCE** + +Use PKCE S256, random state, random nonce, exact configured callback, and exact issuer validation. +Store verifier/nonce/return target under the digest of state for ten minutes and consume it once. +Allow only the fixed return target `/`; do not accept arbitrary `returnTo` URLs. + +- [ ] **Step 4: Enforce the mandatory groups contract and map roles** + +Read `groupsClaim` from the verified ID-token claims. Require a direct array of unique non-empty +strings. Map exact strings through `authorization.groupRoles`, union roles, and ignore all other +strings without logging. A valid identity with no mapped role reaches the authenticated-but- +forbidden state. + +- [ ] **Step 5: Create an OIDC session without persisting tokens** + +Set absolute expiry to `min(now + oidcTtlSeconds, ID-token exp)`. Persist only the derived principal +and authorization/config revision. Clear the state record after both successful and terminal +failed callbacks. + +- [ ] **Step 6: Run OIDC, auth-route, type, and build gates** + +```bash +cd backend +npx vitest run test/oidc-client.test.ts test/auth-routes-oidc.test.ts test/auth-session-store.test.ts +npx tsc --noEmit -p . +npm run build +``` + +- [ ] **Step 7: Commit generic OIDC login** + +```bash +git add backend/src/auth backend/test/oidc-client.test.ts backend/test/auth-routes-oidc.test.ts +git commit -m "feat(auth): add generic OIDC login with mandatory groups" +``` + +### Task 11: Add the Authentik Group Catalog and Shared Auth Diagnostics + +**Files:** +- Create: `backend/src/auth/group-catalog.ts` +- Create: `backend/src/auth/authentik-group-catalog.ts` +- Create: `backend/src/auth/diagnostics.ts` +- Create: `backend/test/authentik-group-catalog.test.ts` +- Create: `backend/test/auth-diagnostics.test.ts` +- Modify: `backend/src/config/secret-bundle.ts` +- Modify: `backend/test/secret-bundle.test.ts` + +**Interfaces:** +- Produces: + +```ts +export interface GroupCatalog { + verifyConfiguredGroups(names: readonly string[], signal: AbortSignal): Promise; +} + +export interface AuthDiagnoser { + inspect(options: { live: boolean; interactive?: boolean; signal?: AbortSignal }): Promise; +} + +export function createAuthDiagnoser(deps: AuthDiagnoserDependencies): AuthDiagnoser; +``` + +- Consumes fixed secret references `THT_OIDC_CLIENT_SECRET` and `THT_AUTHENTIK_API_TOKEN` from the + existing literal secret bundle parser. + +- [ ] **Step 1: Write exact Authentik request and comparison tests** + +Assert each mapped group produces one request with `include_users=false&page_size=2`, bearer auth, +five-second abort, `redirect: "error"`, and encoded exact name. Test 0/1/2 results, pagination/body +overflow, 401/403, invalid JSON, redirect, timeout, and secret redaction. + +Add the defining negative assertion: + +```ts +expect(report.checks).not.toContainEqual(expect.objectContaining({ level: "warning" })); +expect(JSON.stringify(report)).not.toContain("Unmapped Corporate Group"); +``` + +- [ ] **Step 2: Run focused tests and observe missing adapter failures** + +Run: `cd backend && npx vitest run test/authentik-group-catalog.test.ts test/auth-diagnostics.test.ts` + +- [ ] **Step 3: Implement the least-privilege Authentik adapter** + +Use native `fetch`, a fixed operator-configured base origin, no redirects, bounded 1 MiB response, +and an AbortSignal. Return only stable codes and configured group names; discard upstream response +bodies and never enumerate unrelated groups. + +- [ ] **Step 4: Implement static and live diagnosis** + +Static mode validates configuration, file safety, secret presence, local enabled admin, role names, +group map, URL policy, and session root. Live OIDC mode additionally validates discovery/JWKS and +all Authentik mapped groups. Codes are exactly the closed `AuthDiagnosticCode` union in the spec; +`auth_ready` is the sole success code. + +- [ ] **Step 5: Verify redaction under every upstream failure** + +Seed tests with unique client-secret, API-token, cookie, password-hash, and file-path sentinels. +Assert none appears in thrown errors, Fastify logs, human diagnostics, or JSON diagnostics. + +- [ ] **Step 6: Run focused and full auth gates** + +```bash +cd backend +npx vitest run test/auth*.test.ts test/oidc-client.test.ts test/secret-bundle.test.ts +npx tsc --noEmit -p . +``` + +- [ ] **Step 7: Commit Authentik certification logic** + +```bash +git add backend/src/auth backend/src/config/secret-bundle.ts backend/test +git commit -m "feat(auth): validate mapped groups through Authentik" +``` + +### Task 12: Integrate Auth Diagnostics with Workspaces, `tht doctor`, and `tht auth check` + +**Files:** +- Create: `backend/src/auth/diagnostic-command.ts` +- Create: `backend/test/auth-diagnostic-command.test.ts` +- Modify: `backend/src/routes/workspaces.ts` +- Modify: `backend/test/routes-workspaces.test.ts` +- Modify: `backend/src/workspaces/diagnostics.ts` +- Modify: `frontend/src/api/workspaces.ts` +- Modify: `frontend/src/api/workspaces.test.ts` +- Modify: `frontend/src/shell/WorkspaceManager.tsx` +- Modify: `frontend/src/shell/WorkspaceManager.test.tsx` +- Modify: `tools/tht/internal/authconfig/commands.go` +- Modify: `tools/tht/internal/authconfig/commands_test.go` +- Modify: `tools/tht/internal/doctor/report.go` +- Modify: `tools/tht/internal/doctor/report_test.go` +- Modify: `tools/tht/cmd/tht/main_test.go` + +**Interfaces:** +- Extends workspace results with: + +```ts +interface WorkspaceDiagnostics { + activatable: boolean; + diagnostics: Diagnostic[]; + authentication: AuthDiagnostics; +} +``` + +- Produces `tht auth check [--json|--interactive]` and a doctor check named `authentication`. + +- [ ] **Step 1: Write workspace aggregation and CLI delegation tests** + +Assert `/workspaces/validate` uses `inspect({live:false})`, `/workspaces/:id/test` uses +`inspect({live:true})`, and `activatable` is false on auth failure even when workspace connectors +pass. Assert configured missing groups appear as errors and unrelated provider groups never appear. + +In Go, assert exact Compose invocation and redaction for both stopped/running stacks. JSON stdout +must decode as the backend `AuthDiagnostics` contract with no banner. + +- [ ] **Step 2: Run focused backend/frontend/Go tests and observe failures** + +Run: + +```bash +cd backend && npx vitest run test/routes-workspaces.test.ts test/auth-diagnostic-command.test.ts +cd ../frontend && npx vitest run src/api/workspaces.test.ts src/shell/WorkspaceManager.test.tsx +cd ../tools/tht && go test ./internal/authconfig ./internal/doctor ./cmd/tht +``` + +- [ ] **Step 3: Add one backend diagnostic command** + +`node dist/auth/diagnostic-command.js --json` prints only the redacted report. `--interactive` +requires OIDC mode and the discovery document's device authorization endpoint, prints the +verification URI and user code to stderr, validates the resulting ID token and groups, and prints +one final success/failure report. No token is persisted. + +- [ ] **Step 4: Delegate host checks through Compose** + +`tht auth check` invokes a one-shot core command with the installation's normal mounts and secrets; +`tht doctor` uses `exec -T` when core is healthy. Add `authentication` after `configuration` and +before remote workspace/Pi checks. Sanitize both process output streams with all installation +secret values. + +- [ ] **Step 5: Aggregate workspace diagnostics without duplicating auth logic** + +Inject `AuthDiagnoser` into workspace routes. Static draft validation includes the static report; +installation test includes the live report. Preserve all existing connector diagnostic ordering +and messages, and compute overall `activatable` from both components. + +- [ ] **Step 6: Render authentication results in Workspace Manager** + +Show one Authentication section with Passed/Failed and configured-group errors. Do not render or +calculate a list of unmapped groups. Restrict Validate/Test actions to `workspace.manage`. + +- [ ] **Step 7: Run all diagnostic gates** + +```bash +cd backend && npx vitest run test/routes-workspaces.test.ts test/auth-diagnostic-command.test.ts +cd ../frontend && npx vitest run src/api/workspaces.test.ts src/shell/WorkspaceManager.test.tsx +cd ../tools/tht && go test -race ./internal/authconfig ./internal/doctor ./cmd/tht +``` + +- [ ] **Step 8: Commit unified diagnostics** + +```bash +git add backend/src backend/test frontend/src tools/tht +git commit -m "feat(auth): include authentication in workspace and tht diagnostics" +``` + +### Task 13: Wire Authentication into Compose, Setup, Backup, and Restore + +**Files:** +- Modify: `compose.yaml` +- Modify: `deploy/compose.local.yaml` +- Modify: `deploy/compose.server.yaml` +- Modify: `deploy/compose.session-server.yaml.example` +- Modify: `deploy/env/local.env.example` +- Modify: `deploy/env/server.env.example` +- Modify: `deploy/psd/operator.env.example` +- Modify: `deploy/psd/thothii-installation.yaml.example` +- Modify: `docs/install/examples/thothii-installation.local.yaml` +- Modify: `docs/install/examples/thothii-installation.server.yaml` +- Modify: `deploy/secrets/thothii.secrets.example` +- Modify: `deploy/secrets/README.md` +- Modify: `tools/tht/internal/setup/files.go` +- Modify: `tools/tht/internal/setup/files_test.go` +- Modify: `tools/tht/internal/backup/create.go` +- Modify: `tools/tht/internal/backup/create_test.go` +- Modify: `tools/tht/internal/backup/restore.go` +- Modify: `tools/tht/internal/backup/restore_test.go` +- Modify: `tools/tht/internal/doctor/report.go` +- Modify: `tools/tht/internal/doctor/report_test.go` +- Modify: `scripts/unified-deployment-smoke.sh` + +**Interfaces:** +- Produces mount `${THT_AUTH_CONFIG_ROOT}:/run/thothii-auth:ro` and volume + `auth-state:/data/auth`. +- Produces core env `THT_AUTH_CONFIG_FILE=/run/thothii-auth/auth.yaml` and + `THT_AUTH_STATE_ROOT=/data/auth`. +- Preserves server whole-`/data` bind semantics. + +- [ ] **Step 1: Write Compose render and setup tests before YAML changes** + +Assert both profiles render the auth config directory read-only, core alone can access auth state, +the local profile declares `auth-state`, server uses `${THT_DATA_ROOT}/auth` through its `/data` +bind, and workspace-maintenance receives neither user files nor auth state. + +Update doctor volume expectations from seven to eight local persistent volumes. + +- [ ] **Step 2: Run render/setup/doctor tests and observe missing bindings** + +Run: + +```bash +cd tools/tht && go test ./internal/setup ./internal/doctor ./internal/backup +``` + +Expected: tests fail until auth paths and the volume are declared. Compose rendering remains inside +the existing fixture-safe setup/doctor tests so it never depends on an operator's uncommitted env +or secret files. + +- [ ] **Step 3: Add mounts, volume, environment, and examples** + +Remove `AUTH_MODE` from the normal local and server profiles so `auth.yaml` is authoritative, and +keep an explicitly commented deprecated upstream migration example that is valid only when no +auth config exists. Add the two fixed secret-bundle keys: + +```text +THT_OIDC_CLIENT_SECRET= +THT_AUTHENTIK_API_TOKEN= +``` + +Never place example real-looking values in committed files. + +- [ ] **Step 4: Make setup configure authentication before startup** + +The ordered setup workflow becomes: create/validate installation files, configure local/OIDC auth, +validate auth statically, render Compose, build/start, then run aggregate doctor. Non-interactive +setup requires complete auth flags and password-file input for local mode. + +- [ ] **Step 5: Define backup and restore custody** + +Without `--include-secrets`, backup records the auth configuration path but excludes `users.yaml` +and secret values. With `--include-secrets --yes`, include `auth.yaml` and `users.yaml` under the +encrypted/custody-warning secret section. Never include active session or OIDC-state files. +Restore recreates `/data/auth` with private ownership and no active sessions, so every browser must +authenticate again. + +- [ ] **Step 6: Extend deployment smoke assertions** + +Add local bootstrap/login/remember/restart/logout checks and server static OIDC diagnostics with a +fake provider fixture. Assert workspace-maintenance cannot read `/run/thothii-auth` or `/data/auth`. +Assert final cleanup removes only test-scoped containers/volumes. + +- [ ] **Step 7: Run setup, backup, Compose, and smoke gates** + +```bash +cd tools/tht && go test -race ./internal/setup ./internal/config ./internal/backup ./internal/doctor ./cmd/tht +cd ../.. +bash scripts/unified-deployment-smoke.sh +``` + +- [ ] **Step 8: Commit deployment integration** + +```bash +git add compose.yaml deploy tools/tht scripts/unified-deployment-smoke.sh docs/install/examples +git commit -m "feat(auth): integrate authentication with installation lifecycle" +``` + +### Task 14: Document Local Auth, Generic OIDC, Authentik, Groups, and PSD Acceptance + +**Files:** +- Create: `docs/architecture/authentication.md` +- Create: `docs/install/authentication-local.md` +- Create: `docs/install/authentication-oidc.md` +- Create: `docs/install/authentik.md` +- Create: `docs/testing/authentication-manual-acceptance.md` +- Modify: `docs/architecture/overview.md` +- Modify: `docs/install/local.md` +- Modify: `docs/install/server.md` +- Modify: `docs/install/psd-workspace-setup.md` +- Modify: `docs/install/reverse-proxy-caddy.md` +- Modify: `docs/install/reverse-proxy-nginx.md` +- Modify: `docs/guida-utente.md` +- Modify: `docs/index.md` +- Modify: `README.md` +- Modify: `PROJECT_STATE.md` only after automated and manual status is known +- Modify: `mkdocs.yml` if navigation is explicit there + +**Interfaces:** +- Documents the exact YAML, CLI, group claim, group-role mapping, session lifetime, invalidation, + diagnostic codes, Authentik service-account privileges, and PSD acceptance flow from the spec. + +- [ ] **Step 1: Add a documentation contract test** + +Extend the existing docs smoke or add `scripts/auth-docs-smoke.sh` to require: + +```text +tht auth +groups +TOT Admin +THT_OIDC_CLIENT_SECRET +THT_AUTHENTIK_API_TOKEN +Remember me +oidc_mapped_group_missing +``` + +Also fail on `thothii-admin`, host-facing `thothctl auth`, plaintext-password examples, or wording +that claims unmapped OIDC groups generate warnings. + +- [ ] **Step 2: Run the docs smoke and observe missing-document failures** + +Run: `bash scripts/auth-docs-smoke.sh` + +- [ ] **Step 3: Write local and generic OIDC guides** + +Document bootstrap, initial admin, password recovery, remembered/ordinary expiry, logout-all, +restart behavior, all commands, JSON use, same-origin browser requirement, callback URL, mandatory +direct `groups` array, fail-closed unmapped-user behavior, and provider-adapter boundary. + +- [ ] **Step 4: Write the Authentik and PSD guide** + +Include exact operator steps: create OAuth2/OIDC application/provider, register callback, include +`openid profile email`, verify the `groups` claim, create dedicated service account/API token with +group-view permission only, create/confirm `TOT Users` and `TOT Admin`, map them in `auth.yaml`, run +`tht auth check`, run `tht auth check --interactive`, then run workspace Test. State explicitly +that additional Authentik/LDAP groups are ignored silently. + +- [ ] **Step 5: Write the manual acceptance matrix** + +Require one ordinary and one admin PSD test identity. Record expected results for ordinary/admin +route access, missing claim, missing mapped group, wrong API token, group rename, extra unmapped +group, session restart, password/role invalidation, CSRF rejection, logout, and provider outage. +Never record real names, tokens, passwords, LDAP details, or internal URLs in committed evidence. + +- [ ] **Step 6: Run docs build and smoke** + +```bash +bash scripts/auth-docs-smoke.sh +python -m mkdocs build --strict +``` + +- [ ] **Step 7: Commit operator and user documentation** + +```bash +git add docs README.md mkdocs.yml scripts/auth-docs-smoke.sh +git commit -m "docs(auth): document local OIDC and Authentik operation" +``` + +### Task 15: Execute Full Automated and Manual Release Gates + +**Files:** +- Create: `backend/test/fixtures/oidc-provider.mjs` +- Create: `scripts/authentication-smoke.sh` +- Create: `frontend/e2e/auth.spec.ts` +- Modify: `.github/workflows/deployment.yml` +- Modify: `docs/testing/authentication-manual-acceptance.md` +- Modify: `PROJECT_STATE.md` after evidence is retained + +**Interfaces:** +- Produces one retained test report containing commit SHA, image IDs, Node/Pi versions, individual + gate status, and no secret values. +- Consumes every interface and acceptance condition from Tasks 1–14. + +- [ ] **Step 1: Add a deterministic fake OIDC provider and browser E2E** + +The fixture exposes discovery, JWKS, authorization, token, device authorization, and Authentik-like +group-list endpoints on loopback only. It issues signed short-lived ID tokens for ordinary, admin, +missing-groups, malformed-groups, and unmapped-group identities. + +Playwright covers local ordinary/remembered login, backend restart, logout, admin chrome, OIDC +redirect/callback, 403 for an unmapped user, and expired session recovery. + +- [ ] **Step 2: Run every language-level gate** + +```bash +cd tools/tht && go test -race ./... && go build ./cmd/tht +cd ../../backend && npx tsc --noEmit -p . && npx vitest run && npm run build +cd ../frontend && npx tsc -b && npx vitest run && npm run build && npm run e2e +cd ../harness && .venv/bin/ruff check . && .venv/bin/pytest -q +``` + +- [ ] **Step 3: Run Docker, installation, and security smoke gates** + +```bash +cd .. +bash scripts/authentication-smoke.sh +bash scripts/unified-deployment-smoke.sh +bash scripts/auth-docs-smoke.sh +``` + +Verify the built core reports Node `v24.16.0`, Pi starts, auth state survives only the intended +restart, and no sentinel secret appears in logs or artifacts. + +- [ ] **Step 4: Run the opt-in L2 live-session smoke** + +Use the repository's configured PSD/L2 secret layout without copying it into the worktree: + +```bash +cd harness +.venv/bin/pytest -q -m l2 +``` + +Then create one browser session through the real stack and complete a live session workflow as an +ordinary authenticated user. Confirm principal ownership remains stable after resume. + +- [ ] **Step 5: Execute Authentik PSD manual acceptance** + +Follow `docs/testing/authentication-manual-acceptance.md`. Run `tht auth check --interactive`, +workspace Validate/Test, ordinary/admin authorization checks, extra-group no-warning check, and +mapped-group rename failure. Store only sanitized pass/fail evidence under a task-scoped +`.artifacts/manual-acceptance/authentication/` directory. + +- [ ] **Step 6: Update project state with actual evidence** + +Record exact pass counts, retained artifact digest, source commit, Node version, Authentik version, +and whether manual PSD acceptance is PASS or PENDING. Do not mark the feature complete while any +required gate is pending. + +- [ ] **Step 7: Commit final gates and state** + +```bash +git add backend/test/fixtures/oidc-provider.mjs frontend/e2e/auth.spec.ts \ + scripts/authentication-smoke.sh .github/workflows/deployment.yml \ + docs/testing/authentication-manual-acceptance.md PROJECT_STATE.md +git commit -m "test(auth): gate local and Authentik authentication release" +``` + +## Final acceptance checklist + +- [ ] `tht --help` exposes one CLI and the complete `auth` subtree. +- [ ] A fresh local setup creates one admin without plaintext credentials. +- [ ] Ordinary local login and **Remember me** behave with the exact configured lifetimes. +- [ ] A remembered session survives browser and core restart. +- [ ] Password change, role change, disable, logout-all, logout, config change, and restore revoke + the expected sessions. +- [ ] Every state-changing cookie-authenticated route rejects missing/invalid CSRF. +- [ ] Ordinary and admin permission matrices pass in backend and browser tests. +- [ ] OIDC Authorization Code + PKCE validates issuer, signature, audience, expiry, state, nonce, + and mandatory direct `groups`. +- [ ] Authentik group validation fails for configured missing/ambiguous groups. +- [ ] Unmapped Authentik/token groups produce no error, warning, or log entry. +- [ ] Workspace static validation and live Test include authentication and combine `activatable`. +- [ ] `tht auth check`, interactive device check, and aggregate `tht doctor` are redacted and + machine-readable. +- [ ] Frontend stores no token/session secret in Web Storage and SSE uses only the session cookie. +- [ ] Node 24.16, Pi, backend, frontend, harness, Compose, backup/restore, L2, and PSD gates pass. +- [ ] Documentation explains the mandatory `groups` claim and exact group-to-role mapping. diff --git a/docs/superpowers/plans/2026-08-18-thothii-authentication-remediation.md b/docs/superpowers/plans/2026-08-18-thothii-authentication-remediation.md new file mode 100644 index 00000000..347f2ec3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-thothii-authentication-remediation.md @@ -0,0 +1,911 @@ +# ThothII Authentication Important-Finding Remediation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the three remaining Important authentication findings on `feat/thoth-auth`: POSIX local-registry ownership, retained-capability restore staging, and handle-relative Windows claim removal. + +**Architecture:** POSIX registry reads bind every path and descriptor metadata observation to one validated effective UID. Restore staging creates one random private regular file directly below a retained `safeio.PrivateDirectoryHandle`, keeps that directory capability for the entire stream and cleanup lifecycle, and never authorizes cleanup through a pathname. Windows canonical claim removal reuses the already handle-relative `PrivateDirectoryHandle.RemoveClaim` implementation instead of reopening and deleting absolute paths. + +**Tech Stack:** Node.js `24.16.0`, TypeScript, Fastify auth services, Vitest, Go `1.26.5`, `golang.org/x/sys`, native Windows GitHub Actions, Docker Compose release smokes. + +**Spec:** `docs/superpowers/specs/2026-08-16-thothii-authentication-design.md` + +**Review basis:** `.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md` and the Task 15 section of `PROJECT_STATE.md` at finding baseline `178113a`. + +## Global Constraints + +- The only host CLI is `tht`; do not add another executable or revive `thothctl`. +- Production modes are `local` and `oidc`; `none` and `mock` are development/test only, while `upstream` remains a deprecated migration adapter. +- Authentik is the first-release certified OIDC provider; the browser OIDC protocol layer must not contain Authentik-specific login logic. +- The ID token must contain direct claim `groups: string[]`; absent, malformed, indirect, or overage claims fail closed. +- Only configured groups are checked and mapped. Unmapped provider groups are ignored silently and never produce a warning. +- Every configured group must be proven to exist through the configured group-catalog adapter; first release provides `authentik`. +- Local passwords use Argon2id v19 with `m=65536,t=3,p=1`, 16-byte random salt, and 32-byte output. +- A remembered local session has a seven-day idle timeout and thirty-day absolute timeout and survives browser/backend restarts. +- No raw session cookie, password, CSRF token, OIDC token, client secret, Authentik API token, or password hash may enter logs or diagnostic output. +- Browser authentication uses an opaque `HttpOnly`, `SameSite=Lax`, path-scoped cookie; `Secure` is conditional on an HTTPS public URL so loopback HTTP remains functional. +- Every cookie-authenticated state-changing route requires a CSRF token and same-origin browser checks. +- Authentication configuration is installation-global, but static checks appear in workspace validation and live checks appear in workspace connection tests. +- Workspace document content remains in its workspace language; application chrome and new authentication UI strings are English. +- JSON CLI stdout is pristine. Prompts, progress, and human guidance go to stderr. +- Node is exactly `24.16.0`; Docker uses `sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7`. +- Existing session artifacts, workflow persistence, workspace ownership, and Pi RPC behavior must not change. +- Preserve unrelated user changes and the existing untracked `.playwright-cli/` and `.thothctl/` paths. + +--- + +## Execution Contract + +Run the tasks in numerical order. Tasks 2 and 3 both touch `safeio`; they must not be parallelized. + +| Task | Implementer | Reasoning | Mandatory reviewer | +|---|---|---:|---| +| 1 | fresh Terra | high | fresh Terra, ultra | +| 2 | fresh Terra | ultra | fresh Terra, ultra | +| 3 | fresh Terra | ultra | fresh Terra, ultra | +| 4 | fresh Terra | max | fresh Terra, ultra, whole-branch scope | + +For every task: + +1. Dispatch a new Terra implementer that has not worked on any earlier authentication task. +2. Give it only this plan, the design spec, `global-constraints.md`, the Task 15 report, and the current task's relevant files. +3. Require RED-GREEN TDD, focused tests before broad tests, and one task-scoped commit. +4. Dispatch a different, fresh Terra reviewer after the commit. The reviewer is read-only and checks the task diff, tests, security invariants, and scope. +5. Do not start the next numbered task unless the reviewer reports `CLEAN` for Critical/Important issues. A requested fix gets a new Terra implementer and, after its commit, another fresh Terra reviewer. +6. Three review/fix cycles are the hard breaker for one task. If the third review is not `CLEAN`, stop and record the exact blocker; do not weaken an invariant or recast the finding as accepted risk. + +The review of Task 4 is also the final whole-branch review over `178113a..HEAD`. A native-Windows test that was only cross-compiled is `PENDING`, never `PASS`. + +## Scope Boundaries and Design Choices + +- Do not change login, role, session, CSRF, OIDC, group-mapping, or CLI behavior except where a test proves an unintended dependency on one of the three fixes. +- Do not introduce a second filesystem abstraction. Extend `safeio.PrivateDirectoryHandle` with one streaming-file method and reuse its existing handle-relative `RemoveRegular` and `RemoveClaim` operations. +- Do not create a temporary staging subdirectory. A unique archive file directly under the retained `restore-staging` directory removes an unnecessary cleanup capability and still allows candidate and recovery archives to coexist. +- Do not test POSIX ownership by requiring root or calling `chown`. Vitest must override returned metadata while the production code continues to use the real `node:fs` API. +- Do not treat the known Ruff, MkDocs, installation-wording, Pi model-policy, deployment-coupling, L2, PSD, provider-readiness, or Windows-Docker statuses as fixed by this plan. Re-run and report them honestly where the release matrix requires them. +- Do not put credentials, internal endpoints, provider identities, registry names, raw paths containing secrets, or unsanitized Docker output into retained evidence. + +## Target File Structure + +### Backend ownership boundary + +- `backend/src/auth/local-registry.ts` — validates one effective UID across every POSIX `lstat` and `fstat` observation. +- `backend/test/local-registry.test.ts` — injects foreign UID metadata at path-file, descriptor-file, path-directory, and descriptor-directory boundaries. + +### Retained restore staging + +- `tools/tht/internal/safeio/private_root.go` — adds the cross-platform streaming regular-file interface. +- `tools/tht/internal/safeio/private_root_unix.go` — creates an exclusive owner-private stream with `openat` under the retained directory descriptor. +- `tools/tht/internal/safeio/private_root_windows.go` — creates an exclusive owner-private stream with NT `RootDirectory`-relative access. +- `tools/tht/internal/safeio/files_test.go` — proves streaming creation and handle-relative removal through the public directory capability. +- `tools/tht/internal/backup/preflight.go` — owns the retained staging directory capability until `stagedArchive.Close` finishes. +- `tools/tht/internal/backup/preflight_test.go` — updates lifecycle, failure-cleanup, and same-filesystem assertions for direct-child staging. +- `tools/tht/internal/backup/preflight_unix_test.go` — proves cleanup targets the pinned Unix directory after a lexical ancestor swap. +- `tools/tht/internal/backup/preflight_windows_test.go` — proves native Windows blocks the swap while the no-delete directory handle is retained. +- `.github/workflows/deployment.yml` — executes the security tests natively in the existing `windows-clone` job. + +### Windows claim removal + +- `tools/tht/internal/safeio/claim_windows.go` — delegates canonical removal to a retained `PrivateDirectoryHandle`. +- `tools/tht/internal/safeio/claim_windows_test.go` — proves native Windows ancestor replacement cannot redirect deletion. + +### Certification evidence + +- `.artifacts/task-15/automated-gates.json` — sanitized machine-readable results bound to the final source commit. +- `.artifacts/task-15/unified-docker-images.json` — exact final-smoke image identities without registry names. +- `.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md` — remediation and release-gate narrative. +- `PROJECT_STATE.md` — current status, exact source SHA, review verdict, PASS/FAIL/PENDING matrix. + +--- + +### Task 1: Require Effective-UID Ownership for the POSIX Local Registry + +**Files:** +- Modify: `backend/src/auth/local-registry.ts:53-193,229-250` +- Modify: `backend/test/local-registry.test.ts:1-190` + +**Interfaces:** +- Produces: `runtimeOwner(): number`, which returns a non-negative safe integer from `process.geteuid()` or throws `local_user_registry_invalid`. +- Produces: `fileMetadata(info: Stats, owner: number): FileIdentity` and `directoryMetadata(info: Stats, owner: number): DirectoryIdentity`. +- Produces: `registryIdentity(path: string, owner: number)`, `readBounded(path: string, owner: number)`, and `load(path: string, owner: number)`. +- Preserves: `createLocalUserRegistry(usersPath, options)` and every public `LocalUserRegistry` method. + +- [ ] **Step 1: Add a controllable metadata wrapper to the existing Vitest file** + +Place a hoisted state object before the imports from `local-registry.ts`, mock only `node:fs` metadata calls, and leave all real filesystem mutation/open/read calls intact: + +```ts +type OwnershipObservation = + | "file-lstat" + | "file-fstat" + | "directory-lstat" + | "directory-fstat"; + +const ownershipOverride = vi.hoisted(() => ({ + observation: undefined as OwnershipObservation | undefined, + uid: undefined as number | undefined, +})); + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const replaceUid = (value: T, uid: number): T => new Proxy(value, { + get(target, property) { + if (property === "uid") return uid; + const member = Reflect.get(target, property, target); + return typeof member === "function" ? member.bind(target) : member; + }, + }); + const maybeReplace = ( + value: T, + source: "lstat" | "fstat", + ): T => { + const kind = value.isDirectory() ? "directory" : "file"; + return ownershipOverride.observation === `${kind}-${source}` && ownershipOverride.uid !== undefined + ? replaceUid(value, ownershipOverride.uid) + : value; + }; + return { + ...actual, + lstatSync(path: import("node:fs").PathLike) { + return maybeReplace(actual.lstatSync(path), "lstat"); + }, + fstatSync(fd: number) { + return maybeReplace(actual.fstatSync(fd), "fstat"); + }, + }; +}); +``` + +Reset both fields in the existing `afterEach` before deleting fixture files so one case cannot contaminate the next. + +- [ ] **Step 2: Add four foreign-owner rejection cases** + +Add a POSIX-only table test after the existing unsafe-metadata test: + +```ts +test.runIf(process.platform !== "win32").each([ + "file-lstat", + "file-fstat", + "directory-lstat", + "directory-fstat", +] as const)("rejects foreign ownership at the %s boundary", async (observation) => { + const fixture = writeRegistry(registryYaml(userYaml())); + const owner = process.geteuid(); + ownershipOverride.observation = observation; + ownershipOverride.uid = owner === 0 ? 1 : owner - 1; + + await expectInvalid( + createLocalUserRegistry(fixture.path).findByUsername("admin"), + ["admin", passwordHash, fixture.path], + ); +}); +``` + +Add the invalid-effective-UID case explicitly: + +```ts +test.runIf(process.platform !== "win32")("fails closed when the effective UID is invalid", async () => { + const fixture = writeRegistry(registryYaml(userYaml())); + const getuid = vi.spyOn(process, "geteuid").mockReturnValue(-1); + try { + await expectInvalid( + createLocalUserRegistry(fixture.path).findByUsername("admin"), + ["admin", passwordHash, fixture.path], + ); + } finally { + getuid.mockRestore(); + } +}); +``` + +Keep the existing Windows bridge test green to prove native Windows does not enter the POSIX owner path. + +- [ ] **Step 3: Run the focused tests and verify RED** + +```bash +cd backend +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx vitest run test/local-registry.test.ts -t "foreign ownership|effective UID" +``` + +Expected: all new POSIX ownership cases fail because `uid` is not checked; the Windows-only bridge case remains excluded or passing according to host platform. + +- [ ] **Step 4: Add one fail-closed effective-UID resolver** + +Use the same invariant already present in `backend/src/auth/config.ts`, but keep this patch local to the registry to avoid unrelated config refactoring: + +```ts +function runtimeOwner(): number { + if (process.platform === "win32" || typeof process.geteuid !== "function") throw invalid(); + const owner = process.geteuid(); + if (!Number.isSafeInteger(owner) || owner < 0) throw invalid(); + return owner; +} +``` + +Add `uid: number` to both `FileIdentity` and `DirectoryIdentity`; include it in `sameFileIdentity` and `sameDirectoryIdentity`. + +- [ ] **Step 5: Thread one owner through every POSIX observation** + +Change the metadata helpers so every path and descriptor stat uses the same owner captured at the start of `currentPosix()`: + +```ts +function fileMetadata(info: Stats, owner: number): FileIdentity { + if (!info.isFile() || info.uid !== owner || info.nlink !== 1 || (info.mode & 0o7777) !== 0o600) { + throw invalid(); + } + if (info.size < 0 || info.size > MAX_USERS_YAML_BYTES) throw invalid(); + return { dev: info.dev, ino: info.ino, uid: info.uid, size: info.size, mtimeMs: info.mtimeMs }; +} + +function directoryMetadata(info: Stats, owner: number): DirectoryIdentity { + if (!info.isDirectory() || info.uid !== owner || (info.mode & 0o7777) !== 0o700) throw invalid(); + return { dev: info.dev, ino: info.ino, uid: info.uid, mode: info.mode & 0o7777 }; +} +``` + +`directoryIdentity`, `registryIdentity`, `readBounded`, and `load` must all require `owner`. `currentPosix` captures it once and passes it through both cache probes and both load attempts: + +```ts +function currentPosix(): LocalUserRecord[] { + try { + const owner = runtimeOwner(); + const before = registryIdentity(usersPath, owner); + if (cached && sameIdentity(cached.identity, before)) return cached.records; + for (let attempt = 0; attempt < 2; attempt += 1) { + const loaded = load(usersPath, owner); + if (sameIdentity(loaded.identity, registryIdentity(usersPath, owner))) { + cached = loaded; + return loaded.records; + } + } + } catch { + throw invalid(); + } + throw invalid(); +} +``` + +Do not call `runtimeOwner()` inside individual metadata helpers: changing the expected owner between observations would weaken the snapshot invariant. + +- [ ] **Step 6: Run focused and complete Node 24 gates** + +```bash +cd backend +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx vitest run test/local-registry.test.ts +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx tsc --noEmit -p . +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx vitest run +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npm run build +``` + +Expected: focused and full backend suites pass; no error contains a user name, hash, or registry path. + +- [ ] **Step 7: Commit the ownership remediation** + +```bash +git add backend/src/auth/local-registry.ts backend/test/local-registry.test.ts +git commit -m "fix(auth): require local registry ownership" +``` + +**Mandatory Terra review gate:** Review the task commit against the finding. Confirm all eight POSIX metadata observations in `readBounded` plus the file-and-directory observations in both cache probes flow through owner-checking helpers, invalid/missing `geteuid` fails closed, and the Windows bridge remains unchanged. Verdict must be `CLEAN` before Task 2. + +--- + +### Task 2: Retain the Restore-Staging Capability Through Stream and Cleanup + +**Files:** +- Modify: `tools/tht/internal/safeio/private_root.go:8-23` +- Modify: `tools/tht/internal/safeio/private_root_unix.go:162-202` +- Modify: `tools/tht/internal/safeio/private_root_windows.go:512-539` +- Modify: `tools/tht/internal/safeio/files_test.go` +- Modify: `tools/tht/internal/backup/preflight.go:97-103,261-374` +- Modify: `tools/tht/internal/backup/preflight_test.go:60-101,403-488` +- Modify: `tools/tht/internal/backup/preflight_unix_test.go` +- Modify: `tools/tht/internal/backup/preflight_windows_test.go` +- Modify: `.github/workflows/deployment.yml:125-153` + +**Interfaces:** +- Produces: `PrivateDirectoryHandle.CreateRegularFile(name string) (*os.File, bool, error)`. +- Contract: `created=false, file=nil, err=nil` means a safe existing leaf prevented exclusive creation; any unsafe existing leaf returns `ErrUnsafeFile`. +- Contract: a successful caller owns the returned `*os.File`; the directory handle remains the sole cleanup authority through `RemoveRegular(name)`. +- Produces: `newStagingArchiveName() (string, error)` returning `archive-<32 lowercase hex>.zip` from 16 cryptographically random bytes. +- Produces: `stagedArchive{file *os.File, parent safeio.PrivateDirectoryHandle, name string, path string}`; `path` is diagnostic only and is never passed to a remove operation. + +- [ ] **Step 1: Add a failing cross-platform streaming-capability test** + +In `tools/tht/internal/safeio/files_test.go`, open a private directory, call the new method, stream bytes, rewind/read them, and remove the leaf through the same directory capability: + +```go +func TestPrivateDirectoryCreatesAndRemovesStreamingRegularFile(t *testing.T) { + temporaryRoot, err := filepath.EvalSymlinks(os.TempDir()) + if err != nil { + t.Fatal(err) + } + root, err := os.MkdirTemp(temporaryRoot, "tht-safeio-stream-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + if err := ProtectPrivateDirectory(root); err != nil { + t.Fatal(err) + } + directory, found, err := OpenPrivateDirectory(root, true) + if err != nil || !found { + t.Fatalf("OpenPrivateDirectory() = found %v, err %v", found, err) + } + defer directory.Close() + + file, created, err := directory.CreateRegularFile("archive-stream.zip") + if err != nil || !created || file == nil { + t.Fatalf("CreateRegularFile() = file %v, created %v, err %v", file, created, err) + } + if _, err := file.Write([]byte("private archive")); err != nil { + t.Fatal(err) + } + if err := file.Sync(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + removed, err := directory.RemoveRegular("archive-stream.zip") + if err != nil || !removed { + t.Fatalf("RemoveRegular() = removed %v, err %v", removed, err) + } +} +``` + +Add an adjacent case that pre-creates a safe `0600` leaf and expects `(nil, false, nil)`, then replaces a different leaf with a symlink/reparse point and expects `ErrUnsafeFile`: + +```go +if created, err := directory.CreateRegular("existing.zip", []byte("existing")); err != nil || !created { + t.Fatalf("CreateRegular(existing.zip) = created %v, err %v", created, err) +} +if file, created, err := directory.CreateRegularFile("existing.zip"); err != nil || created || file != nil { + t.Fatalf("CreateRegularFile(existing.zip) = file %v, created %v, err %v", file, created, err) +} +if created, err := directory.CreateRegular("target.zip", []byte("target")); err != nil || !created { + t.Fatalf("CreateRegular(target.zip) = created %v, err %v", created, err) +} +testsupport.SymlinkOrSkip(t, filepath.Join(root, "target.zip"), filepath.Join(root, "linked.zip")) +if file, created, err := directory.CreateRegularFile("linked.zip"); !errors.Is(err, ErrUnsafeFile) || created || file != nil { + t.Fatalf("CreateRegularFile(linked.zip) = file %v, created %v, err %v", file, created, err) +} +``` + +- [ ] **Step 2: Add failing StageArchive cleanup-race tests** + +Add `TestStageArchiveCloseUsesPinnedRootAfterAncestorSwap` in both platform files. Install the existing test hook before staging and react only to `before-stage-archive-remove`. + +The Unix case must: + +1. complete `StageArchive` first; +2. rename `result.stagingRoot` to `result.stagingRoot + "-original"` inside the close hook; +3. replace the lexical root with a symlink to an owner-private outside directory containing `sentinel`; +4. call `staged.Close()`; +5. prove the random archive is absent below the moved original root and the outside sentinel is unchanged. + +The Windows case must attempt the same root rename in the close hook and require the rename to fail while the retained no-delete directory handle is live. It then requires `staged.Close()` to succeed, the archive to be absent, and an outside sentinel to remain unchanged. + +Use these hook bodies so the race point is deterministic and occurs after streaming, immediately before removal: + +```go +// Unix hook. +restoreHook := safeio.SetPrivateDirectoryTestHookForTest(func(stage string) { + if stage != "before-stage-archive-remove" || swapped { + return + } + swapped = true + if err := os.Rename(result.stagingRoot, movedRoot); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, result.stagingRoot); err != nil { + t.Fatal(err) + } +}) +defer restoreHook() +``` + +```go +// Windows hook. +restoreHook := safeio.SetPrivateDirectoryTestHookForTest(func(stage string) { + if stage != "before-stage-archive-remove" || attemptedSwap { + return + } + attemptedSwap = true + if err := os.Rename(result.stagingRoot, result.stagingRoot+"-moved"); err == nil { + t.Fatal("staging-root rename succeeded while Close retained its directory handle") + } +}) +defer restoreHook() +``` + +After `staged.Close`, require the hook boolean to be true. The Unix test checks `os.Stat(filepath.Join(movedRoot, stagedName))` returns `os.ErrNotExist`; the Windows test checks the original root contains no archive leaf. Both read the outside sentinel and compare its exact original bytes. + +Update existing assertions that reference `staged.directory`: direct-child staging means `filepath.Dir(staged.path) == result.stagingRoot` in the normal case. + +- [ ] **Step 3: Run focused tests and verify RED** + +```bash +cd tools/tht +go test ./internal/safeio ./internal/backup -run 'TestPrivateDirectoryCreatesAndRemovesStreamingRegularFile|TestStageArchiveCloseUsesPinnedRootAfterAncestorSwap' -count=1 +``` + +Expected: compile failure because `CreateRegularFile`, `parent`, and `name` do not exist, or the old pathname cleanup is redirected by the Unix swap. + +- [ ] **Step 4: Add the minimal streaming method to `PrivateDirectoryHandle`** + +Import `os` in `private_root.go` and add exactly one method: + +```go +type PrivateDirectoryHandle interface { + Close() error + Validate() error + OpenChild(name string, ensure bool) (PrivateDirectoryHandle, bool, error) + CreateRegular(name string, contents []byte) (bool, error) + CreateRegularFile(name string) (*os.File, bool, error) + ReadRegular(name string, maximum int64) ([]byte, bool, error) + ReplaceRegular(name string, contents []byte) error + RemoveRegular(name string) (bool, error) + ListPage(maximumEntries int, afterName string, validName func(string) bool, validLinks func(string, uint64) bool) (PrivateDirectoryPage, error) + ClaimRegular(source, claim string) (bool, error) + ReadClaim(source, claim string, maximum int64) ([]byte, bool, error) + RemoveClaim(source, claim string) (bool, error) +} +``` + +Do not add `CreateTemporaryDirectory`, `RemoveDirectory`, or pathname-returning cleanup methods. + +- [ ] **Step 5: Implement exclusive streaming creation on Unix** + +Use `unix.Openat(directory.descriptor, name, O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC|O_NOFOLLOW, 0600)`. On success: + +- wrap the descriptor in `os.NewFile`; +- apply `Fchmod(0600)`; +- require `privateUnixRootRegular(stat, 1)` after `Fstat`; +- require `directory.Validate()` and `Fsync(directory.descriptor)` before return; +- on every failure, close the descriptor and `Unlinkat(directory.descriptor, name, 0)`. + +On `EEXIST`, inspect the existing leaf with `requirePrivateUnixRootRegularAt(..., 1)` and return `(nil, false, nil)` only for a safe existing regular file. Every other condition returns `ErrUnsafeFile`. + +- [ ] **Step 6: Implement exclusive streaming creation on Windows** + +Reuse `createWindowsPrivateRegularAt(directory.handle, name)` so the owner-only DACL is installed in the same NT relative create. Before detaching the handle: + +- require a one-link regular non-reparse file; +- require `directory.Validate()`; +- convert the retained file handle with `os.NewFile` and clear the wrapper handle to prevent double close; +- on failure call `closeAndDeleteWindowsPrivateRegular` while the parent handle is still retained. + +If relative open proves that a safe one-link leaf already exists, return `(nil, false, nil)`; unsafe or ambiguous failures return `ErrUnsafeFile`. + +- [ ] **Step 7: Replace StageArchive pathname ownership with the retained directory capability** + +Add the bounded, cryptographically random leaf-name helper: + +```go +func newStagingArchiveName() (string, error) { + value := make([]byte, 16) + if _, err := rand.Read(value); err != nil { + return "", err + } + return "archive-" + hex.EncodeToString(value) + ".zip", nil +} +``` + +After the capacity check, open the staging root once: + +```go +parent, found, err := safeio.OpenPrivateDirectory(result.stagingRoot, true) +if err != nil || !found { + return nil, errors.New("open private restore staging root") +} +``` + +Generate up to eight 16-byte random names. For each name, call `parent.CreateRegularFile(name)`; retry only the safe collision result. If no file is created, close the parent and return a sanitized creation error. + +Construct: + +```go +staged := &stagedArchive{ + file: file, + parent: parent, + name: name, + path: filepath.Join(result.stagingRoot, name), +} +``` + +Keep the existing source revalidation, bounded streaming, digest, `Sync`, and rewind logic unchanged. Remove `os.MkdirTemp`, `ProtectPrivateDirectory`, the nested `archive.zip`, and every `os.Remove` cleanup path from `StageArchive` and `stagedArchive.Close`. + +- [ ] **Step 8: Make Close exhaustive, capability-relative, and idempotent** + +`Close` must perform all cleanup attempts in this order even if an earlier operation fails: + +1. close `staged.file` and set it to `nil`; +2. call `safeio.NotifyPrivateDirectoryTestHookForTest("before-stage-archive-remove")`; +3. call `staged.parent.RemoveRegular(staged.name)` and require `removed=true` on the first close; +4. close `staged.parent` and set it to `nil`; +5. clear `name` and `path`; +6. return only `destroy private restore staging archive` if any operation failed. + +A second `Close()` returns `nil`. No error may contain the staging path or archive bytes. + +- [ ] **Step 9: Add the native Windows test gate** + +In the existing `windows-clone` job, after Go setup and before the clone-contract script, add: + +```yaml + - name: Run native Windows retained-capability tests + working-directory: tools/tht + run: go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1 +``` + +This job is the native execution authority. A Linux/macOS cross-compile only proves buildability. + +- [ ] **Step 10: Run focused, broad, race, and cross-compile gates** + +```bash +cd tools/tht +gofmt -w internal/safeio/private_root.go internal/safeio/private_root_unix.go \ + internal/safeio/private_root_windows.go internal/safeio/files_test.go \ + internal/backup/preflight.go internal/backup/preflight_test.go \ + internal/backup/preflight_unix_test.go internal/backup/preflight_windows_test.go +go test ./internal/safeio ./internal/backup -count=1 +go test -race ./internal/safeio ./internal/backup -count=1 +go vet ./internal/safeio ./internal/backup +GOOS=windows GOARCH=amd64 go test -c ./internal/safeio -o /tmp/tht-safeio-windows.test.exe +GOOS=windows GOARCH=amd64 go test -c ./internal/backup -o /tmp/tht-backup-windows.test.exe +``` + +Expected: all host tests pass and both Windows test executables compile. Record native execution as pending until the workflow job actually runs. + +- [ ] **Step 11: Commit the retained-staging remediation** + +```bash +git add tools/tht/internal/safeio/private_root.go \ + tools/tht/internal/safeio/private_root_unix.go \ + tools/tht/internal/safeio/private_root_windows.go \ + tools/tht/internal/safeio/files_test.go \ + tools/tht/internal/backup/preflight.go \ + tools/tht/internal/backup/preflight_test.go \ + tools/tht/internal/backup/preflight_unix_test.go \ + tools/tht/internal/backup/preflight_windows_test.go \ + .github/workflows/deployment.yml +git commit -m "fix(backup): retain staging cleanup capability" +``` + +**Mandatory Terra review gate:** Confirm no `StageArchive` creation or cleanup decision uses `os.MkdirTemp`, `os.Remove`, or a re-resolved staging pathname; `path` is diagnostic only; failure cleanup uses the retained parent; Unix swap cleanup removes only the moved-root archive; Windows native coverage is wired into CI; all handles close on every exit. Verdict must be `CLEAN` before Task 3. + +--- + +### Task 3: Remove Windows Claims Through the Retained Parent Handle + +**Files:** +- Modify: `tools/tht/internal/safeio/claim_windows.go:126-160` +- Create: `tools/tht/internal/safeio/claim_windows_test.go` + +**Interfaces:** +- Consumes: `OpenPrivateDirectory(path string, ensure bool) (PrivateDirectoryHandle, bool, error)`. +- Consumes: `PrivateDirectoryHandle.RemoveClaim(source, claim string) (bool, error)`. +- Preserves: `RemoveCanonicalPrivateClaim(source, claim string) (bool, error)` and its absent/orphan semantics. +- Produces test hook stage: `after-canonical-private-claim-parent-open`. + +- [ ] **Step 1: Add a native-Windows ancestor-swap test** + +Create a `//go:build windows` test in package `safeio`. It must: + +1. create and protect one private directory; +2. create `state.json` as an owner-private regular file; +3. create `state.claim` through `ClaimCanonicalPrivateRegular` so both names refer to the verified two-link file; +4. create a separate protected outside directory with a sentinel that must survive; +5. install `SetPrivateDirectoryTestHookForTest` and react to `after-canonical-private-claim-parent-open`; +6. attempt to rename the protected source directory and require Windows to reject the rename while the retained no-delete handle is live; +7. call `RemoveCanonicalPrivateClaim` and require `(true, nil)`; +8. prove both original names are absent and every outside file is unchanged. + +Use a private-file helper and the following main test shape: + +```go +func createWindowsPrivateTestFile(t *testing.T, path string, contents []byte) { + t.Helper() + file, err := CreateCanonicalNewPrivateFile(path) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write(contents); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} + +func TestRemoveCanonicalPrivateClaimRetainsParentDuringDeletion(t *testing.T) { + parent := filepath.Join(t.TempDir(), "claims") + if err := os.Mkdir(parent, 0o700); err != nil { + t.Fatal(err) + } + if err := ProtectPrivateDirectory(parent); err != nil { + t.Fatal(err) + } + source := filepath.Join(parent, "state.json") + claim := filepath.Join(parent, "state.claim") + createWindowsPrivateTestFile(t, source, []byte("state")) + if claimed, err := ClaimCanonicalPrivateRegular(source, claim); err != nil || !claimed { + t.Fatalf("ClaimCanonicalPrivateRegular() = claimed %v, err %v", claimed, err) + } + + outside := filepath.Join(t.TempDir(), "outside") + if err := os.Mkdir(outside, 0o700); err != nil { + t.Fatal(err) + } + if err := ProtectPrivateDirectory(outside); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(outside, "sentinel") + createWindowsPrivateTestFile(t, sentinel, []byte("outside-safe")) + attemptedSwap := false + restoreHook := SetPrivateDirectoryTestHookForTest(func(stage string) { + if stage != "after-canonical-private-claim-parent-open" || attemptedSwap { + return + } + attemptedSwap = true + if err := os.Rename(parent, parent+"-moved"); err == nil { + t.Fatal("claim parent rename succeeded while removal retained its handle") + } + }) + defer restoreHook() + + removed, err := RemoveCanonicalPrivateClaim(source, claim) + if err != nil || !removed || !attemptedSwap { + t.Fatalf("RemoveCanonicalPrivateClaim() = removed %v, attempted %v, err %v", removed, attemptedSwap, err) + } + for _, path := range []string{source, claim} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("removed path %q still exists: %v", filepath.Base(path), err) + } + } + if contents, err := os.ReadFile(sentinel); err != nil || string(contents) != "outside-safe" { + t.Fatalf("outside sentinel = %q, err %v", contents, err) + } +} +``` + +Add adjacent cases for an orphan one-link claim (`false, nil`, orphan preserved) and a mismatched two-file pair (`ErrUnsafeFile`, neither file removed). Build the mismatch as two independent private files, each with its own auxiliary hard link, so both source and claim have link count two but different file identities. These cases lock in current recovery semantics while changing deletion authority. + +- [ ] **Step 2: Cross-compile the new test and verify RED behavior by inspection** + +```bash +cd tools/tht +GOOS=windows GOARCH=amd64 go test -c ./internal/safeio -o /tmp/tht-safeio-claim-red.test.exe +``` + +Expected: compilation succeeds against the current API, but the new hook is never observed and the test would fail natively because the current function closes retained handles before absolute-path `DeleteFile` calls. Do not label this compile-only result as an executed RED test. + +- [ ] **Step 3: Replace absolute-path deletion with one retained directory operation** + +Implement the Windows function with a named return so a parent-close failure can fail closed: + +```go +func removeCanonicalPrivateClaim(source, claim string) (removed bool, resultErr error) { + parentPath := filepath.Dir(source) + sourceName := filepath.Base(source) + claimName := filepath.Base(claim) + if parentPath != filepath.Dir(claim) || !validPrivateLeafName(sourceName) || !validPrivateLeafName(claimName) { + return false, ErrUnsafeFile + } + directory, found, err := OpenPrivateDirectory(parentPath, false) + if err != nil || !found { + return false, ErrUnsafeFile + } + defer func() { + if closeErr := directory.Close(); closeErr != nil && resultErr == nil { + resultErr = ErrUnsafeFile + } + }() + NotifyPrivateDirectoryTestHookForTest("after-canonical-private-claim-parent-open") + return directory.RemoveClaim(sourceName, claimName) +} +``` + +Delete the old `openWindowsPrivateRegular`/close/`windows.DeleteFile` sequence from this function. Do not duplicate link-count, identity, orphan, or delete-on-close logic: `windowsPrivateDirectory.RemoveClaim` already implements those checks relative to `directory.handle`. + +- [ ] **Step 4: Run Windows compilation plus host-wide Go gates** + +```bash +cd tools/tht +gofmt -w internal/safeio/claim_windows.go internal/safeio/claim_windows_test.go +GOOS=windows GOARCH=amd64 go test -c ./internal/safeio -o /tmp/tht-safeio-claim-windows.test.exe +go test ./internal/safeio ./internal/authstorage -count=1 +go test -race ./... +go vet ./... +go build ./cmd/tht +``` + +Expected: cross-compile, safeio/authstorage semantics, race suite, vet, and CLI build pass. Native ancestor-swap execution remains pending until Task 4 runs the Windows workflow. + +- [ ] **Step 5: Commit the Windows claim remediation** + +```bash +git add tools/tht/internal/safeio/claim_windows.go \ + tools/tht/internal/safeio/claim_windows_test.go +git commit -m "fix(auth): remove Windows claims by retained handle" +``` + +**Mandatory Terra review gate:** Confirm `removeCanonicalPrivateClaim` contains no `DeleteFile` and performs no deletion after closing the parent capability; `RemoveClaim` remains the single identity/link/orphan authority; the test attempts an ancestor replacement and protects outside sentinels; Unix behavior is untouched. Verdict must be `CLEAN` before Task 4. + +--- + +### Task 4: Re-Certify the Remediation and Refresh Sanitized Evidence + +**Files:** +- Modify: `.artifacts/task-15/automated-gates.json` +- Modify: `.artifacts/task-15/unified-docker-images.json` only after a new immutable-source Docker smoke +- Modify: `.superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md` +- Modify: `PROJECT_STATE.md` + +**Interfaces:** +- Consumes every test and workflow gate from Tasks 1-3. +- Produces one source-bound PASS/FAIL/PENDING matrix with hashes for retained artifacts. +- Preserves the historical Task 15 evidence as provenance; new results supersede rather than rewrite historical source SHAs. + +- [ ] **Step 1: Freeze and record the source under test** + +After Tasks 1-3 and their reviews are clean: + +```bash +git status --short +AUTH_REMEDIATION_SOURCE="$(git rev-parse HEAD)" +printf '%s\n' "$AUTH_REMEDIATION_SOURCE" +``` + +Only `.playwright-cli/` and `.thothctl/` may be untracked. Record the resulting commit as `AUTH_REMEDIATION_SOURCE` in the operator notes. If any tracked source changes after this point, discard downstream certification results and restart this task from Step 1. + +- [ ] **Step 2: Run all Go security and build gates** + +```bash +cd tools/tht +go test ./internal/safeio ./internal/backup ./internal/authstorage -count=1 +go test -race ./... +go vet ./... +go build ./cmd/tht +GOOS=windows GOARCH=amd64 go test -c ./internal/safeio -o /tmp/tht-safeio-final-windows.test.exe +GOOS=windows GOARCH=amd64 go test -c ./internal/backup -o /tmp/tht-backup-final-windows.test.exe +GOOS=windows GOARCH=amd64 go test -c ./internal/authstorage -o /tmp/tht-authstorage-final-windows.test.exe +GOOS=windows GOARCH=amd64 go build -o /tmp/tht-final-windows.exe ./cmd/tht +``` + +Record package counts and exact failures. Cross-compilation is a separate `PASS` row and never substitutes for native Windows execution. + +- [ ] **Step 3: Run the exact Node 24 backend and frontend gates** + +```bash +cd backend +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH node --version +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx tsc --noEmit -p . +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx vitest run +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npm run build +cd ../frontend +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx tsc -b +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npx vitest run +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npm run build +PATH=/Users/mp/.nvm/versions/node/v24.16.0/bin:$PATH npm run e2e -- --grep "authentication|F1" +``` + +The first command must print `v24.16.0`. Run the existing sentinel leak scan in `scripts/authentication-smoke.sh`; retain no browser credential or token. + +- [ ] **Step 4: Run harness and static/documentation gates without laundering baselines** + +```bash +cd ../harness +.venv/bin/pytest -q +.venv/bin/ruff check . +cd .. +bash scripts/auth-docs-smoke.sh +``` + +Record current counts. Existing baseline failures remain `FAIL` unless the exact command is now green; this remediation task does not authorize unrelated fixes. + +- [ ] **Step 5: Execute the native Windows authority** + +Push or dispatch only if the execution session has explicit repository authorization. The remote branch must contain `AUTH_REMEDIATION_SOURCE`: + +```bash +AUTH_REMEDIATION_SOURCE="$(git rev-parse HEAD)" +git push origin HEAD:feat/thoth-auth +gh workflow run deployment.yml --ref feat/thoth-auth -f windows_docker_startup=false +AUTH_WINDOWS_RUN_ID="" +for AUTH_WINDOWS_LOOKUP_ATTEMPT in {1..12}; do + AUTH_WINDOWS_RUN_ID="$(gh run list --workflow deployment.yml --branch feat/thoth-auth \ + --event workflow_dispatch --limit 10 --json databaseId,headSha \ + --jq "map(select(.headSha == \"$AUTH_REMEDIATION_SOURCE\"))[0].databaseId // empty")" + if [[ -n "$AUTH_WINDOWS_RUN_ID" ]]; then break; fi + sleep 5 +done +test -n "$AUTH_WINDOWS_RUN_ID" +gh run watch "$AUTH_WINDOWS_RUN_ID" --exit-status +gh run view "$AUTH_WINDOWS_RUN_ID" --json jobs \ + --jq '.jobs[] | select(.name == "Windows clone and Compose contract") | {name,conclusion,url}' +``` + +Wait for the matching source SHA and require the `Windows clone and Compose contract` job, including `Run native Windows retained-capability tests`, to pass. Retain the run URL/ID and the two focused test names, not raw runner logs. If dispatch or a native runner is unavailable, record `PENDING: native Windows execution unavailable` and do not mark the three-finding remediation complete. + +- [ ] **Step 6: Run shell, Compose, authentication, and final Docker gates** + +Run lightweight contracts first: + +```bash +bash -n scripts/*.sh +bash scripts/authentication-smoke.sh +bash scripts/auth-docs-smoke.sh +docker compose -f compose.yaml config --quiet +docker compose -f compose.yaml -f compose.unified.yaml config --quiet +``` + +Then run the unified Docker smoke exactly once against the frozen source: + +```bash +timeout --signal=TERM --kill-after=45s 32m bash scripts/unified-deployment-smoke.sh +``` + +Require task-scoped cleanup and five-image source traceability. A failed Docker smoke stays `FAIL`; do not rerun it against changed source without restarting at Step 1. + +- [ ] **Step 7: Run optional external gates only when their prerequisites exist** + +```bash +cd harness +.venv/bin/pytest -q -m l2 +``` + +Follow `docs/testing/authentication-manual-acceptance.md` for real PSD/Authentik acceptance only when real identity/access is available. Missing L2 secrets, Authentik access, PSD identities, or a provider port are `PENDING` with the exact prerequisite category; they are not remediation failures and are not PASS. + +- [ ] **Step 8: Refresh machine-readable and narrative evidence** + +Update `.artifacts/task-15/automated-gates.json` with: + +- `source_commit` equal to `AUTH_REMEDIATION_SOURCE`; +- UTC start/end timestamps; +- exact Node, Go, and Pi versions; +- focused ownership, StageArchive, and Windows claim test status; +- native Windows run ID/status distinct from cross-compile status; +- backend/frontend/harness counts; +- Docker run ID and cleanup status; +- unchanged known FAIL/PENDING rows where still applicable. + +Regenerate `.artifacts/task-15/unified-docker-images.json` only from the new Docker run and retain digests without registry names. Compute both SHA-256 values and place them in the Task 15 report. + +Update `PROJECT_STATE.md` so its leading Task 15 section states separately: + +- whether all three Important findings are closed by a clean final review; +- whether native Windows execution passed; +- whether authentication is implementation-complete; +- whether release acceptance remains blocked by unrelated FAIL/PENDING gates. + +- [ ] **Step 9: Commit only sanitized certification evidence** + +```bash +git add .artifacts/task-15/automated-gates.json \ + .artifacts/task-15/unified-docker-images.json \ + .superpowers/sdd/2026-08-16-thothii-authentication/task-15-report.md \ + PROJECT_STATE.md +git commit -m "docs(auth): record remediation certification" +``` + +Before committing, search the staged diff for fixture secrets, tokens, internal endpoints, user identities, and registry names. The report may contain hashes, versions, test counts, job IDs, and sanitized failure categories only. + +**Mandatory final Terra review gate:** Review `178113a..HEAD`, not only the evidence commit. Reproduce focused tests, inspect every affected security boundary, verify native-Windows evidence is executed rather than inferred, and issue separate verdicts for (a) the three Important findings and (b) overall release readiness. The remediation is complete only when verdict (a) is `CLEAN`; overall release readiness must remain `PENDING` or `FAIL` wherever unrelated gates still require it. + +--- + +## Final Acceptance Checklist + +- [ ] Every POSIX `lstat`/`fstat` metadata path for `users.yaml` and its parent requires the same valid effective UID. +- [ ] Foreign ownership at file-path, file-descriptor, directory-path, and directory-descriptor observations fails with only `local_user_registry_invalid`. +- [ ] `StageArchive` retains one `PrivateDirectoryHandle` from file creation through `Close` cleanup. +- [ ] `StageArchive` has no pathname-authorized file or directory removal and no nested temporary directory. +- [ ] Unix ancestor replacement cannot redirect staging cleanup; native Windows blocks replacement while the retained handle is live. +- [ ] Windows canonical claim removal delegates to `PrivateDirectoryHandle.RemoveClaim` and contains no post-close `DeleteFile` call. +- [ ] Native Windows executes both retained-capability race tests; cross-compilation is recorded separately. +- [ ] A fresh Terra reviewer reports no Critical/Important finding after every task. +- [ ] A fresh final Terra reviewer reports the three original Important findings `CLEAN` over `178113a..HEAD`. +- [ ] Evidence is bound to one immutable source SHA, sanitized, hashed, and honest about all remaining FAIL/PENDING release gates. diff --git a/docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md b/docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md new file mode 100644 index 00000000..5714168c --- /dev/null +++ b/docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md @@ -0,0 +1,674 @@ +# Git-backed Workspace Registry Design + +**Date:** 2026-08-03 +**Status:** Approved design +**Scope:** Portable workspace definition, CRUD UI, Git publication, local bindings, validation, and runtime revision pinning + +## 1. Purpose + +ThothII must manage workspace configuration as part of its own domain. A workspace must not depend on Chirone, Aritmolab, Supabase, or on the application that happens to provide a data warehouse, vector database, or embedding service. + +The same logical workspaces must be usable by: + +- the production ThothII server; +- a local ThothII installation running in Docker on macOS, Windows, or Linux; +- future ThothII installations connected to different data warehouses, vector stores, and model providers. + +A remote Git repository is the single source of truth. Each ThothII installation maintains a persistent local checkout, resolves installation-specific connectivity through local variables and secret files, and exchanges changes through pull and push. + +## 2. Goals + +- Provide a CRUD workspace page reachable from the right sidebar. +- Store canonical workspace definitions as versioned YAML in a generic Git repository. +- Support GitHub, Gitea, GitLab, and other standard Git servers without provider-specific APIs. +- Keep credentials and private keys outside the repository while defining their required variable names deterministically. +- Support direct database connections, REST access, and SSH-tunnelled connections. +- Treat vector collection and embedding configuration as one coherent semantic index. +- Keep user choices such as active workspace, LLM, and reasoning level local to the browser until an identity system exists. +- Validate free-form values formally, semantically, and—when local bindings exist—operationally. +- Pin every new session to an immutable workspace revision. +- Continue operating from the last valid snapshot when the Git remote is temporarily unavailable. +- Retain browser-mediated export/import as an offline fallback, not as the primary synchronization mechanism. + +## 3. Non-goals + +The first release will not provide: + +- embedded user authentication or per-person server profiles; +- automatic continuous synchronization; +- Git pull-request workflows; +- editing or storing secret values in the workspace UI; +- provider-specific GitHub or Gitea APIs; +- automatic conflict merging; +- a Supabase or SQLite source of truth; +- a user-selectable embedding model for a shared vector collection. + +## 4. Configuration layers + +ThothII separates configuration into three layers. + +### 4.1 Shared workspace definition + +The Git repository contains logical, shared configuration: + +- workspace identity and display metadata; +- DWH engine, logical database/schema, and supported access transports; +- vector-store type and collection; +- embedding provider contract, model, dimensions, and distance metric; +- LLM default and allowlist; +- language and supported workflow capabilities; +- the deterministic installation-variable contract. + +### 4.2 Installation bindings + +Each ThothII installation supplies operational values locally: + +- transport selected for each connector; +- host names, ports, base URLs, and tunnel targets; +- users and non-secret connection parameters; +- password, API-key, certificate, and private-key file paths; +- Git remote credentials, CA, and SSH known-hosts file; +- application data paths for sessions, artifacts, checkout, and snapshots. + +Installation bindings are excluded from the workspace repository. Absolute storage paths formerly represented by `roots` belong to this layer and are not shown in the ordinary workspace form. + +### 4.3 Browser preferences + +Until ThothII has a reliable user identity, these values are stored in browser-local storage: + +- active workspace; +- selected LLM provider/model; +- reasoning level; +- unfinished workspace drafts; +- visual preferences. + +These values are not included in Git or export bundles. The resolved workspace, model, and reasoning level are copied into each session manifest for reproducibility. + +## 5. Git repository contract + +The repository has this canonical layout: + +```text +thoth-workspaces.yaml +workspaces/ + .yaml +contracts/ + .env.example +docs/ + .md +``` + +`thoth-workspaces.yaml` declares the repository schema version. The YAML file is authoritative. The environment example and documentation are deterministic generated artifacts committed by the same publish operation. + +Workspace IDs must match: + +```text +^[a-z][a-z0-9-]{2,62}$ +``` + +The ID is an immutable technical identifier. Renaming the display label does not rename variables, files, or session references. Changing the ID is a migration operation outside ordinary edit mode. + +## 6. Workspace schema + +The initial canonical shape is: + +```yaml +workspace: + schema_version: 2 + id: psd-clinical + name: Policlinico San Donato + description: Clinical data warehouse workspace + language: it + +dwh: + engine: postgres + database: postgres + schema: datawarehouse + supported_transports: + - postgres_direct + - rest_api + - ssh_tunnel + +semantic_index: + vector_store: + engine: pgvector + database: postgres + schema: vectors + collection: clinical_documents + dimensions: 768 + distance: cosine + supported_transports: + - pgvector_direct + - rest_api + - ssh_tunnel + vector_writer: {} # optional; enables a distinct, locally bound reversible diagnostic writer + embedding: + provider: ollama_compatible + model: nomic-embed-text-v2-moe + dimensions: 768 + +diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: bearer + response: { database: database, schema: schema } + vector_rest: + metadata: + method: GET + path: /vector/metadata + auth: bearer + response: { collection: collection, dimensions: dimensions, distance: distance } + reversible_probe: + method: POST + path: /vector/diagnostic-probe + auth: bearer + response: { operation: operation } + embedding: + method: GET + path: /models + auth: none + response: { model: model, dimensions: dimensions } + +llm_policy: + default: zai/glm-5.2 + allowed: + - zai/glm-5.2 + - openai/gpt-5 +``` + +The exact machine schema is maintained by `WorkspaceSchema` and versioned with explicit migrations. Unknown keys are rejected by default so misspellings do not silently change runtime behavior. + +### 6.1 Version migration and operational state + +Schema version 2 makes `semantic_index.vector_store.database` and `.schema` mandatory. They +identify the vector service independently of the DWH, even when both happen to use the same +PostgreSQL instance. + +Version 1 descriptors remain readable and listable so operators can discover legacy Git content. +They are marked `migration_required` and may not generate installation bindings, runtime +configuration, diagnostics, or publication artifacts. Migration is an explicit UI/transformer +action that supplies the vector database/schema; it must never infer either value from the DWH. +The resulting descriptor is written as schema version 2 and then passes normal operational +validation. + +The migration also preserves least privilege: `vector_writer` is optional and never inferred from +the reader binding. A v2 descriptor without it is valid and operates reader-only. If it is +declared, its local `VECTOR_WRITER_API_KEY_FILE` is distinct from the reader API-key file and is +used only by the explicitly requested reversible writer diagnostic. + +### 6.2 Semantic-index invariant + +`semantic_index` is atomic. The vector collection, vector dimensions, distance metric, embedding provider, embedding model, and embedding dimensions describe one index contract. + +The following are validation errors: + +- vector and embedding dimensions differ; +- the selected collection reports different dimensions or distance metric; +- the embedding endpoint does not expose the declared model; +- read and write bindings resolve to incompatible vector collections; +- indexing and retrieval resolve to different embedding contracts. + +Changing collection, embedding model, dimensions, or metric is presented as replacing or migrating the semantic index, not as an individual user preference. + +### 6.3 Declared diagnostic protocol + +Diagnostics are declarative and strict. `dwh_rest` declares the DWH ping method, origin-relative +path, authentication mode, and JSON fields that must equal the canonical DWH database/schema. +`vector_rest.metadata` does the same for collection, dimensions, and distance. `embedding` declares +the model/dimensions response fields. Only `GET` and `POST`, `none`/`bearer`/`x-api-key` +authentication, origin-relative paths without a query or fragment, and identifier-shaped response +field names are accepted. + +For `auth: none`, the binding resolver, runtime renderer, and diagnostic connector all omit the +API-key requirement. Credential-backed declarations retain their local secret-file requirement. + +`vector_rest.reversible_probe`, when present, is an authenticated POST with a declared response +field that must echo each requested `create`/`remove` operation. It is called with a generated +diagnostic record create request and a matching remove request, with cleanup retried in `finally`. +An upsert-only service cannot be declared as this probe. All ordinary diagnostics remain read-only. +The complete request, response, timeout, reader-only fallback, SSH, and private-CA limitations are +the operator contract in [Workspace diagnostic protocol](../../workspace-diagnostic-protocol.md). + +## 7. Deterministic installation-variable naming + +The environment namespace is derived from the immutable workspace ID: + +```text +psd-clinical -> PSD_CLINICAL +``` + +Every variable starts with `THT_WS__`. Connector roles and suffixes are defined by ThothII and cannot be invented in the form. + +### 7.1 DWH variables + +```dotenv +THT_WS_PSD_CLINICAL_DWH_TRANSPORT= +THT_WS_PSD_CLINICAL_DWH_HOST= +THT_WS_PSD_CLINICAL_DWH_PORT= +THT_WS_PSD_CLINICAL_DWH_BASE_URL= +THT_WS_PSD_CLINICAL_DWH_USER= +THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE= +THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE= +THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE= +``` + +### 7.2 Vector-store variables + +```dotenv +THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT= +THT_WS_PSD_CLINICAL_VECTOR_HOST= +THT_WS_PSD_CLINICAL_VECTOR_PORT= +THT_WS_PSD_CLINICAL_VECTOR_BASE_URL= +THT_WS_PSD_CLINICAL_VECTOR_USER= +THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE= +THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE= +THT_WS_PSD_CLINICAL_VECTOR_TLS_CA_FILE= +``` + +The collection and dimensions remain in the canonical workspace because they define the shared semantic index. + +### 7.3 Embedding variables + +```dotenv +THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL= +THT_WS_PSD_CLINICAL_EMBEDDING_API_KEY_FILE= +THT_WS_PSD_CLINICAL_EMBEDDING_TLS_CA_FILE= +``` + +The embedding model and dimensions remain in the canonical workspace. + +### 7.4 Optional vector-writer variable + +Only a descriptor declaring `semantic_index.vector_writer: {}` generates this local secret-file +binding. It is never generated for a reader-only workspace: + +```dotenv +THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE= +``` + +The generated workspace documentation and `.env.example` must render this exact `_FILE` variable +when the optional writer exists. The path must be distinct from +`THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE`; neither file's content is rendered. + +### 7.5 SSH tunnel variables + +For any connector role `` that selects `ssh_tunnel`, ThothII requires: + +```dotenv +THT_WS_PSD_CLINICAL__SSH_HOST= +THT_WS_PSD_CLINICAL__SSH_PORT= +THT_WS_PSD_CLINICAL__SSH_USER= +THT_WS_PSD_CLINICAL__SSH_PRIVATE_KEY_FILE= +THT_WS_PSD_CLINICAL__SSH_KNOWN_HOSTS_FILE= +THT_WS_PSD_CLINICAL__SSH_TARGET_HOST= +THT_WS_PSD_CLINICAL__SSH_TARGET_PORT= +``` + +Secret values use `*_FILE` variables. The application reads the file at runtime and never serializes its content into API responses, logs, Git commits, diagnostics, or export bundles. + +The generated `.env.example`, generated workspace documentation, UI installation-requirements panel, and runtime validator are all derived from the same binding schema. + +## 8. Supported transports + +Transport behavior is encapsulated behind connector adapters. + +### 8.1 Direct + +Direct adapters connect to the configured host and port with the native protocol. PostgreSQL +direct access uses a supplied CA file when present and otherwise requires runtime system trust; +certificate verification is never disabled. Vector direct access uses the native vector-store +protocol or database driver. + +### 8.2 REST API + +REST adapters use a base URL, an optional API-key file, TLS validation, and a documented capabilities endpoint. A REST adapter must expose enough metadata to validate schema or collection identity and semantic-index compatibility. + +The present diagnostic adapter cannot load a private CA from a REST `*_TLS_CA_FILE` binding. It +therefore refuses that diagnostic rather than weakening certificate verification. Operators must +use a runtime-trusted HTTPS chain, direct/SSH transport with native PostgreSQL CA handling, or a +trusted TLS-termination boundary. + +### 8.3 SSH tunnel + +SSH adapters verify the remote host against an explicit known-hosts file, open a temporary local +tunnel, and pass the resulting endpoint to the corresponding direct adapter, including its +verified private-CA-or-system-trust policy. The direct adapter connects to loopback but uses the +original `SSH_TARGET_HOST` as the TLS server name, so certificate hostname validation remains +bound to the remote target. Host-key checking cannot be disabled by the form. + +Transport selection is installation-specific because a production server may connect directly while a laptop reaches the same logical resource through REST or SSH. + +## 9. Backend architecture + +### 9.1 `WorkspaceSchema` + +- Parses canonical YAML. +- Rejects unknown or malformed fields. +- Applies explicit schema migrations. +- Produces canonical serialization. +- Generates binding requirements and documentation. + +### 9.2 `GitWorkspaceRepository` + +- Owns the persistent checkout. +- Reports remote, branch, current commit, dirty state, and divergence. +- Performs fetch, fast-forward pull, diff, commit, and push using argument-safe process execution. +- Uses installation-mounted Git credentials and trust configuration. +- Never accepts repository paths or shell fragments from API requests. + +### 9.3 `WorkspaceRegistry` + +- Lists and reads workspaces from a validated repository revision. +- Creates, updates, duplicates, and deletes workspace documents. +- Enforces workspace IDs and revision preconditions. +- Coordinates publish under a repository lock. +- Materializes immutable validated snapshots. + +### 9.4 `BindingResolver` + +- Generates deterministic environment names. +- Determines required and conditional variables from the selected transports. +- Reads normal variables and secret files. +- Returns sanitized missing/invalid diagnostics without values. + +### 9.5 `WorkspaceDiagnostics` + +- Runs connector-specific operational checks. +- Verifies the semantic-index invariant against live capabilities. +- Separates errors from warnings and local non-activatability. +- Uses read-only probes by default. + +A vector write probe is an explicit action. It writes a uniquely named temporary record in a diagnostic namespace or transaction and removes it before returning. It is not part of ordinary save or publish. + +When no writer descriptor or distinct local writer file is present, the same workspace remains +reader-only and the write probe is omitted; no reader credential is repurposed for writing. + +## 10. Persistent server and local layout + +Both production and local Docker deployments use: + +```text +/data/workspace-registry/ + repo/ # persistent Git checkout + snapshots/ # immutable validated revisions + state/ # active revision and repository metadata + locks/ # short-lived publish locks +``` + +The application image remains read-only. Git credentials, CA files, SSH keys, and known-hosts files are mounted under `/run/secrets` or another installation-controlled secret root. + +On startup: + +1. Clone the configured remote if no checkout exists. +2. Otherwise load the checkout and attempt fetch/pull. +3. Validate the complete candidate repository revision. +4. Atomically activate the new snapshot only if all workspace files and generated contracts are valid. +5. If the remote is unavailable or the candidate is invalid, retain the last valid snapshot and report degraded registry status. + +Database, vector, and embedding servers do not run the workspace manager. Only a ThothII installation needs outbound Git access. Gitea may be colocated with the production ThothII host. + +## 11. Git workflow and concurrency + +### 11.1 Browser drafts + +Drafts remain in browser-local storage and contain: + +- remote fingerprint; +- branch; +- workspace ID; +- base commit; +- base workspace blob checksum; +- form data and update timestamp. + +Drafts do not modify the shared checkout. + +### 11.2 Publish + +Publish is explicit and displays the canonical field-level diff. The backend then: + +1. Acquires the repository publish lock. +2. Fetches the remote branch. +3. Compares the submitted base commit and workspace checksum with the remote. +4. If only other workspaces changed, reapplies the draft on the new remote head. +5. If the same workspace changed, returns HTTP 409 with base, local, and remote field differences. +6. Validates the complete resulting repository. +7. Writes YAML and generated files atomically. +8. Creates a commit with the configured technical identity. +9. Pushes the configured branch. +10. Materializes and activates the validated snapshot. + +If a concurrent push wins after step 3, the backend fetches once more. It retries only when the target workspace is unchanged; otherwise it returns a conflict. + +Without embedded authentication, commits use a technical author such as `ThothII Workspace Manager` and include an installation-ID trailer. They do not claim a human identity. + +### 11.3 Pull + +Pull fetches the remote, requires fast-forward history, validates the complete candidate revision, and activates it atomically. Browser drafts whose base revision becomes stale remain available but are visibly marked as requiring reconciliation. + +### 11.4 Delete + +Delete creates a draft deletion and is published as a Git commit. A workspace referenced by an active runtime cannot be deleted. Historical session snapshots remain available, and Git history provides repository-level recovery. + +## 12. CRUD user experience + +The right sidebar exposes `Workspace management`, opening a dedicated page with a workspace list and editable detail area. + +The form sections are: + +1. General. +2. DWH. +3. Semantic index. +4. LLM policy. +5. Installation requirements. +6. Git status and history. + +Actions are: + +- New; +- Duplicate; +- Delete; +- Save draft; +- Discard changes; +- Pull; +- Publish; +- Export; +- Import; +- Test on this installation. + +Closed choices are used whenever the domain is enumerable: + +- database engine; +- transport type; +- vector-store engine; +- embedding provider; +- distance metric; +- TLS mode; +- language; +- LLM provider/model returned by Pi; +- embedding model returned by a reachable provider. + +Free text or numeric controls are used for names, descriptions, IDs, database/schema/collection identifiers, ports, dimensions, timeouts, and URLs. They display field-level constraints before submission and server validation errors after submission. + +The installation-requirements section shows the exact required, optional, and transport-conditional variable names. It never displays resolved secret values. + +## 13. Validation model + +### 13.1 Formal validation + +- YAML and schema version are valid. +- Required fields are present. +- Unknown fields are rejected. +- IDs and database identifiers match their allowed syntax. +- Ports are integers from 1 through 65535. +- Dimensions and timeouts are positive and within configured safety limits. +- URLs use supported schemes. +- enum values come from the closed schema lists. + +### 13.2 Static semantic validation + +- Selected transports are supported by the corresponding connector. +- Required fields for each transport can be derived unambiguously. +- Embedding and vector dimensions match. +- LLM default belongs to the allowlist. +- Duplicate workspace IDs and generated environment namespaces are rejected. +- Generated documentation exactly matches the binding contract. + +Save draft may retain incomplete local form state in the browser. Publish requires formal and static semantic validation to pass. + +### 13.3 Local operational validation + +- Required variables exist. +- Secret files are regular, readable files within approved secret roots. +- DNS, TCP, TLS, and authentication succeed. +- DWH database and schema exist and are readable. +- REST capabilities match the declared logical resource. +- SSH host verification and tunnel opening succeed. +- Vector collection, dimensions, metric, and read capability match. +- Embedding endpoint exposes the declared model and returns the expected dimensions for a controlled probe. +- A requested writer probe has a declared reversible POST operation, distinct writer credential, + and successful bounded cleanup; otherwise it is omitted without weakening reader validation. + +A portable workspace can be valid but not activatable on a particular installation. Publish is allowed in that state; starting a new session on that installation is not. + +## 14. API contract + +```text +GET /workspace-registry/status +POST /workspace-registry/pull +GET /workspaces +GET /workspaces/:id +POST /workspaces/validate +POST /workspaces/:id/test +POST /workspaces/publish +GET /workspaces/:id/export +POST /workspaces/import +``` + +The status response contains sanitized remote identity, branch, active commit, divergence, last successful sync, last validation result, and degraded status. + +Validation accepts a structured workspace draft rather than arbitrary YAML text. Publish accepts create, update, or delete intent plus base revision metadata. No endpoint accepts a filesystem path. + +Operational errors use stable codes that distinguish: + +- invalid configuration; +- missing local binding; +- non-activatable workspace; +- stale revision; +- field conflict; +- Git remote unavailable; +- Git authentication failure; +- non-fast-forward history; +- push rejection; +- connector unavailable; +- semantic-index incompatibility. + +## 15. Offline export/import fallback + +Export returns `.thoth-workspace.zip` containing: + +```text +manifest.json +workspace.yaml +contract.env.example +README.md +``` + +The manifest contains bundle schema version, workspace ID, source commit, file checksums, and creation timestamp. It contains no secrets or browser preferences. + +Import uploads the bundle to the currently open ThothII installation. The backend validates archive size, entry count, entry names, checksums, schema, and semantics. A successful import returns a browser draft; it does not write or publish directly. + +The browser can therefore download from one ThothII installation and upload to another without direct server-to-server access. Git remains the authoritative synchronization mechanism. + +## 16. Session integration + +New-session creation sends the browser-selected workspace ID, LLM provider/model, and reasoning level. The backend: + +1. Resolves the active validated workspace snapshot. +2. Verifies local activatability. +3. Validates the LLM choice against workspace policy and Pi availability. +4. Starts the harness with the immutable snapshot path. +5. Persists workspace ID, workspace revision, provider, model, and reasoning level in the session manifest. + +Resume uses the persisted snapshot revision even after later pull or publish operations. Snapshot retention cannot remove revisions referenced by resumable sessions. + +Legacy sessions without workspace revision use the existing compatibility resolution and receive a visible legacy warning. New sessions always require a revision. + +## 17. Security constraints + +- No secret value appears in Git, generated documentation, API payloads, logs, diagnostics, browser storage, or export bundles. +- Secret references use approved `*_FILE` variables and approved secret roots. +- Workspace and archive names cannot influence filesystem paths. +- Import prevents zip-slip, symlinks, excessive file count, and excessive expanded size. +- Git commands receive fixed argument arrays; user input is never passed through a shell. +- Git SSH uses explicit known-hosts verification. +- REST and direct TLS validation cannot be disabled silently. +- Diagnostics sanitize provider errors before returning them to the browser. +- `auth: none` diagnostics neither require nor read an API-key file; authenticated REST + diagnostics still require the declared local secret file. +- Production CORS remains same-origin; absence of embedded authentication does not imply cross-origin write access. +- The first release allows every user who can access the ThothII application to publish workspace changes. This limitation is documented until an authorization layer is introduced. + +## 18. Migration + +The migration path is: + +1. Introduce the versioned canonical schema and parser. +2. Convert existing `harness/workspaces` and deployment descriptors into repository fixtures. +3. Generate deterministic environment contracts and compare them with current Compose variables. +4. Configure the persistent registry volume and Git remote. +5. Import the current PSD workspace as the first canonical revision. +6. Keep legacy reads available during a bounded compatibility period. +7. Switch new sessions to validated snapshots and revision pinning. +8. Remove regex-based workspace metadata parsing after all active configurations use schema version 1. + +Migration never copies secret values into Git. Existing absolute roots become installation-level storage configuration. + +## 19. Documentation deliverables + +- Workspace schema reference with field descriptions and examples. +- Generated documentation for every workspace. +- Generated `.env.example` for every workspace. +- A detailed local-installation manual for Docker Desktop on macOS and for a local Docker engine on PC. It must cover prerequisites, clone/checkout or remote bootstrap, local Git credentials, persistent volumes, installation bindings, secret files, Docker Compose startup, first pull, workspace diagnostics, local publish, update, backup, and rollback. +- A detailed server-installation manual. It must cover service account and filesystem ownership, persistent registry volume, remote Git and Gitea configuration, HTTPS/SSH Git credentials, CA and known-hosts mounts, secret-file layout and permissions, Compose deployment, first bootstrap, firewall and outbound Git requirements, same-origin reverse-proxy exposure, health/status verification, pull/publish operations, upgrade, backup, degraded-mode recovery, and rollback to a prior validated snapshot. +- The two manuals must distinguish values that are shared in Git from installation-local bindings and secret files. Both must include complete direct PostgreSQL, REST, and SSH-tunnel examples and a troubleshooting table keyed by the stable diagnostic error codes. +- Docker Compose examples for server and local installations, referenced by the corresponding manual and tested as runnable examples. +- Direct PostgreSQL, REST, and SSH-tunnel examples. +- Vector/embedding compatibility explanation. +- Git remote setup for HTTPS and SSH. +- Gitea deployment example. +- Pull, draft, publish, conflict, export, and import operator guide. +- Diagnostic command and error-code reference. +- Migration guide from current PSD configuration. + +## 20. Testing strategy + +- Unit tests for schema parsing, canonical serialization, migrations, and unknown-key rejection. +- Unit tests for deterministic environment naming and conditional binding requirements. +- Valid and invalid fixtures for direct, REST, and SSH transports. +- Semantic-index fixtures covering model, dimensions, metric, collection, and read/write mismatch. +- Temporary local Git remotes for clone, pull, publish, retry, divergence, and same-file conflicts. +- Atomic-write and lock tests. +- API tests for CRUD, stale revisions, stable error codes, and sanitized responses. +- Import tests for checksum failure, zip-slip, symlinks, archive limits, and malformed schemas. +- Frontend tests for closed choices, free-field errors, drafts, diffs, conflicts, and installation requirements. +- End-to-end tests using a local Git remote and simulated connectors. +- Deployment tests proving persistent checkout and last-valid-snapshot fallback across container replacement. +- Session tests proving revision pinning and resume after a newer workspace publish. +- Documentation verification that executes the manual's local and server Compose examples in isolated test fixtures, including initial bootstrap and recovery from an unavailable Git remote. + +## 21. Acceptance criteria + +The feature is complete when: + +- a workspace can be created, edited, duplicated, deleted, pulled, and published from the UI; +- two browsers cannot silently overwrite the same workspace revision; +- server and local Docker installations can consume the same Git repository; +- each installation can bind the same logical workspace through different transports; +- generated variable names and documentation are deterministic and tested; +- no secret value enters Git or an export bundle; +- vector collection and embedding compatibility is enforced; +- a remote outage leaves the last valid snapshot usable; +- every new session records and resumes with an immutable workspace revision; +- detailed, tested installation manuals exist for local PC/Mac Docker deployments and for server deployments; +- existing PSD configuration can be migrated without embedding PSD-specific behavior in the core schema. diff --git a/docs/superpowers/specs/2026-08-04-unified-compose-deployment-design.md b/docs/superpowers/specs/2026-08-04-unified-compose-deployment-design.md new file mode 100644 index 00000000..8c11b6ce --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-unified-compose-deployment-design.md @@ -0,0 +1,293 @@ +# Unified Docker Compose Deployment Design + +**Date:** 2026-08-04 + +**Status:** approved in conversation, pending written-spec review + +## Objective + +ThothII ships as one autonomous Docker Compose application that runs unchanged on a developer +PC/Mac or on a server. ThothII has no runtime, build, network, path, proxy, configuration, or +documentation dependency on PSD, Chirone, `omics_portal`, or any other application that happens +to provide databases or vector services. + +## Architecture + +The distribution contains the same `frontend` and `core` images in every environment. A portable +base Compose file defines services, health checks, internal networking, named volumes, registry +storage, and configuration contracts. Small local and server overrides select host exposure, +storage bindings, authentication policy, restart policy, and operational limits without copying +the complete service definitions. + +```text +Browser -> frontend container -> core container + |-> Git workspace registry + |-> configured external DWH endpoint + |-> configured external VectorDB endpoint + |-> configured external embedding/LLM endpoints + `-> configured session persistence +``` + +The frontend calls the core over the private Compose network through same-origin proxying. The +browser never needs the core port in the server profile. On a workstation, the frontend is bound +to loopback and the core may be bound to loopback for diagnostics. On a server, a generic external +reverse proxy forwards to the frontend host port; that proxy is an operator concern and is not a +ThothII dependency. + +## Compose layout + +- `compose.yaml`: portable base stack and the only complete service definition. +- `deploy/compose.local.yaml`: loopback ports, local named volumes, `AUTH_MODE=none`, workstation + defaults, and local installation identity. +- `deploy/compose.server.yaml`: frontend host binding suitable for a reverse proxy, no public core + binding, server storage policy, configurable authentication, restart policy, and resource limits. +- `deploy/compose.git-ssh.yaml` and `deploy/compose.git-https.yaml`: mutually exclusive Git secret + mounts and trust configuration. +- `deploy/compose.connector-secrets.yaml`: explicit connector secret mounts selected by an + installation. +- `.env.example`: non-secret common variables and documented absolute host paths. +- `deploy/env/local.env.example` and `deploy/env/server.env.example`: profile-specific examples + containing names and safe defaults, never credentials. + +The standard commands are intentionally symmetric: + +```sh +docker compose -f compose.yaml -f deploy/compose.local.yaml build +docker compose -f compose.yaml -f deploy/compose.local.yaml up -d +``` + +```sh +docker compose -f compose.yaml -f deploy/compose.server.yaml build +docker compose -f compose.yaml -f deploy/compose.server.yaml up -d +``` + +Published images remain optional. A server can build from a release checkout or consume pinned +images through an operator override without changing the application architecture. + +## Configuration and secrets + +Portable workspace descriptors remain in the Git workspace registry. Installation-specific +endpoints, ports, usernames, CA paths, and secret-file bindings remain local. Secret contents are +mounted as files and never enter Git, browser drafts, image layers, Compose output, or generated +workspace artifacts. + +The same deterministic naming contract continues to apply: +`THT_WS___` for bindings and `_FILE`/`_SOURCE` for secret paths. The +base Compose accepts generic DWH, vector, embedding, LLM, Git, and session-storage endpoints; none +has a default hostname, path, or network associated with PSD or `omics_portal`. + +## Networking and exposure + +The base stack owns a private Compose network. `frontend` reaches `core` by service name. External +DWH, vector, Git, embedding, and LLM services are reached through operator-configured DNS names or +URLs. `host.docker.internal` may be documented as a workstation option but is not hard-coded as a +product dependency. + +The local profile binds user-facing ports to `127.0.0.1`. The server profile exposes only the +frontend host port needed by a generic reverse proxy. Examples for Nginx and Caddy document TLS, +forwarded identity, websocket/SSE behavior, and timeouts, but neither proxy is embedded into or +required by the core architecture. + +## Product boundary and external services + +DWH, VectorDB, and embedding services are always external to the ThothII product boundary and +Compose lifecycle. The mandatory ThothII stack neither defines nor starts them, never uses +`depends_on` for them, and does not assume their implementation, installation path, container +name, Docker network, or host. The same rule applies when all services happen to run on the same +physical server: from ThothII's perspective they remain independently operated services reached +through configurable addresses, ports, URLs, TLS settings, credentials, database/schema names, +and vector collections. + +An operator may address a co-resident service through a routable host address, a DNS name, a +documented host-gateway alias, or an explicitly configured external Docker network. None of these +becomes a product default. In particular, `127.0.0.1` inside `core` always means the core container, +not the Docker host; the installation guides must show the correct Mac/Windows Docker Desktop and +Linux server alternatives. + +LLM endpoints follow the same configurable external-service model, while the Pi coding-agent +runtime itself is part of ThothII as described below. + +## Embedded Pi runtime + +Pi is an internal runtime dependency of ThothII and is installed at a pinned version while building +the `core` image. The backend starts the image-bundled Pi executable; it never searches for or +bind-mounts a Pi installation from the host. Therefore a clean Windows PC, Mac, Linux workstation, +or server needs only Docker, Docker Compose, and Git to build and run ThothII. + +Pi configuration and provider credentials are supplied to the container through the documented +ThothII configuration and secret-file contracts. Pi writable state may use the ThothII-managed +`/home/thoth/.pi` volume, but the binary and package installation remain immutable image content. +Upgrading Pi requires changing the pinned build argument, rebuilding the image, and passing the +normal ThothII regression and image-smoke gates. The container health/smoke test verifies that Pi +exists in the image and can be invoked without any host executable. + +## Pi management for non-technical operators + +Pi management uses a hybrid interface so an operator does not need Docker expertise while image +updates remain reproducible and recoverable. Configuration and diagnostics are available in a +ThothII “Pi Management” page; host lifecycle and upgrades are performed by a small ThothII control +command named `thothctl`. + +The Pi Management page shows the bundled Pi version, runtime state, configured provider, default +model and reasoning level, writable Pi-state location, and sanitized diagnostics. It supports +editing non-secret installation defaults, selecting only supported values, validating free-form +fields, testing provider credentials without displaying them, running a Pi smoke request, and +viewing sanitized logs. Per-user model/reasoning preferences remain browser-local until an +authentication system provides durable user identities; installation defaults and Pi runtime +configuration are stored in the mounted ThothII settings/Pi volumes. + +The page may report that a newer supported Pi version exists, but it does not control Docker and +does not mutate the package inside a running container. It presents the exact `thothctl pi update` +command appropriate to the installation. On a loopback-only single-user installation, the normal +configuration functions are available. On a server, privileged Pi management requires trusted +upstream authentication/authorization; without it, privileged controls are disabled and host-side +`thothctl` remains the only update path. + +`thothctl` provides the stable operator commands: + +```text +thothctl status +thothctl doctor +thothctl logs +thothctl update +thothctl backup +thothctl restore +thothctl pi status +thothctl pi doctor +thothctl pi configure +thothctl pi test +thothctl pi update +thothctl pi logs +``` + +The tool wraps validated Docker Compose operations and uses the same behavior on Windows, macOS, +and Linux. Distribution may use a small native executable or platform launchers, but command names, +prompts, exit codes, backups, and rollback semantics are identical. Interactive configuration asks +plain-language questions, offers closed choices where possible, writes only local non-secret +configuration, and directs credentials into protected secret files. + +`thothctl pi update` never runs `npm install` in the live container. It checks compatibility, +records the current image/configuration, builds or pulls an image containing the selected pinned Pi +version, recreates `core`, verifies health and `pi --version`, runs a smoke request, and rolls back +to the recorded image if verification fails. The update preserves `/data` and `/home/thoth/.pi` +volumes and prints a concise recovery result. + +For advanced support, documentation may expose `docker compose exec core pi ...`, but no browser +shell is enabled by default. The `core` container never mounts the Docker socket. A future updater +service or web-triggered image update requires a separate authenticated design and is outside this +scope. + +## Persistence + +Named volumes are the portable default for workstation installations. Server documentation shows +explicit bind mounts under an operator-selected root such as `/srv/thothii`, with UID/GID and +backup requirements. The same container paths are used in both profiles: + +- `/data/workspace-registry` for Git checkout, immutable snapshots, leases, state, and locks; +- `/data/settings` for application settings; +- `/data/sessions` for filesystem sessions when selected; +- `/home/thoth/.pi` for Pi runtime state; +- `/run/secrets` for read-only secret files. + +Shared PostgreSQL session storage remains optional. It is required only when multiple ThothII +installations must see and resume the same sessions. + +## Cross-platform source and line endings + +The repository gains a root `.gitattributes` that makes line endings deterministic independently +of a developer's global Git configuration: + +```gitattributes +* text=auto +*.sh text eol=lf +Dockerfile* text eol=lf +*.Dockerfile text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.py text eol=lf +*.md text eol=lf +*.ps1 text eol=crlf +``` + +Executable shell scripts keep their executable bit and LF bytes. CI and a local verification +script scan Docker entrypoints, shell scripts, Compose/YAML, and Dockerfiles for carriage returns. +The Docker build also fails early with a clear message if an executable copied into an image has +CRLF. Documentation covers Git for Windows and WSL2, recommends cloning inside the WSL filesystem +for Linux-container work, and provides a safe one-time renormalization procedure for existing +clones. The documented process does not require changing global `core.autocrlf`. + +## Build and release behavior + +The local build uses the repository checkout as a BuildKit context and builds both images with +blocking TypeScript checks. The current non-blocking frontend typecheck is changed into a build +gate. A validation command renders each supported Compose combination before build. Image tags +include a local default and may be overridden with a release version or digest. + +Build inputs exclude `.git`, worktrees, local `.env` files, secrets, test output, caches, and +workspace runtime data through `.dockerignore`. Builds must work from macOS, Windows/WSL2, and +Linux without host-language runtimes or a host Pi installation beyond Docker, Compose, and Git. + +## Migration from the current deployment + +The current PSD/portal-oriented root Compose is replaced by the portable base. Reusable settings +from existing local and production overrides are folded into the new local/server overrides. +Portal network aliases, absolute Chirone paths, external `localllm_default`, PSD evidence mounts, +and `/datamart-builder` build arguments are removed from the product defaults. + +Existing operators migrate by copying only intentional values into the new environment and secret +files, rendering Compose, backing up volumes, building the new stack, and validating health and +workspace-registry status before switching the proxy. Legacy Compose examples remain in an +archive or are removed only after their replacement documentation and migration checks exist. + +## Error handling and operability + +Compose rendering fails when required non-secret values are absent. Entrypoints report stable, +sanitized errors for unreadable secret files, invalid registry configuration, incompatible line +endings, and storage permissions. Health checks distinguish process liveness from registry and +connector readiness. Remote outages preserve the last valid workspace snapshot as already +specified by the Git registry design. + +## Documentation + +Two complete guides are maintained against the same architecture: + +- local PC/Mac installation: Docker Desktop/Engine prerequisites, Windows/WSL2 line endings, + clone, environment creation, local build, first start, browser URL, update, backup, and recovery; +- server installation: service account, directories, firewall, generic reverse proxy, environment + and secrets, local image build or pinned image use, startup, health, upgrade, rollback, backup, + and recovery. + +Both guides include a non-technical operator section for `thothctl`, Pi configuration, Pi upgrade, +failed-upgrade rollback, and obtaining sanitized diagnostic output for support. Windows examples +use native PowerShell commands or a packaged executable rather than assuming a Unix shell. + +Both guides use copy-pastable commands validated by scripts. They explain which configuration is +shared in Git, which is installation-local, and how a server that only provides DWH/VectorDB is +consumed without installing ThothII there. + +## Verification strategy + +Automated gates cover Compose rendering for local/server plus each Git transport, LF enforcement, +Docker builds, container health, frontend-to-core same-origin routing, registry bootstrap and +offline fallback, secret non-disclosure, and absence of PSD/Chirone/portal dependencies in active +deployment files. Image tests also prove that the pinned Pi executable is available inside `core` +and that no host Pi path or Docker socket is mounted or required. Contract tests cover every +`thothctl pi` command, non-interactive exit codes, update rollback, volume preservation, secret +redaction, and parity of Windows/macOS/Linux launchers. Existing backend, frontend, harness, +registry, and installation-document tests remain required. + +Manual acceptance covers a clean macOS build, a clean Windows Docker Desktop/WSL2 build from a +GitHub clone, a Linux server deployment behind a generic proxy, an update after `git pull`, volume +persistence, and restoration from backup. + +## Non-goals + +- Bundling a DWH, production VectorDB, embedding server, LLM server, or reverse proxy into the + mandatory ThothII stack, even when those services are co-resident on the same physical host. +- Making PSD, Chirone, or `omics_portal` supported product profiles. +- Synchronizing local filesystem sessions between installations without shared session storage. +- Implementing persistent runtime SSH tunnels for workspace connectors in this migration. +- Providing a browser terminal or allowing the application container to control the Docker daemon. diff --git a/docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md b/docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md new file mode 100644 index 00000000..6ecd72b9 --- /dev/null +++ b/docs/superpowers/specs/2026-08-10-p2-p6-workspace-preprocessing-design.md @@ -0,0 +1,273 @@ +# P2–P6 Registry-Aware Workspace Preprocessing Design + +**Status:** Reviewed execution design; P1 automated and manual acceptance PASS +**Date:** 2026-08-10 +**Source:** `docs/prd/2026-08-09-workspace-preprocessing-prd.md` D2–D6 + +## 1. Goal and delivery protocol + +Connect the existing preprocessing engine to a revision-pinned schema-v3 Git workspace without +requiring Python, Node, or Pi on the host. Delivery remains on +`codex/git-workspace-registry`, with separately reviewable commits and a hard user checkpoint after +each of P2, P3, P4, P5, and P6. + +Each plan has its own clean-state automated process goal. Focused tests run while implementing each +plan; the aggregate P2–P6 Docker/process smoke and full repository verification run only after P6. +`docs/testing/p2-p6-manual-verification.md` is the single living manual walkthrough and records one +independent section and decision for each plan. + +## 2. Chosen architecture + +The existing native `thothctl` binary becomes the only host interface. New `workspace` commands use +the installation descriptor to reconstruct the exact Compose project and launch a one-shot +maintenance process from the selected `core` image. The process uses the installation's registry, +sessions, Qdrant, embedding, Git credentials, connector bindings, and secret mounts. It does not +call a running backend HTTP server and does not require a host language runtime. + +```text +thothctl --installation ... workspace + -> docker compose ... run --rm workspace-maintenance + -> compiled Node operator entrypoint in the core image + -> WorkspaceRegistry + runtime renderer + installation bindings + -> restrictive temporary harness config + -> existing tht commands + -> DWH / artifacts / Qdrant / internal Ollama +``` + +The operator entrypoint shares production classes with the backend, but is a separate process and +interface. It writes pristine JSON to stdout, bounded sanitized diagnostics to stderr, and never +returns a secret value or rendered secret-bearing transport URL. + +## 3. Common identity and persistence + +Every operation binds these values before doing work: + +- workspace ID; +- exact 40-hex active Git commit; +- catalog entry (`thoth-workspaces.yaml`) and exact canonical descriptor snapshot (the catalog blob + and descriptor blob at that same commit; a docs-only or content-only commit is still a distinct + revision even when the descriptor blob is unchanged, because the commit is authoritative); +- installation-local bindings resolved under configured secret roots; +- runtime roots beneath `/data/sessions/`; +- internal Qdrant/Ollama contract; +- the existing harness DWH ownership binding in P2 and its compatible, versioned P3 refinement. + +Mutable preprocessing state and outputs remain beneath the workspace runtime boundary. Operator job +state is an atomic, versioned JSON document under +`/data/sessions//preprocessing/`. It records the operation, revision, existing +harness ownership binding, child run IDs, completed stages, manual checkpoint, and terminal status. +P2 resume is same-revision only. P5 deliberately upgrades FK review to a controlled revision +transition and does not pretend an old same-revision resume token remains valid after a Git push. + +P3 separates reusable DWH-derived state (keyed by the revision-independent effective DWH binding) +from revision-scoped curated/semantic state. Schema/Evidence points and reads bind +`workspace_revision`; their point identities include that revision. Memory/solved records remain +workspace-wide. The harness config gains an explicit workspace-global `paths.memory` root rendered +as `/data/sessions//memory`; legacy configs without it continue to resolve memory +beneath `artifacts/memory` until explicitly migrated. All memory commands, locking, registry JSONL, +and projection rebuild use the explicit root when present. Revision-scoped artifact/corpus roots +therefore cannot split the canonical memory registry. DWH generations may be safely reused when +their existing effective DWH binding is unchanged. + +## 4. P2 — host preprocessing CLI + +### Command contract + +The initial public family is intentionally small: + +```text +thothctl ... workspace inspect --workspace [--json] +thothctl ... workspace preprocess dwh --workspace [--resume ] [--json] +thothctl ... workspace schema suggest-fks --workspace [tht-safe options] [--json] +thothctl ... workspace schema check --workspace [--json] +thothctl ... workspace index-schema --workspace [--json] +thothctl ... workspace preprocess evidence --workspace [--dry-run] [--resume ] [--json] +thothctl ... workspace preprocess run --workspace [--resume ] [--json] +``` + +`preprocess run` orders DWH introspection/LSH, FK review, schema indexing, and Evidence. When newly +suggested FK changes need human review it records `manual_review_required` and exits without +indexing schema or Evidence. A later explicit resume continues only after `schema check` succeeds. +A fixture with already-curated FK can complete without a human pause. + +P2 proves the complete chain with controlled REST DWH and HTTP Evidence fixtures. A filesystem +Evidence source is parsed and rendered but the operational command stops with the stable code +`evidence_materialization_required`; commit-addressed filesystem consumption belongs to P6. + +The command never edits or pushes the registry repository. Curators use an ordinary review clone. +P2 retains the existing engine's generation publication, idempotency, dry-run, and child resume +semantics rather than adding a second preprocessing engine. + +P2 includes the missing machine interfaces in the harness: JSON forms for FK suggest/check and +schema indexing, Evidence job identity from `runtime_identity.workspace_id` rather than a temporary +config filename, and bounded safe ingress/egress for `--from-sql` and annotation export. Host input +is a readable regular non-symlink file, is size-bounded by `thothctl`, and is streamed over the +one-shot process stdin rather than mounted as an arbitrary host directory. + +The one-shot job is a dedicated Compose profile/service, not `compose run core`. It has no Pi auth +mount, no writable Pi state, and an entrypoint that does not run Pi trust initialization. Git and +connector override generators attach only the credentials required by the selected workspace +operation to this service. + +## 5. P3 — effective configuration and `.tht-dwh` ownership + +P2 initially consumes the existing harness schema-v1 ownership contract unchanged. P3 makes that +contract reproducible across the operator and session paths without silently invalidating existing +generations. The current exclusion of `session_storage` and `runtime_identity` is preserved: a Git +content-only commit must not force DWH introspection when the effective DWH configuration is +unchanged. + +P3 introduces a versioned shared canonicalizer for the non-secret effective DWH/preprocessing +configuration and a stable logical config-source identity. It replaces dependence on random +temporary config filenames while retaining a compatibility reader and explicit migration for +existing `OWNER.json` schema-v1 roots. It also adds the explicit workspace-global memory root and +a migration that copies and verifies one legacy canonical JSONL under the workspace lock before +rebuilding its Qdrant projection; conflicting legacy registries fail closed. No in-place +reinterpretation is allowed. + +Reusable DWH cache roots are keyed by the versioned effective DWH binding. Revision-scoped runtime +roots receive verified physical/LSH snapshots from that cache, while annotations, corpus ACTIVE, +and schema/Evidence Qdrant records remain revision-specific. Qdrant schema/Evidence point IDs and +queries include `workspace_revision`; memory/solved identities and queries remain workspace-wide. + +P3 proves that the operator and a session render the same effective DWH binding, that semantically +identical revisions reuse it, and that a changed endpoint/transport/database/schema/root-affecting +policy fails closed. Documentation explains `.tht-dwh`, immutable generations, `OWNER.json`, +`ACTIVE`, input versus config fingerprints, safe migration, regeneration, and recovery. + +## 6. P4 — Qdrant collection lifecycle + +A shared TypeScript collection manager owns Qdrant collection and payload-index reconciliation. +Session admission and the operator path both call it. + +Self-heal may: + +- create a missing collection with exactly 1024 dimensions and cosine distance; +- create any missing required keyword index; +- tolerate an already-compatible concurrent creator and re-read final state. + +Self-heal never mutates incompatible dimensions, distance, or index types. Those return +`semantic_index_incompatible`. + +The host CLI adds guarded collection inspection and rebuild. Rebuild requires the exact workspace +ID, exact collection name repeated as confirmation, and an explicit destructive flag. It uses a +cross-process quiescence protocol rather than trusting the one-shot job: `thothctl` acquires the +installation lifecycle lock, asks the running backend to durably activate maintenance, verifies +the complete session inventory is closed/finalized/archived, and polls a new loopback-only internal +quiescence endpoint until both admission leases and `PiProcessManager.count()` are zero. It then +stops `core`, rechecks that the container is stopped, and starts the dedicated maintenance service. +The maintenance marker prevents a racing restart from admitting work. This backend-mediated drain +is an explicit exception to the ordinary preprocessing path's no-backend-HTTP rule. + +The job also verifies no preprocessing lock is held. It deletes only the descriptor-owned +collection, recreates the complete contract, verifies it, and emits JSON. No prefix matching or +global Qdrant mutation is allowed. A durable rebuild state is written before deletion; success +restarts core and clears maintenance only after health verification. Failure after deletion leaves +maintenance active and provides an explicit recovery/recreate command rather than claiming +rollback of lost vector data. + +## 7. P5 — curated FK annotations in Git + +The canonical path is fixed, not descriptor-configurable: + +```text +/schema/annotations.yaml +``` + +The registry validates that the object is a regular Git blob at the same commit as the descriptor. +Absence remains compatible and produces an empty canonical annotation set plus a warning until a +curator publishes one. Symlinks, submodules, trees at the file path, cross-namespace paths, and +malformed annotations are rejected. + +On activation and before preprocessing/session use, the exact blob is read with fixed Git argv, +validated by the harness annotation parser, and atomically synchronized to the immutable +revision-qualified runtime root: + +```text +/data/sessions//revisions//artifacts/mschema/annotations.yaml +``` + +P3 changes operator and session rendering so `artifacts` and `indexes` select that exact revision +root; the shared session-manifest root remains `/data/sessions//sessions`. Existing +workspace-global artifact roots are treated as legacy input and require the explicit P3 migration; +there is no mutable compatibility symlink or pointer used by pinned runtimes. + +The synchronized file has restrictive mode and an adjacent ownership manifest containing +workspace, commit, blob ID, content digest, and destination. A newer active revision writes a +different directory, so a pinned historical runtime continues to receive its own revision. +`physical.yaml` remains generated locally and is never published. + +The annotation blob is bounded (16 MiB), UTF-8, and parsed before synchronization. The +preprocessing CLI never pushes curated content. P2 same-revision local review is superseded in P5 +by an explicit controlled transition: after commit/push/pull, the operator reviews the current Git +blob against the recorded candidate, runs `workspace schema accept --run --yes`, and records +the accepted candidate/current-blob digests and new revision. Continuation requires that exact +accepted blob and compatible reusable DWH binding; otherwise it starts a new run. An empty file or +`schema check` alone is not evidence of human review. + +## 8. P6 — commit-addressed Evidence materialization + +For filesystem Evidence, the registry materializes exactly +`/evidence` from the pinned commit into an immutable revision content root. +It does not consume the mobile registry checkout and does not resolve against author files. + +Materialization uses fixed Git plumbing to enumerate object type, mode, path, object ID, and bytes. +It rejects every symlink at any depth, gitlink/submodule, device/FIFO/socket, unsupported mode, +absolute/traversing/non-normalized path, cross-workspace namespace, duplicate normalized path, +oversized individual source object, or object identity change. No archive is extracted by a shell. + +Files are written with no-follow/exclusive semantics beneath a fresh owned staging directory. +Every file is hashed and recorded in a bounded manifest. Installation-local non-secret limits bound +entry count, cumulative bytes, path/segment bytes, and manifest bytes; conservative defaults are +documented and may be raised deliberately for large repositories. Materialization streams blobs +and performs a disk-space preflight, so per-file-valid adversarial trees cannot exhaust memory or +inodes silently. The complete tree and manifest are fsynced and atomically renamed only after all +checks pass. A subsequent consumer revalidates destination ownership and the manifest before +reuse. Partial staging is removed without following links. + +The runtime renderer receives the verified immutable content root, after which the existing +filesystem Evidence adapter may discover only beneath that root. Corpus ACTIVE and Evidence vector +records are revision-scoped, and retrieval requires the pinned revision. Retention keeps +materialized and derived roots for every retained/pinned workspace revision and removes only +unreferenced, manifest-owned roots. P6 also makes P2's filesystem Evidence path operational and +removes the temporary stable stop. + +## 9. Error and output contract + +Stable public codes include: + +- `workspace_not_found` / `workspace_not_activatable`; +- `binding_missing`; +- `preprocessing_conflict`; +- `preprocessing_resume_mismatch`; +- `manual_review_required`; +- `evidence_materialization_required` (P2–P5 only); +- `effective_config_mismatch`; +- `semantic_index_incompatible`; +- `annotation_invalid`; +- `evidence_materialization_unsafe`. + +JSON output contains status, stable code, workspace ID, revision, operation/run ID, completed stage +names, counts, and safe artifact identities. It excludes endpoint credentials, secret contents, +query-bearing signed URLs, raw child stderr, and arbitrary exception strings. + +One writer lock per workspace serializes preprocessing, annotation synchronization, collection +rebuild, and materialization publication where they could conflict. Read-only inspection remains +concurrent. + +## 10. Verification and manual acceptance + +Each Px provides one clean-state process goal with unique ownership under `.artifacts/p-...`, +no retry, fixture-only secrets, machine/readable reports, secret scan, exact cleanup, and retained +`--keep` mode. Focused unit/integration/type/lint tests are evidence for that Px. After each Px the +user receives a report and an independently runnable manual section, and work stops for explicit +authorization. + +After P6, one aggregate test starts from a new Git registry and installation, executes P2 through P6 +with real local Git, REST fixtures, Qdrant, and Ollama, proves DWH/FK/schema/Evidence outputs, +repeats for idempotency, proves a second installation can consume the same Git workspace with its +own local state, exercises negative security cases, and cleans only owned resources. Only then are +the full harness/backend/frontend suites and builds run. + +A GUI/backend preprocessing endpoint, real PSD migration, SSH runtime transport, and policy-driven +long-term GC beyond the existing engine remain outside P2–P6. diff --git a/docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md b/docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md new file mode 100644 index 00000000..895e1bb7 --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-p1-1-workspace-directory-registry-design.md @@ -0,0 +1,255 @@ +# P1.1 Workspace-Directory Git Registry Design + +**Status:** Proposed for owner approval +**Date:** 2026-08-11 +**Supersedes:** The repository-layout and descriptor-publication portions of P1; P1's Evidence configuration, immutable revision, local-secret, rendering, and verification contracts remain in force. +**Deferred:** Any edits to P2–P6. Their impact will be planned only after P1.1 manual acceptance. + +## 1. Goal + +Make one shared Git repository read naturally as a catalog of self-contained workspaces. Each +workspace owns one directory containing its technical descriptor and, when Evidence is embedded, +its curated Evidence tree. A root catalog establishes the canonical workspace IDs, names, and +descriptions. ThothII may create a missing descriptor once as a bootstrap convenience, but after +that the descriptor is curator-owned and may be changed or removed only through ordinary Git +review and push. + +P1.1 is a correction to P1, not the preprocessing project. It performs no DWH introspection, +Evidence acquisition, materialization, embeddings, Qdrant writes, ACTIVE publication, FK curation, +or retention execution. + +## 2. Chosen repository contract + +```text +thoth-workspaces.git/ +├── thoth-workspaces.yaml +├── psd/ +│ ├── workspace.yaml +│ └── evidence/ +│ └── ... curated Evidence files ... +├── external-research/ +│ └── workspace.yaml # Evidence may instead be HTTP or S3 +└── workspace-docs/ + ├── psd/ + │ ├── contract.env.example + │ └── README.md + └── external-research/ + ├── contract.env.example + └── README.md +``` + +The fixed paths are: + +```text +catalog thoth-workspaces.yaml +workspace descriptor /workspace.yaml +embedded filesystem Evidence /evidence +future curated FK annotations /schema/annotations.yaml # P5, not P1.1 +public generated docs workspace-docs//{contract.env.example,README.md} +``` + +Internal installation snapshots deliberately remain flat: + +```text +/workspace-registry/snapshots//.yaml +``` + +This avoids changing ThtRunner's trusted-snapshot contract, historical session pins, runtime lease +files, or resume behavior. Repository layout and internal snapshot layout are separate contracts. + +## 3. Root catalog + +The curator owns `thoth-workspaces.yaml`. The API never creates, edits, deletes, stages, or cleans +it. Its strict initial shape is: + +```yaml +schema_version: 1 +workspaces: + - id: psd + name: Policlinico San Donato + description: Data warehouse clinico del Policlinico San Donato +``` + +Rules: + +- IDs use the existing `^[a-z][a-z0-9-]{2,62}$` contract, are unique, and cannot equal the reserved API directory `workspace-docs`. +- `name` is required; `description` is optional. Existing trim/nonblank, Unicode, and safe-error behavior used by descriptor metadata are reused; P1.1 introduces no new string-length limit. +- Unknown keys, duplicate YAML keys, aliases/tags, multiple documents, malformed encodings, and + duplicate IDs are rejected. +- Catalog order is the workspace display order. +- A catalog entry may temporarily have no descriptor. This is the only bootstrap state and is + represented publicly as `configuration_required`; it is not session-activatable. +- `workspace-docs` is a reserved top-level API directory and cannot be a workspace ID. +- The dedicated registry accepts only catalog-listed workspace directories plus the reserved + generated-docs directory and explicitly allowed root control files. An unlisted workspace + directory/descriptor, or a descriptor whose `workspace.id`, `workspace.name`, or optional + `workspace.description` differs from the catalog, is invalid. Activation fails atomically and + retains the prior valid snapshot. +- A present but empty or malformed descriptor is not "empty" for bootstrap. It is curator content + and is rejected; the API never replaces it. + +The descriptor retains `id`, `name`, and `description` so exports and immutable runtime snapshots +remain self-contained. The catalog is authoritative, and exact equality prevents two names for one +workspace. + +## 4. Ownership and write policy + +There are three writers with disjoint authority: + +| Path | Owner | ThothII API behavior | +| --- | --- | --- | +| `thoth-workspaces.yaml` | curator | read and validate only | +| `/workspace.yaml` | curator after bootstrap | create only if absent at the exact base commit; never overwrite or delete | +| `/evidence/**` | curator | read Git objects only; never write, stage, clean, or materialize in P1.1 | +| `workspaces//schema/**` | curator/future P5 | untouched by P1.1 | +| `workspace-docs//*` | API | deterministic generated files only | + +"Absent" means no Git object exists at `/workspace.yaml` in the exact pulled base +commit. A zero-byte file, comments-only YAML, symlink, submodule, tree, or malformed document counts +as present and is never overwritten. + +A browser/API bootstrap succeeds only when: + +1. the catalog entry already exists at the request's exact `baseCommit`; +2. the descriptor path is absent at that commit and remains absent after the pull; +3. request metadata exactly matches the catalog; +4. filesystem Evidence, when selected, already exists as a Git tree at + `/evidence` in that same base commit; +5. the complete descriptor passes schema-v3 and operational publication checks. + +The API then commits only the new descriptor and generated docs. Update and delete requests against +an existing descriptor return a stable `workspace_curator_owned` conflict response and do not +change any Git object. Curator deletion means removing the descriptor or catalog/directory through +Git. A retained session snapshot remains available under the existing retention rules. + +The managed checkout must no longer run a directory-wide clean under `workspaces/`. Failure cleanup +is confined to the exact descriptor/docs files written by the failed API operation and proves their +pre-operation identity before removal. + +## 5. Synchronization and generated docs + +Startup/bootstrap may pull, validate, and activate curator bytes but never pushes as a side effect +of a status/read request. This deliberately means committed `workspace-docs` can remain stale until +an explicit synchronization action; immutable local snapshots and exports always derive fresh docs +from the validated active descriptor and never consume stale Git docs. The explicit +`/workspace-registry/pull` operator action remains the synchronization boundary: + +1. pull the curator commit; +2. validate catalog, descriptors, namespace ownership, Evidence roots, and semantic-index ownership; +3. compute deterministic `workspace-docs/` bytes; +4. if docs differ, create one docs-only follow-up commit without touching catalog, descriptors, or + workspace content; +5. validate and activate the resulting exact commit. + +Every API write transaction records a bounded per-path journal before mutation: prior Git object type, +mode, blob identity and bytes for tracked generated docs, or explicit absence, plus the intended +post-write identity. If bootstrap/docs push races, is rejected, or fails, the prior valid active +snapshot remains active; overwritten/deleted generated docs are restored byte-for-byte to their +prior objects, newly created absent-before files are removed, and curator paths are never cleaned. +A later explicit pull retries from a fresh remote head. The API removes stale generated docs only for workspaces that the curator has +removed from the catalog or returned to `configuration_required`. + +A docs-only follow-up commit is an authoritative workspace revision, as every active workspace is +pinned to the complete Git commit rather than only to its descriptor blob. Automated acceptance +must show that descriptor and Evidence blob identities are unchanged across that docs-only commit. + +## 6. API and browser behavior + +`GET /workspaces` is catalog-driven and returns every catalog entry in catalog order with: + +- canonical ID, display name, and description from the catalog; +- `configurationState: ready | configuration_required`; +- `file: /workspace.yaml`; +- an immutable revision only for `ready` entries. + +`language` is deliberately not a summary field because an unconfigured catalog slot has no +descriptor language. It remains available from the descriptor detail for `ready` workspaces and is +selected in the bootstrap draft before creation. + +Descriptor-only routes (`GET /workspaces/:id`, diagnostics, export, session admission) reject a +`configuration_required` entry as `workspace_not_activatable`. + +`POST /workspaces/validate` remains a context-free schema check. It does not claim catalog +agreement or publication eligibility. `POST /workspaces/publish` becomes bootstrap-create only. +Legacy update/delete payloads are recognized and rejected as `workspace_curator_owned` rather than +silently reinterpreted. + +The browser: + +- lists catalog slots, including those requiring configuration; +- offers an editable, browser-local bootstrap draft only for `configuration_required` entries; +- locks catalog-owned ID/name/description in that form; +- requires explicit validation and confirmation before the one create; +- turns the workspace read-only immediately after creation; +- keeps Pull/Sync, Validate, installation Test, Export, and safe Evidence summary for existing + workspaces; +- removes update, delete, duplicate, field-conflict merge, and publish-existing controls; +- versions or purges old update/deletion drafts so stale localStorage cannot restore write access; +- treats imported bundles as bootstrap drafts only when they match an existing unconfigured + catalog slot. + +Existing descriptors are edited in the curator clone and become active after commit, push, and +installation pull. + +## 7. Evidence and external sources + +For filesystem Evidence, schema v3 now requires exactly: + +```yaml +evidence: + source: + type: filesystem + uri: /evidence +``` + +The lexical path invariant and same-commit Git-tree check remain P1.1 responsibilities. Recursive +materialization, nested symlink rejection, byte acquisition, preprocessing, and indexing remain P6 +or later. + +HTTP and S3 descriptor shapes, local `*_FILE` bindings, secret handling, timeout/limit policy, +runtime rendering, and `tht config check` remain as delivered by P1. Those workspaces need no local +`evidence/` directory. Evidence may also remain absent for compatibility. + +## 8. Rejected alternatives + +1. **Keep P1's three top-level source trees.** Rejected because it does not make a workspace a + self-contained Git unit and does not match the desired curator model. +2. **Place descriptors and Evidence together but let both API and curator update descriptors.** + Rejected because it creates two authorities, restores field-level conflict merging, and risks + overwriting reviewed Git content. +3. **Chosen: API bootstrap once, then curator ownership.** This preserves a convenient initial + form while making ordinary Git review the single authority for all subsequent descriptor and + content changes. + +## 9. Compatibility and migration + +P1.1 is a repository-contract cutover, not a dual-format reader. New code rejects the old flat +layout and a repository without `thoth-workspaces.yaml`. Existing repositories are migrated in one +curator-reviewed commit: + +```text +workspaces/.yaml -> /workspace.yaml +workspace-content//evidence/** -> /evidence/** +(create thoth-workspaces.yaml from reviewed descriptor metadata) +``` + +No automatic in-product migrator rewrites a remote. The installation upgrades only after the +migration commit is available. Historical immutable installation snapshots and retained session +pins keep their current internal shape. + +P2–P6 currently assume P1's old source paths in several places. P1.1 records that impact but does +not edit those plans. After P1.1 automated and manual acceptance, a separate owner-approved plan +will revise P2–P6. + +## 10. Verification boundary + +P1.1 must have independent automated and manual evidence. Accepted retained P1 artifacts remain immutable historical evidence; the old P1 process commands are not release gates for the superseding repository contract. The automated run starts from a clean +local Git remote and proves catalog authority, missing-descriptor bootstrap, curator modification, +API non-overwrite, nested Evidence identity, docs-only reconciliation, immutable snapshots, +runtime render determinism, `tht config check`, negative cases, secret absence, and exact cleanup. +It does not invoke preprocessing or Qdrant/Ollama/DWH services. + +The manual environment is new and independent. The reviewer personally performs the bootstrap, +refusal, curator-edit, pull/sync, UI read-only, Git-object, export, render, config-check, secret-scan, +and cleanup checks. Project state remains `P1.1 manual acceptance: PENDING` until the reviewer +records approval. diff --git a/docs/superpowers/specs/2026-08-14-pi-management-operator-workflow-design.md b/docs/superpowers/specs/2026-08-14-pi-management-operator-workflow-design.md new file mode 100644 index 00000000..fcf52e66 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-pi-management-operator-workflow-design.md @@ -0,0 +1,215 @@ +# Pi Management operator workflow design + +**Date:** 2026-08-14 +**Status:** Proposed + +## Context + +ThothII runs Pi only inside the Docker Compose `core` service. The current Pi Management dialog +mixes four different operations in one long instruction block: + +1. editing the host-side Pi provider catalog; +2. editing the host-side enabled-model policy; +3. selecting application defaults; and +4. upgrading the Pi version bundled in the `core` image. + +It also presents `thothctl pi configure` as mandatory even though the GUI already performs the same +provider/model/reasoning default update, repeats update guidance in a second section, copies an +incomplete `thothctl pi update` command, and exposes developer-only information about Vite and +frontend image rebuilding. + +There is also a real lifecycle gap. `thothctl pi update` safely replaces `core` when the Pi version +changes, but there is no Pi-specific command that reloads changed `models.json`, `settings.json`, or +credentials without changing the image. The documented fallback, `thothctl stop` followed by +`thothctl start`, restarts the whole installation. + +## Goals + +- Give a normal operator a short, structured, platform-specific workflow. +- Explain every operator-facing term before using it, especially `PI_AUTH_FILE`. +- Keep application defaults, Pi configuration files, credentials, configuration reload, and Pi + version updates conceptually separate. +- Add one safe command that recreates only `core` after host configuration or credentials change. +- Preserve the existing update transaction's session-drain, maintenance, verification, and + recovery guarantees. +- Remove duplicate, incomplete, native-Pi, developer-only, and raw-Compose guidance from the GUI. + +## Non-goals + +- The browser will not receive Docker access or a shell. +- The browser will not display or accept provider credentials. +- The GUI will not edit `deploy/pi/models.json`, `deploy/pi/settings.json`, or the credential file. +- `pi restart` will not build, pull, select, or upgrade an image. +- The general Pi internals document may continue to describe Pi's native paths, but it must clearly + state that ThothII operators edit the mounted host sources instead. + +## Operator concepts + +The revised interface will use the following terms consistently: + +- **Project root:** the ThothII checkout directory containing `compose.yaml` and the `deploy/` + directory. +- **Provider catalog:** `deploy/pi/models.json`. It declares provider endpoints and available model + metadata. A provider entry explains `baseUrl`, `api`, `models`, model `id`, and model `name`. +- **Enabled-model policy:** `deploy/pi/settings.json`. Its `enabledModels` array contains + `provider/model` identifiers that Pi is allowed to expose. +- **Application defaults:** provider, model, and reasoning stored in ThothII's persistent application + settings. The GUI's **Save defaults** action and `thothctl pi configure` are alternative interfaces + to this same setting; an operator does not run both. +- **Credential file:** the protected JSON file on the host whose location is assigned to + `PI_AUTH_FILE` in the installation environment file. Docker Compose mounts it read-only for Pi. + The GUI reports only whether a usable credential exists and never reveals its value. +- **Configuration reload:** recreation of the existing `core` container without changing its image. +- **Pi update:** replacement of the selected `core` image with an explicitly versioned build or an + immutable digest-pinned image. + +## New `thothctl pi restart` command + +### Interface + +```text +thothctl --installation pi restart --yes [--drain] +``` + +`--yes` is mandatory. Without `--drain`, the command refuses to proceed when active sessions +exist. With `--drain`, it closes new-session admission and waits until active sessions finish. +It never terminates active sessions merely because `--drain` was supplied. + +### Required behavior + +The command must: + +1. use the installation-aware Compose runner and durable current-image selector; +2. acquire the same exclusive lifecycle lock used by Pi update and rollback; +3. refuse to start when an interrupted update or restart requires recovery; +4. activate the durable maintenance gate before waiting for sessions; +5. validate the currently mounted Pi provider/model configuration before recreating `core`; +6. recreate only `core`, with `--no-deps`, `--force-recreate`, and a bounded health wait; +7. retain the exact currently selected image reference and never build or pull an image; +8. verify core health, bundled Pi version boundaries, mount/configuration identity, and the isolated + Pi/provider smoke after recreation; +9. clear maintenance and lifecycle state only after all verification succeeds; and +10. return sanitized, actionable failures without exposing credentials or raw configuration. + +If failure occurs before container mutation, the command clears maintenance and leaves the running +container untouched. If failure occurs after recreation, it leaves admission closed and records +recovery state. The operator repairs the reported host/Docker/configuration problem and uses the +documented maintenance recovery flow. The command must not silently claim success after a partial +restart. + +### Success output + +Success reports that the existing Pi image was retained, `core` was recreated, and readiness and +smoke checks passed. It does not print credentials or their contents. + +### Help and compatibility + +- `thothctl pi` usage text will list `restart --yes [--drain]`. +- Linux, macOS, and Windows builds expose identical command semantics. +- Existing `pi update`, `rollback`, `maintenance`, `doctor`, `test`, `logs`, and `configure` + behavior remains compatible. + +## Pi Management dialog redesign + +### Header instructions + +The update section begins with the exact short lead-in: + +> Using the host terminal: + +The existing Linux, macOS, and Windows tabs remain closed initially. Opening a tab shows an ordered +workflow made of short paragraphs, labels, lists, and code blocks rather than uninterrupted prose. + +Each tab contains: + +1. **Open the project root.** State that `deploy/` is directly in the ThothII project root, beside + `compose.yaml`. +2. **Edit the provider catalog.** Name the platform-appropriate path and explain the relevant + `models.json` fields in a compact definition list. +3. **Enable the model.** Name `deploy/pi/settings.json` and explain the `provider/model` values in + `enabledModels`. +4. **Set credentials.** Explain where to find `PI_AUTH_FILE`, what it points to, that the file stays + on the host, and how to protect it (`0600` on Linux/macOS, user-only ACL on Windows). Never show + real secret values. +5. **Reload configuration.** Show one platform-specific, directly executable `pi restart --yes + --drain` command using `~` for the user's home directory. +6. **Update the Pi version when needed.** Show one `pi update --version --source build + --yes --drain` command and explain that `` must be replaced with the desired pinned + version. Present digest-pinned `--source pull` as a clearly labelled advanced alternative, not + part of the normal path. +7. **Recover from an update failure.** Keep `pi maintenance status`, `pi logs`, `pi rollback --yes`, + and `pi maintenance recover --yes` in a compact secondary subsection. + +Linux and macOS use `~/bin/thothctl` and `~/thothii-installation.yaml`. Windows uses PowerShell, +`~\bin\thothctl-windows-amd64.exe`, and `~\thothii-installation.yaml` with `Resolve-Path` where +PowerShell requires expansion. + +### Application defaults + +The existing provider/model/reasoning form remains. Its description will say positively that it +selects defaults for new Pi work and stores no credentials. It will not tell the operator to run +`thothctl pi configure`; that command remains a terminal alternative documented outside the normal +GUI workflow. + +### Readiness, test, and diagnostics + +- Keep bundled Pi version, readiness sequence, **Save defaults**, and **Test saved defaults**. +- Keep the bounded sanitized diagnostics view. +- Rewrite descriptions into short, concrete sentences. Explain that diagnostics contain at most + 200 lines and omit declared secret values. + +### Removed UI + +- Remove “not this browser page”. +- Remove the duplicated bottom **Update Pi on the host** section. +- Remove the incomplete global `UPDATE_COMMAND` and its copy action. +- Remove the developer-only `:8080`/`:5173` and frontend rebuild note. +- Remove mandatory `pi configure`, explicit `stop`/`start`, and redundant post-update + `status`/`doctor`/`test` sequences from the platform tabs. +- Remove all native-Pi operator paths such as `~/.pi/agent/...` from the GUI. + +## Documentation changes + +- Update `docs/contracts/tht-pi.md` with the restart safety and recovery contract. +- Update `docs/install/pi-management.md` to separate GUI defaults, configuration reload, version + update, and failure recovery. +- Add a prominent ThothII-operator note to `docs/general/pi-configuration.md`: native Pi paths + describe container internals; operators edit `deploy/pi/...` and the host credential file. +- Update command/help verification scripts and any README command inventory that claims to list the + complete Pi lifecycle surface. + +## Testing + +### Go/CLI + +- Parser tests for required `--yes`, optional `--drain`, unknown flags, and extra arguments. +- Restart refuses active sessions without `--drain` and waits with it. +- Restart uses the durable current-image selector and recreates only `core` without build or pull. +- Pre-mutation validation failure leaves the container untouched and clears maintenance. +- Post-mutation verification failure leaves safe recovery state and maintenance active. +- Success verifies health/version/configuration/smoke and clears maintenance. +- Concurrent update/restart/rollback operations share the lifecycle lock. +- Output and failures remain sanitized on Linux and Windows paths. + +### Frontend + +- All platform tabs start closed. +- Each platform shows structured Docker-only instructions and its executable restart command. +- `PI_AUTH_FILE`, `models.json`, and `settings.json` are explained in operator language. +- No native-Pi paths, duplicated update section, incomplete copy command, or developer-only port + guidance remains. +- Existing save, test, readiness, scrolling, and diagnostics behavior remains covered. + +### Verification + +- Run all `tools/thothctl` Go tests and build the supported binaries. +- Run relevant documentation/command-contract scripts. +- Run the complete frontend test suite, TypeScript build, and production bundle build. +- Rebuild only the frontend service and verify the revised bundle and healthy service on port 8080. + +## Rollout and recovery + +The frontend change is independently deployable, but it must not advertise `pi restart` until the +corresponding `thothctl` binary has been built and made available to operators. Existing commands +remain unchanged. If deployment of the new binary is deferred, the GUI must retain the prior +supported stop/start fallback rather than display a nonexistent command. diff --git a/docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md b/docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md new file mode 100644 index 00000000..1ff3dd47 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-thothctl-discovery-and-pi-update-design.md @@ -0,0 +1,44 @@ +# Simplified `thothctl` installation selection and Pi update design + +## Decision + +Keep every existing `thothctl pi` subcommand. Make the installation descriptor optional on the command line and resolve it automatically when the operator runs from the project tree. Keep `--installation ` as an explicit override for non-standard locations or multiple installations. + +The normal Pi update becomes: + +```sh +thothctl pi update +``` + +When `--version` is omitted, `thothctl` reads the single default `ARG PI_VERSION=` from `docker/core.Dockerfile` in the selected installation's project directory and uses that pinned version with the existing transactional build/update path. An explicit `--version ` remains supported. The command does not fetch an arbitrary npm “latest”; the repository pin, lockfile, image labels, and executable must remain consistent. + +## Installation resolution + +Resolution order is: + +1. an explicit `--installation /thothii-installation.yaml`; +2. `THOTHII_INSTALLATION`, when set to an absolute descriptor path; +3. automatic discovery from the current working directory and its parents. + +Automatic discovery examines only the exact descriptor at each directory level and the immediate `deploy/*/thothii-installation.yaml` locations. It never recursively scans `.artifacts`, home directories, or unrelated descendants. A candidate must be a regular file and must pass `config.Load`. One valid candidate is selected. No candidates or multiple valid candidates produce an actionable error that names the expected locations and explains how to use `--installation`. + +The resolver is shared by all existing top-level commands, not only `pi`, so `thothctl status`, `start`, `stop`, `doctor`, `workspace`, and the Pi commands have the same invocation rules. Existing explicit invocations remain valid. + +## Safety and compatibility + +- No existing Pi subcommand is removed or renamed. +- Existing advanced `pi update --source ... --image ... --yes --drain` syntax remains accepted for compatibility. +- The short update path selects build mode and preserves the existing lifecycle lock, maintenance gate, session handling, candidate verification, image selector promotion, and rollback/recovery behavior. +- The default update uses the current checkout's declared Pi pin; changing to a newer Pi release still requires updating the repository pin and lockfile in the normal source-update workflow. +- Errors and automatic-discovery diagnostics never expose secret file contents. + +## User-facing examples + +```sh +thothctl pi update +thothctl pi update --version 0.81.0 +thothctl pi status +thothctl pi restart --yes --drain +thothctl --installation ~/operator/thothii-installation.yaml pi update +``` + diff --git a/docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md b/docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md new file mode 100644 index 00000000..aee2b137 --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-unified-tht-cli-product-step-design.md @@ -0,0 +1,235 @@ +# Unified `tht` CLI and Product Setup Design + +Date: 2026-08-15 + +## Objective + +Turn the repository into a product that can be cloned, bootstrapped once, and then operated with a +single normal system command named `tht`. The operator must not build Go manually, create a `bin` +directory, remember an installation descriptor path, or use raw Docker commands for normal +installation, lifecycle, Pi updates, backup, or restore. + +## Confirmed decisions + +1. The only product command name is `tht`. +2. `thothctl` is removed completely. There is no compatibility alias, wrapper, deprecation period, + or second installed command. +3. The host operator implementation remains a native Go binary so macOS, Linux, and Windows hosts + do not require Go, Python, or a virtualenv. +4. The existing Python workflow CLI remains inside `core` under the same command name `tht`. + Backend and Pi continue to use it there. It is not installed on the host and is not presented in + operator documentation. No `tht-runtime` command or namespace is introduced. +5. The command audit is binding: 55 commands are `MAINTAIN`, 8 are `ENHANCE`, and 14 are `ERASE`. + The detailed decision matrix is in + `docs/reports/2026-08-15-tht-command-maintain-erase-enhance.md`. +6. Workflow commands called by Pi, the gate, the backend, workspace maintenance, or the session + migrator are machine contracts. Their semantics, pristine JSON output, stdin behavior, and exit + codes are not changed merely to simplify the operator help. +7. The public operator help stays small. Internal workflow commands do not appear in host help. +8. `--installation` remains available as an optional override. Normal commands discover the + installation from the current project root or worktree. +9. `backup` and `restore` are introduced as first-class product commands. +10. The final implementation is installed on this Mac and used to rebuild/restart the live stack on + port 8080. + +## Command boundary + +### Host operator CLI + +The native host command exposes: + +```text +tht setup +tht version +tht start [--build] +tht stop +tht status +tht doctor [--json] +tht logs +tht update [--check-only] [--yes] [--drain] +tht backup [--output PATH] [--include-secrets --yes] [--drain] +tht restore ARCHIVE --yes [--drain] +tht sessions migrate --yes +tht remove [--yes ID...] +tht pi ... +tht workspace ... +``` + +`tht pi` preserves `status`, `doctor`, `test`/`check`, `configure`, `restart`, `update`, +`rollback`, `maintenance`, and `logs`. `tht workspace` preserves every currently implemented +workspace operation, including the two vector operations missing from the current help. + +### Container workflow CLI + +The Python command tree remains the deterministic protocol used by Pi and the backend. The host +installation does not expose these commands as operator shortcuts. This prevents a human operator +from bypassing reviewer gates while avoiding a risky rewrite of the live workflow. + +The `MAINTAIN`, `ENHANCE`, and `ERASE` decisions apply exactly as recorded in the command audit. +`MAINTAIN` commands retain their current path. `ENHANCE` commands receive the approved safety or +diagnostic improvements. `ERASE` commands disappear from Typer registration and active docs; domain +functions still used by canonical pipelines remain internal libraries. + +## Bootstrap and PATH installation + +A freshly cloned repository necessarily needs one bootstrap action before `tht` exists: + +```bash +bash scripts/install-tht.sh +``` + +Windows uses: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/install-tht.ps1 +``` + +The scripts require Docker, build the correct native binary using the repository-pinned Docker +builder, verify it, and install it atomically: + +- macOS and Linux: `/usr/local/bin/tht`, requesting elevation only for the final atomic install; +- Windows: `%LOCALAPPDATA%\ThothII\bin\tht.exe`, adding that directory to the user PATH when needed. + +The user never selects a platform binary, runs Go, creates a `bin` directory, or invokes a +project-relative executable. Re-running the installer upgrades the installed command idempotently. + +## Project and worktree discovery + +Every host command starts from the current working directory and walks parents until it finds the +ThothII root contract (`compose.yaml`, `deploy/`, and the repository marker). A Git worktree root is +treated exactly like the main checkout. + +Installation resolution order is: + +1. explicit `--installation PATH`; +2. `THOTHII_INSTALLATION`; +3. one valid `thothii-installation.yaml` in the current directory or immediate `deploy/*`; +4. the same search while walking parent directories. + +Zero candidates produce a setup-oriented error. Multiple candidates produce a bounded list and +require `--installation`; no arbitrary recursive search is allowed. + +## `tht setup` + +`tht setup` is idempotent and defaults to the local profile on macOS, Windows, and workstation +Linux. `--profile server` selects server behavior. Generated installation files live under +`deploy//`, where `deploy` is explicitly documented as a directory in the project +or worktree root. + +The setup flow: + +1. verifies root/worktree identity, Docker, Compose, supported architecture, and line endings; +2. creates or validates the installation descriptor and non-secret environment files; +3. asks plain-language questions and stores secret file paths, never secret values in the + descriptor; +4. creates protected secret-file templates only after explicit confirmation and never overwrites an + existing file; +5. renders and validates Compose configuration; +6. builds the ThothII images from the current checkout; +7. starts the stack with `docker compose up --detach --remove-orphans`; +8. waits for bounded health checks; +9. runs installation diagnostics and Pi diagnostics; +10. prints the URL and the exact descriptor selected. + +`tht setup --configure-only` stops after validated configuration. `tht setup` never silently +replaces a descriptor, environment file, secret file, or generated state belonging to another +installation. + +## Lifecycle and product update + +- `tht start` starts the selected installation without rebuilding. +- `tht start --build` builds current-checkout images before startup. +- `tht stop` stops the installation while preserving state. +- `tht update --check-only` retains its current non-mutating validation behavior. +- `tht update` becomes the complete product update: lifecycle lock, active-session check/drain, + backup checkpoint, image build from the current checkout, controlled recreation, health checks, + diagnostics, and rollback to the recorded images when verification fails. + +Product update and Pi update remain separate. `tht update` updates ThothII. `tht pi update` updates +only Pi in `core`. + +## Pi management + +`tht pi update [--version VERSION]` makes the version optional. Without `--version`, it queries the +latest stable version of the pinned Pi package from the authoritative package registry. A lookup +failure stops before mutation and tells the operator to retry or supply `--version`; it never +silently substitutes an older pin. + +The command builds or pulls a candidate, verifies the Pi executable, `PI_VERSION`, and image label, +recreates only `core`, checks health, runs the smoke test, preserves volumes, and rolls back on +failure. Interactive terminals receive one clear confirmation; non-interactive execution requires +`--yes`. Model/provider selection remains the responsibility of `tht pi configure` and is not a +required argument to Pi update. + +The project release pin in `docker/core.Dockerfile` remains the clean-build default. Installation +update state records the selected newer image/version so ordinary restart does not revert it. + +## Backup and restore + +`tht backup` creates a versioned manifest and checksummed archive in +`~/.thothii/backups//` unless `--output` is supplied. It acquires the lifecycle +lock, refuses active work unless `--drain` is accepted, obtains a consistent stopped snapshot, and +restarts/verifies a previously running installation. + +The backup includes: + +- installation descriptor, non-secret environment/configuration, generated overrides, and source + revision metadata; +- installation-owned settings, Pi state, workspace registry, workspace secrets volume, sessions, + Qdrant data, and embedding-model volume; +- server bind roots returned by the installation preservation contract; +- a manifest of external secret-file paths and digests. + +Secret-file contents are excluded by default. `--include-secrets --yes` includes them and marks the +archive sensitive; the file is created with owner-only permissions. A backup without secrets is +restorable only when all referenced secret files still exist and match preflight requirements. + +`tht restore ARCHIVE --yes` validates schema version, checksums, installation identity, target +ownership, secret prerequisites, disk space, and stopped/quiescent state before mutation. It creates +a rollback checkpoint, restores only manifest-listed paths/volumes, starts the stack when it was +previously running, and runs health, `doctor`, `pi doctor`, and workspace inspection. Failure keeps +the target in a recoverable stopped state and prints the checkpoint path. + +## Pi Management frontend + +The frontend uses only Docker-based instructions and only the command `tht`. The section: + +- begins fully collapsed; +- uses separate Linux, macOS, and Windows environment panels, with none open initially; +- is shorter than the current panel and has a working vertical scrollbar; +- begins with the exact concise wording `Using the host terminal`; +- explains that `deploy` is a directory in the project/worktree root beside `compose.yaml`; +- explains that `deploy/pi/models.json` and `deploy/pi/settings.json` are host files mounted + read-only into `core`, so the operator edits the host files, not files inside the container; +- explains credential-file concepts in plain language rather than presenting environment-variable + names without context; +- shows direct commands such as `tht pi configure`, `tht pi restart`, `tht pi update`, + `tht pi status`, `tht pi doctor`, and `tht pi test`; +- contains no Go build, `~/bin`, `./bin`, `./tht`, `thothctl`, or mandatory descriptor path. + +The sanitized-log control must either display the bounded, sanitized `core` log response or fail +with a visible error. The live model selector must reflect the mounted Pi configuration, including +the existing GLM 5.3 change after `core` is recreated. + +## Documentation + +Active user, installation, architecture, CLI-contract, testing, and README documentation is +rewritten around the bootstrap-plus-setup flow and the direct `tht` command. Historical +`docs/superpowers` plans/specs remain historical records; the new spec supersedes them. + +Documentation examples run from the project/worktree root, refer to the home directory as `~`, and +do not teach manual Go builds or manually constructed installation paths for normal use. Advanced +sections may document optional `--installation` and non-interactive flags. + +## Safety and acceptance + +- Existing user changes to `deploy/pi/models.json` and `deploy/pi/settings.json` are preserved. +- Unrelated dirty-worktree files are not overwritten or committed accidentally. +- JSON contracts remain pristine and secrets are sanitized from stdout, stderr, logs, archives, and + failure messages. +- Tests cover macOS/Linux shell installation, Windows PowerShell installation, root/worktree + discovery, setup idempotency, lifecycle rollback, Pi latest-version lookup, command audit, + backup/restore, frontend layout/copy, and active-document command examples. +- Final acceptance installs `tht` on this Mac, verifies `command -v tht`, confirms `thothctl` is + absent, updates the live stack, checks healthy services, opens port 8080, verifies Pi Management, + and confirms GLM 5.3 is selectable. diff --git a/docs/superpowers/specs/2026-08-16-thothii-authentication-design.md b/docs/superpowers/specs/2026-08-16-thothii-authentication-design.md new file mode 100644 index 00000000..35787b19 --- /dev/null +++ b/docs/superpowers/specs/2026-08-16-thothii-authentication-design.md @@ -0,0 +1,495 @@ +# ThothII Authentication Design + +## Objective + +Add first-party authentication and authorization to ThothII without bundling an identity +manager. The same product build must support: + +1. a standalone PC or Mac with local users stored in protected operator files; and +2. a server deployment using a standards-based OIDC provider, with Authentik as the first + certified provider for the PSD integration. + +The design replaces implicit trust in a local browser with an explicit authenticated session, +keeps the host operator interface exclusively under `tht`, and extends the existing workspace +validation and installation diagnostics with authentication checks. + +## Confirmed decisions + +- The only host CLI is `tht`. No `thothii-admin`, `thothctl`, or separate authentication binary + is introduced. +- Production authentication modes are `local` and `oidc`. +- `none` and `mock` remain test/development-only. Existing `upstream` support remains as a + deprecated migration adapter until direct OIDC deployment has passed PSD acceptance. +- Local passwords use Argon2id. Passwords are never stored or logged in plaintext. +- A local user can select **Remember me**. The resulting session survives browser and ThothII + restarts until its idle or absolute expiry. +- OIDC login is provider-neutral. Authentik is the provider certified by automated and PSD + acceptance tests in the first release. +- OIDC must return a `groups` claim in the ID token as a JSON array of strings. Missing, + malformed, indirect, or overage-style group claims fail authentication. +- OIDC groups are mapped to ThothII roles in installation configuration. Unmapped groups are + ignored silently, without errors or warnings. +- Every configured group must be proven to exist in the identity manager. OIDC itself cannot + enumerate groups, so this proof uses a provider-specific group-catalog adapter. The first + adapter is `authentik`. +- Authentication configuration is installation-global. Workspace validation still reports its + status and workspace connection tests include its live connectivity checks. +- Browser tokens are never stored in `localStorage`, `sessionStorage`, or JavaScript-readable + cookies. The browser receives only an opaque `HttpOnly` session cookie. + +## Non-goals for the first release + +- Bundling Authentik, Keycloak, LDAP, or another identity manager with ThothII. +- Implementing LDAP authentication directly in ThothII. +- Persisting OIDC access, ID, or refresh tokens after login. +- Supporting arbitrary provider management APIs through a user-programmable HTTP adapter. +- Building a web UI for local-user administration. Local users are managed through `tht auth`. +- Adding fine-grained workspace-specific ACLs. Authorization remains installation-wide. +- Guaranteeing group-catalog validation for an arbitrary OIDC provider without a supported + catalog adapter. + +## Runtime architecture + +The Fastify backend owns authentication, browser sessions, and authorization. The React frontend +only renders login state and sends same-origin requests. The Python harness receives a trusted, +already-authorized principal from the backend exactly as it does today. + +```text +browser + -> local login -----------------------> Fastify auth service + -> OIDC Authorization Code + PKCE ----> Fastify auth service ----> OIDC provider + | + +--> durable opaque sessions + +--> group -> role -> permission mapping + +--> authorized ThothII routes + +tht auth / tht doctor + -> host configuration and local-user files + -> container-local AuthDiagnoser + -> OIDC discovery/JWKS + -> Authentik group API +``` + +## Installation files and storage + +Every installation descriptor gains one non-secret authentication location: + +```yaml +authentication: + configDirectory: /absolute/operator-controlled/thothii-auth +``` + +The directory is mounted read-only into `core` at `/run/thothii-auth`. It contains: + +```text +auth.yaml non-secret mode, URL, lifetime, OIDC, and group-role configuration +users.yaml local user IDs, Argon2id hashes, roles, enabled state, and auth revision +``` + +The host directory must be private to the operator. On POSIX its mode is `0700`, and both files +are regular, single-link, non-symlink files with mode `0600`. Windows uses an equivalent +owner-only ACL. `tht` performs bounded reads, known-field YAML decoding, and atomic replacement. + +Durable browser session state lives under `/data/auth`, backed by a new Compose volume named +`auth-state` for local installations and by the existing server `/data` bind for server +installations: + +```text +/data/auth/sessions/.json +/data/auth/oidc/.json +``` + +Raw cookie tokens, derived CSRF tokens, and raw OIDC tokens are never written to disk. Session +files contain only the principal, authorization snapshot, timestamps, and revision numbers. + +## Authentication configuration contract + +`auth.yaml` is strict, versioned YAML. The local form is: + +```yaml +version: 1 +mode: local +publicUrl: http://127.0.0.1:8080 +session: + regularTtlSeconds: 43200 + regularIdleSeconds: 7200 + rememberTtlSeconds: 2592000 + rememberIdleSeconds: 604800 + oidcTtlSeconds: 28800 +local: + usersFile: users.yaml +``` + +The OIDC/Authentik form is: + +```yaml +version: 1 +mode: oidc +publicUrl: https://thothii.example.org +session: + regularTtlSeconds: 43200 + regularIdleSeconds: 7200 + rememberTtlSeconds: 2592000 + rememberIdleSeconds: 604800 + oidcTtlSeconds: 28800 +oidc: + issuer: https://authentik.example.org/application/o/thothii/ + clientId: thothii + clientSecretRef: THT_OIDC_CLIENT_SECRET + scopes: + - openid + - profile + - email + groupsClaim: groups +groupCatalog: + driver: authentik + baseUrl: https://authentik.example.org + apiTokenRef: THT_AUTHENTIK_API_TOKEN +authorization: + groupRoles: + TOT Users: + - user + TOT Admin: + - admin +``` + +`publicUrl` has no path, query, fragment, or embedded credential. OIDC mode requires HTTPS except +for an explicit loopback test configuration. The callback is always +`/api/auth/oidc/callback`; arbitrary redirect URIs and arbitrary post-login redirects +are not accepted. + +The referenced secret names are read from the existing mounted ThothII secret bundle. Secret +values never appear in `auth.yaml`, command arguments, JSON diagnostics, logs, or frontend data. + +## Local-user registry + +`users.yaml` has this exact shape: + +```yaml +version: 1 +users: + - id: 6ba7b810-9dad-4ed1-80b4-00c04fd430c8 + username: admin + displayName: Local administrator + passwordHash: $argon2id$v=19$m=65536,t=3,p=1$AAECAwQFBgcICQoLDA0ODw$DRo8ZSPI8G5OCvnFFapbVEjP69aDjy1Sw9i2743cPC4 + roles: + - admin + enabled: true + authRevision: 1 +``` + +- `id` is an immutable UUIDv4 generated by `tht` and is the local OIDC-like `subject`. +- `username` is 3–64 ASCII characters, starts with an alphanumeric character, and then uses only + alphanumerics plus `.`, `_`, `@`, or `-`. Lookup is ASCII case-insensitive while display spelling + is retained. `displayName` remains unrestricted Unicode text after control-character rejection. +- Passwords are accepted from a TTY or an explicit `--password-file`, never a command argument. +- Password length is 12 to 1024 UTF-8 bytes. The upper bound prevents accidental memory abuse; + no composition rule is imposed. +- New hashes use Argon2id v19 with 64 MiB memory, three passes, parallelism one, a 16-byte random + salt, and a 32-byte output. Parameters remain embedded in the PHC string for future rehashing. +- Every password, role, enabled-state, or logout-all change increments `authRevision`. +- At least one enabled administrator must remain. Commands that would disable or demote the last + administrator fail before writing. + +The Go CLI hashes with `golang.org/x/crypto/argon2`; Node verifies with the Node 24 native Argon2 +API. Shared fixed test vectors prove cross-language compatibility. + +## Roles and permissions + +External groups and local records map to two stable roles: + +```text +user + session.use + +admin (inherits user) + session.read_all + session.manage_all + settings.manage + workspace.manage + workspace.secrets.manage + pi.manage + auth.diagnostics.read +``` + +Session ownership remains enforced through `issuer + subject`. `session.use` never permits access +to another user's session. `isAdmin` remains temporarily available as the derived value +`roles.includes("admin")` for compatibility, but route authorization is based on permissions. + +Route policy is: + +| Surface | Required authority | +|---|---| +| `/health`, local login, OIDC start/callback | Public protocol endpoint | +| `/me`, effective settings/models, workspace read, own sessions and SSE | Authenticated `user` | +| `scope=all`, operations on another user's session | `session.read_all` / `session.manage_all` | +| Settings writes | `settings.manage` | +| Workspace registry pull/bootstrap/validation/test | `workspace.manage` | +| Workspace secret writes/deletes | `workspace.secrets.manage` | +| Pi management routes | `pi.manage` | +| Authentication diagnostics | `auth.diagnostics.read` or host operator through `tht` | + +An OIDC user whose token contains no mapped group is authenticated but receives no role. Protected +application routes return `403` with `code: "auth_not_authorized"`. + +## Durable browser sessions + +Successful authentication creates a cryptographically random 256-bit cookie token. The server +stores only its SHA-256 digest as the session filename. The record contains: + +```ts +interface AuthSessionRecord { + version: 1; + issuer: string; + subject: string; + displayName?: string; + method: "local" | "oidc" | "upstream"; + roles: readonly ("user" | "admin")[]; + permissions: readonly Permission[]; + userAuthRevision?: number; + authConfigRevision: string; + remembered: boolean; + createdAt: string; + lastSeenAt: string; + idleExpiresAt: string; + absoluteExpiresAt: string; +} +``` + +Session behavior is: + +- Ordinary local login: session cookie with no `Max-Age`, 2-hour idle expiry, 12-hour absolute + expiry. Closing the browser removes the browser cookie. +- Local **Remember me**: persistent cookie, 7-day idle expiry, 30-day absolute expiry. It survives + browser and ThothII restarts. +- OIDC: maximum eight-hour ThothII session, never longer than the validated ID-token expiry. The + identity provider may independently remember its SSO login. +- `lastSeenAt` is written at most once every five minutes to bound filesystem writes. +- Expired records are pruned at startup and every fifteen minutes. +- Local sessions are invalid as soon as `authRevision` differs, the user is absent/disabled, or + the configured role set changes. +- All sessions are invalid on the first request after `authConfigRevision` differs following an + authentication configuration reload. +- Logout deletes the server record and expires the cookie. +- Backup restore intentionally invalidates remembered sessions; session records are not restored + as active credentials. + +The cookie is named `thothii_session`, is `HttpOnly`, `SameSite=Lax`, `Path=/`, has no `Domain`, +and uses `Secure` whenever `publicUrl` is HTTPS. Local loopback HTTP deliberately omits `Secure` so +the browser can use the cookie. + +## CSRF and browser boundary + +Cookie authentication makes CSRF protection mandatory for every state-changing application +route. The backend derives a separate 256-bit CSRF token from the raw session cookie with +domain-separated HKDF-SHA-256; the derivation is one-way and nothing additional is persisted. +`/me` returns the derived token to the same-origin frontend, which keeps it in memory and sends it +as `X-ThothII-CSRF` for `POST`, `PUT`, `PATCH`, and `DELETE` requests. + +The backend requires all of the following for a cookie-authenticated state change: + +1. a valid session; +2. a constant-time match of the CSRF token; +3. a matching `Origin` when the browser supplies one; +4. `Sec-Fetch-Site: same-origin` when Fetch Metadata is present. + +The local login POST requires a same-origin `Origin`; OIDC login uses OIDC `state`, `nonce`, and +PKCE. Frontend and API are supported as one browser origin. Development uses a Vite `/api` proxy +rather than credentialed cross-origin requests. + +## Local authentication flow + +The backend exposes: + +```text +GET /auth/config public mode and login capabilities, no secrets +POST /auth/local/login username, password, remember +POST /auth/logout authenticated + CSRF +GET /me principal, roles, permissions, CSRF token +``` + +Login errors use one generic `invalid_credentials` response for unknown, disabled, and wrong- +password users. A dummy Argon2 verification runs for unknown users. Rate limits apply per source +address and normalized username, and concurrent Argon2 operations are bounded. + +## Generic OIDC flow + +OIDC uses `openid-client` 6.8.5, Authorization Code Flow, PKCE S256, `state`, and `nonce`. The +backend performs provider discovery, validates issuer, signature, audience, expiry, nonce, and +authorization response, then reads the ID-token claims. + +`groupsClaim` is mandatory and resolves to a direct array of non-empty strings. The first release +does not follow distributed claims, provider overage links, or Graph-style group expansion. Such a +token fails with `oidc_groups_claim_invalid` rather than silently granting ordinary access. + +Mapped roles are the union of all exactly matched group names. Additional token groups are ignored +without logging or warnings. No OIDC token is sent to the frontend or persisted after principal +and session creation. + +## Authentik group-catalog adapter + +The certified Authentik adapter calls its documented API with a dedicated service-account bearer +token. For each configured mapping key it requests: + +```text +GET /api/v3/core/groups/?name=&include_users=false&page_size=2 +``` + +The adapter requires exactly one exact-name result. Zero results produce +`oidc_mapped_group_missing`; more than one produces `oidc_mapped_group_ambiguous`. It never +enumerates or compares unrelated groups, so extra Authentik groups produce neither warning nor +error. + +Outbound requests use HTTPS, fixed operator-controlled origins, five-second timeouts, no redirect +following, bounded JSON bodies, and redacted errors. The API token has only group-view permission +and is distinct from the OIDC client secret. + +Authentik is connected to LDAP in PSD, but ThothII validates the groups visible in Authentik. A +group present only in LDAP and not represented in Authentik is correctly treated as missing. + +## Authentication diagnostics + +One backend `AuthDiagnoser` returns a redacted machine contract: + +```ts +interface AuthDiagnostics { + ready: boolean; + mode: "local" | "oidc" | "upstream" | "none" | "mock"; + checks: readonly AuthDiagnostic[]; +} + +interface AuthDiagnostic { + level: "error" | "info"; + code: AuthDiagnosticCode; + message: string; + field?: string; +} +``` + +`AuthDiagnosticCode` is the closed union: + +```text +auth_ready +auth_config_incomplete +auth_config_invalid +auth_session_store_invalid +local_user_registry_invalid +local_admin_missing +oidc_secret_missing +oidc_discovery_unreachable +oidc_issuer_mismatch +oidc_jwks_unreachable +oidc_group_catalog_unreachable +oidc_group_catalog_unauthorized +oidc_mapped_group_missing +oidc_mapped_group_ambiguous +oidc_groups_claim_invalid +oidc_device_flow_unavailable +``` + +Static checks cover configuration, file safety, secret references, local administrators, role +names, group mappings, URL policy, and session storage. Live OIDC checks cover discovery, issuer, +JWKS, Authentik API authentication, and every configured group. + +The same implementation is consumed by: + +- application startup for fatal static configuration errors; +- `POST /workspaces/validate` for static completeness; +- `POST /workspaces/:id/test` for live connectivity and group existence; +- `tht auth check [--json]`; +- aggregate `tht doctor [--json]`. + +Workspace test results retain existing DWH/Qdrant/embedding diagnostics and add an +`authentication` section. Overall `activatable` is false when authentication is not ready. + +An Authentik-backed `tht auth check --interactive` uses OIDC Device Authorization when the +provider advertises it. It prints the verification URI and user code, waits for completion, and +validates a real ID token including `groups`. Absence of a device endpoint is reported explicitly; +ordinary browser login remains usable for generic OIDC providers. + +## Host CLI contract + +The host-facing command surface added to `tht` is: + +```text +tht auth configure --mode local [--public-url URL] [--admin-user USER \ + --admin-display-name NAME --password-file FILE] +tht auth configure --mode oidc --public-url URL --issuer URL --client-id ID \ + --authentik-base-url URL --user-group GROUP --admin-group GROUP +tht auth status [--json] +tht auth check [--interactive] [--json] + +tht auth user list [--json] +tht auth user add USERNAME --role user|admin [--display-name NAME] [--password-file FILE] +tht auth user set-password USERNAME [--password-file FILE] +tht auth user enable USERNAME +tht auth user disable USERNAME +tht auth user grant USERNAME --role user|admin +tht auth user revoke USERNAME --role user|admin +tht auth user logout-all USERNAME --yes +``` + +Interactive password prompts disable terminal echo and require confirmation. JSON stdout remains +pristine; progress and prompts go to stderr. User commands refuse OIDC mode. Configuration writes +never overwrite an existing valid configuration without explicit confirmation. + +`tht setup` creates the protected authentication directory and invokes local or OIDC +configuration before starting the stack. `tht doctor` adds an `authentication` check and redacts +the OIDC client secret, Authentik API token, passwords, hashes, cookie values, and CSRF values. + +## Node and dependency baseline + +The implementation aligns the runtime and CI on Node `24.16.0`. Docker uses the multi-platform +official image digest: + +```text +node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 +``` + +Required added dependencies are: + +- backend `openid-client` 6.8.5; +- backend `@fastify/cookie` 11.1.2; +- backend `@fastify/rate-limit` 11.2.0; +- Go `golang.org/x/crypto` 0.55.0. +- Go `golang.org/x/term` 0.45.0. + +No Node Argon2 native addon or Authentik SDK is added. Node upgrade acceptance requires backend +and frontend typechecks, builds, unit tests, Playwright, Docker smoke, Pi runtime smoke, and the L2 +live-session smoke. A regression blocks the upgrade and the authentication release; it is not +waived merely to gain native Argon2. + +## Deployment and migration + +- Local profiles move from implicit `AUTH_MODE=none` to configured `mode: local` in `auth.yaml` and require an + initial administrator before startup is considered valid. +- Server profiles move from trusted-proxy `AUTH_MODE=upstream` to `mode: oidc` in `auth.yaml` after Authentik + setup. `upstream` remains available during the migration window but is marked deprecated. +- `auth.yaml` is the sole production source of truth for `local` and `oidc`. `AUTH_MODE` remains + accepted only for `none`, `mock`, and deprecated `upstream` when no auth configuration exists. +- Existing session/artifact storage is not migrated or re-owned by authentication work. +- Reverse proxies must preserve the configured public origin and callback path. ThothII trusts + forwarded scheme/host only under the existing explicit server proxy boundary. +- PSD acceptance uses Authentik connected to corporate LDAP, two real groups mapped to `user` and + `admin`, one ordinary test user, and one administrative test user. + +## Documentation and acceptance + +The release must document: + +- standalone local setup, initial admin, Remember me, timeout behavior, password recovery, and + session invalidation; +- the mandatory direct `groups` claim contract and failure behavior; +- group-to-role and role-to-permission mapping, including exact case sensitivity; +- Authentik provider, scope/property mapping, service account, API token, callback, group mapping, + token rotation, and PSD LDAP relationship; +- why unmapped identity-manager groups are ignored without warnings; +- generic OIDC support versus provider-specific group-catalog certification; +- all `tht auth` commands and JSON contracts; +- authentication checks inside workspace validation and `tht doctor`; +- reverse-proxy and same-origin cookie requirements. + +Release acceptance requires cross-language Argon2 vectors, local remembered-session restart tests, +local invalidation tests, authorization matrix tests, CSRF tests, OIDC protocol tests with a local +fake OP, Authentik API contract tests, a real Authentik integration test, workspace diagnostic +tests, frontend login tests, Compose smoke tests, and the existing full regression suites. diff --git a/docs/testing/authentication-manual-acceptance.md b/docs/testing/authentication-manual-acceptance.md new file mode 100644 index 00000000..c1a6ec51 --- /dev/null +++ b/docs/testing/authentication-manual-acceptance.md @@ -0,0 +1,69 @@ +# Authentication manual acceptance + +This is a release-gate checklist, not evidence. Use one ordinary PSD test identity and one admin +PSD test identity supplied through the approved test-identity process. Record only sanitized +pass/fail results, timestamps, build identity, and diagnostic codes. Do not record names, internal +URLs, directory/LDAP details, tokens, passwords, hashes, cookies, or realistic secret examples. +Keep the retained result under `.artifacts/manual-acceptance/authentication//` with a +sanitized digest. Do not retain raw browser traces, Compose environments, provider exports, or +unbounded logs. If the approved identities or access are unavailable, record **PENDING** rather +than inferring a PASS. + +## Preconditions and ordering + +1. Confirm retained Task 13 evidence for the restore prerequisites before certification: the + lifecycle lock is acquired before target-dependent preflight, archive bytes and hashes are + staged/revalidated inside that lock immediately before extraction, and checkpointing requires + an opaque installation-bound transaction capability. Manual acceptance never substitutes for + those automated concurrency and mutation tests. +2. Set the installation and workspace identifiers, then inspect the active workspace with the + native host CLI. This replaces the former Workspace Validate/Test wording: + + ```bash + export THT_BIN=tht + export INSTALLATION=/absolute/path/to/thothii-installation.yaml + export WORKSPACE_ID=psd-clinical + "$THT_BIN" --installation "$INSTALLATION" \ + workspace inspect --workspace "$WORKSPACE_ID" --json + ``` + +3. Run `"$THT_BIN" --installation "$INSTALLATION" auth check --json` for live non-interactive + diagnosis, then `auth check --interactive` where Device Authorization is available. +4. Run `"$THT_BIN" --installation "$INSTALLATION" doctor --json` and confirm this exact report order: `descriptor`, `files`, `docker`, + `compose`, `configuration`, `authentication`, `services`, `core-http`, `frontend-http`, + `workspace-registry`, `workflow`, `pi`. +4. Confirm the exact direct `groups` claim for both identities and the mappings `TOT Users → user` + and `TOT Admin → admin`. Confirm extra upstream groups are ignored without warning. + +## Matrix + +| Scenario | Expected result | +|---|---| +| Ordinary identity opens its own application/session routes | Allowed; admin-only routes return `403`. | +| Admin identity opens admin routes | Allowed according to the `admin` permission set. | +| Browser callback token omits `groups` | Callback returns HTTP 401 `oidc_callback_failed`; the internal reason is not exposed. | +| Browser callback token has malformed, indirect, or overage groups | Callback returns HTTP 401 `oidc_callback_failed`; the internal reason is not exposed. | +| Interactive diagnostic receives missing or invalid groups | Diagnostic fails with `oidc_groups_claim_invalid`. | +| Token has no mapped group | Principal has no role; protected routes return `403`; no warning is emitted. | +| A configured group is absent from Authentik | Check fails with `oidc_mapped_group_missing`. | +| Catalog token is wrong or lacks group-view-only access | Live check fails redacted with `oidc_group_catalog_unauthorized`. | +| Mapped group is renamed | The next check fails closed until configuration and provider agree. | +| Token adds an unrelated group | Login and authorization are unchanged; no warning is emitted. | +| Authenticated PSD identity creates a known-good session | SSE connects, the session is created, and the first reviewer gate appears without unexpected `401`/`403` responses. | +| Backend restarts with Remember me | Remembered local session survives within its TTL. | +| Password/role/enable revision changes | Affected local sessions are rejected and reauthentication is required. | +| CSRF or cross-origin mutation is attempted | Request is rejected. | +| Logout | Cookie expires and the server session is deleted. | +| Provider outage | Live check reports `oidc_discovery_unreachable`; browser login fails closed without exposing credentials. | +| Restore is completed | Sessions and OIDC state are absent; all users must reauthenticate. | + +## Status at Task 15 + +The hermetic browser suite now covers the loopback provider discovery/JWKS/device/group-list +surface and the complete OIDC Authorization Code + PKCE callback, including direct `groups` +fail-closed cases. It also covers local ordinary, remembered/restart, logout, and administrator +flows. This deterministic evidence does not replace the manual PSD/AuthentiK acceptance. + +Native Windows behavioral execution, approved PSD/AuthentiK identities and access, interactive +device acceptance, and external L2 remain **PENDING** until actual retained evidence exists. Do +not mark the feature or this matrix release-complete while any required gate remains pending. diff --git a/docs/testing/p1-manual-acceptance.md b/docs/testing/p1-manual-acceptance.md new file mode 100644 index 00000000..14b2b748 --- /dev/null +++ b/docs/testing/p1-manual-acceptance.md @@ -0,0 +1,107 @@ +# P1 manual configuration acceptance + +This walkthrough is an independent human gate for the P1 workspace configuration process. The +reviewer—not the helper—performs the HTTP, Git, export, rendering, and `tht` checks and judges the +result. Automation never creates `VERDICT.md`, never records PASS, and never consumes or copies +`.artifacts/p1-integration`. + +## Prerequisites + +From a clean repository checkout, Task 8 must already be implemented. Install Node/npm, `python3`, +and Git, `curl`, `unzip`/`zipinfo`, `lsof`, and the harness development environment so +`harness/.venv/bin/tht` is executable. +Ports `127.0.0.1:8791` and `127.0.0.1:8792` must be free. The helper builds and serves only the +production backend; it does not start Docker or the frontend. + +## Lifecycle + +Run these commands from the repository root: + +```bash +./scripts/p1-manual-acceptance.sh prepare +./scripts/p1-manual-acceptance.sh serve +./scripts/p1-manual-acceptance.sh stop +./scripts/p1-manual-acceptance.sh cleanup +``` + +All four actions serialize on the stable repository-root +`.p1-manual-acceptance.lifecycle.lock`; the helper retains and revalidates repository, artifact, +manual-parent, and owned-root identities throughout each transaction. `prepare` acquires that lock +before prerequisite checks and the backend build, exclusively creates +`.artifacts/manual-acceptance/p1/`, and immediately publishes a `PREPARING` ownership record before +populating the lab. That ownership-first record makes an interrupted population cleanable. A +successful prepare atomically advances it to `READY` after creating fresh Git history, fixtures, +secret files, concrete request/inspection commands, `GUIDE.md`, and the single regular +`logs/backend.log` with mode `0600`. It records the log identity and the production entrypoint's +path/device/inode/size/SHA-256, creates no supervisor or readiness-status file, leaves status +`PENDING` and the server stopped, and refuses an existing root. Use guarded `stop` and `cleanup` +rather than deleting or reusing state manually. + +`serve` revalidates the bound `backend/dist/server.js` identity and bytes, the immutable +post-build manifest of every regular `backend/dist` file (path, size, SHA-256, device, inode), +every owned root/runtime/log ancestor, the absence of a legacy supervisor, and the original log +identity before spawning. The log, the production entrypoint, and the distribution manifest are +opened with no-follow semantics; the entrypoint and manifest descriptors are passed directly to the +child, and an immutable preload makes Node load the already verified entrypoint bytes and the +complete verified `backend/dist` module graph rather than a later pathname replacement. At startup +the preload hash-verifies every manifest file and serves only those cached verified bytes for any +import below `backend/dist`, so a same-path regular replacement is refused (before or during +serving) and can never execute. The child remains the production Node entrypoint itself: +`node --import data:text/javascript;base64, backend/dist/server.js` followed by +six ownership, control, and entrypoint-identity arguments (plus the manifest descriptor on fd 4). +The preload owns the authenticated fixed `127.0.0.1:8792` control channel and bounded watchdog, and +tracks the HTTP server that this same process successfully binds to `127.0.0.1:8791`. Before publishing the +`RUNNING` PID record, the parent requires exact nonce-bound control acknowledgements that identify +that owned listener, a 2xx `GET /health`, stable listener generation and entrypoint identity, and a +final authenticated status check. A foreign health listener cannot satisfy readiness. A startup or +non-2xx failure requests nonce-authenticated STOP (or lets the watchdog self-exit) and leaves no PID +record after the child exits. + +`stop` revalidates the exact executable, immutable preload, bound production entrypoint identity and +bytes, arguments, repository cwd/root, and process start identity, then requests STOP over the +nonce-authenticated cooperative channel and requires the exact acknowledgement. The controlled +process closes its owned listener and exits itself; the tool never sends a numeric terminating +signal. Ambiguous, stale, or starting records remain for operator inspection. `cleanup` uses opened, +no-follow directory identities to rename and remove only the exact stopped owned fixed root. Foreign +siblings and automated integration artifacts are outside its cleanup boundary. + +After `prepare`, follow the 14 ordered steps in the generated absolute-path `GUIDE.md`. Personally run each generated `http-01` through `http-14` curl script in numeric order; they save the exact status, three validation, three sequential publication, pull, three read responses, and three ZIP exports. Each publication derives its current base commit with a bounded parser from the preceding saved API response, with no placeholder base. Run the five numbered negative validation scripts separately at checklist step 10. The render commands validate the bounded saved read response, +its commit-addressed owned snapshot path, the saved publish commit, the installed Git HEAD, and the +bounded `snapshot.json` manifest of that commit: they bind the snapshot bytes to the manifest digest, +the saved revision blob to the manifest revision, and the manifest blob to the installed Git commit +(`git rev-parse :workspaces/.yaml` plus `git hash-object` of the snapshot bytes) before +calling the acceptance-only production renderer with the expected `--snapshot-sha256`. The renderer +revalidates the bounded `snapshot.json` (`head`, `files[.yaml]`) and reads the snapshot exactly +once with no-follow semantics, rendering only the digest-verified bytes. It imports the built +`ThtRunner`, resolves bindings from environment paths, copies one lease with mode `0600` through an +opened no-follow `rendered` directory descriptor, rejects an output-parent identity swap, and +releases the lease in `finally`. For each exported ZIP, invoke the generated extractor with the exact expected workspace ID +(`p1-filesystem`, `p1-http`, or `p1-s3`); its `python3` helper opens the source once, stages and +revalidates its SHA-256, anchors every extraction and cleanup operation to an opened no-follow +`exports/extracted` directory descriptor, and binds both the manifest and parsed descriptor identity +to that expected ID. It verifies exactly four regular entries and publishes only their exact checked +bytes. The generated secret scan reads bounded filesystem content and name/path bytes outside the +direct `fixture-secrets` payload directory, discovers every bounded `.git` repository under the lab +(plus the owned bare remote), and enumerates every reachable or unreachable object. It +scans raw blob, commit, tree, and tag bytes plus loose-ref names. Findings and operational diagnostics +redact canary-bearing paths and values. The absence gate rejects directories as well as files, +including the canonical `artifacts/evidence` tree and preprocessing, materialization, embedding, +Qdrant, ACTIVE, or retention names. Do not inspect or print raw secret-file contents; only inspect +ownership/mode/path metadata and canary absence outside `fixture-secrets`. + +## Failures and verdict + +On failure, run `stop` if the owned server is running and preserve the entire fixed root for review. +Do not run `cleanup` until evidence is no longer needed. A reviewer creates `VERDICT.md` only after the +walkthrough, containing: + +- reviewer identity; +- UTC timestamp; +- an explicit result for every one of the 14 generated checklist steps; +- observations and failure evidence; +- exactly `manual acceptance: PASS` or `manual acceptance: FAIL`. + +Passing `bash scripts/test-p1-manual-acceptance.sh` proves only that the tooling guards work. It does +not perform or approve manual acceptance and leaves the project-level manual status PENDING. + +Expected safe outcomes are one production Node PID owning both listeners on `127.0.0.1:8791` and the authenticated control port `127.0.0.1:8792`; 2xx positive responses; non-2xx negative validations without Git or snapshot mutation; an empty render diff; two successful `tht config check` calls; no manifest, Evidence/export, secret, or out-of-scope-artifact finding; and no PID or listener on either port after `stop`. diff --git a/docs/testing/p11-manual-acceptance.md b/docs/testing/p11-manual-acceptance.md new file mode 100644 index 00000000..1f860ba4 --- /dev/null +++ b/docs/testing/p11-manual-acceptance.md @@ -0,0 +1,89 @@ +# P1.1 manual acceptance + +This walkthrough is the separate human gate for the P1.1 workspace-directory registry. +It is independent from both `.artifacts/p1-integration/**` and `.artifacts/p11-integration/**`. +The helper prepares and serves the lab, but the reviewer performs the registry, Git, UI, export, +render, `tht`, refusal, secret-scan, and cleanup checks and records the verdict. + +## Prerequisites + +- clean repository checkout with the P1.1 implementation present; +- `node`, `npm`, `git`, `curl`, and `python3` available; +- built production assets: + +```bash +npm --prefix backend run build +npm --prefix frontend run build +``` + +- executable harness CLI at `harness/.venv/bin/tht`; +- free loopback ports `127.0.0.1:8791` and `127.0.0.1:8792`. + +## Lifecycle commands + +Run from the repository root: + +```bash +./scripts/p11-manual-acceptance.sh prepare +./scripts/p11-manual-acceptance.sh serve +./scripts/p11-manual-acceptance.sh stop +./scripts/p11-manual-acceptance.sh cleanup +``` + +The fixed lab root is: + +```text +.artifacts/manual-acceptance/p11/ +``` + +Expected lifecycle behavior: + +- `prepare` creates the fixed root, ownership record, bare remote, curator clone, root catalog, + nested filesystem evidence, fixture secrets, request fixtures, generated command scripts, and + `GUIDE.md`; it leaves status `PENDING`, performs no reviewer publish operation, and never writes + `VERDICT.md`. +- `serve` starts the production backend on `127.0.0.1:8791` and a production-built frontend preview + on `127.0.0.1:8792`, recording exact ownership for both. +- `stop` refuses foreign or partial ownership and stops only the two owned loopback processes. +- `cleanup` refuses live state and removes only `.artifacts/manual-acceptance/p11/`. + +## Reviewer workflow + +After `prepare`, open the generated `.artifacts/manual-acceptance/p11/GUIDE.md` and personally: + +1. inspect the catalog, nested descriptor/evidence layout, ownership, and secret-path bindings; +2. serve both surfaces and verify the owned listeners; +3. list `configuration_required` slots; +4. validate and bootstrap-create descriptors exactly once; +5. inspect catalog/descriptor/evidence/docs Git object IDs; +6. retry create/update/delete and verify refusal plus unchanged object IDs; +7. make a curator descriptor+catalog edit, push, pull, and verify the API did not rewrite curator bytes; +8. make an evidence-only commit and inspect the new revision identity; +9. verify the live UI shows read-only existing workspaces and bootstrap-only editing for missing slots; +10. exercise export/import under bootstrap-only rules; +11. render twice, diff the results, and run `tht config check`; +12. run negative catalog/path/secret cases and a bounded secret scan; +13. stop the lab, verify both listeners are gone, write `VERDICT.md`, and only then cleanup if desired. + +## Expected outcomes + +- `prepare` produces a fresh P1.1-only lab and leaves no `VERDICT.md`. +- `serve` exposes only the owned loopback backend and frontend preview. +- positive API operations succeed once; curator-owned follow-up mutations are refused safely; +- curator Git changes become active only after pull; +- renders are deterministic; `tht config check -c ` succeeds; +- secret scans find no canaries outside the fixture-secret boundary; +- after `stop`, nothing remains listening on `127.0.0.1:8791` or `127.0.0.1:8792`. + +## Verdict format + +The reviewer creates `VERDICT.md` manually. Include: + +- reviewer identity; +- UTC timestamp; +- result for each checklist step; +- observations and failure evidence; +- exactly one final line: `manual acceptance: PASS` or `manual acceptance: FAIL`. + +Passing `bash scripts/test-p11-manual-acceptance.sh` proves only the tooling/lifecycle guards. It +does not perform or approve manual acceptance. diff --git a/docs/testing/p2-p6-manual-verification.md b/docs/testing/p2-p6-manual-verification.md new file mode 100644 index 00000000..4b1b956e --- /dev/null +++ b/docs/testing/p2-p6-manual-verification.md @@ -0,0 +1,218 @@ +# P2–P6 Manual Verification Walkthrough + +> Living document. Each section is completed with exact released commands and artifacts during its +> corresponding plan. Automated integration and manual acceptance use separate clean state. + +## Global rules + +- Use a new temporary operator root and a new private fixture Git remote for each Px. +- Never use production PSD credentials in a retained report or screenshot. +- Keep descriptor/content in Git; keep endpoints, bindings, credentials, and certificates in the + installation-local protected directory. +- Do not print secret files, rendered signed URLs, Compose environments, or unbounded logs. +- Record the ThothII commit, workspace commit, installation descriptor path, Compose project name, + command exit status, and report path. +- A focused manual PASS does not replace the automated process goal. + +## P2 — Host preprocessing CLI + +**Status:** P2 implementation complete; automated integration PASS; manual acceptance PENDING. + +Manual goal: from a clean local installation, use only `tht` on the host to inspect one +registry workspace and execute the controlled REST-DWH/HTTP-Evidence preprocessing path without a +host Python or Node runtime. Use a fresh operator root and a fresh fixture Git remote; never reuse +the automated `.artifacts/p2-integration/**` state. + +Commands (contract: `docs/contracts/workspace-preprocessing-cli.md`): + +```bash +tht --installation /thothii-installation.yaml workspace inspect --workspace --json +tht --installation /thothii-installation.yaml workspace preprocess dwh --workspace --json +tht --installation /thothii-installation.yaml workspace preprocess dwh --workspace --resume --json +tht --installation /thothii-installation.yaml workspace schema suggest-fks --workspace --from-sql .sql --output .yaml --json +tht --installation /thothii-installation.yaml workspace schema check --workspace --annotations .yaml --reviewed-candidates --json +tht --installation /thothii-installation.yaml workspace index-schema --workspace --json +tht --installation /thothii-installation.yaml workspace preprocess evidence --workspace --dry-run --json +tht --installation /thothii-installation.yaml workspace preprocess evidence --workspace --json +tht --installation /thothii-installation.yaml workspace preprocess run --workspace --json +``` + +Checks: + +1. installation/render preflight (`inspect` returns exact revision + catalog/descriptor digests); +2. DWH introspection+LSH succeeds, rerun is `unchanged`, `--resume ` is `unchanged`/`succeeded`; +3. `schema suggest-fks` returns pristine JSON with `suggestedFksYaml` and a `manual_review_required` + block (exit 3) when candidates exist; the suggested YAML digest equals the reported digest; +4. `schema check --annotations --reviewed-candidates ` succeeds after review; +5. `index-schema` counts against a pre-provisioned compatible collection and rerun is `unchanged`; +6. HTTP Evidence `--dry-run` returns `dry_run`, the real run publishes, rerun is `unchanged`, an input + mutation produces a new generation/ACTIVE; +7. filesystem Evidence returns a stable `evidence_materialization_required` block with no partial + corpus/vector publication; +8. negatives: missing workspace (`workspace_not_activatable`), resume of a nonexistent run + (`preprocessing_resume_mismatch`), invalid annotations digest (`annotation_invalid`), no-Evidence + skip warning, no collection creation, no backend/Pi/frontend listener; +9. secret scan over retained artifacts and exact owned-resource cleanup. + +Decision: **PENDING** (independent manual gate; automation never records PASS). + +## P3 — Effective configuration and `.tht-dwh` + +**Status:** P3 implementation complete; automated integration PASS; manual acceptance PASS (owner approval 2026-08-13). + +Manual goal: prove that the operator CLI and application sessions derive the same effective +configuration, that a content-only revision reuses the prepared DWH generation (fast, `unchanged`), +that a DWH-affecting change fails closed and regenerates, that the workspace memory migration is +safe, and that search records are revision-scoped. See `docs/contracts/tht-dwh.md`. + +Checks: + +1. run `tht ... workspace preprocess dwh` twice with only an Evidence/content change between + them: the second run reports `unchanged` and does not re-introspect; +2. change a DWH-affecting field (host/port/database/schema/user/collection) in the descriptor, + push, pull: the next run refuses the old generation and regenerates, with a clear + `effective_config_mismatch`-style outcome and no mixed artifacts; +3. inspect `.tht-dwh` generations: immutable directories, `OWNER.json` with the canonical + fingerprints, `ACTIVE` pointer; old generations still present; +4. memory: after the guarded migration the workspace uses + `/sessions//memory/`; the JSONL registry and Qdrant projection are + rebuilt and consistent; a conflicting legacy registry fails closed; +5. search records: schema/Evidence points carry the pinned `workspace_revision`; memory/solved + records remain workspace-wide; +6. documentation: `docs/contracts/tht-dwh.md` matches the observed behavior. + +Decision: **PASS** (owner approval 2026-08-13). +## P4 — Qdrant bootstrap and guarded rebuild + +**Status:** superseded by the "P4 Qdrant collection lifecycle" section below (implemented; manual acceptance PASS). + +Manual goal: prove admission creates a missing compatible collection and indexes, refuses an +incompatible collection, and permits destructive rebuild only under durable maintenance with no +active readers/jobs and exact repeated confirmation. + +Checks to fill during P4: + +1. missing-collection self-heal; +2. missing-index self-heal; +3. dimensions/distance/index-type refusal; +4. confirmation mismatch refusal; +5. active-reader/job refusal; +6. successful drained rebuild; +7. interrupted rebuild recovery with maintenance retained. + +Decision: **PASS** (owner approval 2026-08-13; see the section below). + + +## P4 Qdrant collection lifecycle + +Manual goal: verify admission self-heal and the guarded rebuild through the real product surface. + +Checks to complete during P4 manual acceptance (decision: **PASS** (owner approval 2026-08-13)): + +1. On a fresh installation with no Qdrant collection, a session admission creates the + descriptor collection with exactly 1024 dimensions, cosine distance, and the 8 required + keyword payload indexes (`content_hash`, `document_id`, `kind`, `record_key`, + `record_kind`, `vector_generation`, `workspace_id`, `workspace_revision`). +2. A pre-existing collection with incompatible dimensions/distance (e.g. 768-dim or dot) + is refused with `semantic_index_incompatible` and is never mutated. +3. `tht ... workspace vector inspect --workspace --json` reports the collection + contract without mutation (pristine JSON, exit 0). +4. `tht ... workspace vector rebuild --workspace --collection + --confirm --destroy` deletes and recreates the descriptor-owned collection and + verifies the recreated contract; a mismatched `--confirm` or a missing `--destroy` is + refused (exit 2) without touching the collection. +5. Rebuild writes durable state before deletion, deletes only the descriptor collection, + and the recreated collection preserves the P3 revision-scoped payload contract. + +## P5 — Curated FK annotations in Git + +**Status:** P5 implementation complete; automated integration PASS; manual acceptance PASS (owner approval 2026-08-13). + +Manual goal: curate `/schema/annotations.yaml` in an author clone, publish it, pull the new +revision, and prove the revision-pinned sync and the explicit `schema accept` review, without ever +pushing curated content from the operator CLI. + +Commands (contract: `docs/contracts/workspace-preprocessing-cli.md`): + +```bash +tht --installation /thothii-installation.yaml workspace schema suggest-fks --workspace --from-sql .sql --output .yaml --json +# curate the candidate into /schema/annotations.yaml in the author clone, then commit/push/pull +tht --installation /thothii-installation.yaml workspace schema accept --workspace --run --yes --json +tht --installation /thothii-installation.yaml workspace preprocess run --workspace --resume --json +``` + +Checks: + +1. `schema suggest-fks` returns pristine JSON with `suggestedFksYaml` and a `manual_review_required` + block (exit 3) when candidates exist; the suggested YAML digest equals the reported digest; +2. after commit/push/pull, activation reads `/schema/annotations.yaml` as a regular Git blob at + the same commit as the descriptor and synchronizes it to + `/sessions//revisions//artifacts/mschema/annotations.yaml` with a restrictive + mode and an adjacent ownership manifest `{ workspace, commit, blobId, contentDigest, destination }`; +3. two revisions write two different directories; a session pinned to an older revision reads its own + revision's annotations; +4. `schema accept --run --yes` records the accepted candidate/current-blob digests and the new + revision; missing `--yes`, an unknown run, an empty file, a malformed blob, or a blob not matching + the recorded candidate is refused (exit 1, `annotation_invalid`) without recording a review; +5. `preprocess run --resume ` continues only with the exact accepted blob digest and compatible + DWH binding; otherwise it records a new `manual_review_required` checkpoint; +6. negatives: symlink/tree-at-path, cross-namespace, oversized (>16 MiB), non-UTF-8, and malformed + annotation objects are refused at activation without mutating the snapshot or runtime roots; +7. the operator CLI never stages/commits/pushes curated content; secret scan and exact owned-resource + cleanup pass. + +Decision: **PASS** (owner approval 2026-08-13). +## P6 — Commit-addressed Evidence materialization + +**Status:** P6 implementation complete; automated integration PASS; manual acceptance PASS (owner approval 2026-08-13). + +Manual goal: materialize filesystem Evidence from the pinned Git commit, inspect its bounded +manifest, preprocess/index it, retrieve only the pinned revision, and exercise unsafe-tree and +aggregate-limit failures without partial publication. + +Commands (contract: `docs/contracts/workspace-preprocessing-cli.md`): + +```bash +tht --installation /thothii-installation.yaml workspace inspect --workspace --json +tht --installation /thothii-installation.yaml workspace preprocess evidence --workspace --dry-run --json +tht --installation /thothii-installation.yaml workspace preprocess evidence --workspace --json +tht --installation /thothii-installation.yaml workspace preprocess evidence --workspace --json # idempotent rerun +``` + +Checks: + +1. exact commit/tree/object identities: after activation the materialized root is + `/snapshots///evidence` and its sibling manifest + `/evidence.manifest.json` records `workspace`, `commit`, `tree`, per-file `oid`/`digest`, + `entryCount`, `totalBytes`; `snapshot.json` chains the manifest digest; +2. successful atomic materialization: every regular blob is present byte-for-byte; the manifest + digests match; +3. manifest and file digest verification: re-activation reuses a valid root and fails closed on a + tampered manifest; +4. filesystem Evidence dry-run/run/idempotency: `--dry-run` returns `dry_run`, the real run + publishes, rerun is `unchanged`; +5. revision-filtered Qdrant retrieval and corpus ACTIVE: Evidence records carry the pinned + `workspace_revision`; +6. nested symlink/gitlink/traversal/special-file refusal: a commit introducing one of these fails + activation (`workspace_invalid`) and the previous valid revision stays active; +7. file-count/total-byte/path/manifest limit refusal: an oversized or over-count tree fails closed + without a partial publication; +8. retention while pinned and owned cleanup after release: the materialized root persists for a + pinned revision and is removed with its snapshot directory once unreferenced. + +Decision: **PASS** (owner approval 2026-08-13). +## Final aggregate P2–P6 verification + +**Status:** runnable; automated integration PASS; manual acceptance PENDING. + +The automated aggregate (run `p2p6-ee542112c526ef0d4c25ddf6c8bc164b`, report +`.artifacts/p2p6-integration/...`) already executed the complete DWH → FK → schema → filesystem +Evidence chain, idempotency, revision isolation, a second installation, unsafe-tree/bound negatives, +secret scan, and exact cleanup. + +The final manual pass will start with a new registry and two independent installations. It will +run the complete DWH → FK → schema → filesystem Evidence chain, prove idempotency and revision +isolation, confirm the second installation uses its own secrets/state, and compare its observations +to the retained aggregate automated report. + +Decision: **PENDING**. diff --git a/docs/workspace-diagnostic-protocol.md b/docs/workspace-diagnostic-protocol.md new file mode 100644 index 00000000..fef6b03e --- /dev/null +++ b/docs/workspace-diagnostic-protocol.md @@ -0,0 +1,151 @@ +# Workspace diagnostic protocol + +This is the operator contract for testing a workspace on one ThothII installation. The Git-shared +descriptor declares what can be checked; the installation supplies only the selected DWH +transport and local secret-file bindings. No secret value, certificate content, SSH key, or +response body belongs in the descriptor, generated `.env.example` files, or diagnostic output. + +## Scope and safety rules + + +Schema v3 is the only accepted workspace descriptor. +Schema v1 and v2 workspace descriptors are rejected before activation. +- Diagnostics do not run for a rejected descriptor. There is no in-product migrator or automatic + conversion; the Git repository must already contain reviewed v3 descriptors. + +- One workspace owns one Qdrant collection. +- Qdrant and Ollama are internal services. Operators do not bind external vector or embedding + transports for active manuals or supported diagnostics. +- Each diagnostic is bounded by the configured timeout. Redirects are rejected, response bodies + stay inside the adapter, and browser-visible errors are limited to `binding_missing`, + `connector_unavailable`, and `semantic_index_incompatible`. + +## Canonical descriptor contract + +```yaml +workspace: + schema_version: 3 + id: psd-clinical + name: PSD Clinical + language: it + +dwh: + engine: postgres + database: warehouse + schema: datawarehouse + supported_transports: [postgres_direct, rest_api, ssh_tunnel] + +semantic_index: + vector_store: + engine: qdrant + collection: psd-clinical + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 + +diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: bearer + response: { database: database, schema: schema } +``` + +The semantic-index contract is fixed: + +- `engine: qdrant` +- collection name equals the workspace-owned portable identifier +- `qwen3-embedding:0.6b` +- `1024` dimensions +- cosine distance + +If any active collection reports a different model pairing, dimension, or distance, diagnostics +must return `semantic_index_incompatible` rather than silently rewriting data. + +## Installation-local variable contract + +Replace `` with the immutable workspace ID converted to upper case with hyphens changed +to underscores. For example, `psd-clinical` becomes `PSD_CLINICAL`. Set only the variables for the +selected DWH transport. Every `*_FILE` value is an absolute path to a regular, readable file +inside an approved local secret root; it is never the secret itself. + +| Connector and transport | Required local variables | +| --- | --- | +| DWH selection | `THT_WS__DWH_TRANSPORT` | +| DWH `postgres_direct` | `THT_WS__DWH_HOST`, `THT_WS__DWH_PORT`, `THT_WS__DWH_USER`, `THT_WS__DWH_PASSWORD_FILE`; optional `THT_WS__DWH_TLS_CA_FILE` | +| DWH `rest_api` | `THT_WS__DWH_BASE_URL`; `THT_WS__DWH_API_KEY_FILE` only for `bearer`/`x-api-key`; optional `THT_WS__DWH_TLS_CA_FILE` | +| DWH `ssh_tunnel` | `THT_WS__DWH_USER`, `THT_WS__DWH_PASSWORD_FILE`, `THT_WS__DWH_SSH_HOST`, `THT_WS__DWH_SSH_PORT`, `THT_WS__DWH_SSH_USER`, `THT_WS__DWH_SSH_PRIVATE_KEY_FILE`, `THT_WS__DWH_SSH_KNOWN_HOSTS_FILE`, `THT_WS__DWH_SSH_TARGET_HOST`, `THT_WS__DWH_SSH_TARGET_PORT`; optional `THT_WS__DWH_TLS_CA_FILE` | + +There are no supported `THT_WS__VECTOR_*` or +`THT_WS__EMBEDDING_*` installation bindings in the active operator contract. + +## DWH diagnostic + +For direct PostgreSQL and SSH-tunnelled PostgreSQL, the diagnostic connects with the declared +`dwh.database`, checks TLS and authentication, then executes exactly: + +```sql +SELECT current_database() AS database, current_schema() AS schema +``` + +Both returned values must equal the descriptor's DWH database and schema. + +For REST, the descriptor-declared request is for example: + +```text +POST _DWH_BASE_URL>/rpc/ping +Authorization: Bearer +``` + +It has no request body. A 2xx response must be a JSON object whose declared `database` and +`schema` fields match the descriptor. + +## Internal semantic-service diagnostic + +Schema-v3 workspace diagnostics also verify the internal semantic infrastructure through backend +configuration: + +- Qdrant must be reachable at the installation-owned internal URL. +- The workspace-owned collection must exist or be creatable with `1024` dimensions and cosine + distance. +- Ollama must provide `qwen3-embedding:0.6b`. +- A bounded embed probe must return exactly `1024` dimensions. + +These checks use the private Compose services and never require operator-supplied vector or +embedding URLs, transports, or credentials. + +## SSH host verification and tunnel lifecycle + +For DWH `ssh_tunnel`, the known-hosts file is mandatory and is verified before a connection is +accepted. The tunnel is a short-lived loopback forward for the diagnostic only. The effective +OpenSSH constraints are: + +```text +-N -v +-o BatchMode=yes +-o ExitOnForwardFailure=yes +-o StrictHostKeyChecking=yes +-o UserKnownHostsFile=_SSH_KNOWN_HOSTS_FILE +-i _SSH_PRIVATE_KEY_FILE +-p _SSH_PORT +-L 127.0.0.1::_SSH_TARGET_HOST:_SSH_TARGET_PORT +_SSH_USER@_SSH_HOST +``` + +The local listener is `127.0.0.1` only. The process is terminated in cleanup after the direct +probe, on timeout, or on failure. + +In this release, `ssh_tunnel` remains a diagnostic-only DWH transport. A successful probe is +followed by `workspace_not_activatable`, and `POST /sessions` rejects the workspace before +persisting a manifest or starting Pi. This restriction does not apply to SSH transport for the +workspace Git remote. + +## Reader-only fallback + +A workspace may be fully valid in Git but non-activatable locally when a required DWH binding, +secret file, host verification, TLS check, or declared DWH diagnostic fails. That state does not +alter the shared descriptor and does not permit a new session on that installation. It may still +be published and activated elsewhere with valid local bindings. diff --git a/frontend/e2e/auth.spec.ts b/frontend/e2e/auth.spec.ts new file mode 100644 index 00000000..6d94437b --- /dev/null +++ b/frontend/e2e/auth.spec.ts @@ -0,0 +1,183 @@ +import { expect, test, type Page } from "@playwright/test"; +import { createAuthenticationStack } from "./fixtures/auth-stack.mjs"; + +test.describe.configure({ mode: "serial" }); +// The only HTTPS navigation in this file is the test-scoped loopback provider. +test.use({ ignoreHTTPSErrors: true }); + +let stack: Awaited>; + +test.beforeAll(async () => { + stack = await createAuthenticationStack(); +}); + +test.afterAll(async () => { + await stack?.close(); +}); + +test("the loopback fixture exposes signed OIDC discovery, device authorization, and AuthentiK group lookup", async () => { + await expect(stack.providerSurface()).resolves.toEqual({ + discovery: true, + jwks: true, + deviceAuthorization: true, + deviceToken: true, + groupList: true, + runtimeCredential: process.env.THT_TASK15_SENTINEL !== undefined, + }); +}); + +test("the production OIDC client rejects a wrong registration and wrong mounted client secret", async ({ page }) => { + await stack.useOidcMode("ordinary", "wrong-client-id"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expect(page.locator("body")).toContainText("invalid_request"); + await expect(page.getByTestId("app-shell")).toHaveCount(0); + + await stack.useOidcMode("ordinary", "wrong-client-secret"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectOidcCallbackDenied(page); +}); + +test("production diagnostics reject a wrong group token and pass with the correct mounted secrets", async () => { + await stack.useOidcMode("ordinary", "wrong-api-token"); + await expect(stack.authDiagnostics()).resolves.toMatchObject({ + status: 1, + report: { ready: false, checks: [{ code: "oidc_group_catalog_unauthorized" }] }, + }); + + await stack.useOidcMode("ordinary", "correct"); + await expect(stack.authDiagnostics()).resolves.toMatchObject({ + status: 0, + report: { ready: true, checks: [{ code: "auth_ready" }] }, + }); +}); + +async function expectShell(page: Page): Promise { + await expect(page.getByTestId("app-shell")).toBeVisible({ timeout: 30_000 }); +} + +async function signInLocally(page: Page, account: "ordinary" | "admin", remember = false): Promise { + await page.getByLabel("Username").fill(stack.localAccount(account).username); + await page.getByLabel("Password").fill(stack.localAccount(account).password); + const rememberControl = page.getByRole("checkbox", { name: /remember me/i }); + if (remember) await rememberControl.check(); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expectShell(page); +} + +async function browserSession(page: Page): Promise<{ status: number; body: Record }> { + return page.evaluate(async () => { + const response = await fetch("/api/me", { credentials: "same-origin" }); + return { status: response.status, body: await response.json() as Record }; + }); +} + +async function expectNoWebStorageTokens(page: Page): Promise { + const entries = await page.evaluate(() => { + const values = (storage: Storage) => Array.from({ length: storage.length }, (_unused, index) => { + const key = storage.key(index) ?? ""; + return [key, storage.getItem(key) ?? ""]; + }); + return [...values(localStorage), ...values(sessionStorage)]; + }); + expect(entries.filter(([key, value]) => /(?:access|refresh|id)?[_-]?token|bearer|jwt/i.test(`${key}\n${value}`))).toEqual([]); +} + +async function signInWithOidc(page: Page): Promise { + await page.getByRole("button", { name: /continue with single sign-on/i }).click(); +} + +async function expectOidcCallbackDenied(page: Page): Promise { + await expect(page.locator("body")).toContainText("OIDC sign-in could not be completed", { timeout: 30_000 }); + await expect(page.getByTestId("app-shell")).toHaveCount(0); + await expectNoWebStorageTokens(page); +} + +test("local ordinary and remembered sessions survive restart, logout, and keep tokens out of Web Storage", async ({ page }) => { + await stack.useLocalMode(); + await page.goto(stack.publicUrl); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + + await signInLocally(page, "ordinary"); + expect((await browserSession(page)).body.roles).toEqual(["user"]); + await expectNoWebStorageTokens(page); + + await stack.restartBackend(); + await page.reload(); + await expectShell(page); + + await page.getByRole("button", { name: "Log out", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + + await signInLocally(page, "ordinary", true); + const remembered = (await page.context().cookies(stack.publicUrl)).find((cookie) => cookie.name === "thothii_session"); + expect(remembered?.httpOnly).toBe(true); + expect(remembered?.expires ?? -1).toBeGreaterThan(Date.now() / 1_000); + + await stack.restartBackend(); + await page.reload(); + await expectShell(page); + await expectNoWebStorageTokens(page); + + await page.getByRole("button", { name: "Log out", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + expect((await page.context().cookies(stack.publicUrl)).some((cookie) => cookie.name === "thothii_session")).toBe(false); +}); + +test("local administrator receives the administrator role", async ({ page }) => { + await stack.useLocalMode(); + await page.goto(stack.publicUrl); + await signInLocally(page, "admin"); + expect((await browserSession(page)).body.roles).toEqual(["admin"]); + await expectNoWebStorageTokens(page); +}); + +test("OIDC Authorization Code plus PKCE redirects back and maps ordinary and administrator groups", async ({ page }) => { + await stack.useOidcMode("ordinary"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectShell(page); + expect((await browserSession(page)).body.roles).toEqual(["user"]); + expect(stack.lastAuthorization()).toMatchObject({ codeChallengeMethod: "S256", pkceVerified: true }); + await expectNoWebStorageTokens(page); + + await page.getByRole("button", { name: "Log out", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + stack.setOidcIdentity("admin"); + await signInWithOidc(page); + await expectShell(page); + expect((await browserSession(page)).body.roles).toEqual(["admin"]); + await expectNoWebStorageTokens(page); +}); + +test("OIDC unmapped, missing, and malformed groups fail closed; an expired token can recover", async ({ page }) => { + await stack.useOidcMode("unmapped"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expect(page.getByRole("heading", { name: "Access not permitted" })).toBeVisible({ timeout: 30_000 }); + await expectNoWebStorageTokens(page); + + await page.context().clearCookies(); + stack.setOidcIdentity("missing-groups"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectOidcCallbackDenied(page); + + stack.setOidcIdentity("malformed-groups"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectOidcCallbackDenied(page); + + stack.setOidcIdentity("expired"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectOidcCallbackDenied(page); + + stack.setOidcIdentity("ordinary"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectShell(page); + expect((await browserSession(page)).body.roles).toEqual(["user"]); + await expectNoWebStorageTokens(page); +}); diff --git a/frontend/e2e/f1.spec.ts b/frontend/e2e/f1.spec.ts index 16d0dedf..919f14e8 100644 --- a/frontend/e2e/f1.spec.ts +++ b/frontend/e2e/f1.spec.ts @@ -1,4 +1,5 @@ -import { test, expect } from "@playwright/test"; +import { expect, test, type Page } from "@playwright/test"; +import { createAuthenticationStack } from "./fixtures/auth-stack.mjs"; /** * E2E F1 loop — hermetic (no VPN, no real Pi, no real Python). @@ -9,20 +10,42 @@ import { test, expect } from "@playwright/test"; * which emits the f1_disambiguation.json scenario (select widget with * "interpretazione A" / "interpretazione B"). * - * Flow: open app → open NewSessionDialog → fill question → submit (Crea) + * Flow: authenticate locally → focus the new-session composer → fill question → submit (Send) * → wait for F1 select widget → click an option → assert no error shown. */ -test("F1 loop: new question → F1 widget → respond", async ({ page }) => { - await page.goto("/"); +test.describe.configure({ mode: "serial" }); - // Open the new-session dialog. - await page.getByRole("button", { name: /nuova/i }).click(); +let stack: Awaited>; + +test.beforeAll(async () => { + stack = await createAuthenticationStack({ withF1Workspace: true }); + await stack.useLocalMode(); +}); + +test.afterAll(async () => { + await stack?.close(); +}); + +async function signInLocally(page: Page): Promise { + const account = stack.localAccount("ordinary"); + await page.getByLabel("Username").fill(account.username); + await page.getByLabel("Password").fill(account.password); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(page.getByTestId("app-shell")).toBeVisible({ timeout: 30_000 }); +} + +test("F1 loop: new question → F1 widget → respond", async ({ page }) => { + await page.goto(stack.publicUrl); + await signInLocally(page); + + // Focus the composer for a new session. + await page.getByRole("button", { name: "New session", exact: true }).click(); // Fill in the question. - await page.getByLabel(/domanda/i).fill("quante cardioversioni nel 2024"); + await page.getByLabel("New question").fill("quante cardioversioni nel 2024"); // Submit — the backend creates the session (fake-tht) and spawns Pi (fake-pi). - await page.getByRole("button", { name: /^crea$/i }).click(); + await page.getByRole("button", { name: "Send", exact: true }).click(); // Wait for the F1 disambiguation select widget to appear. // The fake-pi emits extension_ui_request → backend bridges to SSE → frontend @@ -41,5 +64,5 @@ test("F1 loop: new question → F1 widget → respond", async ({ page }) => { await expect( page.getByRole("button", { name: /interpretazione A/i }), ).not.toBeVisible({ timeout: 10_000 }); - await expect(page.locator("body")).not.toContainText(/errore/i); + await expect(page.locator("body")).not.toContainText(/error/i); }); diff --git a/frontend/e2e/fixtures/auth-stack.mjs b/frontend/e2e/fixtures/auth-stack.mjs new file mode 100644 index 00000000..1a730fde --- /dev/null +++ b/frontend/e2e/fixtures/auth-stack.mjs @@ -0,0 +1,625 @@ +import { spawn } from "node:child_process"; +import { argon2 } from "node:crypto"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { createServer as createHttpServer } from "node:http"; +import { request as httpsRequest } from "node:https"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { startFakeOidcProvider } from "../../../backend/test/fixtures/oidc-provider.mjs"; + +const __dir = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(__dir, "../../.."); +const frontendRoot = join(repositoryRoot, "frontend"); +const backendRoot = join(repositoryRoot, "backend"); +const harnessRoot = join(repositoryRoot, "harness"); +const thtRoot = join(repositoryRoot, "tools", "tht"); +const fakePi = join(frontendRoot, "e2e", "fixtures", "fake-pi.mjs"); +const fakeTht = join(frontendRoot, "e2e", "fixtures", "fake-tht.mjs"); +const FIXTURE_CLIENT_ID = "thothii-e2e-client"; +const DEFAULT_FIXTURE_CLIENT_SECRET = "e2e-client-secret-not-a-production-secret"; +const DEFAULT_FIXTURE_API_TOKEN = "e2e-group-catalog-token-not-a-production-secret"; +const WRONG_CLIENT_ID = "thothii-e2e-unregistered-client"; +const WRONG_CLIENT_SECRET = "e2e-wrong-client-secret-not-production"; +const WRONG_API_TOKEN = "e2e-wrong-api-token-not-production"; +const OIDC_CREDENTIAL_VARIANTS = new Set([ + "correct", "wrong-client-id", "wrong-client-secret", "wrong-api-token", +]); + +function safeError(code) { + return new Error(code); +} + +function resolveFixtureCredentials() { + const runtimeCredential = process.env.THT_TASK15_SENTINEL; + if (runtimeCredential === undefined) { + return Object.freeze({ + clientSecret: DEFAULT_FIXTURE_CLIENT_SECRET, + apiToken: DEFAULT_FIXTURE_API_TOKEN, + runtime: false, + }); + } + if (runtimeCredential.length < 24 || runtimeCredential.length > 512 || /[\r\n\0]/u.test(runtimeCredential)) { + throw safeError("e2e_runtime_fixture_credential_invalid"); + } + return Object.freeze({ clientSecret: runtimeCredential, apiToken: runtimeCredential, runtime: true }); +} + +function buildAuthenticationStorageBridge(output) { + const result = spawn("go", ["build", "-o", output, "./cmd/tht"], { + cwd: thtRoot, + stdio: "ignore", + }); + return new Promise((resolveBuild, rejectBuild) => { + result.once("error", () => rejectBuild(safeError("e2e_auth_storage_build_failed"))); + result.once("exit", (code) => code === 0 ? resolveBuild() : rejectBuild(safeError("e2e_auth_storage_build_failed"))); + }); +} + +function secureDirectory(path) { + mkdirSync(path, { recursive: true, mode: 0o700 }); + chmodSync(path, 0o700); +} + +function runFixtureCommand(command, args, cwd) { + const child = spawn(command, args, { cwd, stdio: "ignore" }); + return new Promise((resolveCommand, rejectCommand) => { + child.once("error", () => rejectCommand(safeError("e2e_workspace_fixture_command_failed"))); + child.once("exit", (code) => code === 0 + ? resolveCommand() + : rejectCommand(safeError("e2e_workspace_fixture_command_failed"))); + }); +} + +const F1_WORKSPACE_ID = "fixture-workspace"; +const F1_WORKSPACE_DESCRIPTOR = `workspace: + schema_version: 3 + id: fixture-workspace + name: Fixture workspace + language: en +dwh: + engine: postgres + database: fixture + schema: fixture + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: fixture-workspace + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] + default: zai/glm-5.2 +`; + +async function prepareF1Workspace(root) { + const source = join(root, "workspace-source"); + const remote = join(root, "workspace-remote.git"); + secureDirectory(source); + secureDirectory(join(source, F1_WORKSPACE_ID)); + writeSecure(join(source, "thoth-workspaces.yaml"), [ + "schema_version: 1", + "workspaces:", + ` - id: ${F1_WORKSPACE_ID}`, + " name: Fixture workspace", + "", + ].join("\n")); + writeSecure(join(source, F1_WORKSPACE_ID, "workspace.yaml"), F1_WORKSPACE_DESCRIPTOR); + + await runFixtureCommand("git", ["init", "--bare", "--initial-branch=main", remote], root); + chmodSync(remote, 0o700); + await runFixtureCommand("git", ["init", "--initial-branch=main"], source); + await runFixtureCommand("git", ["config", "user.name", "ThothII E2E Fixture"], source); + await runFixtureCommand("git", ["config", "user.email", "thothii-e2e@example.invalid"], source); + await runFixtureCommand("git", ["add", "-A"], source); + await runFixtureCommand("git", ["commit", "-m", "Create deterministic fixture workspace"], source); + await runFixtureCommand("git", ["remote", "add", "origin", remote], source); + await runFixtureCommand("git", ["push", "origin", "main"], source); + return { id: F1_WORKSPACE_ID, remote }; +} + +function writeSecure(path, value) { + writeFileSync(path, value, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); +} + +async function testPasswordHash(password) { + const salt = Buffer.from("thothii-e2e-salt"); + const message = Buffer.from(password, "utf8"); + let digest; + try { + digest = await new Promise((resolveDigest, rejectDigest) => { + argon2("argon2id", { + message, + nonce: salt, + memory: 65_536, + passes: 3, + parallelism: 1, + tagLength: 32, + }, (error, derived) => error || !derived ? rejectDigest(error ?? safeError("e2e_password_hash_failed")) : resolveDigest(derived)); + }); + return `$argon2id$v=19$m=65536,t=3,p=1$${salt.toString("base64").replaceAll("=", "")}$${digest.toString("base64").replaceAll("=", "")}`; + } finally { + message.fill(0); + salt.fill(0); + digest?.fill(0); + } +} + +function pause(milliseconds) { + return new Promise((resolvePause) => setTimeout(resolvePause, milliseconds)); +} + +function providerJson(url, caFile, options = {}) { + return new Promise((resolveResponse, rejectResponse) => { + const body = options.body ?? ""; + const request = httpsRequest(url, { + method: options.method ?? "GET", + ca: readFileSync(caFile), + headers: { + accept: "application/json", + ...(body.length === 0 ? {} : { + "content-length": String(Buffer.byteLength(body)), + "content-type": "application/x-www-form-urlencoded", + }), + ...options.headers, + }, + }, (response) => { + const chunks = []; + let size = 0; + response.on("data", (chunk) => { + size += chunk.length; + if (size > 64 * 1024) request.destroy(safeError("e2e_provider_response_too_large")); + else chunks.push(chunk); + }); + response.once("error", () => rejectResponse(safeError("e2e_provider_response_failed"))); + response.once("end", () => { + try { + resolveResponse({ status: response.statusCode ?? 0, body: JSON.parse(Buffer.concat(chunks).toString("utf8")) }); + } catch { + rejectResponse(safeError("e2e_provider_response_invalid")); + } + }); + }); + request.once("error", () => rejectResponse(safeError("e2e_provider_request_failed"))); + request.end(body); + }); +} + +async function freeLoopbackPort() { + const server = createServer(); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen({ host: "127.0.0.1", port: 0 }, resolveListen); + }); + const address = server.address(); + await new Promise((resolveClose) => server.close(resolveClose)); + if (!address || typeof address === "string") throw safeError("e2e_loopback_port_unavailable"); + return address.port; +} + +const F1_QDRANT_INDEXES = Object.freeze([ + "content_hash", "document_id", "kind", "record_key", + "record_kind", "vector_generation", "workspace_id", "workspace_revision", +]); + +async function startFakeQdrant() { + const payloadSchema = Object.fromEntries(F1_QDRANT_INDEXES.map((field) => [field, { data_type: "keyword" }])); + const server = createHttpServer((request, response) => { + const path = new URL(request.url ?? "/", "http://loopback.invalid").pathname; + if (request.method !== "GET" || path !== `/collections/${F1_WORKSPACE_ID}`) { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ + result: { + config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, + payload_schema: payloadSchema, + }, + })); + }); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen({ host: "127.0.0.1", port: 0 }, resolveListen); + }); + const address = server.address(); + if (!address || typeof address === "string") { + await new Promise((resolveClose) => server.close(resolveClose)); + throw safeError("e2e_qdrant_loopback_port_unavailable"); + } + return { + baseUrl: `http://127.0.0.1:${address.port}/`, + close: () => new Promise((resolveClose) => server.close(resolveClose)), + }; +} + +function managedProcess(command, args, options) { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let exited = false; + let diagnostic = ""; + const captureDiagnostic = (chunk) => { + const sanitized = String(chunk) + .replace(/https?:\/\/[^\s)]+/g, "[url]") + .replace(/(?:THT_[A-Z_]+|PI_[A-Z_]+|AUTH_MODE)=\S+/g, "$1=[redacted]") + .replace(/[A-Za-z0-9_-]{43,}/g, "[redacted]"); + diagnostic = `${diagnostic}${sanitized}`.slice(-800); + }; + child.once("exit", () => { exited = true; }); + child.stdout?.on("data", captureDiagnostic); + child.stderr?.on("data", captureDiagnostic); + return { + child, + exited: () => exited, + diagnostic: () => diagnostic.replace(/\s+/g, " ").trim(), + async close() { + if (exited) return; + child.kill("SIGTERM"); + for (let attempt = 0; attempt < 50 && !exited; attempt += 1) await pause(20); + if (!exited) child.kill("SIGKILL"); + for (let attempt = 0; attempt < 50 && !exited; attempt += 1) await pause(20); + }, + }; +} + +function oneShotJson(command, args, options) { + return new Promise((resolveCommand) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ["ignore", "pipe", "ignore"], + }); + let output = ""; + let outputValid = true; + let settled = false; + const finish = (status) => { + if (settled) return; + settled = true; + clearTimeout(timer); + let report; + try { + report = outputValid ? JSON.parse(output) : undefined; + } catch { + report = undefined; + } + resolveCommand({ + status: Number.isInteger(status) ? status : 1, + report: report && typeof report === "object" + ? report + : { ready: false, mode: "none", checks: [{ code: "fixture_command_invalid" }] }, + }); + }; + child.stdout?.on("data", (chunk) => { + if (!outputValid) return; + output += String(chunk); + if (Buffer.byteLength(output) > 64 * 1024) { + output = ""; + outputValid = false; + child.kill("SIGKILL"); + } + }); + child.once("error", () => finish(1)); + child.once("exit", (status) => finish(status)); + const timer = setTimeout(() => child.kill("SIGKILL"), 35_000); + }); +} + +async function waitForOk(url, processHandle) { + for (let attempt = 0; attempt < 300; attempt += 1) { + if (processHandle.exited()) { + const detail = processHandle.diagnostic(); + throw safeError(detail ? `e2e_service_stopped_during_startup:${detail}` : "e2e_service_stopped_during_startup"); + } + try { + const response = await fetch(url, { redirect: "error" }); + if (response.ok) return; + } catch { + // The process is expected to race its listener setup. + } + await pause(100); + } + throw safeError("e2e_service_startup_timeout"); +} + +function cleanBackendEnvironment(overrides) { + const env = { ...process.env }; + for (const name of [ + "AUTH_MODE", "THT_AUTH_CONFIG_FILE", "THT_AUTH_STATE_ROOT", "THT_SECRETS_FILE", "NODE_EXTRA_CA_CERTS", + "SETTINGS_FILE", "THT_MAINTENANCE_FILE", "THT_WORKSPACE_REGISTRY_ROOT", "THT_WORKSPACE_SECRET_STORE_ROOT", + "THT_WORKSPACE_SECRET_RUNTIME_ROOT", "THT_WORKSPACE_GIT_REMOTE", "THT_WORKSPACE_GIT_BRANCH", + "THT_WORKSPACE_SECRET_ROOTS", "THT_WORKSPACE_INSTALLATION_ID", "THT_LEGACY_WORKSPACE_MODE", + "THT_DWH_PRECHECK", "THT_INTERNAL_QDRANT_URL", "THT_CONFIG", "THT_TASK15_SENTINEL", + ]) delete env[name]; + for (const name of Object.keys(env)) { + if (name.startsWith("THT_WS_")) delete env[name]; + } + return { ...env, ...overrides }; +} + +export async function createAuthenticationStack({ withF1Workspace = false } = {}) { + const credentials = resolveFixtureCredentials(); + const root = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-e2e-")); + secureDirectory(root); + const stateRoot = join(root, "auth-state"); + const registryRoot = join(root, "workspace-registry"); + const workspaceSecretRoot = join(root, "workspace-secrets"); + const workspaceRuntimeRoot = join(root, "workspace-runtime"); + const fixtureSecretRoot = join(root, "fixture-runtime-secrets"); + const providerRoot = join(root, "provider"); + const authConfigFile = join(root, "auth.yaml"); + const usersFile = join(root, "users.yaml"); + const secretsFile = join(root, "test.secrets"); + const settingsFile = join(root, "settings.json"); + const maintenanceFile = join(root, "maintenance.json"); + const authStorageBinary = join(root, "tht-auth-storage"); + const fixtureDwhPasswordFile = join(fixtureSecretRoot, "fixture-dwh-password"); + const fixtureDwhCaFile = join(fixtureSecretRoot, "fixture-dwh-ca.pem"); + for (const path of [stateRoot, registryRoot, workspaceSecretRoot, workspaceRuntimeRoot, fixtureSecretRoot, providerRoot]) secureDirectory(path); + for (const child of ["sessions", "oidc"]) secureDirectory(join(stateRoot, child)); + + const [frontendPort, backendPort] = await Promise.all([freeLoopbackPort(), freeLoopbackPort()]); + const publicUrl = `http://127.0.0.1:${frontendPort}`; + const backendUrl = `http://127.0.0.1:${backendPort}`; + const workspace = withF1Workspace ? await prepareF1Workspace(root) : undefined; + const provider = await startFakeOidcProvider({ + directory: providerRoot, + registration: { + clientId: FIXTURE_CLIENT_ID, + clientSecret: credentials.clientSecret, + redirectUri: new URL("/api/auth/oidc/callback", publicUrl).href, + }, + apiToken: credentials.apiToken, + }); + await buildAuthenticationStorageBridge(authStorageBinary); + const localPassword = "e2e-local-password"; + const passwordHash = await testPasswordHash(localPassword); + const accounts = Object.freeze({ + ordinary: Object.freeze({ username: "ordinary", password: localPassword }), + admin: Object.freeze({ username: "administrator", password: localPassword }), + }); + writeSecure(usersFile, JSON.stringify({ + version: 1, + users: [ + { + id: "11111111-1111-4111-8111-111111111111", + username: accounts.ordinary.username, + displayName: "Fixture ordinary", + passwordHash, + roles: ["user"], + enabled: true, + authRevision: 1, + }, + { + id: "22222222-2222-4222-8222-222222222222", + username: accounts.admin.username, + displayName: "Fixture administrator", + passwordHash, + roles: ["admin"], + enabled: true, + authRevision: 1, + }, + ], + })); + function writeOidcSecrets(variant) { + if (!OIDC_CREDENTIAL_VARIANTS.has(variant)) throw safeError("e2e_oidc_credential_variant_invalid"); + const selectedClientSecret = variant === "wrong-client-secret" ? WRONG_CLIENT_SECRET : credentials.clientSecret; + const selectedApiToken = variant === "wrong-api-token" ? WRONG_API_TOKEN : credentials.apiToken; + writeSecure(secretsFile, [ + `THT_OIDC_CLIENT_SECRET=${selectedClientSecret}`, + `THT_AUTHENTIK_API_TOKEN=${selectedApiToken}`, + "", + ].join("\n")); + } + writeOidcSecrets("correct"); + if (workspace) { + writeSecure(fixtureDwhPasswordFile, "fixture-password-not-a-secret\n"); + writeSecure(fixtureDwhCaFile, "fixture-ca-not-a-production-certificate\n"); + writeSecure(settingsFile, JSON.stringify({ + workspace: workspace.id, + provider: "zai", + model: "glm-5.2", + thinking: "medium", + })); + } + + let mode = undefined; + let backend = undefined; + let qdrant = undefined; + const frontend = managedProcess(join(frontendRoot, "node_modules", ".bin", "vite"), [ + "--host", "127.0.0.1", "--port", String(frontendPort), "--strictPort", + ], { + cwd: frontendRoot, + env: { + ...process.env, + THT_FRONTEND_API_UPSTREAM: backendUrl, + }, + }); + try { + await waitForOk(publicUrl, frontend); + qdrant = workspace ? await startFakeQdrant() : undefined; + + const localConfig = () => ({ + version: 1, + mode: "local", + publicUrl, + session: { + regularTtlSeconds: 600, + regularIdleSeconds: 600, + rememberTtlSeconds: 2_592_000, + rememberIdleSeconds: 604_800, + }, + local: { usersFile: "users.yaml" }, + }); + const oidcConfig = (variant = "correct") => ({ + version: 1, + mode: "oidc", + publicUrl, + session: { + regularTtlSeconds: 600, + regularIdleSeconds: 600, + oidcTtlSeconds: 60, + }, + oidc: { + issuer: provider.issuer, + clientId: variant === "wrong-client-id" ? WRONG_CLIENT_ID : FIXTURE_CLIENT_ID, + clientSecretRef: "THT_OIDC_CLIENT_SECRET", + scopes: ["openid", "profile", "groups"], + groupsClaim: "groups", + }, + groupCatalog: { + driver: "authentik", + baseUrl: provider.baseUrl, + apiTokenRef: "THT_AUTHENTIK_API_TOKEN", + }, + authorization: { + groupRoles: { + "fixture-users": ["user"], + "fixture-admin": ["admin"], + }, + }, + }); + + function backendEnvironment() { + return cleanBackendEnvironment({ + NODE_ENV: "test", + HOST: "127.0.0.1", + PORT: String(backendPort), + PI_BIN: fakePi, + THT_BIN: fakeTht, + THT_AUTH_STORAGE_BIN: authStorageBinary, + THT_HARNESS_DIR: harnessRoot, + THT_AUTH_CONFIG_FILE: authConfigFile, + THT_AUTH_STATE_ROOT: stateRoot, + THT_SECRETS_FILE: secretsFile, + NODE_EXTRA_CA_CERTS: provider.caFile, + SETTINGS_FILE: settingsFile, + THT_MAINTENANCE_FILE: maintenanceFile, + THT_WORKSPACE_REGISTRY_ROOT: registryRoot, + THT_WORKSPACE_SECRET_STORE_ROOT: workspaceSecretRoot, + THT_WORKSPACE_SECRET_RUNTIME_ROOT: workspaceRuntimeRoot, + THT_WORKSPACE_INSTALLATION_ID: "e2e", + THT_DATA_ROOT: join(root, "data"), + ...(qdrant ? { THT_INTERNAL_QDRANT_URL: qdrant.baseUrl } : {}), + ...(workspace ? { + THT_WORKSPACE_GIT_REMOTE: workspace.remote, + THT_WORKSPACE_GIT_BRANCH: "main", + THT_WORKSPACE_SECRET_ROOTS: fixtureSecretRoot, + THT_WS_FIXTURE_WORKSPACE_DWH_TRANSPORT: "postgres_direct", + THT_WS_FIXTURE_WORKSPACE_DWH_HOST: "127.0.0.1", + THT_WS_FIXTURE_WORKSPACE_DWH_PORT: "5432", + THT_WS_FIXTURE_WORKSPACE_DWH_USER: "fixture", + THT_WS_FIXTURE_WORKSPACE_DWH_PASSWORD_FILE: fixtureDwhPasswordFile, + THT_WS_FIXTURE_WORKSPACE_DWH_TLS_CA_FILE: fixtureDwhCaFile, + } : {}), + }); + } + + async function startBackend() { + if (mode === undefined) throw safeError("e2e_auth_mode_not_configured"); + backend = managedProcess(join(backendRoot, "node_modules", ".bin", "tsx"), ["src/server.ts"], { + cwd: backendRoot, + env: backendEnvironment(), + }); + await waitForOk(`${backendUrl}/health`, backend); + } + + async function restartBackend() { + await backend?.close(); + backend = undefined; + await startBackend(); + } + + return { + publicUrl, + localAccount(account) { + const found = accounts[account]; + if (!found) throw safeError("e2e_local_account_unknown"); + return found; + }, + lastAuthorization() { + return provider.lastAuthorization(); + }, + async providerSurface() { + const discovery = await providerJson(`${provider.issuer}.well-known/openid-configuration`, provider.caFile); + const jwks = await providerJson(`${provider.issuer}jwks`, provider.caFile); + const device = await providerJson(`${provider.issuer}device_authorization`, provider.caFile, { + method: "POST", + body: new URLSearchParams({ + client_id: FIXTURE_CLIENT_ID, + client_secret: credentials.clientSecret, + }).toString(), + }); + const deviceCode = typeof device.body?.device_code === "string" ? device.body.device_code : ""; + const deviceToken = deviceCode.length === 0 ? { status: 0, body: {} } : await providerJson(`${provider.issuer}token`, provider.caFile, { + method: "POST", + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + client_id: FIXTURE_CLIENT_ID, + client_secret: credentials.clientSecret, + device_code: deviceCode, + }).toString(), + }); + const groups = await providerJson(`${provider.baseUrl}/api/v3/core/groups/?name=fixture-users`, provider.caFile, { + headers: { authorization: `Bearer ${credentials.apiToken}` }, + }); + return { + discovery: discovery.status === 200 && discovery.body?.issuer === provider.issuer, + jwks: jwks.status === 200 && jwks.body?.keys?.[0]?.alg === "RS256" && jwks.body?.keys?.[0]?.use === "sig", + deviceAuthorization: device.status === 200 && typeof device.body?.device_code === "string" + && typeof device.body?.verification_uri === "string", + deviceToken: deviceToken.status === 200 && typeof deviceToken.body?.id_token === "string", + groupList: groups.status === 200 && groups.body?.results?.[0]?.name === "fixture-users", + runtimeCredential: credentials.runtime, + }; + }, + setOidcIdentity(identity) { + provider.setIdentity(identity); + }, + async useLocalMode() { + writeSecure(authConfigFile, JSON.stringify(localConfig())); + mode = "local"; + await restartBackend(); + }, + async useOidcMode(identity, variant = "correct") { + if (!OIDC_CREDENTIAL_VARIANTS.has(variant)) throw safeError("e2e_oidc_credential_variant_invalid"); + provider.setIdentity(identity); + writeOidcSecrets(variant); + writeSecure(authConfigFile, JSON.stringify(oidcConfig(variant))); + mode = "oidc"; + await restartBackend(); + }, + async authDiagnostics() { + if (mode !== "oidc") throw safeError("e2e_oidc_mode_required"); + return oneShotJson( + join(backendRoot, "node_modules", ".bin", "tsx"), + ["src/auth/diagnostic-command.ts", "--json"], + { cwd: backendRoot, env: backendEnvironment() }, + ); + }, + restartBackend, + async close() { + await backend?.close(); + await frontend.close(); + await provider.close(); + await qdrant?.close(); + rmSync(root, { recursive: true, force: true }); + }, + }; + } catch (error) { + await backend?.close(); + await frontend.close(); + await provider.close(); + await qdrant?.close(); + rmSync(root, { recursive: true, force: true }); + throw error; + } +} diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 42a43771..580ba972 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -1,50 +1,14 @@ import { defineConfig, devices } from "@playwright/test"; -import { resolve } from "node:path"; - -const __dir = new URL(".", import.meta.url).pathname; -const fakePi = resolve(__dir, "e2e/fixtures/fake-pi.mjs"); -const fakeTht = resolve(__dir, "e2e/fixtures/fake-tht.mjs"); -const harnessDir = resolve(__dir, "..", "harness"); export default defineConfig({ testDir: "./e2e", timeout: 60_000, retries: process.env.CI ? 2 : 0, - use: { - baseURL: "http://localhost:5199", - }, + workers: 1, projects: [ { name: "chromium", use: { ...devices["Desktop Chrome"] }, }, ], - webServer: [ - { - // Backend — wired to both fakes; no VPN/Pi/Python required. - command: "npm run dev", - cwd: resolve(__dir, "..", "backend"), - url: "http://localhost:8799/health", - timeout: 30_000, - reuseExistingServer: !process.env.CI, - env: { - PI_BIN: fakePi, - THT_BIN: fakeTht, - THT_HARNESS_DIR: harnessDir, - PORT: "8799", - AUTH_MODE: "none", - }, - }, - { - // Frontend dev server pointing at the hermetic backend. - command: "npm run dev -- --port 5199", - cwd: __dir, - url: "http://localhost:5199", - timeout: 30_000, - reuseExistingServer: !process.env.CI, - env: { - VITE_BACKEND_URL: "http://localhost:8799", - }, - }, - ], }); diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 0ad1a8e5..b9b03201 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -1,7 +1,15 @@ import { render, screen } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; +import { server } from "./test/msw"; import { App } from "./App"; -test("App renders the shell with create affordance", () => { +test("App renders the shell with create affordance after authentication", async () => { + server.use( + http.get("/api/settings", () => HttpResponse.json({ workspace: "psd", provider: "zai", model: "glm-5.2", thinking: "medium" })), + http.get("/api/workspaces", () => HttpResponse.json([])), + http.get("/api/models", () => HttpResponse.json({ models: [] })), + http.get("/api/sessions", () => HttpResponse.json([])), + ); render(); - expect(screen.getByRole("button", { name: /new/i })).toBeInTheDocument(); + expect(await screen.findByRole("button", { name: /new/i })).toBeInTheDocument(); }); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 97b8d367..3ac9b56c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,11 +1,11 @@ import { QueryClientProvider } from "@tanstack/react-query"; import { queryClient } from "./app/queryClient"; -import { AppShell } from "./shell/AppShell"; +import { AuthGate } from "./auth/AuthGate"; export function App() { return ( - + ); } diff --git a/frontend/src/api/auth.test.ts b/frontend/src/api/auth.test.ts new file mode 100644 index 00000000..5e022557 --- /dev/null +++ b/frontend/src/api/auth.test.ts @@ -0,0 +1,64 @@ +import { beforeEach, expect, test, vi } from "vitest"; +import { http, HttpResponse } from "msw"; +import { beginOidcLogin, logout } from "./auth"; +import { server } from "../test/msw"; +import { clearAuthState, setAuthState } from "../auth/authState"; + +const user = { + issuer: "local", subject: "user-a", roles: ["user"] as const, + permissions: ["session.use"], isAdmin: false, csrfToken: "a".repeat(43), session: null, +}; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((onResolve) => { resolve = onResolve; }); + return { promise, resolve }; +} + +beforeEach(() => { + clearAuthState(); + setAuthState(user); +}); + +test("OIDC navigation waits for a successful pending logout response", async () => { + const gate = deferred(); + let logoutStarted!: () => void; + const started = new Promise((resolve) => { logoutStarted = resolve; }); + server.use(http.post("/api/auth/logout", async () => { + logoutStarted(); + await gate.promise; + return new HttpResponse(null, { status: 204 }); + })); + const logoutPromise = logout(); + await started; + const navigate = vi.fn(); + const oidc = beginOidcLogin(navigate); + + await Promise.resolve(); + expect(navigate).not.toHaveBeenCalled(); + gate.resolve(); + await Promise.all([logoutPromise, oidc]); + expect(navigate).toHaveBeenCalledOnce(); + expect(navigate).toHaveBeenCalledWith("/api/auth/oidc/login"); +}); + +test("OIDC navigation waits for a failed pending logout response before continuing", async () => { + const gate = deferred(); + let logoutStarted!: () => void; + const started = new Promise((resolve) => { logoutStarted = resolve; }); + server.use(http.post("/api/auth/logout", async () => { + logoutStarted(); + await gate.promise; + return HttpResponse.json({ code: "auth_unavailable" }, { status: 503 }); + })); + const logoutPromise = logout().catch(() => undefined); + await started; + const navigate = vi.fn(); + const oidc = beginOidcLogin(navigate); + + await Promise.resolve(); + expect(navigate).not.toHaveBeenCalled(); + gate.resolve(); + await Promise.all([logoutPromise, oidc]); + expect(navigate).toHaveBeenCalledWith("/api/auth/oidc/login"); +}); diff --git a/frontend/src/api/auth.ts b/frontend/src/api/auth.ts new file mode 100644 index 00000000..baac8539 --- /dev/null +++ b/frontend/src/api/auth.ts @@ -0,0 +1,132 @@ +import { ApiError, apiFetch } from "./client"; +import type { AuthenticatedUser, AuthPublicConfig, AuthRole, AuthSessionInfo } from "./types"; +import { clearAuthStateIfCurrent, getAuthGeneration, setAuthState } from "../auth/authState"; + +const authModes = new Set(["local", "oidc", "upstream", "none", "mock"]); +const roles = new Set(["user", "admin"]); +const sessionMethods = new Set(["local", "oidc", "upstream"]); +let pendingLogoutResponse: Promise | null = null; +const oidcLoginPath = "/api/auth/oidc/login"; + +function record(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; +} + +function parseAuthConfig(value: unknown): AuthPublicConfig { + const source = record(value); + if (!source || !authModes.has(source.mode as AuthPublicConfig["mode"]) + || typeof source.localLogin !== "boolean" || typeof source.oidcLogin !== "boolean") { + throw new Error("Authentication service returned an invalid public configuration"); + } + return { + mode: source.mode as AuthPublicConfig["mode"], + localLogin: source.localLogin, + oidcLogin: source.oidcLogin, + }; +} + +function parseUser(value: unknown): AuthenticatedUser { + const source = record(value); + if (!source || typeof source.issuer !== "string" || typeof source.subject !== "string") { + throw new Error("Authentication service returned an invalid user"); + } + const parsedRoles = Array.isArray(source.roles) + ? source.roles.filter((role): role is AuthRole => typeof role === "string" && roles.has(role as AuthRole)) + : []; + const parsedPermissions = Array.isArray(source.permissions) + ? source.permissions.filter((permission): permission is string => typeof permission === "string") + : []; + const csrfToken = source.csrfToken === undefined || source.csrfToken === null + ? null + : typeof source.csrfToken === "string" ? source.csrfToken : null; + const sessionSource = source.session === null || source.session === undefined + ? undefined + : record(source.session); + const session = sessionSource + && sessionMethods.has(sessionSource.method as AuthSessionInfo["method"]) + && typeof sessionSource.remembered === "boolean" + && typeof sessionSource.idleExpiresAt === "string" + && typeof sessionSource.absoluteExpiresAt === "string" + ? { + method: sessionSource.method as AuthSessionInfo["method"], + remembered: sessionSource.remembered, + idleExpiresAt: sessionSource.idleExpiresAt, + absoluteExpiresAt: sessionSource.absoluteExpiresAt, + } + : null; + return { + issuer: source.issuer, + subject: source.subject, + ...(typeof source.displayName === "string" ? { displayName: source.displayName } : {}), + roles: parsedRoles, + permissions: parsedPermissions, + isAdmin: source.isAdmin === true, + csrfToken, + session, + }; +} + +export async function getAuthConfig(): Promise { + return parseAuthConfig(await apiFetch("/auth/config")); +} + +export async function getMe(): Promise { + return parseUser(await apiFetch("/me")); +} + +export async function loginLocal(username: string, password: string, remember: boolean): Promise { + if (pendingLogoutResponse) await pendingLogoutResponse; + await apiFetch("/auth/local/login", { + method: "POST", + body: JSON.stringify({ username, password, remember }), + }); + const user = await getMe(); + setAuthState(user); + return user; +} + +/** + * OIDC creates its browser session through a top-level same-origin navigation. A previous logout + * response may still carry a clearing Set-Cookie, so it must settle before this navigation begins. + */ +export async function beginOidcLogin(navigate: (path: string) => void = (path) => window.location.assign(path)): Promise { + const pending = pendingLogoutResponse; + if (pending) { + try { + await pending; + } catch { + // A failed logout must release the coordinator; the current browser cookie remains the + // backend's authority during the following OIDC handshake. + } + } + navigate(oidcLoginPath); +} + +export async function logout(): Promise { + const logoutGeneration = getAuthGeneration(); + if (pendingLogoutResponse) { + await pendingLogoutResponse; + return false; + } + let cleared = false; + const response = (async () => { + try { + await apiFetch("/auth/logout", { method: "POST" }); + } finally { + cleared = clearAuthStateIfCurrent(logoutGeneration); + } + })(); + let trackedResponse: Promise; + trackedResponse = response.finally(() => { + if (pendingLogoutResponse === trackedResponse) pendingLogoutResponse = null; + }); + pendingLogoutResponse = trackedResponse; + await trackedResponse; + return cleared; +} + +export function authErrorStatus(error: unknown): number | undefined { + return error instanceof ApiError ? error.status : undefined; +} diff --git a/frontend/src/api/backend-url-cases.json b/frontend/src/api/backend-url-cases.json index 6648ede9..6daccf5a 100644 --- a/frontend/src/api/backend-url-cases.json +++ b/frontend/src/api/backend-url-cases.json @@ -1,15 +1,15 @@ [ - { "value": "", "valid": true }, - { "value": "/", "valid": true }, { "value": "/api", "valid": true }, - { "value": "/api/", "valid": true }, - { "value": "/datamart-builder/api", "valid": true }, - { "value": "/datamart-builder/api/", "valid": true }, - { "value": "http://localhost:8787", "valid": true }, - { "value": "https://api.example.test/v1", "valid": true }, - { "value": "https://api.example.test/base/path/", "valid": true }, - { "value": "http://127.0.0.1:1/api", "valid": true }, - { "value": "http://[::1]:8787/api", "valid": true }, + { "value": "", "valid": false }, + { "value": "/", "valid": false }, + { "value": "/api/", "valid": false }, + { "value": "/datamart-builder/api", "valid": false }, + { "value": "/datamart-builder/api/", "valid": false }, + { "value": "http://localhost:8787", "valid": false }, + { "value": "https://api.example.test/v1", "valid": false }, + { "value": "https://api.example.test/base/path/", "valid": false }, + { "value": "http://127.0.0.1:1/api", "valid": false }, + { "value": "http://[::1]:8787/api", "valid": false }, { "value": "/backend", "valid": false }, { "value": "api", "valid": false }, { "value": "//evil.test", "valid": false }, diff --git a/frontend/src/api/backend-url-policy.json b/frontend/src/api/backend-url-policy.json index 05087223..470194fa 100644 --- a/frontend/src/api/backend-url-policy.json +++ b/frontend/src/api/backend-url-policy.json @@ -1,5 +1,5 @@ { - "relativeBases": ["", "/", "/api", "/api/", "/datamart-builder/api", "/datamart-builder/api/"], + "relativeBases": ["/api"], "absolutePattern": "^https?://(?:\\[[0-9A-Fa-f:.]+\\]|[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?)(?::[0-9]+)?(?:/[^\\s?#]*)?/?$", "maxPort": 65535, "queryAllowed": false, diff --git a/frontend/src/api/client.test.ts b/frontend/src/api/client.test.ts index bf9bcfbb..7f6474f2 100644 --- a/frontend/src/api/client.test.ts +++ b/frontend/src/api/client.test.ts @@ -1,11 +1,25 @@ import { http, HttpResponse } from "msw"; import { server } from "../test/msw"; -import { apiFetch } from "./client"; +import { apiErrorMessage, apiFetch, ApiError, assertSameOriginRequestUrl } from "./client"; +import { clearAuthState, setAuthState } from "../auth/authState"; + +const authenticated = { + issuer: "local", + subject: "user-1", + roles: ["user"] as const, + permissions: ["session.use"], + isAdmin: false, + csrfToken: "c".repeat(43), + session: null, +}; + +beforeEach(() => setAuthState(authenticated)); +afterEach(() => clearAuthState()); test("body-less POST omits content-type (avoids Fastify empty-body 400)", async () => { let contentType: string | null = "unset"; server.use( - http.post("http://localhost:8787/sessions/s1/resume", ({ request }) => { + http.post("/api/sessions/s1/resume", ({ request }) => { contentType = request.headers.get("content-type"); return HttpResponse.json({ id: "s1" }); }), @@ -17,7 +31,7 @@ test("body-less POST omits content-type (avoids Fastify empty-body 400)", async test("POST with a body sends application/json content-type", async () => { let contentType: string | null = null; server.use( - http.post("http://localhost:8787/sessions/s1/steer", ({ request }) => { + http.post("/api/sessions/s1/steer", ({ request }) => { contentType = request.headers.get("content-type"); return new HttpResponse(null, { status: 204 }); }), @@ -25,3 +39,258 @@ test("POST with a body sends application/json content-type", async () => { await apiFetch("/sessions/s1/steer", { method: "POST", body: JSON.stringify({ text: "hi" }) }); expect(contentType).toContain("application/json"); }); + +test("same-origin requests include credentials and overwrite the CSRF header from memory", async () => { + let observed: { credentials: string | null; csrf: string | null } | undefined; + const fetchSpy = vi.spyOn(globalThis, "fetch"); + server.use( + http.post("/api/sessions/s1/steer", ({ request }) => { + observed = { + credentials: request.headers.get("credentials"), + csrf: request.headers.get("x-thothii-csrf"), + }; + return new HttpResponse(null, { status: 204 }); + }), + ); + + await apiFetch("/sessions/s1/steer", { + method: "POST", + headers: { "X-ThothII-CSRF": "attacker-supplied" }, + body: JSON.stringify({ text: "hi" }), + }); + + expect(observed?.csrf).toBe("c".repeat(43)); + expect(observed?.credentials).toBeNull(); + expect(fetchSpy.mock.calls.at(-1)?.[1]).toMatchObject({ credentials: "same-origin" }); + fetchSpy.mockRestore(); +}); + +test("a 401 clears the in-memory auth state and advances its generation", async () => { + const before = (await import("../auth/authState")).getAuthGeneration(); + server.use(http.get("/api/private", () => new HttpResponse(null, { status: 401 }))); + + await expect(apiFetch("/private")).rejects.toMatchObject({ status: 401 }); + + const state = await import("../auth/authState"); + expect(state.getAuthState()).toBeNull(); + expect(state.getAuthGeneration()).toBeGreaterThan(before); +}); + +test("a delayed 401 from user A cannot clear user B after a new login", async () => { + let release!: () => void; + const delayed = new Promise((resolve) => { release = resolve; }); + server.use(http.get("/api/stale-request", async () => { + await delayed; + return new HttpResponse(null, { status: 401 }); + })); + + const request = apiFetch("/stale-request"); + const userB = { ...authenticated, subject: "user-b", csrfToken: "b".repeat(43) }; + setAuthState(userB); + release(); + + await expect(request).rejects.toMatchObject({ status: 401 }); + expect((await import("../auth/authState")).getAuthState()).toMatchObject({ subject: "user-b" }); +}); + +test("bounds streamed error bodies and never exposes raw HTML or secrets", async () => { + const secret = "TOP-SECRET-token-123"; + const hugeBody = `${secret}${"x".repeat(20_000)}`; + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(hugeBody.slice(0, 4_000))); + controller.enqueue(new TextEncoder().encode(hugeBody.slice(4_000))); + controller.close(); + }, + }), { status: 500, headers: { "content-type": "text/html", "content-length": "1" } }), + ); + + try { + const result = await apiFetch("/oversized").catch((error: unknown) => error); + expect(result).toBeInstanceOf(ApiError); + const failure = result as ApiError; + expect(failure.status).toBe(500); + expect(failure.message).not.toContain(secret); + expect(failure.message).not.toContain(""); + expect(failure.bodyText).not.toContain(secret); + expect(failure.payload).toBeUndefined(); + } finally { + fetchSpy.mockRestore(); + } +}); + +test("keeps only a known safe bounded JSON error payload", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ + code: "dwh_unreachable", + error: "The data warehouse is unreachable.", + }), { status: 503, headers: { "content-type": "application/json" } }), + ); + + try { + const result = await apiFetch("/known-error").catch((error: unknown) => error); + const failure = result as ApiError; + expect(failure).toMatchObject({ + status: 503, + code: "dwh_unreachable", + payload: { code: "dwh_unreachable" }, + }); + expect(failure.bodyText).toBe(""); + expect(failure.message).toBe("The database is unreachable. Please retry."); + } finally { + fetchSpy.mockRestore(); + } +}); + +test("derives local messages without retaining a malicious known-code message", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ + code: "dwh_unreachable", + message: "Bearer eyJhbGciOiJIUzI1NiJ9.password=do-not-show", + error: "password=do-not-show", + }), { status: 503, headers: { "content-type": "application/json" } }), + ); + + try { + const failure = await apiFetch("/known-malicious").catch((error: unknown) => error) as ApiError; + expect(failure.code).toBe("dwh_unreachable"); + expect(failure.message).toBe("The database is unreachable. Please retry."); + expect(failure.bodyText).toBe(""); + expect(failure.payload).toEqual({ code: "dwh_unreachable" }); + expect(failure.message).not.toMatch(/Bearer|password|html|do-not-show/i); + } finally { + fetchSpy.mockRestore(); + } +}); + +test("uses a generic local message for unknown malicious codes", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ + code: "unknown-secret-code", + message: "Bearer eyJhbGciOiJIUzI1NiJ9", + error: "password=do-not-show", + }), { status: 500, headers: { "content-type": "application/json" } }), + ); + + try { + const failure = await apiFetch("/unknown-malicious").catch((error: unknown) => error) as ApiError; + expect(failure.code).toBeUndefined(); + expect(failure.message).toBe("Request failed. Please try again."); + expect(failure.bodyText).toBe(""); + expect(failure.payload).toBeUndefined(); + expect(apiErrorMessage(failure)).toBe("Request failed. Please try again."); + } finally { + fetchSpy.mockRestore(); + } +}); + +test("releases the response reader after a successful bounded read", async () => { + const reader = { + read: vi.fn() + .mockResolvedValueOnce({ done: false, value: new TextEncoder().encode("{not-json") }) + .mockResolvedValueOnce({ done: true, value: undefined }), + cancel: vi.fn().mockResolvedValue(undefined), + releaseLock: vi.fn(), + }; + const response = new Response(new ReadableStream(), { status: 500 }); + vi.spyOn(response.body!, "getReader").mockReturnValue(reader as unknown as ReadableStreamReader>); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(response); + + try { + await expect(apiFetch("/reader-success")).rejects.toBeInstanceOf(ApiError); + expect(reader.cancel).not.toHaveBeenCalled(); + expect(reader.releaseLock).toHaveBeenCalledOnce(); + } finally { + fetchSpy.mockRestore(); + } +}); + +test("cancels and releases the response reader on overflow", async () => { + const reader = { + read: vi.fn() + .mockResolvedValueOnce({ done: false, value: new Uint8Array(9 * 1024) }), + cancel: vi.fn().mockResolvedValue(undefined), + releaseLock: vi.fn(), + }; + const response = new Response(new ReadableStream(), { status: 500 }); + vi.spyOn(response.body!, "getReader").mockReturnValue(reader as unknown as ReadableStreamReader>); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(response); + + try { + await expect(apiFetch("/reader-overflow")).rejects.toBeInstanceOf(ApiError); + expect(reader.cancel).toHaveBeenCalledOnce(); + expect(reader.releaseLock).toHaveBeenCalledOnce(); + } finally { + fetchSpy.mockRestore(); + } +}); + +test("cancels and releases the response reader when a read throws", async () => { + const reader = { + read: vi.fn().mockRejectedValue(new Error("stream broke")), + cancel: vi.fn().mockResolvedValue(undefined), + releaseLock: vi.fn(), + }; + const response = new Response(new ReadableStream(), { status: 500 }); + vi.spyOn(response.body!, "getReader").mockReturnValue(reader as unknown as ReadableStreamReader>); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(response); + + try { + await expect(apiFetch("/reader-throws")).rejects.toThrow("stream broke"); + expect(reader.cancel).toHaveBeenCalledOnce(); + expect(reader.releaseLock).toHaveBeenCalledOnce(); + } finally { + fetchSpy.mockRestore(); + } +}); + +test("rejects malformed and sensitive JSON error bodies without surfacing their content", async () => { + const bodies = [ + "{not-json", + JSON.stringify({ code: "unknown_secret_code", error: "password=super-secret" }), + ]; + + for (const body of bodies) { + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(body, { status: 500, headers: { "content-type": "application/json" } }), + ); + try { + const result = await apiFetch("/unsafe-error").catch((error: unknown) => error); + const failure = result as ApiError; + expect(failure.payload).toBeUndefined(); + expect(failure.message).not.toContain("super-secret"); + expect(failure.message).not.toContain("not-json"); + } finally { + fetchSpy.mockRestore(); + } + } + + const knownCode = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ code: "auth_forbidden", error: "token=super-secret" }), { + status: 403, headers: { "content-type": "application/json" }, + }), + ); + try { + const result = await apiFetch("/known-sensitive-error").catch((error: unknown) => error); + const failure = result as ApiError; + expect(failure.payload).toEqual({ code: "auth_forbidden" }); + expect(failure.message).toBe("Access is not permitted."); + expect(failure.message).not.toContain("super-secret"); + } finally { + knownCode.mockRestore(); + } +}); + +test("refuses a cross-origin request before sending credentials", () => { + expect(() => assertSameOriginRequestUrl("https://attacker.example/api/me")).toThrow(/same-origin/i); +}); + +test("preserves explicit 403 and 503 statuses for presentation", async () => { + server.use( + http.get("/api/forbidden", () => HttpResponse.json({ code: "auth_forbidden" }, { status: 403 })), + http.get("/api/unavailable", () => HttpResponse.json({ code: "auth_unavailable" }, { status: 503 })), + ); + await expect(apiFetch("/forbidden")).rejects.toBeInstanceOf(ApiError); + await expect(apiFetch("/unavailable")).rejects.toMatchObject({ status: 503 }); +}); diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index acae0e53..ec92d02b 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,34 +1,195 @@ import { backendBaseUrl as BASE, joinBackendPath } from "./runtime-config"; +import { + clearAuthStateIfCurrent, + getAuthGeneration, + getAuthState, +} from "../auth/authState"; + +const MAX_ERROR_BODY_BYTES = 8 * 1024; +const safeErrorCodes = new Set([ + "auth_forbidden", "auth_invalid_credentials", "auth_not_authorized", "auth_unavailable", + "auth_not_implemented", "authentication_required", "invalid_credentials", "login_rate_limited", + "csrf_failed", "csrf_invalid", "dwh_unreachable", "model_unavailable", + "workspace_invalid", "binding_missing", "workspace_not_activatable", "workspace_stale", + "git_unavailable", "git_auth_failed", "git_non_fast_forward", "connector_unavailable", + "semantic_index_incompatible", "pi_management_forbidden", "pi_management_unavailable", + "pi_management_invalid_config", "pi_management_write_failed", +]); + +type SafeErrorPayload = { + code: string; +}; + +const localCodeMessages: Record = { + auth_forbidden: "Access is not permitted.", + auth_invalid_credentials: "Invalid username or password.", + auth_not_authorized: "Access is not permitted.", + auth_unavailable: "Authentication is temporarily unavailable. Try again.", + auth_not_implemented: "This sign-in method is not available.", + authentication_required: "Please sign in to continue.", + invalid_credentials: "Invalid username or password.", + login_rate_limited: "Too many sign-in attempts. Try again later.", + csrf_failed: "The security check failed. Please retry.", + csrf_invalid: "The security check failed. Please retry.", + dwh_unreachable: "The database is unreachable. Please retry.", + model_unavailable: "The model provider is unavailable. Please retry.", + workspace_invalid: "The workspace configuration is invalid.", + binding_missing: "The workspace is missing a required binding.", + workspace_not_activatable: "The workspace cannot be activated.", + workspace_stale: "The workspace has changed. Refresh and try again.", + git_unavailable: "The workspace repository is unavailable.", + git_auth_failed: "The workspace repository could not be authenticated.", + git_non_fast_forward: "The workspace repository has moved. Refresh and try again.", + connector_unavailable: "A workspace connector is unavailable.", + semantic_index_incompatible: "The workspace semantic index is incompatible.", + pi_management_forbidden: "Pi management is not permitted", + pi_management_unavailable: "Pi management is unavailable.", + pi_management_invalid_config: "The Pi configuration is invalid.", + pi_management_write_failed: "The Pi configuration could not be saved.", +}; + +const localStatusMessages: Record = { + 401: "Please sign in to continue.", + 403: "Access is not permitted.", + 404: "The requested resource was not found.", + 409: "The request conflicts with current workspace state.", + 429: "Too many requests. Try again later.", + 500: "Request failed. Please try again.", + 502: "The service is unavailable. Please retry.", + 503: "The service is temporarily unavailable. Please retry.", +}; + +const GENERIC_ERROR_MESSAGE = "Request failed. Please try again."; + +function localErrorMessage(status: number, code?: string): string { + return (code && localCodeMessages[code]) || localStatusMessages[status] || GENERIC_ERROR_MESSAGE; +} /** - * Error thrown for non-2xx responses. `.message` stays " " for - * backward compatibility; `.status` and `.payload` (parsed JSON body, if any) - * let callers branch on a specific failure — e.g. a `code: "dwh_unreachable"`. + * Error thrown for non-2xx responses. The message contains only status and a + * whitelisted error code; `.payload` contains a bounded, sanitized JSON shape. */ export class ApiError extends Error { - constructor(readonly status: number, readonly bodyText: string, readonly payload: unknown) { - super(`${status} ${bodyText}`); + constructor( + readonly status: number, + readonly bodyText: string, + readonly payload: SafeErrorPayload | undefined, + ) { + super(localErrorMessage(status, payload?.code)); this.name = "ApiError"; } + + get code(): string | undefined { + return this.payload?.code; + } +} + +export function apiErrorMessage(error: unknown): string { + return error instanceof ApiError ? error.message : GENERIC_ERROR_MESSAGE; +} + +function parseSafeErrorPayload(text: string, truncated: boolean): SafeErrorPayload | undefined { + if (truncated || text.length === 0) return undefined; + let parsed: unknown; + try { parsed = JSON.parse(text); } catch { return undefined; } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return undefined; + const source = parsed as Record; + if (typeof source.code !== "string" || !safeErrorCodes.has(source.code)) return undefined; + return { code: source.code }; +} + +async function readBoundedText(response: Response): Promise<{ text: string; truncated: boolean }> { + const reader = response.body?.getReader(); + if (!reader) return { text: "", truncated: false }; + const decoder = new TextDecoder(); + let text = ""; + let bytes = 0; + let truncated = false; + try { + while (true) { + const next = await reader.read(); + if (next.done) break; + const remaining = MAX_ERROR_BODY_BYTES - bytes; + if (remaining <= 0) { + truncated = true; + await reader.cancel(); + break; + } + const chunk = next.value.byteLength > remaining ? next.value.slice(0, remaining) : next.value; + bytes += chunk.byteLength; + text += decoder.decode(chunk, { stream: next.value.byteLength <= remaining }); + if (next.value.byteLength > remaining) { + truncated = true; + await reader.cancel(); + break; + } + } + } catch (error) { + try { await reader.cancel(); } catch { /* preserve the original read failure */ } + throw error; + } finally { + text += decoder.decode(); + try { reader.releaseLock(); } catch { /* a completed browser reader may already be released */ } + } + return { text, truncated }; +} + +function requestHeaders(init: RequestInit | undefined): Headers { + const headers = new Headers(init?.headers); + // Fetch supplies the multipart boundary for FormData. Declaring JSON here + // would prevent Fastify from parsing an imported workspace bundle. + if ( + init?.body != null + && !(typeof FormData !== "undefined" && init.body instanceof FormData) + && !headers.has("content-type") + ) { + headers.set("content-type", "application/json"); + } + return headers; +} + +async function request(path: string, init?: RequestInit): Promise { + const url = joinBackendPath(BASE, path); + assertSameOriginRequestUrl(url); + const dispatchGeneration = getAuthGeneration(); + const headers = requestHeaders(init); + const method = (init?.method ?? "GET").toUpperCase(); + if (["POST", "PUT", "PATCH", "DELETE"].includes(method)) { + headers.delete("X-ThothII-CSRF"); + const csrfToken = getAuthState()?.csrfToken; + if (csrfToken) headers.set("X-ThothII-CSRF", csrfToken); + } + const res = await fetch(url, { + ...init, + credentials: "same-origin", + headers, + }); + if (res.status === 401) clearAuthStateIfCurrent(dispatchGeneration); + if (!res.ok) { + const { text, truncated } = await readBoundedText(res); + const payload = parseSafeErrorPayload(text, truncated); + throw new ApiError(res.status, "", payload); + } + return res; +} + +/** Refuse a credentialed cross-origin base before the browser can send a request. */ +export function assertSameOriginRequestUrl(url: string): void { + if (typeof window === "undefined") { + if (/^https?:\/\//i.test(url)) throw new Error("The browser must use the same-origin /api route"); + return; + } + const parsed = new URL(url, window.location.origin); + if (parsed.origin !== window.location.origin) { + throw new Error("The browser must use the same-origin /api route"); + } } export async function apiFetch(path: string, init?: RequestInit): Promise { // Only declare a JSON content-type when we actually send a body. Body-less // POSTs (resume, close) would otherwise make Fastify reject the empty body // with FST_ERR_CTP_EMPTY_JSON_BODY (400). - const headers: Record = { - ...(init?.headers as Record | undefined), - }; - if (init?.body != null && !("content-type" in headers) && !("Content-Type" in headers)) { - headers["content-type"] = "application/json"; - } - const res = await fetch(joinBackendPath(BASE, path), { ...init, headers }); - if (!res.ok) { - const bodyText = await res.text().catch(() => ""); - let payload: unknown; - try { payload = bodyText ? JSON.parse(bodyText) : undefined; } catch { payload = undefined; } - throw new ApiError(res.status, bodyText, payload); - } + const res = await request(path, init); if (res.status === 204) return undefined as T; // Accepted fire-and-forget endpoints may legitimately return 202 with no // representation. Keep apiFetch useful for both 202 and 204 contracts. @@ -36,4 +197,9 @@ export async function apiFetch(path: string, init?: RequestInit): Promise return body ? (JSON.parse(body) as T) : (undefined as T); } +/** Download registry bundles without attempting to parse their ZIP body as JSON. */ +export async function apiFetchBlob(path: string, init?: RequestInit): Promise { + return (await request(path, init)).blob(); +} + export { BASE }; diff --git a/frontend/src/api/pi-management.test.ts b/frontend/src/api/pi-management.test.ts new file mode 100644 index 00000000..52968706 --- /dev/null +++ b/frontend/src/api/pi-management.test.ts @@ -0,0 +1,67 @@ +import { http, HttpResponse } from "msw"; +import { server } from "../test/msw"; +import { + asPiManagementApiError, + getPiManagementLogs, + getPiManagementOptions, + getPiManagementStatus, + runPiManagementTest, + savePiManagementConfig, +} from "./pi-management"; + +test("Pi management client calls only the sanctioned sanitized endpoints", async () => { + const calls: Array<{ method: string; path: string; body?: unknown }> = []; + server.use( + http.get("/api/pi-management/status", ({ request }) => { + calls.push({ method: request.method, path: new URL(request.url).pathname }); + return HttpResponse.json({ ready: true, version: "0.80.3", credentials: "present", config: {}, checkedAt: "2026-08-05T10:00:00.000Z" }); + }), + http.get("/api/pi-management/options", ({ request }) => { + calls.push({ method: request.method, path: new URL(request.url).pathname }); + return HttpResponse.json({ providers: ["zai"], models: [{ provider: "zai", id: "glm-5.2" }], reasoning: ["low", "medium", "high"], checkedAt: "2026-08-05T10:00:00.000Z" }); + }), + http.put("/api/pi-management/config", async ({ request }) => { + calls.push({ method: request.method, path: new URL(request.url).pathname, body: await request.json() }); + return HttpResponse.json({ provider: "zai", model: "glm-5.2", reasoning: "high", updatedAt: "2026-08-05T10:00:00.000Z" }); + }), + http.post("/api/pi-management/test", ({ request }) => { + calls.push({ method: request.method, path: new URL(request.url).pathname }); + return HttpResponse.json({ ready: true, checkedAt: "2026-08-05T10:00:00.000Z" }); + }), + http.get("/api/pi-management/logs", ({ request }) => { + calls.push({ method: request.method, path: new URL(request.url).pathname }); + return HttpResponse.json({ lines: ["Pi smoke check succeeded"], checkedAt: "2026-08-05T10:00:00.000Z" }); + }), + ); + + await expect(getPiManagementStatus()).resolves.toMatchObject({ version: "0.80.3", ready: true, credentials: "present" }); + await expect(getPiManagementOptions()).resolves.toMatchObject({ providers: ["zai"] }); + await expect(savePiManagementConfig({ provider: "zai", model: "glm-5.2", reasoning: "high" })).resolves.toMatchObject({ reasoning: "high" }); + await expect(runPiManagementTest()).resolves.toMatchObject({ ready: true }); + await expect(getPiManagementLogs()).resolves.toMatchObject({ lines: ["Pi smoke check succeeded"] }); + + expect(calls).toEqual([ + { method: "GET", path: "/api/pi-management/status" }, + { method: "GET", path: "/api/pi-management/options" }, + { method: "PUT", path: "/api/pi-management/config", body: { provider: "zai", model: "glm-5.2", reasoning: "high" } }, + { method: "POST", path: "/api/pi-management/test" }, + { method: "GET", path: "/api/pi-management/logs" }, + ]); + expect(calls.some((call) => /update|terminal|shell/i.test(call.path))).toBe(false); +}); + +test("Pi management client exposes the stable forbidden message without raw response text", async () => { + server.use(http.get("/api/pi-management/status", () => + HttpResponse.json({ code: "pi_management_forbidden", error: "Pi management is not permitted", raw: "token=do-not-show" }, { status: 403 }), + )); + + await expect(getPiManagementStatus()).rejects.toSatisfy((error: unknown) => { + expect(asPiManagementApiError(error)).toEqual({ + status: 403, + code: "pi_management_forbidden", + message: "Pi management is not permitted", + }); + expect(asPiManagementApiError(error)?.message).not.toContain("token"); + return true; + }); +}); diff --git a/frontend/src/api/pi-management.ts b/frontend/src/api/pi-management.ts new file mode 100644 index 00000000..2b12a801 --- /dev/null +++ b/frontend/src/api/pi-management.ts @@ -0,0 +1,75 @@ +import { ApiError, apiErrorMessage, apiFetch } from "./client"; + +export type PiReasoning = "low" | "medium" | "high"; + +export interface PiInstallationConfig { + provider: string; + model: string; + reasoning: PiReasoning; +} + +export interface PiManagementStatus { + version?: string; + ready: boolean; + credentials: "present" | "missing"; + config: Partial; + checkedAt: string; + message?: string; +} + +export interface PiManagementOptions { + providers: string[]; + models: Array<{ provider: string; id: string }>; + reasoning: PiReasoning[]; + checkedAt: string; +} + +export interface PiManagementTestResult { + ready: boolean; + checkedAt: string; + message?: string; +} + +export interface PiManagementLogs { + lines: string[]; + checkedAt: string; +} + +export type PiManagementApiErrorCode = + | "pi_management_forbidden" + | "pi_management_unavailable" + | "pi_management_invalid_config" + | "pi_management_write_failed"; + +export interface PiManagementApiError { + status: number; + code: PiManagementApiErrorCode; + message: string; +} + +const errorCodes = new Set([ + "pi_management_forbidden", + "pi_management_unavailable", + "pi_management_invalid_config", + "pi_management_write_failed", +]); + +/** Narrows the sanctioned error code and derives its message locally. */ +export function asPiManagementApiError(error: unknown): PiManagementApiError | undefined { + if (!(error instanceof ApiError)) return undefined; + const code = error.code; + if (typeof code !== "string" || !errorCodes.has(code as PiManagementApiErrorCode)) { + return undefined; + } + return { status: error.status, code: code as PiManagementApiErrorCode, message: apiErrorMessage(error) }; +} + +export const getPiManagementStatus = () => apiFetch("/pi-management/status"); +export const getPiManagementOptions = () => apiFetch("/pi-management/options"); +export const savePiManagementConfig = (config: PiInstallationConfig) => + apiFetch("/pi-management/config", { + method: "PUT", body: JSON.stringify(config), + }); +export const runPiManagementTest = () => + apiFetch("/pi-management/test", { method: "POST" }); +export const getPiManagementLogs = () => apiFetch("/pi-management/logs"); diff --git a/frontend/src/api/runtime-config.test.ts b/frontend/src/api/runtime-config.test.ts index 0753694d..5589dddb 100644 --- a/frontend/src/api/runtime-config.test.ts +++ b/frontend/src/api/runtime-config.test.ts @@ -1,40 +1,26 @@ import { describe, expect, it } from "vitest"; import { backendBaseUrl, joinBackendPath, resolveBackendUrl } from "./runtime-config"; -import cases from "./backend-url-cases.json"; describe("resolveBackendUrl", () => { - it("uses the runtime-injected backend URL", () => { - expect(resolveBackendUrl({ backendBaseUrl: "/api" })).toBe("/api"); + it("uses same-origin /api by default", () => { + expect(resolveBackendUrl()).toBe("/api"); }); - it("falls back to the Vite backend URL", () => { - expect(resolveBackendUrl(undefined)).toBe(import.meta.env.VITE_BACKEND_URL ?? ""); + it("does not allow a browser-facing backend override", () => { + expect(() => resolveBackendUrl("https://api.example.test")).toThrow(/same-origin/i); }); - it("preserves the client default when Vite has no configured backend", () => { - expect(backendBaseUrl).toBe(import.meta.env.VITE_BACKEND_URL ?? "http://localhost:8787"); + it("keeps the exported browser base on /api", () => { + expect(backendBaseUrl).toBe("/api"); }); it.each(["/backend", "api", "//evil.test", "ftp://example.test", "https://user:pass@example.test"])( - "rejects unsupported backend URL %j", + "rejects any browser-facing backend URL %j", (backendBaseUrl) => { - expect(() => resolveBackendUrl({ backendBaseUrl })).toThrow(/BACKEND_BASE_URL/); + expect(() => resolveBackendUrl(backendBaseUrl)).toThrow(/same-origin/i); }, ); - - it.each(["", "/", "/api", "/api/", "http://localhost:8787", "https://api.example.test/v1"])( - "accepts supported backend URL %j", - (backendBaseUrl) => { - expect(resolveBackendUrl({ backendBaseUrl })).toBe(backendBaseUrl); - }, - ); - - it.each(cases)("applies the canonical policy to $value", ({ value, valid }) => { - const resolve = () => resolveBackendUrl({ backendBaseUrl: value }); - if (valid) expect(resolve()).toBe(value); - else expect(resolve).toThrow(/BACKEND_BASE_URL/); - }); }); describe("joinBackendPath", () => { diff --git a/frontend/src/api/runtime-config.ts b/frontend/src/api/runtime-config.ts index 9cfab5f5..4533bad3 100644 --- a/frontend/src/api/runtime-config.ts +++ b/frontend/src/api/runtime-config.ts @@ -1,33 +1,6 @@ -import policy from "./backend-url-policy.json"; - -export interface RuntimeConfig { - backendBaseUrl?: string; -} - -declare global { - interface Window { - __THOTHII_CONFIG__?: RuntimeConfig; - } -} - -export function resolveBackendUrl(config: RuntimeConfig | undefined): string { - const value = config?.backendBaseUrl ?? import.meta.env.VITE_BACKEND_URL ?? ""; - if (policy.relativeBases.includes(value)) return value; - try { - if (!new RegExp(policy.absolutePattern).test(value)) throw new Error("syntax"); - const authority = value.replace(/^https?:\/\//, "").split("/", 1)[0]; - const suffix = authority.startsWith("[") - ? authority.slice(authority.indexOf("]") + 1) - : authority.slice(authority.lastIndexOf(":")); - const port = suffix.startsWith(":") ? suffix.slice(1) : ""; - if (port && (port.length > 5 || Number(port) > policy.maxPort)) throw new Error("port"); - return value; - } catch { - // Fall through to the single actionable runtime error below. - } - throw new Error( - "Invalid BACKEND_BASE_URL: use empty/root, /api, or a valid http(s) base without credentials, query, or fragment", - ); +export function resolveBackendUrl(value?: string): string { + if (value === undefined || value === "/api") return "/api"; + throw new Error("Invalid backend URL: the browser must use the same-origin /api route"); } export function joinBackendPath(base: string, path: string): string { @@ -36,6 +9,4 @@ export function joinBackendPath(base: string, path: string): string { return `${normalizedBase}/${normalizedPath}`; } -export const backendBaseUrl = - resolveBackendUrl(typeof window === "undefined" ? undefined : window.__THOTHII_CONFIG__) || - "http://localhost:8787"; +export const backendBaseUrl = resolveBackendUrl(); diff --git a/frontend/src/api/sessions.test.ts b/frontend/src/api/sessions.test.ts index c3491feb..996bb93c 100644 --- a/frontend/src/api/sessions.test.ts +++ b/frontend/src/api/sessions.test.ts @@ -1,27 +1,198 @@ import { http, HttpResponse } from "msw"; import { server } from "../test/msw"; +import { canonicalWorkspaceFixture, workspaceRevisionFixture, workspaceSummaryFixture } from "../test/workspace-fixtures"; import { createSession, getMe, listSessions, prewarmRuntime, resumeSession } from "./sessions"; import { renameSession, setSessionGroup, archiveSession, unarchiveSession, deleteSession, getSessionDocuments, } from "./sessions"; +import { workspacePreferences } from "../workspaces/preferences"; -test("createSession POSTs only {question} and returns the id", async () => { +test("createSession seeds ephemeral selections and posts them", async () => { + localStorage.clear(); let body: unknown = null; server.use( - http.post("http://localhost:8787/sessions", async ({ request }) => { + http.get("/api/settings", () => HttpResponse.json({ + workspace: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + })), + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision: workspaceRevisionFixture("psd-clinical"), + }), + }])), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical"), + revision: { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot" }, + })), + http.post("/api/sessions", async ({ request }) => { body = await request.json(); return HttpResponse.json({ id: "s1" }); }), ); expect(await createSession({ question: "q" })).toEqual({ id: "s1" }); - expect(body).toEqual({ question: "q" }); + expect(body).toEqual({ + question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + }); + expect(workspacePreferences.load()).toEqual({ + workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + }); + expect(localStorage.getItem("thothii.workspace-registry.v1.preferences")).toBeNull(); +}); + +test("createSession does not POST when a selected summary aliases another workspace name", async () => { + workspacePreferences.save({ + workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + }); + let posted = false; + server.use( + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision: workspaceRevisionFixture("psd-clinical"), + }), + name: "other-workspace", + }])), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical"), + revision: { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }, + })), + http.post("/api/sessions", () => { + posted = true; + return HttpResponse.json({ id: "s1" }); + }), + ); + + await expect(createSession({ question: "q" })).rejects.toMatchObject({ + message: "Could not load workspace registry. Please retry.", + }); + expect(posted).toBe(false); +}); + +test.each([ + ["historical state", { state: "operational" }], + ["an unknown revision field", { generation: 1 }], + ["a malformed revision", { commit: "not-a-commit" }], +])("createSession does not POST when the selected summary revision has %s", async (_case, revisionPatch) => { + workspacePreferences.save({ + workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + }); + const revision = { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), + snapshotPath: "/snapshot", ...revisionPatch, + }; + let posted = false; + server.use( + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision: revision as any, + }), + }])), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical"), + revision: { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }, + })), + http.post("/api/sessions", () => { + posted = true; + return HttpResponse.json({ id: "s1" }); + }), + ); + + await expect(createSession({ question: "q" })).rejects.toMatchObject({ + message: "Could not load workspace registry. Please retry.", + }); + expect(posted).toBe(false); +}); + +test("createSession replaces a stale ephemeral workspace with the current installation default", async () => { + workspacePreferences.save({ + workspaceId: "retired-workspace", provider: "zai", model: "glm-5.2", thinking: "low", + }); + let body: unknown; + server.use( + http.get("/api/settings", () => HttpResponse.json({ workspace: "psd-clinical" })), + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision: workspaceRevisionFixture("psd-clinical"), + }), + }])), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical"), + revision: { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }, + })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + + await expect(createSession({ question: "q" })).resolves.toEqual({ id: "s1" }); + expect(body).toEqual({ + question: "q", workspaceId: "psd-clinical", + provider: "zai", model: "glm-5.2", thinking: "low", + }); + expect(workspacePreferences.load()).toEqual({ + workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + }); +}); + +test("createSession preserves a local legacy selection when the registry is empty", async () => { + workspacePreferences.save({ + workspaceId: "legacy-workspace", provider: "zai", model: "glm-5.2", thinking: "low", + }); + let body: unknown; + server.use( + http.get("/api/workspaces", () => HttpResponse.json([])), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + + await expect(createSession({ question: "q" })).resolves.toEqual({ id: "s1" }); + expect(body).toEqual({ + question: "q", workspaceId: "legacy-workspace", + provider: "zai", model: "glm-5.2", thinking: "low", + }); +}); + +test("createSession rejects a workspace summary that omits the canonical revision", async () => { + localStorage.clear(); + let posted = false; + server.use( + http.get("/api/settings", () => HttpResponse.json({ + workspace: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + })), + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + configurationState: "configuration_required", + }), + }])), + http.post("/api/sessions", async () => { + posted = true; + return HttpResponse.json({ id: "s1" }); + }), + ); + + await expect(createSession({ question: "q" })).rejects.toMatchObject({ + message: "Could not load workspace registry. Please retry.", + }); + expect(posted).toBe(false); }); test.each([202, 204])("prewarmRuntime accepts a body-less %s response", async (status) => { let called = false; server.use( - http.post("http://localhost:8787/runtime/prewarm", () => { + http.post("/api/runtime/prewarm", () => { called = true; return new HttpResponse(null, { status }); }), @@ -32,7 +203,7 @@ test.each([202, 204])("prewarmRuntime accepts a body-less %s response", async (s test("listSessions defaults to the current user's scope", async () => { let scope: string | null = null; - server.use(http.get("http://localhost:8787/sessions", ({ request }) => { + server.use(http.get("/api/sessions", ({ request }) => { scope = new URL(request.url).searchParams.get("scope"); return HttpResponse.json([{ id: "s1", status: "open", question: "q", summary: null, created_at: "t", updated_at: null, author: null }]); })); @@ -43,7 +214,7 @@ test("listSessions defaults to the current user's scope", async () => { test("listSessions requests the selected administrator scope", async () => { let scope: string | null = null; - server.use(http.get("http://localhost:8787/sessions", ({ request }) => { + server.use(http.get("/api/sessions", ({ request }) => { scope = new URL(request.url).searchParams.get("scope"); return HttpResponse.json([]); })); @@ -52,7 +223,7 @@ test("listSessions requests the selected administrator scope", async () => { }); test("getMe fetches the typed authenticated principal", async () => { - server.use(http.get("http://localhost:8787/me", () => + server.use(http.get("/api/me", () => HttpResponse.json({ issuer: "portal", subject: "42", displayName: "Alice", isAdmin: true }), )); await expect(getMe()).resolves.toEqual({ @@ -61,7 +232,7 @@ test("getMe fetches the typed authenticated principal", async () => { }); test("resumeSession returns the typed runtime disposition", async () => { - server.use(http.post("http://localhost:8787/sessions/s1/resume", () => + server.use(http.post("/api/sessions/s1/resume", () => HttpResponse.json({ id: "s1", alreadyActive: false }))); const result: { id: string; alreadyActive: boolean } = await resumeSession("s1"); @@ -70,7 +241,7 @@ test("resumeSession returns the typed runtime disposition", async () => { test("renameSession POSTs {name}", async () => { let body: unknown = null; - server.use(http.post("http://localhost:8787/sessions/s1/rename", async ({ request }) => { + server.use(http.post("/api/sessions/s1/rename", async ({ request }) => { body = await request.json(); return new HttpResponse(null, { status: 204 }); })); @@ -80,7 +251,7 @@ test("renameSession POSTs {name}", async () => { test("setSessionGroup POSTs {group}", async () => { let body: unknown = null; - server.use(http.post("http://localhost:8787/sessions/s1/group", async ({ request }) => { + server.use(http.post("/api/sessions/s1/group", async ({ request }) => { body = await request.json(); return new HttpResponse(null, { status: 204 }); })); @@ -91,9 +262,9 @@ test("setSessionGroup POSTs {group}", async () => { test("archive / unarchive / delete hit the right verbs+paths", async () => { const hits: string[] = []; server.use( - http.post("http://localhost:8787/sessions/s1/archive", () => { hits.push("archive"); return new HttpResponse(null, { status: 204 }); }), - http.post("http://localhost:8787/sessions/s1/unarchive", () => { hits.push("unarchive"); return new HttpResponse(null, { status: 204 }); }), - http.delete("http://localhost:8787/sessions/s1", () => { hits.push("delete"); return new HttpResponse(null, { status: 204 }); }), + http.post("/api/sessions/s1/archive", () => { hits.push("archive"); return new HttpResponse(null, { status: 204 }); }), + http.post("/api/sessions/s1/unarchive", () => { hits.push("unarchive"); return new HttpResponse(null, { status: 204 }); }), + http.delete("/api/sessions/s1", () => { hits.push("delete"); return new HttpResponse(null, { status: 204 }); }), ); await archiveSession("s1"); await unarchiveSession("s1"); @@ -102,7 +273,7 @@ test("archive / unarchive / delete hit the right verbs+paths", async () => { }); test("getSessionDocuments GETs the array", async () => { - server.use(http.get("http://localhost:8787/sessions/s1/documents", () => + server.use(http.get("/api/sessions/s1/documents", () => HttpResponse.json([{ phase: "—", key: "question", title: "t", format: "text", content: "q" }]), )); const docs = await getSessionDocuments("s1"); diff --git a/frontend/src/api/sessions.ts b/frontend/src/api/sessions.ts index 2067ecfd..7284db61 100644 --- a/frontend/src/api/sessions.ts +++ b/frontend/src/api/sessions.ts @@ -1,16 +1,177 @@ import { apiFetch } from "./client"; +import { + captureAuthOperation, + isAuthOperationCurrent, + requireAuthOperationPrecondition, + type AuthOperationPrecondition, +} from "../auth/authOperation"; +import { getSettings } from "./settings"; +import { getWorkspace, listWorkspaces } from "./workspaces"; +import { + WORKSPACE_POLICY_ERROR, WORKSPACE_SUMMARY_ERROR, WorkspaceSelectionError, workspacePolicyGate, + workspacePreferences, type WorkspacePreference, +} from "../workspaces/drafts"; import type { - Principal, ResumeSessionResult, SessionScope, SessionSummary, SessionDocument, UiResponse, + AuthenticatedUser, ResumeSessionResult, SessionScope, SessionSummary, SessionDocument, UiResponse, } from "./types"; -export const createSession = (i: { question: string; name?: string }) => - apiFetch<{ id: string }>("/sessions", { method: "POST", body: JSON.stringify(i) }); +type NewSessionInput = { question: string; name?: string } & Partial; + +async function selectedPreferences(precondition?: AuthOperationPrecondition): Promise { + requireAuthOperationPrecondition(precondition); + const saved = workspacePreferences.load(); + if (saved.workspaceId && saved.provider && saved.model && saved.thinking) return saved; + + // A direct new-session entry point can run before the composer has mounted. Seed current + // application memory from the installation defaults once, then keep choices ephemeral. + const legacy = await getSettings(); + requireAuthOperationPrecondition(precondition); + return workspacePreferences.save({ + workspaceId: saved.workspaceId ?? legacy.workspace, + provider: saved.provider ?? legacy.provider, + model: saved.model ?? legacy.model, + thinking: saved.thinking ?? legacy.thinking, + }); +} + +function reconcileWorkspacePolicy(preferences: WorkspacePreference, allowed: readonly string[], defaultModel?: string) { + if (allowed.length === 0) throw new WorkspaceSelectionError(WORKSPACE_POLICY_ERROR); + const selected = preferences.provider && preferences.model + ? `${preferences.provider}/${preferences.model}` + : undefined; + if (selected && allowed.includes(selected)) return preferences; + const replacement = defaultModel && allowed.includes(defaultModel) ? defaultModel : allowed[0]; + const separator = replacement.indexOf("/"); + if (separator <= 0 || separator === replacement.length - 1) { + throw new WorkspaceSelectionError(WORKSPACE_POLICY_ERROR); + } + return workspacePreferences.save({ + ...preferences, + provider: replacement.slice(0, separator), + model: replacement.slice(separator + 1), + }); +} + +async function ensureWorkspaceSelectionPolicy(precondition?: AuthOperationPrecondition): Promise { + while (true) { + requireAuthOperationPrecondition(precondition); + const preferences = workspacePreferences.load(); + const workspaceId = preferences.workspaceId; + if (!workspaceId) return preferences; + workspacePolicyGate.beginSummary(workspaceId); + let workspaces: Awaited>; + try { + workspaces = await listWorkspaces(); + } catch { + requireAuthOperationPrecondition(precondition); + if (workspacePreferences.load().workspaceId !== workspaceId) continue; + workspacePolicyGate.rejectSummary(workspaceId); + throw new WorkspaceSelectionError(WORKSPACE_SUMMARY_ERROR); + } + requireAuthOperationPrecondition(precondition); + if (workspacePreferences.load().workspaceId !== workspaceId) continue; + const workspace = workspaces.find((candidate) => candidate.id === workspaceId); + if (!workspace) { + if (workspaces.length === 0) { + workspacePolicyGate.allowLegacy(workspaceId); + return workspacePreferences.load(); + } + let installationDefault: string | undefined; + try { + installationDefault = (await getSettings()).workspace; + } catch { + requireAuthOperationPrecondition(precondition); + if (workspacePreferences.load().workspaceId !== workspaceId) continue; + workspacePolicyGate.rejectSummary(workspaceId); + throw new WorkspaceSelectionError(WORKSPACE_SUMMARY_ERROR); + } + requireAuthOperationPrecondition(precondition); + if (workspacePreferences.load().workspaceId !== workspaceId) continue; + const replacement = workspaces.find( + (candidate) => candidate.id === installationDefault && candidate.revision, + ) ?? workspaces.find((candidate) => candidate.revision); + if (!replacement) { + requireAuthOperationPrecondition(precondition); + workspacePolicyGate.reject(workspaceId); + throw new WorkspaceSelectionError(WORKSPACE_POLICY_ERROR); + } + requireAuthOperationPrecondition(precondition); + workspacePreferences.save({ ...preferences, workspaceId: replacement.id }); + continue; + } + if (!workspace.revision) { + workspacePolicyGate.reject(workspaceId); + throw new WorkspaceSelectionError(WORKSPACE_POLICY_ERROR); + } + workspacePolicyGate.select(workspaceId); + const outcome = await Promise.race([ + getWorkspace(workspaceId).then( + (record) => ({ kind: "record" as const, record }), + () => ({ kind: "error" as const }), + ), + workspacePolicyGate.waitForCurrent(() => workspacePreferences.load()).then( + (selection) => ({ kind: "selection" as const, selection }), + ), + ]); + requireAuthOperationPrecondition(precondition); + if (outcome.kind === "selection") { + if (outcome.selection.workspaceId !== workspaceId) continue; + return outcome.selection; + } + if (outcome.kind === "error") { + requireAuthOperationPrecondition(precondition); + if (workspacePreferences.load().workspaceId !== workspaceId) continue; + workspacePolicyGate.reject(workspaceId); + throw new WorkspaceSelectionError(WORKSPACE_POLICY_ERROR); + } + const { record } = outcome; + requireAuthOperationPrecondition(precondition); + if (workspacePreferences.load().workspaceId !== workspaceId) continue; + let selection: WorkspacePreference; + try { + selection = reconcileWorkspacePolicy( + preferences, + record.workspace.llm_policy.allowed, + record.workspace.llm_policy.default, + ); + } catch (error) { + requireAuthOperationPrecondition(precondition); + workspacePolicyGate.reject(workspaceId); + throw error; + } + requireAuthOperationPrecondition(precondition); + workspacePolicyGate.resolve(workspaceId); + requireAuthOperationPrecondition(precondition); + if (workspacePreferences.load().workspaceId === workspaceId) return selection; + } +} + +export async function createSession(i: NewSessionInput, precondition?: AuthOperationPrecondition) { + const initiatingOperation = captureAuthOperation(); + const effectivePrecondition = precondition ?? (initiatingOperation ? { + operation: initiatingOperation, + isCurrent: () => isAuthOperationCurrent(initiatingOperation, { + sessionId: null, + disposalEpoch: initiatingOperation.disposalEpoch, + }), + } : undefined); + requireAuthOperationPrecondition(effectivePrecondition); + await selectedPreferences(effectivePrecondition); + requireAuthOperationPrecondition(effectivePrecondition); + const selection = await ensureWorkspaceSelectionPolicy(effectivePrecondition); + requireAuthOperationPrecondition(effectivePrecondition); + return apiFetch<{ id: string }>("/sessions", { + method: "POST", + body: JSON.stringify({ ...i, ...selection }), + }); +} /** Best-effort warm-up; callers must not await it before showing the composer. */ export const prewarmRuntime = () => apiFetch("/runtime/prewarm", { method: "POST" }); -export const getMe = () => apiFetch("/me"); +/** Compatibility export for existing shell tests/callers; AuthGate uses the validated auth API. */ +export const getMe = () => apiFetch("/me"); export const listSessions = (scope: SessionScope = "mine") => apiFetch(`/sessions?scope=${scope}`); diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 1f6b05d4..61d1b4f1 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -117,11 +117,44 @@ export interface SessionSummary { export type SessionScope = "mine" | "all"; +export type AuthRole = "user" | "admin"; + +export interface AuthSessionInfo { + method: "local" | "oidc" | "upstream"; + remembered: boolean; + idleExpiresAt: string; + absoluteExpiresAt: string; +} + +/** Safe browser representation returned by Task 8's /me route. */ +export interface AuthenticatedUser { + issuer: string; + subject: string; + displayName?: string; + roles: readonly AuthRole[]; + permissions: readonly string[]; + isAdmin: boolean; + /** Legacy upstream/none modes deliberately return null: there is no CSRF token. */ + csrfToken: string | null; + /** Legacy upstream/none modes deliberately return null: there is no cookie session. */ + session: AuthSessionInfo | null; +} + +export interface AuthPublicConfig { + mode: "local" | "oidc" | "upstream" | "none" | "mock"; + localLogin: boolean; + oidcLogin: boolean; +} + export interface Principal { issuer: string; subject: string; displayName?: string; + roles?: readonly AuthRole[]; + permissions?: readonly string[]; isAdmin: boolean; + csrfToken?: string | null; + session?: AuthSessionInfo | null; } export interface ResumeSessionResult { diff --git a/frontend/src/api/workspaces.test.ts b/frontend/src/api/workspaces.test.ts new file mode 100644 index 00000000..643fbb5a --- /dev/null +++ b/frontend/src/api/workspaces.test.ts @@ -0,0 +1,177 @@ +import { expect, test } from "vitest"; +import { http, HttpResponse } from "msw"; +import { server } from "../test/msw"; +import { canonicalWorkspaceFixture, workspaceRevisionFixture, workspaceSummaryFixture } from "../test/workspace-fixtures"; +import { + forgetWorkspaceSecret, + getWorkspace, + getWorkspaceRuntimeConfiguration, + listWorkspaces, + saveWorkspaceSecrets, + testWorkspace, + validateWorkspace, +} from "./workspaces"; + +const workspace = canonicalWorkspaceFixture("psd-clinical"); +const revision = workspaceRevisionFixture("psd-clinical"); + +const runtimeConfiguration = { + workspaceId: "psd-clinical", + revision, + configurationState: "configuration_required", + requirements: [{ + id: "dwh.password", + connector: "dwh", + label: "Data warehouse password", + description: "Password used by the selected data warehouse connection.", + input: "password", + required: true, + configured: false, + }], +} as const; + +test("decodes read-only workspace summaries with a revision in every readiness state", async () => { + const ready = workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision, + }); + const needsSecrets = { + ...ready, + configurationState: "configuration_required" as const, + }; + server.use(http.get("/api/workspaces", () => HttpResponse.json([ready, needsSecrets]))); + + await expect(listWorkspaces()).resolves.toEqual([ready, needsSecrets]); +}); + +test("accepts the immutable workspace revision contract", async () => { + server.use(http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ workspace, revision }))); + + await expect(getWorkspace("psd-clinical")).resolves.toEqual({ workspace, revision }); +}); + +test("decodes runtime requirements but rejects any secret value returned by the server", async () => { + server.use(http.get( + "/api/workspaces/psd-clinical/runtime-configuration", + () => HttpResponse.json(runtimeConfiguration), + )); + await expect(getWorkspaceRuntimeConfiguration("psd-clinical")) + .resolves.toEqual(runtimeConfiguration); + + server.use(http.get( + "/api/workspaces/psd-clinical/runtime-configuration", + () => HttpResponse.json({ + ...runtimeConfiguration, + requirements: [{ ...runtimeConfiguration.requirements[0], value: "leaked-secret" }], + }), + )); + await expect(getWorkspaceRuntimeConfiguration("psd-clinical")) + .rejects.toThrow("invalid runtime configuration"); +}); + +test("blind secret replacement sends values once and returns status only", async () => { + let requestBody: unknown; + server.use(http.put("/api/workspaces/psd-clinical/secrets", async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json({ + ...runtimeConfiguration, + configurationState: "ready", + requirements: [{ ...runtimeConfiguration.requirements[0], configured: true }], + }); + })); + + const response = await saveWorkspaceSecrets("psd-clinical", { + "dwh.password": "one-time-value", + }); + + expect(requestBody).toEqual({ values: { "dwh.password": "one-time-value" } }); + expect(response.configurationState).toBe("ready"); + expect(JSON.stringify(response)).not.toContain("one-time-value"); +}); + +test("forget targets one declared requirement", async () => { + let called = false; + server.use(http.delete("/api/workspaces/psd-clinical/secrets/dwh.password", () => { + called = true; + return HttpResponse.json(runtimeConfiguration); + })); + + await expect(forgetWorkspaceSecret("psd-clinical", "dwh.password")) + .resolves.toEqual(runtimeConfiguration); + expect(called).toBe(true); +}); + +test("decodes the shared authentication diagnostics on static validation and live connection tests", async () => { + const authentication = { + ready: false, + mode: "oidc" as const, + checks: [{ + level: "error" as const, + code: "oidc_mapped_group_missing" as const, + field: "Thoth Administrators", + message: "A configured authorization group does not exist.", + }], + }; + server.use( + http.post("/api/workspaces/validate", () => HttpResponse.json({ + workspace, + contract: {}, + activatable: false, + diagnostics: [], + authentication, + })), + http.post("/api/workspaces/psd-clinical/test", () => HttpResponse.json({ + activatable: false, + diagnostics: [], + authentication, + })), + ); + + await expect(validateWorkspace(workspace)).resolves.toMatchObject({ + activatable: false, + authentication, + }); + await expect(testWorkspace("psd-clinical")).resolves.toMatchObject({ + activatable: false, + authentication, + }); +}); + +test.each([ + ["ready with error", { ready: true, mode: "oidc", checks: [{ level: "error", code: "oidc_secret_missing", message: "failure" }] }], + ["failed with ready", { ready: false, mode: "oidc", checks: [{ level: "info", code: "auth_ready", message: "ready" }] }], + ["failed without error", { ready: false, mode: "oidc", checks: [{ level: "info", code: "auth_config_invalid", message: "info" }] }], + ["duplicate", { ready: false, mode: "oidc", checks: [ + { level: "error", code: "oidc_secret_missing", message: "one" }, + { level: "error", code: "oidc_secret_missing", message: "two" }, + ] }], + ["attacker field", { ready: false, mode: "oidc", checks: [{ level: "error", code: "oidc_secret_missing", message: "failure", field: "attacker-field-SENTINEL" }] }], + ["control", { ready: false, mode: "oidc", checks: [{ level: "error", code: "oidc_mapped_group_missing", message: "failure", field: "bad\u0085field" }] }], + ["unexpected property", { ready: false, mode: "oidc", checks: [{ level: "error", code: "oidc_secret_missing", message: "failure", attacker: "field" }] }], +])("rejects hostile authentication diagnostics: %s", async (_name, authentication) => { + server.use(http.post("/api/workspaces/validate", () => HttpResponse.json({ + workspace, + contract: {}, + activatable: false, + diagnostics: [], + authentication, + }))); + + await expect(validateWorkspace(workspace)).rejects.toThrow("invalid authentication diagnostics"); +}); + +test("rejects a workspace claimed activatable when authentication is not ready", async () => { + server.use(http.post("/api/workspaces/validate", () => HttpResponse.json({ + workspace, + contract: {}, + activatable: true, + diagnostics: [], + authentication: { + ready: false, + mode: "oidc", + checks: [{ level: "error", code: "oidc_secret_missing", message: "Authentication is unavailable." }], + }, + }))); + + await expect(validateWorkspace(workspace)).rejects.toThrow("invalid diagnostic result"); +}); diff --git a/frontend/src/api/workspaces.ts b/frontend/src/api/workspaces.ts index 43c612bc..682ccdff 100644 --- a/frontend/src/api/workspaces.ts +++ b/frontend/src/api/workspaces.ts @@ -1,4 +1,468 @@ -import { apiFetch } from "./client"; +import { ApiError, apiErrorMessage, apiFetch } from "./client"; +import { sanitizeCanonicalWorkspace } from "../workspaces/drafts"; -export const listWorkspaces = () => - apiFetch<{ name: string; file: string }[]>("/workspaces"); +export type WorkspaceErrorCode = + | "workspace_invalid" | "binding_missing" | "workspace_not_activatable" + | "workspace_stale" | "git_unavailable" + | "git_auth_failed" | "git_non_fast_forward" + | "connector_unavailable" | "semantic_index_incompatible"; + +export interface RestDiagnosticRequest { + method: "GET" | "POST"; + path: string; + auth: "none" | "bearer" | "x-api-key"; +} + +export interface CanonicalDiagnostics { + dwh_rest?: RestDiagnosticRequest & { response: { database: string; schema: string } }; +} + +export interface EvidencePolicy { + max_chunk_chars: number; + retain_published_generations: number; +} + +export type EvidenceSource = + | { + type: "filesystem"; + uri: string; + patterns: string[]; + max_bytes: number; + } + | { + type: "http"; + uris: string[]; + authentication: "none" | "signed_urls_file"; + connect_timeout_ms: number; + read_timeout_ms: number; + max_bytes: number; + max_redirects: number; + allow_private_hosts: boolean; + max_cache_bytes: number; + } + | { + type: "s3"; + uri: string; + endpoint_url?: string; + region?: string; + credentials: "ambient" | "static_files"; + trusted_endpoint: boolean; + allow_private_endpoint: boolean; + allow_insecure_endpoint: boolean; + max_bytes: number; + max_objects: number; + max_pages: number; + page_size: number; + }; + +export interface WorkspaceEvidence { + source: EvidenceSource; + policy: EvidencePolicy; +} + +export interface CanonicalWorkspace { + workspace: { + schema_version: 3; + id: string; + name: string; + description?: string; + language: "en" | "it"; + }; + dwh: { + engine: "postgres"; + database: string; + schema: string; + port?: number; + timeout_ms?: number; + supported_transports: ("postgres_direct" | "rest_api" | "ssh_tunnel")[]; + }; + semantic_index: { + vector_store: { + engine: "qdrant"; + collection: string; + dimensions: 1024; + distance: "cosine"; + } + embedding: { + provider: "ollama_internal"; + model: "qwen3-embedding:0.6b"; + dimensions: 1024; + }; + }; + llm_policy: { default?: `${string}/${string}`; allowed: `${string}/${string}`[] }; + diagnostics?: CanonicalDiagnostics; + evidence?: WorkspaceEvidence; +} + +export interface WorkspaceRevision { + id: string; + commit: string; + blob: string; + snapshotPath: string; +} + +export interface WorkspaceSummary { + id: string; + /** Kept for compatibility with the existing workspace selector. */ + name: string; + file: string; + displayName: string; + description?: string; + configurationState: "ready" | "configuration_required"; + revision?: WorkspaceRevision; +} + +export interface WorkspaceRecord { + workspace: CanonicalWorkspace; + revision: WorkspaceRevision; +} + +export interface WorkspaceRegistryStatus { + branch: string; + head?: string; + ahead: number; + behind: number; + degraded: boolean; + lastError?: WorkspaceErrorCode; + repository?: { + host: string; + repository: string; + transport: "https" | "ssh" | "local"; + }; +} + +export interface WorkspaceDiagnostic { + level: "error" | "warning" | "info"; + code: WorkspaceErrorCode | "binding_ok"; + field?: string; + message: string; +} + +export type AuthDiagnosticCode = + | "auth_ready" + | "auth_config_incomplete" + | "auth_config_invalid" + | "auth_session_store_invalid" + | "local_user_registry_invalid" + | "local_admin_missing" + | "oidc_secret_missing" + | "oidc_discovery_unreachable" + | "oidc_issuer_mismatch" + | "oidc_jwks_unreachable" + | "oidc_group_catalog_unreachable" + | "oidc_group_catalog_unauthorized" + | "oidc_mapped_group_missing" + | "oidc_mapped_group_ambiguous" + | "oidc_groups_claim_invalid" + | "oidc_device_flow_unavailable"; + +export interface AuthDiagnostic { + level: "error" | "info"; + code: AuthDiagnosticCode; + message: string; + field?: string; +} + +export interface AuthDiagnostics { + ready: boolean; + mode: "local" | "oidc" | "upstream" | "none" | "mock"; + checks: AuthDiagnostic[]; +} + +export interface WorkspaceDiagnostics { + activatable: boolean; + diagnostics: WorkspaceDiagnostic[]; + authentication: AuthDiagnostics; +} + +export interface WorkspaceValidationResult extends WorkspaceDiagnostics { + workspace: CanonicalWorkspace; + contract: unknown; +} + +export interface WorkspaceSecretRequirement { + id: string; + connector: "dwh" | "evidence"; + label: string; + description: string; + input: "password" | "textarea"; + required: boolean; + configured: boolean; +} + +export interface WorkspaceRuntimeConfiguration { + workspaceId: string; + revision: WorkspaceRevision; + configurationState: "ready" | "configuration_required"; + requirements: WorkspaceSecretRequirement[]; +} + +export interface WorkspaceApiError { + status: number; + code: WorkspaceErrorCode; + message: string; + fields?: string[]; +} + +const workspaceErrorCodes = new Set([ + "workspace_invalid", "binding_missing", "workspace_not_activatable", "workspace_stale", + "git_unavailable", "git_auth_failed", "git_non_fast_forward", "connector_unavailable", + "semantic_index_incompatible", +]); + +function object(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; +} + +const authDiagnosticCodes = new Set([ + "auth_ready", "auth_config_incomplete", "auth_config_invalid", "auth_session_store_invalid", + "local_user_registry_invalid", "local_admin_missing", "oidc_secret_missing", + "oidc_discovery_unreachable", "oidc_issuer_mismatch", "oidc_jwks_unreachable", + "oidc_group_catalog_unreachable", "oidc_group_catalog_unauthorized", "oidc_mapped_group_missing", + "oidc_mapped_group_ambiguous", "oidc_groups_claim_invalid", "oidc_device_flow_unavailable", +]); + +function text(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 512 + && value.trim() === value && !/\p{Cc}/u.test(value); +} + +function decodeAuthentication(value: unknown): AuthDiagnostics { + const source = exactObject(value, ["ready", "mode", "checks"]); + if (!source || typeof source.ready !== "boolean" + || typeof source.mode !== "string" + || !["local", "oidc", "upstream", "none", "mock"].includes(source.mode) + || !Array.isArray(source.checks) || source.checks.length === 0 || source.checks.length > 129) { + throw new Error("Workspace API returned invalid authentication diagnostics"); + } + const seen = new Set(); + const checks = source.checks.map((item): AuthDiagnostic => { + const raw = object(item); + const check = raw && exactObject(raw, raw.field === undefined + ? ["level", "code", "message"] + : ["level", "code", "message", "field"]); + if (!check || (check.level !== "error" && check.level !== "info") + || typeof check.code !== "string" || !authDiagnosticCodes.has(check.code as AuthDiagnosticCode) + || !text(check.message) || (check.field !== undefined && !text(check.field))) { + throw new Error("Workspace API returned invalid authentication diagnostics"); + } + if (check.field !== undefined + && check.code !== "oidc_mapped_group_missing" && check.code !== "oidc_mapped_group_ambiguous") { + throw new Error("Workspace API returned invalid authentication diagnostics"); + } + const key = `${check.code}\u0000${check.field ?? ""}`; + if (seen.has(key)) throw new Error("Workspace API returned invalid authentication diagnostics"); + seen.add(key); + return { + level: check.level, + code: check.code as AuthDiagnosticCode, + message: check.message, + ...(check.field === undefined ? {} : { field: check.field }), + }; + }); + if (source.ready) { + if (checks.length !== 1 || checks[0].level !== "info" || checks[0].code !== "auth_ready" + || checks[0].field !== undefined) throw new Error("Workspace API returned invalid authentication diagnostics"); + } else if (!checks.some(({ level }) => level === "error") + || checks.some(({ code }) => code === "auth_ready")) { + throw new Error("Workspace API returned invalid authentication diagnostics"); + } + return { ready: source.ready, mode: source.mode as AuthDiagnostics["mode"], checks }; +} + +function decodeWorkspaceDiagnostics(value: unknown): WorkspaceDiagnostics { + const source = object(value); + if (!source || typeof source.activatable !== "boolean" || !Array.isArray(source.diagnostics)) { + throw new Error("Workspace API returned an invalid diagnostic result"); + } + const diagnostics = source.diagnostics.map((item): WorkspaceDiagnostic => { + const diagnostic = object(item); + if (!diagnostic || (diagnostic.level !== "error" && diagnostic.level !== "warning" && diagnostic.level !== "info") + || typeof diagnostic.code !== "string" || !text(diagnostic.message) + || (diagnostic.field !== undefined && !text(diagnostic.field))) { + throw new Error("Workspace API returned an invalid diagnostic result"); + } + return { + level: diagnostic.level, + code: diagnostic.code as WorkspaceDiagnostic["code"], + message: diagnostic.message, + ...(diagnostic.field === undefined ? {} : { field: diagnostic.field }), + }; + }); + const authentication = decodeAuthentication(source.authentication); + if (source.activatable && !authentication.ready) { + throw new Error("Workspace API returned an invalid diagnostic result"); + } + return { activatable: source.activatable, diagnostics, authentication }; +} + +function exactObject(value: unknown, keys: readonly string[]): Record | undefined { + const source = object(value); + return source && Object.keys(source).every((key) => keys.includes(key)) ? source : undefined; +} + +function workspaceRevision(value: unknown, expectedId: string): WorkspaceRevision | undefined { + const source = exactObject(value, ["id", "commit", "blob", "snapshotPath"]); + if (!source) return undefined; + const { id, commit, blob, snapshotPath } = source; + if ( + id !== expectedId + || typeof id !== "string" || !/^[a-z][a-z0-9-]{2,62}$/.test(id) + || typeof commit !== "string" || !/^[0-9a-f]{40}$/.test(commit) + || typeof blob !== "string" || !/^[0-9a-f]{40}$/.test(blob) + || typeof snapshotPath !== "string" || snapshotPath.length === 0 + || snapshotPath.trim() !== snapshotPath || /[\x00-]/u.test(snapshotPath) + ) return undefined; + return { id, commit, blob, snapshotPath }; +} + +function workspaceSummary(value: unknown): WorkspaceSummary | undefined { + const source = exactObject(value, [ + "id", "name", "file", "displayName", "description", "configurationState", "revision", + ]); + if (!source) return undefined; + const { id, name, file, displayName, description, configurationState } = source; + const validText = (candidate: unknown) => typeof candidate === "string" + && candidate.length > 0 + && candidate.trim() === candidate; + if ( + typeof id !== "string" || !/^[a-z][a-z0-9-]{2,62}$/.test(id) + || name !== id || file !== `${id}/workspace.yaml` + || !validText(displayName) + || (description !== undefined && !validText(description)) + || (configurationState !== "ready" && configurationState !== "configuration_required") + ) return undefined; + const revision = source.revision === undefined + ? undefined + : workspaceRevision(source.revision, id); + if ( + !revision + ) return undefined; + return { + id, + name: name as string, + file: file as string, + displayName: displayName as string, + ...(description === undefined ? {} : { description: description as string }), + configurationState, + revision, + }; +} + +function requireWorkspaceRevision(value: unknown, expectedId: string): WorkspaceRevision { + const revision = workspaceRevision(value, expectedId); + if (!revision) throw new Error("Workspace API returned an invalid workspace revision"); + return revision; +} + +/** Localized registry error data; it intentionally excludes the raw response body. */ +export function asWorkspaceApiError(error: unknown): WorkspaceApiError | undefined { + if (!(error instanceof ApiError)) return undefined; + const code = error.code; + if (typeof code !== "string" || !workspaceErrorCodes.has(code as WorkspaceErrorCode)) { + return undefined; + } + return { status: error.status, code: code as WorkspaceErrorCode, message: apiErrorMessage(error) }; +} + +function requireCanonicalWorkspace(value: unknown): CanonicalWorkspace { + const workspace = sanitizeCanonicalWorkspace(value); + if (!workspace) throw new Error("Workspace API returned an invalid canonical workspace"); + return workspace; +} + +export const listWorkspaces = async (): Promise => { + const response = await apiFetch("/workspaces"); + if (!Array.isArray(response)) throw new Error("Workspace API returned an invalid workspace summary"); + const summaries = response.map(workspaceSummary); + if (summaries.some((summary) => !summary)) { + throw new Error("Workspace API returned an invalid workspace summary"); + } + return summaries as WorkspaceSummary[]; +}; + +export const getWorkspace = async (id: string): Promise => { + const response = await apiFetch(`/workspaces/${encodeURIComponent(id)}`); + const source = object(response); + if (!source) throw new Error("Workspace API returned an invalid workspace record"); + const workspace = requireCanonicalWorkspace(source.workspace); + return { + workspace, + revision: requireWorkspaceRevision(source.revision, workspace.workspace.id), + }; +}; + +export const getWorkspaceRegistryStatus = () => apiFetch("/workspace-registry/status"); +export const pullWorkspaceRegistry = () => apiFetch("/workspace-registry/pull", { method: "POST" }); + +export const validateWorkspace = async (workspace: CanonicalWorkspace): Promise => { + const safe = requireCanonicalWorkspace(workspace); + const response = await apiFetch("/workspaces/validate", { + method: "POST", body: JSON.stringify({ workspace: safe }), + }); + const source = object(response); + if (!source) throw new Error("Workspace API returned an invalid validation result"); + return { + workspace: requireCanonicalWorkspace(source.workspace), + contract: source.contract, + ...decodeWorkspaceDiagnostics(source), + }; +}; + +export const testWorkspace = async (id: string): Promise => + decodeWorkspaceDiagnostics(await apiFetch(`/workspaces/${encodeURIComponent(id)}/test`, { method: "POST" })); + +function runtimeConfiguration(value: unknown, expectedId: string): WorkspaceRuntimeConfiguration { + const source = exactObject(value, [ + "workspaceId", "revision", "configurationState", "requirements", + ]); + if ( + !source + || source.workspaceId !== expectedId + || (source.configurationState !== "ready" && source.configurationState !== "configuration_required") + || !Array.isArray(source.requirements) + ) throw new Error("Workspace API returned an invalid runtime configuration"); + const requirements = source.requirements.map((value) => { + const requirement = exactObject(value, [ + "id", "connector", "label", "description", "input", "required", "configured", + ]); + if ( + !requirement + || typeof requirement.id !== "string" || !/^[a-z0-9][a-z0-9._-]{1,127}$/.test(requirement.id) + || (requirement.connector !== "dwh" && requirement.connector !== "evidence") + || typeof requirement.label !== "string" || requirement.label.length === 0 + || typeof requirement.description !== "string" || requirement.description.length === 0 + || (requirement.input !== "password" && requirement.input !== "textarea") + || typeof requirement.required !== "boolean" + || typeof requirement.configured !== "boolean" + ) throw new Error("Workspace API returned an invalid runtime configuration"); + return requirement as unknown as WorkspaceSecretRequirement; + }); + return { + workspaceId: expectedId, + revision: requireWorkspaceRevision(source.revision, expectedId), + configurationState: source.configurationState, + requirements, + }; +} + +export const getWorkspaceRuntimeConfiguration = async (id: string) => runtimeConfiguration( + await apiFetch(`/workspaces/${encodeURIComponent(id)}/runtime-configuration`), + id, +); + +export const saveWorkspaceSecrets = async (id: string, values: Readonly>) => ( + runtimeConfiguration(await apiFetch(`/workspaces/${encodeURIComponent(id)}/secrets`, { + method: "PUT", + body: JSON.stringify({ values }), + }), id) +); + +export const forgetWorkspaceSecret = async (id: string, requirementId: string) => ( + runtimeConfiguration(await apiFetch( + `/workspaces/${encodeURIComponent(id)}/secrets/${encodeURIComponent(requirementId)}`, + { method: "DELETE" }, + ), id) +); diff --git a/frontend/src/auth/AuthGate.test.tsx b/frontend/src/auth/AuthGate.test.tsx new file mode 100644 index 00000000..3a022ce4 --- /dev/null +++ b/frontend/src/auth/AuthGate.test.tsx @@ -0,0 +1,150 @@ +import { cleanup, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { http, HttpResponse, delay } from "msw"; +import { beforeEach, afterEach, describe, expect, test, vi } from "vitest"; +import { StrictMode } from "react"; +import { AuthGate } from "./AuthGate"; +import { clearAuthState, getAuthGeneration, getAuthState, setAuthState } from "./authState"; +import { server } from "../test/msw"; + +vi.mock("../shell/AppShell", () => ({ + AppShell: () => ( +
+ Authenticated shell + +
+ ), +})); + +const user = { + issuer: "local", + subject: "user-1", + displayName: "Analyst", + roles: ["user"] as const, + permissions: ["session.use"], + isAdmin: false, + csrfToken: "c".repeat(43), + session: { + method: "local" as const, + remembered: false, + idleExpiresAt: "2026-08-17T10:00:00.000Z", + absoluteExpiresAt: "2026-08-17T20:00:00.000Z", + }, +}; + +const localConfig = { mode: "local", localLogin: true, oidcLogin: false }; + +beforeEach(() => { + clearAuthState(); + server.use( + http.get("/api/auth/config", () => HttpResponse.json(localConfig)), + http.get("/api/me", () => HttpResponse.json(user)), + ); +}); + +afterEach(() => { + cleanup(); + clearAuthState(); +}); + +describe("AuthGate", () => { + test("shows a loading state while /me is unresolved", async () => { + server.use(http.get("/api/me", async () => { + await delay(100); + return HttpResponse.json(user); + })); + + render(); + + expect(screen.getByRole("status", { name: /checking access/i })).toBeInTheDocument(); + expect(screen.queryByTestId("authenticated-shell")).not.toBeInTheDocument(); + }); + + test("renders the authenticated shell from the safe /me DTO", async () => { + render(); + + expect(await screen.findByTestId("authenticated-shell")).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: /sign in/i })).not.toBeInTheDocument(); + }); + + test("returns to local login after an expired session 401", async () => { + server.use(http.get("/api/me", () => new HttpResponse(null, { status: 401 }))); + + render(); + + expect(await screen.findByRole("heading", { name: /sign in to thothii/i })).toBeInTheDocument(); + expect(screen.getByLabelText(/password/i)).toBeInTheDocument(); + }); + + test("presents a forbidden /me response explicitly", async () => { + setAuthState(user); + const generation = getAuthGeneration(); + server.use(http.get("/api/me", () => HttpResponse.json( + { code: "auth_not_authorized", error: "This operation is not permitted" }, + { status: 403 }, + ))); + + render(); + + expect(await screen.findByRole("heading", { name: /access not permitted/i })).toBeInTheDocument(); + expect(screen.getByText(/signed in without permission/i)).toBeInTheDocument(); + expect(getAuthState()).toMatchObject({ subject: "user-1", csrfToken: "c".repeat(43) }); + expect(getAuthGeneration()).toBe(generation); + }); + + test("offers retry when the authentication provider is unavailable", async () => { + let attempts = 0; + server.use( + http.get("/api/me", () => { + attempts += 1; + return attempts === 1 + ? HttpResponse.json({ code: "auth_unavailable" }, { status: 503 }) + : HttpResponse.json(user); + }), + ); + + render(); + + expect(await screen.findByRole("heading", { name: /authentication unavailable/i })).toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: /retry/i })); + expect(await screen.findByTestId("authenticated-shell")).toBeInTheDocument(); + }); + + test("accepts the nullable legacy /me session shape without inventing a token", async () => { + server.use(http.get("/api/me", () => HttpResponse.json({ + issuer: "portal", + subject: "legacy-user", + displayName: "Legacy user", + roles: ["user"], + permissions: ["session.use"], + isAdmin: false, + csrfToken: null, + session: null, + }))); + + render(); + + expect(await screen.findByTestId("authenticated-shell")).toBeInTheDocument(); + }); + + test("does not write credentials or tokens to browser storage", async () => { + const storageWrites = (["setItem", "removeItem", "clear"] as const).map((method) => + vi.spyOn(Storage.prototype, method)); + render(); + + await screen.findByTestId("authenticated-shell"); + for (const write of storageWrites) expect(write).not.toHaveBeenCalled(); + for (const write of storageWrites) write.mockRestore(); + }); + + test("does not emit act warnings while StrictMode authenticates", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + render(); + expect(await screen.findByTestId("authenticated-shell")).toBeInTheDocument(); + expect(errors.mock.calls.flat().join(" ")).not.toMatch(/not wrapped in act/i); + } finally { + errors.mockRestore(); + } + }); +}); diff --git a/frontend/src/auth/AuthGate.tsx b/frontend/src/auth/AuthGate.tsx new file mode 100644 index 00000000..19e25d86 --- /dev/null +++ b/frontend/src/auth/AuthGate.tsx @@ -0,0 +1,111 @@ +import { useCallback, useEffect, useState } from "react"; +import { ApiError } from "../api/client"; +import { authErrorStatus, getAuthConfig, getMe } from "../api/auth"; +import type { AuthenticatedUser, AuthPublicConfig } from "../api/types"; +import { AppShell } from "../shell/AppShell"; +import { + clearAuthStateIfCurrent, + getAuthGeneration, + getAuthState, + isAuthGenerationCurrent, + setAuthState, + useAuthGeneration, + useAuthUser, +} from "./authState"; +import { LoginPage } from "./LoginPage"; +import { Button } from "../components/ui/button"; + +type GateStatus = "loading" | "login" | "authenticated" | "forbidden" | "unavailable"; + +function AuthenticatedContent({ onExpired }: { onExpired: () => void }) { + const user = useAuthUser(); + const authGeneration = useAuthGeneration(); + useEffect(() => { + if (!user) onExpired(); + }, [onExpired, user]); + return user + ? + : null; +} + +export function AuthGate() { + const [status, setStatus] = useState("loading"); + const [config, setConfig] = useState(); + const [attempt, setAttempt] = useState(0); + + const retry = useCallback(() => setAttempt((value) => value + 1), []); + + useEffect(() => { + let cancelled = false; + const load = async () => { + setStatus("loading"); + const loadGeneration = getAuthGeneration(); + try { + const publicConfig = await getAuthConfig(); + if (cancelled || !isAuthGenerationCurrent(loadGeneration)) return; + setConfig(publicConfig); + try { + const authenticated = await getMe(); + if (cancelled || !isAuthGenerationCurrent(loadGeneration)) return; + setAuthState(authenticated); + setStatus("authenticated"); + } catch (error) { + if (cancelled) return; + const statusCode = authErrorStatus(error); + if (statusCode === 401) { + if (isAuthGenerationCurrent(loadGeneration)) clearAuthStateIfCurrent(loadGeneration); + if (getAuthState() === null) setStatus("login"); + return; + } + if (!isAuthGenerationCurrent(loadGeneration)) return; + if (statusCode === 403) setStatus("forbidden"); + else if (statusCode === 503) setStatus("unavailable"); + else setStatus("login"); + } + } catch (error) { + if (cancelled || !isAuthGenerationCurrent(loadGeneration)) return; + if (error instanceof ApiError && error.status === 503) setStatus("unavailable"); + else setStatus("unavailable"); + } + }; + void load(); + return () => { cancelled = true; }; + }, [attempt]); + + if (status === "loading") { + return

Checking access…

; + } + + if (status === "authenticated") { + return setStatus("login")} />; + } + + if (status === "unavailable") { + return ( +
+
+

ThothII access

+

Authentication unavailable

+

The authentication provider could not be reached. Try again in a moment.

+ +
+
+ ); + } + + if (status === "forbidden") { + return ( +
+
+

ThothII access

+

Access not permitted

+

You are signed in without permission to use this workspace. Contact the installation administrator.

+
+
+ ); + } + + return config ? { + setStatus("authenticated"); + }} onRetry={retry} /> : null; +} diff --git a/frontend/src/auth/LoginPage.test.tsx b/frontend/src/auth/LoginPage.test.tsx new file mode 100644 index 00000000..faddf437 --- /dev/null +++ b/frontend/src/auth/LoginPage.test.tsx @@ -0,0 +1,274 @@ +import { act, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { http, HttpResponse, delay } from "msw"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { StrictMode } from "react"; +import { LoginPage } from "./LoginPage"; +import { clearAuthState, setAuthState } from "./authState"; +import * as authApi from "../api/auth"; +import { server } from "../test/msw"; + +const localConfig = { mode: "local", localLogin: true, oidcLogin: false } as const; +const oidcConfig = { mode: "oidc", localLogin: false, oidcLogin: true } as const; +const authenticated = { + issuer: "local", + subject: "admin-1", + roles: ["admin"] as const, + permissions: ["session.use", "pi.manage"], + isAdmin: true, + csrfToken: "c".repeat(43), + session: { + method: "local" as const, + remembered: true, + idleExpiresAt: "2026-08-17T10:00:00.000Z", + absoluteExpiresAt: "2026-09-16T10:00:00.000Z", + }, +}; + +beforeEach(() => clearAuthState()); +afterEach(() => clearAuthState()); + +describe("LoginPage", () => { + test("shows an unchecked local Remember me control and clears the password after failure", async () => { + let submittedPassword = ""; + server.use(http.post("/api/auth/local/login", async ({ request }) => { + const body = await request.json() as { password: string; remember: boolean }; + submittedPassword = body.password; + expect(body.remember).toBe(false); + return HttpResponse.json({ code: "invalid_credentials" }, { status: 401 }); + })); + + render(); + + expect(screen.getByRole("checkbox", { name: /remember me/i })).not.toBeChecked(); + await userEvent.type(screen.getByLabelText(/username/i), "alice"); + await userEvent.type(screen.getByLabelText(/password/i), "not-a-real-password"); + await userEvent.click(screen.getByRole("button", { name: /sign in/i })); + + expect(await screen.findByRole("alert")).toHaveTextContent(/invalid username or password/i); + expect(submittedPassword).toBe("not-a-real-password"); + expect(screen.getByLabelText(/password/i)).toHaveValue(""); + }); + + test("submits a remembered local login once and returns the safe user", async () => { + let requests = 0; + server.use( + http.post("/api/auth/local/login", async ({ request }) => { + requests += 1; + const body = await request.json() as { remember: boolean }; + expect(body.remember).toBe(true); + await delay(20); + return HttpResponse.json({}); + }), + http.get("/api/me", () => HttpResponse.json(authenticated)), + ); + const onAuthenticated = vi.fn(); + render(); + + await userEvent.click(screen.getByRole("checkbox", { name: /remember me/i })); + await userEvent.type(screen.getByLabelText(/username/i), "admin"); + await userEvent.type(screen.getByLabelText(/password/i), "correct-password"); + const submit = screen.getByRole("button", { name: /sign in/i }); + await Promise.all([userEvent.click(submit), userEvent.click(submit)]); + + expect(requests).toBe(1); + await vi.waitFor(() => expect(onAuthenticated).toHaveBeenCalledWith(expect.objectContaining({ subject: "admin-1" }))); + expect(screen.getByLabelText(/password/i)).toHaveValue(""); + }); + + test("shows OIDC only when public configuration enables it", () => { + const { rerender } = render(); + expect(screen.queryByRole("link", { name: /single sign-on/i })).not.toBeInTheDocument(); + + rerender(); + expect(screen.getByRole("button", { name: /single sign-on/i })).toBeEnabled(); + }); + + test("routes every rendered SSO affordance through the held logout barrier", async () => { + let releaseLogout!: () => void; + let logoutStarted!: () => void; + const logoutGate = new Promise((resolve) => { releaseLogout = resolve; }); + const logoutRequest = new Promise((resolve) => { logoutStarted = resolve; }); + server.use(http.post("/api/auth/logout", async () => { + logoutStarted(); + await logoutGate; + return new HttpResponse(null, { status: 204 }); + })); + setAuthState({ ...authenticated, subject: "user-a" }); + const logoutPromise = authApi.logout(); + await logoutRequest; + + const navigate = vi.fn(); + const beginOidcLogin = authApi.beginOidcLogin; + const begin = vi.spyOn(authApi, "beginOidcLogin") + .mockImplementation(() => beginOidcLogin(navigate)); + try { + render(); + const buttons = screen.getAllByRole("button", { name: /single sign-on/i }); + expect(screen.queryAllByRole("link", { name: /single sign-on/i })).toHaveLength(0); + expect(buttons).toHaveLength(1); + + await userEvent.click(buttons[0]); + await Promise.resolve(); + expect(begin).toHaveBeenCalledOnce(); + expect(navigate).not.toHaveBeenCalled(); + + releaseLogout(); + await Promise.all([logoutPromise, vi.waitFor(() => expect(navigate).toHaveBeenCalledWith("/api/auth/oidc/login"))]); + } finally { + releaseLogout(); + begin.mockRestore(); + } + }); + + test("does not dispatch local login until an in-flight logout response settles", async () => { + let releaseLogout!: () => void; + let logoutStarted!: () => void; + let loginDispatched = false; + const logoutGate = new Promise((resolve) => { releaseLogout = resolve; }); + const logoutRequest = new Promise((resolve) => { logoutStarted = resolve; }); + server.use( + http.post("/api/auth/logout", async () => { + logoutStarted(); + await logoutGate; + return new HttpResponse(null, { status: 204 }); + }), + http.post("/api/auth/local/login", () => { + loginDispatched = true; + return HttpResponse.json({}); + }), + http.get("/api/me", () => HttpResponse.json(authenticated)), + ); + setAuthState({ ...authenticated, subject: "user-a" }); + const logoutPromise = authApi.logout(); + await logoutRequest; + const loginPromise = authApi.loginLocal("admin", "password", false); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(loginDispatched).toBe(false); + releaseLogout(); + await expect(logoutPromise).resolves.toBe(true); + await loginPromise; + expect(loginDispatched).toBe(true); + }); + + test("a failed logout response does not deadlock the next local login", async () => { + let logoutCalls = 0; + let loginDispatched = false; + server.use( + http.post("/api/auth/logout", () => { + logoutCalls += 1; + return HttpResponse.json({ code: "auth_unavailable" }, { status: 503 }); + }), + http.post("/api/auth/local/login", () => { + loginDispatched = true; + return HttpResponse.json({}); + }), + http.get("/api/me", () => HttpResponse.json(authenticated)), + ); + setAuthState({ ...authenticated, subject: "user-a" }); + await expect(authApi.logout()).rejects.toMatchObject({ status: 503 }); + await authApi.loginLocal("admin", "password", false); + expect(logoutCalls).toBe(1); + expect(loginDispatched).toBe(true); + }); + + test("uses explicit provider-unavailable copy with retry affordance", async () => { + server.use(http.post("/api/auth/local/login", () => HttpResponse.json( + { code: "auth_unavailable" }, { status: 503 }, + ))); + const retry = vi.fn(); + render(); + + await userEvent.type(screen.getByLabelText(/username/i), "alice"); + await userEvent.type(screen.getByLabelText(/password/i), "correct-password"); + await userEvent.click(screen.getByRole("button", { name: /sign in/i })); + + expect(await screen.findByRole("alert")).toHaveTextContent(/temporarily unavailable/i); + await userEvent.click(screen.getByRole("button", { name: /retry/i })); + expect(retry).toHaveBeenCalledOnce(); + }); + + test("never writes the password, remember choice, or returned token to browser storage", async () => { + server.use( + http.post("/api/auth/local/login", () => HttpResponse.json({})), + http.get("/api/me", () => HttpResponse.json(authenticated)), + ); + const storageWrites = (["setItem", "removeItem", "clear"] as const).map((method) => + vi.spyOn(Storage.prototype, method)); + render(); + await userEvent.type(screen.getByLabelText(/username/i), "alice"); + await userEvent.type(screen.getByLabelText(/password/i), "correct-password"); + await userEvent.click(screen.getByRole("button", { name: /sign in/i })); + await vi.waitFor(() => expect(screen.getByLabelText(/password/i)).toHaveValue("")); + for (const write of storageWrites) expect(write).not.toHaveBeenCalled(); + for (const write of storageWrites) write.mockRestore(); + }); + + test("settles a successful login on the current StrictMode mount", async () => { + server.use( + http.post("/api/auth/local/login", () => HttpResponse.json({})), + http.get("/api/me", () => HttpResponse.json(authenticated)), + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); + const onAuthenticated = vi.fn(); + try { + render(); + await userEvent.type(screen.getByLabelText(/username/i), "admin"); + await userEvent.type(screen.getByLabelText(/password/i), "correct-password"); + await userEvent.click(screen.getByRole("button", { name: /sign in/i })); + await vi.waitFor(() => expect(onAuthenticated).toHaveBeenCalledOnce()); + expect(screen.getByLabelText(/password/i)).toHaveValue(""); + expect(screen.getByRole("button", { name: /sign in/i })).toBeEnabled(); + expect(errors.mock.calls.flat().join(" ")).not.toMatch(/not wrapped in act/i); + } finally { + errors.mockRestore(); + } + }); + + test("settles a failed login and clears the password on the current StrictMode mount", async () => { + server.use(http.post("/api/auth/local/login", () => HttpResponse.json({ code: "invalid_credentials" }, { status: 401 }))); + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + render(); + await userEvent.type(screen.getByLabelText(/username/i), "admin"); + await userEvent.type(screen.getByLabelText(/password/i), "wrong-password"); + await userEvent.click(screen.getByRole("button", { name: /sign in/i })); + expect(await screen.findByRole("alert")).toHaveTextContent(/invalid username or password/i); + expect(screen.getByLabelText(/password/i)).toHaveValue(""); + expect(screen.getByRole("button", { name: /sign in/i })).toBeEnabled(); + expect(errors.mock.calls.flat().join(" ")).not.toMatch(/not wrapped in act/i); + } finally { + errors.mockRestore(); + } + }); + + test("does not update an unmounted StrictMode login attempt", async () => { + let release!: () => void; + const pending = new Promise((resolve) => { release = resolve; }); + let settled!: () => void; + const settledAfterUnmount = new Promise((resolve) => { settled = resolve; }); + const loginLocal = vi.spyOn(authApi, "loginLocal").mockImplementation(async () => { + try { + await pending; + throw new Error("deferred invalid credentials"); + } finally { + settled(); + } + }); + const errors = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + const view = render(); + await userEvent.type(screen.getByLabelText(/username/i), "admin"); + await userEvent.type(screen.getByLabelText(/password/i), "wrong-password"); + await userEvent.click(screen.getByRole("button", { name: /sign in/i })); + view.unmount(); + release(); + await act(async () => { await settledAfterUnmount; }); + expect(loginLocal).toHaveBeenCalledOnce(); + expect(errors.mock.calls.flat().join(" ")).not.toMatch(/can't perform a react state update|not wrapped in act/i); + } finally { + loginLocal.mockRestore(); + errors.mockRestore(); + } + }); +}); diff --git a/frontend/src/auth/LoginPage.tsx b/frontend/src/auth/LoginPage.tsx new file mode 100644 index 00000000..7449a9a6 --- /dev/null +++ b/frontend/src/auth/LoginPage.tsx @@ -0,0 +1,155 @@ +import { useEffect, useRef, useState } from "react"; +import type { FormEvent } from "react"; +import { AlertTriangle, ArrowRight, LockKeyhole } from "lucide-react"; +import { ApiError } from "../api/client"; +import { beginOidcLogin, loginLocal } from "../api/auth"; +import type { AuthenticatedUser, AuthPublicConfig } from "../api/types"; +import { Button } from "../components/ui/button"; + +interface LoginPageProps { + config: AuthPublicConfig; + onAuthenticated: (user: AuthenticatedUser) => void; + onRetry?: () => void; +} + +function loginError(error: unknown): { message: string; retry: boolean } { + if (error instanceof ApiError && error.status === 503) { + return { message: "Authentication is temporarily unavailable. Try again.", retry: true }; + } + if (error instanceof ApiError && error.status === 403) { + return { message: "This sign-in request was rejected. Open ThothII from its configured address and try again.", retry: false }; + } + return { message: "Invalid username or password.", retry: false }; +} + +export function LoginPage({ config, onAuthenticated, onRetry }: LoginPageProps) { + const localLogin = config.mode === "local" && config.localLogin; + const oidcLogin = config.mode === "oidc" && config.oidcLogin; + const formRef = useRef(null); + const passwordRef = useRef(null); + const mountedRef = useRef(true); + const submittingRef = useRef(false); + const attemptRef = useRef(0); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState<{ message: string; retry: boolean }>(); + + useEffect(() => { + mountedRef.current = true; + return () => { mountedRef.current = false; }; + }, []); + + async function submit(event: FormEvent) { + event.preventDefault(); + if (submittingRef.current) return; + const form = event.currentTarget; + const values = new FormData(form); + const username = String(values.get("username") ?? ""); + const password = String(values.get("password") ?? ""); + const remember = values.get("remember") === "on"; + const attempt = ++attemptRef.current; + submittingRef.current = true; + setError(undefined); + setSubmitting(true); + try { + const user = await loginLocal(username, password, remember); + if (mountedRef.current && attemptRef.current === attempt) onAuthenticated(user); + } catch (failure) { + if (mountedRef.current && attemptRef.current === attempt) setError(loginError(failure)); + } finally { + submittingRef.current = false; + if (!mountedRef.current || attemptRef.current !== attempt) return; + // Passwords never enter React state and are cleared after every attempt, + // including operational failures and successful authentication. + form.reset(); + setSubmitting(false); + passwordRef.current?.focus(); + } + } + + function startOidcLogin() { + void beginOidcLogin(); + } + + return ( +
+
+
+

+ From intent to SQL, with Human In The Loop +

+

AI generates, you guide and approve.

+
+ +
+
+ +

Sign in to ThothII

+

Use your installation account to continue.

+
+ + {error && ( +
+

{error.message}

+ {error.retry && onRetry && ( + + )} +
+ )} + + {localLogin && ( +
+
+ + +
+
+ + +
+ + +
+ )} + {oidcLogin && ( + + )} + + {!localLogin && !config.oidcLogin && ( +

+ No browser sign-in method is enabled for this installation. +

+ )} +
+
+
+ ); +} diff --git a/frontend/src/auth/authOperation.test.ts b/frontend/src/auth/authOperation.test.ts new file mode 100644 index 00000000..ccbd92cf --- /dev/null +++ b/frontend/src/auth/authOperation.test.ts @@ -0,0 +1,40 @@ +import { beforeEach, expect, test } from "vitest"; +import { clearAuthState, setAuthState } from "./authState"; +import { captureAuthOperation, isAuthOperationCurrent } from "./authOperation"; + +const userA = { + issuer: "local", subject: "user-a", roles: ["user"] as const, + permissions: ["session.use"], isAdmin: false, csrfToken: "a".repeat(43), session: null, +}; + +beforeEach(() => { + clearAuthState(); + setAuthState(userA); +}); + +test("captures identity, generation, session target, and disposal epoch", () => { + const operation = captureAuthOperation({ sessionId: "s1", disposalEpoch: 4 }); + + expect(operation).toMatchObject({ + authGeneration: expect.any(Number), + issuer: "local", + subject: "user-a", + sessionId: "s1", + disposalEpoch: 4, + }); + expect(isAuthOperationCurrent(operation!, { sessionId: "s1", disposalEpoch: 4 })).toBe(true); +}); + +test("rejects a changed identity, target session, or disposal epoch", () => { + const operation = captureAuthOperation({ sessionId: "s1", disposalEpoch: 4 }); + setAuthState({ ...userA, subject: "user-b", csrfToken: "b".repeat(43) }); + + expect(isAuthOperationCurrent(operation!, { sessionId: "s1", disposalEpoch: 4 })).toBe(false); + expect(isAuthOperationCurrent(operation!, { sessionId: "s2", disposalEpoch: 4 })).toBe(false); + expect(isAuthOperationCurrent(operation!, { sessionId: "s1", disposalEpoch: 5 })).toBe(false); +}); + +test("does not start an authenticated operation without a principal", () => { + clearAuthState(); + expect(captureAuthOperation()).toBeNull(); +}); diff --git a/frontend/src/auth/authOperation.ts b/frontend/src/auth/authOperation.ts new file mode 100644 index 00000000..28316744 --- /dev/null +++ b/frontend/src/auth/authOperation.ts @@ -0,0 +1,65 @@ +import { getAuthGeneration, getAuthState } from "./authState"; + +export type AuthOperationGuard = Readonly<{ + authGeneration: number; + issuer: string; + subject: string; + sessionId: string | null; + disposalEpoch: number; +}>; + +export type AuthOperationPrecondition = Readonly<{ + operation: AuthOperationGuard; + isCurrent: () => boolean; +}>; + +export class StaleAuthOperationError extends Error { + constructor() { + super("The authenticated operation is no longer current"); + this.name = "StaleAuthOperationError"; + } +} + +export function captureAuthOperation(options: { + sessionId?: string | null; + disposalEpoch?: number; +} = {}): AuthOperationGuard | null { + const user = getAuthState(); + if (!user) return null; + return { + authGeneration: getAuthGeneration(), + issuer: user.issuer, + subject: user.subject, + sessionId: options.sessionId ?? null, + disposalEpoch: options.disposalEpoch ?? 0, + }; +} + +export function isAuthOperationCurrent( + operation: AuthOperationGuard | null, + options: { sessionId?: string | null; disposalEpoch: number }, +): boolean { + if (!operation) return false; + const user = getAuthState(); + return Boolean( + user + && getAuthGeneration() === operation.authGeneration + && user.issuer === operation.issuer + && user.subject === operation.subject + && operation.sessionId === (options.sessionId ?? null) + && operation.disposalEpoch === options.disposalEpoch, + ); +} + +export function requireCurrentAuthOperation( + operation: AuthOperationGuard | null, + options: { sessionId?: string | null; disposalEpoch: number }, +): asserts operation is AuthOperationGuard { + if (!isAuthOperationCurrent(operation, options)) throw new StaleAuthOperationError(); +} + +export function requireAuthOperationPrecondition( + precondition: AuthOperationPrecondition | undefined, +): void { + if (precondition && !precondition.isCurrent()) throw new StaleAuthOperationError(); +} diff --git a/frontend/src/auth/authState.test.ts b/frontend/src/auth/authState.test.ts new file mode 100644 index 00000000..b4437cc6 --- /dev/null +++ b/frontend/src/auth/authState.test.ts @@ -0,0 +1,57 @@ +import { http, HttpResponse } from "msw"; +import { afterEach, beforeEach, expect, test } from "vitest"; +import { apiFetch } from "../api/client"; +import { queryClient } from "../app/queryClient"; +import { server } from "../test/msw"; +import { useSessionStore } from "../store/sessionStore"; +import { clearAuthState, getAuthState, setAuthState } from "./authState"; + +const userA = { + issuer: "local", subject: "user-a", roles: ["user"] as const, permissions: ["session.use"], isAdmin: false, + csrfToken: "a".repeat(43), session: null, +}; +const userB = { ...userA, subject: "user-b", csrfToken: "b".repeat(43) }; + +beforeEach(() => { + queryClient.clear(); + useSessionStore.getState().resetSession(); + clearAuthState(); +}); + +afterEach(() => { + queryClient.clear(); + useSessionStore.getState().resetSession(); + clearAuthState(); +}); + +function seedUserAData() { + setAuthState(userA); + queryClient.setQueryData(["sessions", "mine"], [{ id: "a-session" }]); + useSessionStore.getState().applyEvent({ type: "text_delta", text: "A transcript" }); +} + +test("an ordinary API 401 scrubs A data before B can log in", async () => { + server.use(http.get("/api/ordinary-expiry", () => new HttpResponse(null, { status: 401 }))); + seedUserAData(); + + await expect(apiFetch("/ordinary-expiry")).rejects.toMatchObject({ status: 401 }); + expect(getAuthState()).toBeNull(); + expect(queryClient.getQueryData(["sessions", "mine"])).toBeUndefined(); + expect(useSessionStore.getState().transcript).toEqual([]); + + setAuthState(userB); + expect(getAuthState()).toMatchObject({ subject: "user-b" }); + expect(queryClient.getQueryData(["sessions", "mine"])).toBeUndefined(); + expect(useSessionStore.getState().transcript).toEqual([]); +}); + +test("logout followed by B login cannot retain A cache or live transcript", () => { + seedUserAData(); + + clearAuthState(); + setAuthState(userB); + + expect(getAuthState()).toMatchObject({ subject: "user-b" }); + expect(queryClient.getQueryData(["sessions", "mine"])).toBeUndefined(); + expect(useSessionStore.getState().transcript).toEqual([]); +}); diff --git a/frontend/src/auth/authState.ts b/frontend/src/auth/authState.ts new file mode 100644 index 00000000..fb46a65e --- /dev/null +++ b/frontend/src/auth/authState.ts @@ -0,0 +1,71 @@ +import { useSyncExternalStore } from "react"; +import type { AuthenticatedUser } from "../api/types"; +import { queryClient } from "../app/queryClient"; +import { useSessionStore } from "../store/sessionStore"; + +let current: AuthenticatedUser | null = null; +let generation = 0; +const listeners = new Set<() => void>(); + +function notify() { + for (const listener of listeners) listener(); +} + +function scrubUserBoundState(): void { + queryClient.clear(); + useSessionStore.getState().resetSession(); +} + +/** Authentication is intentionally process-local; no browser storage is involved. */ +export function getAuthState(): AuthenticatedUser | null { + return current; +} + +export function setAuthState(user: AuthenticatedUser): void { + scrubUserBoundState(); + current = user; + generation += 1; + notify(); +} + +export function clearAuthState(): void { + scrubUserBoundState(); + current = null; + generation += 1; + notify(); +} + +export function isAuthGenerationCurrent(expectedGeneration: number): boolean { + return generation === expectedGeneration; +} + +export function clearAuthStateIfCurrent(expectedGeneration: number): boolean { + if (!isAuthGenerationCurrent(expectedGeneration)) return false; + clearAuthState(); + return true; +} + +export function getAuthGeneration(): number { + return generation; +} + +export function subscribeAuthState(listener: () => void): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} + +export function useAuthState(): AuthenticatedUser | null { + return useSyncExternalStore(subscribeAuthState, getAuthState, getAuthState); +} + +export function useAuthUser(): AuthenticatedUser | null { + return useAuthState(); +} + +export function useAuthGeneration(): number { + return useSyncExternalStore(subscribeAuthState, getAuthGeneration, getAuthGeneration); +} + +export function hasPermission(user: Pick | null | undefined, permission: string): boolean { + return user?.permissions.includes(permission) ?? false; +} diff --git a/frontend/src/shell/AppShell.auth.test.tsx b/frontend/src/shell/AppShell.auth.test.tsx new file mode 100644 index 00000000..3a13d381 --- /dev/null +++ b/frontend/src/shell/AppShell.auth.test.tsx @@ -0,0 +1,210 @@ +import { act, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { http, HttpResponse } from "msw"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { AppShell } from "./AppShell"; +import { clearAuthState, getAuthGeneration, getAuthState, setAuthState, useAuthGeneration, useAuthUser } from "../auth/authState"; +import { server } from "../test/msw"; +import { useSessionStore } from "../store/sessionStore"; + +function renderShell(user: { + subject: string; + isAdmin: boolean; + roles: readonly ("user" | "admin")[]; + permissions: readonly string[]; +}) { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + setAuthState({ + issuer: "local", ...user, csrfToken: null, session: null, + }); + return render(); +} + +function KeyedAuthenticatedShell() { + const user = useAuthUser(); + const generation = useAuthGeneration(); + return user ? : null; +} + +beforeEach(() => { + clearAuthState(); + useSessionStore.getState().resetSession(); + server.use( + http.get("/api/sessions", () => HttpResponse.json([])), + http.get("/api/settings", () => HttpResponse.json({ workspace: "default", provider: "test", model: "test", thinking: "low" })), + http.get("/api/workspaces", () => HttpResponse.json([])), + http.get("/api/workspace-registry/status", () => HttpResponse.json({ + branch: "main", ahead: 0, behind: 0, degraded: false, + })), + http.get("/api/models", () => HttpResponse.json({ models: [] })), + http.get("/api/health/dwh", () => HttpResponse.json({ ok: true })), + ); +}); + +describe("authenticated shell permissions", () => { + test("shows only read-safe workspace chrome to a session user", async () => { + renderShell({ subject: "user-1", isAdmin: false, roles: ["user"], permissions: ["session.use"] }); + + expect(await screen.findByRole("button", { name: "Workspace management" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Pi management" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "All sessions" })).not.toBeInTheDocument(); + }); + + test("shows management and all-session chrome only for exact permissions", async () => { + renderShell({ + subject: "admin-1", + isAdmin: true, + roles: ["admin"], + permissions: ["session.use", "session.read_all", "workspace.manage", "workspace.secrets.manage", "pi.manage"], + }); + + expect(await screen.findByRole("button", { name: "Pi management" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "All sessions" })).toBeInTheDocument(); + }); + + test("logout revokes the cookie session and clears in-memory auth", async () => { + const user = { + issuer: "local" as const, + subject: "admin-1", + roles: ["admin"] as const, + permissions: ["session.use", "pi.manage"], + isAdmin: true, + csrfToken: "c".repeat(43), + session: null, + }; + setAuthState(user); + let logoutCalls = 0; + server.use(http.post("/api/auth/logout", () => { + logoutCalls += 1; + return new HttpResponse(null, { status: 204 }); + })); + renderShell(user); + + await userEvent.click(screen.getByRole("button", { name: "Log out" })); + + await vi.waitFor(() => expect(logoutCalls).toBe(1)); + expect(getAuthState()).toBeNull(); + }); + + test("a stale logout continuation cannot scrub user B after the logout response settles", async () => { + const userA = { + issuer: "local" as const, subject: "user-a", roles: ["user"] as const, + permissions: ["session.use"] as const, isAdmin: false, + csrfToken: "a".repeat(43), session: null, + }; + const userB = { ...userA, subject: "user-b", csrfToken: "b".repeat(43) }; + let releaseLogout!: () => void; + let logoutStarted!: () => void; + let logoutSettled!: () => void; + const logoutGate = new Promise((resolve) => { releaseLogout = resolve; }); + const started = new Promise((resolve) => { logoutStarted = resolve; }); + const settled = new Promise((resolve) => { logoutSettled = resolve; }); + server.use(http.post("/api/auth/logout", async () => { + logoutStarted(); + try { + await logoutGate; + return new HttpResponse(null, { status: 204 }); + } finally { + logoutSettled(); + } + })); + + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + setAuthState(userA); + render(); + await userEvent.click(screen.getByRole("button", { name: "Log out" })); + await started; + + act(() => setAuthState(userB)); + act(() => { + client.setQueryData(["b-only"], { owner: "user-b" }); + useSessionStore.getState().applyEvent({ type: "text_delta", text: "B transcript" }); + }); + releaseLogout(); + await act(async () => { await settled; }); + + expect(getAuthState()).toMatchObject({ subject: "user-b" }); + expect(client.getQueryData(["b-only"])).toEqual({ owner: "user-b" }); + expect(useSessionStore.getState().transcript).toEqual([{ role: "assistant", text: "B transcript" }]); + expect(screen.getByRole("button", { name: "Log out" })).toBeInTheDocument(); + }); + + test("handles a failed shell logout without an unhandled rejection", async () => { + const user = { + issuer: "local" as const, subject: "user-a", roles: ["user"] as const, + permissions: ["session.use"] as const, isAdmin: false, + csrfToken: "a".repeat(43), session: null, + }; + const rejection = vi.fn(); + process.on("unhandledRejection", rejection); + server.use(http.post("/api/auth/logout", () => HttpResponse.json( + { code: "auth_unavailable" }, { status: 503 }, + ))); + try { + renderShell(user); + await userEvent.click(screen.getByRole("button", { name: "Log out" })); + await waitFor(() => expect(getAuthState()).toBeNull()); + await new Promise((resolve) => setImmediate(resolve)); + expect(rejection).not.toHaveBeenCalled(); + } finally { + process.off("unhandledRejection", rejection); + } + }); + + test("permission chrome follows current auth state after a stale admin identity disappears", async () => { + renderShell({ + subject: "admin-1", isAdmin: true, roles: ["admin"], + permissions: ["session.use", "session.read_all", "workspace.manage", "pi.manage"], + }); + expect(await screen.findByRole("button", { name: "Pi management" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "All sessions" })).toBeInTheDocument(); + + act(() => clearAuthState()); + expect(screen.queryByRole("button", { name: "Pi management" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "All sessions" })).not.toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Workspace management" })); + expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Update workspace repository" })).not.toBeInTheDocument(); + }); + + test("remounts the real shell so user-A panel and transcript state cannot survive user-B", async () => { + const userA = { + issuer: "local" as const, + subject: "user-a", + roles: ["user"] as const, + permissions: ["session.use"] as const, + isAdmin: false, + }; + const userB = { ...userA, subject: "user-b" }; + const panelSession = { + id: "panel-a", status: "finalized", question: "User A governed question", summary: null, + created_at: "2026-08-17T10:00:00Z", updated_at: null, author: "user-a", name: null, + group: null, archived: false, + }; + server.use( + http.get("/api/sessions", () => HttpResponse.json([panelSession])), + http.get("/api/sessions/panel-a/documents", () => HttpResponse.json([ + { key: "question", title: "Question", phase: "F1", format: "markdown", content: "A-private-document" }, + ])), + ); + setAuthState({ ...userA, csrfToken: null, session: null }); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + function renderUserShell() { + return render(); + } + renderUserShell(); + + await userEvent.click(await screen.findByTestId("session-item-panel-a")); + expect(await screen.findByText("A-private-document")).toBeInTheDocument(); + act(() => useSessionStore.getState().applyEvent({ type: "text_delta", text: "A-private-transcript" })); + expect(useSessionStore.getState().transcript).toEqual([{ role: "assistant", text: "A-private-transcript" }]); + + act(() => setAuthState({ ...userB, csrfToken: null, session: null })); + + await waitFor(() => expect(screen.queryByText("A-private-document")).not.toBeInTheDocument()); + expect(screen.queryByRole("heading", { name: "User A governed question" })).not.toBeInTheDocument(); + expect(useSessionStore.getState().transcript).toEqual([]); + expect(getAuthGeneration()).toBeGreaterThan(0); + }); +}); diff --git a/frontend/src/shell/AppShell.new-session.test.tsx b/frontend/src/shell/AppShell.new-session.test.tsx index e1020410..e570ffd4 100644 --- a/frontend/src/shell/AppShell.new-session.test.tsx +++ b/frontend/src/shell/AppShell.new-session.test.tsx @@ -6,6 +6,8 @@ import { server } from "../test/msw"; import { FakeEventSource } from "../test/fakeEventSource"; import { useSessionStore } from "../store/sessionStore"; import { AppShell } from "./AppShell"; +import { clearAuthState, setAuthState } from "../auth/authState"; +import { workspacePreferences } from "../workspaces/preferences"; function renderShell() { const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); @@ -13,22 +15,29 @@ function renderShell() { } beforeEach(() => { + clearAuthState(); + setAuthState({ + issuer: "test", subject: "test", roles: ["user"], permissions: ["session.use"], + isAdmin: false, csrfToken: null, session: null, + }); + localStorage.clear(); FakeEventSource.instances = []; (globalThis as any).EventSource = FakeEventSource; useSessionStore.getState().resetSession(); server.use( - http.get("http://localhost:8787/sessions", () => HttpResponse.json([])), - http.get("http://localhost:8787/settings", () => + http.get("/api/me", () => HttpResponse.json({ issuer: "test", subject: "test", displayName: "Test", isAdmin: false })), + http.get("/api/sessions", () => HttpResponse.json([])), + http.get("/api/settings", () => HttpResponse.json({ workspace: "default", provider: "test", model: "test", thinking: "low" })), - http.get("http://localhost:8787/workspaces", () => HttpResponse.json([])), - http.get("http://localhost:8787/models", () => HttpResponse.json({ models: [] })), + http.get("/api/workspaces", () => HttpResponse.json([])), + http.get("/api/models", () => HttpResponse.json({ models: [] })), ); }); test("New session starts prewarm without delaying composer focus", async () => { let prewarmStarted = false; server.use( - http.post("http://localhost:8787/runtime/prewarm", async () => { + http.post("/api/runtime/prewarm", async () => { prewarmStarted = true; await delay(100); return new HttpResponse(null, { status: 202 }); @@ -49,7 +58,7 @@ test("records the prompt without central duplication, then opens the live log", let releaseCreate!: () => void; const createMayFinish = new Promise((resolve) => { releaseCreate = resolve; }); server.use( - http.post("http://localhost:8787/sessions", async () => { + http.post("/api/sessions", async () => { await createMayFinish; return HttpResponse.json({ id: "s-new" }); }), @@ -79,7 +88,7 @@ test("records the prompt without central duplication, then opens the live log", test("a failed create restores the landing view and preserves the question for retry", async () => { server.use( - http.post("http://localhost:8787/sessions", () => + http.post("/api/sessions", () => new HttpResponse("unavailable", { status: 503 })), ); renderShell(); @@ -97,7 +106,7 @@ test("a failed create restores the landing view and preserves the question for r test("a DWH-unreachable precheck shows a specific alert and preserves the question", async () => { server.use( - http.post("http://localhost:8787/sessions", () => + http.post("/api/sessions", () => HttpResponse.json( { error: "Cannot start a session: the database is unreachable. Check the VPN connection and try again.", @@ -121,22 +130,16 @@ test("a DWH-unreachable precheck shows a specific alert and preserves the questi expect(FakeEventSource.instances).toHaveLength(0); }); -test("model selector shows the three Pi-enabled models and persists the selected provider", async () => { - let saved: unknown; +test("model selector shows the three Pi-enabled models and stores the selected provider locally", async () => { server.use( - http.get("http://localhost:8787/settings", () => HttpResponse.json({ + http.get("/api/settings", () => HttpResponse.json({ workspace: "default", provider: "zai", model: "glm-5.2", thinking: "low", })), - http.get("http://localhost:8787/models", () => HttpResponse.json({ models: [ + http.get("/api/models", () => HttpResponse.json({ models: [ { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, { provider: "deepseek", id: "deepseek-v4-flash", name: "DeepSeek V4 Flash", reasoning: true }, { provider: "local-qwen", id: "qwen3.6-35b-a3b", name: "Qwen3.6 35B A3B Local", reasoning: false }, ] })), - http.put("http://localhost:8787/settings", async ({ request }) => { - const body = await request.json() as Record; - saved = body; - return HttpResponse.json(body); - }), ); renderShell(); @@ -147,9 +150,22 @@ test("model selector shows the three Pi-enabled models and persists the selected ]); }); await userEvent.selectOptions(select, "qwen3.6-35b-a3b"); - await waitFor(() => expect(saved).toMatchObject({ - provider: "local-qwen", model: "qwen3.6-35b-a3b", - })); + await waitFor(() => expect(workspacePreferences.load()).toEqual({ + workspaceId: "default", provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low", + })); +}); + +test("opens Workspace management from the right sidebar without interrupting the shell", async () => { + server.use( + http.get("/api/workspace-registry/status", () => + HttpResponse.json({ branch: "main", ahead: 0, behind: 0, degraded: false })), + ); + renderShell(); + + await userEvent.click(screen.getByRole("button", { name: "Workspace management" })); + + expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeVisible(); + expect(screen.getByTestId("app-shell")).toHaveAttribute("data-activity-layout", "closed"); }); diff --git a/frontend/src/shell/AppShell.notifications.test.tsx b/frontend/src/shell/AppShell.notifications.test.tsx new file mode 100644 index 00000000..9b1e9e74 --- /dev/null +++ b/frontend/src/shell/AppShell.notifications.test.tsx @@ -0,0 +1,29 @@ +import { act, render, screen } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; +import { App } from "../App"; +import { queryClient } from "../app/queryClient"; +import { useSessionStore } from "../store/sessionStore"; +import { server } from "../test/msw"; + +beforeEach(() => { + queryClient.clear(); + useSessionStore.getState().resetSession(); + server.use( + http.get("/api/me", () => HttpResponse.json({ issuer: "local", subject: "dev", isAdmin: true })), + http.get("/api/sessions", () => HttpResponse.json([])), + http.get("/api/settings", () => HttpResponse.json({ workspace: "psd-clinical" })), + http.get("/api/workspaces", () => HttpResponse.json([])), + http.get("/api/models", () => HttpResponse.json({ models: [] })), + ); +}); +test("session errors queued in the store become visible notifications", async () => { + render(); + await screen.findByTestId("app-shell"); + + act(() => useSessionStore.getState().pushToast({ + level: "error", + text: "The selected model is unavailable for this subscription.", + })); + + expect(await screen.findByText("The selected model is unavailable for this subscription.")).toBeInTheDocument(); +}); diff --git a/frontend/src/shell/AppShell.session-mgmt.test.tsx b/frontend/src/shell/AppShell.session-mgmt.test.tsx index 71f909f9..3b2b3550 100644 --- a/frontend/src/shell/AppShell.session-mgmt.test.tsx +++ b/frontend/src/shell/AppShell.session-mgmt.test.tsx @@ -5,10 +5,22 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { server } from "../test/msw"; import { FakeEventSource } from "../test/fakeEventSource"; import { AppShell } from "./AppShell"; +import type { AuthenticatedUser } from "../api/types"; +import { clearAuthState, setAuthState } from "../auth/authState"; import { useSessionStore } from "../store/sessionStore"; import { ACTIVITY_PANEL_STORAGE_KEY } from "./useActivityPanelResize"; -function wrap() { +const regularUser: AuthenticatedUser = { + issuer: "portal", subject: "alice", displayName: "Alice", roles: ["user"] as const, + permissions: ["session.use", "pi.manage"] as const, isAdmin: false, csrfToken: null, session: null, +}; +const adminUser: AuthenticatedUser = { + ...regularUser, subject: "alice-id", roles: ["admin"] as const, + permissions: ["session.use", "session.read_all", "pi.manage"] as const, isAdmin: true, +}; + +function wrap(user: AuthenticatedUser = regularUser) { + if (user !== regularUser) setAuthState(user); const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return render(); } @@ -47,6 +59,8 @@ function deferred() { } beforeEach(() => { + clearAuthState(); + setAuthState(regularUser); localStorage.clear(); FakeEventSource.instances = []; ControlledResizeObserver.instances = []; @@ -56,20 +70,25 @@ beforeEach(() => { window.matchMedia = vi.fn().mockReturnValue({ matches: true, addEventListener: vi.fn(), removeEventListener: vi.fn() }); useSessionStore.getState().resetSession(); server.use( - http.get("http://localhost:8787/me", () => - HttpResponse.json({ issuer: "portal", subject: "alice", displayName: "Alice", isAdmin: false }), + http.get("/api/me", () => + HttpResponse.json({ issuer: "portal", subject: "alice", displayName: "Alice", isAdmin: false, permissions: ["session.use", "pi.manage"] }), ), - http.get("http://localhost:8787/sessions", () => HttpResponse.json(LIST)), - http.get("http://localhost:8787/sessions/:id/documents", () => HttpResponse.json([ + http.get("/api/settings", () => HttpResponse.json({ workspace: "psd", provider: "zai", model: "glm-5.2", thinking: "medium" })), + http.get("/api/workspaces", () => HttpResponse.json([])), + http.get("/api/models", () => HttpResponse.json({ models: [] })), + http.get("/api/sessions", () => HttpResponse.json(LIST)), + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 1 })), + http.get("/api/sessions/:id/documents", () => HttpResponse.json([ { phase: "—", key: "question", title: "Domanda originale", format: "text", content: "Attiva uno" }, ])), - http.post("http://localhost:8787/sessions/:id/archive", () => new HttpResponse(null, { status: 204 })), + http.post("/api/sessions/:id/archive", () => new HttpResponse(null, { status: 204 })), + http.post("/api/runtime/prewarm", () => new HttpResponse(null, { status: 202 })), ); }); test("regular users load only their sessions and never see administrator controls", async () => { let scope: string | null = null; - server.use(http.get("http://localhost:8787/sessions", ({ request }) => { + server.use(http.get("/api/sessions", ({ request }) => { scope = new URL(request.url).searchParams.get("scope"); return HttpResponse.json(LIST); })); @@ -80,13 +99,54 @@ test("regular users load only their sessions and never see administrator control expect(screen.queryByText(/administrator view/i)).not.toBeInTheDocument(); }); +test("Pi management preserves the open session summary and the model activity timeline", async () => { + const user = userEvent.setup(); + server.use( + http.get("/api/pi-management/status", () => HttpResponse.json({ + version: "0.80.3", ready: true, + credentials: "present", + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-05T10:00:00.000Z", + })), + http.get("/api/pi-management/options", () => HttpResponse.json({ + providers: ["zai"], models: [{ provider: "zai", id: "glm-5.2" }], + reasoning: ["low", "medium", "high"], checkedAt: "2026-08-05T10:00:00.000Z", + })), + http.post("/api/sessions/:id/resume", () => resumeResult("s1")), + http.get("/api/sessions/:id", () => HttpResponse.json({ phase: 4 })), + ); + wrap(); + + await user.click(await screen.findByText("Attiva uno")); + expect(await screen.findByRole("complementary", { name: "Session summary" })).toHaveTextContent("Domanda originale"); + await user.click(screen.getByRole("button", { name: "Pi management" })); + expect(await screen.findByRole("heading", { name: "Pi management" })).toBeVisible(); + const hiddenSummary = document.querySelector('aside[aria-label="Session summary"]'); + expect(hiddenSummary).toHaveAttribute("aria-hidden", "true"); + expect(hiddenSummary).toHaveTextContent("Attiva uno"); + await user.click(screen.getByRole("button", { name: "Close Pi management" })); + await waitFor(() => { + expect(screen.getByRole("complementary", { name: "Session summary" })).toHaveTextContent("Domanda originale"); + }); + + await user.click(screen.getByRole("button", { name: "Resume" })); + await screen.findByRole("button", { name: "Show model activity" }); + act(() => useSessionStore.getState().setLastUserEntry({ kind: "input", text: "Preserved activity" })); + await user.click(screen.getByRole("button", { name: "Pi management" })); + await user.click(await screen.findByRole("button", { name: "Close Pi management" })); + await waitFor(() => expect(screen.queryByRole("heading", { name: "Pi management" })).not.toBeInTheDocument()); + await user.click(screen.getByRole("button", { name: "Show model activity" })); + expect(await screen.findByRole("heading", { name: "Model activity" })).toBeVisible(); + expect(screen.getByLabelText("Model activity timeline")).toHaveTextContent("Preserved activity"); +}); + test("administrators can explicitly switch to all sessions and see owners", async () => { let scope = ""; server.use( - http.get("http://localhost:8787/me", () => + http.get("/api/me", () => HttpResponse.json({ issuer: "portal", subject: "alice-id", displayName: "Alice", isAdmin: true }), ), - http.get("http://localhost:8787/sessions", ({ request }) => { + http.get("/api/sessions", ({ request }) => { scope = new URL(request.url).searchParams.get("scope") ?? ""; return HttpResponse.json([ { ...LIST[0], author: "Alice" }, @@ -94,7 +154,7 @@ test("administrators can explicitly switch to all sessions and see owners", asyn ]); }), ); - wrap(); + wrap(adminUser); await screen.findByRole("button", { name: "All sessions" }); expect(screen.getByRole("button", { name: "My sessions" })).toHaveAttribute("aria-pressed", "true"); expect(screen.getByRole("button", { name: "All sessions" })).toHaveAttribute("aria-pressed", "false"); @@ -109,19 +169,19 @@ test("administrators can explicitly switch to all sessions and see owners", asyn test("administrator confirms before deleting a same-named user's session", async () => { let deletes = 0; server.use( - http.get("http://localhost:8787/me", () => + http.get("/api/me", () => HttpResponse.json({ issuer: "portal", subject: "alice-id", displayName: "Alice", isAdmin: true }), ), - http.get("http://localhost:8787/sessions", () => HttpResponse.json([ + http.get("/api/sessions", () => HttpResponse.json([ { ...LIST[0], author: "Alice" }, { ...LIST[1], id: "s3", question: "Second session", archived: false, author: "Bob" }, ])), - http.delete("http://localhost:8787/sessions/:id", () => { + http.delete("/api/sessions/:id", () => { deletes += 1; return new HttpResponse(null, { status: 204 }); }), ); - wrap(); + wrap(adminUser); await userEvent.click(await screen.findByRole("button", { name: "All sessions" })); await screen.findByText("Owner: Alice"); await userEvent.click(screen.getByRole("checkbox", { name: "Select Attiva uno" })); @@ -136,18 +196,18 @@ test("administrator confirms before archiving a same-named user's session", asyn let archives = 0; const confirm = vi.spyOn(window, "confirm").mockReturnValue(false); server.use( - http.get("http://localhost:8787/me", () => + http.get("/api/me", () => HttpResponse.json({ issuer: "portal", subject: "alice-id", displayName: "Alice", isAdmin: true }), ), - http.get("http://localhost:8787/sessions", () => HttpResponse.json([ + http.get("/api/sessions", () => HttpResponse.json([ { ...LIST[0], author: "Alice" }, ])), - http.post("http://localhost:8787/sessions/:id/archive", () => { + http.post("/api/sessions/:id/archive", () => { archives += 1; return new HttpResponse(null, { status: 204 }); }), ); - wrap(); + wrap(adminUser); await userEvent.click(await screen.findByRole("button", { name: "All sessions" })); await screen.findByText("Owner: Alice"); await userEvent.click(screen.getByRole("button", { name: "Session actions" })); @@ -166,7 +226,7 @@ test("active list shows group header and hides archived sessions", async () => { test("ungrouped sessions render after groups with no 'No group' label", async () => { server.use( - http.get("http://localhost:8787/sessions", () => HttpResponse.json([ + http.get("/api/sessions", () => HttpResponse.json([ { id: "g1", status: "open", question: "In gruppo", summary: null, created_at: "2026-01-02T00:00:00Z", updated_at: null, author: null, name: null, group: "Aritmologia", archived: false }, { id: "u1", status: "open", question: "Senza gruppo", summary: null, created_at: "2026-01-03T00:00:00Z", updated_at: null, author: null, name: null, group: null, archived: false }, ])), @@ -242,13 +302,13 @@ test("supports keyboard session resizing and hides its divider when the split is test("clicking a session with a live runtime reconnects to its gate instead of the panel", async () => { let resumed: string | null = null; server.use( - http.get("http://localhost:8787/sessions", () => + http.get("/api/sessions", () => HttpResponse.json([{ ...LIST[0], active: true }])), - http.post("http://localhost:8787/sessions/:id/resume", ({ params }) => { + http.post("/api/sessions/:id/resume", ({ params }) => { resumed = params.id as string; return resumeResult(resumed, true); // warm runtime → alreadyActive }), - http.get("http://localhost:8787/sessions/:id", () => + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 1 })), ); wrap(); @@ -260,7 +320,7 @@ test("clicking a session with a live runtime reconnects to its gate instead of t }); test("New session closes an open session detail panel", async () => { - server.use(http.post("http://localhost:8787/runtime/prewarm", () => + server.use(http.post("/api/runtime/prewarm", () => HttpResponse.json({ status: "warming" }, { status: 202 }))); wrap(); await userEvent.click(await screen.findByText("Attiva uno")); // cold session → panel opens @@ -287,7 +347,7 @@ test("Resume from the panel activates the session and closes the panel", async ( activityLog: [{ kind: "status", phase: "F7", text: "Stale prior activity", level: "info" }], }); server.use( - http.post("http://localhost:8787/sessions/:id/resume", ({ params }) => { + http.post("/api/sessions/:id/resume", ({ params }) => { resumed = params.id as string; return resumeResult(resumed); }), @@ -306,8 +366,8 @@ test("Resume from the panel activates the session and closes the panel", async ( test("Resume paints the re-entry phase from the manifest after the cold Resume succeeds", async () => { useSessionStore.getState().resetSession(); server.use( - http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1")), - http.get("http://localhost:8787/sessions/:id", () => + http.post("/api/sessions/:id/resume", () => resumeResult("s1")), + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 4 })), ); wrap(); @@ -320,11 +380,11 @@ test("Resume paints the re-entry phase from the manifest after the cold Resume s test("an already-active same-session Resume preserves its EventSource and store", async () => { let resumeCalls = 0; server.use( - http.post("http://localhost:8787/sessions/:id/resume", () => { + http.post("/api/sessions/:id/resume", () => { resumeCalls += 1; return resumeResult("s1", resumeCalls > 1); }), - http.get("http://localhost:8787/sessions/:id", () => + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 1 })), ); wrap(); @@ -350,14 +410,14 @@ test("concurrent same-id Resume invocations share one cold request and replaceme const coldGate = deferred(); const coldStarted = deferred(); server.use( - http.post("http://localhost:8787/sessions/:id/resume", async () => { + http.post("/api/sessions/:id/resume", async () => { resumeCalls += 1; if (resumeCalls === 1) return resumeResult("s1", false); if (resumeCalls === 2) coldStarted.resolve(); await coldGate.promise; return resumeResult("s1", false); }), - http.get("http://localhost:8787/sessions/:id", () => + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 1 })), ); wrap(); @@ -379,7 +439,7 @@ test("concurrent same-id Resume invocations share one cold request and replaceme await waitFor(() => expect(FakeEventSource.instances).toHaveLength(2)); const replacement = FakeEventSource.instances[1]; expect(oldSource.closed).toBe(true); - expect(replacement.url).toBe("http://localhost:8787/sessions/s1/events"); + expect(replacement.url).toBe("/api/sessions/s1/events"); expect(useSessionStore.getState().activityLog).toEqual([ { kind: "lifecycle", phase: null, text: "Resuming session" }, ]); @@ -402,13 +462,13 @@ test("a committed Resume releases same-id single-flight before its manifest sett const firstS1ManifestGate = deferred(); const firstS1ManifestStarted = deferred(); server.use( - http.get("http://localhost:8787/sessions", () => HttpResponse.json([LIST[0], other])), - http.post("http://localhost:8787/sessions/:id/resume", ({ params }) => { + http.get("/api/sessions", () => HttpResponse.json([LIST[0], other])), + http.post("/api/sessions/:id/resume", ({ params }) => { const id = params.id as string; if (id === "s1") s1ResumeCalls += 1; return resumeResult(id); }), - http.get("http://localhost:8787/sessions/:id", async ({ params }) => { + http.get("/api/sessions/:id", async ({ params }) => { if (params.id === "s1") { s1ManifestCalls += 1; if (s1ManifestCalls === 1) { @@ -447,9 +507,9 @@ test("a committed Resume releases same-id single-flight before its manifest sett test("cold same-session Resume keeps the old stream until success then receives post-clear events once", async () => { server.use( - http.post("http://localhost:8787/sessions/:id/resume", () => + http.post("/api/sessions/:id/resume", () => resumeResult("s1")), - http.get("http://localhost:8787/sessions/:id", () => + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 1 })), ); wrap(); @@ -463,7 +523,7 @@ test("cold same-session Resume keeps the old stream until success then receives let markStarted!: () => void; const resumeStarted = new Promise((resolve) => { markStarted = resolve; }); const resumeReleased = new Promise((resolve) => { releaseResume = resolve; }); - server.use(http.post("http://localhost:8787/sessions/:id/resume", async () => { + server.use(http.post("/api/sessions/:id/resume", async () => { markStarted(); await resumeReleased; return resumeResult("s1"); @@ -488,7 +548,7 @@ test("cold same-session Resume keeps the old stream until success then receives await waitFor(() => expect(FakeEventSource.instances).toHaveLength(2)); expect(first.closed).toBe(true); const replacement = FakeEventSource.instances[1]; - expect(replacement.url).toBe("http://localhost:8787/sessions/s1/events"); + expect(replacement.url).toBe("/api/sessions/s1/events"); expect(useSessionStore.getState().transcript).toEqual([]); expect(useSessionStore.getState().activityLog).toEqual([ { kind: "lifecycle", phase: null, text: "Resuming session" }, @@ -520,9 +580,9 @@ test("cold same-session Resume keeps the old stream until success then receives test("a failed same-session Resume preserves its source, activity, and document panel", async () => { server.use( - http.post("http://localhost:8787/sessions/:id/resume", () => + http.post("/api/sessions/:id/resume", () => resumeResult("s1")), - http.get("http://localhost:8787/sessions/:id", () => + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 1 })), ); wrap(); @@ -533,7 +593,7 @@ test("a failed same-session Resume preserves its source, activity, and document act(() => first.emitNamed("info", { type: "info", text: "Keep me" }, "4")); const before = useSessionStore.getState().activityLog.map((entry) => ({ ...entry })); - server.use(http.post("http://localhost:8787/sessions/:id/resume", () => + server.use(http.post("/api/sessions/:id/resume", () => new HttpResponse(null, { status: 409 }))); await userEvent.click(screen.getByTestId("session-item-s1")); await userEvent.click(await screen.findByRole("button", { name: /resume/i })); @@ -552,10 +612,10 @@ test("resuming a different already-active session binds it only after success", created_at: "2026-01-03T00:00:00Z", }; server.use( - http.get("http://localhost:8787/sessions", () => HttpResponse.json([LIST[0], other])), - http.post("http://localhost:8787/sessions/:id/resume", ({ params }) => + http.get("/api/sessions", () => HttpResponse.json([LIST[0], other])), + http.post("/api/sessions/:id/resume", ({ params }) => resumeResult(params.id as string, params.id === "s3")), - http.get("http://localhost:8787/sessions/:id", ({ params }) => + http.get("/api/sessions/:id", ({ params }) => HttpResponse.json({ id: params.id, status: "open", phase: params.id === "s3" ? 3 : 1 })), ); wrap(); @@ -588,8 +648,8 @@ test("competing Resume requests for different ids commit only the latest intent" const s1Started = deferred(); const s3Started = deferred(); server.use( - http.get("http://localhost:8787/sessions", () => HttpResponse.json([LIST[0], other])), - http.post("http://localhost:8787/sessions/:id/resume", async ({ params }) => { + http.get("/api/sessions", () => HttpResponse.json([LIST[0], other])), + http.post("/api/sessions/:id/resume", async ({ params }) => { const id = params.id as string; if (id === "s1") { s1Started.resolve(); @@ -600,7 +660,7 @@ test("competing Resume requests for different ids commit only the latest intent" } return resumeResult(id); }), - http.get("http://localhost:8787/sessions/:id", ({ params }) => + http.get("/api/sessions/:id", ({ params }) => HttpResponse.json({ id: params.id, status: "open", phase: params.id === "s3" ? 3 : 1 })), ); wrap(); @@ -638,10 +698,10 @@ test("a stale Resume manifest cannot repaint the latest session phase", async () const s1ManifestGate = deferred(); const s1ManifestStarted = deferred(); server.use( - http.get("http://localhost:8787/sessions", () => HttpResponse.json([LIST[0], other])), - http.post("http://localhost:8787/sessions/:id/resume", ({ params }) => + http.get("/api/sessions", () => HttpResponse.json([LIST[0], other])), + http.post("/api/sessions/:id/resume", ({ params }) => resumeResult(params.id as string)), - http.get("http://localhost:8787/sessions/:id", async ({ params }) => { + http.get("/api/sessions/:id", async ({ params }) => { if (params.id === "s1") { s1ManifestStarted.resolve(); await s1ManifestGate.promise; @@ -671,12 +731,12 @@ test("starting a new question invalidates a pending Resume intent", async () => const resumeGate = deferred(); const resumeStarted = deferred(); server.use( - http.post("http://localhost:8787/sessions/:id/resume", async () => { + http.post("/api/sessions/:id/resume", async () => { resumeStarted.resolve(); await resumeGate.promise; return resumeResult("s1"); }), - http.post("http://localhost:8787/runtime/prewarm", () => + http.post("/api/runtime/prewarm", () => HttpResponse.json({ status: "warming" }, { status: 202 })), ); wrap(); @@ -694,17 +754,74 @@ test("starting a new question invalidates a pending Resume intent", async () => expect(screen.getByText(/type your question/i)).toBeInTheDocument(); }); +test("a delayed Resume from user A cannot activate a session after user B logs in", async () => { + const resumeGate = deferred(); + const resumeStarted = deferred(); + const resumeSettled = deferred(); + server.use( + http.post("/api/sessions/:id/resume", async () => { + resumeStarted.resolve(); + try { + await resumeGate.promise; + return resumeResult("s1"); + } finally { + resumeSettled.resolve(); + } + }), + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 4 })), + ); + wrap(); + + await userEvent.click(await screen.findByText("Attiva uno")); + await userEvent.click(await screen.findByRole("button", { name: /resume/i })); + await resumeStarted.promise; + act(() => setAuthState({ ...regularUser, subject: "bob", displayName: "Bob" })); + resumeGate.resolve(); + await act(async () => { await resumeSettled.promise; }); + expect(FakeEventSource.instances).toHaveLength(0); + expect(useSessionStore.getState().currentPhase).toBeNull(); + expect(screen.queryByRole("button", { name: "Hide model activity" })).not.toBeInTheDocument(); +}); + +test("a delayed session deletion from user A cannot refresh or toast into user B", async () => { + const deleteGate = deferred(); + const deleteStarted = deferred(); + const deleteSettled = deferred(); + server.use(http.delete("/api/sessions/:id", async () => { + deleteStarted.resolve(); + try { + await deleteGate.promise; + return new HttpResponse(null, { status: 204 }); + } finally { + deleteSettled.resolve(); + } + })); + wrap(); + + await screen.findByText("Attiva uno"); + screen.getByRole("checkbox", { name: "Select Attiva uno" }).focus(); + await userEvent.keyboard(" "); + await userEvent.click(await screen.findByRole("button", { name: "Delete 1 selected sessions" })); + await deleteStarted.promise; + act(() => setAuthState({ ...regularUser, subject: "bob", displayName: "Bob" })); + deleteGate.resolve(); + await act(async () => { await deleteSettled.promise; }); + expect(useSessionStore.getState().toasts).toEqual([]); + expect(screen.queryByText("Deleted 1 of 1 sessions.")).not.toBeInTheDocument(); + expect(screen.getByTestId("session-item-s1")).toBeInTheDocument(); +}); + test("a successful Delete invalidates an earlier pending Resume for the same target", async () => { const resumeGate = deferred(); const resumeStarted = deferred(); const deleteCompleted = deferred(); server.use( - http.post("http://localhost:8787/sessions/:id/resume", async () => { + http.post("/api/sessions/:id/resume", async () => { resumeStarted.resolve(); await resumeGate.promise; return resumeResult("s1"); }), - http.delete("http://localhost:8787/sessions/:id", ({ params }) => { + http.delete("/api/sessions/:id", ({ params }) => { expect(params.id).toBe("s1"); deleteCompleted.resolve(); return new HttpResponse(null, { status: 204 }); @@ -731,14 +848,14 @@ test("a successful Delete detaches a Resume that commits while Delete is pending const deleteGate = deferred(); const deleteStarted = deferred(); server.use( - http.delete("http://localhost:8787/sessions/:id", async ({ params }) => { + http.delete("/api/sessions/:id", async ({ params }) => { expect(params.id).toBe("s1"); deleteStarted.resolve(); await deleteGate.promise; return new HttpResponse(null, { status: 204 }); }), - http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1")), - http.get("http://localhost:8787/sessions/:id", () => + http.post("/api/sessions/:id/resume", () => resumeResult("s1")), + http.get("/api/sessions/:id", () => HttpResponse.json({ id: "s1", status: "open", phase: 1 })), ); wrap(); @@ -765,12 +882,12 @@ test("deleting another session does not invalidate a pending Resume", async () = const resumeStarted = deferred(); const deleteCompleted = deferred(); server.use( - http.post("http://localhost:8787/sessions/:id/resume", async () => { + http.post("/api/sessions/:id/resume", async () => { resumeStarted.resolve(); await resumeGate.promise; return resumeResult("s1"); }), - http.delete("http://localhost:8787/sessions/:id", ({ params }) => { + http.delete("/api/sessions/:id", ({ params }) => { expect(params.id).toBe("s2"); deleteCompleted.resolve(); return new HttpResponse(null, { status: 204 }); @@ -802,8 +919,8 @@ test("deleting active A preserves a pending Resume for different session B", asy const s3ResumeGate = deferred(); const s3ResumeStarted = deferred(); server.use( - http.get("http://localhost:8787/sessions", () => HttpResponse.json([LIST[0], other])), - http.post("http://localhost:8787/sessions/:id/resume", async ({ params }) => { + http.get("/api/sessions", () => HttpResponse.json([LIST[0], other])), + http.post("/api/sessions/:id/resume", async ({ params }) => { const id = params.id as string; if (id === "s3") { s3ResumeStarted.resolve(); @@ -811,9 +928,9 @@ test("deleting active A preserves a pending Resume for different session B", asy } return resumeResult(id); }), - http.get("http://localhost:8787/sessions/:id", ({ params }) => + http.get("/api/sessions/:id", ({ params }) => HttpResponse.json({ id: params.id, status: "open", phase: params.id === "s3" ? 3 : 1 })), - http.delete("http://localhost:8787/sessions/:id", ({ params }) => { + http.delete("/api/sessions/:id", ({ params }) => { expect(params.id).toBe("s1"); return new HttpResponse(null, { status: 204 }); }), @@ -847,12 +964,12 @@ test("a failed Delete does not invalidate a pending Resume for its target", asyn const resumeStarted = deferred(); const deleteFailed = deferred(); server.use( - http.post("http://localhost:8787/sessions/:id/resume", async () => { + http.post("/api/sessions/:id/resume", async () => { resumeStarted.resolve(); await resumeGate.promise; return resumeResult("s1"); }), - http.delete("http://localhost:8787/sessions/:id", ({ params }) => { + http.delete("/api/sessions/:id", ({ params }) => { expect(params.id).toBe("s1"); deleteFailed.resolve(); return new HttpResponse(null, { status: 500 }); @@ -876,7 +993,7 @@ test("a failed Delete does not invalidate a pending Resume for its target", asyn }); test("a failed Resume with no active session preserves the panel and existing activity", async () => { - server.use(http.post("http://localhost:8787/sessions/:id/resume", () => new HttpResponse(null, { status: 409 }))); + server.use(http.post("/api/sessions/:id/resume", () => new HttpResponse(null, { status: 409 }))); useSessionStore.setState({ activityLog: [{ kind: "status", phase: "F7", text: "Preserve activity", level: "info" }], }); @@ -894,7 +1011,7 @@ test("a failed Resume with no active session preserves the panel and existing ac test("closing and reopening Model activity preserves the complete activity log", async () => { wrap(); - server.use(http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1"))); + server.use(http.post("/api/sessions/:id/resume", () => resumeResult("s1"))); await userEvent.click(await screen.findByText("Attiva uno")); await userEvent.click(await screen.findByRole("button", { name: /resume/i })); act(() => { @@ -942,8 +1059,8 @@ test("session finalization shows the completion banner and returns to landing", useSessionStore.getState().resetSession(); let finalized = false; server.use( - http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1")), - http.get("http://localhost:8787/sessions", () => + http.post("/api/sessions/:id/resume", () => resumeResult("s1")), + http.get("/api/sessions", () => HttpResponse.json(finalized ? [{ ...LIST[0], status: "finalized" }] : LIST)), ); wrap(); @@ -966,7 +1083,7 @@ test("session finalization shows the completion banner and returns to landing", test("renaming a group reassigns its members via setSessionGroup", async () => { const groupSets: Array<{ id: string; group: string }> = []; server.use( - http.post("http://localhost:8787/sessions/:id/group", async ({ params, request }) => { + http.post("/api/sessions/:id/group", async ({ params, request }) => { const body = (await request.json()) as { group: string }; groupSets.push({ id: params.id as string, group: body.group }); return new HttpResponse(null, { status: 204 }); @@ -984,7 +1101,7 @@ test("renaming a group reassigns its members via setSessionGroup", async () => { test("opens an accessible resizable activity split and persists pointer width", async () => { - server.use(http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1"))); + server.use(http.post("/api/sessions/:id/resume", () => resumeResult("s1"))); wrap(); await userEvent.click(await screen.findByText("Attiva uno")); await userEvent.click(await screen.findByRole("button", { name: /resume/i })); @@ -1014,7 +1131,7 @@ test("opens an accessible resizable activity split and persists pointer width", test("resizes the activity split with keyboard and exposes responsive drawer classes", async () => { localStorage.setItem(ACTIVITY_PANEL_STORAGE_KEY, "448"); - server.use(http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1"))); + server.use(http.post("/api/sessions/:id/resume", () => resumeResult("s1"))); wrap(); await userEvent.click(await screen.findByText("Attiva uno")); await userEvent.click(await screen.findByRole("button", { name: /resume/i })); @@ -1041,7 +1158,7 @@ test("resizes the activity split with keyboard and exposes responsive drawer cla test("uses the measured app shell for the desktop activity split", async () => { localStorage.setItem(ACTIVITY_PANEL_STORAGE_KEY, "576"); - server.use(http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1"))); + server.use(http.post("/api/sessions/:id/resume", () => resumeResult("s1"))); wrap(); await userEvent.click(await screen.findByText("Attiva uno")); await userEvent.click(await screen.findByRole("button", { name: /resume/i })); @@ -1063,7 +1180,7 @@ test("uses the measured app shell for the desktop activity split", async () => { test("cancels an active resize when the measured shell becomes too narrow", async () => { localStorage.setItem(ACTIVITY_PANEL_STORAGE_KEY, "576"); - server.use(http.post("http://localhost:8787/sessions/:id/resume", () => resumeResult("s1"))); + server.use(http.post("/api/sessions/:id/resume", () => resumeResult("s1"))); wrap(); await userEvent.click(await screen.findByText("Attiva uno")); await userEvent.click(await screen.findByRole("button", { name: /resume/i })); diff --git a/frontend/src/shell/AppShell.session-target.test.tsx b/frontend/src/shell/AppShell.session-target.test.tsx new file mode 100644 index 00000000..e4cba8f3 --- /dev/null +++ b/frontend/src/shell/AppShell.session-target.test.tsx @@ -0,0 +1,108 @@ +import { act, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { beforeEach, expect, test, vi } from "vitest"; +import { http, HttpResponse } from "msw"; +import { AppShell } from "./AppShell"; +import { server } from "../test/msw"; +import { FakeEventSource } from "../test/fakeEventSource"; +import { clearAuthState, setAuthState } from "../auth/authState"; +import { useSessionStore } from "../store/sessionStore"; + +function renderShell() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render(); +} + +function deferred() { + let resolve!: () => void; + const promise = new Promise((onResolve) => { resolve = onResolve; }); + return { promise, resolve }; +} + +beforeEach(() => { + clearAuthState(); + setAuthState({ + issuer: "local", subject: "user-a", roles: ["user"], permissions: ["session.use"], + isAdmin: false, csrfToken: null, session: null, + }); + FakeEventSource.instances = []; + (globalThis as { EventSource: typeof EventSource }).EventSource = FakeEventSource as unknown as typeof EventSource; + useSessionStore.getState().resetSession(); + server.use( + http.get("/api/me", () => HttpResponse.json({ issuer: "local", subject: "user-a", isAdmin: false })), + http.get("/api/settings", () => HttpResponse.json({ workspace: "default", provider: "test", model: "test", thinking: "low" })), + http.get("/api/workspaces", () => HttpResponse.json([])), + http.get("/api/models", () => HttpResponse.json({ models: [] })), + http.post("/api/runtime/prewarm", () => new HttpResponse(null, { status: 202 })), + ); +}); + +test("a held stop for s1 cannot reset the newer active s2 session", async () => { + const closeGate = deferred(); + const closeStarted = deferred(); + const active = (id: string, question: string) => ({ + id, status: "open", question, summary: null, created_at: "2026-01-02T00:00:00Z", + updated_at: null, author: null, name: null, group: null, archived: false, active: true, + }); + server.use( + http.get("/api/sessions", () => HttpResponse.json([active("s1", "Active one"), active("s2", "Active two")])), + http.post("/api/sessions/:id/resume", ({ params }) => HttpResponse.json({ id: params.id, alreadyActive: false })), + http.get("/api/sessions/:id", ({ params }) => HttpResponse.json({ id: params.id, status: "open", phase: 1 })), + http.post("/api/sessions/s1/close", async () => { + closeStarted.resolve(); + await closeGate.promise; + return new HttpResponse(null, { status: 204 }); + }), + ); + renderShell(); + await userEvent.click(await screen.findByTestId("session-item-s1")); + await waitFor(() => expect(FakeEventSource.instances.at(-1)?.url).toContain("/sessions/s1/events")); + await userEvent.click(screen.getByRole("button", { name: /stop and save session/i })); + await userEvent.click(await screen.findByRole("button", { name: "Stop & save" })); + await closeStarted.promise; + + await userEvent.click(screen.getByTestId("session-item-s2")); + await waitFor(() => expect(FakeEventSource.instances.at(-1)?.url).toContain("/sessions/s2/events")); + act(() => useSessionStore.setState({ currentPhase: "F2" })); + closeGate.resolve(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(FakeEventSource.instances.at(-1)?.url).toContain("/sessions/s2/events"); + expect(FakeEventSource.instances.at(-1)?.closed).toBe(false); + expect(useSessionStore.getState().currentPhase).toBe("F2"); +}); + +test("a held new-session completion cannot replace the newer active s2 target", async () => { + const createGate = deferred(); + const createStarted = deferred(); + server.use( + http.get("/api/sessions", () => HttpResponse.json([{ + id: "s2", status: "open", question: "Current session", summary: null, + created_at: "2026-01-02T00:00:00Z", updated_at: null, author: null, name: null, + group: null, archived: false, active: true, + }])), + http.post("/api/sessions", async () => { + createStarted.resolve(); + await createGate.promise; + return HttpResponse.json({ id: "s3" }); + }), + http.post("/api/sessions/:id/resume", ({ params }) => HttpResponse.json({ id: params.id, alreadyActive: false })), + http.get("/api/sessions/:id", ({ params }) => HttpResponse.json({ id: params.id, status: "open", phase: 1 })), + ); + renderShell(); + const composer = screen.getByRole("textbox", { name: /new question/i }); + await userEvent.type(composer, "Held new question"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + await createStarted.promise; + + await userEvent.click(screen.getByTestId("session-item-s2")); + await waitFor(() => expect(FakeEventSource.instances.at(-1)?.url).toContain("/sessions/s2/events")); + act(() => useSessionStore.setState({ currentPhase: "F2" })); + createGate.resolve(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(FakeEventSource.instances.at(-1)?.url).toContain("/sessions/s2/events"); + expect(FakeEventSource.instances.at(-1)?.closed).toBe(false); + expect(useSessionStore.getState().currentPhase).toBe("F2"); +}); diff --git a/frontend/src/shell/AppShell.tsx b/frontend/src/shell/AppShell.tsx index 6a35646d..7041678e 100644 --- a/frontend/src/shell/AppShell.tsx +++ b/frontend/src/shell/AppShell.tsx @@ -3,6 +3,8 @@ import { useSessionStore } from "../store/sessionStore"; import { WidgetHost } from "./WidgetHost"; import { CentralStatus } from "./CentralStatus"; import { ModelActivityPanel } from "./ModelActivityPanel"; +import { WorkspaceManager } from "./WorkspaceManager"; +import { PiManagement } from "./PiManagement"; import { useActivityPanelResize } from "./useActivityPanelResize"; import { useSessionPanelResize } from "./useSessionPanelResize"; import { NavSessions } from "./NavSessions"; @@ -19,25 +21,23 @@ import { Checkbox } from "../components/ui/checkbox"; import { Toaster } from "../components/ui/sonner"; import { toast } from "sonner"; import { - closeSession, getMe, listSessions, resumeSession, getSession, + closeSession, listSessions, resumeSession, getSession, renameSession, setSessionGroup, archiveSession, unarchiveSession, deleteSession, prewarmRuntime, checkDwhHealth, } from "../api/sessions"; +import { logout as logoutUser } from "../api/auth"; import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, } from "../components/ui/dialog"; -import type { Principal, SessionScope, SessionSummary } from "../api/types"; +import type { SessionScope, SessionSummary } from "../api/types"; +import { useAuthGeneration, useAuthUser } from "../auth/authState"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useEffect, useMemo, useRef, useState } from "react"; import type { CSSProperties } from "react"; +import { captureAuthOperation, isAuthOperationCurrent, StaleAuthOperationError, type AuthOperationGuard } from "../auth/authOperation"; -/** - * The page is designed to live INSIDE the Omics Portal chrome (its left sidebar - * + topbar) once embedded, so this shell intentionally has no left rail and no - * top header of its own. The session rail sits on the RIGHT, mirroring the - * portal's left sidebar, and the workflow phases ride a slim strip above the chat. - */ export function AppShell() { + const authenticatedUser = useAuthUser(); const [panelSession, setPanelSession] = useState(null); const { containerRef, @@ -52,9 +52,8 @@ export function AppShell() { resizing: sessionResizing, separatorProps: sessionSeparatorProps, } = useSessionPanelResize(containerRef, panelSession !== null); - // Publish the app area's horizontal geometry as CSS vars on : when the app - // is embedded beside the portal's left sidebar, viewport-fixed dialogs (dialog.tsx) - // must center on the ThothII area, not on the whole browser window. + // Publish the app area's horizontal geometry so viewport-fixed dialogs center on + // the application area rather than the whole browser window. useEffect(() => { const el = containerRef.current; if (!el) return; @@ -81,24 +80,43 @@ export function AppShell() { } as CSSProperties; const [activeSessionId, setActiveSessionId] = useState(null); const activeSessionIdRef = useRef(null); + const activeSessionEpochRef = useRef(0); + const newSessionOperationRef = useRef<{ target: string | null; epoch: number } | null>(null); const resumeInvocationRef = useRef(0); const latestResumeIntentRef = useRef<{ token: number; id: string } | null>(null); const resumeInFlightRef = useRef(new Map; + guard: AuthOperationGuard; }>()); const [streamCursorResetEpoch, setStreamCursorResetEpoch] = useState(0); const [creatingSession, setCreatingSession] = useState(false); const [awaitingQuestion, setAwaitingQuestion] = useState(false); const [sessionScope, setSessionScope] = useState("mine"); - const { data: principal } = useQuery({ queryKey: ["me"], queryFn: getMe, staleTime: Infinity }); + const principal = authenticatedUser; + const permissions = authenticatedUser?.permissions ?? []; + const canReadAllSessions = permissions.includes("session.read_all"); + const canManageWorkspace = permissions.includes("workspace.manage"); + const canManageWorkspaceSecrets = permissions.includes("workspace.secrets.manage"); + const canManagePi = permissions.includes("pi.manage"); + const authGeneration = useAuthGeneration(); const { data: sessions = [] } = useQuery({ - queryKey: ["sessions", sessionScope], queryFn: () => listSessions(sessionScope), refetchInterval: 10_000, + queryKey: ["sessions", sessionScope], + queryFn: async () => { + const guard = captureAuthOperation({ disposalEpoch: operationEpochRef.current }); + if (!guard) throw new StaleAuthOperationError(); + const result = await listSessions(sessionScope); + if (!isAuthOperationCurrent(guard, { disposalEpoch: operationEpochRef.current })) throw new StaleAuthOperationError(); + return result; + }, + refetchInterval: 10_000, }); const composerRef = useRef(null); const queryClient = useQueryClient(); const [showActivity, setShowActivity] = useState(false); + const [workspaceManagerOpen, setWorkspaceManagerOpen] = useState(false); + const [piManagementOpen, setPiManagementOpen] = useState(false); const [activeOpen, setActiveOpen] = useState(true); const [archiveOpen, setArchiveOpen] = useState(false); const [renameTarget, setRenameTarget] = useState(null); @@ -110,11 +128,15 @@ export function AppShell() { const [dwhDown, setDwhDown] = useState(false); const [dwhChecking, setDwhChecking] = useState(true); const [dwhCheckEpoch, setDwhCheckEpoch] = useState(0); + const operationEpochRef = useRef(0); + useEffect(() => () => { operationEpochRef.current += 1; }, []); useEffect(() => { let cancelled = false; + const operation = captureAuthOperation({ disposalEpoch: operationEpochRef.current }); + if (!operation) return () => { cancelled = true; }; setDwhChecking(true); checkDwhHealth().then((r) => { - if (cancelled) return; + if (cancelled || !isAuthOperationCurrent(operation, { disposalEpoch: operationEpochRef.current })) return; setDwhDown(!r.ok); setDwhChecking(false); }); @@ -128,7 +150,10 @@ export function AppShell() { const activeList = sessions.filter((s) => !s.archived); const ungroupedActive = activeList.filter((s) => !s.group); const archivedList = sessions.filter((s) => s.archived); - const refresh = () => queryClient.invalidateQueries({ queryKey: ["sessions"] }); + const refresh = (operation?: AuthOperationGuard | null) => { + if (operation && !isAuthOperationCurrent(operation, { disposalEpoch: operationEpochRef.current })) return; + void queryClient.invalidateQueries({ queryKey: ["sessions"] }); + }; const activeSession = sessions.find((s) => s.id === activeSessionId) ?? null; const finalized = activeSession?.status === "finalized"; const selectedSessions = sessions.filter((session) => selectedSessionIds.has(session.id)); @@ -142,6 +167,7 @@ export function AppShell() { function selectActiveSession(id: string | null) { // Keep async Resume completions synchronized before React commits the state update. + if (activeSessionIdRef.current !== id) activeSessionEpochRef.current += 1; activeSessionIdRef.current = id; setActiveSessionId(id); } @@ -161,6 +187,10 @@ export function AppShell() { }); }, [sessions]); + useEffect(() => { + if (!canReadAllSessions && sessionScope === "all") setSessionScope("mine"); + }, [canReadAllSessions, sessionScope]); + function setSessionSelected(id: string, selected: boolean) { setSelectedSessionIds((current) => { const next = new Set(current); @@ -196,10 +226,12 @@ export function AppShell() { }); } async function doResume(id: string) { + const guard = captureAuthOperation({ sessionId: id, disposalEpoch: operationEpochRef.current }); + if (!guard) return; const token = ++resumeInvocationRef.current; latestResumeIntentRef.current = { token, id }; const inFlight = resumeInFlightRef.current.get(id); - if (inFlight) { + if (inFlight && isAuthOperationCurrent(inFlight.guard, { sessionId: id, disposalEpoch: operationEpochRef.current })) { // Repeated intent for the same target shares one backend lifecycle operation and one // commit path. Updating its token still lets s1→s2→s1 make the final s1 intent authoritative. inFlight.latestToken = token; @@ -209,8 +241,9 @@ export function AppShell() { const operation = { latestToken: token, promise: Promise.resolve(), + guard, }; - operation.promise = runResume(id, operation).finally(() => { + operation.promise = runResume(id, operation, guard).finally(() => { if (resumeInFlightRef.current.get(id) === operation) { resumeInFlightRef.current.delete(id); } @@ -222,9 +255,11 @@ export function AppShell() { async function runResume( id: string, operation: { latestToken: number; promise: Promise }, + guard: AuthOperationGuard, ) { try { const result = await resumeSession(id); + if (!isAuthOperationCurrent(guard, { sessionId: id, disposalEpoch: operationEpochRef.current })) return; const latest = latestResumeIntentRef.current; if (latest?.token !== operation.latestToken || latest.id !== id) return; const reconnectSameSession = activeSessionIdRef.current === id; @@ -257,6 +292,7 @@ export function AppShell() { // The manifest's `phase` is the 1-based current phase (1..8). try { const m = (await getSession(id)) as { phase?: number }; + if (!isAuthOperationCurrent(guard, { sessionId: id, disposalEpoch: operationEpochRef.current })) return; const latestAfterManifest = latestResumeIntentRef.current; if ( latestAfterManifest?.token !== operation.latestToken @@ -270,7 +306,7 @@ export function AppShell() { /* non-fatal: the first gate will set the phase */ } } catch { - if ( + if (isAuthOperationCurrent(guard, { sessionId: id, disposalEpoch: operationEpochRef.current }) && latestResumeIntentRef.current?.token === operation.latestToken && latestResumeIntentRef.current.id === id ) { @@ -279,41 +315,73 @@ export function AppShell() { } } async function move(s: SessionSummary, group: string) { + const guard = captureAuthOperation({ sessionId: s.id, disposalEpoch: operationEpochRef.current }); + if (!guard) return; try { - await setSessionGroup(s.id, group); refresh(); + await setSessionGroup(s.id, group); + if (!isAuthOperationCurrent(guard, { sessionId: s.id, disposalEpoch: operationEpochRef.current })) return; + refresh(guard); } catch { - toast.error("Failed to move session."); + if (isAuthOperationCurrent(guard, { sessionId: s.id, disposalEpoch: operationEpochRef.current })) toast.error("Failed to move session."); } } async function newGroup(s: SessionSummary) { const name = window.prompt("New group:"); if (name && name.trim()) { + const guard = captureAuthOperation({ sessionId: s.id, disposalEpoch: operationEpochRef.current }); + if (!guard) return; try { - await setSessionGroup(s.id, name.trim()); refresh(); + await setSessionGroup(s.id, name.trim()); + if (!isAuthOperationCurrent(guard, { sessionId: s.id, disposalEpoch: operationEpochRef.current })) return; + refresh(guard); } catch { - toast.error("Failed to update group."); + if (isAuthOperationCurrent(guard, { sessionId: s.id, disposalEpoch: operationEpochRef.current })) toast.error("Failed to update group."); } } } - async function renameGroup(oldName: string, newName: string) { + async function renameGroup(oldName: string, newName: string, suppliedGuard?: AuthOperationGuard | null) { const trimmed = newName.trim(); - if (!trimmed || trimmed === oldName) return; + if (!trimmed || trimmed === oldName) return false; + const guard = suppliedGuard ?? captureAuthOperation({ disposalEpoch: operationEpochRef.current }); + if (!guard) return false; try { for (const s of sessions.filter((x) => x.group === oldName)) { await setSessionGroup(s.id, trimmed); + if (!isAuthOperationCurrent(guard, { disposalEpoch: operationEpochRef.current })) return false; } - refresh(); + refresh(guard); + return true; } catch { - toast.error("Failed to rename group."); + if (isAuthOperationCurrent(guard, { disposalEpoch: operationEpochRef.current })) toast.error("Failed to rename group."); + return false; + } + } + async function renameSessionFromDialog(id: string, name: string) { + const guard = captureAuthOperation({ sessionId: id, disposalEpoch: operationEpochRef.current }); + if (!guard) return; + try { + await renameSession(id, name); + if (!isAuthOperationCurrent(guard, { sessionId: id, disposalEpoch: operationEpochRef.current })) return; + setRenameTarget(null); + refresh(guard); + } catch { + if (isAuthOperationCurrent(guard, { sessionId: id, disposalEpoch: operationEpochRef.current })) { + toast.error("Failed to rename session."); + } } } async function toggleArchive(s: SessionSummary) { + const guard = captureAuthOperation({ sessionId: s.id, disposalEpoch: operationEpochRef.current }); + if (!guard) return; try { await (s.archived ? unarchiveSession(s.id) : archiveSession(s.id)); + if (!isAuthOperationCurrent(guard, { sessionId: s.id, disposalEpoch: operationEpochRef.current })) return; if (panelSession?.id === s.id) setPanelSession(null); - refresh(); + refresh(guard); } catch { - toast.error(s.archived ? "Failed to restore session." : "Failed to archive session."); + if (isAuthOperationCurrent(guard, { sessionId: s.id, disposalEpoch: operationEpochRef.current })) { + toast.error(s.archived ? "Failed to restore session." : "Failed to archive session."); + } } } @@ -326,8 +394,11 @@ export function AppShell() { } async function deleteSessions(targets: SessionSummary[]) { + const guard = captureAuthOperation({ disposalEpoch: operationEpochRef.current }); + if (!guard) return; try { const results = await Promise.allSettled(targets.map((session) => deleteSession(session.id))); + if (!isAuthOperationCurrent(guard, { disposalEpoch: operationEpochRef.current })) return; const deletedIds = new Set( targets.filter((_, index) => results[index].status === "fulfilled").map((session) => session.id), ); @@ -337,12 +408,12 @@ export function AppShell() { if (deletedIds.has(panelSession?.id ?? "")) setPanelSession(null); if (deletedActiveSession) { resetSession(); selectActiveSession(null); } setSelectedSessionIds((current) => new Set([...current].filter((id) => !deletedIds.has(id)))); - refresh(); + refresh(guard); if (deletedIds.size !== targets.length) { toast.error(`Deleted ${deletedIds.size} of ${targets.length} sessions.`); } } catch { - toast.error("Failed to delete selected sessions."); + if (isAuthOperationCurrent(guard, { disposalEpoch: operationEpochRef.current })) toast.error("Failed to delete selected sessions."); } } @@ -375,19 +446,42 @@ export function AppShell() { // condition the final workflow step — which ends with no follow-up gate — would // leave the working state on forever. const pendingWidget = useSessionStore((s) => s.pendingWidget); + const sessionToasts = useSessionStore((s) => s.toasts); const resetSession = useSessionStore((s) => s.resetSession); const recordLifecycle = useSessionStore((s) => s.recordLifecycle); const setPhase = useSessionStore((s) => s.setPhase); const setAgentActive = useSessionStore((s) => s.setAgentActive); const lastSystemEvent = useSessionStore((s) => s.lastSystemEvent); const agentActive = useSessionStore((s) => s.agentActive); + const deliveredToastCountRef = useRef(0); + + useEffect(() => { + if (sessionToasts.length < deliveredToastCountRef.current) { + deliveredToastCountRef.current = 0; + } + + for (const notification of sessionToasts.slice(deliveredToastCountRef.current)) { + if (notification.level === "error") toast.error(notification.text); + else if (notification.level === "success") toast.success(notification.text); + else if (notification.level === "warning") toast.warning(notification.text); + else toast.info(notification.text); + } + + deliveredToastCountRef.current = sessionToasts.length; + }, [sessionToasts]); const sessionViewOpen = Boolean(activeSessionId) || creatingSession; const working = sessionViewOpen && !pendingWidget && agentActive; // The workflow bar runs only while the harness works, not while a finalized // session sits idle or a gate awaits the reviewer (pendingWidget). const running = working && !finalized; - useSessionStream(activeSessionId, 0, streamCursorResetEpoch); + useSessionStream( + activeSessionId, + 0, + streamCursorResetEpoch, + Boolean(authenticatedUser), + authGeneration, + ); // A backend "session_exit" system event (e.g. the replay server emitting it // when the reviewer picks "Esci") asks us to leave the live session view and @@ -414,6 +508,7 @@ export function AppShell() { function startNewSession() { invalidateResumeIntent(); + newSessionOperationRef.current = null; resetSession(); // Starting a new question closes any open session detail panel: the reader is // moving away from that session, so its left-hand box must not linger. @@ -428,11 +523,18 @@ export function AppShell() { } function beginSessionCreation() { + newSessionOperationRef.current = { + target: activeSessionIdRef.current, + epoch: activeSessionEpochRef.current, + }; setAwaitingQuestion(false); setCreatingSession(true); } function finishSessionCreation(id: string) { + const operation = newSessionOperationRef.current; + newSessionOperationRef.current = null; + if (!operation || operation.target !== activeSessionIdRef.current || operation.epoch !== activeSessionEpochRef.current) return; // React batches these updates, preserving the provisional session view // while useSessionStream opens the durable session's SSE channel. selectActiveSession(id); @@ -442,23 +544,35 @@ export function AppShell() { } function failSessionCreation(message?: string) { + const operation = newSessionOperationRef.current; + newSessionOperationRef.current = null; + if (!operation || operation.target !== activeSessionIdRef.current || operation.epoch !== activeSessionEpochRef.current) return; setCreatingSession(false); resetSession(); toast.error(message ?? "Failed to create session. Your question is ready to retry."); } async function stopSession() { - if (!activeSessionId) return; + const id = activeSessionIdRef.current; + if (!id) return; + const guard = captureAuthOperation({ sessionId: id, disposalEpoch: operationEpochRef.current }); + if (!guard) return; invalidateResumeIntent(); try { - await closeSession(activeSessionId); + await closeSession(id); } finally { + if (!isAuthOperationCurrent(guard, { sessionId: id, disposalEpoch: operationEpochRef.current }) + || activeSessionIdRef.current !== id) return; resetSession(); selectActiveSession(null); setAwaitingQuestion(false); } } + async function signOut() { + await logoutUser(); + } + return (
)} - {showActivity && setShowActivity(false)} />} + {showActivity && setShowActivity(false)} onOpenWorkspaceManager={() => setWorkspaceManagerOpen(true)} />} {showActivity && desktopSplit && (
)} - {activeSessionId && } + {activeSessionId && } {finalized && !agentActive && (

@@ -569,6 +683,7 @@ export function AppShell() {

- {/* Right session rail — symmetric to the portal's left sidebar */} + {/* Right session rail */} {!showActivity && ( )} + setWorkspaceManagerOpen(false)} + canManageWorkspace={canManageWorkspace} + canManageSecrets={canManageWorkspaceSecrets} + /> + {canManagePi && setPiManagementOpen(false)} />} @@ -772,15 +920,7 @@ export function AppShell() { open initial={renameTarget.name ?? ""} onOpenChange={(o) => { if (!o) setRenameTarget(null); }} - onSubmit={async (name) => { - try { - await renameSession(renameTarget.id, name); - setRenameTarget(null); - refresh(); - } catch { - toast.error("Failed to rename session."); - } - }} + onSubmit={(name) => renameSessionFromDialog(renameTarget.id, name)} /> )} {renameGroupTarget && ( @@ -788,7 +928,13 @@ export function AppShell() { open initial={renameGroupTarget} onOpenChange={(o) => { if (!o) setRenameGroupTarget(null); }} - onSubmit={async (name) => { await renameGroup(renameGroupTarget, name); setRenameGroupTarget(null); }} + onSubmit={async (name) => { + const target = renameGroupTarget; + const guard = captureAuthOperation({ disposalEpoch: operationEpochRef.current }); + if (await renameGroup(target, name, guard) && guard && isAuthOperationCurrent(guard, { disposalEpoch: operationEpochRef.current })) { + setRenameGroupTarget(null); + } + }} title="Rename group" /> )} diff --git a/frontend/src/shell/ModelActivityPanel.tsx b/frontend/src/shell/ModelActivityPanel.tsx index da5f39d1..08db885e 100644 --- a/frontend/src/shell/ModelActivityPanel.tsx +++ b/frontend/src/shell/ModelActivityPanel.tsx @@ -1,5 +1,5 @@ import { useLayoutEffect, useRef } from "react"; -import { X } from "lucide-react"; +import { Settings2, X } from "lucide-react"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; import type { ActivityEntry, ActivityKind } from "../api/types"; @@ -61,9 +61,11 @@ function ActivityRow({ entry }: { entry: ActivityEntry }) { export function ModelActivityPanel({ desktopSplit = false, onClose, + onOpenWorkspaceManager, }: { desktopSplit?: boolean; onClose: () => void; + onOpenWorkspaceManager?: () => void; }) { const activityLog = useSessionStore((s) => s.activityLog); const visibleActivity = activityLog.filter(isVisibleModelActivity); @@ -82,11 +84,16 @@ export function ModelActivityPanel({ ? "static z-auto flex min-w-0 w-[var(--activity-panel-width)] shrink-0 flex-col border-r-0 bg-sidebar" : "fixed inset-y-0 left-0 z-30 flex min-w-0 w-[min(90vw,24rem)] shrink-0 flex-col border-r border-border bg-sidebar" }> -
+

Model activity

- +
+ + +
{ +test("submitting includes ephemeral migrated preferences and calls onCreated", async () => { + localStorage.clear(); let body: unknown = null; server.use( - http.post("http://localhost:8787/sessions", async ({ request }) => { + http.get("/api/settings", () => HttpResponse.json({ + workspace: "default", provider: "zai", model: "glm-5.2", thinking: "low", + })), + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("default", { + displayName: "Default", + revision: workspaceRevisionFixture("default"), + }), + }])), + http.get("/api/workspaces/default", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("default", ["zai/glm-5.2"], "zai/glm-5.2"), + revision: workspaceRevisionFixture("default"), + })), + http.post("/api/sessions", async ({ request }) => { body = await request.json(); return HttpResponse.json({ id: "s1" }); }), @@ -40,7 +56,59 @@ test("submitting posts only { question } and calls onCreated", async () => { await userEvent.type(await screen.findByLabelText(/question/i), "Quante vendite nel 2025?"); await userEvent.click(screen.getByRole("button", { name: /^create$/i })); - await waitFor(() => expect(body).toEqual({ question: "Quante vendite nel 2025?" })); + await waitFor(() => expect(body).toEqual({ + question: "Quante vendite nel 2025?", + workspaceId: "default", provider: "zai", model: "glm-5.2", thinking: "low", + })); + await waitFor(() => expect(onCreated).toHaveBeenCalledWith("s1")); +}); + +test("first-run direct dialog creation waits for registry policy without a mounted footer", async () => { + let body: unknown; + let releasePolicy!: () => void; + let summaryRequestStarted = false; + let policyRequestStarted = false; + const policyMayFinish = new Promise((resolve) => { releasePolicy = resolve; }); + const revision = { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }; + workspacePreferences.save({ + workspaceId: "psd-clinical", provider: "deepseek", model: "deepseek-v4-pro", thinking: "medium", + }); + server.use( + http.get("/api/workspaces", () => { + summaryRequestStarted = true; + return HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { displayName: "PSD Clinical", revision: revision as any }), + }]); + }), + http.get("/api/workspaces/psd-clinical", async () => { + policyRequestStarted = true; + await policyMayFinish; + return HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical", ["zai/glm-5.2"], "zai/glm-5.2"), revision, + }); + }), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + + const { onCreated } = renderDialog(); + await userEvent.click(screen.getByRole("button", { name: /new/i })); + await userEvent.type(await screen.findByLabelText(/question/i), "q"); + await userEvent.click(screen.getByRole("button", { name: /^create$/i })); + + await waitFor(() => expect(summaryRequestStarted).toBe(true)); + await waitFor(() => expect(policyRequestStarted).toBe(true)); + expect(body).toBeUndefined(); + expect(screen.getByRole("button", { name: /creating/i })).toBeDisabled(); + releasePolicy(); + + await waitFor(() => expect(body).toEqual({ + question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "medium", + })); await waitFor(() => expect(onCreated).toHaveBeenCalledWith("s1")); }); diff --git a/frontend/src/shell/NewSessionDialog.tsx b/frontend/src/shell/NewSessionDialog.tsx index de931bc3..0ea380c0 100644 --- a/frontend/src/shell/NewSessionDialog.tsx +++ b/frontend/src/shell/NewSessionDialog.tsx @@ -1,5 +1,5 @@ // frontend/src/shell/NewSessionDialog.tsx -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { Dialog, DialogContent, @@ -8,6 +8,7 @@ import { } from "../components/ui/dialog"; import { Button } from "../components/ui/button"; import { createSession } from "../api/sessions"; +import { captureAuthOperation, isAuthOperationCurrent } from "../auth/authOperation"; interface Props { onCreated: (id: string) => void; @@ -18,6 +19,8 @@ export function NewSessionDialog({ onCreated }: Props) { const [question, setQuestion] = useState(""); const [error, setError] = useState(null); const [busy, setBusy] = useState(false); + const operationEpochRef = useRef(0); + useEffect(() => () => { operationEpochRef.current += 1; }, []); async function handleSubmit(e: React.FormEvent) { e.preventDefault(); @@ -26,16 +29,34 @@ export function NewSessionDialog({ onCreated }: Props) { setError("The question cannot be empty."); return; } + const operation = captureAuthOperation({ disposalEpoch: operationEpochRef.current }); setBusy(true); try { - const { id } = await createSession({ question: question.trim() }); + const { id } = await createSession({ question: question.trim() }, operation ? { + operation, + isCurrent: () => isAuthOperationCurrent(operation, { + sessionId: null, + disposalEpoch: operationEpochRef.current, + }), + } : undefined); + if (operation && !isAuthOperationCurrent(operation, { + sessionId: null, + disposalEpoch: operationEpochRef.current, + })) return; setOpen(false); setQuestion(""); onCreated(id); } catch (err) { + if (operation && !isAuthOperationCurrent(operation, { + sessionId: null, + disposalEpoch: operationEpochRef.current, + })) return; setError(err instanceof Error ? err.message : "Failed to create the session."); } finally { - setBusy(false); + if (!operation || isAuthOperationCurrent(operation, { + sessionId: null, + disposalEpoch: operationEpochRef.current, + })) setBusy(false); } } diff --git a/frontend/src/shell/PiManagement.test.tsx b/frontend/src/shell/PiManagement.test.tsx new file mode 100644 index 00000000..17700591 --- /dev/null +++ b/frontend/src/shell/PiManagement.test.tsx @@ -0,0 +1,465 @@ +import { render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { http, HttpResponse } from "msw"; +import { server } from "../test/msw"; +import { PiManagement } from "./PiManagement"; + +const readyStatus = { + version: "0.80.3", + ready: true, + credentials: "present", + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-05T10:00:00.000Z", +}; + +const options = { + providers: ["zai", "deepseek"], + models: [ + { provider: "zai", id: "glm-5.2" }, + { provider: "deepseek", id: "deepseek-v4" }, + ], + reasoning: ["low", "medium", "high"], + checkedAt: "2026-08-05T10:00:00.000Z", +}; + +function renderManagement() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( undefined} />); +} + +beforeEach(() => { + server.use( + http.get("/api/pi-management/status", () => HttpResponse.json(readyStatus)), + http.get("/api/pi-management/options", () => HttpResponse.json(options)), + ); +}); + +test("loads Pi version and readiness as an accessible operational rail", async () => { + renderManagement(); + + expect(screen.getByText("Loading Pi management…")).toBeVisible(); + expect(await screen.findByRole("heading", { name: "Pi management" })).toBeVisible(); + await screen.findByLabelText("Provider"); + expect(screen.getByRole("dialog", { name: "Pi management" })).toHaveClass("sm:max-w-[min(94vw,58rem)]"); + expect(screen.getByTestId("pi-readiness-rail")).toHaveTextContent("Runtime ready"); + expect(screen.getByRole("status", { name: "Pi readiness" })).toHaveTextContent("Ready"); + expect(screen.getByText("Pi 0.80.3")).toBeVisible(); + expect(screen.getByText("Defaults ready")).toBeVisible(); + expect(screen.getByRole("button", { name: "Test saved defaults" })).toBeVisible(); +}); + +test("uses closed provider, model, and reasoning choices without a secret field or terminal", async () => { + renderManagement(); + + const provider = await screen.findByLabelText("Provider"); + expect(provider).toHaveValue("zai"); + expect(screen.getByLabelText("Model")).toHaveValue("glm-5.2"); + expect(screen.getByLabelText("Reasoning level")).toHaveValue("medium"); + expect(within(provider).getAllByRole("option").map((option) => option.textContent)).toEqual(["zai", "deepseek"]); + expect(screen.getByLabelText("Model")).toHaveTextContent("GLM 5.2"); + expect(screen.queryByRole("textbox", { name: /provider|model|reasoning|credential/i })).not.toBeInTheDocument(); + expect(document.querySelector('input[type="password"]')).toBeNull(); + expect(screen.queryByRole("button", { name: /terminal|shell access/i })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Update Pi" })).not.toBeInTheDocument(); +}); + +test("saves only a selected non-secret configuration", async () => { + const user = userEvent.setup(); + let saved: unknown; + server.use(http.put("/api/pi-management/config", async ({ request }) => { + saved = await request.json(); + return HttpResponse.json({ ...saved as object, updatedAt: "2026-08-05T10:02:00.000Z" }); + })); + renderManagement(); + + await user.selectOptions(await screen.findByLabelText("Provider"), "deepseek"); + await user.selectOptions(screen.getByLabelText("Reasoning level"), "high"); + await user.click(screen.getByRole("button", { name: "Save defaults" })); + + await waitFor(() => expect(saved).toEqual({ provider: "deepseek", model: "deepseek-v4", reasoning: "high" })); + expect(screen.getByRole("status", { name: "Pi management feedback" })).toHaveTextContent("Defaults saved"); +}); + +// Catches a provider switch updating only the saved config while leaving the credential rail +// attached to the previously selected provider. +test("shows authoritative credential presence after saving a different provider", async () => { + const user = userEvent.setup(); + let saved = false; + server.use( + http.get("/api/pi-management/status", () => HttpResponse.json(saved ? { + ...readyStatus, + credentials: "missing", + config: { provider: "deepseek", model: "deepseek-v4", reasoning: "medium" }, + checkedAt: "2026-08-05T10:02:00.000Z", + } : readyStatus)), + http.put("/api/pi-management/config", async ({ request }) => { + saved = true; + return HttpResponse.json({ + ...await request.json() as object, + updatedAt: "2026-08-05T10:01:00.000Z", + }); + }), + ); + renderManagement(); + + expect(await screen.findByText("Credentials present")).toBeVisible(); + await user.selectOptions(screen.getByLabelText("Provider"), "deepseek"); + await user.click(screen.getByRole("button", { name: "Save defaults" })); + + expect(await screen.findByText("Credentials missing")).toBeVisible(); + expect(screen.queryByText("Credentials present")).not.toBeInTheDocument(); + expect(screen.getByLabelText("Provider")).toHaveValue("deepseek"); + expect(screen.getByRole("status", { name: "Pi management feedback" })).toHaveTextContent("Defaults saved"); +}); + +test("runs the saved-configuration test without changing credential presence", async () => { + const user = userEvent.setup(); + let tests = 0; + server.use(http.post("/api/pi-management/test", () => { + tests += 1; + return HttpResponse.json(tests === 1 + ? { ready: true, checkedAt: "2026-08-05T10:02:00.000Z" } + : { ready: false, message: "Pi provider smoke check failed", checkedAt: "2026-08-05T10:03:00.000Z" }); + })); + renderManagement(); + + const testButton = await screen.findByRole("button", { name: "Test saved defaults" }); + expect(screen.getByText("Defaults ready")).toBeVisible(); + expect(screen.queryByText("Changes are not saved yet.")).not.toBeInTheDocument(); + expect(testButton).toBeEnabled(); + await user.click(testButton); + await waitFor(() => expect(tests).toBe(1)); + expect(await screen.findByText("Saved configuration test passed")).toBeVisible(); + expect(screen.getByText("Credentials present")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Test saved defaults" })); + expect(await screen.findByText("Saved configuration test failed")).toBeVisible(); + expect(screen.getByText("Credentials present")).toBeVisible(); +}); + +test("fetches and displays bounded sanitized diagnostic logs only on request", async () => { + const user = userEvent.setup(); + let logRequests = 0; + server.use(http.get("/api/pi-management/logs", () => { + logRequests += 1; + return HttpResponse.json({ lines: ["Pi smoke check succeeded", "provider token=[REDACTED]"], checkedAt: "2026-08-05T10:04:00.000Z" }); + })); + renderManagement(); + + await screen.findByLabelText("Provider"); + expect(logRequests).toBe(0); + await user.click(screen.getByRole("button", { name: "Show sanitized logs" })); + expect(await screen.findByLabelText("Sanitized Pi diagnostics")).toHaveTextContent("provider token=[REDACTED]"); + expect(logRequests).toBe(1); + expect(screen.queryByText("raw-provider-token")).not.toBeInTheDocument(); +}); + +test("explains forbidden management access without offering mutation controls", async () => { + server.use( + http.get("/api/pi-management/status", () => + HttpResponse.json({ code: "pi_management_forbidden", error: "Pi management is not permitted" }, { status: 403 })), + ); + renderManagement(); + + expect(await screen.findByRole("alert", { name: "Pi management unavailable" })).toHaveTextContent("Pi management is not permitted"); + expect(screen.queryByLabelText("Provider")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Save defaults" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Test saved defaults" })).not.toBeInTheDocument(); +}); + +test("shows a seven-step host-terminal workflow in scrollable platform tabs", async () => { + const user = userEvent.setup(); + renderManagement(); + + const tablist = await screen.findByRole("tablist", { name: "Pi host platform" }); + const [linuxTab, macosTab, windowsTab] = within(tablist).getAllByRole("tab"); + expect([linuxTab, macosTab, windowsTab].map((tab) => tab.textContent)).toEqual(["Linux", "macOS", "Windows"]); + expect(screen.getByRole("dialog", { name: "Pi management" })).toHaveClass("max-h-[calc(100vh-6rem)]"); + expect(screen.getByTestId("pi-platform-instructions-scroll")).toHaveClass("overflow-y-scroll"); + expect(screen.queryByRole("tabpanel")).not.toBeInTheDocument(); + expect([linuxTab, macosTab, windowsTab].every((tab) => tab.getAttribute("aria-selected") === "false")).toBe(true); + expect(linuxTab).toHaveAttribute("tabindex", "0"); + expect(macosTab).toHaveAttribute("tabindex", "-1"); + expect(windowsTab).toHaveAttribute("tabindex", "-1"); + for (const tab of [linuxTab, macosTab, windowsTab]) { + const panelId = tab.getAttribute("aria-controls"); + const panel = panelId ? document.getElementById(panelId) : null; + expect(panel).toBeInTheDocument(); + expect(panel).toHaveAttribute("role", "tabpanel"); + expect(panel).toHaveAttribute("aria-labelledby", tab.id); + expect(panel).toHaveAttribute("hidden"); + expect(panel).toHaveAttribute("tabindex", "-1"); + } + expect(screen.getByText("Using the host terminal:")).toBeVisible(); + expect(screen.queryByText(/not this browser page/i)).not.toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Host update" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Copy update command" })).not.toBeInTheDocument(); + expect(screen.queryByText(/:5173/)).not.toBeInTheDocument(); + expect(await screen.findByText("Select the provider, model, and reasoning used for new Pi work. Credentials stay in protected host files.")).toBeVisible(); + expect(screen.getByText("Shows at most 200 recent lines with declared secret values removed.")).toBeVisible(); + + linuxTab.focus(); + await user.keyboard("{ArrowRight}"); + expect(macosTab).toHaveFocus(); + expect(macosTab).toHaveAttribute("aria-selected", "true"); + expect(macosTab).toHaveAttribute("tabindex", "0"); + expect(linuxTab).toHaveAttribute("tabindex", "-1"); + await user.keyboard("{ArrowRight}"); + expect(windowsTab).toHaveFocus(); + expect(windowsTab).toHaveAttribute("aria-selected", "true"); + expect(windowsTab).toHaveAttribute("tabindex", "0"); + expect(macosTab).toHaveAttribute("tabindex", "-1"); + await user.keyboard("{ArrowRight}"); + expect(linuxTab).toHaveFocus(); + expect(linuxTab).toHaveAttribute("aria-selected", "true"); + await user.keyboard("{ArrowLeft}"); + expect(windowsTab).toHaveFocus(); + await user.keyboard("{Home}"); + expect(linuxTab).toHaveFocus(); + await user.keyboard("{End}"); + expect(windowsTab).toHaveFocus(); + + await user.click(windowsTab); + expect(windowsTab).toHaveAttribute("aria-selected", "false"); + expect(screen.queryByRole("tabpanel")).not.toBeInTheDocument(); + await user.keyboard(" "); + expect(windowsTab).toHaveAttribute("aria-selected", "true"); + await user.keyboard(" "); + expect(windowsTab).toHaveAttribute("aria-selected", "false"); + await user.keyboard("{Enter}"); + expect(windowsTab).toHaveAttribute("aria-selected", "true"); + await user.keyboard("{Enter}"); + expect(windowsTab).toHaveAttribute("aria-selected", "false"); + + await user.click(linuxTab); + const linux = screen.getByRole("tabpanel", { name: "Linux" }); + expect(within(linux).getAllByRole("listitem")).toHaveLength(7); + expect(within(linux).getAllByRole("heading", { level: 4 }).map((heading) => heading.textContent)).toEqual([ + "Open the project root", + "Edit the provider catalog", + "Enable the model", + "Check the provider credential", + "Reload Pi configuration", + "Update the Pi version", + "Recover a failed update", + ]); + expect(linux).toHaveTextContent("The deploy directory is beside compose.yaml"); + expect(linux).toHaveTextContent("All commands below start in this project root"); + expect(linux).toHaveTextContent("mkdir -p bin"); + expect(linux).toHaveTextContent("go -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl"); + expect(linux).toHaveTextContent("deploy/pi/models.json"); + expect(linux).toHaveTextContent("deploy/pi/settings.json"); + expect(linux).toHaveTextContent("baseUrl is the provider API endpoint"); + expect(linux).toHaveTextContent("enabledModels uses provider/model identifiers"); + expect(linux).toHaveTextContent("Pi reads the provider API key from a protected file on the host"); + expect(linux).toHaveTextContent("Do not put the key in models.json or settings.json"); + expect(linux).toHaveTextContent("./bin/thothctl pi restart --yes --drain"); + expect(linux).toHaveTextContent("./bin/thothctl pi update"); + expect(linux).toHaveTextContent("./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain"); + expect(linux).toHaveTextContent("The command installs the Pi version pinned in docker/core.Dockerfile"); + expect(linux).toHaveTextContent("./bin/thothctl pi maintenance status"); + expect(linux).toHaveTextContent("./bin/thothctl pi logs"); + expect(linux).toHaveTextContent("./bin/thothctl pi rollback --yes"); + expect(linux).toHaveTextContent("./bin/thothctl pi maintenance recover --yes"); + expect(linux).not.toHaveTextContent("~/bin/"); + expect(linux).not.toHaveTextContent("~/.pi/agent/"); + + await user.click(macosTab); + const macos = screen.getByRole("tabpanel", { name: "macOS" }); + expect(within(macos).getAllByRole("listitem")).toHaveLength(7); + expect(within(macos).getAllByRole("heading", { level: 4 }).map((heading) => heading.textContent)).toEqual([ + "Open the project root", + "Edit the provider catalog", + "Enable the model", + "Check the provider credential", + "Reload Pi configuration", + "Update the Pi version", + "Recover a failed update", + ]); + expect(macos).toHaveTextContent("The deploy directory is beside compose.yaml"); + expect(macos).toHaveTextContent("All commands below start in this project root"); + expect(macos).toHaveTextContent("go -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl"); + expect(macos).toHaveTextContent("deploy/pi/models.json"); + expect(macos).toHaveTextContent("deploy/pi/settings.json"); + expect(macos).toHaveTextContent("./bin/thothctl pi restart --yes --drain"); + expect(macos).toHaveTextContent("./bin/thothctl pi update"); + expect(macos).toHaveTextContent("./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain"); + expect(macos).toHaveTextContent("The command installs the Pi version pinned in docker/core.Dockerfile"); + expect(macos).toHaveTextContent("./bin/thothctl pi maintenance status"); + expect(macos).toHaveTextContent("./bin/thothctl pi logs"); + expect(macos).toHaveTextContent("./bin/thothctl pi rollback --yes"); + expect(macos).toHaveTextContent("./bin/thothctl pi maintenance recover --yes"); + + await user.click(windowsTab); + const windows = screen.getByRole("tabpanel", { name: "Windows" }); + expect(within(windows).getAllByRole("listitem")).toHaveLength(7); + expect(within(windows).getAllByRole("heading", { level: 4 }).map((heading) => heading.textContent)).toEqual([ + "Open the project root", + "Edit the provider catalog", + "Enable the model", + "Check the provider credential", + "Reload Pi configuration", + "Update the Pi version", + "Recover a failed update", + ]); + expect(windows).toHaveTextContent("The deploy directory is beside compose.yaml"); + expect(windows).toHaveTextContent("All commands below start in this project root"); + expect(windows).toHaveTextContent("deploy\\pi\\models.json"); + expect(windows).toHaveTextContent("deploy\\pi\\settings.json"); + expect(windows).toHaveTextContent("New-Item -ItemType Directory -Force bin"); + expect(windows).toHaveTextContent("go -C tools/thothctl build -o ../../bin/thothctl.exe ./cmd/thothctl"); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi restart --yes --drain'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi update'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi update --version --source pull --image @sha256: --yes --drain'); + expect(windows).toHaveTextContent("The command installs the Pi version pinned in docker/core.Dockerfile"); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi maintenance status'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi logs'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi rollback --yes'); + expect(windows).toHaveTextContent('.\\bin\\thothctl.exe pi maintenance recover --yes'); + expect(windows).not.toHaveTextContent("~\\bin\\"); +}); + +test("keeps the required PI_AUTH_FILE guidance in one static text node", async () => { + const user = userEvent.setup(); + renderManagement(); + + const tablist = await screen.findByRole("tablist", { name: "Pi host platform" }); + await user.click(within(tablist).getByRole("tab", { name: "Linux" })); + const credentialStep = screen.getByRole("heading", { name: "Check the provider credential" }).closest("li"); + const guidance = credentialStep?.querySelector("p"); + + expect( + Array.from(guidance?.childNodes ?? []).some( + (node) => node.nodeType === Node.TEXT_NODE + && node.textContent?.includes("Pi reads the provider API key from a protected file on the host"), + ), + ).toBe(true); +}); + +test("reloads installation defaults when the panel is reopened", async () => { + let statusCalls = 0; + server.use(http.get("/api/pi-management/status", () => { + statusCalls += 1; + return HttpResponse.json(statusCalls === 1 + ? readyStatus + : { ...readyStatus, config: { provider: "deepseek", model: "deepseek-v4", reasoning: "high" } }); + })); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const view = render( undefined} />); + + expect(await screen.findByLabelText("Provider")).toHaveValue("zai"); + view.rerender( undefined} />); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Pi management" })).not.toBeInTheDocument()); + view.rerender( undefined} />); + + await waitFor(() => expect(screen.getByLabelText("Provider")).toHaveValue("deepseek")); +}); + +test("shows an explicit recoverable incomplete state when no provider model is available", async () => { + server.use( + http.get("/api/pi-management/status", () => HttpResponse.json({ ...readyStatus, credentials: "present" })), + http.get("/api/pi-management/options", () => HttpResponse.json({ + ...options, + providers: ["zai"], + models: [], + })), + ); + renderManagement(); + + const incomplete = await screen.findByRole("alert", { name: "Pi configuration incomplete" }); + expect(incomplete).toHaveTextContent("No enabled provider and model choices are available"); + expect(incomplete).toHaveTextContent("Check the host-managed Pi model configuration"); + expect(screen.queryByText("Loading Pi management…")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Save defaults" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Test saved defaults" })).toBeDisabled(); +}); + +test("keeps suggested draft choices distinct from invalid persisted defaults until save succeeds", async () => { + let configured = false; + const persisted = { + ...readyStatus, + credentials: "missing", + config: { provider: "retired", model: "old-model", reasoning: "medium" }, + }; + server.use( + http.get("/api/pi-management/status", () => HttpResponse.json(configured ? { + ...readyStatus, + credentials: "missing", + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-05T10:05:00.000Z", + } : persisted)), + http.put("/api/pi-management/config", () => { + configured = true; + return HttpResponse.json({ + provider: "zai", + model: "glm-5.2", + reasoning: "medium", + updatedAt: "2026-08-05T10:05:00.000Z", + }); + }), + ); + const user = userEvent.setup(); + renderManagement(); + + expect(await screen.findByLabelText("Provider")).toHaveValue("zai"); + expect(screen.getByText("Defaults incomplete")).toBeVisible(); + expect(screen.getByText("Suggested choices are not saved yet.")).toBeVisible(); + expect(screen.getByRole("button", { name: "Test saved defaults" })).toBeDisabled(); + + await user.click(screen.getByRole("button", { name: "Save defaults" })); + expect(await screen.findByText("Defaults ready")).toBeVisible(); + expect(screen.getByRole("button", { name: "Test saved defaults" })).toBeEnabled(); +}); + +test.each(["present", "missing"] as const)( + "shows credentials %s from status without inferring them from smoke", + async (credentials) => { + server.use(http.get("/api/pi-management/status", () => HttpResponse.json({ + ...readyStatus, + credentials, + }))); + renderManagement(); + + expect(await screen.findByText(`Credentials ${credentials}`)).toBeVisible(); + }, +); + +test("labels smoke only as a saved-configuration test and resets it when the draft changes", async () => { + server.use( + http.get("/api/pi-management/status", () => HttpResponse.json({ ...readyStatus, credentials: "present" })), + http.post("/api/pi-management/test", () => HttpResponse.json({ + ready: true, + checkedAt: "2026-08-05T10:06:00.000Z", + })), + ); + const user = userEvent.setup(); + renderManagement(); + + await user.click(await screen.findByRole("button", { name: "Test saved defaults" })); + expect(await screen.findByText("Saved configuration test passed")).toBeVisible(); + expect(screen.getByText("Credentials present")).toBeVisible(); + + await user.selectOptions(screen.getByLabelText("Provider"), "deepseek"); + expect(screen.getByText("Saved configuration test not run")).toBeVisible(); + expect(screen.getByText("Save these changes before testing. The test always uses saved defaults.")).toBeVisible(); + expect(screen.getByRole("button", { name: "Test saved defaults" })).toBeDisabled(); + expect(screen.getByText("Credentials present")).toBeVisible(); +}); + +test("a failed saved-configuration test does not change backend credential presence", async () => { + server.use( + http.get("/api/pi-management/status", () => HttpResponse.json({ ...readyStatus, credentials: "present" })), + http.post("/api/pi-management/test", () => HttpResponse.json({ + ready: false, + message: "Pi runtime is unavailable", + checkedAt: "2026-08-05T10:07:00.000Z", + })), + ); + const user = userEvent.setup(); + renderManagement(); + + await user.click(await screen.findByRole("button", { name: "Test saved defaults" })); + expect(await screen.findByText("Saved configuration test failed")).toBeVisible(); + expect(screen.getByText("Credentials present")).toBeVisible(); + expect(screen.queryByText("Credentials missing")).not.toBeInTheDocument(); +}); diff --git a/frontend/src/shell/PiManagement.tsx b/frontend/src/shell/PiManagement.tsx new file mode 100644 index 00000000..19baf848 --- /dev/null +++ b/frontend/src/shell/PiManagement.tsx @@ -0,0 +1,500 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { CheckCircle2, CircleAlert, ClipboardCheck, FlaskConical, LoaderCircle, ScrollText, X } from "lucide-react"; +import { + asPiManagementApiError, + getPiManagementLogs, + getPiManagementOptions, + getPiManagementStatus, + runPiManagementTest, + savePiManagementConfig, + type PiInstallationConfig, + type PiManagementOptions, +} from "../api/pi-management"; +import { Button } from "../components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "../components/ui/dialog"; + +const fieldClass = "h-9 w-full rounded-md border border-input bg-background px-2.5 text-sm shadow-xs outline-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:cursor-not-allowed disabled:opacity-60"; + +type Feedback = { tone: "success" | "error"; message: string } | undefined; +type SmokeState = "passed" | "failed" | undefined; + +function suggestedConfig(status: { config: Partial }, options: PiManagementOptions): PiInstallationConfig | undefined { + const provider = status.config.provider && options.providers.includes(status.config.provider) + ? status.config.provider + : options.providers[0]; + const model = status.config.model && options.models.some((item) => item.provider === provider && item.id === status.config.model) + ? status.config.model + : options.models.find((item) => item.provider === provider)?.id; + const reasoning = status.config.reasoning && options.reasoning.includes(status.config.reasoning) + ? status.config.reasoning + : options.reasoning[0]; + return provider && model && reasoning ? { provider, model, reasoning } : undefined; +} + +function isSupportedConfig( + config: Partial | undefined, + options: PiManagementOptions | undefined, +): config is PiInstallationConfig { + return Boolean( + config?.provider + && config.model + && config.reasoning + && options?.providers.includes(config.provider) + && options.models.some((model) => model.provider === config.provider && model.id === config.model) + && options.reasoning.includes(config.reasoning), + ); +} + +function sameConfig(a: Partial | undefined, b: PiInstallationConfig | undefined): boolean { + return Boolean( + a?.provider === b?.provider + && a?.model === b?.model + && a?.reasoning === b?.reasoning, + ); +} + +function errorMessage(error: unknown, fallback: string): string { + return asPiManagementApiError(error)?.message ?? fallback; +} + +function ReadinessRail({ ready, configured, credentials, smokeState }: { + ready: boolean; + configured: boolean; + credentials: "present" | "missing"; + smokeState: SmokeState; +}) { + return ( +
+ + + + +
+ ); +} + +function RailItem({ label, value, state }: { label: string; value: string; state: "ready" | "attention" | "idle" }) { + const iconClass = state === "ready" ? "text-[oklch(var(--success))]" : state === "attention" ? "text-amber-700 dark:text-amber-400" : "text-muted-foreground"; + return
+

{label}

+

+ {state === "ready" ? : state === "attention" ? : } + {value} +

+
; +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ; +} + +type PiPlatform = "linux" | "macos" | "windows"; + +type PiPlatformDetails = { + modelsPath: string; + settingsPath: string; + terminal: string; + credentialProtection: string; + buildCommand: string; + restartCommand: string; + updateCommand: string; + pullCommand: string; + recoveryCommands: string; +}; + +const piPlatforms: Array<{ id: PiPlatform; label: string; details: PiPlatformDetails }> = [ + { + id: "linux", + label: "Linux", + details: { + modelsPath: "deploy/pi/models.json", + settingsPath: "deploy/pi/settings.json", + terminal: "a terminal", + credentialProtection: "a protected host file with mode 0600", + buildCommand: "mkdir -p bin\ngo -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl", + restartCommand: "./bin/thothctl pi restart --yes --drain", + updateCommand: "./bin/thothctl pi update", + pullCommand: "./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain", + recoveryCommands: "./bin/thothctl pi maintenance status\n./bin/thothctl pi logs\n./bin/thothctl pi rollback --yes\n./bin/thothctl pi maintenance recover --yes", + }, + }, + { + id: "macos", + label: "macOS", + details: { + modelsPath: "deploy/pi/models.json", + settingsPath: "deploy/pi/settings.json", + terminal: "Terminal", + credentialProtection: "a protected host file with mode 0600", + buildCommand: "mkdir -p bin\ngo -C tools/thothctl build -o ../../bin/thothctl ./cmd/thothctl", + restartCommand: "./bin/thothctl pi restart --yes --drain", + updateCommand: "./bin/thothctl pi update", + pullCommand: "./bin/thothctl pi update --version --source pull --image @sha256: --yes --drain", + recoveryCommands: "./bin/thothctl pi maintenance status\n./bin/thothctl pi logs\n./bin/thothctl pi rollback --yes\n./bin/thothctl pi maintenance recover --yes", + }, + }, + { + id: "windows", + label: "Windows", + details: { + modelsPath: "deploy\\pi\\models.json", + settingsPath: "deploy\\pi\\settings.json", + terminal: "PowerShell", + credentialProtection: "a protected host file with a user-only ACL", + buildCommand: "New-Item -ItemType Directory -Force bin | Out-Null\ngo -C tools/thothctl build -o ../../bin/thothctl.exe ./cmd/thothctl", + restartCommand: ".\\bin\\thothctl.exe pi restart --yes --drain", + updateCommand: ".\\bin\\thothctl.exe pi update", + pullCommand: ".\\bin\\thothctl.exe pi update --version --source pull --image @sha256: --yes --drain", + recoveryCommands: ".\\bin\\thothctl.exe pi maintenance status\n.\\bin\\thothctl.exe pi logs\n.\\bin\\thothctl.exe pi rollback --yes\n.\\bin\\thothctl.exe pi maintenance recover --yes", + }, + }, +]; + +function PiCodeBlock({ children, className = "" }: { children: string; className?: string }) { + return 
{children}
; +} + +function PiInstructionSteps({ details }: { details: PiPlatformDetails }) { + return
    +
  1. +

    Open the project root

    +

    Using {details.terminal}, open the current ThothII checkout or worktree root. All commands below start in this project root. The deploy directory is beside compose.yaml. If this checkout has no bin directory, create it and build the local CLI once:

    + {details.buildCommand} +

    The commands below use the executable built in this checkout, so they cannot accidentally target another worktree.

    +
    2. +
  2. +

    Edit the provider catalog

    +

    Edit {details.modelsPath}. It is the provider catalog.

    +
    +
    baseUrl
    is the provider API endpoint.
    +
    api
    selects the provider API format.
    +
    models
    lists that provider's available models.
    +
    id
    is the model identifier.
    +
    name
    is the model name shown to operators.
    +
    +
    3. +
  3. +

    Enable the model

    +

    Edit {details.settingsPath}. It is the enabled-model policy, not the provider catalog.

    +
    +
    enabledModels
    uses provider/model identifiers to choose the models available for new Pi work.
    +
    +
    4. +
  4. +

    Check the provider credential

    +

    Pi reads the provider API key from a protected file on the host. PI_AUTH_FILE tells this installation which file to use; Docker mounts it read-only into the core container. Do not put the key in models.json or settings.json. Keep {details.credentialProtection}.

    +
    5. +
  5. +

    Reload Pi configuration

    +

    After changing the catalog, policy, or selected credential file, reload Pi configuration.

    + {details.restartCommand} +
    6. +
  6. +

    Update the Pi version

    +

    The command installs the Pi version pinned in docker/core.Dockerfile. Use --version <VERSION> only when you deliberately want another version.

    + {details.updateCommand} +

    Advanced: pull an immutable, digest-pinned image.

    + {details.pullCommand} +
    7. +
  7. +

    Recover a failed update

    +

    Check maintenance status and bounded sanitized Pi logs first. For a failed update, use rollback. For restart or update maintenance recovery after repairing the reported problem, use maintenance recovery.

    + {details.recoveryCommands} +
    8. +
; +} + +function PiPlatformInstructions() { + const [activePlatform, setActivePlatform] = useState(); + const [focusedPlatform, setFocusedPlatform] = useState("linux"); + const tabRefs = useRef>({ linux: null, macos: null, windows: null }); + + function activateAndFocus(platform: PiPlatform) { + setFocusedPlatform(platform); + setActivePlatform(platform); + tabRefs.current[platform]?.focus(); + } + + function handleTabKeyDown(event: React.KeyboardEvent, platform: PiPlatform) { + const index = piPlatforms.findIndex((item) => item.id === platform); + const nextPlatform = event.key === "ArrowRight" + ? piPlatforms[(index + 1) % piPlatforms.length] + : event.key === "ArrowLeft" + ? piPlatforms[(index - 1 + piPlatforms.length) % piPlatforms.length] + : event.key === "Home" + ? piPlatforms[0] + : event.key === "End" + ? piPlatforms[piPlatforms.length - 1] + : undefined; + if (!nextPlatform) return; + event.preventDefault(); + activateAndFocus(nextPlatform.id); + } + + return ( +
+

Using the host terminal:

+
+ {piPlatforms.map((platform) => ( + + ))} +
+
+ {piPlatforms.map((platform) => ( + + ))} +
+
+ ); +} + +export function PiManagement({ open, onClose }: { open: boolean; onClose: () => void }) { + const queryClient = useQueryClient(); + const [draft, setDraft] = useState(); + const [feedback, setFeedback] = useState(); + const [smokeState, setSmokeState] = useState(); + const [logsRequested, setLogsRequested] = useState(false); + const statusQuery = useQuery({ queryKey: ["pi-management", "status"], queryFn: getPiManagementStatus, enabled: open }); + const optionsQuery = useQuery({ queryKey: ["pi-management", "options"], queryFn: getPiManagementOptions, enabled: open }); + const logsQuery = useQuery({ queryKey: ["pi-management", "logs"], queryFn: getPiManagementLogs, enabled: open && logsRequested }); + const models = useMemo( + () => optionsQuery.data?.models.filter((model) => model.provider === draft?.provider) ?? [], + [draft?.provider, optionsQuery.data?.models], + ); + const initialDraft = useMemo( + () => statusQuery.data && optionsQuery.data + ? suggestedConfig(statusQuery.data, optionsQuery.data) + : undefined, + [optionsQuery.data, statusQuery.data], + ); + const persistedReady = isSupportedConfig(statusQuery.data?.config, optionsQuery.data); + const validDraft = isSupportedConfig(draft, optionsQuery.data); + const dirty = Boolean(draft && !sameConfig(statusQuery.data?.config, draft)); + + useEffect(() => { + if (!open) { + setDraft(undefined); + setFeedback(undefined); + setSmokeState(undefined); + setLogsRequested(false); + queryClient.removeQueries({ queryKey: ["pi-management"] }); + } + }, [open, queryClient]); + + useEffect(() => { + if (draft || !initialDraft) return; + setDraft(initialDraft); + }, [draft, initialDraft]); + + const saveMutation = useMutation({ + mutationFn: savePiManagementConfig, + onSuccess: async (saved) => { + const config = { provider: saved.provider, model: saved.model, reasoning: saved.reasoning }; + setDraft(config); + setSmokeState(undefined); + await queryClient.invalidateQueries({ queryKey: ["pi-management", "status"] }); + setFeedback({ tone: "success", message: "Defaults saved." }); + }, + onError: (error) => setFeedback({ tone: "error", message: errorMessage(error, "Could not save Pi defaults.") }), + }); + const smokeMutation = useMutation({ + mutationFn: runPiManagementTest, + onSuccess: (result) => { + setSmokeState(result.ready ? "passed" : "failed"); + setFeedback({ + tone: result.ready ? "success" : "error", + message: result.ready + ? "Saved configuration test passed." + : `Saved configuration test failed.${result.message ? ` ${result.message}` : ""}`, + }); + }, + onError: (error) => { + setSmokeState("failed"); + setFeedback({ tone: "error", message: errorMessage(error, "Could not test the saved Pi defaults.") }); + }, + }); + + function saveDefaults() { + if (!draft || !validDraft) { + setFeedback({ tone: "error", message: "Choose a supported provider, model, and reasoning level." }); + return; + } + saveMutation.mutate(draft); + } + + function updateDraft(update: (current: PiInstallationConfig) => PiInstallationConfig) { + setDraft((current) => current ? update(current) : current); + setSmokeState(undefined); + setFeedback(undefined); + } + + function testSavedDefaults() { + if (!persistedReady || dirty) { + setFeedback({ tone: "error", message: "Save supported defaults before running the test." }); + return; + } + smokeMutation.mutate(); + } + + const forbidden = asPiManagementApiError(statusQuery.error)?.code === "pi_management_forbidden"; + const loading = statusQuery.isLoading || optionsQuery.isLoading || Boolean(!draft && initialDraft); + const unavailable = statusQuery.isError || optionsQuery.isError; + + return ( + { if (!nextOpen) onClose(); }}> + + +

Installation controls

+ Pi management + Review the bundled runtime, set safe defaults, and test the saved provider configuration. + + + +
+ {forbidden ? ( +
+

Pi management is not permitted

+

Ask an installation administrator to manage Pi defaults and diagnostics.

+
+ ) : unavailable ? ( +
+

{errorMessage(statusQuery.error ?? optionsQuery.error, "Pi management is unavailable")}

+ +
+ ) : loading ?

Loading Pi management…

: statusQuery.data && optionsQuery.data && ( +
+
+
+

Bundled runtime

+

Pi {statusQuery.data.version ?? "version unavailable"}

+
+

+ {statusQuery.data.ready ? "Ready" : "Needs attention"} +

+
+ + + + {statusQuery.data.message &&

{statusQuery.data.message}

} + {feedback &&

+ {feedback.tone === "error" ? : } + {feedback.message} +

} + +
+
+

Installation defaults

Select the provider, model, and reasoning used for new Pi work. Credentials stay in protected host files.

+
+ {draft ? ( + <> +
+ + + + + + + + + +
+ {dirty &&

+ {persistedReady ? "Changes are not saved yet." : "Suggested choices are not saved yet."} +

} +
+ + +
+ {dirty &&

Save these changes before testing. The test always uses saved defaults.

} + {!dirty && !persistedReady &&

Save supported defaults before testing. The test always uses saved defaults.

} + + ) : ( +
+
+

No enabled provider and model choices are available.

+

Check the host-managed Pi model configuration, then retry this panel.

+
+
+ + + +
+
+ )} +
+ +
+
+

Sanitized diagnostics

Shows at most 200 recent lines with declared secret values removed.

+ +
+ {logsQuery.isError &&

Could not load sanitized Pi diagnostics.

} + {logsQuery.data && 
{logsQuery.data.lines.join("\n") || "No diagnostic lines are available."}
} +
+
+ )} +
+ + + ); +} diff --git a/frontend/src/shell/SessionDocumentsPanel.test.tsx b/frontend/src/shell/SessionDocumentsPanel.test.tsx index bd853fbe..9f477bcc 100644 --- a/frontend/src/shell/SessionDocumentsPanel.test.tsx +++ b/frontend/src/shell/SessionDocumentsPanel.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, within } from "@testing-library/react"; +import { render, screen, waitFor, within } from "@testing-library/react"; import { http, HttpResponse } from "msw"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { server } from "../test/msw"; @@ -23,7 +23,7 @@ const base: SessionSummary = { }; beforeEach(() => { - server.use(http.get("http://localhost:8787/sessions/s1/documents", () => + server.use(http.get("/api/sessions/s1/documents", () => HttpResponse.json([ { phase: "—", key: "question", title: "Original question", format: "text", content: "How many ablations?" }, { phase: "F7", key: "sql", title: "Final SQL", format: "sql", content: "SELECT 1" }, @@ -60,6 +60,7 @@ test("lets text documents adapt to the full resized panel width", async () => { test("shows Resume for resumable, hides it for finalized", async () => { const { rerender } = wrap(); expect(await screen.findByRole("button", { name: /resume/i })).toBeInTheDocument(); + await waitFor(() => expect(document.querySelector(".shiki")).not.toBeNull()); rerender(); expect(screen.queryByRole("button", { name: /resume/i })).not.toBeInTheDocument(); }); @@ -77,7 +78,7 @@ test("a malformed document renders a fallback without taking down its siblings", // unmounts the tree); the good sibling document must still render. // resetHandlers(...) replaces the beforeEach handler so this response is used. server.resetHandlers( - http.get("http://localhost:8787/sessions/s1/documents", () => + http.get("/api/sessions/s1/documents", () => HttpResponse.json([ { phase: "F4", key: "schema", title: "Schema linking", format: "schema-linking", content: '{"joins":[]}' }, { phase: "—", key: "question", title: "Original question", format: "text", content: "SIBLING-SURVIVES" }, @@ -95,7 +96,7 @@ test("a malformed document renders a fallback without taking down its siblings", test("renders the canonical summary order and formats human text as Markdown", async () => { server.resetHandlers( - http.get("http://localhost:8787/sessions/s1/documents", () => + http.get("/api/sessions/s1/documents", () => HttpResponse.json([ { phase: "—", key: "question", title: "Original question", format: "text", content: "Original **question**" }, { phase: "F7", key: "sql", title: "Final SQL", format: "sql", content: "SELECT 1" }, @@ -129,7 +130,7 @@ test("renders the canonical summary order and formats human text as Markdown", a test("renders one approved-then-declined memory list with Markdown details", async () => { server.resetHandlers( - http.get("http://localhost:8787/sessions/s1/documents", () => + http.get("/api/sessions/s1/documents", () => HttpResponse.json([ { phase: "F8", @@ -182,7 +183,7 @@ test("never renders technical approval, promotion, or memory decisions", async ( { type: "column_excluded", subject: "fact_a.note", detail: "Non **pertinente**" }, ].map((decision) => JSON.stringify(decision)).join("\n"); server.resetHandlers( - http.get("http://localhost:8787/sessions/s1/documents", () => + http.get("/api/sessions/s1/documents", () => HttpResponse.json([ { phase: "—", key: "decisions", title: "Decisions", format: "decisions", content: lines }, ]), diff --git a/frontend/src/shell/SteerInput.test.tsx b/frontend/src/shell/SteerInput.test.tsx index efa2d656..ea000f71 100644 --- a/frontend/src/shell/SteerInput.test.tsx +++ b/frontend/src/shell/SteerInput.test.tsx @@ -1,20 +1,61 @@ // frontend/src/shell/SteerInput.test.tsx -import { render, screen, waitFor } from "@testing-library/react"; +import { act, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { http, HttpResponse } from "msw"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { server } from "../test/msw"; +import { canonicalWorkspaceFixture, workspaceRevisionFixture, workspaceSummaryFixture } from "../test/workspace-fixtures"; import { useSessionStore } from "../store/sessionStore"; +import { workspacePreferences } from "../workspaces/preferences"; import { ComposerFooter, ContextGauge, SteerInput } from "./SteerInput"; +import { setAuthState } from "../auth/authState"; + +const userA = { + issuer: "local", subject: "user-a", roles: ["user"] as const, + permissions: ["session.use"], isAdmin: false, csrfToken: "a".repeat(43), session: null, +}; beforeEach(() => { + localStorage.clear(); + setAuthState(userA); server.use( - http.post("http://localhost:8787/sessions/:id/steer", () => + http.post("/api/sessions/:id/steer", () => new HttpResponse(null, { status: 204 }), ), ); }); +test("new sessions send the ephemeral workspace, model, provider, and thinking", async () => { + let body: unknown; + workspacePreferences.save({ + workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "high", + }); + server.use( + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision: { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot" }, + }), + }])), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical"), + revision: { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot" }, + })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + render(); + + await userEvent.type(screen.getByRole("textbox", { name: /new question/i }), "q"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + + await waitFor(() => expect(body).toEqual({ + question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "high", + })); +}); + test("renders a text input and submit button", () => { render(); expect(screen.getByRole("textbox")).toBeInTheDocument(); @@ -24,7 +65,7 @@ test("renders a text input and submit button", () => { test("submitting typed text POSTs to /steer and clears the input", async () => { let captured: unknown = null; server.use( - http.post("http://localhost:8787/sessions/:id/steer", async ({ request, params }) => { + http.post("/api/sessions/:id/steer", async ({ request, params }) => { captured = { id: params.id, body: await request.json() }; return new HttpResponse(null, { status: 204 }); }), @@ -39,10 +80,167 @@ test("submitting typed text POSTs to /steer and clears the input", async () => { expect(input).toHaveValue(""); }); +test("a delayed steer from user A cannot mutate user B's store or composer", async () => { + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/sessions/:id/steer", async () => { + started(); + try { + await held; + return new HttpResponse(null, { status: 204 }); + } finally { + settled(); + } + })); + setAuthState(userA); + const view = render(); + const input = screen.getByRole("textbox"); + await userEvent.type(input, "A-only steer"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + await requestStarted; + + act(() => setAuthState({ ...userA, subject: "user-b", csrfToken: "b".repeat(43) })); + act(() => useSessionStore.getState().setLastUserEntry({ kind: "input", text: "B-entry" })); + release(); + await act(async () => { await requestSettled; }); + + expect(useSessionStore.getState().lastUserEntry).toEqual({ kind: "input", text: "B-entry" }); + expect(input).toHaveValue("A-only steer"); + view.unmount(); +}); + +test("a held steer for s1 cannot complete into the active s2 operation scope", async () => { + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/sessions/s1/steer", async () => { + started(); + try { + await held; + return new HttpResponse(null, { status: 204 }); + } finally { + settled(); + } + })); + + const view = render(); + const input = screen.getByRole("textbox"); + await userEvent.type(input, "Keep s1 isolated"); + await userEvent.click(screen.getByRole("button", { name: /invia|send|steer/i })); + await requestStarted; + + view.rerender(); + act(() => useSessionStore.getState().setLastUserEntry({ kind: "input", text: "s2 owned" })); + release(); + await act(async () => { await requestSettled; }); + + expect(useSessionStore.getState().lastUserEntry).toEqual({ kind: "input", text: "s2 owned" }); + expect(input).toHaveValue("Keep s1 isolated"); +}); + +test("a settings preflight from user A prevents session POST after user B logs in", async () => { + let releaseSettings!: () => void; + let settingsStarted!: () => void; + let settingsSettled!: () => void; + let sessionPosts = 0; + const settingsGate = new Promise((resolve) => { releaseSettings = resolve; }); + const started = new Promise((resolve) => { settingsStarted = resolve; }); + const settled = new Promise((resolve) => { settingsSettled = resolve; }); + server.use( + http.get("/api/settings", async () => { + settingsStarted(); + try { + await settingsGate; + return HttpResponse.json({ workspace: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low" }); + } finally { + settingsSettled(); + } + }), + http.post("/api/sessions", () => { + sessionPosts += 1; + return HttpResponse.json({ id: "a-session" }); + }), + ); + const userB = { ...userA, subject: "user-b", csrfToken: "b".repeat(43) }; + const view = render(); + const input = screen.getByRole("textbox", { name: /new question/i }); + await userEvent.type(input, "A-only question"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + await started; + + act(() => setAuthState(userB)); + act(() => useSessionStore.getState().setLastUserEntry({ kind: "input", text: "B-entry" })); + releaseSettings(); + await act(async () => { await settled; }); + + expect(sessionPosts).toBe(0); + expect(useSessionStore.getState().lastUserEntry).toEqual({ kind: "input", text: "B-entry" }); + view.unmount(); +}); + +test("a workspace-policy preflight from user A prevents session POST after user B logs in", async () => { + workspacePreferences.save({ + workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "low", + }); + let releaseWorkspaces!: () => void; + let workspacesStarted!: () => void; + let workspacesSettled!: () => void; + let sessionPosts = 0; + const workspacesGate = new Promise((resolve) => { releaseWorkspaces = resolve; }); + const started = new Promise((resolve) => { workspacesStarted = resolve; }); + const settled = new Promise((resolve) => { workspacesSettled = resolve; }); + server.use( + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision: workspaceRevisionFixture("psd-clinical"), + }), + }])), + http.get("/api/workspaces/psd-clinical", async () => { + workspacesStarted(); + try { + await workspacesGate; + return HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical"), + revision: workspaceRevisionFixture("psd-clinical"), + }); + } finally { + workspacesSettled(); + } + }), + http.post("/api/sessions", () => { + sessionPosts += 1; + return HttpResponse.json({ id: "a-session" }); + }), + ); + const userB = { ...userA, subject: "user-b", csrfToken: "b".repeat(43) }; + const view = render(); + const input = screen.getByRole("textbox", { name: /new question/i }); + await userEvent.type(input, "A-policy-question"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + await started; + + act(() => setAuthState(userB)); + act(() => useSessionStore.getState().setLastUserEntry({ kind: "input", text: "B-entry" })); + releaseWorkspaces(); + await act(async () => { await settled; }); + + expect(sessionPosts).toBe(0); + expect(useSessionStore.getState().lastUserEntry).toEqual({ kind: "input", text: "B-entry" }); + view.unmount(); +}); + test("pressing Enter in the input submits the steer", async () => { let captured: unknown = null; server.use( - http.post("http://localhost:8787/sessions/:id/steer", async ({ request }) => { + http.post("/api/sessions/:id/steer", async ({ request }) => { captured = await request.json(); return new HttpResponse(null, { status: 204 }); }), @@ -58,7 +256,7 @@ test("pressing Enter in the input submits the steer", async () => { test("does not POST when input is empty", async () => { let called = false; server.use( - http.post("http://localhost:8787/sessions/:id/steer", () => { + http.post("/api/sessions/:id/steer", () => { called = true; return new HttpResponse(null, { status: 204 }); }), @@ -101,11 +299,20 @@ test("the context gauge uses green, yellow, and red at the requested thresholds" test("footer shows cumulative k-token counters after workspace and context gauge after thinking", async () => { server.use( - http.get("http://localhost:8787/settings", () => HttpResponse.json({ + http.get("/api/settings", () => HttpResponse.json({ workspace: "psd", provider: "zai", model: "glm-5.2", thinking: "medium", })), - http.get("http://localhost:8787/workspaces", () => HttpResponse.json([{ name: "psd" }])), - http.get("http://localhost:8787/models", () => HttpResponse.json({ + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd", { + displayName: "PSD", + revision: { id: "psd", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot" }, + }), + }])), + http.get("/api/workspaces/psd", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd"), + revision: { id: "psd", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot" }, + })), + http.get("/api/models", () => HttpResponse.json({ models: [{ provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }], })), ); @@ -131,6 +338,323 @@ test("footer shows cumulative k-token counters after workspace and context gauge expect(thinking.compareDocumentPosition(gauge) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); }); +test("footer limits model choices to the selected workspace policy", async () => { + server.use( + http.get("/api/settings", () => HttpResponse.json({ + workspace: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "medium", + })), + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + revision: { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot" }, + }), + }])), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical"), + revision: { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot" }, + })), + http.get("/api/models", () => HttpResponse.json({ models: [ + { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, + { provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro", reasoning: true }, + ] })), + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render(); + + const selector = await screen.findByRole("combobox", { name: "Model" }); + await waitFor(() => expect(selector).toHaveTextContent("GLM-5.2")); + await waitFor(() => expect(selector).not.toHaveTextContent("DeepSeek V4 Pro")); +}); + +test("switching workspaces replaces an out-of-policy model before session creation", async () => { + let body: unknown; + workspacePreferences.save({ + workspaceId: "research", provider: "deepseek", model: "deepseek-v4-pro", thinking: "medium", + }); + const revision = (id: string) => ({ + id, commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }); + server.use( + http.get("/api/settings", () => HttpResponse.json({ + workspace: "research", provider: "deepseek", model: "deepseek-v4-pro", thinking: "medium", + })), + http.get("/api/workspaces", () => HttpResponse.json([ + workspaceSummaryFixture("research", { displayName: "Research", revision: revision("research") }), + workspaceSummaryFixture("psd-clinical", { displayName: "PSD Clinical", revision: revision("psd-clinical") }), + ])), + http.get("/api/workspaces/research", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("research", ["deepseek/deepseek-v4-pro"]), + revision: revision("research"), + })), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical", ["zai/glm-5.2"], "zai/glm-5.2"), + revision: revision("psd-clinical"), + })), + http.get("/api/models", () => HttpResponse.json({ models: [ + { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, + { provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro", reasoning: true }, + ] })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render(); + + const workspaceSelector = await screen.findByRole("combobox", { name: "Workspace" }); + await waitFor(() => expect(workspaceSelector).toHaveTextContent("psd-clinical")); + await userEvent.selectOptions(workspaceSelector, "psd-clinical"); + await waitFor(() => expect(screen.getByRole("combobox", { name: "Model" })).toHaveValue("glm-5.2")); + expect(screen.getByRole("combobox", { name: "Model" })).not.toHaveTextContent("DeepSeek V4 Pro"); + + await userEvent.type(screen.getByRole("textbox", { name: /new question/i }), "q"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + + await waitFor(() => expect(body).toEqual({ + question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "medium", + })); +}); + +test("immediate submit waits for a switched workspace policy before creating a session", async () => { + let body: unknown; + let releasePolicy!: () => void; + let policyRequestStarted = false; + const policyMayFinish = new Promise((resolve) => { releasePolicy = resolve; }); + workspacePreferences.save({ + workspaceId: "research", provider: "deepseek", model: "deepseek-v4-pro", thinking: "medium", + }); + const revision = (id: string) => ({ + id, commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }); + server.use( + http.get("/api/settings", () => HttpResponse.json({ workspace: "research" })), + http.get("/api/workspaces", () => HttpResponse.json([ + workspaceSummaryFixture("research", { displayName: "Research", revision: revision("research") }), + workspaceSummaryFixture("psd-clinical", { displayName: "PSD Clinical", revision: revision("psd-clinical") }), + ])), + http.get("/api/workspaces/research", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("research", ["deepseek/deepseek-v4-pro"]), revision: revision("research"), + })), + http.get("/api/workspaces/psd-clinical", async () => { + policyRequestStarted = true; + await policyMayFinish; + return HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical", ["zai/glm-5.2"], "zai/glm-5.2"), + revision: revision("psd-clinical"), + }); + }), + http.get("/api/models", () => HttpResponse.json({ models: [ + { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, + { provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro", reasoning: true }, + ] })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render(); + + const workspaceSelector = await screen.findByRole("combobox", { name: "Workspace" }); + await waitFor(() => expect(workspaceSelector).toHaveTextContent("psd-clinical")); + await userEvent.selectOptions(workspaceSelector, "psd-clinical"); + await waitFor(() => expect(policyRequestStarted).toBe(true)); + await userEvent.type(screen.getByRole("textbox", { name: /new question/i }), "q"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + + expect(body).toBeUndefined(); + expect(screen.getByRole("button", { name: /send/i })).toBeDisabled(); + releasePolicy(); + + await waitFor(() => expect(body).toEqual({ + question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "medium", + })); +}); + +test("initial restored workspace waits for its delayed policy before creating a session", async () => { + let body: unknown; + let releasePolicy!: () => void; + let policyRequestStarted = false; + const policyMayFinish = new Promise((resolve) => { releasePolicy = resolve; }); + workspacePreferences.save({ + workspaceId: "psd-clinical", provider: "deepseek", model: "deepseek-v4-pro", thinking: "medium", + }); + const revision = (id: string) => ({ + id, commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }); + server.use( + http.get("/api/settings", () => HttpResponse.json({ workspace: "psd-clinical" })), + http.get("/api/workspaces", () => HttpResponse.json([ + workspaceSummaryFixture("psd-clinical", { displayName: "PSD Clinical", revision: revision("psd-clinical") }), + ])), + http.get("/api/workspaces/psd-clinical", async () => { + policyRequestStarted = true; + await policyMayFinish; + return HttpResponse.json({ + workspace: canonicalWorkspaceFixture("psd-clinical", ["zai/glm-5.2"], "zai/glm-5.2"), + revision: revision("psd-clinical"), + }); + }), + http.get("/api/models", () => HttpResponse.json({ models: [ + { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, + { provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro", reasoning: true }, + ] })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render(); + + await waitFor(() => expect(policyRequestStarted).toBe(true)); + await userEvent.type(screen.getByRole("textbox", { name: /new question/i }), "q"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + + expect(body).toBeUndefined(); + expect(screen.getByRole("button", { name: /send/i })).toBeDisabled(); + releasePolicy(); + + await waitFor(() => expect(body).toEqual({ + question: "q", workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "medium", + })); +}); + +test("initial submit rejects a workspace summary that omits the canonical revision", async () => { + let body: unknown; + let failure: string | undefined; + workspacePreferences.save({ + workspaceId: "broken-workspace", provider: "zai", model: "glm-5.2", thinking: "medium", + }); + server.use( + http.get("/api/settings", () => HttpResponse.json({ workspace: "broken-workspace" })), + http.get("/api/workspaces", () => HttpResponse.json([{ + ...workspaceSummaryFixture("broken-workspace", { + displayName: "Broken workspace", + configurationState: "configuration_required", + }), + }])), + http.get("/api/models", () => HttpResponse.json({ models: [ + { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, + ] })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( { failure = message; }} />); + + await userEvent.type(screen.getByRole("textbox", { name: /new question/i }), "q"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + + await waitFor(() => expect(failure).toBe("Could not load workspace registry. Please retry.")); + expect(body).toBeUndefined(); +}); + +test("failed workspace summaries block creation and report a safe error", async () => { + let body: unknown; + let failure: string | undefined; + let summaryRequestFailed = false; + workspacePreferences.save({ + workspaceId: "psd-clinical", provider: "zai", model: "glm-5.2", thinking: "medium", + }); + server.use( + http.get("/api/settings", () => HttpResponse.json({ workspace: "psd-clinical" })), + http.get("/api/workspaces", () => { + summaryRequestFailed = true; + return new HttpResponse(null, { status: 503 }); + }), + http.get("/api/models", () => HttpResponse.json({ models: [ + { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, + ] })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( + + + { failure = message; }} /> + , + ); + + await waitFor(() => expect(summaryRequestFailed).toBe(true)); + await waitFor(() => expect(client.getQueryState(["workspaces"])?.status).toBe("error")); + await userEvent.type(screen.getByRole("textbox", { name: /new question/i }), "q"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + + await waitFor(() => expect(failure).toBe("Could not load workspace registry. Please retry.")); + expect(body).toBeUndefined(); +}); + +test("submit follows a rapid workspace switch instead of waiting for an abandoned policy", async () => { + let body: unknown; + let releaseC!: () => void; + let bPolicyRequestStarted = false; + let cPolicyRequestStarted = false; + const cPolicyMayFinish = new Promise((resolve) => { releaseC = resolve; }); + workspacePreferences.save({ + workspaceId: "research", provider: "deepseek", model: "deepseek-v4-pro", thinking: "medium", + }); + const revision = (id: string) => ({ + id, commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/snapshot", + }); + server.use( + http.get("/api/settings", () => HttpResponse.json({ workspace: "research" })), + http.get("/api/workspaces", () => HttpResponse.json([ + workspaceSummaryFixture("research", { displayName: "Research", revision: revision("research") }), + workspaceSummaryFixture("workspace-b", { displayName: "Workspace B", revision: revision("workspace-b") }), + workspaceSummaryFixture("workspace-c", { displayName: "Workspace C", revision: revision("workspace-c") }), + ])), + http.get("/api/workspaces/research", () => HttpResponse.json({ + workspace: canonicalWorkspaceFixture("research", ["deepseek/deepseek-v4-pro"]), revision: revision("research"), + })), + http.get("/api/workspaces/workspace-b", async () => { + bPolicyRequestStarted = true; + await new Promise(() => undefined); + return HttpResponse.json({}); + }), + http.get("/api/workspaces/workspace-c", async () => { + cPolicyRequestStarted = true; + await cPolicyMayFinish; + return HttpResponse.json({ + workspace: canonicalWorkspaceFixture("workspace-c", ["zai/glm-5.2"], "zai/glm-5.2"), + revision: revision("workspace-c"), + }); + }), + http.get("/api/models", () => HttpResponse.json({ models: [ + { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true }, + { provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro", reasoning: true }, + ] })), + http.post("/api/sessions", async ({ request }) => { + body = await request.json(); + return HttpResponse.json({ id: "s1" }); + }), + ); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render(); + + const workspaceSelector = await screen.findByRole("combobox", { name: "Workspace" }); + await waitFor(() => expect(screen.getByRole("option", { name: "workspace-b" })).toBeInTheDocument()); + await userEvent.selectOptions(workspaceSelector, "workspace-b"); + await waitFor(() => expect(bPolicyRequestStarted).toBe(true)); + await userEvent.type(screen.getByRole("textbox", { name: /new question/i }), "q"); + await userEvent.click(screen.getByRole("button", { name: /send/i })); + await userEvent.selectOptions(workspaceSelector, "workspace-c"); + await waitFor(() => expect(cPolicyRequestStarted).toBe(true)); + + expect(body).toBeUndefined(); + releaseC(); + + await waitFor(() => expect(body).toEqual({ + question: "q", workspaceId: "workspace-c", provider: "zai", model: "glm-5.2", thinking: "medium", + })); +}); + test("pulses the stop dot only while the harness is working", () => { const { rerender } = render(); const dot = () => diff --git a/frontend/src/shell/SteerInput.tsx b/frontend/src/shell/SteerInput.tsx index f07d59d0..2f51ecb6 100644 --- a/frontend/src/shell/SteerInput.tsx +++ b/frontend/src/shell/SteerInput.tsx @@ -1,13 +1,17 @@ // frontend/src/shell/SteerInput.tsx import { useEffect, useRef, useState } from "react"; import { CornerDownLeft } from "lucide-react"; -import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useQuery } from "@tanstack/react-query"; import { postSteer, createSession } from "../api/sessions"; -import { ApiError } from "../api/client"; -import { getSettings, putSettings, type Settings } from "../api/settings"; -import { listWorkspaces } from "../api/workspaces"; +import { ApiError, apiErrorMessage } from "../api/client"; +import { getSettings } from "../api/settings"; +import { getWorkspace, listWorkspaces } from "../api/workspaces"; import { listModels } from "../api/models"; import { useSessionStore } from "../store/sessionStore"; +import { + WorkspaceSelectionError, workspacePolicyGate, workspacePreferences, type WorkspacePreference, +} from "../workspaces/drafts"; +import { captureAuthOperation, isAuthOperationCurrent } from "../auth/authOperation"; const THINKING_LEVELS = ["low", "medium", "high"] as const; @@ -45,6 +49,14 @@ export function SteerInput({ const setLastUserEntry = useSessionStore((s) => s.setLastUserEntry); const setPhase = useSessionStore((s) => s.setPhase); const taRef = useRef(null); + const operationEpochRef = useRef(0); + useEffect(() => () => { operationEpochRef.current += 1; }, []); + useEffect(() => { + // A new active session owns a new composer operation scope. Invalidate any held request + // before it can write the old session's completion into the new target. + operationEpochRef.current += 1; + setBusy(false); + }, [sessionId]); // Merge our own ref (for autosizing) with the forwarded one (parent focus). function attachRef(el: HTMLTextAreaElement | null) { @@ -65,14 +77,19 @@ export function SteerInput({ async function submit() { const trimmed = text.trim(); if (!trimmed || busy) return; + const operation = captureAuthOperation({ sessionId, disposalEpoch: operationEpochRef.current }); + if (!operation) return; setBusy(true); if (sessionId) { try { await postSteer(sessionId, trimmed); + if (!isAuthOperationCurrent(operation, { sessionId, disposalEpoch: operationEpochRef.current })) return; setLastUserEntry({ kind: "input", text: trimmed }); setText(""); } finally { - setBusy(false); + if (isAuthOperationCurrent(operation, { sessionId, disposalEpoch: operationEpochRef.current })) { + setBusy(false); + } } return; } @@ -83,20 +100,29 @@ export function SteerInput({ setLastUserEntry({ kind: "input", text: trimmed }); onSessionCreating?.(trimmed); try { - const { id } = await createSession({ question: trimmed }); + const { id } = await createSession({ question: trimmed }, { + operation, + isCurrent: () => isAuthOperationCurrent(operation, { + sessionId: null, + disposalEpoch: operationEpochRef.current, + }), + }); + if (!isAuthOperationCurrent(operation, { sessionId: null, disposalEpoch: operationEpochRef.current })) return; onSessionCreated?.(id); setText(""); } catch (error) { + if (!isAuthOperationCurrent(operation, { sessionId: null, disposalEpoch: operationEpochRef.current })) return; // Keep the question in the composer so retrying does not require retyping. - // A DWH-unreachable precheck (local dev, VPN down) carries a specific code; - // surface its message as the alert instead of the generic retry hint. - const payload = error instanceof ApiError - ? (error.payload as { code?: string; error?: string } | undefined) - : undefined; - const alert = payload?.code === "dwh_unreachable" ? payload.error : undefined; + // ApiError messages are derived from the local status/code allowlist; + // never surface response-body text from the server. + const alert = error instanceof ApiError && error.code === "dwh_unreachable" + ? apiErrorMessage(error) + : error instanceof WorkspaceSelectionError ? error.message : undefined; onSessionCreateFailed?.(alert); } finally { - setBusy(false); + if (isAuthOperationCurrent(operation, { sessionId: null, disposalEpoch: operationEpochRef.current })) { + setBusy(false); + } } } @@ -151,33 +177,105 @@ export function SteerInput({ /** * Status strip beneath the composer. Left: the active workspace selector. - * Right: live context usage plus model + thinking-level selectors (these replace - * the old Settings dialog and persist through PUT /settings). + * Right: live context usage plus ephemeral workspace/model/thinking selectors. */ export function ComposerFooter() { - const qc = useQueryClient(); const { data: settings } = useQuery({ queryKey: ["settings"], queryFn: getSettings }); - const { data: workspaces = [] } = useQuery({ queryKey: ["workspaces"], queryFn: listWorkspaces }); + const { data: workspaces = [], isLoading: workspacesLoading, isError: workspaceSummariesError } = useQuery({ + queryKey: ["workspaces"], queryFn: listWorkspaces, + }); const { data: modelsData } = useQuery({ queryKey: ["models"], queryFn: listModels }); const models = modelsData?.models ?? []; const tokenUsage = useSessionStore((state) => state.tokenUsage); + const [preferences, setPreferences] = useState(() => workspacePreferences.load()); - const workspace = settings?.workspace ?? ""; - const model = settings?.model ?? ""; - const thinking = settings?.thinking ?? "medium"; + useEffect(() => { + if (!settings) return; + setPreferences(workspacePreferences.migrate({ + workspaceId: settings.workspace, + provider: settings.provider, + model: settings.model, + thinking: settings.thinking, + })); + }, [settings]); - // PUT /settings replaces the whole object, so always send the merged settings. - async function update(patch: Partial) { - await putSettings({ workspace, provider: settings?.provider, model, thinking, ...patch }); - qc.invalidateQueries({ queryKey: ["settings"] }); + const workspace = preferences.workspaceId ?? settings?.workspace ?? ""; + const selectedWorkspace = workspaces.find((candidate) => candidate.id === workspace); + const selectedWorkspaceHasRevision = Boolean(selectedWorkspace?.revision); + const { data: workspaceRecord, isError: workspacePolicyError } = useQuery({ + queryKey: ["workspace", workspace], + queryFn: () => getWorkspace(workspace), + enabled: selectedWorkspaceHasRevision, + }); + const model = preferences.model ?? settings?.model ?? ""; + const thinking = preferences.thinking ?? settings?.thinking ?? "medium"; + + const allowedModels = workspaceRecord?.workspace.llm_policy.allowed; + const policyModels = allowedModels + ? models.filter((candidate) => allowedModels.includes(`${candidate.provider}/${candidate.id}`)) + : models; + + useEffect(() => { + if (!workspace) { + workspacePolicyGate.clear(); + } else if (workspacesLoading) { + workspacePolicyGate.beginSummary(workspace); + } else if (workspaceSummariesError) { + workspacePolicyGate.rejectSummary(workspace); + } else if (selectedWorkspace && !selectedWorkspaceHasRevision) { + workspacePolicyGate.rejectSummary(workspace); + } else if (selectedWorkspace?.revision) { + workspacePolicyGate.select(workspace); + } else { + workspacePolicyGate.allowLegacy(workspace); + } + }, [selectedWorkspace, workspace, workspaceSummariesError, workspacesLoading]); + + useEffect(() => { + if (!allowedModels?.length) return; + const selected = preferences.provider && preferences.model + ? `${preferences.provider}/${preferences.model}` + : undefined; + if (selected && allowedModels.some((allowed) => allowed === selected)) return; + const replacement = workspaceRecord?.workspace.llm_policy.default + && allowedModels.includes(workspaceRecord.workspace.llm_policy.default) + ? workspaceRecord.workspace.llm_policy.default + : allowedModels[0]; + const separator = replacement.indexOf("/"); + if (separator <= 0 || separator === replacement.length - 1) return; + const next = { + ...preferences, + provider: replacement.slice(0, separator), + model: replacement.slice(separator + 1), + }; + workspacePreferences.save(next); + setPreferences(next); + }, [allowedModels, preferences, workspaceRecord]); + + useEffect(() => { + if (!selectedWorkspace?.revision) return; + if (workspaceRecord) workspacePolicyGate.resolve(workspace); + else if (workspacePolicyError) workspacePolicyGate.reject(workspace); + }, [selectedWorkspace, workspace, workspacePolicyError, workspaceRecord]); + + function update(patch: WorkspacePreference) { + if (patch.workspaceId && patch.workspaceId !== workspace) { + const selected = workspaces.find((candidate) => candidate.id === patch.workspaceId); + if (selected && !selected.revision) workspacePolicyGate.rejectSummary(patch.workspaceId); + else if (selected?.revision) workspacePolicyGate.select(patch.workspaceId); + else workspacePolicyGate.allowLegacy(patch.workspaceId); + } + const next = workspacePreferences.save({ ...preferences, ...patch }); + setPreferences(next); } function onModelChange(id: string) { - const m = models.find((x) => x.id === id); + const m = policyModels.find((x) => x.id === id); update({ model: id, provider: m?.provider }); } - const knownModel = models.some((m) => m.id === model); + const knownModel = policyModels.some((m) => m.id === model); + const showModelFallback = !allowedModels && !knownModel; const contextPct = tokenUsage && tokenUsage.contextWindow > 0 ? tokenUsage.totalTokens / tokenUsage.contextWindow : 0; @@ -185,12 +283,12 @@ export function ComposerFooter() { return (
- update({ workspace: v })}> + update({ workspaceId: v })}> {workspaces.length === 0 ? ( ) : ( - workspaces.map((w) => ( - )) @@ -206,12 +304,12 @@ export function ComposerFooter() {
- {models.length === 0 ? ( + {policyModels.length === 0 ? ( ) : ( <> - {!knownModel && } - {models.map((m) => ( + {showModelFallback && } + {policyModels.map((m) => ( diff --git a/frontend/src/shell/WidgetHost.response.test.tsx b/frontend/src/shell/WidgetHost.response.test.tsx new file mode 100644 index 00000000..8cd39907 --- /dev/null +++ b/frontend/src/shell/WidgetHost.response.test.tsx @@ -0,0 +1,128 @@ +import { act, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { http, HttpResponse } from "msw"; +import { server } from "../test/msw"; +import { useSessionStore } from "../store/sessionStore"; +import { WidgetHost } from "./WidgetHost"; +import { setAuthState } from "../auth/authState"; + +const userA = { + issuer: "local", subject: "user-a", roles: ["user"] as const, + permissions: ["session.use"], isAdmin: false, csrfToken: "a".repeat(43), session: null, +}; + +beforeEach(() => { + useSessionStore.getState().resetSession(); + setAuthState(userA); +}); + +test("a gate choice shows progress, prevents duplicate clicks, and stays silent on success", async () => { + let release!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + let requests = 0; + server.use(http.post("/api/sessions/s1/response", async () => { + requests += 1; + await held; + return new HttpResponse(null, { status: 204 }); + })); + useSessionStore.setState({ + pendingWidget: { + id: "gate-1", + widget: "select", + options: [{ id: "recommended", label: "Recommended answer" }], + }, + }); + + const user = userEvent.setup(); + render(); + const choice = screen.getByRole("button", { name: "Recommended answer" }); + await user.click(choice); + + expect(screen.getByRole("status")).toHaveTextContent("Sending response"); + expect(choice).toBeDisabled(); + await user.click(choice); + expect(requests).toBe(1); + + act(() => release()); + await waitFor(() => expect(useSessionStore.getState().pendingWidget).toBeNull()); + expect(useSessionStore.getState().toasts).toEqual([]); + + act(() => useSessionStore.setState({ + pendingWidget: { + id: "gate-2", + widget: "select", + options: [{ id: "next", label: "Next answer" }], + }, + })); + expect(screen.getByRole("button", { name: "Next answer" })).toBeEnabled(); + expect(screen.queryByRole("status")).not.toBeInTheDocument(); +}); + +test("a delayed gate response from user A cannot clear user B's pending gate", async () => { + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/sessions/s1/response", async () => { + started(); + try { + await held; + return new HttpResponse(null, { status: 204 }); + } finally { + settled(); + } + })); + setAuthState(userA); + useSessionStore.setState({ + pendingWidget: { id: "gate-a", widget: "select", options: [{ id: "a", label: "A answer" }] }, + }); + render(); + await userEvent.click(screen.getByRole("button", { name: "A answer" })); + await requestStarted; + + act(() => setAuthState({ ...userA, subject: "user-b", csrfToken: "b".repeat(43) })); + act(() => useSessionStore.setState({ + pendingWidget: { id: "gate-b", widget: "select", options: [{ id: "b", label: "B answer" }] }, + })); + release(); + await act(async () => { await requestSettled; }); + + expect(useSessionStore.getState().pendingWidget?.id).toBe("gate-b"); + expect(useSessionStore.getState().lastUserEntry).toBeNull(); +}); + +test("a held s1 gate response cannot clear the pending s2 gate", async () => { + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/sessions/s1/response", async () => { + started(); + try { + await held; + return new HttpResponse(null, { status: 204 }); + } finally { + settled(); + } + })); + useSessionStore.setState({ + pendingWidget: { id: "gate-s1", widget: "select", options: [{ id: "s1", label: "Answer s1" }] }, + }); + const view = render(); + await userEvent.click(screen.getByRole("button", { name: "Answer s1" })); + await requestStarted; + + view.rerender(); + act(() => useSessionStore.setState({ + pendingWidget: { id: "gate-s2", widget: "select", options: [{ id: "s2", label: "Answer s2" }] }, + })); + release(); + await act(async () => { await requestSettled; }); + + expect(useSessionStore.getState().pendingWidget?.id).toBe("gate-s2"); + expect(useSessionStore.getState().lastUserEntry).toBeNull(); +}); diff --git a/frontend/src/shell/WidgetHost.test.tsx b/frontend/src/shell/WidgetHost.test.tsx index 95cff725..d2a380af 100644 --- a/frontend/src/shell/WidgetHost.test.tsx +++ b/frontend/src/shell/WidgetHost.test.tsx @@ -1,4 +1,4 @@ -import { render, screen } from "@testing-library/react"; +import { act, cleanup, render, screen } from "@testing-library/react"; import { WidgetHost } from "./WidgetHost"; import { useSessionStore } from "../store/sessionStore"; import type { WidgetDescriptor } from "../api/types"; @@ -17,18 +17,19 @@ beforeEach(() => { useSessionStore.getState().resetSession(); }); afterEach(() => { + cleanup(); errorSpy.mockRestore(); - useSessionStore.getState().resetSession(); + act(() => useSessionStore.getState().resetSession()); }); const gate: WidgetDescriptor = { id: "g1", widget: "select", title: "Choose" }; -test("a crashing widget shows a fallback instead of unmounting the app", () => { +test("a crashing widget shows a fallback instead of unmounting the app", async () => { useSessionStore.setState({ pendingWidget: gate }); - render(); + await act(async () => { render(); }); // No throw escaped: the fallback rendered. - expect(screen.getByRole("alert")).toBeInTheDocument(); - expect(screen.getByText(/couldn't be displayed/i)).toBeInTheDocument(); + expect(await screen.findByRole("alert")).toBeInTheDocument(); + expect(await screen.findByText(/couldn't be displayed/i)).toBeInTheDocument(); // The store (and thus the surrounding app) is still alive and interactive. expect(useSessionStore.getState().pendingWidget).toEqual(gate); }); diff --git a/frontend/src/shell/WidgetHost.tsx b/frontend/src/shell/WidgetHost.tsx index da9ae89c..7b5a60be 100644 --- a/frontend/src/shell/WidgetHost.tsx +++ b/frontend/src/shell/WidgetHost.tsx @@ -1,21 +1,41 @@ +import { useEffect, useRef, useState } from "react"; import { useSessionStore } from "../store/sessionStore"; import { resolve } from "../widgets"; import { postResponse } from "../api/sessions"; import type { UiResponse } from "../api/types"; import { ErrorBoundary } from "../components/ErrorBoundary"; +import { captureAuthOperation, isAuthOperationCurrent } from "../auth/authOperation"; export function WidgetHost({ sessionId }: { sessionId: string | null }) { const pending = useSessionStore((s) => s.pendingWidget); const clearPending = useSessionStore((s) => s.clearPending); const pushToast = useSessionStore((s) => s.pushToast); const setLastUserEntry = useSessionStore((s) => s.setLastUserEntry); + const [responding, setResponding] = useState(false); + const responseInFlight = useRef(false); + const operationEpochRef = useRef(0); + useEffect(() => () => { operationEpochRef.current += 1; }, []); + useEffect(() => { + // A gate belongs to its active session target, not merely the authenticated user. + operationEpochRef.current += 1; + responseInFlight.current = false; + setResponding(false); + }, [sessionId]); if (!pending) return null; const Renderer = resolve(pending.widget); const onRespond = async (r: UiResponse) => { + if (responseInFlight.current) return; + const operation = captureAuthOperation({ sessionId, disposalEpoch: operationEpochRef.current }); + if (!operation) return; + responseInFlight.current = true; + setResponding(true); + if (sessionId) { try { await postResponse(sessionId, r); + if (!isAuthOperationCurrent(operation, { sessionId, disposalEpoch: operationEpochRef.current })) return; } catch (err) { + if (!isAuthOperationCurrent(operation, { sessionId, disposalEpoch: operationEpochRef.current })) return; // Surface feedback and keep the widget pending so the user can retry. pushToast({ level: "error", @@ -24,9 +44,14 @@ export function WidgetHost({ sessionId }: { sessionId: string | null }) { ? `Failed to send response: ${err.message}` : "Failed to send response.", }); + responseInFlight.current = false; + setResponding(false); return; } } + if (!isAuthOperationCurrent(operation, { sessionId, disposalEpoch: operationEpochRef.current })) return; + responseInFlight.current = false; + setResponding(false); setLastUserEntry({ kind: "choice", text: @@ -40,7 +65,16 @@ export function WidgetHost({ sessionId }: { sessionId: string | null }) { // widget render. resetKeys on the descriptor id so the next gate starts clean. return ( - +
+
+ +
+ {responding && ( +

+ Sending response… +

+ )} +
); } diff --git a/frontend/src/shell/WorkspaceManager.test.tsx b/frontend/src/shell/WorkspaceManager.test.tsx new file mode 100644 index 00000000..c318f4e3 --- /dev/null +++ b/frontend/src/shell/WorkspaceManager.test.tsx @@ -0,0 +1,1027 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { http, HttpResponse } from "msw"; +import { beforeEach, expect, test, vi } from "vitest"; +import { server } from "../test/msw"; +import { canonicalWorkspaceFixture, workspaceRevisionFixture, workspaceSummaryFixture } from "../test/workspace-fixtures"; +import { WorkspaceManager } from "./WorkspaceManager"; +import { setAuthState } from "../auth/authState"; +import { queryClient } from "../app/queryClient"; + +const workspace = canonicalWorkspaceFixture("psd-clinical"); +const revision = workspaceRevisionFixture("psd-clinical"); +const authenticatedWorkspaceUser = { + issuer: "local", subject: "workspace-user", roles: ["user"] as const, + permissions: ["workspace.manage", "workspace.secrets.manage"], isAdmin: false, + csrfToken: "w".repeat(43), session: null, +}; +const requirement = { + id: "dwh.password", + connector: "dwh", + label: "Data warehouse password", + description: "Password used by the selected data warehouse connection.", + input: "password", + required: true, + configured: false, +}; + +const readyAuthentication = { + ready: true, + mode: "none", + checks: [{ level: "info", code: "auth_ready", message: "Authentication is ready." }], +}; + +function runtimeConfiguration(configured = false) { + return { + workspaceId: "psd-clinical", + revision, + configurationState: configured ? "ready" : "configuration_required", + requirements: [{ ...requirement, configured }], + }; +} + +function renderManager(onClose = vi.fn()) { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return { + onClose, + ...render( + + + , + ), + }; +} + +function renderDeniedManager() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + , + ); +} + +type ManagerSettings = { + open: boolean; + canManageWorkspace: boolean; + canManageSecrets: boolean; +}; + +function renderManagerWithSettings(initial: Partial = {}) { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const onClose = vi.fn(); + let settings: ManagerSettings = { + open: true, + canManageWorkspace: true, + canManageSecrets: true, + ...initial, + }; + const tree = () => ( + + + + ); + const view = render(tree()); + return { + ...view, + client, + onClose, + rerender(next: Partial) { + settings = { ...settings, ...next }; + view.rerender(tree()); + }, + }; +} + +beforeEach(() => { + localStorage.clear(); + setAuthState(authenticatedWorkspaceUser); + server.use( + http.get("/api/workspace-registry/status", () => HttpResponse.json({ + branch: "main", + head: "a".repeat(40), + ahead: 0, + behind: 0, + degraded: false, + repository: { + host: "git.example.test", + repository: "analytics/thoth-workspaces", + transport: "ssh", + }, + })), + http.get("/api/workspaces", () => HttpResponse.json([ + workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + description: "Clinical data", + configurationState: "configuration_required", + revision, + }), + ])), + http.get("/api/workspaces/psd-clinical", () => HttpResponse.json({ workspace, revision })), + http.get("/api/workspaces/psd-clinical/runtime-configuration", () => ( + HttpResponse.json(runtimeConfiguration()) + )), + ); +}); + +test("defaults workspace mutations and secrets to denied", async () => { + renderDeniedManager(); + expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Update workspace repository" })).not.toBeInTheDocument(); + await userEvent.click(await screen.findByRole("button", { name: "PSD Clinical" })); + expect(await screen.findByRole("heading", { name: "Workspace-specific actions" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Validate workspace source" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Test workspace connections" })).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Runtime secrets" })).not.toBeInTheDocument(); +}); + +test("uses a tall viewport area and keeps the workspace content scrollable", () => { + renderManager(); + + const dialog = screen.getByRole("dialog"); + expect(dialog).toHaveClass( + "h-[86vh]", + "w-[94vw]", + "sm:w-[70vw]", + "max-w-[94vw]", + ); + expect(screen.getByRole("main")).toHaveClass("overflow-y-auto"); +}); + +test("offers a button above the workspace list that returns to Level 1", async () => { + const user = userEvent.setup(); + const onClose = vi.fn(); + renderManager(onClose); + + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + expect(await screen.findByRole("heading", { name: "Workspace-specific actions" })).toBeVisible(); + + const navigation = screen.getByRole("navigation", { name: "Workspaces" }); + const backButton = within(navigation).getByRole("button", { name: "Back to Level 1" }); + expect(backButton).toHaveClass("border-border", "bg-card", "w-full"); + expect(backButton.compareDocumentPosition(within(navigation).getByText("Available workspaces")) + & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + await user.click(backButton); + + expect(onClose).not.toHaveBeenCalled(); + expect(screen.getByTestId("workspace-overview")).toBeVisible(); + expect(screen.queryByRole("heading", { name: "Workspace-specific actions" })).not.toBeInTheDocument(); +}); + +test("level one explains the read-only Git sequence and the repository update button", async () => { + const user = userEvent.setup(); + server.use(http.post("/api/workspace-registry/pull", () => HttpResponse.json({ + branch: "main", head: "b".repeat(40), ahead: 0, behind: 0, degraded: false, + }))); + renderManager(); + + expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeVisible(); + const overview = screen.getByTestId("workspace-overview"); + expect(within(overview).getByText(/Git server such as GitHub, GitLab, or Gitea/i)).toBeVisible(); + expect(within(overview).getByText(/configured during ThothII installation/i)).toBeVisible(); + const repositoryStep = within(overview).getAllByRole("listitem")[0]; + expect(repositoryStep).toHaveTextContent(/create a workspace repository/i); + expect(repositoryStep).toHaveTextContent(/one directory for each workspace/i); + expect(repositoryStep).toHaveTextContent(/thoth-workspaces\.yaml/i); + expect(repositoryStep).toHaveTextContent(/database connection/i); + expect(repositoryStep).toHaveTextContent(/Evidence sources/i); + expect(repositoryStep).toHaveTextContent(/vector-database collection/i); + expect(within(overview).getByRole("link", { name: /workspace authoring instructions on GitHub/i })).toHaveAttribute( + "href", + "https://github.com/mptyl/ThothII/blob/main/docs/install/local-workspace-registry.md#prepare-and-publish-a-workspace-source", + ); + expect(within(overview).getAllByText(/managed read-only checkout/i)).toHaveLength(2); + expect(within(overview).getByText(/current active revision remains unchanged/i)).toBeVisible(); + expect(within(overview).getByText(/No workspace selection is required/i)).toBeVisible(); + expect(await within(overview).findByText("git.example.test/analytics/thoth-workspaces")).toBeVisible(); + + await user.click(screen.getByRole("button", { name: "Update workspace repository" })); + expect(await screen.findByText("Workspace repository updated and validated.")).toBeVisible(); + expect(screen.getByText(/create a workspace repository/i)).toBeInTheDocument(); + expect(screen.queryByText(/import|export|bundle/i)).not.toBeInTheDocument(); +}); + +test("workspace-specific commands remain isolated until a workspace is selected", async () => { + const user = userEvent.setup(); + renderManager(); + + expect(screen.queryByRole("heading", { name: "Workspace-specific actions" })).not.toBeInTheDocument(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + expect(await screen.findByRole("heading", { name: "Workspace-specific actions" })).toBeVisible(); + expect(screen.getByText(/reads this revision without modifying or publishing it/i)).toBeVisible(); + expect(screen.getByText(/checks workspace.yaml and the required workspace directories/i)).toBeVisible(); + expect(screen.getByText(/temporary decrypted credentials/i)).toBeVisible(); + const databaseField = screen.getByText("Database").parentElement; + expect(databaseField).not.toBeNull(); + expect(databaseField).toHaveTextContent("engine: postgres"); + expect(databaseField).toHaveTextContent("database: database"); + expect(databaseField).toHaveTextContent("schema: public"); + expect(screen.getByRole("button", { name: "Validate workspace source" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Test workspace connections" })).toBeVisible(); +}); + +test("keeps validation and connection results inside their respective action cards", async () => { + const user = userEvent.setup(); + server.use( + http.post("/api/workspaces/validate", () => HttpResponse.json({ + workspace, contract: {}, activatable: true, diagnostics: [], authentication: readyAuthentication, + })), + http.post("/api/workspaces/psd-clinical/test", () => HttpResponse.json({ + activatable: false, + diagnostics: [{ level: "error", code: "connector_unavailable", message: "Connector diagnostic failed." }], + authentication: readyAuthentication, + })), + ); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + const validationCard = screen.getByTestId("workspace-validation-card"); + const connectionCard = screen.getByTestId("workspace-connection-card"); + await user.click(within(validationCard).getByRole("button", { name: "Validate workspace source" })); + const validationStatus = await within(validationCard).findByRole("status"); + expect(validationStatus).toHaveTextContent("Workspace source and authentication are valid."); + expect(validationStatus).toHaveClass("text-emerald-700"); + expect(within(connectionCard).queryByText("Workspace source and authentication are valid.")).not.toBeInTheDocument(); + + await user.click(within(connectionCard).getByRole("button", { name: "Test workspace connections" })); + expect(await within(connectionCard).findByRole("alert")).toHaveTextContent( + "connector_unavailable: Connector diagnostic failed.", + ); + expect(within(validationCard).queryByText("connector_unavailable: Connector diagnostic failed.")).not.toBeInTheDocument(); +}); + +test("renders one authentication section with configured-group errors and no unmapped-group list", async () => { + const user = userEvent.setup(); + server.use(http.post("/api/workspaces/validate", () => HttpResponse.json({ + workspace, + contract: {}, + activatable: false, + diagnostics: [], + authentication: { + ready: false, + mode: "oidc", + checks: [{ + level: "error", + code: "oidc_mapped_group_missing", + field: "Thoth Administrators", + message: "A configured authorization group does not exist.", + }], + }, + }))); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + await user.click(screen.getByRole("button", { name: "Validate workspace source" })); + + const section = await screen.findByTestId("workspace-authentication"); + expect(within(section).getByRole("heading", { name: "Authentication" })).toBeVisible(); + expect(within(section).getByText("Failed")).toBeVisible(); + expect(within(section).getByText("oidc_mapped_group_missing: Thoth Administrators — A configured authorization group does not exist.")).toBeVisible(); + expect(within(section).queryByText(/unmapped/i)).not.toBeInTheDocument(); +}); + +test("clears a previous authentication result as soon as validation is retried", async () => { + const user = userEvent.setup(); + let calls = 0; + let releaseRetry!: () => void; + const retryStarted = new Promise((resolve) => { releaseRetry = resolve; }); + server.use(http.post("/api/workspaces/validate", async () => { + calls += 1; + if (calls > 1) await retryStarted; + return HttpResponse.json({ + workspace, contract: {}, activatable: true, diagnostics: [], authentication: readyAuthentication, + }); + })); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + const button = screen.getByRole("button", { name: "Validate workspace source" }); + + await user.click(button); + expect(await screen.findByTestId("workspace-authentication")).toBeVisible(); + + await user.click(button); + await waitFor(() => expect(calls).toBe(2)); + expect(screen.queryByTestId("workspace-authentication")).not.toBeInTheDocument(); + + releaseRetry(); + expect(await screen.findByTestId("workspace-authentication")).toBeVisible(); +}); + +test("ignores an older validation response that completes after a newer connection test", async () => { + const user = userEvent.setup(); + let releaseValidation!: () => void; + let releaseTest!: () => void; + let validationStarted!: () => void; + let testStarted!: () => void; + let validationSettled!: () => void; + const heldValidation = new Promise((resolve) => { releaseValidation = resolve; }); + const heldTest = new Promise((resolve) => { releaseTest = resolve; }); + const validationRequestStarted = new Promise((resolve) => { validationStarted = resolve; }); + const testRequestStarted = new Promise((resolve) => { testStarted = resolve; }); + const validationRequestSettled = new Promise((resolve) => { validationSettled = resolve; }); + server.use( + http.post("/api/workspaces/validate", async () => { + validationStarted(); + try { + await heldValidation; + return HttpResponse.json({ + workspace, contract: {}, activatable: true, diagnostics: [], authentication: readyAuthentication, + }); + } finally { + validationSettled(); + } + }), + http.post("/api/workspaces/psd-clinical/test", async () => { + testStarted(); + await heldTest; + return HttpResponse.json({ + activatable: false, + diagnostics: [], + authentication: { + ready: false, + mode: "oidc", + checks: [{ level: "error", code: "oidc_secret_missing", message: "Newer connection result." }], + }, + }); + }), + ); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + await user.click(screen.getByRole("button", { name: "Validate workspace source" })); + await validationRequestStarted; + await user.click(screen.getByRole("button", { name: "Test workspace connections" })); + await testRequestStarted; + expect(screen.queryByTestId("workspace-authentication")).not.toBeInTheDocument(); + + act(() => releaseTest()); + const authentication = await screen.findByTestId("workspace-authentication"); + expect(within(authentication).getByText(/Newer connection result/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Validate workspace source" })).not.toBeDisabled(); + + act(() => releaseValidation()); + await act(async () => { + await validationRequestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + expect(within(authentication).getByText(/Newer connection result/)).toBeVisible(); + expect(within(authentication).queryByText("Passed")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Validate workspace source" })).not.toBeDisabled(); +}); + +test("does not apply a diagnostic that completes after its dialog is closed and reopened", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/workspaces/validate", async () => { + started(); + try { + await held; + return HttpResponse.json({ + workspace, contract: {}, activatable: true, diagnostics: [], authentication: readyAuthentication, + }); + } finally { + settled(); + } + })); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const onClose = vi.fn(); + const view = render( + + + , + ); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.click(screen.getByRole("button", { name: "Validate workspace source" })); + await requestStarted; + + await user.click(screen.getByRole("button", { name: "Close workspace management" })); + expect(onClose).toHaveBeenCalledTimes(1); + view.rerender( + + + , + ); + view.rerender( + + + , + ); + expect(await screen.findByRole("button", { name: "Validate workspace source" })).not.toBeDisabled(); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + expect(screen.queryByTestId("workspace-authentication")).not.toBeInTheDocument(); +}); + +test("does not apply a diagnostic that completes after returning through Level 1", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/workspaces/validate", async () => { + started(); + try { + await held; + return HttpResponse.json({ + workspace, contract: {}, activatable: true, diagnostics: [], authentication: readyAuthentication, + }); + } finally { + settled(); + } + })); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.click(screen.getByRole("button", { name: "Validate workspace source" })); + await requestStarted; + + await user.click(within(screen.getByRole("navigation", { name: "Workspaces" })) + .getByRole("button", { name: "Back to Level 1" })); + expect(screen.getByTestId("workspace-overview")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "PSD Clinical" })); + expect(await screen.findByRole("button", { name: "Validate workspace source" })).not.toBeDisabled(); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + expect(screen.queryByTestId("workspace-authentication")).not.toBeInTheDocument(); +}); + +test("does not apply a diagnostic after changing workspaces and returning", async () => { + const user = userEvent.setup(); + const alternateId = "other-clinical"; + const alternateWorkspace = canonicalWorkspaceFixture(alternateId); + const alternateRevision = workspaceRevisionFixture(alternateId); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use( + http.get("/api/workspaces", () => HttpResponse.json([ + workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", + description: "Clinical data", + configurationState: "configuration_required", + revision, + }), + workspaceSummaryFixture(alternateId, { + displayName: "Other Clinical", + description: "Other data", + configurationState: "configuration_required", + revision: alternateRevision, + }), + ])), + http.get(`/api/workspaces/${alternateId}`, () => HttpResponse.json({ + workspace: alternateWorkspace, + revision: alternateRevision, + })), + http.get(`/api/workspaces/${alternateId}/runtime-configuration`, () => HttpResponse.json({ + workspaceId: alternateId, + revision: alternateRevision, + configurationState: "configuration_required", + requirements: [{ ...requirement }], + })), + http.post("/api/workspaces/validate", async () => { + started(); + try { + await held; + return HttpResponse.json({ + workspace, contract: {}, activatable: true, diagnostics: [], authentication: readyAuthentication, + }); + } finally { + settled(); + } + }), + ); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.click(screen.getByRole("button", { name: "Validate workspace source" })); + await requestStarted; + + await user.click(screen.getByRole("button", { name: "Other Clinical" })); + expect(await screen.findByRole("heading", { name: "Other Clinical" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "PSD Clinical" })); + expect(await screen.findByRole("button", { name: "Validate workspace source" })).not.toBeDisabled(); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + expect(screen.queryByTestId("workspace-authentication")).not.toBeInTheDocument(); +}); + +test("never renders a hostile authentication field rejected by the API decoder", async () => { + const user = userEvent.setup(); + const attacker = "attacker-field-SENTINEL"; + server.use(http.post("/api/workspaces/validate", () => HttpResponse.json({ + workspace, + contract: {}, + activatable: false, + diagnostics: [], + authentication: { + ready: false, + mode: "oidc", + checks: [{ level: "error", code: "oidc_secret_missing", message: "failure", field: attacker }], + }, + }))); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + await user.click(screen.getByRole("button", { name: "Validate workspace source" })); + + expect(await screen.findByRole("alert")).toBeVisible(); + expect(screen.queryByText(new RegExp(attacker))).not.toBeInTheDocument(); +}); + +test("renders binding_ok as a green connection success", async () => { + const user = userEvent.setup(); + server.use( + http.post("/api/workspaces/psd-clinical/test", () => HttpResponse.json({ + activatable: true, + diagnostics: [{ level: "info", code: "binding_ok", message: "Installation bindings and diagnostics succeeded." }], + authentication: readyAuthentication, + })), + ); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + const connectionCard = screen.getByTestId("workspace-connection-card"); + await user.click(within(connectionCard).getByRole("button", { name: "Test workspace connections" })); + const connectionStatus = await within(connectionCard).findByRole("status"); + expect(connectionStatus).toHaveTextContent("binding_ok: Installation bindings and diagnostics succeeded."); + expect(connectionStatus).toHaveClass("text-emerald-700"); + expect(within(connectionCard).queryByRole("alert")).not.toBeInTheDocument(); +}); + +test("secret fields are write-only, clear after blind save, and may be forgotten", async () => { + const user = userEvent.setup(); + let savedBody: unknown; + server.use( + http.put("/api/workspaces/psd-clinical/secrets", async ({ request }) => { + savedBody = await request.json(); + return HttpResponse.json(runtimeConfiguration(true)); + }), + http.delete("/api/workspaces/psd-clinical/secrets/dwh.password", () => ( + HttpResponse.json(runtimeConfiguration(false)) + )), + ); + renderManager(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + + const input = await screen.findByLabelText("Data warehouse password"); + expect(input).toHaveValue(""); + expect(input).toHaveAttribute("type", "password"); + expect(screen.getByText("Not configured")).toBeVisible(); + await user.type(input, "one-time-password"); + await user.click(screen.getByRole("button", { name: "Save entered secrets" })); + + await waitFor(() => expect(savedBody).toEqual({ + values: { "dwh.password": "one-time-password" }, + })); + expect(input).toHaveValue(""); + expect(await screen.findByText("Configured")).toBeVisible(); + expect(screen.queryByDisplayValue("one-time-password")).not.toBeInTheDocument(); + expect(localStorage.length).toBe(0); + + await user.click(screen.getByRole("button", { name: "Forget stored Data warehouse password" })); + expect(await screen.findByText("Not configured")).toBeVisible(); +}); + +test("a delayed runtime-secret save from user A cannot repopulate user B's cache or notice", async () => { + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.put("/api/workspaces/psd-clinical/secrets", async () => { + started(); + try { + await held; + return HttpResponse.json(runtimeConfiguration(true)); + } finally { + settled(); + } + })); + queryClient.clear(); + setAuthState({ issuer: "local", subject: "user-a", roles: ["user"], permissions: ["workspace.secrets.manage"], isAdmin: false, csrfToken: "a".repeat(43), session: null }); + render( + + + , + ); + await userEvent.click(await screen.findByRole("button", { name: "PSD Clinical" })); + const input = await screen.findByLabelText("Data warehouse password"); + await userEvent.type(input, "a-secret"); + await userEvent.click(screen.getByRole("button", { name: "Save entered secrets" })); + await requestStarted; + + act(() => setAuthState({ issuer: "local", subject: "user-b", roles: ["user"], permissions: ["workspace.secrets.manage"], isAdmin: false, csrfToken: "b".repeat(43), session: null })); + expect(queryClient.getQueryData(["workspace-runtime-configuration", "psd-clinical"])).toBeUndefined(); + release(); + await act(async () => { await requestSettled; }); + + expect(queryClient.getQueryData(["workspace-runtime-configuration", "psd-clinical"])).toBeUndefined(); + expect(screen.queryByText("Runtime secrets saved. Stored values remain hidden.")).not.toBeInTheDocument(); +}); + +test("a deferred secret save cannot update a reopened dialog", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.put("/api/workspaces/psd-clinical/secrets", async () => { + started(); + try { + await held; + return HttpResponse.json(runtimeConfiguration(true)); + } finally { + settled(); + } + })); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.type(await screen.findByLabelText("Data warehouse password"), "one-time-password"); + await user.click(screen.getByRole("button", { name: "Save entered secrets" })); + await requestStarted; + + await user.click(screen.getByRole("button", { name: "Close workspace management" })); + manager.rerender({ open: false }); + manager.rerender({ open: true }); + await user.type(await screen.findByLabelText("Data warehouse password"), "new-secret"); + expect(await screen.findByRole("button", { name: "Save entered secrets" })).not.toBeDisabled(); + const cacheBeforeRelease = manager.client.getQueryData(["workspace-runtime-configuration", "psd-clinical"]); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect(manager.client.getQueryData(["workspace-runtime-configuration", "psd-clinical"])).toEqual(cacheBeforeRelease); + expect(screen.queryByText("Runtime secrets saved. Stored values remain hidden.")).not.toBeInTheDocument(); +}); + +test("a deferred secret save cannot update a workspace selected again after a cross-workspace transition", async () => { + const user = userEvent.setup(); + const alternateId = "other-clinical"; + const alternateWorkspace = canonicalWorkspaceFixture(alternateId); + const alternateRevision = workspaceRevisionFixture(alternateId); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use( + http.get("/api/workspaces", () => HttpResponse.json([ + workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", description: "Clinical data", configurationState: "configuration_required", revision, + }), + workspaceSummaryFixture(alternateId, { + displayName: "Other Clinical", description: "Other data", configurationState: "configuration_required", revision: alternateRevision, + }), + ])), + http.get(`/api/workspaces/${alternateId}`, () => HttpResponse.json({ workspace: alternateWorkspace, revision: alternateRevision })), + http.get(`/api/workspaces/${alternateId}/runtime-configuration`, () => HttpResponse.json({ + workspaceId: alternateId, revision: alternateRevision, configurationState: "configuration_required", requirements: [{ ...requirement }], + })), + http.put("/api/workspaces/psd-clinical/secrets", async () => { + started(); + try { + await held; + return HttpResponse.json(runtimeConfiguration(true)); + } finally { + settled(); + } + }), + ); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.type(await screen.findByLabelText("Data warehouse password"), "one-time-password"); + await user.click(screen.getByRole("button", { name: "Save entered secrets" })); + await requestStarted; + + await user.click(screen.getByRole("button", { name: "Other Clinical" })); + expect(await screen.findByRole("heading", { name: "Other Clinical" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "PSD Clinical" })); + await user.type(await screen.findByLabelText("Data warehouse password"), "new-secret"); + expect(await screen.findByRole("button", { name: "Save entered secrets" })).not.toBeDisabled(); + const cacheBeforeRelease = manager.client.getQueryData(["workspace-runtime-configuration", "psd-clinical"]); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect(manager.client.getQueryData(["workspace-runtime-configuration", "psd-clinical"])).toEqual(cacheBeforeRelease); + expect(screen.queryByText("Runtime secrets saved. Stored values remain hidden.")).not.toBeInTheDocument(); +}); + +test("a deferred secret save cannot update a context after its secret permission changes", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.put("/api/workspaces/psd-clinical/secrets", async () => { + started(); + try { + await held; + return HttpResponse.json(runtimeConfiguration(true)); + } finally { + settled(); + } + })); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.type(await screen.findByLabelText("Data warehouse password"), "one-time-password"); + await user.click(screen.getByRole("button", { name: "Save entered secrets" })); + await requestStarted; + + manager.rerender({ canManageSecrets: false }); + manager.rerender({ canManageSecrets: true }); + await user.type(await screen.findByLabelText("Data warehouse password"), "new-secret"); + expect(await screen.findByRole("button", { name: "Save entered secrets" })).not.toBeDisabled(); + const cacheBeforeRelease = manager.client.getQueryData(["workspace-runtime-configuration", "psd-clinical"]); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect(manager.client.getQueryData(["workspace-runtime-configuration", "psd-clinical"])).toEqual(cacheBeforeRelease); + expect(screen.queryByText("Runtime secrets saved. Stored values remain hidden.")).not.toBeInTheDocument(); +}); + +test("a deferred repository update cannot update a reopened dialog or refetch its cache", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + let statusRequests = 0; + let workspaceRequests = 0; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use( + http.get("/api/workspace-registry/status", () => { + statusRequests++; + return HttpResponse.json({ branch: "main", head: "a".repeat(40), ahead: 0, behind: 0, degraded: false }); + }), + http.get("/api/workspaces", () => { + workspaceRequests++; + return HttpResponse.json([workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", description: "Clinical data", configurationState: "configuration_required", revision, + })]); + }), + http.post("/api/workspace-registry/pull", async () => { + started(); + try { + await held; + return HttpResponse.json({ branch: "main", head: "b".repeat(40), ahead: 0, behind: 0, degraded: false }); + } finally { + settled(); + } + }), + ); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "Update workspace repository" })); + await requestStarted; + + await user.click(screen.getByRole("button", { name: "Close workspace management" })); + manager.rerender({ open: false }); + manager.rerender({ open: true }); + expect(await screen.findByRole("button", { name: "Update workspace repository" })).not.toBeDisabled(); + const requestsBeforeRelease = { statusRequests, workspaceRequests }; + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect({ statusRequests, workspaceRequests }).toEqual(requestsBeforeRelease); + expect(screen.queryByText("Workspace repository updated and validated.")).not.toBeInTheDocument(); +}); + +test("a deferred repository update cannot update a context after a cross-workspace transition", async () => { + const user = userEvent.setup(); + const alternateId = "other-clinical"; + const alternateWorkspace = canonicalWorkspaceFixture(alternateId); + const alternateRevision = workspaceRevisionFixture(alternateId); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + let statusRequests = 0; + let workspaceRequests = 0; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use( + http.get("/api/workspace-registry/status", () => { + statusRequests++; + return HttpResponse.json({ branch: "main", head: "a".repeat(40), ahead: 0, behind: 0, degraded: false }); + }), + http.get("/api/workspaces", () => { + workspaceRequests++; + return HttpResponse.json([ + workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", description: "Clinical data", configurationState: "configuration_required", revision, + }), + workspaceSummaryFixture(alternateId, { + displayName: "Other Clinical", description: "Other data", configurationState: "configuration_required", revision: alternateRevision, + }), + ]); + }), + http.get(`/api/workspaces/${alternateId}`, () => HttpResponse.json({ workspace: alternateWorkspace, revision: alternateRevision })), + http.get(`/api/workspaces/${alternateId}/runtime-configuration`, () => HttpResponse.json({ + workspaceId: alternateId, revision: alternateRevision, configurationState: "configuration_required", requirements: [{ ...requirement }], + })), + http.post("/api/workspace-registry/pull", async () => { + started(); + try { + await held; + return HttpResponse.json({ branch: "main", head: "b".repeat(40), ahead: 0, behind: 0, degraded: false }); + } finally { + settled(); + } + }), + ); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "Update workspace repository" })); + await requestStarted; + + await user.click(screen.getByRole("button", { name: "Other Clinical" })); + expect(await screen.findByRole("heading", { name: "Other Clinical" })).toBeVisible(); + await user.click(within(screen.getByRole("navigation", { name: "Workspaces" })) + .getByRole("button", { name: "Back to Level 1" })); + expect(await screen.findByRole("button", { name: "Update workspace repository" })).not.toBeDisabled(); + const requestsBeforeRelease = { statusRequests, workspaceRequests }; + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect({ statusRequests, workspaceRequests }).toEqual(requestsBeforeRelease); + expect(screen.queryByText("Workspace repository updated and validated.")).not.toBeInTheDocument(); +}); + +test("a deferred repository update cannot update a context after workspace permission changes", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + let statusRequests = 0; + let workspaceRequests = 0; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use( + http.get("/api/workspace-registry/status", () => { + statusRequests++; + return HttpResponse.json({ branch: "main", head: "a".repeat(40), ahead: 0, behind: 0, degraded: false }); + }), + http.get("/api/workspaces", () => { + workspaceRequests++; + return HttpResponse.json([workspaceSummaryFixture("psd-clinical", { + displayName: "PSD Clinical", description: "Clinical data", configurationState: "configuration_required", revision, + })]); + }), + http.post("/api/workspace-registry/pull", async () => { + started(); + try { + await held; + return HttpResponse.json({ branch: "main", head: "b".repeat(40), ahead: 0, behind: 0, degraded: false }); + } finally { + settled(); + } + }), + ); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "Update workspace repository" })); + await requestStarted; + + manager.rerender({ canManageWorkspace: false }); + manager.rerender({ canManageWorkspace: true }); + expect(await screen.findByRole("button", { name: "Update workspace repository" })).not.toBeDisabled(); + const requestsBeforeRelease = { statusRequests, workspaceRequests }; + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect({ statusRequests, workspaceRequests }).toEqual(requestsBeforeRelease); + expect(screen.queryByText("Workspace repository updated and validated.")).not.toBeInTheDocument(); +}); + +test("a deferred validation cannot update a context after workspace permission changes", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/workspaces/validate", async () => { + started(); + try { + await held; + return HttpResponse.json({ workspace, contract: {}, activatable: true, diagnostics: [], authentication: readyAuthentication }); + } finally { + settled(); + } + })); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.click(screen.getByRole("button", { name: "Validate workspace source" })); + await requestStarted; + + manager.rerender({ canManageWorkspace: false }); + manager.rerender({ canManageWorkspace: true }); + expect(await screen.findByRole("button", { name: "Validate workspace source" })).not.toBeDisabled(); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect(screen.queryByTestId("workspace-authentication")).not.toBeInTheDocument(); +}); + +test("a deferred connection test cannot update a context after workspace permission changes", async () => { + const user = userEvent.setup(); + let release!: () => void; + let started!: () => void; + let settled!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + const requestStarted = new Promise((resolve) => { started = resolve; }); + const requestSettled = new Promise((resolve) => { settled = resolve; }); + server.use(http.post("/api/workspaces/psd-clinical/test", async () => { + started(); + try { + await held; + return HttpResponse.json({ activatable: true, diagnostics: [], authentication: readyAuthentication }); + } finally { + settled(); + } + })); + const manager = renderManagerWithSettings(); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.click(screen.getByRole("button", { name: "Test workspace connections" })); + await requestStarted; + + manager.rerender({ canManageWorkspace: false }); + manager.rerender({ canManageWorkspace: true }); + expect(await screen.findByRole("button", { name: "Test workspace connections" })).not.toBeDisabled(); + + act(() => release()); + await act(async () => { + await requestSettled; + await new Promise((resolve) => { setTimeout(resolve, 50); }); + }); + + expect(screen.queryByTestId("workspace-authentication")).not.toBeInTheDocument(); +}); + +test("closing clears unsaved secret fields", async () => { + const user = userEvent.setup(); + const onClose = vi.fn(); + renderManager(onClose); + await user.click(await screen.findByRole("button", { name: "PSD Clinical" })); + await user.type(await screen.findByLabelText("Data warehouse password"), "unsaved-value"); + + await user.click(screen.getByRole("button", { name: "Close workspace management" })); + + expect(onClose).toHaveBeenCalledTimes(1); + expect(screen.queryByDisplayValue("unsaved-value")).not.toBeInTheDocument(); +}); diff --git a/frontend/src/shell/WorkspaceManager.tsx b/frontend/src/shell/WorkspaceManager.tsx new file mode 100644 index 00000000..c0678f42 --- /dev/null +++ b/frontend/src/shell/WorkspaceManager.tsx @@ -0,0 +1,666 @@ +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { + AlertCircle, + ArrowLeft, + CheckCircle2, + ClipboardCheck, + FlaskConical, + GitPullRequest, + KeyRound, + Trash2, + X, +} from "lucide-react"; + +import { + asWorkspaceApiError, + forgetWorkspaceSecret, + getWorkspace, + getWorkspaceRegistryStatus, + getWorkspaceRuntimeConfiguration, + listWorkspaces, + pullWorkspaceRegistry, + saveWorkspaceSecrets, + testWorkspace, + validateWorkspace, + type AuthDiagnostics, + type WorkspaceRuntimeConfiguration, +} from "../api/workspaces"; +import { + captureAuthOperation, + isAuthOperationCurrent, + StaleAuthOperationError, + type AuthOperationGuard, +} from "../auth/authOperation"; +import { Button } from "../components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "../components/ui/dialog"; + +function QueryError({ name, message, retryLabel, onRetry }: { + name: string; + message: string; + retryLabel: string; + onRetry: () => void; +}) { + return ( +
+

{message}

+
+
+ ); +} + +function publicError(error: unknown, fallback: string): string { + const safe = asWorkspaceApiError(error); + return safe ? `${safe.code}: ${safe.message}` : fallback; +} + +function stateLabel(state: "ready" | "configuration_required"): string { + return state === "ready" ? "Ready" : "Runtime configuration required"; +} + +const workspaceAuthoringGuideUrl = + "https://github.com/mptyl/ThothII/blob/main/docs/install/local-workspace-registry.md#prepare-and-publish-a-workspace-source"; + +type WorkspaceOperationContext = Readonly<{ + open: boolean; + canManageWorkspace: boolean; + canManageSecrets: boolean; +}>; + +type WorkspaceOperationGuard = Readonly<{ + operation: AuthOperationGuard; + context: WorkspaceOperationContext; +}>; + +export function WorkspaceManager({ + open, + onClose, + canManageWorkspace = false, + canManageSecrets = false, +}: { + open: boolean; + onClose: () => void; + canManageWorkspace?: boolean; + canManageSecrets?: boolean; +}) { + const queryClient = useQueryClient(); + const [selectedId, setSelectedId] = useState(); + const [secretValues, setSecretValues] = useState>({}); + const [notice, setNotice] = useState(); + const [diagnostics, setDiagnostics] = useState([]); + const [validationNotice, setValidationNotice] = useState(); + const [validationDiagnostics, setValidationDiagnostics] = useState([]); + const [connectionNotice, setConnectionNotice] = useState(); + const [connectionDiagnostics, setConnectionDiagnostics] = useState([]); + const [authentication, setAuthentication] = useState(); + const [busyAction, setBusyAction] = useState(); + const operationEpochRef = useRef(0); + const diagnosticEpochRef = useRef(0); + const selectedIdRef = useRef(selectedId); + const contextRef = useRef({ open, canManageWorkspace, canManageSecrets }); + const committedContextRef = useRef(contextRef.current); + selectedIdRef.current = selectedId; + contextRef.current = { open, canManageWorkspace, canManageSecrets }; + + const clearWorkspacePresentation = () => { + setNotice(undefined); + setDiagnostics([]); + setValidationNotice(undefined); + setValidationDiagnostics([]); + setConnectionNotice(undefined); + setConnectionDiagnostics([]); + setAuthentication(undefined); + }; + + const cancelWorkspaceQueries = () => { + void queryClient.cancelQueries({ queryKey: ["workspace-repository-status"] }); + void queryClient.cancelQueries({ queryKey: ["workspaces"] }); + void queryClient.cancelQueries({ queryKey: ["workspace"] }); + void queryClient.cancelQueries({ queryKey: ["workspace-runtime-configuration"] }); + }; + + const clearWorkspaceCaches = () => { + queryClient.removeQueries({ queryKey: ["workspace-repository-status"] }); + queryClient.removeQueries({ queryKey: ["workspaces"] }); + queryClient.removeQueries({ queryKey: ["workspace"] }); + queryClient.removeQueries({ queryKey: ["workspace-runtime-configuration"] }); + }; + + // This is the sole invalidation boundary for deferred workspace work. Call it before capturing + // an operation guard; doing it after capture would immediately invalidate the new operation. + const invalidateWorkspaceContext = ({ + clearSecrets = true, + clearCaches = false, + }: { clearSecrets?: boolean; clearCaches?: boolean } = {}) => { + operationEpochRef.current += 1; + diagnosticEpochRef.current += 1; + cancelWorkspaceQueries(); + if (clearCaches) clearWorkspaceCaches(); + setBusyAction(undefined); + if (clearSecrets) { + setSecretValues({}); + } + clearWorkspacePresentation(); + }; + + useEffect(() => () => { + operationEpochRef.current += 1; + diagnosticEpochRef.current += 1; + }, []); + + useLayoutEffect(() => { + const previous = committedContextRef.current; + const current = contextRef.current; + const openChanged = previous.open !== current.open; + const permissionsChanged = previous.canManageWorkspace !== current.canManageWorkspace + || previous.canManageSecrets !== current.canManageSecrets; + if (openChanged || permissionsChanged) { + invalidateWorkspaceContext({ clearCaches: permissionsChanged }); + } + committedContextRef.current = current; + }, [open, canManageWorkspace, canManageSecrets]); + + function captureWorkspaceOperation(targetId?: string): WorkspaceOperationGuard | null { + const context = contextRef.current; + if (!context.open) return null; + const operation = captureAuthOperation({ + sessionId: targetId ?? null, + disposalEpoch: operationEpochRef.current, + }); + return operation ? { operation, context } : null; + } + + function isWorkspaceOperationCurrent(guard: WorkspaceOperationGuard | null, targetId?: string): boolean { + if (!guard) return false; + const context = contextRef.current; + return context.open === guard.context.open + && context.canManageWorkspace === guard.context.canManageWorkspace + && context.canManageSecrets === guard.context.canManageSecrets + && isAuthOperationCurrent(guard.operation, { + sessionId: targetId ?? null, + disposalEpoch: operationEpochRef.current, + }); + } + + async function guardedQuery(request: () => Promise, targetId?: string): Promise { + const guard = captureWorkspaceOperation(targetId); + if (!guard) throw new StaleAuthOperationError(); + const result = await request(); + const currentTargetId = targetId === undefined ? undefined : selectedIdRef.current; + if (!isWorkspaceOperationCurrent(guard, currentTargetId)) throw new StaleAuthOperationError(); + return result; + } + + const statusQuery = useQuery({ + queryKey: ["workspace-repository-status"], + queryFn: () => guardedQuery(getWorkspaceRegistryStatus), + enabled: open, + }); + const workspacesQuery = useQuery({ + queryKey: ["workspaces"], + queryFn: () => guardedQuery(listWorkspaces), + enabled: open, + }); + const workspaces = workspacesQuery.data ?? []; + const selectedSummary = useMemo( + () => workspaces.find(({ id }) => id === selectedId), + [selectedId, workspaces], + ); + const detailQuery = useQuery({ + queryKey: ["workspace", selectedId], + queryFn: () => guardedQuery(() => getWorkspace(selectedId!), selectedId), + enabled: Boolean(open && selectedId), + }); + const runtimeQuery = useQuery({ + queryKey: ["workspace-runtime-configuration", selectedId], + queryFn: () => guardedQuery(() => getWorkspaceRuntimeConfiguration(selectedId!), selectedId), + enabled: Boolean(open && selectedId), + }); + + const close = () => { + invalidateWorkspaceContext(); + onClose(); + }; + + const selectWorkspace = (id: string) => { + invalidateWorkspaceContext(); + setSelectedId(id); + }; + + const showLevelOne = () => { + invalidateWorkspaceContext(); + setSelectedId(undefined); + }; + + async function updateRepository() { + invalidateWorkspaceContext({ clearCaches: true }); + const guard = captureWorkspaceOperation(); + if (!guard) return; + const targetId = selectedIdRef.current; + setBusyAction("repository"); + try { + await pullWorkspaceRegistry(); + if (!isWorkspaceOperationCurrent(guard)) return; + await Promise.all([ + statusQuery.refetch(), + workspacesQuery.refetch(), + targetId ? detailQuery.refetch() : Promise.resolve(), + targetId ? runtimeQuery.refetch() : Promise.resolve(), + ]); + if (!isWorkspaceOperationCurrent(guard)) return; + setNotice("Workspace repository updated and validated."); + } catch (error) { + if (isWorkspaceOperationCurrent(guard)) { + setDiagnostics([publicError(error, "git_unavailable: Workspace repository could not be updated")]); + } + } finally { + if (isWorkspaceOperationCurrent(guard)) setBusyAction(undefined); + } + } + + async function validateSource() { + if (!detailQuery.data) return; + const targetId = selectedId; + const source = detailQuery.data.workspace; + invalidateWorkspaceContext({ clearSecrets: false }); + const guard = captureWorkspaceOperation(targetId); + if (!guard) return; + const diagnosticEpoch = diagnosticEpochRef.current; + setBusyAction("validate"); + try { + const result = await validateWorkspace(source); + if (diagnosticEpoch !== diagnosticEpochRef.current || + !isWorkspaceOperationCurrent(guard, selectedIdRef.current)) return; + setAuthentication(result.authentication); + setValidationNotice(result.activatable ? "Workspace source and authentication are valid." : "Workspace source is valid."); + } catch (error) { + if (diagnosticEpoch === diagnosticEpochRef.current && + isWorkspaceOperationCurrent(guard, selectedIdRef.current)) { + setValidationDiagnostics([publicError(error, "workspace_invalid: Workspace validation could not be completed")]); + } + } finally { + if (diagnosticEpoch === diagnosticEpochRef.current && + isWorkspaceOperationCurrent(guard, selectedIdRef.current)) setBusyAction(undefined); + } + } + + async function testConnections() { + if (!selectedId) return; + const targetId = selectedId; + invalidateWorkspaceContext({ clearSecrets: false }); + const guard = captureWorkspaceOperation(targetId); + if (!guard) return; + const diagnosticEpoch = diagnosticEpochRef.current; + setBusyAction("test"); + try { + const result = await testWorkspace(targetId); + if (diagnosticEpoch !== diagnosticEpochRef.current || + !isWorkspaceOperationCurrent(guard, selectedIdRef.current)) return; + setAuthentication(result.authentication); + const issues = result.diagnostics.filter(({ level }) => level !== "info"); + const informational = result.diagnostics.find(({ level }) => level === "info"); + setConnectionDiagnostics(issues.map(({ code, message }) => `${code}: ${message}`)); + if (issues.length === 0) { + setConnectionNotice(result.activatable + ? informational + ? `${informational.code}: ${informational.message}` + : "Workspace connections are valid." + : "Workspace connection test completed."); + } + } catch (error) { + if (diagnosticEpoch === diagnosticEpochRef.current && + isWorkspaceOperationCurrent(guard, selectedIdRef.current)) { + setConnectionDiagnostics([publicError(error, "connector_unavailable: Workspace connections could not be tested")]); + } + } finally { + if (diagnosticEpoch === diagnosticEpochRef.current && + isWorkspaceOperationCurrent(guard, selectedIdRef.current)) setBusyAction(undefined); + } + } + + async function saveSecrets() { + if (!selectedId) return; + const targetId = selectedId; + const values = Object.fromEntries( + Object.entries(secretValues).filter(([, value]) => value.length > 0), + ); + if (Object.keys(values).length === 0) return; + invalidateWorkspaceContext({ clearSecrets: false }); + const guard = captureWorkspaceOperation(targetId); + if (!guard) return; + setBusyAction("save-secrets"); + try { + const configuration = await saveWorkspaceSecrets(targetId, values); + if (!isWorkspaceOperationCurrent(guard, selectedIdRef.current)) return; + queryClient.setQueryData( + ["workspace-runtime-configuration", targetId], + configuration, + ); + setSecretValues({}); + await workspacesQuery.refetch(); + if (!isWorkspaceOperationCurrent(guard, selectedIdRef.current)) return; + setNotice("Runtime secrets saved. Stored values remain hidden."); + } catch (error) { + if (isWorkspaceOperationCurrent(guard, selectedIdRef.current)) { + setDiagnostics([publicError(error, "workspace_invalid: Runtime secrets could not be saved")]); + } + } finally { + if (isWorkspaceOperationCurrent(guard, selectedIdRef.current)) setBusyAction(undefined); + } + } + + async function forgetSecret(requirementId: string) { + if (!selectedId) return; + const targetId = selectedId; + invalidateWorkspaceContext({ clearSecrets: false }); + const guard = captureWorkspaceOperation(targetId); + if (!guard) return; + setBusyAction(`forget:${requirementId}`); + try { + const configuration = await forgetWorkspaceSecret(targetId, requirementId); + if (!isWorkspaceOperationCurrent(guard, selectedIdRef.current)) return; + queryClient.setQueryData( + ["workspace-runtime-configuration", targetId], + configuration, + ); + setSecretValues((current) => ({ ...current, [requirementId]: "" })); + await workspacesQuery.refetch(); + if (!isWorkspaceOperationCurrent(guard, selectedIdRef.current)) return; + setNotice("Stored secret forgotten."); + } catch (error) { + if (isWorkspaceOperationCurrent(guard, selectedIdRef.current)) { + setDiagnostics([publicError(error, "workspace_invalid: Stored secret could not be forgotten")]); + } + } finally { + if (isWorkspaceOperationCurrent(guard, selectedIdRef.current)) setBusyAction(undefined); + } + } + + const repository = statusQuery.data?.repository; + const repositoryLabel = repository + ? `${repository.host}/${repository.repository}` + : "the repository configured for this ThothII installation"; + const runtime = runtimeQuery.data; + const hasEnteredSecrets = Object.values(secretValues).some((value) => value.length > 0); + + return ( + { if (!nextOpen) close(); }}> + + + Workspace management + + Read, validate, and complete the runtime configuration of workspaces supplied by the installation repository. + + + + +
+ + +
+ {notice && ( +

+ {notice} +

+ )} + {diagnostics.length > 0 && ( +
+ {diagnostics.map((diagnostic) => ( +

+ {diagnostic} +

+ ))} +
+ )} + + {!selectedSummary ? ( +
+
+

Level 1 · Repository

+

How workspaces reach ThothII

+
+
    +
  1. Create a workspace repository in any local directory you choose. Add one directory for each workspace you want ThothII to manage. At the repository root, thoth-workspaces.yaml lists those workspaces; each workspace directory contains its own workspace.yaml, which declares the database connection, the Evidence sources, and the ThothII vector-database collection used during the process.
    2. +
  2. Publish that source by committing and pushing it to a repository hosted by a Git server such as GitHub, GitLab, or Gitea.
    3. +
  3. The repository address, branch, and read-only Git credentials are configured during ThothII installation. This installation reads {repositoryLabel} on branch {statusQuery.data?.branch ?? "main"}.
    4. +
  4. ThothII fetches the configured branch into its managed read-only checkout, validates the complete candidate revision, and activates it only when validation succeeds. It never edits, commits, pushes, or publishes workspace source.
    5. +
+

+ Follow the workspace authoring instructions on GitHub for the required layout and validation rules. +

+ {canManageWorkspace &&
+
+
+

Update workspace repository

+

Fetches the configured branch directly into the managed read-only checkout and validates it. No workspace selection is required. If candidate validation fails, the current active revision remains unchanged.

+
+ +
+
} + {!canManageWorkspace && ( +

+ You can inspect workspaces. Workspace updates, validation, and connection tests require workspace management permission. +

+ )} +

Select a workspace from the left only for workspace-specific validation, runtime credentials, and connection tests.

+
+ ) : ( +
+
+

Level 2 · Selected workspace

+

{selectedSummary.displayName}

+

{selectedSummary.id}

+
+ + {(detailQuery.isLoading || runtimeQuery.isLoading) &&

Loading workspace configuration…

} + {(detailQuery.isError || runtimeQuery.isError) && ( + { void Promise.all([detailQuery.refetch(), runtimeQuery.refetch()]); }} /> + )} + + {detailQuery.data && runtime && ( + <> +
+

Workspace-specific actions

+

The actions below apply only to {selectedSummary.displayName}. ThothII reads this revision without modifying or publishing it.

+
+ +
+
Source file
{selectedSummary.file}
+
Active revision
{detailQuery.data.revision.commit}
+
+
Database
+
+ engine: {detailQuery.data.workspace.dwh.engine} + database: {detailQuery.data.workspace.dwh.database} + schema: {detailQuery.data.workspace.dwh.schema} +
+
+
Runtime status
{stateLabel(runtime.configurationState)}
+
+ +
+
+

Validate workspace source

+

Checks workspace.yaml and the required workspace directories against the supported workspace schema. No source file is changed.

+ {validationNotice && ( +

+ {validationNotice} +

+ )} + {validationDiagnostics.length > 0 && ( +
+ {validationDiagnostics.map((diagnostic) => ( +

+ {diagnostic} +

+ ))} +
+ )} + {canManageWorkspace && } +
+
+

Test workspace connections

+

Uses temporary decrypted credentials to verify the configured data warehouse and Evidence source. Temporary files are deleted after the test.

+ {connectionNotice && ( +

+ {connectionNotice} +

+ )} + {connectionDiagnostics.length > 0 && ( +
+ {connectionDiagnostics.map((diagnostic) => ( +

+ {diagnostic} +

+ ))} +
+ )} + {canManageWorkspace && } +
+
+ + {authentication &&
+
+

Authentication

+ + {authentication.ready ? "Passed" : "Failed"} + +
+ {authentication.checks.some(({ level }) => level === "error") &&
+ {authentication.checks.filter(({ level }) => level === "error").map(({ code, field, message }) => ( +

+ + {code}: {field === undefined ? message : `${field} — ${message}`} +

+ ))} +
} +
} + + {canManageSecrets &&
+
+ +
+

Runtime secrets

+

Enter only new or replacement values. Stored values are never displayed. Saving replaces the selected secret and clears the form field.

+
+
+ {runtime.requirements.length === 0 ? ( +

This workspace does not require user-provided runtime secrets for its selected connectors.

+ ) : ( +
+ {runtime.requirements.map((requirement) => ( +
+
+ + + {requirement.configured ? "Configured" : "Not configured"} + +
+

{requirement.description}{requirement.required ? " Required for this workspace." : " Optional."}

+ {requirement.input === "textarea" ? ( +